From 2dd2878263ef8e2092a82850b4005b1da81bf199 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Wed, 7 Aug 2019 10:51:34 +0800 Subject: [PATCH 01/17] planner: add aggregation hints `TIDB_HASHAGG` and `TIDB_STREAMAGG` (#11364) --- cmd/explaintest/r/topn_push_down.result | 16 +-- executor/join_test.go | 24 ++-- planner/core/exhaust_physical_plans.go | 76 ++++++++++-- planner/core/expression_rewriter.go | 5 +- planner/core/logical_plan_builder.go | 65 +++++----- planner/core/logical_plans.go | 5 + planner/core/physical_plan_test.go | 158 +++++++++++++++++++++++- planner/core/planbuilder.go | 1 + 8 files changed, 285 insertions(+), 65 deletions(-) diff --git a/cmd/explaintest/r/topn_push_down.result b/cmd/explaintest/r/topn_push_down.result index 9deb403ecceca..4aba3b9758704 100644 --- a/cmd/explaintest/r/topn_push_down.result +++ b/cmd/explaintest/r/topn_push_down.result @@ -240,19 +240,19 @@ explain select /*+ TIDB_SMJ(t1, t2) */ * from t t1 join t t2 on t1.a = t2.a limi id count task operator info Limit_11 5.00 root offset:0, count:5 └─MergeJoin_12 5.00 root inner join, left key:test.t1.a, right key:test.t2.a - ├─IndexReader_14 4.00 root index:IndexScan_13 - │ └─IndexScan_13 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo - └─IndexReader_16 4.00 root index:IndexScan_15 - └─IndexScan_15 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + ├─IndexReader_15 4.00 root index:IndexScan_14 + │ └─IndexScan_14 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + └─IndexReader_17 4.00 root index:IndexScan_16 + └─IndexScan_16 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo explain select /*+ TIDB_SMJ(t1, t2) */ * from t t1 left join t t2 on t1.a = t2.a where t2.a is null limit 5; id count task operator info Limit_12 5.00 root offset:0, count:5 └─Selection_13 5.00 root isnull(test.t2.a) └─MergeJoin_14 5.00 root left outer join, left key:test.t1.a, right key:test.t2.a - ├─IndexReader_16 4.00 root index:IndexScan_15 - │ └─IndexScan_15 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo - └─IndexReader_18 4.00 root index:IndexScan_17 - └─IndexScan_17 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + ├─IndexReader_17 4.00 root index:IndexScan_16 + │ └─IndexScan_16 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + └─IndexReader_19 4.00 root index:IndexScan_18 + └─IndexScan_18 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo explain select /*+ TIDB_HJ(t1, t2) */ * from t t1 join t t2 on t1.a = t2.a limit 5; id count task operator info Limit_11 5.00 root offset:0, count:5 diff --git a/executor/join_test.go b/executor/join_test.go index 6c2211538be29..68cf928cf6dcd 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -143,13 +143,13 @@ func (s *testSuite2) TestJoin(c *C) { tk.MustQuery("select /*+ TIDB_INLJ(t1) */ * from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", " 4 5")) tk.MustQuery("select /*+ TIDB_INLJ(t) */ avg(t.b) from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1.5000")) - // Test that two conflict hints will return error. - err := tk.ExecToErr("select /*+ TIDB_INLJ(t) TIDB_SMJ(t) */ * from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) - err = tk.ExecToErr("select /*+ TIDB_INLJ(t) TIDB_HJ(t) */ from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) - err = tk.ExecToErr("select /*+ TIDB_SMJ(t) TIDB_HJ(t) */ from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) + // Test that two conflict hints will return warning. + tk.MustExec("select /*+ TIDB_INLJ(t) TIDB_SMJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + tk.MustExec("select /*+ TIDB_INLJ(t) TIDB_HJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + tk.MustExec("select /*+ TIDB_SMJ(t) TIDB_HJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") @@ -888,11 +888,11 @@ func (s *testSuite2) TestMergejoinOrder(c *C) { tk.MustExec("insert into t2 select a*100, b*100 from t1;") tk.MustQuery("explain select /*+ TIDB_SMJ(t2) */ * from t1 left outer join t2 on t1.a=t2.a and t1.a!=3 order by t1.a;").Check(testkit.Rows( - "MergeJoin_15 10000.00 root left outer join, left key:test.t1.a, right key:test.t2.a, left cond:[ne(test.t1.a, 3)]", - "├─TableReader_11 10000.00 root data:TableScan_10", - "│ └─TableScan_10 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo", - "└─TableReader_13 6666.67 root data:TableScan_12", - " └─TableScan_12 6666.67 cop table:t2, range:[-inf,3), (3,+inf], keep order:true, stats:pseudo", + "MergeJoin_20 10000.00 root left outer join, left key:test.t1.a, right key:test.t2.a, left cond:[ne(test.t1.a, 3)]", + "├─TableReader_12 10000.00 root data:TableScan_11", + "│ └─TableScan_11 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo", + "└─TableReader_14 6666.67 root data:TableScan_13", + " └─TableScan_13 6666.67 cop table:t2, range:[-inf,3), (3,+inf], keep order:true, stats:pseudo", )) tk.MustExec("set @@tidb_init_chunk_size=1") diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index bd9fac359ba4d..ed2c31e0c0e17 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -118,7 +118,7 @@ func (p *PhysicalMergeJoin) tryToGetChildReqProp(prop *property.PhysicalProperty } func (p *LogicalJoin) getMergeJoin(prop *property.PhysicalProperty) []PhysicalPlan { - joins := make([]PhysicalPlan, 0, len(p.leftProperties)) + joins := make([]PhysicalPlan, 0, len(p.leftProperties)+1) // The leftProperties caches all the possible properties that are provided by its children. for _, lhsChildProperty := range p.leftProperties { offsets := getMaxSortPrefix(lhsChildProperty, p.LeftJoinKeys) @@ -159,10 +159,10 @@ func (p *LogicalJoin) getMergeJoin(prop *property.PhysicalProperty) []PhysicalPl joins = append(joins, mergeJoin) } } - // If TiDB_SMJ hint is existed && no join keys in children property, - // it should to enforce merge join. - if len(joins) == 0 && (p.preferJoinType&preferMergeJoin) > 0 { - return p.getEnforcedMergeJoin(prop) + // If TiDB_SMJ hint is existed, it should consider enforce merge join, + // because we can't trust lhsChildProperty completely. + if (p.preferJoinType & preferMergeJoin) > 0 { + joins = append(joins, p.getEnforcedMergeJoin(prop)...) } return joins @@ -1171,11 +1171,36 @@ func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) []P panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } +func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { + _, desc := prop.AllSameOrder() + enforcedAggs := make([]PhysicalPlan, 0, len(wholeTaskTypes)) + childProp := &property.PhysicalProperty{ + ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), + Enforced: true, + Items: property.ItemsFromCols(la.groupByCols, desc), + } + + for _, taskTp := range wholeTaskTypes { + copiedChildProperty := new(property.PhysicalProperty) + *copiedChildProperty = *childProp // It's ok to not deep copy the "cols" field. + copiedChildProperty.TaskTp = taskTp + + agg := basePhysicalAgg{ + GroupByItems: la.GroupByItems, + AggFuncs: la.AggFuncs, + }.initForStream(la.ctx, la.stats.ScaleByExpectCnt(prop.ExpectedCnt), copiedChildProperty) + agg.SetSchema(la.schema.Clone()) + enforcedAggs = append(enforcedAggs, agg) + } + return enforcedAggs +} + func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { all, desc := prop.AllSameOrder() - if len(la.possibleProperties) == 0 || !all { + if !all { return nil } + for _, aggFunc := range la.AggFuncs { if aggFunc.Mode == aggregation.FinalMode { return nil @@ -1186,7 +1211,7 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P return nil } - streamAggs := make([]PhysicalPlan, 0, len(la.possibleProperties)*(len(wholeTaskTypes)-1)) + streamAggs := make([]PhysicalPlan, 0, len(la.possibleProperties)*(len(wholeTaskTypes)-1)+len(wholeTaskTypes)) childProp := &property.PhysicalProperty{ ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), } @@ -1217,6 +1242,11 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P streamAggs = append(streamAggs, agg) } } + // If STREAM_AGG hint is existed, it should consider enforce stream aggregation, + // because we can't trust possibleChildProperty completely. + if (la.preferAggType & preferStreamAgg) > 0 { + streamAggs = append(streamAggs, la.getEnforcedStreamAggs(prop)...) + } return streamAggs } @@ -1237,9 +1267,35 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy } func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { - aggs := make([]PhysicalPlan, 0, len(la.possibleProperties)+1) - aggs = append(aggs, la.getHashAggs(prop)...) - aggs = append(aggs, la.getStreamAggs(prop)...) + preferHash := (la.preferAggType & preferHashAgg) > 0 + preferStream := (la.preferAggType & preferStreamAgg) > 0 + if preferHash && preferStream { + errMsg := "Optimizer aggregation hints are conflicted" + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + la.preferAggType = 0 + preferHash, preferStream = false, false + } + + hashAggs := la.getHashAggs(prop) + if hashAggs != nil && preferHash { + return hashAggs + } + + streamAggs := la.getStreamAggs(prop) + if streamAggs != nil && preferStream { + return streamAggs + } + + if streamAggs == nil && preferStream { + errMsg := "Optimizer Hint TIDB_STREAMAGG is inapplicable" + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + } + + aggs := make([]PhysicalPlan, 0, len(hashAggs)+len(streamAggs)) + aggs = append(aggs, hashAggs...) + aggs = append(aggs, streamAggs...) return aggs } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 2f7cbea61fa24..42e3f50f36938 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -758,10 +758,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, v *ast.Patte join.attachOnConds(expression.SplitCNFItems(checkCondition)) // Set join hint for this join. if er.b.TableHints() != nil { - er.err = join.setPreferredJoinType(er.b.TableHints()) - if er.err != nil { - return v, true - } + join.setPreferredJoinType(er.b.TableHints()) } er.p = join } else { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 517746047d580..b6831a5931727 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -54,6 +54,10 @@ const ( TiDBIndexNestedLoopJoin = "tidb_inlj" // TiDBHashJoin is hint enforce hash join. TiDBHashJoin = "tidb_hj" + // TiDBHashAgg is hint enforce hash aggregation. + TiDBHashAgg = "tidb_hashagg" + // TiDBStreamAgg is hint enforce stream aggregation. + TiDBStreamAgg = "tidb_streamagg" ) const ( @@ -137,6 +141,9 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu plan4Agg.GroupByItems = gbyItems plan4Agg.SetSchema(schema4Agg) plan4Agg.collectGroupByColumns() + if hint := b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } return plan4Agg, aggIndexMap, nil } @@ -327,9 +334,9 @@ func extractTableAlias(p LogicalPlan) *model.CIStr { return nil } -func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) error { +func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) { if hintInfo == nil { - return nil + return } lhsAlias := extractTableAlias(p.children[0]) @@ -355,9 +362,10 @@ func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) error { // If there're multiple join types and one of them is not index join hint, // then there is a conflict of join types. if bits.OnesCount(p.preferJoinType) > 1 && (p.preferJoinType^preferRightAsIndexInner^preferLeftAsIndexInner) > 0 { - return errors.New("Join hints are conflict, you can only specify one type of join") + errMsg := "Join hints are conflict, you can only specify one type of join" + warning := ErrInternal.GenWithStack(errMsg) + p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } - return nil } func resetNotNullFlag(schema *expression.Schema, start, end int) { @@ -424,10 +432,7 @@ func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (Logica joinPlan.redundantSchema = expression.MergeSchema(lRedundant, rRedundant) // Set preferred join algorithm if some join hints is specified by user. - err = joinPlan.setPreferredJoinType(b.TableHints()) - if err != nil { - return nil, err - } + joinPlan.setPreferredJoinType(b.TableHints()) // "NATURAL JOIN" doesn't have "ON" or "USING" conditions. // @@ -1931,8 +1936,9 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi return resultList, nil } -func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { +func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { var sortMergeTables, INLJTables, hashJoinTables []hintTableInfo + var preferAggType uint for _, hint := range hints { switch hint.HintName.L { case TiDBMergeJoin: @@ -1941,19 +1947,20 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { INLJTables = tableNames2HintTableInfo(hint.Tables) case TiDBHashJoin: hashJoinTables = tableNames2HintTableInfo(hint.Tables) + case TiDBHashAgg: + preferAggType |= preferHashAgg + case TiDBStreamAgg: + preferAggType |= preferStreamAgg default: // ignore hints that not implemented } } - if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables) > 0 { - b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ - sortMergeJoinTables: sortMergeTables, - indexNestedLoopJoinTables: INLJTables, - hashJoinTables: hashJoinTables, - }) - return true - } - return false + b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ + sortMergeJoinTables: sortMergeTables, + indexNestedLoopJoinTables: INLJTables, + hashJoinTables: hashJoinTables, + preferAggType: preferAggType, + }) } func (b *PlanBuilder) popTableHints() { @@ -1983,10 +1990,10 @@ func (b *PlanBuilder) TableHints() *tableHintInfo { } func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p LogicalPlan, err error) { - if b.pushTableHints(sel.TableHints) { - // table hints are only visible in the current SELECT statement. - defer b.popTableHints() - } + b.pushTableHints(sel.TableHints) + // table hints are only visible in the current SELECT statement. + defer b.popTableHints() + if sel.SelectStmtOpts != nil { origin := b.inStraightJoin b.inStraightJoin = sel.SelectStmtOpts.StraightJoin @@ -2532,10 +2539,9 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio } func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { - if b.pushTableHints(update.TableHints) { - // table hints are only visible in the current UPDATE statement. - defer b.popTableHints() - } + b.pushTableHints(update.TableHints) + // table hints are only visible in the current UPDATE statement. + defer b.popTableHints() // update subquery table should be forbidden var asNameList []string @@ -2752,10 +2758,9 @@ func extractTableAsNameForUpdate(p LogicalPlan, asNames map[*model.TableInfo][]* } func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) (Plan, error) { - if b.pushTableHints(delete.TableHints) { - // table hints are only visible in the current DELETE statement. - defer b.popTableHints() - } + b.pushTableHints(delete.TableHints) + // table hints are only visible in the current DELETE statement. + defer b.popTableHints() sel := &ast.SelectStmt{ Fields: &ast.FieldList{}, diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 411937b4a315f..9011603d8ca33 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -98,6 +98,8 @@ const ( preferRightAsIndexInner preferHashJoin preferMergeJoin + preferHashAgg + preferStreamAgg ) // LogicalJoin is the logical join plan. @@ -247,6 +249,9 @@ type LogicalAggregation struct { // groupByCols stores the columns that are group-by items. groupByCols []*expression.Column + // preferAggType stores preferred aggregation algorithm type. + preferAggType uint + possibleProperties [][]*expression.Column inputCount float64 // inputCount is the input count of this plan. } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index a6c65e1e8c25c..6d41b9fa6d0f8 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1517,7 +1517,7 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { } } -func (s *testPlanSuite) TestIndexJoinHint(c *C) { +func (s *testPlanSuite) TestJoinHints(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() c.Assert(err, IsNil) @@ -1572,3 +1572,159 @@ func (s *testPlanSuite) TestIndexJoinHint(c *C) { } } } + +func (s *testPlanSuite) TestAggregationHints(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + warning string + }{ + // without Aggregation hints + { + sql: "select count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", + warning: "", + }, + { + sql: "select count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", + warning: "", + }, + // with Aggregation hints + { + sql: "select /*+ TIDB_HASHAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", + warning: "", + }, + { + sql: "select /*+ TIDB_STREAMAGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", + warning: "", + }, + // test conflict warning + { + sql: "select /*+ TIDB_HASHAGG() TIDB_STREAMAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", + warning: "[planner:1815]Optimizer aggregation hints are conflicted", + }, + } + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + if test.warning == "" { + c.Assert(len(warnings), Equals, 0) + } else { + c.Assert(len(warnings), Equals, 1) + c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning) + c.Assert(warnings[0].Err.Error(), Equals, test.warning) + } + } +} + +func (s *testPlanSuite) TestHintScope(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + }{ + // join hints + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t3) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)->Sort}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t3.c,test.t2.a)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + // aggregation hints + { + sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select /*+ TIDB_HASHAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", + }, + { + sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select /*+ TIDB_STREAMAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Sort->Projection->StreamAgg->HashAgg->Projection", + }, + { + sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->HashAgg->Projection", + }, + { + sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", + }, + } + + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(warnings, HasLen, 0, comment) + } +} diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index d9c3e5fb9619f..a6efe8941258b 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -56,6 +56,7 @@ type tableHintInfo struct { indexNestedLoopJoinTables []hintTableInfo sortMergeJoinTables []hintTableInfo hashJoinTables []hintTableInfo + preferAggType uint } type hintTableInfo struct { From 489c094f4199fda1c3756724b247cae8e5ea6457 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Wed, 14 Aug 2019 19:31:01 +0800 Subject: [PATCH 02/17] planner: rename optimizer hints (#11673) --- executor/index_lookup_join_test.go | 8 +-- go.mod | 4 +- go.sum | 8 +-- planner/core/exhaust_physical_plans.go | 8 +-- planner/core/logical_plan_builder.go | 36 +++++++------ planner/core/physical_plan_test.go | 73 +++++++++++++++++++++----- 6 files changed, 96 insertions(+), 41 deletions(-) diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index faa4230e22590..2fec9fa6e5e13 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -131,17 +131,17 @@ func (s *testSuite1) TestInapplicableIndexJoinHint(c *C) { tk.MustExec(`create table t1(a bigint, b bigint);`) tk.MustExec(`create table t2(a bigint, b bigint);`) tk.MustQuery(`select /*+ TIDB_INLJ(t1, t2) */ * from t1, t2;`).Check(testkit.Rows()) - tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1, t2) */ is inapplicable without column equal ON condition`)) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ INL_JOIN(t1, t2) */ or /*+ TIDB_INLJ(t1, t2) */ is inapplicable without column equal ON condition`)) tk.MustQuery(`select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a;`).Check(testkit.Rows()) - tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1, t2) */ is inapplicable`)) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ INL_JOIN(t1, t2) */ or /*+ TIDB_INLJ(t1, t2) */ is inapplicable`)) tk.MustExec(`drop table if exists t1, t2;`) tk.MustExec(`create table t1(a bigint, b bigint, index idx_a(a));`) tk.MustExec(`create table t2(a bigint, b bigint);`) tk.MustQuery(`select /*+ TIDB_INLJ(t1) */ * from t1 left join t2 on t1.a=t2.a;`).Check(testkit.Rows()) - tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1) */ is inapplicable`)) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ INL_JOIN(t1) */ or /*+ TIDB_INLJ(t1) */ is inapplicable`)) tk.MustQuery(`select /*+ TIDB_INLJ(t2) */ * from t1 right join t2 on t1.a=t2.a;`).Check(testkit.Rows()) - tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t2) */ is inapplicable`)) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ INL_JOIN(t2) */ or /*+ TIDB_INLJ(t2) */ is inapplicable`)) } func (s *testSuite) TestIndexJoinOverflow(c *C) { diff --git a/go.mod b/go.mod index d54d6c8c6addd..5557bafa5b17c 100644 --- a/go.mod +++ b/go.mod @@ -43,8 +43,8 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2 - github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 + github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb + github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 github.com/prometheus/client_golang v0.9.0 diff --git a/go.sum b/go.sum index 090681c0bef8f..eda0d8344f9d5 100644 --- a/go.sum +++ b/go.sum @@ -167,10 +167,10 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2 h1:wPekJBaldlYsCQa6zkahYkTctfoQLGXf4eqCdtxlYQ0= -github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 h1:sqqiviE8oEYXJh3Aq59HO/AhxjsvcRb9ETh0ivFOHXc= -github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= +github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb h1:7VLFE2QHeZIhnpKK0XRESkgXj/PtJguIkDZvW3MinN4= +github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= +github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw= diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index ed2c31e0c0e17..e2ba41fc60db8 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -964,9 +964,11 @@ func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJ defer func() { if !forced && hasIndexJoinHint { // Construct warning message prefix. - errMsg := "Optimizer Hint TIDB_INLJ is inapplicable" + errMsg := "Optimizer Hint INL_JOIN or TIDB_INLJ is inapplicable" if p.hintInfo != nil { - errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(TiDBIndexNestedLoopJoin, p.hintInfo.indexNestedLoopJoinTables)) + errMsg = fmt.Sprintf("Optimizer Hint %s or %s is inapplicable", + restore2JoinHint(HintINLJ, p.hintInfo.indexNestedLoopJoinTables), + restore2JoinHint(TiDBIndexNestedLoopJoin, p.hintInfo.indexNestedLoopJoinTables)) } // Append inapplicable reason. @@ -1288,7 +1290,7 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper } if streamAggs == nil && preferStream { - errMsg := "Optimizer Hint TIDB_STREAMAGG is inapplicable" + errMsg := "Optimizer Hint STREAM_AGG is inapplicable" warning := ErrInternal.GenWithStack(errMsg) la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index b6831a5931727..84f9244edd6be 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -50,14 +50,20 @@ import ( const ( // TiDBMergeJoin is hint enforce merge join. TiDBMergeJoin = "tidb_smj" + // HintSMJ is hint enforce merge join. + HintSMJ = "sm_join" // TiDBIndexNestedLoopJoin is hint enforce index nested loop join. TiDBIndexNestedLoopJoin = "tidb_inlj" + // HintINLJ is hint enforce index nested loop join. + HintINLJ = "inl_join" // TiDBHashJoin is hint enforce hash join. TiDBHashJoin = "tidb_hj" - // TiDBHashAgg is hint enforce hash aggregation. - TiDBHashAgg = "tidb_hashagg" - // TiDBStreamAgg is hint enforce stream aggregation. - TiDBStreamAgg = "tidb_streamagg" + // HintHJ is hint enforce hash join. + HintHJ = "hash_join" + // HintHashAgg is hint enforce hash aggregation. + HintHashAgg = "hash_agg" + // HintStreamAgg is hint enforce stream aggregation. + HintStreamAgg = "stream_agg" ) const ( @@ -1941,15 +1947,15 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { var preferAggType uint for _, hint := range hints { switch hint.HintName.L { - case TiDBMergeJoin: + case TiDBMergeJoin, HintSMJ: sortMergeTables = tableNames2HintTableInfo(hint.Tables) - case TiDBIndexNestedLoopJoin: + case TiDBIndexNestedLoopJoin, HintINLJ: INLJTables = tableNames2HintTableInfo(hint.Tables) - case TiDBHashJoin: + case TiDBHashJoin, HintHJ: hashJoinTables = tableNames2HintTableInfo(hint.Tables) - case TiDBHashAgg: + case HintHashAgg: preferAggType |= preferHashAgg - case TiDBStreamAgg: + case HintStreamAgg: preferAggType |= preferStreamAgg default: // ignore hints that not implemented @@ -1965,19 +1971,19 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { func (b *PlanBuilder) popTableHints() { hintInfo := b.tableHintInfo[len(b.tableHintInfo)-1] - b.appendUnmatchedJoinHintWarning(TiDBIndexNestedLoopJoin, hintInfo.indexNestedLoopJoinTables) - b.appendUnmatchedJoinHintWarning(TiDBMergeJoin, hintInfo.sortMergeJoinTables) - b.appendUnmatchedJoinHintWarning(TiDBHashJoin, hintInfo.hashJoinTables) + b.appendUnmatchedJoinHintWarning(HintINLJ, TiDBIndexNestedLoopJoin, hintInfo.indexNestedLoopJoinTables) + b.appendUnmatchedJoinHintWarning(HintSMJ, TiDBMergeJoin, hintInfo.sortMergeJoinTables) + b.appendUnmatchedJoinHintWarning(HintHJ, TiDBHashJoin, hintInfo.hashJoinTables) b.tableHintInfo = b.tableHintInfo[:len(b.tableHintInfo)-1] } -func (b *PlanBuilder) appendUnmatchedJoinHintWarning(joinType string, hintTables []hintTableInfo) { +func (b *PlanBuilder) appendUnmatchedJoinHintWarning(joinType string, joinTypeAlias string, hintTables []hintTableInfo) { unMatchedTables := extractUnmatchedTables(hintTables) if len(unMatchedTables) == 0 { return } - errMsg := fmt.Sprintf("There are no matching table names for (%s) in optimizer hint %s. Maybe you can use the table alias name", - strings.Join(unMatchedTables, ", "), restore2JoinHint(joinType, hintTables)) + errMsg := fmt.Sprintf("There are no matching table names for (%s) in optimizer hint %s or %s. Maybe you can use the table alias name", + strings.Join(unMatchedTables, ", "), restore2JoinHint(joinType, hintTables), restore2JoinHint(joinTypeAlias, hintTables)) b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack(errMsg)) } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 6d41b9fa6d0f8..0a0f8da0d7e28 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1481,15 +1481,15 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { }{ { sql: "SELECT /*+ TIDB_SMJ(t3, t4) */ * from t t1, t t2 where t1.a = t2.a", - warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ TIDB_SMJ(t3, t4) */. Maybe you can use the table alias name", + warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ SM_JOIN(t3, t4) */ or /*+ TIDB_SMJ(t3, t4) */. Maybe you can use the table alias name", }, { sql: "SELECT /*+ TIDB_HJ(t3, t4) */ * from t t1, t t2 where t1.a = t2.a", - warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ TIDB_HJ(t3, t4) */. Maybe you can use the table alias name", + warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ HASH_JOIN(t3, t4) */ or /*+ TIDB_HJ(t3, t4) */. Maybe you can use the table alias name", }, { sql: "SELECT /*+ TIDB_INLJ(t3, t4) */ * from t t1, t t2 where t1.a = t2.a", - warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ TIDB_INLJ(t3, t4) */. Maybe you can use the table alias name", + warning: "[planner:1815]There are no matching table names for (t3, t4) in optimizer hint /*+ INL_JOIN(t3, t4) */ or /*+ TIDB_INLJ(t3, t4) */. Maybe you can use the table alias name", }, { sql: "SELECT /*+ TIDB_SMJ(t1, t2) */ * from t t1, t t2 where t1.a = t2.a", @@ -1497,7 +1497,7 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { }, { sql: "SELECT /*+ TIDB_SMJ(t3, t4) */ * from t t1, t t2, t t3 where t1.a = t2.a and t2.a = t3.a", - warning: "[planner:1815]There are no matching table names for (t4) in optimizer hint /*+ TIDB_SMJ(t3, t4) */. Maybe you can use the table alias name", + warning: "[planner:1815]There are no matching table names for (t4) in optimizer hint /*+ SM_JOIN(t3, t4) */ or /*+ TIDB_SMJ(t3, t4) */. Maybe you can use the table alias name", }, } for _, test := range tests { @@ -1542,8 +1542,8 @@ func (s *testPlanSuite) TestJoinHints(c *C) { }, { sql: "select /*+ TIDB_INLJ(t1) */ t1.b, t2.a from t t1, t t2 where t1.b = t2.a;", - best: "LeftHashJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t1.b,test.t2.a)", - warning: "[planner:1815]Optimizer Hint /*+ TIDB_INLJ(t1) */ is inapplicable", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.b,test.t2.a)", + warning: "[planner:1815]Optimizer Hint /*+ INL_JOIN(t1) */ or /*+ TIDB_INLJ(t1) */ is inapplicable", }, { sql: "select /*+ TIDB_INLJ(t2) */ t1.b, t2.a from t2 t1, t2 t2 where t1.b=t2.b and t2.c=-1;", @@ -1604,18 +1604,18 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { }, // with Aggregation hints { - sql: "select /*+ TIDB_HASHAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", warning: "", }, { - sql: "select /*+ TIDB_STREAMAGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + sql: "select /*+ STREAM_AGG */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", warning: "", }, // test conflict warning { - sql: "select /*+ TIDB_HASHAGG() TIDB_STREAMAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG STREAM_AGG */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", warning: "[planner:1815]Optimizer aggregation hints are conflicted", }, @@ -1697,19 +1697,19 @@ func (s *testPlanSuite) TestHintScope(c *C) { }, // aggregation hints { - sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select /*+ TIDB_HASHAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + sql: "select /*+ STREAM_AGG */ s, count(s) from (select /*+ HASH_AGG */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", }, { - sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select /*+ TIDB_STREAMAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + sql: "select /*+ HASH_AGG */ s, count(s) from (select /*+ STREAM_AGG */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Sort->Projection->StreamAgg->HashAgg->Projection", }, { - sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + sql: "select /*+ HASH_AGG */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->HashAgg->Projection", }, { - sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + sql: "select /*+ STREAM_AGG */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", }, } @@ -1728,3 +1728,50 @@ func (s *testPlanSuite) TestHintScope(c *C) { c.Assert(warnings, HasLen, 0, comment) } } + +func (s *testPlanSuite) TestHintAlias(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql1 string + sql2 string + }{ + { + sql1: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t3) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + sql2: "select /*+ SM_JOIN(t1) */ t1.a, t1.b from t t1, (select /*+ INL_JOIN(t3) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + }, + { + sql1: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + sql2: "select /*+ HASH_JOIN(t1) */ t1.a, t1.b from t t1, (select /*+ SM_JOIN(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + }, + { + sql1: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + sql2: "select /*+ INL_JOIN(t1) */ t1.a, t1.b from t t1, (select /*+ HASH_JOIN(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + }, + } + ctx := context.TODO() + for i, tt := range tests { + comment := Commentf("case:%v sql1:%s sql2:%s", i, tt.sql1, tt.sql2) + stmt1, err := s.ParseOneStmt(tt.sql1, "", "") + c.Assert(err, IsNil, comment) + stmt2, err := s.ParseOneStmt(tt.sql2, "", "") + c.Assert(err, IsNil, comment) + + p1, err := planner.Optimize(ctx, se, stmt1, s.is) + c.Assert(err, IsNil) + p2, err := planner.Optimize(ctx, se, stmt2, s.is) + c.Assert(err, IsNil) + + c.Assert(core.ToString(p1), Equals, core.ToString(p2)) + } +} From ada3a8291a29c712ece17846ae99e1bb04727ede Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Thu, 15 Aug 2019 13:49:19 +0800 Subject: [PATCH 03/17] revert hint scope & remove test (#11740) --- planner/core/logical_plan_builder.go | 39 +++++++----- planner/core/physical_plan_test.go | 88 ---------------------------- 2 files changed, 23 insertions(+), 104 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 84f9244edd6be..5d85484114e19 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1942,7 +1942,7 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi return resultList, nil } -func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { +func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { var sortMergeTables, INLJTables, hashJoinTables []hintTableInfo var preferAggType uint for _, hint := range hints { @@ -1961,12 +1961,16 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { // ignore hints that not implemented } } - b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ - sortMergeJoinTables: sortMergeTables, - indexNestedLoopJoinTables: INLJTables, - hashJoinTables: hashJoinTables, - preferAggType: preferAggType, - }) + if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables) > 0 || preferAggType != 0 { + b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ + sortMergeJoinTables: sortMergeTables, + indexNestedLoopJoinTables: INLJTables, + hashJoinTables: hashJoinTables, + preferAggType: preferAggType, + }) + return true + } + return false } func (b *PlanBuilder) popTableHints() { @@ -1996,9 +2000,10 @@ func (b *PlanBuilder) TableHints() *tableHintInfo { } func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p LogicalPlan, err error) { - b.pushTableHints(sel.TableHints) - // table hints are only visible in the current SELECT statement. - defer b.popTableHints() + if b.pushTableHints(sel.TableHints) { + // table hints are only visible in the current SELECT statement. + defer b.popTableHints() + } if sel.SelectStmtOpts != nil { origin := b.inStraightJoin @@ -2545,9 +2550,10 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio } func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { - b.pushTableHints(update.TableHints) - // table hints are only visible in the current UPDATE statement. - defer b.popTableHints() + if b.pushTableHints(update.TableHints) { + // table hints are only visible in the current UPDATE statement. + defer b.popTableHints() + } // update subquery table should be forbidden var asNameList []string @@ -2764,9 +2770,10 @@ func extractTableAsNameForUpdate(p LogicalPlan, asNames map[*model.TableInfo][]* } func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) (Plan, error) { - b.pushTableHints(delete.TableHints) - // table hints are only visible in the current DELETE statement. - defer b.popTableHints() + if b.pushTableHints(delete.TableHints) { + // table hints are only visible in the current DELETE statement. + defer b.popTableHints() + } sel := &ast.SelectStmt{ Fields: &ast.FieldList{}, diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 0a0f8da0d7e28..6f10e402ac536 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1641,94 +1641,6 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { } } -func (s *testPlanSuite) TestHintScope(c *C) { - defer testleak.AfterTest(c)() - store, dom, err := newStoreWithBootstrap() - c.Assert(err, IsNil) - defer func() { - dom.Close() - store.Close() - }() - se, err := session.CreateSession4Test(store) - c.Assert(err, IsNil) - _, err = se.Execute(context.Background(), "use test") - c.Assert(err, IsNil) - - tests := []struct { - sql string - best string - }{ - // join hints - { - sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t3) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", - }, - { - sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "MergeInnerJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)->Sort}(test.t1.a,test.t2.a)->Projection", - }, - { - sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", - }, - { - sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", - }, - { - sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", - }, - { - sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "RightHashJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t3.c,test.t2.a)}(test.t1.a,test.t2.a)->Projection", - }, - { - sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "MergeInnerJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", - }, - { - sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", - }, - { - sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", - }, - // aggregation hints - { - sql: "select /*+ STREAM_AGG */ s, count(s) from (select /*+ HASH_AGG */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", - }, - { - sql: "select /*+ HASH_AGG */ s, count(s) from (select /*+ STREAM_AGG */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Sort->Projection->StreamAgg->HashAgg->Projection", - }, - { - sql: "select /*+ HASH_AGG */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->HashAgg->Projection", - }, - { - sql: "select /*+ STREAM_AGG */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", - }, - } - - ctx := context.Background() - for i, test := range tests { - comment := Commentf("case:%v sql:%s", i, test) - stmt, err := s.ParseOneStmt(test.sql, "", "") - c.Assert(err, IsNil, comment) - - p, err := planner.Optimize(ctx, se, stmt, s.is) - c.Assert(err, IsNil) - c.Assert(core.ToString(p), Equals, test.best) - - warnings := se.GetSessionVars().StmtCtx.GetWarnings() - c.Assert(warnings, HasLen, 0, comment) - } -} - func (s *testPlanSuite) TestHintAlias(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() From 20a9758ee0ed4daee2e614839b9218ef493fd08e Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Tue, 20 Aug 2019 10:32:18 +0800 Subject: [PATCH 04/17] planner: fix incorrect table name in explain information (#11780) --- cmd/explaintest/r/explain_complex.result | 36 +++++++++---------- .../r/explain_complex_stats.result | 2 +- cmd/explaintest/r/explain_easy.result | 4 +-- cmd/explaintest/r/tpch.result | 4 +-- planner/core/exhaust_physical_plans.go | 2 +- planner/core/find_best_task.go | 1 + planner/core/logical_plan_builder.go | 8 ++--- planner/core/planbuilder.go | 2 +- 8 files changed, 29 insertions(+), 30 deletions(-) diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result index 6f273706d1e94..4ce1c27af3e57 100644 --- a/cmd/explaintest/r/explain_complex.result +++ b/cmd/explaintest/r/explain_complex.result @@ -126,7 +126,7 @@ Projection_13 1.00 root test.gad.id, test.dd.id, test.gad.aid, test.gad.cm, test └─IndexLookUp_33 3.33 root ├─IndexScan_30 3333.33 cop table:gad, index:t, range:(1478143908,+inf], keep order:false, stats:pseudo └─Selection_32 3.33 cop eq(test.gad.pt, "android"), not(isnull(test.gad.ip)) - └─TableScan_31 3333.33 cop table:st, keep order:false, stats:pseudo + └─TableScan_31 3333.33 cop table:gad, keep order:false, stats:pseudo explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000; id count task operator info Projection_10 0.00 root test.gad.id, test.sdk.id, test.gad.aid, test.gad.cm, test.sdk.dic, test.sdk.ip, test.sdk.t, test.gad.p1, test.gad.p2, test.gad.p3, test.gad.p4, test.gad.p5, test.gad.p6_md5, test.gad.p7_md5, test.gad.ext @@ -135,11 +135,11 @@ Projection_10 0.00 root test.gad.id, test.sdk.id, test.gad.aid, test.gad.cm, tes ├─IndexLookUp_27 0.00 root │ ├─IndexScan_24 3333.33 cop table:gad, index:t, range:(1477971479,+inf], keep order:false, stats:pseudo │ └─Selection_26 0.00 cop eq(test.gad.bm, 0), eq(test.gad.dit, "mac"), eq(test.gad.pt, "ios"), not(isnull(test.gad.dic)) - │ └─TableScan_25 3333.33 cop table:st, keep order:false, stats:pseudo + │ └─TableScan_25 3333.33 cop table:gad, keep order:false, stats:pseudo └─IndexLookUp_17 0.00 root ├─IndexScan_14 10.00 cop table:sdk, index:aid, dic, range: decided by [eq(test.sdk.aid, test.gad.aid)], keep order:false, stats:pseudo └─Selection_16 0.00 cop eq(test.sdk.bm, 0), eq(test.sdk.pt, "ios"), gt(test.sdk.t, 1477971479), not(isnull(test.sdk.mac)), not(isnull(test.sdk.t)) - └─TableScan_15 10.00 cop table:dd, keep order:false, stats:pseudo + └─TableScan_15 10.00 cop table:sdk, keep order:false, stats:pseudo explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5; id count task operator info Projection_5 1.00 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, 3_col_0, 3_col_1 @@ -244,18 +244,18 @@ explain SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.lead id count task operator info Sort_10 1.00 root test.d.left_value:asc └─HashAgg_15 1.00 root group by:test.d.id, funcs:firstrow(test.d.id), firstrow(test.d.ctx), firstrow(test.d.name), firstrow(test.d.left_value), firstrow(test.d.right_value), firstrow(test.d.depth), firstrow(test.d.leader_id), firstrow(test.d.status), firstrow(test.d.created_on), firstrow(test.d.updated_on) - └─Selection_20 0.01 root or(eq(test.ep.user_id, 62), or(eq(test.d.id, 20), eq(test.d.id, 20))) - └─HashLeftJoin_21 0.02 root left outer join, inner:TableReader_55, equal:[eq(test.p.id, test.ep.position_id)] - ├─IndexJoin_29 0.01 root left outer join, inner:IndexLookUp_28, outer key:test.d.id, inner key:test.p.department_id - │ ├─IndexLookUp_45 0.01 root - │ │ ├─IndexScan_42 10.00 cop table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo - │ │ └─Selection_44 0.01 cop eq(test.d.status, 1000) - │ │ └─TableScan_43 10.00 cop table:org_department, keep order:false, stats:pseudo - │ └─IndexLookUp_28 0.01 root - │ ├─Selection_26 9.99 cop not(isnull(test.p.department_id)) - │ │ └─IndexScan_24 10.00 cop table:p, index:department_id, range: decided by [eq(test.p.department_id, test.d.id)], keep order:false, stats:pseudo - │ └─Selection_27 0.01 cop eq(test.p.status, 1000) - │ └─TableScan_25 9.99 cop table:org_position, keep order:false, stats:pseudo - └─TableReader_55 9.99 root data:Selection_54 - └─Selection_54 9.99 cop eq(test.ep.status, 1000), not(isnull(test.ep.position_id)) - └─TableScan_53 10000.00 cop table:ep, range:[-inf,+inf], keep order:false, stats:pseudo + └─Selection_22 0.01 root or(eq(test.ep.user_id, 62), or(eq(test.d.id, 20), eq(test.d.id, 20))) + └─HashLeftJoin_23 0.02 root left outer join, inner:TableReader_57, equal:[eq(test.p.id, test.ep.position_id)] + ├─IndexJoin_32 0.01 root left outer join, inner:IndexLookUp_31, outer key:test.d.id, inner key:test.p.department_id + │ ├─IndexLookUp_47 0.01 root + │ │ ├─IndexScan_44 10.00 cop table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo + │ │ └─Selection_46 0.01 cop eq(test.d.status, 1000) + │ │ └─TableScan_45 10.00 cop table:d, keep order:false, stats:pseudo + │ └─IndexLookUp_31 0.01 root + │ ├─Selection_29 9.99 cop not(isnull(test.p.department_id)) + │ │ └─IndexScan_27 10.00 cop table:p, index:department_id, range: decided by [eq(test.p.department_id, test.d.id)], keep order:false, stats:pseudo + │ └─Selection_30 0.01 cop eq(test.p.status, 1000) + │ └─TableScan_28 9.99 cop table:p, keep order:false, stats:pseudo + └─TableReader_57 9.99 root data:Selection_56 + └─Selection_56 9.99 cop eq(test.ep.status, 1000), not(isnull(test.ep.position_id)) + └─TableScan_55 10000.00 cop table:ep, range:[-inf,+inf], keep order:false, stats:pseudo diff --git a/cmd/explaintest/r/explain_complex_stats.result b/cmd/explaintest/r/explain_complex_stats.result index 419d1ba3834d8..2217d2bed2adb 100644 --- a/cmd/explaintest/r/explain_complex_stats.result +++ b/cmd/explaintest/r/explain_complex_stats.result @@ -147,7 +147,7 @@ Projection_10 170.34 root test.gad.id, test.sdk.id, test.gad.aid, test.gad.cm, t └─IndexLookUp_17 0.25 root ├─IndexScan_14 1.00 cop table:sdk, index:aid, dic, range: decided by [eq(test.sdk.aid, test.gad.aid)], keep order:false └─Selection_16 0.25 cop eq(test.sdk.bm, 0), eq(test.sdk.pt, "ios"), gt(test.sdk.t, 1477971479), not(isnull(test.sdk.mac)), not(isnull(test.sdk.t)) - └─TableScan_15 1.00 cop table:dd, keep order:false + └─TableScan_15 1.00 cop table:sdk, keep order:false explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5; id count task operator info Projection_5 39.28 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, 3_col_0, 3_col_1 diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 2c9ed5f634306..564963b7c141d 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -346,7 +346,7 @@ Projection_11 10000.00 root 9_aux_0 ├─IndexLookUp_29 9.99 root │ ├─IndexScan_26 10.00 cop table:s, index:b, range: decided by [eq(test.s.b, test.t.a)], keep order:false, stats:pseudo │ └─Selection_28 9.99 cop not(isnull(test.s.c)) - │ └─TableScan_27 10.00 cop table:t, keep order:false, stats:pseudo + │ └─TableScan_27 10.00 cop table:s, keep order:false, stats:pseudo └─TableReader_33 1.00 root data:TableScan_32 └─TableScan_32 1.00 cop table:t1, range: decided by [test.s.c], keep order:false, stats:pseudo insert into t values(1, 1, 1), (2, 2 ,2), (3, 3, 3), (4, 3, 4),(5,3,5); @@ -687,7 +687,7 @@ Projection_8 8320.83 root test.t.a, test.t1.a └─UnionScan_14 6656.67 root not(and(ge(test.t1.a, 1), le(test.t1.a, 2))), not(isnull(test.t1.a)) └─TableReader_17 6656.67 root data:Selection_16 └─Selection_16 6656.67 cop not(isnull(test.t1.a)), or(lt(test.t1.a, 1), gt(test.t1.a, 2)) - └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo + └─TableScan_15 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo rollback; drop table if exists t; create table t(a time, b date); diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 45f0cdc023f06..c5e42245f8978 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -1241,11 +1241,11 @@ Projection_25 1.00 root tpch.supplier.s_name, 17_col_0 │ │ └─TableScan_59 1.00 cop table:orders, range: decided by [tpch.l1.l_orderkey], keep order:false │ └─IndexLookUp_55 1.00 root │ ├─IndexScan_53 1.00 cop table:l2, index:L_ORDERKEY, L_LINENUMBER, range: decided by [eq(tpch.l2.l_orderkey, tpch.l1.l_orderkey)], keep order:false - │ └─TableScan_54 1.00 cop table:lineitem, keep order:false + │ └─TableScan_54 1.00 cop table:l2, keep order:false └─IndexLookUp_39 0.80 root ├─IndexScan_36 1.00 cop table:l3, index:L_ORDERKEY, L_LINENUMBER, range: decided by [eq(tpch.l3.l_orderkey, tpch.l1.l_orderkey)], keep order:false └─Selection_38 0.80 cop gt(tpch.l3.l_receiptdate, tpch.l3.l_commitdate) - └─TableScan_37 1.00 cop table:lineitem, keep order:false + └─TableScan_37 1.00 cop table:l3, keep order:false /* Q22 Global Sales Opportunity Query The Global Sales Opportunity Query identifies geographies where there are customers who may be likely to make a diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index e2ba41fc60db8..008c3ee434f54 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -564,7 +564,7 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn } if !isCoveringIndex(ds.schema.Columns, is.Index.Columns, is.Table.PKIsHandle) { // On this way, it's double read case. - ts := PhysicalTableScan{Columns: ds.Columns, Table: is.Table}.Init(ds.ctx) + ts := PhysicalTableScan{Columns: ds.Columns, Table: is.Table, TableAsName: ds.TableAsName}.Init(ds.ctx) ts.SetSchema(is.dataSourceSchema) cop.tablePlan = ts } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 2be6c5982ed22..3664827ffc48a 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -519,6 +519,7 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid ts := PhysicalTableScan{ Columns: ds.Columns, Table: is.Table, + TableAsName: ds.TableAsName, isPartition: ds.isPartition, physicalTableID: ds.physicalTableID, }.Init(ds.ctx) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 5d85484114e19..3fb46e2365f96 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -164,7 +164,7 @@ func (b *PlanBuilder) buildResultSetNode(ctx context.Context, node ast.ResultSet case *ast.UnionStmt: p, err = b.buildUnion(ctx, v) case *ast.TableName: - p, err = b.buildDataSource(ctx, v) + p, err = b.buildDataSource(ctx, v, &x.AsName) default: err = ErrUnsupportedType.GenWithStackByArgs(v) } @@ -172,9 +172,6 @@ func (b *PlanBuilder) buildResultSetNode(ctx context.Context, node ast.ResultSet return nil, err } - if v, ok := p.(*DataSource); ok { - v.TableAsName = &x.AsName - } for _, col := range p.Schema().Columns { col.OrigTblName = col.TblName if x.AsName.L != "" { @@ -2216,7 +2213,7 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) return statsTbl } -func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (LogicalPlan, error) { +func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, asName *model.CIStr) (LogicalPlan, error) { dbName := tn.Schema if dbName.L == "" { dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB) @@ -2276,6 +2273,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L ds := DataSource{ DBName: dbName, + TableAsName: asName, table: tbl, tableInfo: tableInfo, statisticTable: statisticTable, diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index a6efe8941258b..08afddb446837 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -791,7 +791,7 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(ctx context.Context, dbName }.Init(b.ctx) is.stats = property.NewSimpleStats(0) // It's double read case. - ts := PhysicalTableScan{Columns: tblReaderCols, Table: is.Table}.Init(b.ctx) + ts := PhysicalTableScan{Columns: tblReaderCols, Table: is.Table, TableAsName: &tblInfo.Name}.Init(b.ctx) ts.SetSchema(tblSchema) cop := &copTask{indexPlan: is, tablePlan: ts} ts.HandleIdx = pkOffset From 6173f046fae98b9734885059532306504e77637f Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Thu, 22 Aug 2019 12:38:36 +0800 Subject: [PATCH 05/17] *: support query block syntax (#11761) --- go.mod | 2 +- go.sum | 4 ++-- planner/core/physical_plan_test.go | 6 +++--- planner/core/planbuilder.go | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 5557bafa5b17c..0fc50a98896b6 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb + github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index eda0d8344f9d5..0682e365e1842 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb h1:7VLFE2QHeZIhnpKK0XRESkgXj/PtJguIkDZvW3MinN4= -github.com/pingcap/parser v0.0.0-20190814092219-43d2d687aecb/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864 h1:ohPZBjGBLoRaaJi9UFYSgDs2dTty9YEyF+TWs7DBo58= +github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 6f10e402ac536..5f15482b7fdb9 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1604,18 +1604,18 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { }, // with Aggregation hints { - sql: "select /*+ HASH_AGG */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", warning: "", }, { - sql: "select /*+ STREAM_AGG */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + sql: "select /*+ STREAM_AGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", warning: "", }, // test conflict warning { - sql: "select /*+ HASH_AGG STREAM_AGG */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG() STREAM_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", warning: "[planner:1815]Optimizer aggregation hints are conflicted", }, diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 08afddb446837..b2d1f05211f5d 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -64,15 +64,15 @@ type hintTableInfo struct { matched bool } -func tableNames2HintTableInfo(tableNames []model.CIStr) []hintTableInfo { - if len(tableNames) == 0 { +func tableNames2HintTableInfo(hintTables []ast.HintTable) []hintTableInfo { + if len(hintTables) == 0 { return nil } - hintTables := make([]hintTableInfo, 0, len(tableNames)) - for _, tableName := range tableNames { - hintTables = append(hintTables, hintTableInfo{name: tableName}) + hintTableInfos := make([]hintTableInfo, len(hintTables)) + for i, hintTable := range hintTables { + hintTableInfos[i] = hintTableInfo{name: hintTable.TableName} } - return hintTables + return hintTableInfos } func (info *tableHintInfo) ifPreferMergeJoin(tableNames ...*model.CIStr) bool { From 6833d8cffe7cd2455e848b3aa7337920a81f97e1 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Fri, 23 Aug 2019 18:27:46 +0800 Subject: [PATCH 06/17] planner: add index optimizer hint (#11746) --- planner/core/logical_plan_builder.go | 35 +++++++++++--- planner/core/physical_plan_test.go | 72 ++++++++++++++++++++++++++++ planner/core/planbuilder.go | 29 +++++++++-- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 3fb46e2365f96..937bbf7312014 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -64,6 +64,8 @@ const ( HintHashAgg = "hash_agg" // HintStreamAgg is hint enforce stream aggregation. HintStreamAgg = "stream_agg" + // HintIndex is hint enforce using some indexes. + HintIndex = "index" ) const ( @@ -1940,29 +1942,44 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi } func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { - var sortMergeTables, INLJTables, hashJoinTables []hintTableInfo - var preferAggType uint + var ( + sortMergeTables, INLJTables, hashJoinTables []hintTableInfo + indexHintList []indexHintInfo + preferAggType uint + ) for _, hint := range hints { switch hint.HintName.L { case TiDBMergeJoin, HintSMJ: - sortMergeTables = tableNames2HintTableInfo(hint.Tables) + sortMergeTables = append(sortMergeTables, tableNames2HintTableInfo(hint.Tables)...) case TiDBIndexNestedLoopJoin, HintINLJ: - INLJTables = tableNames2HintTableInfo(hint.Tables) + INLJTables = append(INLJTables, tableNames2HintTableInfo(hint.Tables)...) case TiDBHashJoin, HintHJ: - hashJoinTables = tableNames2HintTableInfo(hint.Tables) + hashJoinTables = append(hashJoinTables, tableNames2HintTableInfo(hint.Tables)...) case HintHashAgg: preferAggType |= preferHashAgg case HintStreamAgg: preferAggType |= preferStreamAgg + case HintIndex: + if len(hint.Tables) != 0 && len(hint.Indexes) != 0 { + indexHintList = append(indexHintList, indexHintInfo{ + tblName: hint.Tables[0].TableName, + indexHint: &ast.IndexHint{ + IndexNames: hint.Indexes, + HintType: ast.HintUse, + HintScope: ast.HintForScan, + }, + }) + } default: // ignore hints that not implemented } } - if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables) > 0 || preferAggType != 0 { + if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables)+len(indexHintList) > 0 || preferAggType != 0 { b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ sortMergeJoinTables: sortMergeTables, indexNestedLoopJoinTables: INLJTables, hashJoinTables: hashJoinTables, + indexHintList: indexHintList, preferAggType: preferAggType, }) return true @@ -2248,7 +2265,11 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as return nil, ErrPartitionClauseOnNonpartitioned } - possiblePaths, err := getPossibleAccessPaths(tn.IndexHints, tableInfo) + tblName := *asName + if tblName.L == "" { + tblName = tn.Name + } + possiblePaths, err := b.getPossibleAccessPaths(tn.IndexHints, tableInfo, tblName) if err != nil { return nil, err } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 5f15482b7fdb9..854c862ab2290 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1687,3 +1687,75 @@ func (s *testPlanSuite) TestHintAlias(c *C) { c.Assert(core.ToString(p1), Equals, core.ToString(p2)) } } + +func (s *testPlanSuite) TestIndexHint(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + hasWarn bool + }{ + // simple case + { + sql: "select /*+ INDEX(t, c_d_e) */ * from t", + best: "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", + hasWarn: false, + }, + { + sql: "select /*+ INDEX(t, c_d_e) */ * from t t1", + best: "TableReader(Table(t))", + hasWarn: false, + }, + { + sql: "select /*+ INDEX(t1, c_d_e) */ * from t t1", + best: "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", + hasWarn: false, + }, + { + sql: "select /*+ INDEX(t1, c_d_e), INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))}(test.t1.a,test.t2.b)", + hasWarn: false, + }, + // test multiple indexes + { + sql: "select /*+ INDEX(t, c_d_e, f, g) */ * from t order by f", + best: "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))", + hasWarn: false, + }, + // there will be a warning instead of error when index not exist + { + sql: "select /*+ INDEX(t, no_such_index) */ * from t", + best: "TableReader(Table(t))", + hasWarn: true, + }, + } + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best, comment) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + if test.hasWarn { + c.Assert(warnings, HasLen, 1, comment) + } else { + c.Assert(warnings, HasLen, 0, comment) + } + + } +} diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index b2d1f05211f5d..b36340174a94a 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -56,6 +56,7 @@ type tableHintInfo struct { indexNestedLoopJoinTables []hintTableInfo sortMergeJoinTables []hintTableInfo hashJoinTables []hintTableInfo + indexHintList []indexHintInfo preferAggType uint } @@ -64,6 +65,11 @@ type hintTableInfo struct { matched bool } +type indexHintInfo struct { + tblName model.CIStr + indexHint *ast.IndexHint +} + func tableNames2HintTableInfo(hintTables []ast.HintTable) []hintTableInfo { if len(hintTables) == 0 { return nil @@ -461,7 +467,7 @@ func isPrimaryIndex(indexName model.CIStr) bool { return indexName.L == "primary" } -func getPossibleAccessPaths(indexHints []*ast.IndexHint, tblInfo *model.TableInfo) ([]*accessPath, error) { +func (b *PlanBuilder) getPossibleAccessPaths(indexHints []*ast.IndexHint, tblInfo *model.TableInfo, tblName model.CIStr) ([]*accessPath, error) { publicPaths := make([]*accessPath, 0, len(tblInfo.Indices)+1) publicPaths = append(publicPaths, &accessPath{isTablePath: true}) for _, index := range tblInfo.Indices { @@ -473,7 +479,18 @@ func getPossibleAccessPaths(indexHints []*ast.IndexHint, tblInfo *model.TableInf hasScanHint, hasUseOrForce := false, false available := make([]*accessPath, 0, len(publicPaths)) ignored := make([]*accessPath, 0, len(publicPaths)) - for _, hint := range indexHints { + + // Extract comment-style index hint like /*+ INDEX(t, idx1, idx2) */. + indexHintsLen := len(indexHints) + if hints := b.TableHints(); hints != nil { + for _, hint := range hints.indexHintList { + if hint.tblName == tblName { + indexHints = append(indexHints, hint.indexHint) + } + } + } + + for i, hint := range indexHints { if hint.HintScope != ast.HintForScan { continue } @@ -493,7 +510,13 @@ func getPossibleAccessPaths(indexHints []*ast.IndexHint, tblInfo *model.TableInf for _, idxName := range hint.IndexNames { path := getPathByIndexName(publicPaths, idxName, tblInfo) if path == nil { - return nil, ErrKeyDoesNotExist.GenWithStackByArgs(idxName, tblInfo.Name) + err := ErrKeyDoesNotExist.GenWithStackByArgs(idxName, tblInfo.Name) + // if hint is from comment-style sql hints, we should throw a warning instead of error. + if i < indexHintsLen { + return nil, err + } + b.ctx.GetSessionVars().StmtCtx.AppendWarning(err) + continue } if hint.HintType == ast.HintIgnore { // Collect all the ignored index hints. From c8f7267d4674a3bf5f55f38a5daad0888ad73749 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Thu, 29 Aug 2019 19:14:38 +0900 Subject: [PATCH 07/17] planner/core: support query block hint (#11861) --- executor/show.go | 2 +- go.mod | 2 +- go.sum | 4 +- planner/core/expression_rewriter.go | 5 +- planner/core/hints.go | 156 +++++++++++++++++++++++++++ planner/core/logical_plan_builder.go | 9 +- planner/core/logical_plan_test.go | 49 +++++---- planner/core/optimizer.go | 7 +- planner/core/physical_plan_test.go | 66 ++++++++++++ planner/core/planbuilder.go | 11 +- planner/core/planbuilder_test.go | 5 +- planner/core/prepare_test.go | 2 +- planner/optimize.go | 4 +- 13 files changed, 280 insertions(+), 42 deletions(-) create mode 100644 planner/core/hints.go diff --git a/executor/show.go b/executor/show.go index 2b5553bc16727..710da35a2915f 100644 --- a/executor/show.go +++ b/executor/show.go @@ -383,7 +383,7 @@ func (e *ShowExec) fetchShowColumns(ctx context.Context) error { if tb.Meta().IsView() { // Because view's undertable's column could change or recreate, so view's column type may change overtime. // To avoid this situation we need to generate a logical plan and extract current column types from Schema. - planBuilder := plannercore.NewPlanBuilder(e.ctx, e.is) + planBuilder := plannercore.NewPlanBuilder(e.ctx, e.is, &plannercore.BlockHintProcessor{}) viewLogicalPlan, err := planBuilder.BuildDataSourceFromView(ctx, e.DBName, tb.Meta()) if err != nil { return err diff --git a/go.mod b/go.mod index 0fc50a98896b6..2a976ee72eca9 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864 + github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 0682e365e1842..3d7d00ed95842 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864 h1:ohPZBjGBLoRaaJi9UFYSgDs2dTty9YEyF+TWs7DBo58= -github.com/pingcap/parser v0.0.0-20190822024127-41d48df05864/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c h1:VcQ60QkH7Vv98dbedNKDEO7RfUw+6TE59r/lQokNz/k= +github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 42e3f50f36938..0e37315805601 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -44,8 +44,9 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error return val.Datum, nil } b := &PlanBuilder{ - ctx: sctx, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: sctx, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } if sctx.GetSessionVars().TxnCtx.InfoSchema != nil { b.is = sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) diff --git a/planner/core/hints.go b/planner/core/hints.go new file mode 100644 index 0000000000000..8b88473f78ec7 --- /dev/null +++ b/planner/core/hints.go @@ -0,0 +1,156 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/format" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/sessionctx" +) + +// BlockHintProcessor processes hints at different level of sql statement. +type BlockHintProcessor struct { + QbNameMap map[string]int // Map from query block name to select stmt offset. + QbHints map[int][]*ast.TableOptimizerHint // Group all hints at same query block. + Ctx sessionctx.Context + selectStmtOffset int +} + +// Enter implements Visitor interface. +func (p *BlockHintProcessor) Enter(in ast.Node) (ast.Node, bool) { + switch node := in.(type) { + case *ast.UpdateStmt: + p.checkQueryBlockHints(node.TableHints, 0) + case *ast.DeleteStmt: + p.checkQueryBlockHints(node.TableHints, 0) + case *ast.SelectStmt: + p.selectStmtOffset++ + node.QueryBlockOffset = p.selectStmtOffset + p.checkQueryBlockHints(node.TableHints, node.QueryBlockOffset) + } + return in, false +} + +// Leave implements Visitor interface. +func (p *BlockHintProcessor) Leave(in ast.Node) (ast.Node, bool) { + return in, true +} + +const hintQBName = "qb_name" + +// checkQueryBlockHints checks the validity of query blocks and records the map of query block name to select offset. +func (p *BlockHintProcessor) checkQueryBlockHints(hints []*ast.TableOptimizerHint, offset int) { + var qbName string + for _, hint := range hints { + if hint.HintName.L != hintQBName { + continue + } + if qbName != "" { + p.Ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New(fmt.Sprintf("There are more than two query names in same query block,, using the first one %s", qbName))) + } else { + qbName = hint.QBName.L + } + } + if qbName == "" { + return + } + if p.QbNameMap == nil { + p.QbNameMap = make(map[string]int) + } + if _, ok := p.QbNameMap[qbName]; ok { + p.Ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New(fmt.Sprintf("Duplicate query block name %s, only the first one is effective", qbName))) + } else { + p.QbNameMap[qbName] = offset + } +} + +const ( + defaultUpdateBlockName = "upd_1" + defaultDeleteBlockName = "del_1" + defaultSelectBlockPrefix = "sel_" +) + +type nodeType int + +const ( + typeUpdate nodeType = iota + typeDelete + typeSelect +) + +// getBlockName finds the offset of query block name. It use 0 as offset for top level update or delete, +// -1 for invalid block name. +func (p *BlockHintProcessor) getBlockOffset(blockName model.CIStr, nodeType nodeType) int { + if p.QbNameMap != nil { + level, ok := p.QbNameMap[blockName.L] + if ok { + return level + } + } + // Handle the default query block name. + if nodeType == typeUpdate && blockName.L == defaultUpdateBlockName { + return 0 + } + if nodeType == typeDelete && blockName.L == defaultDeleteBlockName { + return 0 + } + if nodeType == typeSelect && strings.HasPrefix(blockName.L, defaultSelectBlockPrefix) { + suffix := blockName.L[len(defaultSelectBlockPrefix):] + level, err := strconv.ParseInt(suffix, 10, 64) + if err != nil || level > int64(p.selectStmtOffset) { + return -1 + } + return int(level) + } + return -1 +} + +// getHintOffset gets the offset of stmt that the hints take effects. +func (p *BlockHintProcessor) getHintOffset(hint *ast.TableOptimizerHint, nodeType nodeType, currentOffset int) int { + if hint.QBName.L != "" { + return p.getBlockOffset(hint.QBName, nodeType) + } + return currentOffset +} + +// getCurrentStmtHints extracts all hints that take effects at current stmt. +func (p *BlockHintProcessor) getCurrentStmtHints(hints []*ast.TableOptimizerHint, nodeType nodeType, currentOffset int) []*ast.TableOptimizerHint { + if p.QbHints == nil { + p.QbHints = make(map[int][]*ast.TableOptimizerHint) + } + for _, hint := range hints { + if hint.HintName.L == hintQBName { + continue + } + offset := p.getHintOffset(hint, nodeType, currentOffset) + if offset < 0 { + var sb strings.Builder + ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb) + err := hint.Restore(ctx) + // There won't be any error for optimizer hint. + if err == nil { + p.Ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New(fmt.Sprintf("Hint %s is ignored due to unknown query block name", sb.String()))) + } + continue + } + p.QbHints[offset] = append(p.QbHints[offset], hint) + } + return p.QbHints[currentOffset] +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 937bbf7312014..76c34569a2490 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1941,7 +1941,8 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi return resultList, nil } -func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { +func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType nodeType, currentLevel int) bool { + hints = b.hintProcessor.getCurrentStmtHints(hints, nodeType, currentLevel) var ( sortMergeTables, INLJTables, hashJoinTables []hintTableInfo indexHintList []indexHintInfo @@ -2014,7 +2015,7 @@ func (b *PlanBuilder) TableHints() *tableHintInfo { } func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p LogicalPlan, err error) { - if b.pushTableHints(sel.TableHints) { + if b.pushTableHints(sel.TableHints, typeSelect, sel.QueryBlockOffset) { // table hints are only visible in the current SELECT statement. defer b.popTableHints() } @@ -2569,7 +2570,7 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio } func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { - if b.pushTableHints(update.TableHints) { + if b.pushTableHints(update.TableHints, typeUpdate, 0) { // table hints are only visible in the current UPDATE statement. defer b.popTableHints() } @@ -2789,7 +2790,7 @@ func extractTableAsNameForUpdate(p LogicalPlan, asNames map[*model.TableInfo][]* } func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) (Plan, error) { - if b.pushTableHints(delete.TableHints) { + if b.pushTableHints(delete.TableHints, typeDelete, 0) { // table hints are only visible in the current DELETE statement. defer b.popTableHints() } diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index e1dfc14302692..5709d8c8a85c1 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -1798,9 +1798,10 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - colMapper: make(map[*ast.ColumnNameExpr]int), - ctx: MockContext(), - is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + hintProcessor: &BlockHintProcessor{}, } builder.ctx.GetSessionVars().HashJoinConcurrency = 1 _, err = builder.Build(context.TODO(), stmt) @@ -1917,9 +1918,10 @@ func (s *testPlanSuite) TestUnion(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } plan, err := builder.Build(ctx, stmt) if tt.err { @@ -2050,9 +2052,10 @@ func (s *testPlanSuite) TestTopNPushDown(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) @@ -2177,9 +2180,10 @@ func (s *testPlanSuite) TestOuterJoinEliminator(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) @@ -2209,9 +2213,10 @@ func (s *testPlanSuite) TestSelectView(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) @@ -2494,9 +2499,10 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, stmt) if err != nil { @@ -2584,9 +2590,10 @@ func (s *testPlanSuite) TestSkylinePruning(c *C) { c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: MockContext(), + is: s.is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, stmt) if err != nil { diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 04ac09e2be074..a2e76fbe03ac5 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -76,9 +76,10 @@ func BuildLogicalPlan(ctx context.Context, sctx sessionctx.Context, node ast.Nod sctx.GetSessionVars().PlanID = 0 sctx.GetSessionVars().PlanColumnID = 0 builder := &PlanBuilder{ - ctx: sctx, - is: is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: sctx, + is: is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: &BlockHintProcessor{}, } p, err := builder.Build(ctx, node) if err != nil { diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 854c862ab2290..6ddeee0a6f32c 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1756,6 +1756,72 @@ func (s *testPlanSuite) TestIndexHint(c *C) { } else { c.Assert(warnings, HasLen, 0, comment) } + } +} + +func (s *testPlanSuite) TestQueryBlockHint(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + plan string + }{ + { + sql: "select /*+ SM_JOIN(@sel_1 t1), INL_JOIN(@sel_2 t3) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ SM_JOIN(@sel_1 t1), INL_JOIN(@qb t3) */ t1.a, t1.b from t t1, (select /*+ QB_NAME(qb) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ HASH_JOIN(@sel_1 t1), SM_JOIN(@sel_2 t2) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ HASH_JOIN(@sel_1 t1), SM_JOIN(@qb t2) */ t1.a, t1.b from t t1, (select /*+ QB_NAME(qb) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ INL_JOIN(@sel_1 t1), HASH_JOIN(@sel_2 t2) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ INL_JOIN(@sel_1 t1), HASH_JOIN(@qb t2) */ t1.a, t1.b from t t1, (select /*+ QB_NAME(qb) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ HASH_AGG(@sel_1), STREAM_AGG(@sel_2) */ count(*) from t t1 where t1.a < (select count(*) from t t2 where t1.a > t2.a)", + plan: "Apply{TableReader(Table(t))->TableReader(Table(t)->Sel([gt(test.t1.a, test.t2.a)])->StreamAgg)->StreamAgg->Sel([not(isnull(count(*)))])}->HashAgg", + }, + { + sql: "select /*+ STREAM_AGG(@sel_1), HASH_AGG(@qb) */ count(*) from t t1 where t1.a < (select /*+ QB_NAME(qb) */ count(*) from t t2 where t1.a > t2.a)", + plan: "Apply{TableReader(Table(t))->TableReader(Table(t)->Sel([gt(test.t1.a, test.t2.a)])->HashAgg)->HashAgg->Sel([not(isnull(count(*)))])}->StreamAgg", + }, + { + sql: "select /*+ HASH_AGG(@sel_2) */ a, (select count(*) from t t1 where t1.b > t.a) from t where b > (select b from t t2 where t2.b = t.a limit 1)", + plan: "Apply{Apply{TableReader(Table(t))->TableReader(Table(t)->Sel([eq(test.t2.b, test.t.a)])->Limit)->Limit}->TableReader(Table(t)->Sel([gt(test.t1.b, test.t.a)])->HashAgg)->HashAgg}->Projection", + }, + } + ctx := context.TODO() + for i, tt := range tests { + comment := Commentf("case:%v sql: %s", i, tt.sql) + stmt, err := s.ParseOneStmt(tt.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil, comment) + c.Assert(core.ToString(p), Equals, tt.plan, comment) } } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index b36340174a94a..b3a10dde8faec 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -197,6 +197,8 @@ type PlanBuilder struct { inStraightJoin bool windowSpecs map[string]*ast.WindowSpec + + hintProcessor *BlockHintProcessor } // GetVisitInfo gets the visitInfo of the PlanBuilder. @@ -230,11 +232,12 @@ func (b *PlanBuilder) GetOptFlag() uint64 { } // NewPlanBuilder creates a new PlanBuilder. -func NewPlanBuilder(sctx sessionctx.Context, is infoschema.InfoSchema) *PlanBuilder { +func NewPlanBuilder(sctx sessionctx.Context, is infoschema.InfoSchema, processor *BlockHintProcessor) *PlanBuilder { return &PlanBuilder{ - ctx: sctx, - is: is, - colMapper: make(map[*ast.ColumnNameExpr]int), + ctx: sctx, + is: is, + colMapper: make(map[*ast.ColumnNameExpr]int), + hintProcessor: processor, } } diff --git a/planner/core/planbuilder_test.go b/planner/core/planbuilder_test.go index eca40338aa44a..54636648f2abd 100644 --- a/planner/core/planbuilder_test.go +++ b/planner/core/planbuilder_test.go @@ -97,7 +97,8 @@ func (s *testPlanBuilderSuite) TestGetPathByIndexName(c *C) { func (s *testPlanBuilderSuite) TestRewriterPool(c *C) { builder := &PlanBuilder{ - ctx: MockContext(), + ctx: MockContext(), + hintProcessor: &BlockHintProcessor{}, } // Make sure PlanBuilder.getExpressionRewriter() provides clean rewriter from pool. @@ -151,7 +152,7 @@ func (s *testPlanBuilderSuite) TestDisableFold(c *C) { stmt := st.(*ast.SelectStmt) expr := stmt.Fields.Fields[0].Expr - builder := &PlanBuilder{ctx: ctx} + builder := &PlanBuilder{ctx: ctx, hintProcessor: &BlockHintProcessor{}} builder.rewriterCounter++ rewriter := builder.getExpressionRewriter(context.TODO(), nil) c.Assert(rewriter, NotNil) diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index 607afd2a85f3d..984f41537e726 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -163,7 +163,7 @@ func (s *testPlanSuite) TestPrepareCacheDeferredFunction(c *C) { stmt, err := s.ParseOneStmt(sql1, "", "") c.Check(err, IsNil) is := tk.Se.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) - builder := core.NewPlanBuilder(tk.Se, is) + builder := core.NewPlanBuilder(tk.Se, is, &core.BlockHintProcessor{}) p, err := builder.Build(ctx, stmt) c.Check(err, IsNil) execPlan, ok := p.(*core.Execute) diff --git a/planner/optimize.go b/planner/optimize.go index a0f3008802dc0..c9da8c010641d 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -35,7 +35,9 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in // build logical plan sctx.GetSessionVars().PlanID = 0 sctx.GetSessionVars().PlanColumnID = 0 - builder := plannercore.NewPlanBuilder(sctx, is) + hintProcessor := &plannercore.BlockHintProcessor{Ctx: sctx} + node.Accept(hintProcessor) + builder := plannercore.NewPlanBuilder(sctx, is, hintProcessor) p, err := builder.Build(ctx, node) if err != nil { return nil, err From f6b6858772d2e5fc7064900111f534c8ae516bf9 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Wed, 4 Sep 2019 14:08:35 +0800 Subject: [PATCH 08/17] planner: fix aggregation hint didn't work in some cases (#11996) --- planner/core/expression_rewriter.go | 9 ++++ planner/core/logical_plan_builder.go | 9 ++-- planner/core/physical_plan_test.go | 63 +++++++++++++++------- planner/core/rule_aggregation_push_down.go | 22 ++++---- 4 files changed, 70 insertions(+), 33 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 0e37315805601..4f166c6e02039 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -442,6 +442,9 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, v *ast. // it will be rewrote to t.id < (select max(s.id) from s). func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression.Expression, np LogicalPlan, useMin bool, cmpFunc string, all bool) { plan4Agg := LogicalAggregation{}.Init(er.sctx) + if hint := er.b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } plan4Agg.SetChildren(np) // Create a "max" or "min" aggregation. @@ -568,6 +571,9 @@ func (er *expressionRewriter) handleNEAny(lexpr, rexpr expression.Expression, np plan4Agg := LogicalAggregation{ AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, }.Init(er.sctx) + if hint := er.b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ ColName: model.NewCIStr("col_firstRow"), @@ -602,6 +608,9 @@ func (er *expressionRewriter) handleEQAll(lexpr, rexpr expression.Expression, np plan4Agg := LogicalAggregation{ AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, }.Init(er.sctx) + if hint := er.b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ ColName: model.NewCIStr("col_firstRow"), diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 76c34569a2490..2cf879471f86d 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -97,6 +97,9 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu b.optFlag = b.optFlag | flagEliminateProjection plan4Agg := LogicalAggregation{AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(aggFuncList))}.Init(b.ctx) + if hint := b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } schema4Agg := expression.NewSchema(make([]*expression.Column, 0, len(aggFuncList)+p.Schema().Len())...) // aggIdxMap maps the old index to new index after applying common aggregation functions elimination. aggIndexMap := make(map[int]int) @@ -149,9 +152,6 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu plan4Agg.GroupByItems = gbyItems plan4Agg.SetSchema(schema4Agg) plan4Agg.collectGroupByColumns() - if hint := b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType - } return plan4Agg, aggIndexMap, nil } @@ -790,6 +790,9 @@ func (b *PlanBuilder) buildDistinct(child LogicalPlan, length int) (*LogicalAggr AggFuncs: make([]*aggregation.AggFuncDesc, 0, child.Schema().Len()), GroupByItems: expression.Column2Exprs(child.Schema().Clone().Columns[:length]), }.Init(b.ctx) + if hint := b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } plan4Agg.collectGroupByColumns() for _, col := range child.Schema().Columns { aggDesc, err := aggregation.NewAggFuncDesc(b.ctx, ast.AggFuncFirstRow, []expression.Expression{col}, false) diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 6ddeee0a6f32c..79f96c579d1bc 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1587,31 +1587,28 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { c.Assert(err, IsNil) tests := []struct { - sql string - best string - warning string + sql string + best string + warning string + aggPushDown bool }{ // without Aggregation hints { - sql: "select count(*) from t t1, t t2 where t1.a = t2.b", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", - warning: "", + sql: "select count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", }, { - sql: "select count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", - warning: "", + sql: "select count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", }, // with Aggregation hints { - sql: "select /*+ HASH_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", - warning: "", + sql: "select /*+ HASH_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", }, { - sql: "select /*+ STREAM_AGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", - warning: "", + sql: "select /*+ STREAM_AGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", }, // test conflict warning { @@ -1619,24 +1616,50 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", warning: "[planner:1815]Optimizer aggregation hints are conflicted", }, + // additional test + { + sql: "select /*+ STREAM_AGG() */ distinct a from t", + best: "TableReader(Table(t)->StreamAgg)->StreamAgg", + }, + { + sql: "select /*+ HASH_AGG() */ t1.a from t t1 where t1.a < any(select t2.b from t t2)", + best: "LeftHashJoin{TableReader(Table(t)->Sel([if(isnull(test.t1.a), , 1)]))->TableReader(Table(t)->HashAgg)->HashAgg->Sel([ne(agg_col_cnt, 0)])}->Projection->Projection", + }, + { + sql: "select /*+ hash_agg() */ t1.a from t t1 where t1.a != any(select t2.b from t t2)", + best: "LeftHashJoin{TableReader(Table(t)->Sel([if(isnull(test.t1.a), , 1)]))->TableReader(Table(t))->Projection->HashAgg->Sel([ne(agg_col_cnt, 0)])}->Projection->Projection", + }, + { + sql: "select /*+ hash_agg() */ t1.a from t t1 where t1.a = all(select t2.b from t t2)", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection->HashAgg}->Projection->Projection", + }, + { + sql: "select /*+ STREAM_AGG() */ sum(t1.a) from t t1 join t t2 on t1.b = t2.b group by t1.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Sort->Projection->StreamAgg}(test.t2.b,test.t1.b)->HashAgg", + warning: "[planner:1815]Optimizer Hint STREAM_AGG is inapplicable", + aggPushDown: true, + }, } ctx := context.Background() for i, test := range tests { comment := Commentf("case:%v sql:%s", i, test) + se.GetSessionVars().StmtCtx.SetWarnings(nil) + se.GetSessionVars().AllowAggPushDown = test.aggPushDown + stmt, err := s.ParseOneStmt(test.sql, "", "") c.Assert(err, IsNil, comment) p, err := planner.Optimize(ctx, se, stmt, s.is) c.Assert(err, IsNil) - c.Assert(core.ToString(p), Equals, test.best) + c.Assert(core.ToString(p), Equals, test.best, comment) warnings := se.GetSessionVars().StmtCtx.GetWarnings() if test.warning == "" { - c.Assert(len(warnings), Equals, 0) + c.Assert(len(warnings), Equals, 0, comment) } else { - c.Assert(len(warnings), Equals, 1) - c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning) - c.Assert(warnings[0].Err.Error(), Equals, test.warning) + c.Assert(len(warnings), Equals, 1, comment) + c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning, comment) + c.Assert(warnings[0].Err.Error(), Equals, test.warning, comment) } } } diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index 6f7488667b0ac..157f8b6711a7f 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -189,7 +189,7 @@ func (a *aggregationPushDownSolver) decompose(ctx sessionctx.Context, aggFunc *a // tryToPushDownAgg tries to push down an aggregate function into a join path. If all aggFuncs are first row, we won't // process it temporarily. If not, We will add additional group by columns and first row functions. We make a new aggregation operator. // If the pushed aggregation is grouped by unique key, it's no need to push it down. -func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, join *LogicalJoin, childIdx int) (_ LogicalPlan, err error) { +func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, join *LogicalJoin, childIdx int, preferAggType uint) (_ LogicalPlan, err error) { child := join.children[childIdx] if aggregation.IsAllFirstRow(aggFuncs) { return child, nil @@ -204,7 +204,7 @@ func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.Agg return child, nil } } - agg, err := a.makeNewAgg(join.ctx, aggFuncs, gbyCols) + agg, err := a.makeNewAgg(join.ctx, aggFuncs, gbyCols, preferAggType) if err != nil { return nil, err } @@ -247,10 +247,11 @@ func (a *aggregationPushDownSolver) checkAnyCountAndSum(aggFuncs []*aggregation. return false } -func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column) (*LogicalAggregation, error) { +func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, preferAggType uint) (*LogicalAggregation, error) { agg := LogicalAggregation{ - GroupByItems: expression.Column2Exprs(gbyCols), - groupByCols: gbyCols, + GroupByItems: expression.Column2Exprs(gbyCols), + groupByCols: gbyCols, + preferAggType: preferAggType, }.Init(ctx) aggLen := len(aggFuncs) + len(gbyCols) newAggFuncDescs := make([]*aggregation.AggFuncDesc, 0, aggLen) @@ -282,8 +283,9 @@ func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs func (a *aggregationPushDownSolver) pushAggCrossUnion(agg *LogicalAggregation, unionSchema *expression.Schema, unionChild LogicalPlan) LogicalPlan { ctx := agg.ctx newAgg := LogicalAggregation{ - AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(agg.AggFuncs)), - GroupByItems: make([]expression.Expression, 0, len(agg.GroupByItems)), + AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(agg.AggFuncs)), + GroupByItems: make([]expression.Expression, 0, len(agg.GroupByItems)), + preferAggType: agg.preferAggType, }.Init(ctx) newAgg.SetSchema(agg.schema.Clone()) for _, aggFunc := range agg.AggFuncs { @@ -340,7 +342,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e if rightInvalid { rChild = join.children[1] } else { - rChild, err = a.tryToPushDownAgg(rightAggFuncs, rightGbyCols, join, 1) + rChild, err = a.tryToPushDownAgg(rightAggFuncs, rightGbyCols, join, 1, agg.preferAggType) if err != nil { return nil, err } @@ -348,7 +350,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e if leftInvalid { lChild = join.children[0] } else { - lChild, err = a.tryToPushDownAgg(leftAggFuncs, leftGbyCols, join, 0) + lChild, err = a.tryToPushDownAgg(leftAggFuncs, leftGbyCols, join, 0, agg.preferAggType) if err != nil { return nil, err } @@ -380,7 +382,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e } else if union, ok1 := child.(*LogicalUnionAll); ok1 { var gbyCols []*expression.Column gbyCols = expression.ExtractColumnsFromExpressions(gbyCols, agg.GroupByItems, nil) - pushedAgg, err := a.makeNewAgg(agg.ctx, agg.AggFuncs, gbyCols) + pushedAgg, err := a.makeNewAgg(agg.ctx, agg.AggFuncs, gbyCols, agg.preferAggType) if err != nil { return nil, err } From 834b7929a77dea3458566f1c685838148de0698a Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Thu, 12 Sep 2019 11:36:50 +0800 Subject: [PATCH 09/17] planner: add AGG_TO_COP hint (#12043) --- planner/core/exhaust_physical_plans.go | 45 ++++++++++++--- planner/core/expression_rewriter.go | 6 +- planner/core/logical_plan_builder.go | 18 +++--- planner/core/logical_plans.go | 4 +- planner/core/physical_plan_test.go | 64 ++++++++++++++++++++++ planner/core/planbuilder.go | 7 ++- planner/core/rule_aggregation_push_down.go | 24 ++++---- 7 files changed, 136 insertions(+), 32 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 008c3ee434f54..fbd7699176511 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1173,6 +1173,16 @@ func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) []P panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } +func (la *LogicalAggregation) canPushToCop() bool { + // At present, only Aggregation, Limit, TopN can be pushed to cop task, and Projection will be supported in the future. + // When we push task to coprocessor, finishCopTask will close the cop task and create a root task in the current implementation. + // Thus, we can't push two different tasks to coprocessor now, and can only push task to coprocessor when the child is Datasource. + + // TODO: develop this function after supporting push several tasks to coprecessor and supporting Projection to coprocessor. + _, ok := la.children[0].(*DataSource) + return ok +} + func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { _, desc := prop.AllSameOrder() enforcedAggs := make([]PhysicalPlan, 0, len(wholeTaskTypes)) @@ -1182,7 +1192,11 @@ func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalPrope Items: property.ItemsFromCols(la.groupByCols, desc), } - for _, taskTp := range wholeTaskTypes { + taskTypes := []property.TaskType{property.CopSingleReadTaskType, property.CopDoubleReadTaskType} + if !la.aggHints.preferAggToCop { + taskTypes = append(taskTypes, property.RootTaskType) + } + for _, taskTp := range taskTypes { copiedChildProperty := new(property.PhysicalProperty) *copiedChildProperty = *childProp // It's ok to not deep copy the "cols" field. copiedChildProperty.TaskTp = taskTp @@ -1231,7 +1245,11 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P // The table read of "CopDoubleReadTaskType" can't promises the sort // property that the stream aggregation required, no need to consider. - for _, taskTp := range []property.TaskType{property.CopSingleReadTaskType, property.RootTaskType} { + taskTypes := []property.TaskType{property.CopSingleReadTaskType} + if !la.aggHints.preferAggToCop { + taskTypes = append(taskTypes, property.RootTaskType) + } + for _, taskTp := range taskTypes { copiedChildProperty := new(property.PhysicalProperty) *copiedChildProperty = *childProp // It's ok to not deep copy the "cols" field. copiedChildProperty.TaskTp = taskTp @@ -1246,7 +1264,7 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P } // If STREAM_AGG hint is existed, it should consider enforce stream aggregation, // because we can't trust possibleChildProperty completely. - if (la.preferAggType & preferStreamAgg) > 0 { + if (la.aggHints.preferAggType & preferStreamAgg) > 0 { streamAggs = append(streamAggs, la.getEnforcedStreamAggs(prop)...) } return streamAggs @@ -1257,7 +1275,11 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy return nil } hashAggs := make([]PhysicalPlan, 0, len(wholeTaskTypes)) - for _, taskTp := range wholeTaskTypes { + taskTypes := []property.TaskType{property.CopSingleReadTaskType, property.CopDoubleReadTaskType} + if !la.aggHints.preferAggToCop { + taskTypes = append(taskTypes, property.RootTaskType) + } + for _, taskTp := range taskTypes { agg := basePhysicalAgg{ GroupByItems: la.GroupByItems, AggFuncs: la.AggFuncs, @@ -1269,13 +1291,22 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy } func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { - preferHash := (la.preferAggType & preferHashAgg) > 0 - preferStream := (la.preferAggType & preferStreamAgg) > 0 + if la.aggHints.preferAggToCop { + if !la.canPushToCop() { + errMsg := "Optimizer Hint AGG_TO_COP is inapplicable" + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + la.aggHints.preferAggToCop = false + } + } + + preferHash := (la.aggHints.preferAggType & preferHashAgg) > 0 + preferStream := (la.aggHints.preferAggType & preferStreamAgg) > 0 if preferHash && preferStream { errMsg := "Optimizer aggregation hints are conflicted" warning := ErrInternal.GenWithStack(errMsg) la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) - la.preferAggType = 0 + la.aggHints.preferAggType = 0 preferHash, preferStream = false, false } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 4f166c6e02039..1c13d6e348500 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -443,7 +443,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, v *ast. func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression.Expression, np LogicalPlan, useMin bool, cmpFunc string, all bool) { plan4Agg := LogicalAggregation{}.Init(er.sctx) if hint := er.b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType + plan4Agg.aggHints = hint.aggHints } plan4Agg.SetChildren(np) @@ -572,7 +572,7 @@ func (er *expressionRewriter) handleNEAny(lexpr, rexpr expression.Expression, np AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, }.Init(er.sctx) if hint := er.b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType + plan4Agg.aggHints = hint.aggHints } plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ @@ -609,7 +609,7 @@ func (er *expressionRewriter) handleEQAll(lexpr, rexpr expression.Expression, np AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, }.Init(er.sctx) if hint := er.b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType + plan4Agg.aggHints = hint.aggHints } plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 2cf879471f86d..0f9fba9b9b12b 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -66,6 +66,8 @@ const ( HintStreamAgg = "stream_agg" // HintIndex is hint enforce using some indexes. HintIndex = "index" + // HintAggToCop is hint enforce pushing aggregation to coprocessor. + HintAggToCop = "agg_to_cop" ) const ( @@ -98,7 +100,7 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu plan4Agg := LogicalAggregation{AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(aggFuncList))}.Init(b.ctx) if hint := b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType + plan4Agg.aggHints = hint.aggHints } schema4Agg := expression.NewSchema(make([]*expression.Column, 0, len(aggFuncList)+p.Schema().Len())...) // aggIdxMap maps the old index to new index after applying common aggregation functions elimination. @@ -791,7 +793,7 @@ func (b *PlanBuilder) buildDistinct(child LogicalPlan, length int) (*LogicalAggr GroupByItems: expression.Column2Exprs(child.Schema().Clone().Columns[:length]), }.Init(b.ctx) if hint := b.TableHints(); hint != nil { - plan4Agg.preferAggType = hint.preferAggType + plan4Agg.aggHints = hint.aggHints } plan4Agg.collectGroupByColumns() for _, col := range child.Schema().Columns { @@ -1949,7 +1951,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n var ( sortMergeTables, INLJTables, hashJoinTables []hintTableInfo indexHintList []indexHintInfo - preferAggType uint + aggHints aggHintInfo ) for _, hint := range hints { switch hint.HintName.L { @@ -1960,9 +1962,11 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n case TiDBHashJoin, HintHJ: hashJoinTables = append(hashJoinTables, tableNames2HintTableInfo(hint.Tables)...) case HintHashAgg: - preferAggType |= preferHashAgg + aggHints.preferAggType |= preferHashAgg case HintStreamAgg: - preferAggType |= preferStreamAgg + aggHints.preferAggType |= preferStreamAgg + case HintAggToCop: + aggHints.preferAggToCop = true case HintIndex: if len(hint.Tables) != 0 && len(hint.Indexes) != 0 { indexHintList = append(indexHintList, indexHintInfo{ @@ -1978,13 +1982,13 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n // ignore hints that not implemented } } - if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables)+len(indexHintList) > 0 || preferAggType != 0 { + if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables)+len(indexHintList) > 0 || aggHints.preferAggType != 0 || aggHints.preferAggToCop { b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ sortMergeJoinTables: sortMergeTables, indexNestedLoopJoinTables: INLJTables, hashJoinTables: hashJoinTables, indexHintList: indexHintList, - preferAggType: preferAggType, + aggHints: aggHints, }) return true } diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 9011603d8ca33..5fb25a0792ec1 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -249,8 +249,8 @@ type LogicalAggregation struct { // groupByCols stores the columns that are group-by items. groupByCols []*expression.Column - // preferAggType stores preferred aggregation algorithm type. - preferAggType uint + // aggHints stores aggregation hint information. + aggHints aggHintInfo possibleProperties [][]*expression.Column inputCount float64 // inputCount is the input count of this plan. diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 79f96c579d1bc..2bdb401601f06 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1664,6 +1664,70 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { } } +func (s *testPlanSuite) TestAggToCopHint(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "insert into mysql.opt_rule_blacklist values(\"aggregation_eliminate\")") + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "admin reload opt_rule_blacklist") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + warning string + }{ + { + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), INDEX(t) */ sum(a) from t group by a", + best: "TableReader(Table(t)->HashAgg)->HashAgg", + }, + { + sql: "select /*+ AGG_TO_COP(), INDEX(t) */ sum(b) from t group by b", + best: "TableReader(Table(t)->HashAgg)->HashAgg", + }, + { + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), INDEX(t) */ distinct a from t group by a", + best: "TableReader(Table(t)->HashAgg)->HashAgg->HashAgg", + warning: "[planner:1815]Optimizer Hint AGG_TO_COP is inapplicable", + }, + { + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), HASH_JOIN(t1), INDEX(t1), INDEX(t2) */ sum(t1.a) from t t1, t t2 where t1.a = t2.b group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg", + warning: "[planner:1815]Optimizer Hint AGG_TO_COP is inapplicable", + }, + } + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + se.GetSessionVars().StmtCtx.SetWarnings(nil) + + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best, comment) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + if test.warning == "" { + c.Assert(len(warnings), Equals, 0, comment) + } else { + c.Assert(len(warnings), Equals, 1, comment) + c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning, comment) + c.Assert(warnings[0].Err.Error(), Equals, test.warning, comment) + } + } +} + func (s *testPlanSuite) TestHintAlias(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index b3a10dde8faec..aa3daaaedece1 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -57,7 +57,7 @@ type tableHintInfo struct { sortMergeJoinTables []hintTableInfo hashJoinTables []hintTableInfo indexHintList []indexHintInfo - preferAggType uint + aggHints aggHintInfo } type hintTableInfo struct { @@ -70,6 +70,11 @@ type indexHintInfo struct { indexHint *ast.IndexHint } +type aggHintInfo struct { + preferAggType uint + preferAggToCop bool +} + func tableNames2HintTableInfo(hintTables []ast.HintTable) []hintTableInfo { if len(hintTables) == 0 { return nil diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index 157f8b6711a7f..caa6a8dd4fcfe 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -189,7 +189,7 @@ func (a *aggregationPushDownSolver) decompose(ctx sessionctx.Context, aggFunc *a // tryToPushDownAgg tries to push down an aggregate function into a join path. If all aggFuncs are first row, we won't // process it temporarily. If not, We will add additional group by columns and first row functions. We make a new aggregation operator. // If the pushed aggregation is grouped by unique key, it's no need to push it down. -func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, join *LogicalJoin, childIdx int, preferAggType uint) (_ LogicalPlan, err error) { +func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, join *LogicalJoin, childIdx int, aggHints aggHintInfo) (_ LogicalPlan, err error) { child := join.children[childIdx] if aggregation.IsAllFirstRow(aggFuncs) { return child, nil @@ -204,7 +204,7 @@ func (a *aggregationPushDownSolver) tryToPushDownAgg(aggFuncs []*aggregation.Agg return child, nil } } - agg, err := a.makeNewAgg(join.ctx, aggFuncs, gbyCols, preferAggType) + agg, err := a.makeNewAgg(join.ctx, aggFuncs, gbyCols, aggHints) if err != nil { return nil, err } @@ -247,11 +247,11 @@ func (a *aggregationPushDownSolver) checkAnyCountAndSum(aggFuncs []*aggregation. return false } -func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, preferAggType uint) (*LogicalAggregation, error) { +func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, aggHints aggHintInfo) (*LogicalAggregation, error) { agg := LogicalAggregation{ - GroupByItems: expression.Column2Exprs(gbyCols), - groupByCols: gbyCols, - preferAggType: preferAggType, + GroupByItems: expression.Column2Exprs(gbyCols), + groupByCols: gbyCols, + aggHints: aggHints, }.Init(ctx) aggLen := len(aggFuncs) + len(gbyCols) newAggFuncDescs := make([]*aggregation.AggFuncDesc, 0, aggLen) @@ -283,9 +283,9 @@ func (a *aggregationPushDownSolver) makeNewAgg(ctx sessionctx.Context, aggFuncs func (a *aggregationPushDownSolver) pushAggCrossUnion(agg *LogicalAggregation, unionSchema *expression.Schema, unionChild LogicalPlan) LogicalPlan { ctx := agg.ctx newAgg := LogicalAggregation{ - AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(agg.AggFuncs)), - GroupByItems: make([]expression.Expression, 0, len(agg.GroupByItems)), - preferAggType: agg.preferAggType, + AggFuncs: make([]*aggregation.AggFuncDesc, 0, len(agg.AggFuncs)), + GroupByItems: make([]expression.Expression, 0, len(agg.GroupByItems)), + aggHints: agg.aggHints, }.Init(ctx) newAgg.SetSchema(agg.schema.Clone()) for _, aggFunc := range agg.AggFuncs { @@ -342,7 +342,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e if rightInvalid { rChild = join.children[1] } else { - rChild, err = a.tryToPushDownAgg(rightAggFuncs, rightGbyCols, join, 1, agg.preferAggType) + rChild, err = a.tryToPushDownAgg(rightAggFuncs, rightGbyCols, join, 1, agg.aggHints) if err != nil { return nil, err } @@ -350,7 +350,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e if leftInvalid { lChild = join.children[0] } else { - lChild, err = a.tryToPushDownAgg(leftAggFuncs, leftGbyCols, join, 0, agg.preferAggType) + lChild, err = a.tryToPushDownAgg(leftAggFuncs, leftGbyCols, join, 0, agg.aggHints) if err != nil { return nil, err } @@ -382,7 +382,7 @@ func (a *aggregationPushDownSolver) aggPushDown(p LogicalPlan) (_ LogicalPlan, e } else if union, ok1 := child.(*LogicalUnionAll); ok1 { var gbyCols []*expression.Column gbyCols = expression.ExtractColumnsFromExpressions(gbyCols, agg.GroupByItems, nil) - pushedAgg, err := a.makeNewAgg(agg.ctx, agg.AggFuncs, gbyCols, agg.preferAggType) + pushedAgg, err := a.makeNewAgg(agg.ctx, agg.AggFuncs, gbyCols, agg.aggHints) if err != nil { return nil, err } From 57c826703100a26a36605c0168756f5fec6cfcdf Mon Sep 17 00:00:00 2001 From: Mingcong Han Date: Fri, 6 Sep 2019 16:42:20 +0800 Subject: [PATCH 10/17] planner: choose TableScan when use an empty index hint (#12037) --- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plans.go | 10 ++++++++++ planner/core/physical_plan_test.go | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 0f9fba9b9b12b..fe0a0a0658944 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1968,7 +1968,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n case HintAggToCop: aggHints.preferAggToCop = true case HintIndex: - if len(hint.Tables) != 0 && len(hint.Indexes) != 0 { + if len(hint.Tables) != 0 { indexHintList = append(indexHintList, indexHintInfo{ tblName: hint.Tables[0].TableName, indexHint: &ast.IndexHint{ diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 5fb25a0792ec1..ea31b260261c7 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -387,6 +387,16 @@ func getTablePath(paths []*accessPath) *accessPath { return nil } +// getTablePath finds the TablePath from a group of accessPaths. +func getTablePath(paths []*accessPath) *accessPath { + for _, path := range paths { + if path.isTablePath { + return path + } + } + return nil +} + // deriveTablePathStats will fulfill the information that the accessPath need. // And it will check whether the primary key is covered only by point query. func (ds *DataSource) deriveTablePathStats(path *accessPath) (bool, error) { diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 2bdb401601f06..d5a37c38dd2ba 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1820,6 +1820,12 @@ func (s *testPlanSuite) TestIndexHint(c *C) { best: "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))", hasWarn: false, }, + // use TablePath when the hint only contains table. + { + sql: "select /*+ INDEX(t) */ f from t where f > 10", + best: "TableReader(Table(t)->Sel([gt(test.t.f, 10)]))", + hasWarn: false, + }, // there will be a warning instead of error when index not exist { sql: "select /*+ INDEX(t, no_such_index) */ * from t", From 3f6d3250bb948eb03db94a262be8be7795ca3488 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Thu, 12 Sep 2019 11:51:24 +0800 Subject: [PATCH 11/17] planner: add IGNORE_INDEX hint (#12059) --- go.mod | 2 +- go.sum | 4 ++-- planner/core/logical_plan_builder.go | 19 ++++++++++++++++--- planner/core/physical_plan_test.go | 12 +++++++----- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 2a976ee72eca9..1530082711ce2 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c + github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 3d7d00ed95842..ec82908b95d89 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c h1:VcQ60QkH7Vv98dbedNKDEO7RfUw+6TE59r/lQokNz/k= -github.com/pingcap/parser v0.0.0-20190828080649-a621a0f9b06c/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e h1:QeD1wC7bGElAhufSHH4JcIbs1cVdxnGWD3n3gcE5qeY= +github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index fe0a0a0658944..968ea44c6fdcb 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -64,8 +64,10 @@ const ( HintHashAgg = "hash_agg" // HintStreamAgg is hint enforce stream aggregation. HintStreamAgg = "stream_agg" - // HintIndex is hint enforce using some indexes. - HintIndex = "index" + // HintUseIndex is hint enforce using some indexes. + HintUseIndex = "use_index" + // HintIgnoreIndex is hint enforce ignoring some indexes. + HintIgnoreIndex = "ignore_index" // HintAggToCop is hint enforce pushing aggregation to coprocessor. HintAggToCop = "agg_to_cop" ) @@ -1967,7 +1969,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n aggHints.preferAggType |= preferStreamAgg case HintAggToCop: aggHints.preferAggToCop = true - case HintIndex: + case HintUseIndex: if len(hint.Tables) != 0 { indexHintList = append(indexHintList, indexHintInfo{ tblName: hint.Tables[0].TableName, @@ -1978,6 +1980,17 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n }, }) } + case HintIgnoreIndex: + if len(hint.Tables) != 0 { + indexHintList = append(indexHintList, indexHintInfo{ + tblName: hint.Tables[0].TableName, + indexHint: &ast.IndexHint{ + IndexNames: hint.Indexes, + HintType: ast.HintIgnore, + HintScope: ast.HintForScan, + }, + }) + } default: // ignore hints that not implemented } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index d5a37c38dd2ba..c81efa55dc4e3 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1687,20 +1687,20 @@ func (s *testPlanSuite) TestAggToCopHint(c *C) { warning string }{ { - sql: "select /*+ AGG_TO_COP(), HASH_AGG(), INDEX(t) */ sum(a) from t group by a", + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), USE_INDEX(t) */ sum(a) from t group by a", best: "TableReader(Table(t)->HashAgg)->HashAgg", }, { - sql: "select /*+ AGG_TO_COP(), INDEX(t) */ sum(b) from t group by b", + sql: "select /*+ AGG_TO_COP(), USE_INDEX(t) */ sum(b) from t group by b", best: "TableReader(Table(t)->HashAgg)->HashAgg", }, { - sql: "select /*+ AGG_TO_COP(), HASH_AGG(), INDEX(t) */ distinct a from t group by a", + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), USE_INDEX(t) */ distinct a from t group by a", best: "TableReader(Table(t)->HashAgg)->HashAgg->HashAgg", warning: "[planner:1815]Optimizer Hint AGG_TO_COP is inapplicable", }, { - sql: "select /*+ AGG_TO_COP(), HASH_AGG(), HASH_JOIN(t1), INDEX(t1), INDEX(t2) */ sum(t1.a) from t t1, t t2 where t1.a = t2.b group by t1.a", + sql: "select /*+ AGG_TO_COP(), HASH_AGG(), HASH_JOIN(t1), USE_INDEX(t1), USE_INDEX(t2) */ sum(t1.a) from t t1, t t2 where t1.a = t2.b group by t1.a", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg", warning: "[planner:1815]Optimizer Hint AGG_TO_COP is inapplicable", }, @@ -1836,7 +1836,9 @@ func (s *testPlanSuite) TestIndexHint(c *C) { ctx := context.Background() for i, test := range tests { comment := Commentf("case:%v sql:%s", i, test) - stmt, err := s.ParseOneStmt(test.sql, "", "") + se.GetSessionVars().StmtCtx.SetWarnings(nil) + + stmt, err := s.ParseOneStmt(test, "", "") c.Assert(err, IsNil, comment) p, err := planner.Optimize(ctx, se, stmt, s.is) From a1721ef6df262af1d3db9ea43db3edf41aaec24d Mon Sep 17 00:00:00 2001 From: foreyes Date: Mon, 16 Sep 2019 13:22:19 +0800 Subject: [PATCH 12/17] update parser & fix conflict --- go.mod | 2 ++ go.sum | 4 ++-- planner/core/logical_plans.go | 10 ---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 1530082711ce2..909591a6eead9 100644 --- a/go.mod +++ b/go.mod @@ -79,3 +79,5 @@ require ( ) go 1.13 + +replace github.com/pingcap/parser => github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45 diff --git a/go.sum b/go.sum index ec82908b95d89..44f739a018611 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45 h1:NGPWpXNx5OEXXa7eoq4WCo+LA4MCKja6NacXdJk7Bzw= +github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45/go.mod h1:3lbW3qHt81o5eguDSsve9wLN8OtUzvdZrVQ/pngnIg0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= @@ -167,8 +169,6 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e h1:QeD1wC7bGElAhufSHH4JcIbs1cVdxnGWD3n3gcE5qeY= -github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index ea31b260261c7..5fb25a0792ec1 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -387,16 +387,6 @@ func getTablePath(paths []*accessPath) *accessPath { return nil } -// getTablePath finds the TablePath from a group of accessPaths. -func getTablePath(paths []*accessPath) *accessPath { - for _, path := range paths { - if path.isTablePath { - return path - } - } - return nil -} - // deriveTablePathStats will fulfill the information that the accessPath need. // And it will check whether the primary key is covered only by point query. func (ds *DataSource) deriveTablePathStats(path *accessPath) (bool, error) { From 8ea835e5138a309d525c2eff8e50f348b9e6ce5a Mon Sep 17 00:00:00 2001 From: foreyes Date: Mon, 16 Sep 2019 14:37:24 +0800 Subject: [PATCH 13/17] update go.mod go.sum & fix explain test --- cmd/explaintest/r/explain_complex.result | 30 ++++++++++++------------ go.mod | 4 ++-- go.sum | 6 +++-- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result index 4ce1c27af3e57..eeb6051e261e8 100644 --- a/cmd/explaintest/r/explain_complex.result +++ b/cmd/explaintest/r/explain_complex.result @@ -244,18 +244,18 @@ explain SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.lead id count task operator info Sort_10 1.00 root test.d.left_value:asc └─HashAgg_15 1.00 root group by:test.d.id, funcs:firstrow(test.d.id), firstrow(test.d.ctx), firstrow(test.d.name), firstrow(test.d.left_value), firstrow(test.d.right_value), firstrow(test.d.depth), firstrow(test.d.leader_id), firstrow(test.d.status), firstrow(test.d.created_on), firstrow(test.d.updated_on) - └─Selection_22 0.01 root or(eq(test.ep.user_id, 62), or(eq(test.d.id, 20), eq(test.d.id, 20))) - └─HashLeftJoin_23 0.02 root left outer join, inner:TableReader_57, equal:[eq(test.p.id, test.ep.position_id)] - ├─IndexJoin_32 0.01 root left outer join, inner:IndexLookUp_31, outer key:test.d.id, inner key:test.p.department_id - │ ├─IndexLookUp_47 0.01 root - │ │ ├─IndexScan_44 10.00 cop table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo - │ │ └─Selection_46 0.01 cop eq(test.d.status, 1000) - │ │ └─TableScan_45 10.00 cop table:d, keep order:false, stats:pseudo - │ └─IndexLookUp_31 0.01 root - │ ├─Selection_29 9.99 cop not(isnull(test.p.department_id)) - │ │ └─IndexScan_27 10.00 cop table:p, index:department_id, range: decided by [eq(test.p.department_id, test.d.id)], keep order:false, stats:pseudo - │ └─Selection_30 0.01 cop eq(test.p.status, 1000) - │ └─TableScan_28 9.99 cop table:p, keep order:false, stats:pseudo - └─TableReader_57 9.99 root data:Selection_56 - └─Selection_56 9.99 cop eq(test.ep.status, 1000), not(isnull(test.ep.position_id)) - └─TableScan_55 10000.00 cop table:ep, range:[-inf,+inf], keep order:false, stats:pseudo + └─Selection_20 0.01 root or(eq(test.ep.user_id, 62), or(eq(test.d.id, 20), eq(test.d.id, 20))) + └─HashLeftJoin_21 0.02 root left outer join, inner:TableReader_55, equal:[eq(test.p.id, test.ep.position_id)] + ├─IndexJoin_29 0.01 root left outer join, inner:IndexLookUp_28, outer key:test.d.id, inner key:test.p.department_id + │ ├─IndexLookUp_45 0.01 root + │ │ ├─IndexScan_42 10.00 cop table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo + │ │ └─Selection_44 0.01 cop eq(test.d.status, 1000) + │ │ └─TableScan_43 10.00 cop table:d, keep order:false, stats:pseudo + │ └─IndexLookUp_28 0.01 root + │ ├─Selection_26 9.99 cop not(isnull(test.p.department_id)) + │ │ └─IndexScan_24 10.00 cop table:p, index:department_id, range: decided by [eq(test.p.department_id, test.d.id)], keep order:false, stats:pseudo + │ └─Selection_27 0.01 cop eq(test.p.status, 1000) + │ └─TableScan_25 9.99 cop table:p, keep order:false, stats:pseudo + └─TableReader_55 9.99 root data:Selection_54 + └─Selection_54 9.99 cop eq(test.ep.status, 1000), not(isnull(test.ep.position_id)) + └─TableScan_53 10000.00 cop table:ep, range:[-inf,+inf], keep order:false, stats:pseudo diff --git a/go.mod b/go.mod index 909591a6eead9..e2f727a3eb398 100644 --- a/go.mod +++ b/go.mod @@ -43,8 +43,8 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e - github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b + github.com/pingcap/parser v0.0.0-20190910041007-2a177b291004 + github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 github.com/prometheus/client_golang v0.9.0 diff --git a/go.sum b/go.sum index 44f739a018611..12d4f3797e79c 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,10 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= -github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= +github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2 h1:wPekJBaldlYsCQa6zkahYkTctfoQLGXf4eqCdtxlYQ0= +github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 h1:sqqiviE8oEYXJh3Aq59HO/AhxjsvcRb9ETh0ivFOHXc= +github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw= From ab8d7dcb13145b2e02e546595fa25a38048694d9 Mon Sep 17 00:00:00 2001 From: foreyes Date: Mon, 16 Sep 2019 15:16:35 +0800 Subject: [PATCH 14/17] go mod tidy & fix conflict --- go.sum | 2 -- planner/core/physical_plan_test.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index 12d4f3797e79c..677a69f72ef32 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2 h1:wPekJBaldlYsCQa6zkahYkTctfoQLGXf4eqCdtxlYQ0= -github.com/pingcap/parser v0.0.0-20190806084718-1a31cabbaef2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 h1:sqqiviE8oEYXJh3Aq59HO/AhxjsvcRb9ETh0ivFOHXc= github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index c81efa55dc4e3..38583a8fcf9dd 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1835,10 +1835,10 @@ func (s *testPlanSuite) TestIndexHint(c *C) { } ctx := context.Background() for i, test := range tests { - comment := Commentf("case:%v sql:%s", i, test) + comment := Commentf("case:%v sql:%s", i, test.sql) se.GetSessionVars().StmtCtx.SetWarnings(nil) - stmt, err := s.ParseOneStmt(test, "", "") + stmt, err := s.ParseOneStmt(test.sql, "", "") c.Assert(err, IsNil, comment) p, err := planner.Optimize(ctx, se, stmt, s.is) From 2690a388c80ff973344f935002982a4871a6c20c Mon Sep 17 00:00:00 2001 From: foreyes Date: Mon, 16 Sep 2019 16:19:31 +0800 Subject: [PATCH 15/17] update parser --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2f727a3eb398..b2620e2fe0686 100644 --- a/go.mod +++ b/go.mod @@ -80,4 +80,4 @@ require ( go 1.13 -replace github.com/pingcap/parser => github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45 +replace github.com/pingcap/parser => github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f diff --git a/go.sum b/go.sum index 677a69f72ef32..f241c9dcf39b5 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45 h1:NGPWpXNx5OEXXa7eoq4WCo+LA4MCKja6NacXdJk7Bzw= -github.com/foreyes/parser v0.0.0-20190916051548-0cb835cf3e45/go.mod h1:3lbW3qHt81o5eguDSsve9wLN8OtUzvdZrVQ/pngnIg0= +github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f h1:rL2kxdwlFdLmHR8Ic8o6FOkC18pU8/agoS22Z0Jl/1A= +github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f/go.mod h1:3lbW3qHt81o5eguDSsve9wLN8OtUzvdZrVQ/pngnIg0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= From e75d7be31e852876e2c607ebfe9077e3295feb17 Mon Sep 17 00:00:00 2001 From: foreyes Date: Mon, 16 Sep 2019 16:48:24 +0800 Subject: [PATCH 16/17] fix unit test --- planner/core/cbo_test.go | 2 +- planner/core/physical_plan_test.go | 48 +++++++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 8dab6a8829639..8941d841fb1b3 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -712,7 +712,7 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { " └─IndexLookUp_21 0.10 root ", " ├─IndexScan_18 1.00 cop table:t1, index:c, range: decided by [eq(test.t1.c, test.t.c)], keep order:false", " └─Selection_20 0.10 cop eq(test.t1.a, test.t.a)", - " └─TableScan_19 1.00 cop table:t, keep order:false", + " └─TableScan_19 1.00 cop table:t1, keep order:false", )) } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 38583a8fcf9dd..8a26b9c3c865a 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1536,19 +1536,19 @@ func (s *testPlanSuite) TestJoinHints(c *C) { warning string }{ { - sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t2.a, t3.a from t t1, t t2, t t3 where t1.a = t2.a and t2.a = t3.a;", + sql: "select /*+ TIDB_INLJ(t1), USE_INDEX(t1), USE_INDEX(t2), USE_INDEX(t3) */ t1.a, t2.a, t3.a from t t1, t t2, t t3 where t1.a = t2.a and t2.a = t3.a;", best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t1.a)}(test.t3.a,test.t2.a)->Projection", warning: "", }, { - sql: "select /*+ TIDB_INLJ(t1) */ t1.b, t2.a from t t1, t t2 where t1.b = t2.a;", + sql: "select /*+ TIDB_INLJ(t1), USE_INDEX(t1), USE_INDEX(t2) */ t1.b, t2.a from t t1, t t2 where t1.b = t2.a;", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.b,test.t2.a)", warning: "[planner:1815]Optimizer Hint /*+ INL_JOIN(t1) */ or /*+ TIDB_INLJ(t1) */ is inapplicable", }, { sql: "select /*+ TIDB_INLJ(t2) */ t1.b, t2.a from t2 t1, t2 t2 where t1.b=t2.b and t2.c=-1;", best: "IndexJoin{IndexReader(Index(t2.b)[[NULL,+inf]])->IndexReader(Index(t2.b_c)[[NULL,+inf]]->Sel([eq(test.t2.c, -1)]))}(test.t2.b,test.t1.b)->Projection", - warning: "[planner:1815]Optimizer Hint /*+ TIDB_INLJ(t2) */ is inapplicable", + warning: "[planner:1815]Optimizer Hint /*+ INL_JOIN(t2) */ or /*+ TIDB_INLJ(t2) */ is inapplicable", }, } ctx := context.Background() @@ -1595,46 +1595,46 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { // without Aggregation hints { sql: "select count(*) from t t1, t t2 where t1.a = t2.b", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", + best: "LeftHashJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", }, { sql: "select count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", + best: "LeftHashJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->IndexReader(Index(t.c_d_e)[[NULL,+inf]])->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", }, // with Aggregation hints { - sql: "select /*+ HASH_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG(), USE_INDEX(t1), USE_INDEX(t2) */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", }, { - sql: "select /*+ STREAM_AGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + sql: "select /*+ STREAM_AGG(), USE_INDEX(t1), USE_INDEX(t2) */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", }, // test conflict warning { - sql: "select /*+ HASH_AGG() STREAM_AGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ HASH_AGG(), STREAM_AGG(), USE_INDEX(t1), USE_INDEX(t2) */ count(*) from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", warning: "[planner:1815]Optimizer aggregation hints are conflicted", }, // additional test { - sql: "select /*+ STREAM_AGG() */ distinct a from t", + sql: "select /*+ STREAM_AGG(), USE_INDEX(t) */ distinct a from t", best: "TableReader(Table(t)->StreamAgg)->StreamAgg", }, { - sql: "select /*+ HASH_AGG() */ t1.a from t t1 where t1.a < any(select t2.b from t t2)", + sql: "select /*+ HASH_AGG(), USE_INDEX(t1) */ t1.a from t t1 where t1.a < any(select t2.b from t t2)", best: "LeftHashJoin{TableReader(Table(t)->Sel([if(isnull(test.t1.a), , 1)]))->TableReader(Table(t)->HashAgg)->HashAgg->Sel([ne(agg_col_cnt, 0)])}->Projection->Projection", }, { - sql: "select /*+ hash_agg() */ t1.a from t t1 where t1.a != any(select t2.b from t t2)", + sql: "select /*+ hash_agg(), USE_INDEX(t1) */ t1.a from t t1 where t1.a != any(select t2.b from t t2)", best: "LeftHashJoin{TableReader(Table(t)->Sel([if(isnull(test.t1.a), , 1)]))->TableReader(Table(t))->Projection->HashAgg->Sel([ne(agg_col_cnt, 0)])}->Projection->Projection", }, { - sql: "select /*+ hash_agg() */ t1.a from t t1 where t1.a = all(select t2.b from t t2)", + sql: "select /*+ hash_agg(), USE_INDEX(t1) */ t1.a from t t1 where t1.a = all(select t2.b from t t2)", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection->HashAgg}->Projection->Projection", }, { - sql: "select /*+ STREAM_AGG() */ sum(t1.a) from t t1 join t t2 on t1.b = t2.b group by t1.b", + sql: "select /*+ STREAM_AGG(), USE_INDEX(t1), USE_INDEX(t2) */ sum(t1.a) from t t1 join t t2 on t1.b = t2.b group by t1.b", best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Sort->Projection->StreamAgg}(test.t2.b,test.t1.b)->HashAgg", warning: "[planner:1815]Optimizer Hint STREAM_AGG is inapplicable", aggPushDown: true, @@ -1795,40 +1795,40 @@ func (s *testPlanSuite) TestIndexHint(c *C) { }{ // simple case { - sql: "select /*+ INDEX(t, c_d_e) */ * from t", + sql: "select /*+ USE_INDEX(t, c_d_e) */ * from t", best: "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", hasWarn: false, }, { - sql: "select /*+ INDEX(t, c_d_e) */ * from t t1", + sql: "select /*+ USE_INDEX(t, c_d_e) */ * from t t1", best: "TableReader(Table(t))", hasWarn: false, }, { - sql: "select /*+ INDEX(t1, c_d_e) */ * from t t1", + sql: "select /*+ USE_INDEX(t1, c_d_e) */ * from t t1", best: "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", hasWarn: false, }, { - sql: "select /*+ INDEX(t1, c_d_e), INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", + sql: "select /*+ USE_INDEX(t1, c_d_e), USE_INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", best: "LeftHashJoin{IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))}(test.t1.a,test.t2.b)", hasWarn: false, }, // test multiple indexes { - sql: "select /*+ INDEX(t, c_d_e, f, g) */ * from t order by f", + sql: "select /*+ USE_INDEX(t, c_d_e, f, g) */ * from t order by f", best: "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))", hasWarn: false, }, // use TablePath when the hint only contains table. { - sql: "select /*+ INDEX(t) */ f from t where f > 10", + sql: "select /*+ USE_INDEX(t) */ f from t where f > 10", best: "TableReader(Table(t)->Sel([gt(test.t.f, 10)]))", hasWarn: false, }, // there will be a warning instead of error when index not exist { - sql: "select /*+ INDEX(t, no_such_index) */ * from t", + sql: "select /*+ USE_INDEX(t, no_such_index) */ * from t", best: "TableReader(Table(t))", hasWarn: true, }, @@ -1889,19 +1889,19 @@ func (s *testPlanSuite) TestQueryBlockHint(c *C) { }, { sql: "select /*+ INL_JOIN(@sel_1 t1), HASH_JOIN(@sel_2 t2) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", }, { sql: "select /*+ INL_JOIN(@sel_1 t1), HASH_JOIN(@qb t2) */ t1.a, t1.b from t t1, (select /*+ QB_NAME(qb) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", - plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + plan: "IndexJoin{TableReader(Table(t))->LeftHashJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", }, { sql: "select /*+ HASH_AGG(@sel_1), STREAM_AGG(@sel_2) */ count(*) from t t1 where t1.a < (select count(*) from t t2 where t1.a > t2.a)", - plan: "Apply{TableReader(Table(t))->TableReader(Table(t)->Sel([gt(test.t1.a, test.t2.a)])->StreamAgg)->StreamAgg->Sel([not(isnull(count(*)))])}->HashAgg", + plan: "Apply{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->IndexReader(Index(t.c_d_e)[[NULL,+inf]]->Sel([gt(test.t1.a, test.t2.a)])->StreamAgg)->StreamAgg->Sel([not(isnull(count(*)))])}->HashAgg", }, { sql: "select /*+ STREAM_AGG(@sel_1), HASH_AGG(@qb) */ count(*) from t t1 where t1.a < (select /*+ QB_NAME(qb) */ count(*) from t t2 where t1.a > t2.a)", - plan: "Apply{TableReader(Table(t))->TableReader(Table(t)->Sel([gt(test.t1.a, test.t2.a)])->HashAgg)->HashAgg->Sel([not(isnull(count(*)))])}->StreamAgg", + plan: "Apply{IndexReader(Index(t.c_d_e)[[NULL,+inf]])->IndexReader(Index(t.c_d_e)[[NULL,+inf]]->Sel([gt(test.t1.a, test.t2.a)])->HashAgg)->HashAgg->Sel([not(isnull(count(*)))])}->StreamAgg", }, { sql: "select /*+ HASH_AGG(@sel_2) */ a, (select count(*) from t t1 where t1.b > t.a) from t where b > (select b from t t2 where t2.b = t.a limit 1)", From bb90580e9a714393e263f1d7523725358f83117e Mon Sep 17 00:00:00 2001 From: foreyes Date: Tue, 17 Sep 2019 13:22:41 +0800 Subject: [PATCH 17/17] update parser --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b2620e2fe0686..7bca902657042 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190910041007-2a177b291004 + github.com/pingcap/parser v0.0.0-20190917040145-a90dba59f50d github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 @@ -79,5 +79,3 @@ require ( ) go 1.13 - -replace github.com/pingcap/parser => github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f diff --git a/go.sum b/go.sum index f241c9dcf39b5..d13b90cf65f1e 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f h1:rL2kxdwlFdLmHR8Ic8o6FOkC18pU8/agoS22Z0Jl/1A= -github.com/foreyes/parser v0.0.0-20190916075236-bf3dfbfb451f/go.mod h1:3lbW3qHt81o5eguDSsve9wLN8OtUzvdZrVQ/pngnIg0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= @@ -169,6 +167,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/parser v0.0.0-20190917040145-a90dba59f50d h1:mbbWj+NCU7hgwnyXvjtuAHAT89ewvmFgFYJ27HhcGe0= +github.com/pingcap/parser v0.0.0-20190917040145-a90dba59f50d/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9 h1:sqqiviE8oEYXJh3Aq59HO/AhxjsvcRb9ETh0ivFOHXc= github.com/pingcap/pd v0.0.0-20190711034019-ee98bf9063e9/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU=