From bb847f0bd05c2f924a01d88ac33aa222efd39cfd Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 27 Jul 2022 16:46:09 +0800 Subject: [PATCH 1/4] executor: prevent sending cop request for show columns --- executor/infoschema_reader_test.go | 19 +++++++++++++++++++ executor/show.go | 4 ++++ planner/core/expression_rewriter.go | 18 ++++++++++-------- sessionctx/stmtctx/stmtctx.go | 4 ++++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/executor/infoschema_reader_test.go b/executor/infoschema_reader_test.go index 97531b2489140..c81efedb8958c 100644 --- a/executor/infoschema_reader_test.go +++ b/executor/infoschema_reader_test.go @@ -717,3 +717,22 @@ SELECT `).Check(testkit.Rows("t a b")) } } + +// https://github.com/pingcap/tidb/issues/36426. +func TestShowColumnsWithSubQueryView(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk.MustExec("CREATE TABLE added (`id` int(11), `name` text, `some_date` timestamp);") + tk.MustExec("CREATE TABLE incremental (`id` int(11), `name`text, `some_date` timestamp);") + tk.MustExec("create view temp_view as (select * from `added` where id > (select max(id) from `incremental`));") + // Show columns should not send coprocessor request to the storage. + require.NoError(t, failpoint.Enable("tikvclient/tikvStoreSendReqResult", `return("timeout")`)) + tk.MustQuery("show columns from temp_view;").Check(testkit.Rows( + "id int(11) YES ", + "name text YES ", + "some_date timestamp YES ")) + require.NoError(t, failpoint.Disable("tikvclient/tikvStoreSendReqResult")) +} diff --git a/executor/show.go b/executor/show.go index 2f1d366dc7c1e..8b0e26ead2d80 100644 --- a/executor/show.go +++ b/executor/show.go @@ -2061,6 +2061,10 @@ func tryFillViewColumnType(ctx context.Context, sctx sessionctx.Context, is info // multiple goroutines running at the same time while session is not goroutine-safe. // Take joining system table as an example, `fetchBuildSideRows` and `fetchProbeSideChunks` can be run concurrently. return runWithSystemSession(ctx, sctx, func(s sessionctx.Context) error { + s.GetSessionVars().StmtCtx.DisableSubQueryPreprocessing = true + defer func() { + s.GetSessionVars().StmtCtx.DisableSubQueryPreprocessing = false + }() // Retrieve view columns info. planBuilder, _ := plannercore.NewPlanBuilder().Init(s, is, &hint.BlockHintProcessor{}) if viewLogicalPlan, err := planBuilder.BuildDataSourceFromView(ctx, dbName, tbl); err == nil { diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index a58e49ef2b73c..48d2256696cc6 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -825,7 +825,8 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.Ex return v, true } np = er.popExistsSubPlan(np) - if len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { + stmtCtx := er.sctx.GetSessionVars().StmtCtx + if stmtCtx.DisableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { er.p, er.err = er.b.buildSemiApply(er.p, np, nil, er.asScalar, v.Not, hasRewriteHint) if er.err != nil || !er.asScalar { return v, true @@ -833,10 +834,10 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.Ex er.ctxStackAppend(er.p.Schema().Columns[er.p.Schema().Len()-1], er.p.OutputNames()[er.p.Schema().Len()-1]) } else { // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - NthPlanBackup := er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan - er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = -1 + NthPlanBackup := stmtCtx.StmtHints.ForceNthPlan + stmtCtx.StmtHints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = NthPlanBackup + stmtCtx.StmtHints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true @@ -1000,7 +1001,8 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S return v, true } np = er.b.buildMaxOneRow(np) - if len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { + stmtCtx := er.sctx.GetSessionVars().StmtCtx + if stmtCtx.DisableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { er.p = er.b.buildApplyWithJoinType(er.p, np, LeftOuterJoin) if np.Schema().Len() > 1 { newCols := make([]expression.Expression, 0, np.Schema().Len()) @@ -1019,10 +1021,10 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S return v, true } // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - NthPlanBackup := er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan - er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = -1 + NthPlanBackup := stmtCtx.StmtHints.ForceNthPlan + stmtCtx.StmtHints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = NthPlanBackup + stmtCtx.StmtHints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index a13b80a78ae69..41ad5b4b467d7 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -139,6 +139,10 @@ type StatementContext struct { SkipASCIICheck bool SkipUTF8MB4Check bool MultiSchemaInfo *model.MultiSchemaInfo + + // DisableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. + DisableSubQueryPreprocessing bool + // If the select statement was like 'select * from t as of timestamp ...' or in a stale read transaction // or is affected by the tidb_read_staleness session variable, then the statement will be makred as isStaleness // in stmtCtx From f96f2866a5f19249575aedda95029a8e49f1afea Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 27 Jul 2022 17:38:40 +0800 Subject: [PATCH 2/4] move the field from StatementContext to Planbuilder --- executor/show.go | 7 ++----- planner/core/expression_rewriter.go | 21 +++++++++++---------- planner/core/planbuilder.go | 23 +++++++++++++++++++++-- sessionctx/stmtctx/stmtctx.go | 4 ---- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/executor/show.go b/executor/show.go index 8b0e26ead2d80..3439d83aca44f 100644 --- a/executor/show.go +++ b/executor/show.go @@ -2061,12 +2061,9 @@ func tryFillViewColumnType(ctx context.Context, sctx sessionctx.Context, is info // multiple goroutines running at the same time while session is not goroutine-safe. // Take joining system table as an example, `fetchBuildSideRows` and `fetchProbeSideChunks` can be run concurrently. return runWithSystemSession(ctx, sctx, func(s sessionctx.Context) error { - s.GetSessionVars().StmtCtx.DisableSubQueryPreprocessing = true - defer func() { - s.GetSessionVars().StmtCtx.DisableSubQueryPreprocessing = false - }() // Retrieve view columns info. - planBuilder, _ := plannercore.NewPlanBuilder().Init(s, is, &hint.BlockHintProcessor{}) + planBuilder, _ := plannercore.NewPlanBuilder( + plannercore.PlanBuilderOptNoExecution{}).Init(s, is, &hint.BlockHintProcessor{}) if viewLogicalPlan, err := planBuilder.BuildDataSourceFromView(ctx, dbName, tbl); err == nil { viewSchema := viewLogicalPlan.Schema() viewOutputNames := viewLogicalPlan.OutputNames() diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 48d2256696cc6..15d372a7627c5 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -825,8 +825,8 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.Ex return v, true } np = er.popExistsSubPlan(np) - stmtCtx := er.sctx.GetSessionVars().StmtCtx - if stmtCtx.DisableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { + + if er.b.disableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { er.p, er.err = er.b.buildSemiApply(er.p, np, nil, er.asScalar, v.Not, hasRewriteHint) if er.err != nil || !er.asScalar { return v, true @@ -834,10 +834,11 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.Ex er.ctxStackAppend(er.p.Schema().Columns[er.p.Schema().Len()-1], er.p.OutputNames()[er.p.Schema().Len()-1]) } else { // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - NthPlanBackup := stmtCtx.StmtHints.ForceNthPlan - stmtCtx.StmtHints.ForceNthPlan = -1 + hints := er.sctx.GetSessionVars().StmtCtx.StmtHints + NthPlanBackup := hints.ForceNthPlan + hints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - stmtCtx.StmtHints.ForceNthPlan = NthPlanBackup + hints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true @@ -1001,8 +1002,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S return v, true } np = er.b.buildMaxOneRow(np) - stmtCtx := er.sctx.GetSessionVars().StmtCtx - if stmtCtx.DisableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { + if er.b.disableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 { er.p = er.b.buildApplyWithJoinType(er.p, np, LeftOuterJoin) if np.Schema().Len() > 1 { newCols := make([]expression.Expression, 0, np.Schema().Len()) @@ -1021,10 +1021,11 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S return v, true } // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - NthPlanBackup := stmtCtx.StmtHints.ForceNthPlan - stmtCtx.StmtHints.ForceNthPlan = -1 + hints := er.sctx.GetSessionVars().StmtCtx.StmtHints + NthPlanBackup := hints.ForceNthPlan + hints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - stmtCtx.StmtHints.ForceNthPlan = NthPlanBackup + hints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 025d015be3257..9e5911b6706c4 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -518,6 +518,8 @@ type PlanBuilder struct { // hasValidSemijoinHint would tell the outer APPLY/JOIN operator that there's valid hint to be checked later // if there's SEMI_JOIN_REWRITE hint and we find checkSemiJoinHint is true. hasValidSemiJoinHint bool + // disableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. + disableSubQueryPreprocessing bool } type handleColHelper struct { @@ -619,14 +621,31 @@ func (b *PlanBuilder) popSelectOffset() { b.selectOffset = b.selectOffset[:len(b.selectOffset)-1] } +// PlanBuilderOpt is used to adjust the plan builder. +type PlanBuilderOpt interface { + Apply(builder *PlanBuilder) +} + +// PlanBuilderOptNoExecution means the plan builder should not run any executor during Build(). +type PlanBuilderOptNoExecution struct{} + +// Apply implements the interface PlanBuilderOpt. +func (p PlanBuilderOptNoExecution) Apply(builder *PlanBuilder) { + builder.disableSubQueryPreprocessing = true +} + // NewPlanBuilder creates a new PlanBuilder. -func NewPlanBuilder() *PlanBuilder { - return &PlanBuilder{ +func NewPlanBuilder(opts ...PlanBuilderOpt) *PlanBuilder { + builder := &PlanBuilder{ outerCTEs: make([]*cteInfo, 0), colMapper: make(map[*ast.ColumnNameExpr]int), handleHelper: &handleColHelper{id2HandleMapStack: make([]map[int64][]HandleCols, 0)}, correlatedAggMapper: make(map[*ast.AggregateFuncExpr]*expression.CorrelatedColumn), } + for _, opt := range opts { + opt(builder) + } + return builder } // Init initialize a PlanBuilder. diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 41ad5b4b467d7..a13b80a78ae69 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -139,10 +139,6 @@ type StatementContext struct { SkipASCIICheck bool SkipUTF8MB4Check bool MultiSchemaInfo *model.MultiSchemaInfo - - // DisableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. - DisableSubQueryPreprocessing bool - // If the select statement was like 'select * from t as of timestamp ...' or in a stale read transaction // or is affected by the tidb_read_staleness session variable, then the statement will be makred as isStaleness // in stmtCtx From 29b70d691575002d2b00faee19b81db5d8107110 Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 27 Jul 2022 17:41:01 +0800 Subject: [PATCH 3/4] fix build --- planner/core/planbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 9e5911b6706c4..01c04cd3e0c18 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -643,7 +643,7 @@ func NewPlanBuilder(opts ...PlanBuilderOpt) *PlanBuilder { correlatedAggMapper: make(map[*ast.AggregateFuncExpr]*expression.CorrelatedColumn), } for _, opt := range opts { - opt(builder) + opt.Apply(builder) } return builder } From 70a3e4f905be4c725f12071c9602f32d5d259e88 Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 27 Jul 2022 18:06:02 +0800 Subject: [PATCH 4/4] revert improperate changes --- planner/core/expression_rewriter.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 15d372a7627c5..93b5b0d7a9ddc 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -834,11 +834,10 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.Ex er.ctxStackAppend(er.p.Schema().Columns[er.p.Schema().Len()-1], er.p.OutputNames()[er.p.Schema().Len()-1]) } else { // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - hints := er.sctx.GetSessionVars().StmtCtx.StmtHints - NthPlanBackup := hints.ForceNthPlan - hints.ForceNthPlan = -1 + NthPlanBackup := er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan + er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - hints.ForceNthPlan = NthPlanBackup + er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true @@ -1021,11 +1020,10 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S return v, true } // We don't want nth_plan hint to affect separately executed subqueries here, so disable nth_plan temporarily. - hints := er.sctx.GetSessionVars().StmtCtx.StmtHints - NthPlanBackup := hints.ForceNthPlan - hints.ForceNthPlan = -1 + NthPlanBackup := er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan + er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = -1 physicalPlan, _, err := DoOptimize(ctx, er.sctx, er.b.optFlag, np) - hints.ForceNthPlan = NthPlanBackup + er.sctx.GetSessionVars().StmtCtx.StmtHints.ForceNthPlan = NthPlanBackup if err != nil { er.err = err return v, true