From c2171b9fad6355b28ecb17eeafe73c870c3f3e81 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 29 Jun 2023 09:19:55 +0200 Subject: [PATCH 01/26] error out if we have aggregate gtid in handleAggr Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/aggregation_pushing.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index 9a596039906..57a3f5c0620 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -472,6 +472,9 @@ func (ab *aggBuilder) handleAggr(ctx *plancontext.PlanningContext, aggr Aggr) er return errAbortAggrPushing case opcode.AggregateUnassigned: return vterrors.VT12001(fmt.Sprintf("in scatter query: aggregation function '%s'", sqlparser.String(aggr.Original))) + case opcode.AggregateGtid: + // this is only used for SHOW GTID queries that will never contain joins + return vterrors.VT13001("cannot do join with vgtid") default: return errHorizonNotPlanned() } From da016f00d3336f854d08e2761d0e43762ee32d58 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 29 Jun 2023 09:29:51 +0200 Subject: [PATCH 02/26] remove un-required for loop Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/aggregation_pushing.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index 57a3f5c0620..c61ed0adec3 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -114,12 +114,6 @@ func pushDownAggregationThroughFilter( filter *Filter, ) (ops.Operator, *rewrite.ApplyResult, error) { - for _, predicate := range filter.Predicates { - if sqlparser.ContainsAggregation(predicate) { - return nil, nil, errHorizonNotPlanned() - } - } - columnsNeeded := collectColNamesNeeded(ctx, filter) // Create a new aggregator to be placed below the route. From 2139472da18bc41a1621f97c73168afded16c4a1 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 29 Jun 2023 10:07:57 +0200 Subject: [PATCH 03/26] add GetSelectExprs to the operator interface and remove horizon/derived Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- go/vt/vterrors/code.go | 1 + .../planbuilder/operators/aggregator.go | 4 ++++ .../planbuilder/operators/apply_join.go | 4 ++++ go/vt/vtgate/planbuilder/operators/derived.go | 6 +++++- .../vtgate/planbuilder/operators/distinct.go | 4 ++++ go/vt/vtgate/planbuilder/operators/filter.go | 4 ++++ go/vt/vtgate/planbuilder/operators/horizon.go | 6 +++++- .../planbuilder/operators/horizon_planning.go | 2 +- go/vt/vtgate/planbuilder/operators/limit.go | 4 ++++ .../vtgate/planbuilder/operators/operator.go | 20 +++++++++++++++++-- go/vt/vtgate/planbuilder/operators/ops/op.go | 2 ++ .../vtgate/planbuilder/operators/ordering.go | 4 ++++ .../planbuilder/operators/projection.go | 4 ++++ go/vt/vtgate/planbuilder/operators/route.go | 4 ++++ go/vt/vtgate/planbuilder/operators/table.go | 4 ++++ go/vt/vtgate/planbuilder/operators/vindex.go | 4 ++++ 16 files changed, 72 insertions(+), 5 deletions(-) diff --git a/go/vt/vterrors/code.go b/go/vt/vterrors/code.go index 26abd85e49e..7d8f22889d0 100644 --- a/go/vt/vterrors/code.go +++ b/go/vt/vterrors/code.go @@ -73,6 +73,7 @@ var ( VT09012 = errorWithoutState("VT09012", vtrpcpb.Code_FAILED_PRECONDITION, "%s statement with %s tablet not allowed", "This type of statement is not allowed on the given tablet.") VT09013 = errorWithoutState("VT09013", vtrpcpb.Code_FAILED_PRECONDITION, "semi-sync plugins are not loaded", "Durability policy wants Vitess to use semi-sync, but the MySQL instances don't have the semi-sync plugin loaded.") VT09014 = errorWithoutState("VT09014", vtrpcpb.Code_FAILED_PRECONDITION, "vindex cannot be modified", "The vindex cannot be used as table in DML statement") + VT09015 = errorWithoutState("VT09015", vtrpcpb.Code_FAILED_PRECONDITION, "schema tracking required", "This query cannot be planned without more information on the SQL schema. Please turn on schema tracking or add authoritative columns information to your VSchema.") VT10001 = errorWithoutState("VT10001", vtrpcpb.Code_ABORTED, "foreign key constraints are not allowed", "Foreign key constraints are not allowed, see https://vitess.io/blog/2021-06-15-online-ddl-why-no-fk/.") diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 9a5b28bccfa..2e5a931be8a 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -191,6 +191,10 @@ func (a *Aggregator) GetColumns() ([]*sqlparser.AliasedExpr, error) { return a.Columns, nil } +func (a *Aggregator) GetSelectExprs() (sqlparser.SelectExprs, error) { + return transformColumnsToSelectExprs(a) +} + func (a *Aggregator) ShortDescription() string { columnns := slices2.Map(a.Columns, func(from *sqlparser.AliasedExpr) string { return sqlparser.String(from) diff --git a/go/vt/vtgate/planbuilder/operators/apply_join.go b/go/vt/vtgate/planbuilder/operators/apply_join.go index 816ce47a813..41080e39902 100644 --- a/go/vt/vtgate/planbuilder/operators/apply_join.go +++ b/go/vt/vtgate/planbuilder/operators/apply_join.go @@ -183,6 +183,10 @@ func (a *ApplyJoin) GetColumns() ([]*sqlparser.AliasedExpr, error) { return slices2.Map(a.ColumnsAST, joinColumnToAliasedExpr), nil } +func (a *ApplyJoin) GetSelectExprs() (sqlparser.SelectExprs, error) { + return transformColumnsToSelectExprs(a) +} + func (a *ApplyJoin) GetOrdering() ([]ops.OrderBy, error) { return a.LHS.GetOrdering() } diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go index dc6bbd79952..9531a3759ab 100644 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ b/go/vt/vtgate/planbuilder/operators/derived.go @@ -208,13 +208,17 @@ func (d *Derived) GetColumns() (exprs []*sqlparser.AliasedExpr, err error) { for _, expr := range sqlparser.GetFirstSelect(d.Query).SelectExprs { ae, ok := expr.(*sqlparser.AliasedExpr) if !ok { - return nil, errHorizonNotPlanned() + return nil, vterrors.VT09015() } exprs = append(exprs, ae) } return } +func (d *Derived) GetSelectExprs() (sqlparser.SelectExprs, error) { + return sqlparser.GetFirstSelect(d.Query).SelectExprs, nil +} + func (d *Derived) GetOrdering() ([]ops.OrderBy, error) { if d.QP == nil { return nil, vterrors.VT13001("QP should already be here") diff --git a/go/vt/vtgate/planbuilder/operators/distinct.go b/go/vt/vtgate/planbuilder/operators/distinct.go index c120bef8230..d5905f87adb 100644 --- a/go/vt/vtgate/planbuilder/operators/distinct.go +++ b/go/vt/vtgate/planbuilder/operators/distinct.go @@ -112,6 +112,10 @@ func (d *Distinct) GetColumns() ([]*sqlparser.AliasedExpr, error) { return d.Source.GetColumns() } +func (d *Distinct) GetSelectExprs() (sqlparser.SelectExprs, error) { + return d.Source.GetSelectExprs() +} + func (d *Distinct) ShortDescription() string { return "" } diff --git a/go/vt/vtgate/planbuilder/operators/filter.go b/go/vt/vtgate/planbuilder/operators/filter.go index d8dbd8be9de..1c0751f47ad 100644 --- a/go/vt/vtgate/planbuilder/operators/filter.go +++ b/go/vt/vtgate/planbuilder/operators/filter.go @@ -100,6 +100,10 @@ func (f *Filter) GetColumns() ([]*sqlparser.AliasedExpr, error) { return f.Source.GetColumns() } +func (f *Filter) GetSelectExprs() (sqlparser.SelectExprs, error) { + return f.Source.GetSelectExprs() +} + func (f *Filter) GetOrdering() ([]ops.OrderBy, error) { return f.Source.GetOrdering() } diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index d4c30313114..a05d23795eb 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -44,13 +44,17 @@ func (h *Horizon) GetColumns() (exprs []*sqlparser.AliasedExpr, err error) { for _, expr := range sqlparser.GetFirstSelect(h.Select).SelectExprs { ae, ok := expr.(*sqlparser.AliasedExpr) if !ok { - return nil, errHorizonNotPlanned() + return nil, vterrors.VT09015() } exprs = append(exprs, ae) } return } +func (h *Horizon) GetSelectExprs() (sqlparser.SelectExprs, error) { + return sqlparser.GetFirstSelect(h.Select).SelectExprs, nil +} + var _ ops.Operator = (*Horizon)(nil) func (h *Horizon) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (ops.Operator, error) { diff --git a/go/vt/vtgate/planbuilder/operators/horizon_planning.go b/go/vt/vtgate/planbuilder/operators/horizon_planning.go index 3fbff7a586e..4ff4cfeb7dc 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_planning.go @@ -621,7 +621,7 @@ func tryPushingDownDistinct(in *Distinct) (ops.Operator, *rewrite.ApplyResult, e // makeSureOutputIsCorrect uses the original Horizon to make sure that the output columns line up with what the user asked for func makeSureOutputIsCorrect(ctx *plancontext.PlanningContext, oldHorizon ops.Operator, output ops.Operator) (ops.Operator, error) { - cols, err := output.GetColumns() + cols, err := output.GetSelectExprs() if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/limit.go b/go/vt/vtgate/planbuilder/operators/limit.go index 35108965a52..24f6af9ec7c 100644 --- a/go/vt/vtgate/planbuilder/operators/limit.go +++ b/go/vt/vtgate/planbuilder/operators/limit.go @@ -69,6 +69,10 @@ func (l *Limit) GetColumns() ([]*sqlparser.AliasedExpr, error) { return l.Source.GetColumns() } +func (l *Limit) GetSelectExprs() (sqlparser.SelectExprs, error) { + return l.Source.GetSelectExprs() +} + func (l *Limit) GetOrdering() ([]ops.OrderBy, error) { return l.Source.GetOrdering() } diff --git a/go/vt/vtgate/planbuilder/operators/operator.go b/go/vt/vtgate/planbuilder/operators/operator.go index d2ce5cb77d0..6bb953cb418 100644 --- a/go/vt/vtgate/planbuilder/operators/operator.go +++ b/go/vt/vtgate/planbuilder/operators/operator.go @@ -34,6 +34,7 @@ The operators go through a few phases while planning: package operators import ( + "vitess.io/vitess/go/slices2" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" @@ -97,11 +98,15 @@ func (noInputs) SetInputs(ops []ops.Operator) { // AddColumn implements the Operator interface func (noColumns) AddColumn(*plancontext.PlanningContext, *sqlparser.AliasedExpr, bool, bool) (ops.Operator, int, error) { - return nil, 0, vterrors.VT13001("the noColumns operator cannot accept columns") + return nil, 0, vterrors.VT13001("noColumns operators have no column") } func (noColumns) GetColumns() ([]*sqlparser.AliasedExpr, error) { - return nil, vterrors.VT13001("the noColumns operator cannot accept columns") + return nil, vterrors.VT13001("noColumns operators have no column") +} + +func (noColumns) GetSelectExprs() (sqlparser.SelectExprs, error) { + return nil, vterrors.VT13001("noColumns operators have no column") } // AddPredicate implements the Operator interface @@ -135,3 +140,14 @@ func tryTruncateColumnsAt(op ops.Operator, truncateAt int) bool { return tryTruncateColumnsAt(inputs[0], truncateAt) } + +func transformColumnsToSelectExprs(op ops.Operator) (sqlparser.SelectExprs, error) { + columns, err := op.GetColumns() + if err != nil { + return nil, err + } + selExprs := slices2.Map(columns, func(from *sqlparser.AliasedExpr) sqlparser.SelectExpr { + return from + }) + return selExprs, nil +} diff --git a/go/vt/vtgate/planbuilder/operators/ops/op.go b/go/vt/vtgate/planbuilder/operators/ops/op.go index 57e27879861..a0a350633a2 100644 --- a/go/vt/vtgate/planbuilder/operators/ops/op.go +++ b/go/vt/vtgate/planbuilder/operators/ops/op.go @@ -49,6 +49,8 @@ type ( GetColumns() ([]*sqlparser.AliasedExpr, error) + GetSelectExprs() (sqlparser.SelectExprs, error) + ShortDescription() string GetOrdering() ([]OrderBy, error) diff --git a/go/vt/vtgate/planbuilder/operators/ordering.go b/go/vt/vtgate/planbuilder/operators/ordering.go index 360bf87cb23..2ff0901c97b 100644 --- a/go/vt/vtgate/planbuilder/operators/ordering.go +++ b/go/vt/vtgate/planbuilder/operators/ordering.go @@ -76,6 +76,10 @@ func (o *Ordering) GetColumns() ([]*sqlparser.AliasedExpr, error) { return o.Source.GetColumns() } +func (o *Ordering) GetSelectExprs() (sqlparser.SelectExprs, error) { + return o.Source.GetSelectExprs() +} + func (o *Ordering) GetOrdering() ([]ops.OrderBy, error) { return o.Order, nil } diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index a969fdc1129..78c8d74bc1d 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -173,6 +173,10 @@ func (p *Projection) GetColumns() ([]*sqlparser.AliasedExpr, error) { return p.Columns, nil } +func (p *Projection) GetSelectExprs() (sqlparser.SelectExprs, error) { + return transformColumnsToSelectExprs(p) +} + func (p *Projection) GetOrdering() ([]ops.OrderBy, error) { return p.Source.GetOrdering() } diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index 31edd45fb78..d98d26f65f6 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -609,6 +609,10 @@ func (r *Route) GetColumns() ([]*sqlparser.AliasedExpr, error) { return r.Source.GetColumns() } +func (r *Route) GetSelectExprs() (sqlparser.SelectExprs, error) { + return r.Source.GetSelectExprs() +} + func (r *Route) GetOrdering() ([]ops.OrderBy, error) { return r.Source.GetOrdering() } diff --git a/go/vt/vtgate/planbuilder/operators/table.go b/go/vt/vtgate/planbuilder/operators/table.go index a3eeea5e365..081d70cc4df 100644 --- a/go/vt/vtgate/planbuilder/operators/table.go +++ b/go/vt/vtgate/planbuilder/operators/table.go @@ -79,6 +79,10 @@ func (to *Table) GetColumns() ([]*sqlparser.AliasedExpr, error) { return slices2.Map(to.Columns, colNameToExpr), nil } +func (to *Table) GetSelectExprs() (sqlparser.SelectExprs, error) { + return transformColumnsToSelectExprs(to) +} + func (to *Table) GetOrdering() ([]ops.OrderBy, error) { return nil, nil } diff --git a/go/vt/vtgate/planbuilder/operators/vindex.go b/go/vt/vtgate/planbuilder/operators/vindex.go index 252cb7ba2b3..a3c16fea0c4 100644 --- a/go/vt/vtgate/planbuilder/operators/vindex.go +++ b/go/vt/vtgate/planbuilder/operators/vindex.go @@ -86,6 +86,10 @@ func (v *Vindex) GetColumns() ([]*sqlparser.AliasedExpr, error) { return slices2.Map(v.Columns, colNameToExpr), nil } +func (v *Vindex) GetSelectExprs() (sqlparser.SelectExprs, error) { + return transformColumnsToSelectExprs(v) +} + func (v *Vindex) GetOrdering() ([]ops.OrderBy, error) { return nil, nil } From 6ccbcc9508b5a1c6fe055ac0fdbeec30503d3f73 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 29 Jun 2023 16:26:53 +0200 Subject: [PATCH 04/26] support distinct aggregations on the new horizon planner Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/aggregations.go | 5 +- go/vt/vtgate/engine/ordered_aggregate_test.go | 27 ++++---- go/vt/vtgate/planbuilder/horizon_planning.go | 2 +- .../operators/aggregation_pushing.go | 61 +++++++++++++++---- .../planbuilder/operators/aggregator.go | 5 +- .../operators/horizon_expanding.go | 5 -- go/vt/vtgate/planbuilder/operators/phases.go | 33 +++++++--- go/vt/vtgate/planbuilder/ordered_aggregate.go | 2 + .../vtgate/planbuilder/projection_pushing.go | 1 + go/vt/vtgate/planbuilder/show.go | 1 + .../planbuilder/testdata/aggr_cases.json | 52 ++++++++-------- .../tabletmanager/vdiff/table_plan.go | 1 + .../vdiff/workflow_differ_test.go | 2 + go/vt/wrangler/vdiff.go | 1 + go/vt/wrangler/vdiff_test.go | 2 + 15 files changed, 128 insertions(+), 72 deletions(-) diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index 9bdee013cf8..73ba3ec58e1 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -41,7 +41,6 @@ type AggregateParams struct { // These are used only for distinct opcodes. KeyCol int WCol int - WAssigned bool CollationID collations.ID Alias string `json:",omitempty"` @@ -68,7 +67,7 @@ func (ap *AggregateParams) preProcess() bool { func (ap *AggregateParams) String() string { keyCol := strconv.Itoa(ap.Col) - if ap.WAssigned { + if ap.WCol >= 0 { keyCol = fmt.Sprintf("%s|%d", keyCol, ap.WCol) } if ap.CollationID != collations.Unknown { @@ -280,7 +279,7 @@ func convertFields(fields []*querypb.Field, aggrs []*AggregateParams) []*querypb func findComparableCurrentDistinct(row []sqltypes.Value, aggr *AggregateParams) sqltypes.Value { curDistinct := row[aggr.KeyCol] - if aggr.WAssigned && !curDistinct.IsComparable() { + if aggr.WCol >= 0 && !curDistinct.IsComparable() { aggr.KeyCol = aggr.WCol curDistinct = row[aggr.KeyCol] } diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index 83f2cb4ecf4..599290041e8 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -742,11 +742,10 @@ func TestCountDistinctOnVarchar(t *testing.T) { oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - WCol: 2, - WAssigned: true, - Alias: "count(distinct c2)", + Opcode: AggregateCountDistinct, + Col: 1, + WCol: 2, + Alias: "count(distinct c2)", }}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, @@ -806,11 +805,10 @@ func TestCountDistinctOnVarcharWithNulls(t *testing.T) { oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - WCol: 2, - WAssigned: true, - Alias: "count(distinct c2)", + Opcode: AggregateCountDistinct, + Col: 1, + WCol: 2, + Alias: "count(distinct c2)", }}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, @@ -872,11 +870,10 @@ func TestSumDistinctOnVarcharWithNulls(t *testing.T) { oa := &OrderedAggregate{ Aggregates: []*AggregateParams{{ - Opcode: AggregateSumDistinct, - Col: 1, - WCol: 2, - WAssigned: true, - Alias: "sum(distinct c2)", + Opcode: AggregateSumDistinct, + Col: 1, + WCol: 2, + Alias: "sum(distinct c2)", }}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, diff --git a/go/vt/vtgate/planbuilder/horizon_planning.go b/go/vt/vtgate/planbuilder/horizon_planning.go index 1f50b2e19bd..3766d817326 100644 --- a/go/vt/vtgate/planbuilder/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/horizon_planning.go @@ -443,6 +443,7 @@ func generateAggregateParams(aggrs []operators.Aggr, aggrParamOffsets [][]offset aggrParams[idx] = &engine.AggregateParams{ Opcode: opcode, Col: offset, + WCol: -1, Alias: aggr.Alias, Expr: aggr.Original.Expr, Original: aggr.Original, @@ -482,7 +483,6 @@ func addColumnsToOA( Opcode: a.OpCode, Col: o.col, KeyCol: o.col, - WAssigned: o.wsCol >= 0, WCol: o.wsCol, Alias: a.Alias, Original: a.Original, diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index c61ed0adec3..38a572cc14e 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -39,10 +39,16 @@ func tryPushingDownAggregator(ctx *plancontext.PlanningContext, aggregator *Aggr case *ApplyJoin: if ctx.DelegateAggregation { output, applyResult, err = pushDownAggregationThroughJoin(ctx, aggregator, src) + if applyResult != rewrite.SameTree && aggregator.Original { + aggregator.aggregateTheAggregates() + } } case *Filter: if ctx.DelegateAggregation { output, applyResult, err = pushDownAggregationThroughFilter(ctx, aggregator, src) + if applyResult != rewrite.SameTree && aggregator.Original { + aggregator.aggregateTheAggregates() + } } default: return aggregator, rewrite.SameTree, nil @@ -57,23 +63,24 @@ func tryPushingDownAggregator(ctx *plancontext.PlanningContext, aggregator *Aggr } aggregator.Pushed = true - if applyResult != rewrite.SameTree && aggregator.Original { - aggregator.aggregateTheAggregates() - } return } func (a *Aggregator) aggregateTheAggregates() { - for i, aggr := range a.Aggregations { - // Handle different aggregation operations when pushing down through a sharded route. - switch aggr.OpCode { - case opcode.AggregateCount, opcode.AggregateCountStar, opcode.AggregateCountDistinct: - // All count variations turn into SUM above the Route. - // Think of it as we are SUMming together a bunch of distributed COUNTs. - aggr.OriginalOpCode, aggr.OpCode = aggr.OpCode, opcode.AggregateSum - a.Aggregations[i] = aggr - } + for i := range a.Aggregations { + aggregateTheAggregate(a, i) + } +} + +func aggregateTheAggregate(a *Aggregator, i int) { + aggr := a.Aggregations[i] + switch aggr.OpCode { + case opcode.AggregateCount, opcode.AggregateCountStar, opcode.AggregateCountDistinct: + // All count variations turn into SUM above the Route. + // Think of it as we are SUMming together a bunch of distributed COUNTs. + aggr.OriginalOpCode, aggr.OpCode = aggr.OpCode, opcode.AggregateSum + a.Aggregations[i] = aggr } } @@ -95,6 +102,36 @@ func pushDownAggregationThroughRoute( aggrBelowRoute := aggregator.Clone([]ops.Operator{route.Source}).(*Aggregator) aggrBelowRoute.Pushed = false aggrBelowRoute.Original = false + aggrBelowRoute.Aggregations = nil + + for i, aggregation := range aggregator.Aggregations { + if !aggregation.Distinct || exprHasUniqueVindex(ctx, aggregation.Func.GetArg()) { + aggrBelowRoute.Aggregations = append(aggrBelowRoute.Aggregations, aggregation) + aggregateTheAggregate(aggregator, i) + continue + } + innerExpr := aggregation.Func.GetArg() + + if aggregator.DistinctExpr != nil { + if ctx.SemTable.EqualsExpr(aggregator.DistinctExpr, innerExpr) { + // we can handle multiple distinct aggregations, as long as they are aggregating on the same expression + aggrBelowRoute.Columns[aggregation.ColOffset] = aeWrap(innerExpr) + continue + } + return nil, nil, vterrors.VT12001(fmt.Sprintf("only one DISTINCT aggregation is allowed in a SELECT: %s", sqlparser.String(aggregation.Original))) + } + + // We handle a distinct aggregation by turning it into a group by and + // doing the aggregating on the vtgate level instead + aggregator.DistinctExpr = innerExpr + aeDistinctExpr := aeWrap(aggregator.DistinctExpr) + + aggrBelowRoute.Columns[aggregation.ColOffset] = aeDistinctExpr + + groupBy := NewGroupBy(aggregator.DistinctExpr, aggregator.DistinctExpr, aeDistinctExpr) + groupBy.ColOffset = aggregation.ColOffset + aggrBelowRoute.Grouping = append(aggrBelowRoute.Grouping, groupBy) + } // Set the source of the route to the new aggregator placed below the route. route.Source = aggrBelowRoute diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 2e5a931be8a..c09e9628247 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -42,6 +42,9 @@ type ( Grouping []GroupBy Aggregations []Aggr + // We support a single distinct aggregation per aggregator. It is stored here + DistinctExpr sqlparser.Expr + // Pushed will be set to true once this aggregation has been pushed deeper in the tree Pushed bool offsetPlanned bool @@ -331,7 +334,7 @@ func (a *Aggregator) addIfGroupingColumn(ctx *plancontext.PlanningContext, colId continue } - newSrc, offset, err := a.Source.AddColumn(ctx, a.Columns[colIdx], false, false) + newSrc, offset, err := a.Source.AddColumn(ctx, a.Columns[colIdx], false, true) if err != nil { return -1, err } diff --git a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go index 2342d6edb27..59dc9a32b65 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go @@ -75,11 +75,6 @@ func expandHorizon(ctx *plancontext.PlanningContext, horizon horizonLike) (ops.O } func checkInvalid(aggregations []Aggr, horizon horizonLike) error { - for _, aggregation := range aggregations { - if aggregation.Distinct { - return errHorizonNotPlanned() - } - } if _, isDerived := horizon.(*Derived); isDerived { return errHorizonNotPlanned() } diff --git a/go/vt/vtgate/planbuilder/operators/phases.go b/go/vt/vtgate/planbuilder/operators/phases.go index 802d1876fdb..b1155903dd0 100644 --- a/go/vt/vtgate/planbuilder/operators/phases.go +++ b/go/vt/vtgate/planbuilder/operators/phases.go @@ -63,18 +63,27 @@ func addOrderBysForAggregations(ctx *plancontext.PlanningContext, root ops.Opera return in, rewrite.SameTree, nil } - requireOrdering, err := needsOrdering(aggrOp, ctx) + requireOrdering, err := needsOrdering(ctx, aggrOp) if err != nil { return nil, nil, err } if !requireOrdering { return in, rewrite.SameTree, nil } + orderBys := slices2.Map(aggrOp.Grouping, func(from GroupBy) ops.OrderBy { + return from.AsOrderBy() + }) + if aggrOp.DistinctExpr != nil { + orderBys = append(orderBys, ops.OrderBy{ + Inner: &sqlparser.Order{ + Expr: aggrOp.DistinctExpr, + }, + SimplifiedExpr: aggrOp.DistinctExpr, + }) + } aggrOp.Source = &Ordering{ Source: aggrOp.Source, - Order: slices2.Map(aggrOp.Grouping, func(from GroupBy) ops.OrderBy { - return from.AsOrderBy() - }), + Order: orderBys, } return in, rewrite.NewTree("added ordering before aggregation", in), nil } @@ -82,19 +91,25 @@ func addOrderBysForAggregations(ctx *plancontext.PlanningContext, root ops.Opera return rewrite.TopDown(root, TableID, visitor, stopAtRoute) } -func needsOrdering(in *Aggregator, ctx *plancontext.PlanningContext) (bool, error) { - if len(in.Grouping) == 0 { +func needsOrdering(ctx *plancontext.PlanningContext, in *Aggregator) (bool, error) { + requiredOrder := slices2.Map(in.Grouping, func(from GroupBy) sqlparser.Expr { + return from.SimplifiedExpr + }) + if in.DistinctExpr != nil { + requiredOrder = append(requiredOrder, in.DistinctExpr) + } + if len(requiredOrder) == 0 { return false, nil } srcOrdering, err := in.Source.GetOrdering() if err != nil { return false, err } - if len(srcOrdering) < len(in.Grouping) { + if len(srcOrdering) < len(requiredOrder) { return true, nil } - for idx, gb := range in.Grouping { - if !ctx.SemTable.EqualsExprWithDeps(srcOrdering[idx].SimplifiedExpr, gb.SimplifiedExpr) { + for idx, gb := range requiredOrder { + if !ctx.SemTable.EqualsExprWithDeps(srcOrdering[idx].SimplifiedExpr, gb) { return true, nil } } diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index 7e2554d8c3f..a4f7cda487a 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -277,6 +277,7 @@ func (oa *orderedAggregate) pushAggr(pb *primitiveBuilder, expr *sqlparser.Alias Col: innerCol, Alias: expr.ColumnName(), OrigOpcode: origOpcode, + WCol: -1, }) } else { newBuilder, _, innerCol, err := planProjection(pb, oa.input, expr, origin) @@ -288,6 +289,7 @@ func (oa *orderedAggregate) pushAggr(pb *primitiveBuilder, expr *sqlparser.Alias Opcode: opcode, Col: innerCol, OrigOpcode: origOpcode, + WCol: -1, }) } diff --git a/go/vt/vtgate/planbuilder/projection_pushing.go b/go/vt/vtgate/planbuilder/projection_pushing.go index acb01308b01..ab77398f50c 100644 --- a/go/vt/vtgate/planbuilder/projection_pushing.go +++ b/go/vt/vtgate/planbuilder/projection_pushing.go @@ -144,6 +144,7 @@ func pushProjectionIntoOA(ctx *plancontext.PlanningContext, expr *sqlparser.Alia Alias: expr.ColumnName(), Expr: expr.Expr, Original: expr, + WCol: -1, }) return offset, true, nil } diff --git a/go/vt/vtgate/planbuilder/show.go b/go/vt/vtgate/planbuilder/show.go index 132ca55715b..9bccd3b5e39 100644 --- a/go/vt/vtgate/planbuilder/show.go +++ b/go/vt/vtgate/planbuilder/show.go @@ -568,6 +568,7 @@ func buildShowVGtidPlan(show *sqlparser.ShowBasic, vschema plancontext.VSchema) Opcode: popcode.AggregateGtid, Col: 1, Alias: "global vgtid_executed", + WCol: -1, }, }, TruncateColumnCount: 2, diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index a1c8d01ad24..f229d79a05c 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -686,9 +686,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -798,9 +798,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -855,9 +855,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -901,7 +901,7 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1|3) AS min(distinct col2)", + "Aggregates": "min(1) AS min(distinct col2)", "GroupBy": "(0|2)", "ResultColumns": 2, "Inputs": [ @@ -912,9 +912,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -981,9 +981,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -2221,9 +2221,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, weight_string(col1), col2, weight_string(col2)", + "FieldQuery": "select col1, col2, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|3) ASC, (1|4) ASC", - "Query": "select col1, col2, col2, weight_string(col1), weight_string(col2) from `user` group by col1, weight_string(col1), col2, weight_string(col2) order by col1 asc, col2 asc", + "Query": "select col1, col2, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -3571,9 +3571,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` where 1 != 1 group by col1, weight_string(col1), col3, weight_string(col3)", + "FieldQuery": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` where 1 != 1 group by col1, col3, weight_string(col1), weight_string(col3)", "OrderBy": "(0|3) ASC, (2|4) ASC", - "Query": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` group by col1, weight_string(col1), col3, weight_string(col3) order by col1 asc, col3 asc", + "Query": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` group by col1, col3, weight_string(col1), weight_string(col3) order by col1 asc, col3 asc", "Table": "`user`" } ] @@ -3687,9 +3687,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select val2, val1, count(*), weight_string(val2), weight_string(val1) from `user` where 1 != 1 group by val2, weight_string(val2), val1, weight_string(val1)", + "FieldQuery": "select val2, val1, count(*), weight_string(val2), weight_string(val1) from `user` where 1 != 1 group by val2, val1, weight_string(val2), weight_string(val1)", "OrderBy": "(0|3) ASC, (1|4) ASC", - "Query": "select val2, val1, count(*), weight_string(val2), weight_string(val1) from `user` group by val2, weight_string(val2), val1, weight_string(val1) order by val2 asc, val1 asc", + "Query": "select val2, val1, count(*), weight_string(val2), weight_string(val1) from `user` group by val2, val1, weight_string(val2), weight_string(val1) order by val2 asc, val1 asc", "Table": "`user`" } ] @@ -3777,9 +3777,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select tcol1, tcol2, tcol2, weight_string(tcol1), weight_string(tcol2) from `user` where 1 != 1 group by tcol1, weight_string(tcol1), tcol2, weight_string(tcol2)", + "FieldQuery": "select tcol1, tcol2, tcol2, weight_string(tcol1), weight_string(tcol2) from `user` where 1 != 1 group by tcol1, tcol2, weight_string(tcol1), weight_string(tcol2)", "OrderBy": "(0|3) ASC, (1|4) ASC", - "Query": "select tcol1, tcol2, tcol2, weight_string(tcol1), weight_string(tcol2) from `user` group by tcol1, weight_string(tcol1), tcol2, weight_string(tcol2) order by tcol1 asc, tcol2 asc", + "Query": "select tcol1, tcol2, tcol2, weight_string(tcol1), weight_string(tcol2) from `user` group by tcol1, tcol2, weight_string(tcol1), weight_string(tcol2) order by tcol1 asc, tcol2 asc", "Table": "`user`" } ] @@ -3799,8 +3799,8 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "count_distinct(0|4) AS count(distinct tcol2), sum_count_star(2) AS count(*), sum_distinct(3|4) AS sum(distinct tcol2)", - "GroupBy": "(1|5)", + "Aggregates": "count_distinct(0|5) AS count(distinct tcol2), sum_count_star(2) AS count(*), sum_distinct(3|5) AS sum(distinct tcol2)", + "GroupBy": "(1|4)", "ResultColumns": 4, "Inputs": [ { @@ -3810,9 +3810,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select tcol2, tcol1, count(*), tcol2, weight_string(tcol2), weight_string(tcol1) from `user` where 1 != 1 group by tcol2, weight_string(tcol2), tcol1, weight_string(tcol1)", - "OrderBy": "(1|5) ASC, (0|4) ASC", - "Query": "select tcol2, tcol1, count(*), tcol2, weight_string(tcol2), weight_string(tcol1) from `user` group by tcol2, weight_string(tcol2), tcol1, weight_string(tcol1) order by tcol1 asc, tcol2 asc", + "FieldQuery": "select tcol2, tcol1, count(*), tcol2, weight_string(tcol1), weight_string(tcol2) from `user` where 1 != 1 group by tcol1, tcol2, weight_string(tcol1), weight_string(tcol2)", + "OrderBy": "(1|4) ASC, (0|5) ASC", + "Query": "select tcol2, tcol1, count(*), tcol2, weight_string(tcol1), weight_string(tcol2) from `user` group by tcol1, tcol2, weight_string(tcol1), weight_string(tcol2) order by tcol1 asc, tcol2 asc", "Table": "`user`" } ] @@ -6280,7 +6280,7 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "min(0) AS min(textcol1), max(1) AS max(textcol2), sum_distinct(2 COLLATE latin1_swedish_ci) AS sum(distinct textcol1), count_distinct(3 COLLATE latin1_swedish_ci) AS count(distinct textcol1)", + "Aggregates": "min(0 COLLATE latin1_swedish_ci) AS min(textcol1), max(1 COLLATE latin1_swedish_ci) AS max(textcol2), sum_distinct(2 COLLATE latin1_swedish_ci) AS sum(distinct textcol1), count_distinct(3 COLLATE latin1_swedish_ci) AS count(distinct textcol1)", "Inputs": [ { "OperatorType": "Route", @@ -6311,7 +6311,7 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1) AS min(textcol1), max(2) AS max(textcol2), sum_distinct(3 COLLATE latin1_swedish_ci) AS sum(distinct textcol1), count_distinct(4 COLLATE latin1_swedish_ci) AS count(distinct textcol1)", + "Aggregates": "min(1 COLLATE latin1_swedish_ci) AS min(textcol1), max(2 COLLATE latin1_swedish_ci) AS max(textcol2), sum_distinct(3 COLLATE latin1_swedish_ci) AS sum(distinct textcol1), count_distinct(4 COLLATE latin1_swedish_ci) AS count(distinct textcol1)", "GroupBy": "0", "Inputs": [ { diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go index d5dc11f7691..43bfbe502d9 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go @@ -113,6 +113,7 @@ func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName str aggregates = append(aggregates, &engine.AggregateParams{ Opcode: opcode.SupportedAggregates[aggregateFuncType], Col: len(sourceSelect.SelectExprs) - 1, + WCol: -1, }) } } diff --git a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go index 31a014a28f4..67ba246fd7d 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go +++ b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go @@ -440,9 +440,11 @@ func TestBuildPlanSuccess(t *testing.T) { aggregates: []*engine.AggregateParams{{ Opcode: opcode.AggregateSum, Col: 2, + WCol: -1, }, { Opcode: opcode.AggregateSum, Col: 3, + WCol: -1, }}, }, }, { diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 098f4f18ff1..c626e9adc67 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -695,6 +695,7 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer aggregates = append(aggregates, &engine.AggregateParams{ Opcode: opcode.SupportedAggregates[aggregateFuncType], Col: len(sourceSelect.SelectExprs) - 1, + WCol: -1, }) } } diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index ef2857e55cf..2f99de07e16 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -403,9 +403,11 @@ func TestVDiffPlanSuccess(t *testing.T) { Aggregates: []*engine.AggregateParams{{ Opcode: opcode.AggregateSum, Col: 2, + WCol: -1, }, { Opcode: opcode.AggregateSum, Col: 3, + WCol: -1, }}, GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1}}, Input: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), From d52f903a11a90ae9b936e98a8c89270c460b6ff6 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 21 Jun 2023 15:59:55 +0200 Subject: [PATCH 05/26] stop pushing of aggregation filtering into derived tables Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/derived.go | 20 +-------- .../planbuilder/testdata/aggr_cases.json | 45 ++++++++++++------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go index 9531a3759ab..1ddc84cfc56 100644 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ b/go/vt/vtgate/planbuilder/operators/derived.go @@ -17,8 +17,6 @@ limitations under the License. package operators import ( - "io" - "golang.org/x/exp/slices" "vitess.io/vitess/go/vt/sqlparser" @@ -131,8 +129,7 @@ func (d *Derived) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser. } newExpr := semantics.RewriteDerivedTableExpression(expr, tableInfo) - if !canBePushedDownIntoDerived(newExpr) { - // if we have an aggregation, we don't want to push it inside + if sqlparser.ContainsAggregation(newExpr) { return &Filter{Source: d, Predicates: []sqlparser.Expr{expr}}, nil } d.Source, err = d.Source.AddPredicate(ctx, newExpr) @@ -142,21 +139,6 @@ func (d *Derived) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser. return d, nil } -func canBePushedDownIntoDerived(expr sqlparser.Expr) (canBePushed bool) { - canBePushed = true - _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { - switch node.(type) { - case *sqlparser.Max, *sqlparser.Min: - // empty by default - case sqlparser.AggrFunc: - canBePushed = false - return false, io.EOF - } - return true, nil - }, expr) - return -} - func (d *Derived) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { col, ok := expr.Expr.(*sqlparser.ColName) if !ok { diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index f229d79a05c..d2cad3d6c33 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -5308,26 +5308,41 @@ "Instructions": { "OperatorType": "SimpleProjection", "Columns": [ - 0 + 1 ], "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Ordered", - "Aggregates": "max(1) AS bazo", - "GroupBy": "(0|2)", + "OperatorType": "Filter", + "Predicate": "bazo between 100 and 200", "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", - "OrderBy": "(0|2) ASC", - "Query": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` having max(baz) between 100 and 200) as f group by foo, weight_string(foo) order by foo asc", - "Table": "`user`" + "OperatorType": "SimpleProjection", + "Columns": [ + 1, + 0 + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "max(1) AS bazo", + "GroupBy": "(0|2)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", + "OrderBy": "(0|2) ASC", + "Query": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", + "Table": "`user`" + } + ] + } + ] } ] } From 3f8406352fc24768e19cb4a898ef0d302251c153 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 22 Jun 2023 11:27:23 +0200 Subject: [PATCH 06/26] wip - enable derived tables in the new horizon planner Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 21 ++ .../operators/{logical.go => ast2op.go} | 81 ++++-- go/vt/vtgate/planbuilder/operators/derived.go | 1 + go/vt/vtgate/planbuilder/operators/horizon.go | 16 + .../operators/horizon_expanding.go | 11 - .../vtgate/planbuilder/operators/operator.go | 6 +- .../planbuilder/operators/queryprojection.go | 3 +- .../vtgate/planbuilder/operators/subquery.go | 2 +- .../planbuilder/testdata/aggr_cases.json | 46 ++- .../planbuilder/testdata/from_cases.json | 32 +- .../testdata/info_schema57_cases.json | 51 +--- .../testdata/info_schema80_cases.json | 51 +--- .../planbuilder/testdata/select_cases.json | 44 +-- .../planbuilder/testdata/union_cases.json | 273 +----------------- .../testdata/unsupported_cases.json | 2 +- 15 files changed, 154 insertions(+), 486 deletions(-) rename go/vt/vtgate/planbuilder/operators/{logical.go => ast2op.go} (90%) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index c09e9628247..c970cf421f8 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -121,6 +121,19 @@ func (a *Aggregator) isDerived() bool { } func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { + if a.TableID != nil { + // this is a derived table, and so we have to translate the column names to the internal expression + col, ok := expr.Expr.(*sqlparser.ColName) + if !ok { + panic(4) + // return nil, 0, vterrors.VT13001("don't know how to handle this") + } + for i, column := range a.Columns { + if col.Name.EqualString(column.ColumnName()) { + return a, i, nil + } + } + } if addToGroupBy { return nil, 0, vterrors.VT13001("did not expect to add group by here") } @@ -178,6 +191,10 @@ func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser } func (a *Aggregator) GetColumns() ([]*sqlparser.AliasedExpr, error) { + if _, isSourceDerived := a.Source.(*Derived); isSourceDerived { + return a.Columns, nil + } + // we update the incoming columns, so we know about any new columns that have been added // in the optimization phase, other operators could be pushed down resulting in additional columns for aggregator. // Aggregator should be made aware of these to truncate them in final result. @@ -403,3 +420,7 @@ func (a *Aggregator) internalAddColumn(ctx *plancontext.PlanningContext, aliased } var _ ops.Operator = (*Aggregator)(nil) + +// func (a *Aggregator) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { +// +// } diff --git a/go/vt/vtgate/planbuilder/operators/logical.go b/go/vt/vtgate/planbuilder/operators/ast2op.go similarity index 90% rename from go/vt/vtgate/planbuilder/operators/logical.go rename to go/vt/vtgate/planbuilder/operators/ast2op.go index 41396070089..74ae9179f85 100644 --- a/go/vt/vtgate/planbuilder/operators/logical.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -30,8 +30,8 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -// createLogicalOperatorFromAST creates an operator tree that represents the input SELECT or UNION query -func createLogicalOperatorFromAST(ctx *plancontext.PlanningContext, selStmt sqlparser.Statement) (op ops.Operator, err error) { +// translateQueryToOp creates an operator tree that represents the input SELECT or UNION query +func translateQueryToOp(ctx *plancontext.PlanningContext, selStmt sqlparser.Statement) (op ops.Operator, err error) { switch node := selStmt.(type) { case *sqlparser.Select: op, err = createOperatorFromSelect(ctx, node) @@ -53,6 +53,36 @@ func createLogicalOperatorFromAST(ctx *plancontext.PlanningContext, selStmt sqlp return op, nil } +// func removeUnnecessaryOrdering(ctx *plancontext.PlanningContext, op ops.Operator) (ops.Operator, error) { +// horizon, ok := op.(horizonLike) +// if !ok { +// return op, nil +// } +// +// qp, err := horizon.getQP(ctx) +// if err != nil { +// return nil, err +// } +// +// if !qp.HasAggr { +// return op, nil +// } +// +// visit := func(op ops.Operator, lhsTables semantics.TableSet, isRoot bool) (ops.Operator, *rewrite.ApplyResult, error) { +// +// } +// +// shouldVisit := func(o ops.Operator) rewrite.VisitRule { +// switch o.(type) { +// case *Aggregator, *Route: +// return false +// default: +// return true +// } +// } +// rewrite.TopDown(op, TableID, visit, shouldVisit) +// } + // createOperatorFromSelect creates an operator tree that represents the input SELECT query func createOperatorFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.Select) (ops.Operator, error) { subq, err := createSubqueryFromStatement(ctx, sel) @@ -74,21 +104,20 @@ func createOperatorFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.S addColumnEquality(ctx, expr) } } - if subq == nil { - return &Horizon{ - Source: op, - Select: sel, - }, nil + + if subq != nil { + subq.Outer = op + op = subq } - subq.Outer = op + return &Horizon{ - Source: subq, + Source: op, Select: sel, }, nil } func createOperatorFromUnion(ctx *plancontext.PlanningContext, node *sqlparser.Union) (ops.Operator, error) { - opLHS, err := createLogicalOperatorFromAST(ctx, node.Left) + opLHS, err := translateQueryToOp(ctx, node.Left) if err != nil { return nil, err } @@ -97,7 +126,7 @@ func createOperatorFromUnion(ctx *plancontext.PlanningContext, node *sqlparser.U if isRHSUnion { return nil, vterrors.VT12001("nesting of UNIONs on the right-hand side") } - opRHS, err := createLogicalOperatorFromAST(ctx, node.Right) + opRHS, err := translateQueryToOp(ctx, node.Right) if err != nil { return nil, err } @@ -530,10 +559,10 @@ func modifyForAutoinc(ins *sqlparser.Insert, vTable *vindexes.Table) (*Generate, return gen, nil } -func getOperatorFromTableExpr(ctx *plancontext.PlanningContext, tableExpr sqlparser.TableExpr) (ops.Operator, error) { +func getOperatorFromTableExpr(ctx *plancontext.PlanningContext, tableExpr sqlparser.TableExpr, onlyTable bool) (ops.Operator, error) { switch tableExpr := tableExpr.(type) { case *sqlparser.AliasedTableExpr: - return getOperatorFromAliasedTableExpr(ctx, tableExpr) + return getOperatorFromAliasedTableExpr(ctx, tableExpr, onlyTable) case *sqlparser.JoinTableExpr: return getOperatorFromJoinTableExpr(ctx, tableExpr) case *sqlparser.ParenTableExpr: @@ -544,11 +573,11 @@ func getOperatorFromTableExpr(ctx *plancontext.PlanningContext, tableExpr sqlpar } func getOperatorFromJoinTableExpr(ctx *plancontext.PlanningContext, tableExpr *sqlparser.JoinTableExpr) (ops.Operator, error) { - lhs, err := getOperatorFromTableExpr(ctx, tableExpr.LeftExpr) + lhs, err := getOperatorFromTableExpr(ctx, tableExpr.LeftExpr, false) if err != nil { return nil, err } - rhs, err := getOperatorFromTableExpr(ctx, tableExpr.RightExpr) + rhs, err := getOperatorFromTableExpr(ctx, tableExpr.RightExpr, false) if err != nil { return nil, err } @@ -563,7 +592,7 @@ func getOperatorFromJoinTableExpr(ctx *plancontext.PlanningContext, tableExpr *s } } -func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr *sqlparser.AliasedTableExpr) (ops.Operator, error) { +func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr *sqlparser.AliasedTableExpr, onlyTable bool) (ops.Operator, error) { tableID := ctx.SemTable.TableSetFor(tableExpr) switch tbl := tableExpr.Expr.(type) { case sqlparser.TableName: @@ -591,7 +620,7 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr qg.Tables = append(qg.Tables, qt) return qg, nil case *sqlparser.DerivedTable: - inner, err := createLogicalOperatorFromAST(ctx, tbl.Select) + inner, err := translateQueryToOp(ctx, tbl.Select) if err != nil { return nil, err } @@ -599,12 +628,26 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr inner = horizon.Source } + selStmt := sqlparser.CloneSelectStatement(tbl.Select) + if onlyTable { + selStmt.SetOrderBy(nil) + } + sel, isSel := selStmt.(*sqlparser.Select) + if !isSel { + return nil, errHorizonNotPlanned() + } + qp, err := CreateQPFromSelect(ctx, sel) + if err != nil { + return nil, err + } + return &Derived{ TableId: tableID, Alias: tableExpr.As.String(), Source: inner, - Query: tbl.Select, + Query: selStmt, ColumnAliases: tableExpr.Columns, + QP: qp, }, nil default: return nil, vterrors.VT13001(fmt.Sprintf("unable to use: %T", tbl)) @@ -614,7 +657,7 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr func crossJoin(ctx *plancontext.PlanningContext, exprs sqlparser.TableExprs) (ops.Operator, error) { var output ops.Operator for _, tableExpr := range exprs { - op, err := getOperatorFromTableExpr(ctx, tableExpr) + op, err := getOperatorFromTableExpr(ctx, tableExpr, len(exprs) == 1) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go index 1ddc84cfc56..0623c0480e9 100644 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ b/go/vt/vtgate/planbuilder/operators/derived.go @@ -52,6 +52,7 @@ func (d *Derived) Clone(inputs []ops.Operator) ops.Operator { Columns: slices.Clone(d.Columns), ColumnsOffset: slices.Clone(d.ColumnsOffset), TableId: d.TableId, + QP: d.QP, } } diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index a05d23795eb..45ec06fb4b7 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -116,3 +116,19 @@ func (h *Horizon) setQP(qp *QueryProjection) { func (h *Horizon) ShortDescription() string { return "" } + +//func (h *Horizon) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { +// src, ok := h.Source.(*Derived) +// if !ok { +// return h, rewrite.SameTree, nil +// } +// qp, err := src.getQP(ctx) +// if err != nil { +// return nil, nil, err +// } +// +// src.Query.(*sqlparser.Select).OrderBy = nil +// qp.OrderExprs = nil +// +// return h, rewrite.SameTree, nil +//} diff --git a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go index 59dc9a32b65..1ca64a1c081 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go @@ -74,13 +74,6 @@ func expandHorizon(ctx *plancontext.PlanningContext, horizon horizonLike) (ops.O return op, rewrite.NewTree("expand horizon into smaller components", op), nil } -func checkInvalid(aggregations []Aggr, horizon horizonLike) error { - if _, isDerived := horizon.(*Derived); isDerived { - return errHorizonNotPlanned() - } - return nil -} - func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizonLike) (out ops.Operator, err error) { qp, err := horizon.getQP(ctx) if err != nil { @@ -112,10 +105,6 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizo return nil, err } - if err := checkInvalid(aggregations, horizon); err != nil { - return nil, err - } - a := &Aggregator{ Source: horizon.src(), Original: true, diff --git a/go/vt/vtgate/planbuilder/operators/operator.go b/go/vt/vtgate/planbuilder/operators/operator.go index 6bb953cb418..8fa73e882fe 100644 --- a/go/vt/vtgate/planbuilder/operators/operator.go +++ b/go/vt/vtgate/planbuilder/operators/operator.go @@ -54,11 +54,15 @@ type ( // PlanQuery creates a query plan for a given SQL statement func PlanQuery(ctx *plancontext.PlanningContext, stmt sqlparser.Statement) (ops.Operator, error) { - op, err := createLogicalOperatorFromAST(ctx, stmt) + op, err := translateQueryToOp(ctx, stmt) if err != nil { return nil, err } + if op, err = compact(ctx, op); err != nil { + return nil, err + } + if err = checkValid(op); err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index cde1c7a8d02..dec8b88f498 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -836,9 +836,8 @@ func (qp *QueryProjection) GetColumnCount() int { // We don't currently support planning for operators having derived tables. func checkAggregationSupported(op ops.Operator) error { return rewrite.Visit(op, func(operator ops.Operator) error { - _, isDerived := operator.(*Derived) projection, isProjection := operator.(*Projection) - if isDerived || (isProjection && projection.TableID != nil) { + if isProjection && projection.TableID != nil { return errHorizonNotPlanned() } return nil diff --git a/go/vt/vtgate/planbuilder/operators/subquery.go b/go/vt/vtgate/planbuilder/operators/subquery.go index 632042bf20e..8966c30e192 100644 --- a/go/vt/vtgate/planbuilder/operators/subquery.go +++ b/go/vt/vtgate/planbuilder/operators/subquery.go @@ -111,7 +111,7 @@ func createSubqueryFromStatement(ctx *plancontext.PlanningContext, stmt sqlparse } subq := &SubQuery{} for _, sq := range ctx.SemTable.SubqueryMap[stmt] { - opInner, err := createLogicalOperatorFromAST(ctx, sq.Subquery.Select) + opInner, err := translateQueryToOp(ctx, sq.Subquery.Select) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index d2cad3d6c33..a5eaadd6c9a 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -362,28 +362,20 @@ "QueryType": "SELECT", "Original": "select a from (select count(*) as a from user) t", "Instructions": { - "OperatorType": "SimpleProjection", - "Columns": [ - 0 - ], + "OperatorType": "Aggregate", + "Variant": "Scalar", + "Aggregates": "sum_count_star(0) AS a", "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Scalar", - "Aggregates": "sum_count_star(0) AS a", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select count(*) as a from `user` where 1 != 1", - "Query": "select count(*) as a from `user`", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*) as a from `user` where 1 != 1", + "Query": "select count(*) as a from `user`", + "Table": "`user`" } ] }, @@ -1939,7 +1931,7 @@ "Sharded": true }, "FieldQuery": "select count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where 1 != 1) as a where 1 != 1", - "Query": "select count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id order by user_extra.extra asc) as a", + "Query": "select count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id) as a", "Table": "`user`, user_extra" } ] @@ -1989,7 +1981,7 @@ "Sharded": true }, "FieldQuery": "select col from (select `user`.col, user_extra.extra from `user`, user_extra where 1 != 1) as a where 1 != 1", - "Query": "select col from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id order by user_extra.extra asc) as a", + "Query": "select col from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id) as a", "Table": "`user`, user_extra" }, "TablesUsed": [ @@ -2020,7 +2012,7 @@ }, "FieldQuery": "select col, count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where 1 != 1) as a where 1 != 1 group by col", "OrderBy": "0 ASC", - "Query": "select col, count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id order by user_extra.extra asc) as a group by col order by col asc", + "Query": "select col, count(*) from (select `user`.col, user_extra.extra from `user`, user_extra where `user`.id = user_extra.user_id) as a group by col order by col asc", "Table": "`user`, user_extra" } ] @@ -4911,8 +4903,8 @@ "Sharded": true }, "FieldQuery": "select id, val1, weight_string(val1) from `user` where 1 != 1", - "OrderBy": "(1|2) ASC, (1|2) ASC", - "Query": "select id, val1, weight_string(val1) from `user` where val2 < 4 order by val1 asc, val1 asc limit :__upper_limit", + "OrderBy": "(1|2) ASC", + "Query": "select id, val1, weight_string(val1) from `user` where val2 < 4 order by val1 asc limit :__upper_limit", "Table": "`user`" } ] @@ -4955,7 +4947,7 @@ "Inputs": [ { "OperatorType": "Filter", - "Predicate": ":1 = 1", + "Predicate": "count(*) = 1", "Inputs": [ { "OperatorType": "Aggregate", @@ -5290,7 +5282,7 @@ "Sharded": true }, "FieldQuery": "select t1.portalId, t1.flowId from (select portalId, flowId, count(*) as `count` from user_extra where 1 != 1 group by user_id, flowId) as t1 where 1 != 1", - "Query": "select t1.portalId, t1.flowId from (select portalId, flowId, count(*) as `count` from user_extra where localDate > :v1 group by user_id, flowId order by null) as t1 where `count` >= :v2", + "Query": "select t1.portalId, t1.flowId from (select portalId, flowId, count(*) as `count` from user_extra where localDate > :v1 group by user_id, flowId) as t1 where `count` >= :v2", "Table": "user_extra" }, "TablesUsed": [ diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index 8ef50577bc0..d6c12fd9b77 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -4896,28 +4896,20 @@ "QueryType": "SELECT", "Original": "select a as k from (select count(*) as a from user) t", "Instructions": { - "OperatorType": "SimpleProjection", - "Columns": [ - 0 - ], + "OperatorType": "Aggregate", + "Variant": "Scalar", + "Aggregates": "sum_count_star(0) AS a", "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Scalar", - "Aggregates": "sum_count_star(0) AS a", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select count(*) as a from `user` where 1 != 1", - "Query": "select count(*) as a from `user`", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*) as a from `user` where 1 != 1", + "Query": "select count(*) as a from `user`", + "Table": "`user`" } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json index d96ac0d9caf..08efdb97fc5 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json @@ -1147,22 +1147,7 @@ "comment": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select sum(found) from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", - "Query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", - "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "union as a derived table", @@ -1208,22 +1193,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select found from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select found from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", - "Query": "select found from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", - "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "merge system schema queries as long as they have any same table_schema", @@ -1454,22 +1424,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select TABLE_NAME from information_schema.tables t where t.TABLE_SCHEMA = 'a' union select TABLE_NAME from information_schema.columns) dt", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where 1 != 1 union select TABLE_NAME from information_schema.`columns` where 1 != 1) as dt where 1 != 1", - "Query": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where t.TABLE_SCHEMA = :__vtschemaname /* VARCHAR */ union select TABLE_NAME from information_schema.`columns`) as dt", - "SysTableTableSchema": "[VARCHAR(\"a\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "merge even one side have schema name in subquery", diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json index 6363203b88a..dd0ef9622fd 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json @@ -1229,22 +1229,7 @@ "comment": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select sum(found) from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", - "Query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", - "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "union as a derived table", @@ -1290,22 +1275,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select found from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select found from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", - "Query": "select found from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", - "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "merge system schema queries as long as they have any same table_schema", @@ -1536,22 +1506,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select TABLE_NAME from information_schema.tables t where t.TABLE_SCHEMA = 'a' union select TABLE_NAME from information_schema.columns) dt", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where 1 != 1 union select TABLE_NAME from information_schema.`columns` where 1 != 1) as dt where 1 != 1", - "Query": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where t.TABLE_SCHEMA = :__vtschemaname /* VARCHAR */ union select TABLE_NAME from information_schema.`columns`) as dt", - "SysTableTableSchema": "[VARCHAR(\"a\")]", - "Table": "information_schema.`tables`" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "merge even one side have schema name in subquery", diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index e23bea5e0d4..95d9f6e1016 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -4215,49 +4215,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select found from (select id as found from user union all (select id from unsharded)) as t", - "Instructions": { - "OperatorType": "SimpleProjection", - "Columns": [ - 0 - ], - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id as found from `user` where 1 != 1", - "Query": "select id as found from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select id from unsharded where 1 != 1", - "Query": "select id from unsharded", - "Table": "unsharded" - } - ] - } - ] - }, - "TablesUsed": [ - "main.unsharded", - "user.user" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "use output column containing data from both sides of the join", diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index ef76413b5c7..057123fcb00 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -348,25 +348,7 @@ "Table": "`user`" } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select * from user union all select * from user_extra) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select * from (select * from `user` where 1 != 1 union all select * from user_extra where 1 != 1) as t where 1 != 1", - "Query": "select * from (select * from `user` union all select * from user_extra) as t", - "Table": "`user`" - }, - "TablesUsed": [ - "user.user", - "user.user_extra" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "union operations in derived table, without star expression (FROM)¡", @@ -386,25 +368,7 @@ "Table": "`user`" } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select col1,col2 from (select col1, col2 from user union all select col1, col2 from user_extra) as t", - "Instructions": { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select col1, col2 from (select col1, col2 from `user` where 1 != 1 union all select col1, col2 from user_extra where 1 != 1) as t where 1 != 1", - "Query": "select col1, col2 from (select col1, col2 from `user` union all select col1, col2 from user_extra) as t", - "Table": "`user`" - }, - "TablesUsed": [ - "user.user", - "user.user_extra" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "union all between two scatter selects, with order by", @@ -1747,41 +1711,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from ((select id from user union select id+1 from user) union select user_id from user_extra) as t", - "Instructions": { - "OperatorType": "SimpleProjection", - "Columns": [ - 0 - ], - "Inputs": [ - { - "OperatorType": "Distinct", - "Collations": [ - "(0:1)" - ], - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1 union all select id + 1, weight_string(id + 1) from `user` where 1 != 1 union select user_id, weight_string(user_id) from user_extra where 1 != 1", - "Query": "select id, weight_string(id) from `user` union all select id + 1, weight_string(id + 1) from `user` union select user_id, weight_string(user_id) from user_extra", - "Table": "`user`" - } - ] - } - ] - }, - "TablesUsed": [ - "user.user", - "user.user_extra" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "gen4 optimises away ORDER BY when it's safe to do", @@ -2177,90 +2107,7 @@ ] } }, - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select tbl2.id FROM ((select id from user order by id limit 5) union all (select id from user order by id desc limit 5)) as tbl1 INNER JOIN user as tbl2 ON tbl1.id = tbl2.id", - "Instructions": { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0", - "JoinVars": { - "tbl1_id": 0 - }, - "TableName": "`user`_`user`", - "Inputs": [ - { - "OperatorType": "SimpleProjection", - "Columns": [ - 0 - ], - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Limit", - "Count": "INT64(5)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "(0|1) ASC", - "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", - "ResultColumns": 1, - "Table": "`user`" - } - ] - }, - { - "OperatorType": "Limit", - "Count": "INT64(5)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "(0|1) DESC", - "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", - "ResultColumns": 1, - "Table": "`user`" - } - ] - } - ] - } - ] - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select tbl2.id from `user` as tbl2 where 1 != 1", - "Query": "select tbl2.id from `user` as tbl2 where tbl2.id = :tbl1_id", - "Table": "`user`", - "Values": [ - ":tbl1_id" - ], - "Vindex": "user_index" - } - ] - }, - "TablesUsed": [ - "user.user" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "ambiguous LIMIT", @@ -2294,129 +2141,25 @@ "comment": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", "query": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", "v3-plan": "VT12001: unsupported: expression on results of a cross-shard subquery", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", - "Instructions": { - "OperatorType": "SimpleProjection", - "Columns": [ - 2 - ], - "Inputs": [ - { - "OperatorType": "Distinct", - "Collations": [ - "(0:1)" - ], - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id + 42 as foo, weight_string(id + 42), 1 from `user` where 1 != 1", - "Query": "select distinct id + 42 as foo, weight_string(id + 42), 1 from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 + id as foo, weight_string(1 + id), 1 from unsharded where 1 != 1", - "Query": "select distinct 1 + id as foo, weight_string(1 + id), 1 from unsharded", - "Table": "unsharded" - } - ] - } - ] - } - ] - }, - "TablesUsed": [ - "main.unsharded", - "user.user" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "systable union query in derived table with constraint on outside (without star projection)", "query": "select * from (select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `COLUMN_NAME` = 'primary'", "v3-plan": "VT12001: unsupported: filtering on results of cross-shard subquery", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `COLUMN_NAME` = 'primary'", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", - "Query": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.COLUMN_NAME = 'primary' union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.COLUMN_NAME = 'primary') as kcu", - "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", - "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", - "Table": "information_schema.key_column_usage" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "pushes predicate on both sides of UNION", "query": "select * from (select name, id as foo from user union select 'extra', user_id from user_extra) X where X.foo = 3", "v3-plan": "VT12001: unsupported: filtering on results of cross-shard subquery", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select name, id as foo from user union select 'extra', user_id from user_extra) X where X.foo = 3", - "Instructions": { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select X.`name`, X.foo from (select `name`, id as foo from `user` where 1 != 1 union select 'extra', user_id from user_extra where 1 != 1) as X where 1 != 1", - "Query": "select X.`name`, X.foo from (select `name`, id as foo from `user` where id = 3 union select 'extra', user_id from user_extra where user_id = 3) as X", - "Table": "`user`", - "Values": [ - "INT64(3)" - ], - "Vindex": "user_index" - }, - "TablesUsed": [ - "user.user", - "user.user_extra" - ] - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "systable union query in derived table with constraint on outside (star projection)", "query": "select * from (select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `constraint_name` = 'primary'", "v3-plan": "VT03019: column constraint_name not found", - "gen4-plan": { - "QueryType": "SELECT", - "Original": "select * from (select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `constraint_name` = 'primary'", - "Instructions": { - "OperatorType": "Route", - "Variant": "DBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", - "Query": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary' union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary') as kcu", - "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", - "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", - "Table": "information_schema.key_column_usage" - } - } + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "unknown columns are OK as long as the whole query is unsharded", diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 330de6c7fd3..42102652b78 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -407,7 +407,7 @@ "comment": "aggregation on union", "query": "select sum(col) from (select col from user union all select col from unsharded) t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": "VT12001: unsupported: using aggregation on top of a *planbuilder.concatenateGen4 plan" + "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" }, { "comment": "insert having subquery in row values", From 64aa0fbe83fae6b37e3e704d4dd763ee5dd9679e Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 22 Jun 2023 11:38:07 +0200 Subject: [PATCH 07/26] Bring back support for union inside derived tables Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/gen4_planner.go | 2 +- go/vt/vtgate/planbuilder/horizon_planning.go | 2 +- go/vt/vtgate/planbuilder/operators/ast2op.go | 12 +- go/vt/vtgate/planbuilder/operators/derived.go | 2 +- go/vt/vtgate/planbuilder/operators/horizon.go | 6 +- .../planbuilder/operators/queryprojection.go | 18 +- .../operators/queryprojection_test.go | 4 +- .../testdata/info_schema80_cases.json | 51 +++- .../planbuilder/testdata/select_cases.json | 44 ++- .../planbuilder/testdata/union_cases.json | 273 +++++++++++++++++- .../testdata/unsupported_cases.json | 2 +- 11 files changed, 383 insertions(+), 33 deletions(-) diff --git a/go/vt/vtgate/planbuilder/gen4_planner.go b/go/vt/vtgate/planbuilder/gen4_planner.go index 87853dca885..de52b22b37a 100644 --- a/go/vt/vtgate/planbuilder/gen4_planner.go +++ b/go/vt/vtgate/planbuilder/gen4_planner.go @@ -592,7 +592,7 @@ func planHorizon(ctx *plancontext.PlanningContext, plan logicalPlan, in sqlparse } func planOrderByOnUnion(ctx *plancontext.PlanningContext, plan logicalPlan, union *sqlparser.Union) (logicalPlan, error) { - qp, err := operators.CreateQPFromUnion(ctx, union) + qp, err := operators.CreateQPFromSelectStatement(ctx, union) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/horizon_planning.go b/go/vt/vtgate/planbuilder/horizon_planning.go index 3766d817326..caf20cb4961 100644 --- a/go/vt/vtgate/planbuilder/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/horizon_planning.go @@ -69,7 +69,7 @@ func (hp *horizonPlanning) planHorizon(ctx *plancontext.PlanningContext, plan lo } var err error - hp.qp, err = operators.CreateQPFromSelect(ctx, hp.sel) + hp.qp, err = operators.CreateQPFromSelectStatement(ctx, hp.sel) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 74ae9179f85..6df7fb45063 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -628,15 +628,11 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr inner = horizon.Source } - selStmt := sqlparser.CloneSelectStatement(tbl.Select) + stmt := sqlparser.CloneSelectStatement(tbl.Select) if onlyTable { - selStmt.SetOrderBy(nil) + stmt.SetOrderBy(nil) } - sel, isSel := selStmt.(*sqlparser.Select) - if !isSel { - return nil, errHorizonNotPlanned() - } - qp, err := CreateQPFromSelect(ctx, sel) + qp, err := CreateQPFromSelectStatement(ctx, stmt) if err != nil { return nil, err } @@ -645,7 +641,7 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr TableId: tableID, Alias: tableExpr.As.String(), Source: inner, - Query: selStmt, + Query: stmt, ColumnAliases: tableExpr.Columns, QP: qp, }, nil diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go index 0623c0480e9..3ac8b5029a2 100644 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ b/go/vt/vtgate/planbuilder/operators/derived.go @@ -232,7 +232,7 @@ func (d *Derived) getQP(ctx *plancontext.PlanningContext) (*QueryProjection, err if d.QP != nil { return d.QP, nil } - qp, err := CreateQPFromSelect(ctx, d.Query.(*sqlparser.Select)) + qp, err := CreateQPFromSelectStatement(ctx, d.Query) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index 45ec06fb4b7..5813798689a 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -101,7 +101,7 @@ func (h *Horizon) getQP(ctx *plancontext.PlanningContext) (*QueryProjection, err if h.QP != nil { return h.QP, nil } - qp, err := CreateQPFromSelect(ctx, h.Select.(*sqlparser.Select)) + qp, err := CreateQPFromSelectStatement(ctx, h.Select) if err != nil { return nil, err } @@ -117,7 +117,7 @@ func (h *Horizon) ShortDescription() string { return "" } -//func (h *Horizon) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { +// func (h *Horizon) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { // src, ok := h.Source.(*Derived) // if !ok { // return h, rewrite.SameTree, nil @@ -131,4 +131,4 @@ func (h *Horizon) ShortDescription() string { // qp.OrderExprs = nil // // return h, rewrite.SameTree, nil -//} +// } diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index dec8b88f498..a8eb5fa4741 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -208,8 +208,8 @@ func (s SelectExpr) GetAliasedExpr() (*sqlparser.AliasedExpr, error) { } } -// CreateQPFromSelect creates the QueryProjection for the input *sqlparser.Select -func CreateQPFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.Select) (*QueryProjection, error) { +// createQPFromSelect creates the QueryProjection for the input *sqlparser.Select +func createQPFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.Select) (*QueryProjection, error) { qp := &QueryProjection{ Distinct: sel.Distinct, } @@ -316,8 +316,8 @@ func (qp *QueryProjection) addSelectExpressions(sel *sqlparser.Select) error { return nil } -// CreateQPFromUnion creates the QueryProjection for the input *sqlparser.Union -func CreateQPFromUnion(ctx *plancontext.PlanningContext, union *sqlparser.Union) (*QueryProjection, error) { +// createQPFromUnion creates the QueryProjection for the input *sqlparser.Union +func createQPFromUnion(ctx *plancontext.PlanningContext, union *sqlparser.Union) (*QueryProjection, error) { qp := &QueryProjection{} sel := sqlparser.GetFirstSelect(union) @@ -881,3 +881,13 @@ func CompareRefInt(a *int, b *int) bool { } return *a < *b } + +func CreateQPFromSelectStatement(ctx *plancontext.PlanningContext, stmt sqlparser.SelectStatement) (*QueryProjection, error) { + switch sel := stmt.(type) { + case *sqlparser.Select: + return createQPFromSelect(ctx, sel) + case *sqlparser.Union: + return createQPFromUnion(ctx, sel) + } + return nil, vterrors.VT13001("Can only create query projection from Union and Select statements") +} diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection_test.go b/go/vt/vtgate/planbuilder/operators/queryprojection_test.go index 2a89cd10716..7c92b716d7c 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection_test.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection_test.go @@ -87,7 +87,7 @@ func TestQP(t *testing.T) { _, err = semantics.Analyze(sel, "", &semantics.FakeSI{}) require.NoError(t, err) - qp, err := CreateQPFromSelect(ctx, sel) + qp, err := createQPFromSelect(ctx, sel) if tcase.expErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tcase.expErr) @@ -194,7 +194,7 @@ func TestQPSimplifiedExpr(t *testing.T) { _, err = semantics.Analyze(sel, "", &semantics.FakeSI{}) require.NoError(t, err) ctx := &plancontext.PlanningContext{SemTable: semantics.EmptySemTable()} - qp, err := CreateQPFromSelect(ctx, sel) + qp, err := createQPFromSelect(ctx, sel) require.NoError(t, err) require.Equal(t, tc.expected[1:], qp.toString()) }) diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json index dd0ef9622fd..6363203b88a 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json @@ -1229,7 +1229,22 @@ "comment": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select sum(found) from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", + "Query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", + "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "union as a derived table", @@ -1275,7 +1290,22 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select found from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select found from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", + "Query": "select found from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", + "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "merge system schema queries as long as they have any same table_schema", @@ -1506,7 +1536,22 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select TABLE_NAME from information_schema.tables t where t.TABLE_SCHEMA = 'a' union select TABLE_NAME from information_schema.columns) dt", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where 1 != 1 union select TABLE_NAME from information_schema.`columns` where 1 != 1) as dt where 1 != 1", + "Query": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where t.TABLE_SCHEMA = :__vtschemaname /* VARCHAR */ union select TABLE_NAME from information_schema.`columns`) as dt", + "SysTableTableSchema": "[VARCHAR(\"a\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "merge even one side have schema name in subquery", diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index 95d9f6e1016..e23bea5e0d4 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -4215,7 +4215,49 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select found from (select id as found from user union all (select id from unsharded)) as t", + "Instructions": { + "OperatorType": "SimpleProjection", + "Columns": [ + 0 + ], + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id as found from `user` where 1 != 1", + "Query": "select id as found from `user`", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select id from unsharded where 1 != 1", + "Query": "select id from unsharded", + "Table": "unsharded" + } + ] + } + ] + }, + "TablesUsed": [ + "main.unsharded", + "user.user" + ] + } }, { "comment": "use output column containing data from both sides of the join", diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index 057123fcb00..ef76413b5c7 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -348,7 +348,25 @@ "Table": "`user`" } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select * from user union all select * from user_extra) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select * from (select * from `user` where 1 != 1 union all select * from user_extra where 1 != 1) as t where 1 != 1", + "Query": "select * from (select * from `user` union all select * from user_extra) as t", + "Table": "`user`" + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } }, { "comment": "union operations in derived table, without star expression (FROM)¡", @@ -368,7 +386,25 @@ "Table": "`user`" } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select col1,col2 from (select col1, col2 from user union all select col1, col2 from user_extra) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select col1, col2 from (select col1, col2 from `user` where 1 != 1 union all select col1, col2 from user_extra where 1 != 1) as t where 1 != 1", + "Query": "select col1, col2 from (select col1, col2 from `user` union all select col1, col2 from user_extra) as t", + "Table": "`user`" + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } }, { "comment": "union all between two scatter selects, with order by", @@ -1711,7 +1747,41 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from ((select id from user union select id+1 from user) union select user_id from user_extra) as t", + "Instructions": { + "OperatorType": "SimpleProjection", + "Columns": [ + 0 + ], + "Inputs": [ + { + "OperatorType": "Distinct", + "Collations": [ + "(0:1)" + ], + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1 union all select id + 1, weight_string(id + 1) from `user` where 1 != 1 union select user_id, weight_string(user_id) from user_extra where 1 != 1", + "Query": "select id, weight_string(id) from `user` union all select id + 1, weight_string(id + 1) from `user` union select user_id, weight_string(user_id) from user_extra", + "Table": "`user`" + } + ] + } + ] + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } }, { "comment": "gen4 optimises away ORDER BY when it's safe to do", @@ -2107,7 +2177,90 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select tbl2.id FROM ((select id from user order by id limit 5) union all (select id from user order by id desc limit 5)) as tbl1 INNER JOIN user as tbl2 ON tbl1.id = tbl2.id", + "Instructions": { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0", + "JoinVars": { + "tbl1_id": 0 + }, + "TableName": "`user`_`user`", + "Inputs": [ + { + "OperatorType": "SimpleProjection", + "Columns": [ + 0 + ], + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Limit", + "Count": "INT64(5)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "(0|1) ASC", + "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", + "ResultColumns": 1, + "Table": "`user`" + } + ] + }, + { + "OperatorType": "Limit", + "Count": "INT64(5)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "(0|1) DESC", + "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", + "ResultColumns": 1, + "Table": "`user`" + } + ] + } + ] + } + ] + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select tbl2.id from `user` as tbl2 where 1 != 1", + "Query": "select tbl2.id from `user` as tbl2 where tbl2.id = :tbl1_id", + "Table": "`user`", + "Values": [ + ":tbl1_id" + ], + "Vindex": "user_index" + } + ] + }, + "TablesUsed": [ + "user.user" + ] + } }, { "comment": "ambiguous LIMIT", @@ -2141,25 +2294,129 @@ "comment": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", "query": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", "v3-plan": "VT12001: unsupported: expression on results of a cross-shard subquery", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select 1 from (select id+42 as foo from user union select 1+id as foo from unsharded) as t", + "Instructions": { + "OperatorType": "SimpleProjection", + "Columns": [ + 2 + ], + "Inputs": [ + { + "OperatorType": "Distinct", + "Collations": [ + "(0:1)" + ], + "Inputs": [ + { + "OperatorType": "Concatenate", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id + 42 as foo, weight_string(id + 42), 1 from `user` where 1 != 1", + "Query": "select distinct id + 42 as foo, weight_string(id + 42), 1 from `user`", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select 1 + id as foo, weight_string(1 + id), 1 from unsharded where 1 != 1", + "Query": "select distinct 1 + id as foo, weight_string(1 + id), 1 from unsharded", + "Table": "unsharded" + } + ] + } + ] + } + ] + }, + "TablesUsed": [ + "main.unsharded", + "user.user" + ] + } }, { "comment": "systable union query in derived table with constraint on outside (without star projection)", "query": "select * from (select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `COLUMN_NAME` = 'primary'", "v3-plan": "VT12001: unsupported: filtering on results of cross-shard subquery", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select kcu.`COLUMN_NAME` from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `COLUMN_NAME` = 'primary'", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", + "Query": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.COLUMN_NAME = 'primary' union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.COLUMN_NAME = 'primary') as kcu", + "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", + "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", + "Table": "information_schema.key_column_usage" + } + } }, { "comment": "pushes predicate on both sides of UNION", "query": "select * from (select name, id as foo from user union select 'extra', user_id from user_extra) X where X.foo = 3", "v3-plan": "VT12001: unsupported: filtering on results of cross-shard subquery", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select name, id as foo from user union select 'extra', user_id from user_extra) X where X.foo = 3", + "Instructions": { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select X.`name`, X.foo from (select `name`, id as foo from `user` where 1 != 1 union select 'extra', user_id from user_extra where 1 != 1) as X where 1 != 1", + "Query": "select X.`name`, X.foo from (select `name`, id as foo from `user` where id = 3 union select 'extra', user_id from user_extra where user_id = 3) as X", + "Table": "`user`", + "Values": [ + "INT64(3)" + ], + "Vindex": "user_index" + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } }, { "comment": "systable union query in derived table with constraint on outside (star projection)", "query": "select * from (select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `constraint_name` = 'primary'", "v3-plan": "VT03019: column constraint_name not found", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'user_extra' union select * from `information_schema`.`key_column_usage` `kcu` where `kcu`.`table_schema` = 'user' and `kcu`.`table_name` = 'music') `kcu` where `constraint_name` = 'primary'", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", + "Query": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary' union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary') as kcu", + "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", + "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", + "Table": "information_schema.key_column_usage" + } + } }, { "comment": "unknown columns are OK as long as the whole query is unsharded", diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 42102652b78..330de6c7fd3 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -407,7 +407,7 @@ "comment": "aggregation on union", "query": "select sum(col) from (select col from user union all select col from unsharded) t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": "VT12001: unsupported: using aggregation on top of a *planbuilder.concatenateGen4 plan" }, { "comment": "insert having subquery in row values", From 3280d7914c7662d87222ea97c7d233eb4452fb0a Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 22 Jun 2023 11:40:19 +0200 Subject: [PATCH 08/26] fix typo in error Signed-off-by: Florent Poinsard Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/queryprojection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index a8eb5fa4741..14e66cbce3f 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -889,5 +889,5 @@ func CreateQPFromSelectStatement(ctx *plancontext.PlanningContext, stmt sqlparse case *sqlparser.Union: return createQPFromUnion(ctx, sel) } - return nil, vterrors.VT13001("Can only create query projection from Union and Select statements") + return nil, vterrors.VT13001("can only create query projection from Union and Select statements") } From c5d7eb81fc946f08423ca549d9eedfaa1b7796ea Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 22 Jun 2023 14:14:38 +0200 Subject: [PATCH 09/26] update test expectations Signed-off-by: Andres Taylor --- .../testdata/info_schema57_cases.json | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json index 08efdb97fc5..d96ac0d9caf 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json @@ -1147,7 +1147,22 @@ "comment": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select sum(found) from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", + "Query": "select sum(found) from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", + "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "union as a derived table", @@ -1193,7 +1208,22 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select found from (select 1 as found from information_schema.`tables` where table_schema = 'music' union all (select 1 as found from information_schema.views where table_schema = 'music' limit 1)) as t", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select found from (select 1 as found from information_schema.`tables` where 1 != 1 union all (select 1 as found from information_schema.views where 1 != 1)) as t where 1 != 1", + "Query": "select found from (select 1 as found from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */ union all (select 1 as found from information_schema.views where table_schema = :__vtschemaname /* VARCHAR */ limit 1)) as t", + "SysTableTableSchema": "[VARCHAR(\"music\"), VARCHAR(\"music\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "merge system schema queries as long as they have any same table_schema", @@ -1424,7 +1454,22 @@ ] } }, - "gen4-plan": "VT12001: unsupported: query cannot be fully operator planned" + "gen4-plan": { + "QueryType": "SELECT", + "Original": "select * from (select TABLE_NAME from information_schema.tables t where t.TABLE_SCHEMA = 'a' union select TABLE_NAME from information_schema.columns) dt", + "Instructions": { + "OperatorType": "Route", + "Variant": "DBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where 1 != 1 union select TABLE_NAME from information_schema.`columns` where 1 != 1) as dt where 1 != 1", + "Query": "select dt.TABLE_NAME from (select TABLE_NAME from information_schema.`tables` as t where t.TABLE_SCHEMA = :__vtschemaname /* VARCHAR */ union select TABLE_NAME from information_schema.`columns`) as dt", + "SysTableTableSchema": "[VARCHAR(\"a\")]", + "Table": "information_schema.`tables`" + } + } }, { "comment": "merge even one side have schema name in subquery", From 2e31f832e70d1b32e3d86706cdda78676edef26c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 22 Jun 2023 15:43:30 +0200 Subject: [PATCH 10/26] use semantics.RewriteDerivedTableExpression instead of manual rewriting of ColNames Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 43 ++++------ .../planbuilder/operators/projection.go | 8 ++ .../planbuilder/testdata/aggr_cases.json | 78 +++++++------------ .../planbuilder/testdata/select_cases.json | 16 ++-- 4 files changed, 61 insertions(+), 84 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index c970cf421f8..f9b9b845c31 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -61,17 +61,12 @@ type ( ) func (a *Aggregator) Clone(inputs []ops.Operator) ops.Operator { - return &Aggregator{ - Source: inputs[0], - Columns: slices.Clone(a.Columns), - Grouping: slices.Clone(a.Grouping), - Aggregations: slices.Clone(a.Aggregations), - Pushed: a.Pushed, - offsetPlanned: a.offsetPlanned, - Original: a.Original, - ResultColumns: a.ResultColumns, - QP: a.QP, - } + kopy := *a + kopy.Source = inputs[0] + kopy.Columns = slices.Clone(a.Columns) + kopy.Grouping = slices.Clone(a.Grouping) + kopy.Aggregations = slices.Clone(a.Aggregations) + return &kopy } func (a *Aggregator) Inputs() []ops.Operator { @@ -122,21 +117,13 @@ func (a *Aggregator) isDerived() bool { func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { if a.TableID != nil { - // this is a derived table, and so we have to translate the column names to the internal expression - col, ok := expr.Expr.(*sqlparser.ColName) - if !ok { - panic(4) - // return nil, 0, vterrors.VT13001("don't know how to handle this") - } - for i, column := range a.Columns { - if col.Name.EqualString(column.ColumnName()) { - return a, i, nil - } + derivedTBL, err := ctx.SemTable.TableInfoFor(*a.TableID) + if err != nil { + return nil, 0, err } + expr.Expr = semantics.RewriteDerivedTableExpression(expr.Expr, derivedTBL) } - if addToGroupBy { - return nil, 0, vterrors.VT13001("did not expect to add group by here") - } + // Aggregator is little special and cannot work if the input offset are not matched with the aggregation columns. // So, before pushing anything from above the aggregator offset planning needs to be completed. err := a.planOffsets(ctx) @@ -154,6 +141,10 @@ func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser } } + if addToGroupBy { + return nil, 0, vterrors.VT13001("did not expect to add group by here") + } + // If weight string function is received from above operator. Then check if we have a group on the expression used. // If it is found, then continue to push it down but with addToGroupBy true so that is the added to group by sql down in the AddColumn. // This also set the weight string column offset so that we would not need to add it later in aggregator operator planOffset. @@ -420,7 +411,3 @@ func (a *Aggregator) internalAddColumn(ctx *plancontext.PlanningContext, aliased } var _ ops.Operator = (*Aggregator)(nil) - -// func (a *Aggregator) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { -// -// } diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index 78c8d74bc1d..f3dd3483207 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -120,6 +120,14 @@ func (p *Projection) isDerived() bool { } func (p *Projection) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { + if p.TableID != nil { + derivedTBL, err := ctx.SemTable.TableInfoFor(*p.TableID) + if err != nil { + return nil, 0, err + } + expr.Expr = semantics.RewriteDerivedTableExpression(expr.Expr, derivedTBL) + } + if offset, found := canReuseColumn(ctx, p.Columns, expr.Expr, extractExpr); found { return p, offset, nil } diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index a5eaadd6c9a..b8940a663bd 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -5300,7 +5300,7 @@ "Instructions": { "OperatorType": "SimpleProjection", "Columns": [ - 1 + 0 ], "Inputs": [ { @@ -5308,31 +5308,22 @@ "Predicate": "bazo between 100 and 200", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": [ - 1, - 0 - ], + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "max(1) AS bazo", + "GroupBy": "(0|3), (2|4)", "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Ordered", - "Aggregates": "max(1) AS bazo", - "GroupBy": "(0|2)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", - "OrderBy": "(0|2) ASC", - "Query": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select foo, max(baz) as bazo, bazo, weight_string(foo), weight_string(bazo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, bazo", + "OrderBy": "(0|3) ASC, (2|4) ASC", + "Query": "select foo, max(baz) as bazo, bazo, weight_string(foo), weight_string(bazo) from (select foo, baz from `user`) as f group by foo, bazo order by foo asc, bazo asc", + "Table": "`user`" } ] } @@ -5355,7 +5346,7 @@ "Instructions": { "OperatorType": "SimpleProjection", "Columns": [ - 1 + 0 ], "Inputs": [ { @@ -5363,31 +5354,22 @@ "Predicate": "bazo between 100 and 200", "Inputs": [ { - "OperatorType": "SimpleProjection", - "Columns": [ - 1, - 0 - ], + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum_count(1) AS bazo", + "GroupBy": "(0|3), 2", "Inputs": [ { - "OperatorType": "Aggregate", - "Variant": "Ordered", - "Aggregates": "sum_count(1) AS bazo", - "GroupBy": "(0|2)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select foo, count(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", - "OrderBy": "(0|2) ASC", - "Query": "select foo, count(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select foo, count(baz) as bazo, bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, bazo", + "OrderBy": "(0|3) ASC, 2 ASC", + "Query": "select foo, count(baz) as bazo, bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, bazo order by foo asc, bazo asc", + "Table": "`user`" } ] } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index e23bea5e0d4..c760156ae05 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -7620,24 +7620,24 @@ "Instructions": { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:1", + "JoinColumnIndexes": "R:0,L:0", "JoinVars": { - "t_id": 0 + "t_id": 1 }, "TableName": "user_extra_`user`", "Inputs": [ { "OperatorType": "SimpleProjection", "Columns": [ - 0, - 1 + 1, + 0 ], "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count_star(1) AS b", - "GroupBy": "(0|3), (2|4)", + "GroupBy": "(2|3), (0|4)", "Inputs": [ { "OperatorType": "Route", @@ -7646,9 +7646,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id, count(*) as b, req, weight_string(id), weight_string(req) from user_extra where 1 != 1 group by id, weight_string(id), req, weight_string(req)", - "OrderBy": "(0|3) ASC, (2|4) ASC", - "Query": "select id, count(*) as b, req, weight_string(id), weight_string(req) from user_extra group by id, weight_string(id), req, weight_string(req) order by id asc, req asc", + "FieldQuery": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra where 1 != 1 group by req, id", + "OrderBy": "(2|3) ASC, (0|4) ASC", + "Query": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra group by req, id order by req asc, id asc", "Table": "user_extra" } ] From 9a11f4409929044a28a3b8d0016b353acafc32a9 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 27 Jun 2023 11:13:40 +0200 Subject: [PATCH 11/26] work on making derived tables with aggregation work Signed-off-by: Andres Taylor --- .../operators/aggregation_pushing.go | 4 +- .../planbuilder/operators/aggregator.go | 46 +++++++++++++++-- .../planbuilder/operators/offset_planning.go | 50 +++++++++++++------ .../planbuilder/operators/projection.go | 21 ++++++-- go/vt/vtgate/planbuilder/operators/route.go | 1 + .../planbuilder/testdata/aggr_cases.json | 16 +++--- .../planbuilder/testdata/select_cases.json | 4 +- 7 files changed, 105 insertions(+), 37 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index 38a572cc14e..6d20cf211a5 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -99,9 +99,7 @@ func pushDownAggregationThroughRoute( } // Create a new aggregator to be placed below the route. - aggrBelowRoute := aggregator.Clone([]ops.Operator{route.Source}).(*Aggregator) - aggrBelowRoute.Pushed = false - aggrBelowRoute.Original = false + aggrBelowRoute := aggregator.SplitAggregatorBelowRoute(route.Inputs()) aggrBelowRoute.Aggregations = nil for i, aggregation := range aggregator.Aggregations { diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index f9b9b845c31..003ea7388eb 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -115,8 +115,29 @@ func (a *Aggregator) isDerived() bool { return a.TableID != nil } +func (a *Aggregator) findCol(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (int, error) { + if a.isDerived() { + derivedTBL, err := ctx.SemTable.TableInfoFor(*a.TableID) + if err != nil { + return 0, err + } + expr = semantics.RewriteDerivedTableExpression(expr, derivedTBL) + } + if offset, found := canReuseColumn(ctx, a.Columns, expr, extractExpr); found { + return offset, nil + } + return -1, nil +} + func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { - if a.TableID != nil { + offset, err := a.findCol(ctx, expr.Expr) + if err != nil { + return nil, 0, err + } + if offset >= 0 { + return a, offset, nil + } + if a.isDerived() { derivedTBL, err := ctx.SemTable.TableInfoFor(*a.TableID) if err != nil { return nil, 0, err @@ -126,7 +147,7 @@ func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser // Aggregator is little special and cannot work if the input offset are not matched with the aggregation columns. // So, before pushing anything from above the aggregator offset planning needs to be completed. - err := a.planOffsets(ctx) + err = a.planOffsets(ctx) if err != nil { return nil, 0, err } @@ -207,9 +228,12 @@ func (a *Aggregator) GetSelectExprs() (sqlparser.SelectExprs, error) { } func (a *Aggregator) ShortDescription() string { - columnns := slices2.Map(a.Columns, func(from *sqlparser.AliasedExpr) string { + columns := slices2.Map(a.Columns, func(from *sqlparser.AliasedExpr) string { return sqlparser.String(from) }) + if a.Alias != "" { + columns = append([]string{"derived[" + a.Alias + "]"}, columns...) + } org := "" if a.Original { @@ -217,7 +241,7 @@ func (a *Aggregator) ShortDescription() string { } if len(a.Grouping) == 0 { - return fmt.Sprintf("%s%s", org, strings.Join(columnns, ", ")) + return fmt.Sprintf("%s%s", org, strings.Join(columns, ", ")) } var grouping []string @@ -225,7 +249,7 @@ func (a *Aggregator) ShortDescription() string { grouping = append(grouping, sqlparser.String(gb.SimplifiedExpr)) } - return fmt.Sprintf("%s%s group by %s", org, strings.Join(columnns, ", "), strings.Join(grouping, ",")) + return fmt.Sprintf("%s%s group by %s", org, strings.Join(columns, ", "), strings.Join(grouping, ",")) } func (a *Aggregator) GetOrdering() ([]ops.OrderBy, error) { @@ -410,4 +434,16 @@ func (a *Aggregator) internalAddColumn(ctx *plancontext.PlanningContext, aliased return offset, nil } +// SplitAggregatorBelowRoute returns the aggregator that will live under the Route. +// This is used when we are splitting the aggregation so one part is done +// at the mysql level and one part at the vtgate level +func (a *Aggregator) SplitAggregatorBelowRoute(input []ops.Operator) *Aggregator { + newOp := a.Clone(input).(*Aggregator) + newOp.Pushed = false + newOp.Original = false + newOp.Alias = "" + newOp.TableID = nil + return newOp +} + var _ ops.Operator = (*Aggregator)(nil) diff --git a/go/vt/vtgate/planbuilder/operators/offset_planning.go b/go/vt/vtgate/planbuilder/operators/offset_planning.go index 31c36d609fb..37232f07952 100644 --- a/go/vt/vtgate/planbuilder/operators/offset_planning.go +++ b/go/vt/vtgate/planbuilder/operators/offset_planning.go @@ -110,7 +110,7 @@ func useOffsets(ctx *plancontext.PlanningContext, expr sqlparser.Expr, op ops.Op } getColumns := func() []*sqlparser.AliasedExpr { return columns } - visitor := getVisitor(ctx, getColumns, found, notFound) + visitor := getVisitor2(ctx, getColumns, found, notFound) // The cursor replace is not available while walking `down`, so `up` is used to do the replacement. up := func(cursor *sqlparser.CopyOnWriteCursor) { @@ -137,10 +137,6 @@ func addColumnsToInput(ctx *plancontext.PlanningContext, root ops.Operator) (ops return in, rewrite.SameTree, nil } - columns, err := filter.GetColumns() - if err != nil { - return nil, nil, err - } proj, areOnTopOfProj := filter.Source.(selectExpressions) if !areOnTopOfProj { // not much we can do here @@ -152,19 +148,12 @@ func addColumnsToInput(ctx *plancontext.PlanningContext, root ops.Operator) (ops _, addToGroupBy := e.(*sqlparser.ColName) proj.addColumnWithoutPushing(aeWrap(e), addToGroupBy) addedColumns = true - columns, err = proj.GetColumns() return nil } - getColumns := func() []*sqlparser.AliasedExpr { - return columns - } - visitor := getVisitor(ctx, getColumns, found, notFound) + visitor := getVisitor(ctx, proj, found, notFound) for _, expr := range filter.Predicates { - sqlparser.CopyOnRewrite(expr, visitor, nil, ctx.SemTable.CopyDependenciesOnSQLNodes) - if err != nil { - return nil, nil, err - } + _ = sqlparser.CopyOnRewrite(expr, visitor, nil, ctx.SemTable.CopyDependenciesOnSQLNodes) } if addedColumns { return in, rewrite.NewTree("added columns because filter needs it", in), nil @@ -177,6 +166,39 @@ func addColumnsToInput(ctx *plancontext.PlanningContext, root ops.Operator) (ops } func getVisitor( + ctx *plancontext.PlanningContext, + proj selectExpressions, + found func(sqlparser.Expr, int), + notFound func(sqlparser.Expr) error, +) func(node, parent sqlparser.SQLNode) bool { + var err error + return func(node, parent sqlparser.SQLNode) bool { + if err != nil { + return false + } + e, ok := node.(sqlparser.Expr) + if !ok { + return true + } + var offset int + offset, err = proj.findCol(ctx, e) + if err != nil { + return false + } + if offset >= 0 { + found(e, offset) + return false + } + + if fetchByOffset(e) { + err = notFound(e) + return false + } + + return true + } +} +func getVisitor2( ctx *plancontext.PlanningContext, getColumns func() []*sqlparser.AliasedExpr, found func(sqlparser.Expr, int), diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index f3dd3483207..37b6cc0c7f4 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -119,18 +119,29 @@ func (p *Projection) isDerived() bool { return p.TableID != nil } -func (p *Projection) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { - if p.TableID != nil { +func (p *Projection) findCol(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (int, error) { + if p.isDerived() { derivedTBL, err := ctx.SemTable.TableInfoFor(*p.TableID) if err != nil { - return nil, 0, err + return 0, err } - expr.Expr = semantics.RewriteDerivedTableExpression(expr.Expr, derivedTBL) + expr = semantics.RewriteDerivedTableExpression(expr, derivedTBL) + } + if offset, found := canReuseColumn(ctx, p.Columns, expr, extractExpr); found { + return offset, nil } + return -1, nil +} - if offset, found := canReuseColumn(ctx, p.Columns, expr.Expr, extractExpr); found { +func (p *Projection) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { + offset, err := p.findCol(ctx, expr.Expr) + if err != nil { + return nil, 0, err + } + if offset >= 0 { return p, offset, nil } + sourceOp, offset, err := p.Source.AddColumn(ctx, expr, true, addToGroupBy) if err != nil { return nil, 0, err diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index d98d26f65f6..ae2e2e432e1 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -582,6 +582,7 @@ type selectExpressions interface { ops.Operator addColumnWithoutPushing(expr *sqlparser.AliasedExpr, addToGroupBy bool) int isDerived() bool + findCol(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (int, error) } func addColumnToInput(operator ops.Operator, expr *sqlparser.AliasedExpr, addToGroupBy bool) (bool, int) { diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index b8940a663bd..6f93a3ab2a1 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -5311,7 +5311,7 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "max(1) AS bazo", - "GroupBy": "(0|3), (2|4)", + "GroupBy": "(0|2)", "Inputs": [ { "OperatorType": "Route", @@ -5320,9 +5320,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select foo, max(baz) as bazo, bazo, weight_string(foo), weight_string(bazo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, bazo", - "OrderBy": "(0|3) ASC, (2|4) ASC", - "Query": "select foo, max(baz) as bazo, bazo, weight_string(foo), weight_string(bazo) from (select foo, baz from `user`) as f group by foo, bazo order by foo asc, bazo asc", + "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", + "OrderBy": "(0|2) ASC", + "Query": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", "Table": "`user`" } ] @@ -5357,7 +5357,7 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum_count(1) AS bazo", - "GroupBy": "(0|3), 2", + "GroupBy": "(0|2)", "Inputs": [ { "OperatorType": "Route", @@ -5366,9 +5366,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select foo, count(baz) as bazo, bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, bazo", - "OrderBy": "(0|3) ASC, 2 ASC", - "Query": "select foo, count(baz) as bazo, bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, bazo order by foo asc, bazo asc", + "FieldQuery": "select foo, count(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", + "OrderBy": "(0|2) ASC", + "Query": "select foo, count(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index c760156ae05..cf730f526ad 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -7646,9 +7646,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra where 1 != 1 group by req, id", + "FieldQuery": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra where 1 != 1 group by req, id, weight_string(req), weight_string(id)", "OrderBy": "(2|3) ASC, (0|4) ASC", - "Query": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra group by req, id order by req asc, id asc", + "Query": "select id, count(*) as b, req, weight_string(req), weight_string(id) from user_extra group by req, id, weight_string(req), weight_string(id) order by req asc, id asc", "Table": "user_extra" } ] From 7796a6ed1b922198bfa61af343ab5f19280f0c6e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 27 Jun 2023 11:17:37 +0200 Subject: [PATCH 12/26] refactor code Signed-off-by: Andres Taylor --- .../planbuilder/operators/offset_planning.go | 45 ++++--------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/offset_planning.go b/go/vt/vtgate/planbuilder/operators/offset_planning.go index 37232f07952..42ec0fe606c 100644 --- a/go/vt/vtgate/planbuilder/operators/offset_planning.go +++ b/go/vt/vtgate/planbuilder/operators/offset_planning.go @@ -110,7 +110,12 @@ func useOffsets(ctx *plancontext.PlanningContext, expr sqlparser.Expr, op ops.Op } getColumns := func() []*sqlparser.AliasedExpr { return columns } - visitor := getVisitor2(ctx, getColumns, found, notFound) + findCol := func(ctx *plancontext.PlanningContext, e sqlparser.Expr) (int, error) { + return slices.IndexFunc(getColumns(), func(expr *sqlparser.AliasedExpr) bool { + return ctx.SemTable.EqualsExprWithDeps(expr.Expr, e) + }), nil + } + visitor := getVisitor(ctx, findCol, found, notFound) // The cursor replace is not available while walking `down`, so `up` is used to do the replacement. up := func(cursor *sqlparser.CopyOnWriteCursor) { @@ -150,7 +155,7 @@ func addColumnsToInput(ctx *plancontext.PlanningContext, root ops.Operator) (ops addedColumns = true return nil } - visitor := getVisitor(ctx, proj, found, notFound) + visitor := getVisitor(ctx, proj.findCol, found, notFound) for _, expr := range filter.Predicates { _ = sqlparser.CopyOnRewrite(expr, visitor, nil, ctx.SemTable.CopyDependenciesOnSQLNodes) @@ -167,7 +172,7 @@ func addColumnsToInput(ctx *plancontext.PlanningContext, root ops.Operator) (ops func getVisitor( ctx *plancontext.PlanningContext, - proj selectExpressions, + findCol func(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (int, error), found func(sqlparser.Expr, int), notFound func(sqlparser.Expr) error, ) func(node, parent sqlparser.SQLNode) bool { @@ -181,42 +186,10 @@ func getVisitor( return true } var offset int - offset, err = proj.findCol(ctx, e) - if err != nil { - return false - } - if offset >= 0 { - found(e, offset) - return false - } - - if fetchByOffset(e) { - err = notFound(e) - return false - } - - return true - } -} -func getVisitor2( - ctx *plancontext.PlanningContext, - getColumns func() []*sqlparser.AliasedExpr, - found func(sqlparser.Expr, int), - notFound func(sqlparser.Expr) error, -) func(node, parent sqlparser.SQLNode) bool { - var err error - return func(node, parent sqlparser.SQLNode) bool { + offset, err = findCol(ctx, e) if err != nil { return false } - e, ok := node.(sqlparser.Expr) - if !ok { - return true - } - offset := slices.IndexFunc(getColumns(), func(expr *sqlparser.AliasedExpr) bool { - return ctx.SemTable.EqualsExprWithDeps(expr.Expr, e) - }) - if offset >= 0 { found(e, offset) return false From b134c928fcd2991d7821afa896cbe1613c752c8b Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 27 Jun 2023 12:00:56 +0200 Subject: [PATCH 13/26] enable horizon planning in more situations Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/ast2op.go | 2 +- .../operators/horizon_expanding.go | 5 - .../planbuilder/operators/queryprojection.go | 13 -- .../planbuilder/testdata/aggr_cases.json | 151 +++++++----------- .../planbuilder/testdata/tpch_cases.json | 30 ++-- 5 files changed, 78 insertions(+), 123 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 6df7fb45063..58148357e59 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -629,7 +629,7 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr } stmt := sqlparser.CloneSelectStatement(tbl.Select) - if onlyTable { + if onlyTable && stmt.GetLimit() == nil { stmt.SetOrderBy(nil) } qp, err := CreateQPFromSelectStatement(ctx, stmt) diff --git a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go index 1ca64a1c081..a03c9a95904 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go @@ -95,11 +95,6 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizo return out, nil } - err = checkAggregationSupported(horizon) - if err != nil { - return nil, err - } - aggregations, complexAggr, err := qp.AggregationExpressions(ctx, true) if err != nil { return nil, err diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 14e66cbce3f..1e1a8ed28ce 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -29,7 +29,6 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine/opcode" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" - "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/rewrite" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" ) @@ -832,18 +831,6 @@ func (qp *QueryProjection) GetColumnCount() int { return len(qp.SelectExprs) - qp.AddedColumn } -// checkAggregationSupported checks if the aggregation is supported on the given operator tree or not. -// We don't currently support planning for operators having derived tables. -func checkAggregationSupported(op ops.Operator) error { - return rewrite.Visit(op, func(operator ops.Operator) error { - projection, isProjection := operator.(*Projection) - if isProjection && projection.TableID != nil { - return errHorizonNotPlanned() - } - return nil - }) -} - func checkForInvalidGroupingExpressions(expr sqlparser.Expr) error { return sqlparser.Walk(func(node sqlparser.SQLNode) (bool, error) { if _, isAggregate := node.(sqlparser.AggrFunc); isAggregate { diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 6f93a3ab2a1..04f7d7f8b9e 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -4005,13 +4005,13 @@ { "OperatorType": "Projection", "Expressions": [ - "[COLUMN 2] * COALESCE([COLUMN 3], INT64(1)) as sum(col)" + "[COLUMN 0] * [COLUMN 1] as sum(col)" ], "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,R:1", + "JoinColumnIndexes": "L:0,R:0", "TableName": "`user`_user_extra", "Inputs": [ { @@ -4021,8 +4021,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.col as col, 32, sum(col) from `user` where 1 != 1", - "Query": "select `user`.col as col, 32, sum(col) from `user`", + "FieldQuery": "select sum(col) from (select `user`.col as col, 32 from `user` where 1 != 1) as t where 1 != 1", + "Query": "select sum(col) from (select `user`.col as col, 32 from `user`) as t", "Table": "`user`" }, { @@ -4032,8 +4032,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select 1, count(*) from user_extra where 1 != 1 group by 1", - "Query": "select 1, count(*) from user_extra group by 1", + "FieldQuery": "select count(*) from user_extra where 1 != 1 group by .0", + "Query": "select count(*) from user_extra group by .0", "Table": "user_extra" } ] @@ -4728,27 +4728,19 @@ "Aggregates": "count(0) AS count(city)", "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "[COLUMN 2] as count(city)" - ], + "OperatorType": "Limit", + "Count": "INT64(10)", "Inputs": [ { - "OperatorType": "Limit", - "Count": "INT64(10)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select phone, id, city from `user` where 1 != 1", - "Query": "select phone, id, city from `user` where id > 12 limit :__upper_limit", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select city from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", + "Query": "select city from (select phone, id, city from `user` where id > 12) as x limit :__upper_limit", + "Table": "`user`" } ] } @@ -4772,27 +4764,19 @@ "Aggregates": "count_star(0) AS count(*)", "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "[COLUMN 0] as count(*)" - ], + "OperatorType": "Limit", + "Count": "INT64(10)", "Inputs": [ { - "OperatorType": "Limit", - "Count": "INT64(10)", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select phone, id, city from `user` where 1 != 1", - "Query": "select phone, id, city from `user` where id > 12 limit :__upper_limit", - "Table": "`user`" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from (select phone, id, city from `user` where 1 != 1) as x where 1 != 1", + "Query": "select 1 from (select phone, id, city from `user` where id > 12) as x limit :__upper_limit", + "Table": "`user`" } ] } @@ -4815,47 +4799,39 @@ "Aggregates": "count(0) AS count(col)", "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "[COLUMN 0] as count(col)" - ], + "OperatorType": "Limit", + "Count": "INT64(10)", "Inputs": [ { - "OperatorType": "Limit", - "Count": "INT64(10)", + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "R:0", + "JoinVars": { + "user_id": 0 + }, + "TableName": "`user`_user_extra", "Inputs": [ { - "OperatorType": "Join", - "Variant": "LeftJoin", - "JoinColumnIndexes": "R:0", - "JoinVars": { - "user_id": 0 + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true }, - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select `user`.id from `user` where 1 != 1", - "Query": "select `user`.id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select user_extra.col as col from user_extra where 1 != 1", - "Query": "select user_extra.col as col from user_extra where user_extra.id = :user_id", - "Table": "user_extra" - } - ] + "FieldQuery": "select `user`.id from `user` where 1 != 1", + "Query": "select `user`.id from `user`", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select user_extra.col from user_extra where 1 != 1", + "Query": "select user_extra.col from user_extra where user_extra.id = :user_id", + "Table": "user_extra" } ] } @@ -4884,12 +4860,9 @@ "ResultColumns": 2, "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "[COLUMN 1] as val1", - "[COLUMN 0] as count(*)", - "[COLUMN 2]" - ], + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(0|2) ASC", "Inputs": [ { "OperatorType": "Limit", @@ -4902,9 +4875,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id, val1, weight_string(val1) from `user` where 1 != 1", - "OrderBy": "(1|2) ASC", - "Query": "select id, val1, weight_string(val1) from `user` where val2 < 4 order by val1 asc limit :__upper_limit", + "FieldQuery": "select val1, 1, weight_string(val1), weight_string(val1) from (select id, val1 from `user` where 1 != 1) as x where 1 != 1", + "OrderBy": "(0|3) ASC", + "Query": "select val1, 1, weight_string(val1), weight_string(val1) from (select id, val1 from `user` where val2 < 4) as x order by val1 asc limit :__upper_limit", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 33152cfc5a5..3e97f2c01fa 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -586,18 +586,18 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,R:1,L:2,L:3,L:5,R:2,R:3,R:4,L:6,L:8,R:5,R:6,R:7,L:9,L:10,L:11,R:8,R:9,R:10,L:12", + "JoinColumnIndexes": "L:0,L:1,R:0,L:3,L:4,L:6,L:7,L:8,R:1,L:9,L:11,L:12,L:13,R:2,L:14,L:15,L:16,R:3,R:4,R:5,L:17", "JoinVars": { - "l_suppkey": 1 + "l_suppkey": 2 }, "TableName": "lineitem_orders_supplier_nation", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,L:0,L:2,L:3,L:0,R:0,L:2,L:6,R:2,L:7,L:4,R:1,L:8", + "JoinColumnIndexes": "L:0,L:1,L:2,L:4,L:5,L:2,L:0,L:1,L:1,L:4,L:8,L:9,L:10,L:10,L:11,L:6,R:1,L:12", "JoinVars": { - "l_orderkey": 1 + "l_orderkey": 3 }, "TableName": "lineitem_orders", "Inputs": [ @@ -608,8 +608,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select l_suppkey, l_orderkey, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(l_orderkey), weight_string(l_suppkey), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where 1 != 1 group by l_orderkey, weight_string(l_orderkey), l_suppkey, weight_string(l_suppkey), l_year, weight_string(l_year)", - "Query": "select l_suppkey, l_orderkey, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(l_orderkey), weight_string(l_suppkey), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31') group by l_orderkey, weight_string(l_orderkey), l_suppkey, weight_string(l_suppkey), l_year, weight_string(l_year)", + "FieldQuery": "select shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(shipping.`lineitem.l_orderkey`), weight_string(shipping.`lineitem.l_suppkey`), weight_string(shipping.`orders.o_custkey`), weight_string(shipping.`n1.n_name`), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where 1 != 1 group by shipping.`lineitem.l_orderkey`, weight_string(shipping.`lineitem.l_orderkey`), shipping.`lineitem.l_suppkey`, weight_string(shipping.`lineitem.l_suppkey`), shipping.`orders.o_custkey`, weight_string(shipping.`orders.o_custkey`), shipping.`n1.n_name`, weight_string(shipping.`n1.n_name`), l_year, weight_string(l_year)", + "Query": "select shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(shipping.`lineitem.l_orderkey`), weight_string(shipping.`lineitem.l_suppkey`), weight_string(shipping.`orders.o_custkey`), weight_string(shipping.`n1.n_name`), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31') group by shipping.`lineitem.l_orderkey`, weight_string(shipping.`lineitem.l_orderkey`), shipping.`lineitem.l_suppkey`, weight_string(shipping.`lineitem.l_suppkey`), shipping.`orders.o_custkey`, weight_string(shipping.`orders.o_custkey`), shipping.`n1.n_name`, weight_string(shipping.`n1.n_name`), l_year, weight_string(l_year)", "Table": "lineitem" }, { @@ -619,8 +619,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select o_custkey, count(*), weight_string(o_custkey) from orders where 1 != 1 group by o_custkey, weight_string(o_custkey)", - "Query": "select o_custkey, count(*), weight_string(o_custkey) from orders where o_orderkey = :l_orderkey group by o_custkey, weight_string(o_custkey)", + "FieldQuery": "select 1, count(*) from orders where 1 != 1 group by 1", + "Query": "select 1, count(*) from orders where o_orderkey = :l_orderkey group by 1", "Table": "orders", "Values": [ ":l_orderkey" @@ -632,7 +632,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,R:1,R:0,R:0,R:1,R:3,R:3,R:4,L:1,R:2,R:5", + "JoinColumnIndexes": "R:0,R:0,R:2,L:1,R:1,R:3", "JoinVars": { "s_nationkey": 0 }, @@ -645,8 +645,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select s_nationkey, count(*), weight_string(s_nationkey) from supplier where 1 != 1 group by s_nationkey, weight_string(s_nationkey)", - "Query": "select s_nationkey, count(*), weight_string(s_nationkey) from supplier where s_suppkey = :l_suppkey group by s_nationkey, weight_string(s_nationkey)", + "FieldQuery": "select shipping.`supplier.s_nationkey`, count(*), weight_string(shipping.`supplier.s_nationkey`) from supplier where 1 != 1 group by shipping.`supplier.s_nationkey`, weight_string(shipping.`supplier.s_nationkey`)", + "Query": "select shipping.`supplier.s_nationkey`, count(*), weight_string(shipping.`supplier.s_nationkey`) from supplier where s_suppkey = :l_suppkey group by shipping.`supplier.s_nationkey`, weight_string(shipping.`supplier.s_nationkey`)", "Table": "supplier", "Values": [ ":l_suppkey" @@ -660,8 +660,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select n1.n_name, n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where 1 != 1 group by n1.n_name, weight_string(n1.n_name), supp_nation, weight_string(supp_nation)", - "Query": "select n1.n_name, n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where n1.n_nationkey = :s_nationkey group by n1.n_name, weight_string(n1.n_name), supp_nation, weight_string(supp_nation)", + "FieldQuery": "select n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where 1 != 1 group by supp_nation, weight_string(supp_nation)", + "Query": "select n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where n1.n_nationkey = :s_nationkey group by supp_nation, weight_string(supp_nation)", "Table": "nation", "Values": [ ":s_nationkey" @@ -688,8 +688,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select c_nationkey, count(*), weight_string(c_nationkey) from customer where 1 != 1 group by c_nationkey, weight_string(c_nationkey)", - "Query": "select c_nationkey, count(*), weight_string(c_nationkey) from customer where c_custkey = :o_custkey group by c_nationkey, weight_string(c_nationkey)", + "FieldQuery": "select shipping.`customer.c_nationkey`, count(*), weight_string(shipping.`customer.c_nationkey`) from customer where 1 != 1 group by shipping.`customer.c_nationkey`, weight_string(shipping.`customer.c_nationkey`)", + "Query": "select shipping.`customer.c_nationkey`, count(*), weight_string(shipping.`customer.c_nationkey`) from customer where c_custkey = :o_custkey group by shipping.`customer.c_nationkey`, weight_string(shipping.`customer.c_nationkey`)", "Table": "customer", "Values": [ ":o_custkey" From 3da1903bb550b1b88b1093319c817130b3c61d90 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 29 Jun 2023 21:04:08 +0200 Subject: [PATCH 14/26] update test expectations Signed-off-by: Andres Taylor --- go/vt/vtgate/executor_select_test.go | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index dcaf55c6aa0..ed5e72fd04c 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -3688,15 +3688,15 @@ func TestSelectAggregationNoData(t *testing.T) { }, { sql: `select count(*) from (select col1, col2 from user limit 2) x`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2", "int64|int64")), - expSandboxQ: "select col1, col2 from `user` limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "int64")), + expSandboxQ: "select 1 from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"count(*)" type:INT64]`, expRow: `[[INT64(0)]]`, }, { sql: `select col2, count(*) from (select col1, col2 from user limit 2) x group by col2`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col2)", "int64|int64|varbinary")), - expSandboxQ: "select col1, col2, weight_string(col2) from `user` order by col2 asc limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col2|1|weight_string(col2)", "int64|int64|varbinary")), + expSandboxQ: "select col2, 1, weight_string(col2) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col2" type:INT64 name:"count(*)" type:INT64]`, expRow: `[]`, }, @@ -3772,71 +3772,71 @@ func TestSelectAggregationData(t *testing.T) { }, { sql: `select count(*) from (select col1, col2 from user limit 2) x`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2", "int64|int64"), "1|2", "2|1"), - expSandboxQ: "select col1, col2 from `user` limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("1", "int64"), "1", "1"), + expSandboxQ: "select 1 from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"count(*)" type:INT64]`, expRow: `[[INT64(2)]]`, }, { sql: `select col2, count(*) from (select col1, col2 from user limit 9) x group by col2`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col2)", "int64|int64|varbinary"), "3|1|NULL", "2|2|NULL"), - expSandboxQ: "select col1, col2, weight_string(col2) from `user` order by col2 asc limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col2|1|weight_string(col2)", "int64|int64|varbinary"), "3|1|NULL", "2|1|NULL"), + expSandboxQ: "select col2, 1, weight_string(col2) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col2" type:INT64 name:"count(*)" type:INT64]`, - expRow: `[[INT64(1) INT64(8)] [INT64(2) INT64(1)]]`, + expRow: `[[INT64(2) INT64(4)] [INT64(3) INT64(5)]]`, }, { sql: `select count(col1) from (select id, col1 from user limit 2) x`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("id|col1", "int64|varchar"), "3|a", "2|b"), - expSandboxQ: "select id, col1 from `user` limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar"), "a", "b"), + expSandboxQ: "select col1 from (select id, col1 from `user`) as x limit :__upper_limit", expField: `[name:"count(col1)" type:INT64]`, expRow: `[[INT64(2)]]`, }, { sql: `select count(col1), col2 from (select col2, col1 from user limit 9) x group by col2`, - sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col2|col1|weight_string(col2)", "int64|varchar|varbinary"), "3|a|NULL", "2|b|NULL"), - expSandboxQ: "select col2, col1, weight_string(col2) from `user` order by col2 asc limit :__upper_limit", + sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col2)", "varchar|int64|varbinary"), "a|3|NULL", "b|2|NULL"), + expSandboxQ: "select col1, col2, weight_string(col2) from (select col2, col1 from `user`) as x limit :__upper_limit", expField: `[name:"count(col1)" type:INT64 name:"col2" type:INT64]`, - expRow: `[[INT64(8) INT64(2)] [INT64(1) INT64(3)]]`, + expRow: `[[INT64(4) INT64(2)] [INT64(5) INT64(3)]]`, }, { sql: `select col1, count(col2) from (select col1, col2 from user limit 9) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|int64|varbinary"), "a|1|a", "b|null|b"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"count(col2)" type:INT64]`, - expRow: `[[VARCHAR("a") INT64(8)] [VARCHAR("b") INT64(0)]]`, + expRow: `[[VARCHAR("a") INT64(5)] [VARCHAR("b") INT64(0)]]`, }, { sql: `select col1, count(col2) from (select col1, col2 from user limit 32) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|int64|varbinary"), "null|1|null", "null|null|null", "a|1|a", "b|null|b"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"count(col2)" type:INT64]`, expRow: `[[NULL INT64(8)] [VARCHAR("a") INT64(8)] [VARCHAR("b") INT64(0)]]`, }, { sql: `select col1, sum(col2) from (select col1, col2 from user limit 4) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|int64|varbinary"), "a|3|a"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"sum(col2)" type:DECIMAL]`, expRow: `[[VARCHAR("a") DECIMAL(12)]]`, }, { sql: `select col1, sum(col2) from (select col1, col2 from user limit 4) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|varchar|varbinary"), "a|2|a"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"sum(col2)" type:FLOAT64]`, expRow: `[[VARCHAR("a") FLOAT64(8)]]`, }, { sql: `select col1, sum(col2) from (select col1, col2 from user limit 4) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|varchar|varbinary"), "a|x|a"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"sum(col2)" type:FLOAT64]`, expRow: `[[VARCHAR("a") FLOAT64(0)]]`, }, { sql: `select col1, sum(col2) from (select col1, col2 from user limit 4) x group by col1`, sandboxRes: sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1|col2|weight_string(col1)", "varchar|varchar|varbinary"), "a|null|a"), - expSandboxQ: "select col1, col2, weight_string(col1) from `user` order by col1 asc limit :__upper_limit", + expSandboxQ: "select col1, col2, weight_string(col1) from (select col1, col2 from `user`) as x limit :__upper_limit", expField: `[name:"col1" type:VARCHAR name:"sum(col2)" type:FLOAT64]`, expRow: `[[VARCHAR("a") NULL]]`, }, From c9fcec9f5badebb76df39f224c862631b7b4dec4 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 29 Jun 2023 21:43:02 +0200 Subject: [PATCH 15/26] make sure to always use a method to create aggregate params Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/aggregations.go | 17 +- go/vt/vtgate/engine/ordered_aggregate_test.go | 199 +++++------------- go/vt/vtgate/planbuilder/horizon_planning.go | 29 +-- .../planbuilder/operator_transformers.go | 17 +- go/vt/vtgate/planbuilder/ordered_aggregate.go | 19 +- .../vtgate/planbuilder/projection_pushing.go | 12 +- go/vt/vtgate/planbuilder/show.go | 7 +- .../tabletmanager/vdiff/table_plan.go | 10 +- .../vdiff/workflow_differ_test.go | 13 +- go/vt/wrangler/vdiff.go | 10 +- go/vt/wrangler/vdiff_test.go | 13 +- 11 files changed, 109 insertions(+), 237 deletions(-) diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index 73ba3ec58e1..32c481feed5 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -52,6 +52,19 @@ type AggregateParams struct { OrigOpcode AggregateOpcode } +func NewAggregateParam(opcode AggregateOpcode, col int, alias string) *AggregateParams { + return &AggregateParams{ + Opcode: opcode, + Col: col, + Alias: alias, + WCol: -1, + } +} + +func (ap *AggregateParams) WAssigned() bool { + return ap.WCol >= 0 +} + func (ap *AggregateParams) isDistinct() bool { return ap.Opcode == AggregateCountDistinct || ap.Opcode == AggregateSumDistinct } @@ -67,7 +80,7 @@ func (ap *AggregateParams) preProcess() bool { func (ap *AggregateParams) String() string { keyCol := strconv.Itoa(ap.Col) - if ap.WCol >= 0 { + if ap.WAssigned() { keyCol = fmt.Sprintf("%s|%d", keyCol, ap.WCol) } if ap.CollationID != collations.Unknown { @@ -279,7 +292,7 @@ func convertFields(fields []*querypb.Field, aggrs []*AggregateParams) []*querypb func findComparableCurrentDistinct(row []sqltypes.Value, aggr *AggregateParams) sqltypes.Value { curDistinct := row[aggr.KeyCol] - if aggr.WCol >= 0 && !curDistinct.IsComparable() { + if aggr.WAssigned() && !curDistinct.IsComparable() { aggr.KeyCol = aggr.WCol curDistinct = row[aggr.KeyCol] } diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index 599290041e8..c4edac39171 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -62,10 +62,7 @@ func TestOrderedAggregateExecute(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -98,12 +95,11 @@ func TestOrderedAggregateExecuteTruncate(t *testing.T) { )}, } + aggr := NewAggregateParam(AggregateSum, 1, "") + aggr.OrigOpcode = AggregateCountStar + oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - OrigOpcode: AggregateCountStar, - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{aggr}, GroupByKeys: []*GroupByParams{{KeyCol: 2}}, TruncateColumnCount: 2, Input: fp, @@ -142,10 +138,7 @@ func TestOrderedAggregateStreamExecute(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -185,10 +178,7 @@ func TestOrderedAggregateStreamExecuteTruncate(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 2}}, TruncateColumnCount: 2, Input: fp, @@ -232,32 +222,6 @@ func TestOrderedAggregateGetFields(t *testing.T) { assert.Equal(got, input) } -func TestOrderedAggregateGetFieldsTruncate(t *testing.T) { - assert := assert.New(t) - result := sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "col|count(*)|weight_string(col)", - "varchar|decimal|varbinary", - ), - ) - fp := &fakePrimitive{results: []*sqltypes.Result{result}} - - oa := &OrderedAggregate{ - TruncateColumnCount: 2, - Input: fp, - } - - got, err := oa.GetFields(context.Background(), nil, nil) - assert.NoError(err) - wantResult := sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "col|count(*)", - "varchar|decimal", - ), - ) - utils.MustMatch(t, wantResult, got) -} - func TestOrderedAggregateInputFail(t *testing.T) { fp := &fakePrimitive{sendErr: errors.New("input fail")} @@ -319,17 +283,11 @@ func TestOrderedAggregateExecuteCountDistinct(t *testing.T) { )}, } + aggr1 := NewAggregateParam(AggregateCountDistinct, 1, "count(distinct col2)") + aggr2 := NewAggregateParam(AggregateSum, 2, "") + aggr2.OrigOpcode = AggregateCountStar oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - Alias: "count(distinct col2)", - }, { - // Also add a count(*) - OrigOpcode: AggregateCountStar, - Opcode: AggregateSum, - Col: 2, - }}, + Aggregates: []*AggregateParams{aggr1, aggr2}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -395,16 +353,13 @@ func TestOrderedAggregateStreamCountDistinct(t *testing.T) { )}, } + aggr2 := NewAggregateParam(AggregateSum, 2, "") + aggr2.OrigOpcode = AggregateCountDistinct + oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - Alias: "count(distinct col2)", - }, { - Opcode: AggregateSum, - OrigOpcode: AggregateCountStar, - Col: 2, - }}, + Aggregates: []*AggregateParams{ + NewAggregateParam(AggregateCountDistinct, 1, "count(distinct col2)"), + aggr2}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -484,14 +439,10 @@ func TestOrderedAggregateSumDistinctGood(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSumDistinct, - Col: 1, - Alias: "sum(distinct col2)", - }, { - Opcode: AggregateSum, - Col: 2, - }}, + Aggregates: []*AggregateParams{ + NewAggregateParam(AggregateSumDistinct, 1, "sum(distinct col2)"), + NewAggregateParam(AggregateSum, 2, ""), + }, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -533,11 +484,7 @@ func TestOrderedAggregateSumDistinctTolerateError(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSumDistinct, - Col: 1, - Alias: "sum(distinct col2)", - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSumDistinct, 1, "sum(distinct col2)")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -569,10 +516,7 @@ func TestOrderedAggregateKeysFail(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -602,10 +546,7 @@ func TestOrderedAggregateMergeFail(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -642,20 +583,12 @@ func TestOrderedAggregateMergeFail(t *testing.T) { func TestMerge(t *testing.T) { assert := assert.New(t) oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }, { - Opcode: AggregateSum, - Col: 2, - }, { - Opcode: AggregateMin, - Col: 3, - }, { - Opcode: AggregateMax, - Col: 4, - }}, - } + Aggregates: []*AggregateParams{ + NewAggregateParam(AggregateSum, 1, ""), + NewAggregateParam(AggregateSum, 2, ""), + NewAggregateParam(AggregateMin, 3, ""), + NewAggregateParam(AggregateMax, 4, ""), + }} fields := sqltypes.MakeTestFields( "a|b|c|d|e", "int64|int64|decimal|in32|varbinary", @@ -703,11 +636,7 @@ func TestOrderedAggregateExecuteGtid(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateGtid, - Col: 1, - Alias: "vgtid", - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateGtid, 1, "vgtid")}, TruncateColumnCount: 2, Input: fp, } @@ -740,13 +669,10 @@ func TestCountDistinctOnVarchar(t *testing.T) { )}, } + aggr := NewAggregateParam(AggregateCountDistinct, 1, "count(distinct c2)") + aggr.WCol = 2 oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - WCol: 2, - Alias: "count(distinct c2)", - }}, + Aggregates: []*AggregateParams{aggr}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, TruncateColumnCount: 2, @@ -803,13 +729,10 @@ func TestCountDistinctOnVarcharWithNulls(t *testing.T) { )}, } + aggr := NewAggregateParam(AggregateCountDistinct, 1, "count(distinct c2)") + aggr.WCol = 2 oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - WCol: 2, - Alias: "count(distinct c2)", - }}, + Aggregates: []*AggregateParams{aggr}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, TruncateColumnCount: 2, @@ -868,13 +791,10 @@ func TestSumDistinctOnVarcharWithNulls(t *testing.T) { )}, } + aggr := NewAggregateParam(AggregateSumDistinct, 1, "sum(distinct c2)") + aggr.WCol = 2 oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSumDistinct, - Col: 1, - WCol: 2, - Alias: "sum(distinct c2)", - }}, + Aggregates: []*AggregateParams{aggr}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, TruncateColumnCount: 2, @@ -936,15 +856,10 @@ func TestMultiDistinct(t *testing.T) { } oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateCountDistinct, - Col: 1, - Alias: "count(distinct c2)", - }, { - Opcode: AggregateSumDistinct, - Col: 2, - Alias: "sum(distinct c3)", - }}, + Aggregates: []*AggregateParams{ + NewAggregateParam(AggregateCountDistinct, 1, "count(distinct c2)"), + NewAggregateParam(AggregateSumDistinct, 2, "sum(distinct c3)"), + }, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -1001,10 +916,7 @@ func TestOrderedAggregateCollate(t *testing.T) { collationID, _ := collationEnv.LookupID("utf8mb4_0900_ai_ci") oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0, CollationID: collationID}}, Input: fp, } @@ -1043,10 +955,7 @@ func TestOrderedAggregateCollateAS(t *testing.T) { collationID, _ := collationEnv.LookupID("utf8mb4_0900_as_ci") oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0, CollationID: collationID}}, Input: fp, } @@ -1087,10 +996,7 @@ func TestOrderedAggregateCollateKS(t *testing.T) { collationID, _ := collationEnv.LookupID("utf8mb4_ja_0900_as_cs_ks") oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateSum, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateSum, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0, CollationID: collationID}}, Input: fp, } @@ -1172,11 +1078,7 @@ func TestGroupConcatWithAggrOnEngine(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { fp := &fakePrimitive{results: []*sqltypes.Result{tcase.inputResult}} oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateGroupConcat, - Col: 1, - Alias: "group_concat(c2)", - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateGroupConcat, 1, "group_concat(c2)")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } @@ -1255,10 +1157,7 @@ func TestGroupConcat(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { fp := &fakePrimitive{results: []*sqltypes.Result{tcase.inputResult}} oa := &OrderedAggregate{ - Aggregates: []*AggregateParams{{ - Opcode: AggregateGroupConcat, - Col: 1, - }}, + Aggregates: []*AggregateParams{NewAggregateParam(AggregateGroupConcat, 1, "")}, GroupByKeys: []*GroupByParams{{KeyCol: 0}}, Input: fp, } diff --git a/go/vt/vtgate/planbuilder/horizon_planning.go b/go/vt/vtgate/planbuilder/horizon_planning.go index caf20cb4961..8f78959d72d 100644 --- a/go/vt/vtgate/planbuilder/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/horizon_planning.go @@ -440,15 +440,11 @@ func generateAggregateParams(aggrs []operators.Aggr, aggrParamOffsets [][]offset } } - aggrParams[idx] = &engine.AggregateParams{ - Opcode: opcode, - Col: offset, - WCol: -1, - Alias: aggr.Alias, - Expr: aggr.Original.Expr, - Original: aggr.Original, - OrigOpcode: aggr.OpCode, - } + aggrParam := engine.NewAggregateParam(opcode, offset, aggr.Alias) + aggrParam.Expr = aggr.Original.Expr + aggrParam.Original = aggr.Original + aggrParam.OrigOpcode = aggr.OpCode + aggrParams[idx] = aggrParam } return aggrParams, nil } @@ -479,15 +475,12 @@ func addColumnsToOA( count++ a := aggregationExprs[offset] collID := ctx.SemTable.CollationForExpr(a.Func.GetArg()) - oa.aggregates = append(oa.aggregates, &engine.AggregateParams{ - Opcode: a.OpCode, - Col: o.col, - KeyCol: o.col, - WCol: o.wsCol, - Alias: a.Alias, - Original: a.Original, - CollationID: collID, - }) + aggr := engine.NewAggregateParam(a.OpCode, o.col, a.Alias) + aggr.KeyCol = o.col + aggr.WCol = o.wsCol + aggr.Original = a.Original + aggr.CollationID = collID + oa.aggregates = append(oa.aggregates, aggr) } lastOffset := distinctOffsets[len(distinctOffsets)-1] distinctIdx := 0 diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index cde1b756a9a..1b599ec2f78 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -89,16 +89,13 @@ func transformAggregator(ctx *plancontext.PlanningContext, op *operators.Aggrega if aggr.OpCode == opcode.AggregateUnassigned { return nil, vterrors.VT12001(fmt.Sprintf("in scatter query: aggregation function '%s'", sqlparser.String(aggr.Original))) } - oa.aggregates = append(oa.aggregates, &engine.AggregateParams{ - Opcode: aggr.OpCode, - Col: aggr.ColOffset, - Alias: aggr.Alias, - Expr: aggr.Func, - Original: aggr.Original, - OrigOpcode: aggr.OriginalOpCode, - WCol: aggr.WSOffset, - CollationID: aggr.GetCollation(ctx), - }) + aggrParam := engine.NewAggregateParam(aggr.OpCode, aggr.ColOffset, aggr.Alias) + aggrParam.Expr = aggr.Func + aggrParam.Original = aggr.Original + aggrParam.OrigOpcode = aggr.OriginalOpCode + aggrParam.WCol = aggr.WSOffset + aggrParam.CollationID = aggr.GetCollation(ctx) + oa.aggregates = append(oa.aggregates, aggrParam) } for _, groupBy := range op.Grouping { oa.groupByKeys = append(oa.groupByKeys, &engine.GroupByParams{ diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index a4f7cda487a..490935889f5 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -272,25 +272,18 @@ func (oa *orderedAggregate) pushAggr(pb *primitiveBuilder, expr *sqlparser.Alias case popcode.AggregateSum: opcode = popcode.AggregateSumDistinct } - oa.aggregates = append(oa.aggregates, &engine.AggregateParams{ - Opcode: opcode, - Col: innerCol, - Alias: expr.ColumnName(), - OrigOpcode: origOpcode, - WCol: -1, - }) + aggr := engine.NewAggregateParam(opcode, innerCol, expr.ColumnName()) + aggr.OrigOpcode = origOpcode + oa.aggregates = append(oa.aggregates, aggr) } else { newBuilder, _, innerCol, err := planProjection(pb, oa.input, expr, origin) if err != nil { return nil, 0, err } pb.plan = newBuilder - oa.aggregates = append(oa.aggregates, &engine.AggregateParams{ - Opcode: opcode, - Col: innerCol, - OrigOpcode: origOpcode, - WCol: -1, - }) + aggr := engine.NewAggregateParam(opcode, innerCol, "") + aggr.OrigOpcode = origOpcode + oa.aggregates = append(oa.aggregates, aggr) } // Build a new rc with oa as origin because it's semantically different diff --git a/go/vt/vtgate/planbuilder/projection_pushing.go b/go/vt/vtgate/planbuilder/projection_pushing.go index ab77398f50c..8772f882c78 100644 --- a/go/vt/vtgate/planbuilder/projection_pushing.go +++ b/go/vt/vtgate/planbuilder/projection_pushing.go @@ -138,14 +138,10 @@ func pushProjectionIntoOA(ctx *plancontext.PlanningContext, expr *sqlparser.Alia if err != nil { return 0, false, err } - node.aggregates = append(node.aggregates, &engine.AggregateParams{ - Opcode: popcode.AggregateAnyValue, - Col: offset, - Alias: expr.ColumnName(), - Expr: expr.Expr, - Original: expr, - WCol: -1, - }) + aggr := engine.NewAggregateParam(popcode.AggregateAnyValue, offset, expr.ColumnName()) + aggr.Expr = expr.Expr + aggr.Original = expr + node.aggregates = append(node.aggregates, aggr) return offset, true, nil } diff --git a/go/vt/vtgate/planbuilder/show.go b/go/vt/vtgate/planbuilder/show.go index 9bccd3b5e39..7a4c5abc14c 100644 --- a/go/vt/vtgate/planbuilder/show.go +++ b/go/vt/vtgate/planbuilder/show.go @@ -564,12 +564,7 @@ func buildShowVGtidPlan(show *sqlparser.ShowBasic, vschema plancontext.VSchema) } return &engine.OrderedAggregate{ Aggregates: []*engine.AggregateParams{ - { - Opcode: popcode.AggregateGtid, - Col: 1, - Alias: "global vgtid_executed", - WCol: -1, - }, + engine.NewAggregateParam(popcode.AggregateGtid, 1, "global vgtid_executed"), }, TruncateColumnCount: 2, Input: send, diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go index 43bfbe502d9..e669dbd9a33 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go @@ -109,12 +109,10 @@ func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName str // this won't work: "select count(*) from (select id from t limit 1)" // since vreplication only handles simple tables (no joins/derived tables) this is fine for now // but will need to be revisited when we add such support to vreplication - aggregateFuncType := "sum" - aggregates = append(aggregates, &engine.AggregateParams{ - Opcode: opcode.SupportedAggregates[aggregateFuncType], - Col: len(sourceSelect.SelectExprs) - 1, - WCol: -1, - }) + aggregates = append(aggregates, engine.NewAggregateParam( + /*opcode*/ opcode.AggregateSum, + /*offset*/ len(sourceSelect.SelectExprs)-1, + /*alias*/ "")) } } default: diff --git a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go index 67ba246fd7d..daf7360b895 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go +++ b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go @@ -437,15 +437,10 @@ func TestBuildPlanSuccess(t *testing.T) { Expr: &sqlparser.ColName{Name: sqlparser.NewIdentifierCI("c1")}, Direction: sqlparser.AscOrder, }}, - aggregates: []*engine.AggregateParams{{ - Opcode: opcode.AggregateSum, - Col: 2, - WCol: -1, - }, { - Opcode: opcode.AggregateSum, - Col: 3, - WCol: -1, - }}, + aggregates: []*engine.AggregateParams{ + engine.NewAggregateParam(opcode.AggregateSum, 2, ""), + engine.NewAggregateParam(opcode.AggregateSum, 3, ""), + }, }, }, { // date conversion on import. diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index c626e9adc67..761412f3726 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -691,12 +691,10 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer // this won't work: "select count(*) from (select id from t limit 1)" // since vreplication only handles simple tables (no joins/derived tables) this is fine for now // but will need to be revisited when we add such support to vreplication - aggregateFuncType := "sum" - aggregates = append(aggregates, &engine.AggregateParams{ - Opcode: opcode.SupportedAggregates[aggregateFuncType], - Col: len(sourceSelect.SelectExprs) - 1, - WCol: -1, - }) + aggregates = append(aggregates, engine.NewAggregateParam( + /*opcode*/ opcode.AggregateSum, + /*offset*/ len(sourceSelect.SelectExprs)-1, + /*alias*/ "")) } } default: diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index 2f99de07e16..500cc9a4080 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -400,15 +400,10 @@ func TestVDiffPlanSuccess(t *testing.T) { pkCols: []int{0}, selectPks: []int{0}, sourcePrimitive: &engine.OrderedAggregate{ - Aggregates: []*engine.AggregateParams{{ - Opcode: opcode.AggregateSum, - Col: 2, - WCol: -1, - }, { - Opcode: opcode.AggregateSum, - Col: 3, - WCol: -1, - }}, + Aggregates: []*engine.AggregateParams{ + engine.NewAggregateParam(opcode.AggregateSum, 2, ""), + engine.NewAggregateParam(opcode.AggregateSum, 3, ""), + }, GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1}}, Input: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), }, From 92d28deb19a404a98f91a96a28755f62b5160d4b Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 30 Jun 2023 07:26:09 +0200 Subject: [PATCH 16/26] cleanup and refactoring Signed-off-by: Andres Taylor --- .../operators/aggregation_pushing.go | 33 ++++++++++++------- go/vt/vtgate/planbuilder/operators/ast2op.go | 31 ----------------- go/vt/vtgate/planbuilder/operators/horizon.go | 16 --------- 3 files changed, 21 insertions(+), 59 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index 6d20cf211a5..63a4b820284 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -102,6 +102,25 @@ func pushDownAggregationThroughRoute( aggrBelowRoute := aggregator.SplitAggregatorBelowRoute(route.Inputs()) aggrBelowRoute.Aggregations = nil + err := pushDownAggregations(ctx, aggregator, aggrBelowRoute) + if err != nil { + return nil, nil, err + } + + // Set the source of the route to the new aggregator placed below the route. + route.Source = aggrBelowRoute + + if !aggregator.Original { + // we only keep the root aggregation, if this aggregator was created + // by splitting one and pushing under a join, we can get rid of this one + return aggregator.Source, rewrite.NewTree("push aggregation under route - remove original", aggregator), nil + } + + return aggregator, rewrite.NewTree("push aggregation under route - keep original", aggregator), nil +} + +// pushDownAggregations splits aggregations between the original aggregator and the one we are pushing down +func pushDownAggregations(ctx *plancontext.PlanningContext, aggregator *Aggregator, aggrBelowRoute *Aggregator) error { for i, aggregation := range aggregator.Aggregations { if !aggregation.Distinct || exprHasUniqueVindex(ctx, aggregation.Func.GetArg()) { aggrBelowRoute.Aggregations = append(aggrBelowRoute.Aggregations, aggregation) @@ -116,7 +135,7 @@ func pushDownAggregationThroughRoute( aggrBelowRoute.Columns[aggregation.ColOffset] = aeWrap(innerExpr) continue } - return nil, nil, vterrors.VT12001(fmt.Sprintf("only one DISTINCT aggregation is allowed in a SELECT: %s", sqlparser.String(aggregation.Original))) + return vterrors.VT12001(fmt.Sprintf("only one DISTINCT aggregation is allowed in a SELECT: %s", sqlparser.String(aggregation.Original))) } // We handle a distinct aggregation by turning it into a group by and @@ -130,17 +149,7 @@ func pushDownAggregationThroughRoute( groupBy.ColOffset = aggregation.ColOffset aggrBelowRoute.Grouping = append(aggrBelowRoute.Grouping, groupBy) } - - // Set the source of the route to the new aggregator placed below the route. - route.Source = aggrBelowRoute - - if !aggregator.Original { - // we only keep the root aggregation, if this aggregator was created - // by splitting one and pushing under a join, we can get rid of this one - return aggregator.Source, rewrite.NewTree("push aggregation under route - remove original", aggregator), nil - } - - return aggregator, rewrite.NewTree("push aggregation under route - keep original", aggregator), nil + return nil } func pushDownAggregationThroughFilter( diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 58148357e59..6b368f78fce 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -53,37 +53,6 @@ func translateQueryToOp(ctx *plancontext.PlanningContext, selStmt sqlparser.Stat return op, nil } -// func removeUnnecessaryOrdering(ctx *plancontext.PlanningContext, op ops.Operator) (ops.Operator, error) { -// horizon, ok := op.(horizonLike) -// if !ok { -// return op, nil -// } -// -// qp, err := horizon.getQP(ctx) -// if err != nil { -// return nil, err -// } -// -// if !qp.HasAggr { -// return op, nil -// } -// -// visit := func(op ops.Operator, lhsTables semantics.TableSet, isRoot bool) (ops.Operator, *rewrite.ApplyResult, error) { -// -// } -// -// shouldVisit := func(o ops.Operator) rewrite.VisitRule { -// switch o.(type) { -// case *Aggregator, *Route: -// return false -// default: -// return true -// } -// } -// rewrite.TopDown(op, TableID, visit, shouldVisit) -// } - -// createOperatorFromSelect creates an operator tree that represents the input SELECT query func createOperatorFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.Select) (ops.Operator, error) { subq, err := createSubqueryFromStatement(ctx, sel) if err != nil { diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index 5813798689a..719c0330a6c 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -116,19 +116,3 @@ func (h *Horizon) setQP(qp *QueryProjection) { func (h *Horizon) ShortDescription() string { return "" } - -// func (h *Horizon) Compact(ctx *plancontext.PlanningContext) (ops.Operator, *rewrite.ApplyResult, error) { -// src, ok := h.Source.(*Derived) -// if !ok { -// return h, rewrite.SameTree, nil -// } -// qp, err := src.getQP(ctx) -// if err != nil { -// return nil, nil, err -// } -// -// src.Query.(*sqlparser.Select).OrderBy = nil -// qp.OrderExprs = nil -// -// return h, rewrite.SameTree, nil -// } From 3cb619e64bfbd8ccad74f11a63ee85cbd553c2ab Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 30 Jun 2023 07:35:18 +0200 Subject: [PATCH 17/26] use weight strings for min/max Signed-off-by: Andres Taylor --- .../planbuilder/operators/queryprojection.go | 7 +--- .../planbuilder/testdata/aggr_cases.json | 37 ++++++++++--------- .../testdata/postprocess_cases.json | 8 ++-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 1e1a8ed28ce..423ef90f106 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -111,14 +111,11 @@ type ( func (aggr Aggr) NeedWeightString(ctx *plancontext.PlanningContext) bool { switch aggr.OpCode { - case opcode.AggregateCountDistinct, opcode.AggregateSumDistinct: + case opcode.AggregateCountDistinct, opcode.AggregateSumDistinct, opcode.AggregateMin, opcode.AggregateMax: return ctx.SemTable.NeedsWeightString(aggr.Func.GetArg()) - case opcode.AggregateMin, opcode.AggregateMax, opcode.AggregateGroupConcat: - // currently this returns false, as aggregation engine primitive does not support the usage of weight_string - // for comparison. If Min/Max column is non-comparable then it will fail at runtime. + default: return false } - return false } func (aggr Aggr) GetCollation(ctx *plancontext.PlanningContext) collations.ID { diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 04f7d7f8b9e..2966ab8a86a 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -893,7 +893,7 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1) AS min(distinct col2)", + "Aggregates": "min(1|3) AS min(distinct col2)", "GroupBy": "(0|2)", "ResultColumns": 2, "Inputs": [ @@ -904,9 +904,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1)", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2, weight_string(col1), weight_string(col2)", "OrderBy": "(0|2) ASC, (1|3) ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1) order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2, weight_string(col1), weight_string(col2) order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -3552,7 +3552,7 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1) AS min(distinct id), sum_distinct(2|4) AS sum(distinct col3)", + "Aggregates": "min(1|4) AS min(distinct id), sum_distinct(2|5) AS sum(distinct col3)", "GroupBy": "(0|3)", "ResultColumns": 3, "Inputs": [ @@ -3563,9 +3563,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` where 1 != 1 group by col1, col3, weight_string(col1), weight_string(col3)", - "OrderBy": "(0|3) ASC, (2|4) ASC", - "Query": "select col1, min(distinct id), col3, weight_string(col1), weight_string(col3) from `user` group by col1, col3, weight_string(col1), weight_string(col3) order by col1 asc, col3 asc", + "FieldQuery": "select col1, min(distinct id), col3, weight_string(col1), weight_string(id), weight_string(col3) from `user` where 1 != 1 group by col1, col3, weight_string(col1), weight_string(id), weight_string(col3)", + "OrderBy": "(0|3) ASC, (2|5) ASC", + "Query": "select col1, min(distinct id), col3, weight_string(col1), weight_string(id), weight_string(col3) from `user` group by col1, col3, weight_string(col1), weight_string(id), weight_string(col3) order by col1 asc, col3 asc", "Table": "`user`" } ] @@ -5283,7 +5283,7 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "max(1) AS bazo", + "Aggregates": "max(1|3) AS bazo", "GroupBy": "(0|2)", "Inputs": [ { @@ -5293,9 +5293,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo)", + "FieldQuery": "select foo, max(baz) as bazo, weight_string(foo), weight_string(baz) from (select foo, baz from `user` where 1 != 1) as f where 1 != 1 group by foo, weight_string(foo), weight_string(baz)", "OrderBy": "(0|2) ASC", - "Query": "select foo, max(baz) as bazo, weight_string(foo) from (select foo, baz from `user`) as f group by foo, weight_string(foo) order by foo asc", + "Query": "select foo, max(baz) as bazo, weight_string(foo), weight_string(baz) from (select foo, baz from `user`) as f group by foo, weight_string(foo), weight_string(baz) order by foo asc", "Table": "`user`" } ] @@ -6425,14 +6425,14 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1) AS min(user_extra.foo), max(3) AS max(user_extra.bar)", + "Aggregates": "min(1|5) AS min(user_extra.foo), max(3|6) AS max(user_extra.bar)", "GroupBy": "0, (2|4)", "ResultColumns": 4, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:2", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:2,R:2,R:3", "JoinVars": { "user_col": 0 }, @@ -6457,8 +6457,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select min(user_extra.foo), max(user_extra.bar) from user_extra where 1 != 1 group by .0", - "Query": "select min(user_extra.foo), max(user_extra.bar) from user_extra where user_extra.bar = :user_col group by .0", + "FieldQuery": "select min(user_extra.foo), max(user_extra.bar), weight_string(user_extra.foo), weight_string(user_extra.bar) from user_extra where 1 != 1 group by .0, weight_string(user_extra.foo), weight_string(user_extra.bar)", + "Query": "select min(user_extra.foo), max(user_extra.bar), weight_string(user_extra.foo), weight_string(user_extra.bar) from user_extra where user_extra.bar = :user_col group by .0, weight_string(user_extra.foo), weight_string(user_extra.bar)", "Table": "user_extra" } ] @@ -6481,12 +6481,13 @@ "Instructions": { "OperatorType": "Aggregate", "Variant": "Scalar", - "Aggregates": "max(0) AS max(u.foo * ue.bar)", + "Aggregates": "max(0|1) AS max(u.foo * ue.bar)", + "ResultColumns": 1, "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0", + "JoinColumnIndexes": "R:0,R:1", "JoinVars": { "u_foo": 0 }, @@ -6510,8 +6511,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select :u_foo * ue.bar from user_extra as ue where 1 != 1", - "Query": "select :u_foo * ue.bar from user_extra as ue", + "FieldQuery": "select :u_foo * ue.bar, weight_string(:u_foo * ue.bar) from user_extra as ue where 1 != 1", + "Query": "select :u_foo * ue.bar, weight_string(:u_foo * ue.bar) from user_extra as ue", "Table": "user_extra" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 0991a324f6e..ca521685cd6 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -3303,13 +3303,13 @@ { "OperatorType": "Aggregate", "Variant": "Ordered", - "Aggregates": "min(1) AS min(a.id)", + "Aggregates": "min(1|3) AS min(a.id)", "GroupBy": "(0|2)", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:1,L:0,L:2", + "JoinColumnIndexes": "L:1,L:0,L:2,L:3", "JoinVars": { "a_tcol1": 1 }, @@ -3322,9 +3322,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select min(a.id), a.tcol1, weight_string(a.tcol1) from `user` as a where 1 != 1 group by a.tcol1, weight_string(a.tcol1)", + "FieldQuery": "select min(a.id), a.tcol1, weight_string(a.tcol1), weight_string(a.id) from `user` as a where 1 != 1 group by a.tcol1, weight_string(a.tcol1), weight_string(a.id)", "OrderBy": "(1|2) ASC", - "Query": "select min(a.id), a.tcol1, weight_string(a.tcol1) from `user` as a group by a.tcol1, weight_string(a.tcol1) order by a.tcol1 asc", + "Query": "select min(a.id), a.tcol1, weight_string(a.tcol1), weight_string(a.id) from `user` as a group by a.tcol1, weight_string(a.tcol1), weight_string(a.id) order by a.tcol1 asc", "Table": "`user`" }, { From 77d630aa90ea4b488730e14e3a42257d1b8ff91d Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Fri, 30 Jun 2023 09:11:26 +0200 Subject: [PATCH 18/26] update list of error code Signed-off-by: Florent Poinsard --- go/vt/vterrors/code.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/vterrors/code.go b/go/vt/vterrors/code.go index 7d8f22889d0..2aa9827c084 100644 --- a/go/vt/vterrors/code.go +++ b/go/vt/vterrors/code.go @@ -137,6 +137,7 @@ var ( VT09012, VT09013, VT09014, + VT09015, VT10001, VT12001, VT13001, From b320671921cd61d577586d0fdd1bf275b30e6900 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 30 Jun 2023 15:31:09 +0200 Subject: [PATCH 19/26] disallow aggregation on top of aggregation with a clearer error message Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/aggregator.go | 3 +++ go/vt/vtgate/planbuilder/testdata/unsupported_cases.json | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index 003ea7388eb..a40fad16b82 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -351,6 +351,9 @@ func (a *Aggregator) addIfAggregationColumn(ctx *plancontext.PlanningContext, co return 0, err } if aggr.ColOffset != offset { + if _, srcIsAlsoAggr := a.Source.(*Aggregator); srcIsAlsoAggr { + return 0, vterrors.VT12001("aggregation on top of aggregation not supported") + } return -1, vterrors.VT13001(fmt.Sprintf("aggregation column on wrong index: want: %d, got: %d", colIdx, offset)) } diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 330de6c7fd3..cc020dac2af 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -459,5 +459,11 @@ "comment": "extremum on input from both sides", "query": "insert into music(user_id, id) select foo, bar from music on duplicate key update id = id+1", "plan": "VT12001: unsupported: DML cannot update vindex column" + }, + { + "comment": "aggregation on top of aggregation not supported", + "query": "select distinct count(*) from user, (select distinct count(*) from user) X", + "v3-plan": "VT12001: unsupported: cross-shard query with aggregates", + "gen4-plan": "VT12001: unsupported: aggregation on top of aggregation not supported" } ] From f016a89014fadf326e71de310c01f5195bf35d9e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 30 Jun 2023 16:20:11 +0200 Subject: [PATCH 20/26] fail min/max queries on types we cant compare Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/aggregations.go | 37 +++++++++---------- go/vt/vtgate/engine/opcode/constants.go | 18 +++++++++ go/vt/vtgate/engine/ordered_aggregate_test.go | 27 ++++++++++++++ .../planbuilder/operators/aggregator.go | 4 +- .../planbuilder/operators/queryprojection.go | 9 +---- 5 files changed, 67 insertions(+), 28 deletions(-) diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index 32c481feed5..abe542c1e4c 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -20,6 +20,8 @@ import ( "fmt" "strconv" + "vitess.io/vitess/go/vt/vterrors" + "google.golang.org/protobuf/proto" "vitess.io/vitess/go/mysql/collations" @@ -53,31 +55,22 @@ type AggregateParams struct { } func NewAggregateParam(opcode AggregateOpcode, col int, alias string) *AggregateParams { - return &AggregateParams{ + out := &AggregateParams{ Opcode: opcode, Col: col, Alias: alias, WCol: -1, } + if opcode.NeedsComparableValues() { + out.KeyCol = col + } + return out } func (ap *AggregateParams) WAssigned() bool { return ap.WCol >= 0 } -func (ap *AggregateParams) isDistinct() bool { - return ap.Opcode == AggregateCountDistinct || ap.Opcode == AggregateSumDistinct -} - -func (ap *AggregateParams) preProcess() bool { - switch ap.Opcode { - case AggregateCountDistinct, AggregateSumDistinct, AggregateGtid, AggregateCount, AggregateGroupConcat: - return true - default: - return false - } -} - func (ap *AggregateParams) String() string { keyCol := strconv.Itoa(ap.Col) if ap.WAssigned() { @@ -173,7 +166,7 @@ func merge( ) ([]sqltypes.Value, []sqltypes.Value, error) { result := sqltypes.CopyRow(row1) for index, aggr := range aggregates { - if aggr.isDistinct() { + if aggr.Opcode.IsDistinct() { if row2[aggr.KeyCol].IsNull() { continue } @@ -206,8 +199,14 @@ func merge( } result[aggr.Col], err = evalengine.NullSafeAdd(value, v2, fields[aggr.Col].Type) case AggregateMin: + if aggr.WAssigned() && !row2[aggr.Col].IsComparable() { + return minMaxWeightStringError() + } result[aggr.Col], err = evalengine.Min(row1[aggr.Col], row2[aggr.Col], aggr.CollationID) case AggregateMax: + if aggr.WAssigned() && !row2[aggr.Col].IsComparable() { + return minMaxWeightStringError() + } result[aggr.Col], err = evalengine.Max(row1[aggr.Col], row2[aggr.Col], aggr.CollationID) case AggregateCountDistinct: result[aggr.Col], err = evalengine.NullSafeAdd(row1[aggr.Col], countOne, fields[aggr.Col].Type) @@ -253,6 +252,10 @@ func merge( return result, curDistincts, nil } +func minMaxWeightStringError() ([]sqltypes.Value, []sqltypes.Value, error) { + return nil, nil, vterrors.VT12001("min/max on types that are not comparable is not supported") +} + func convertFinal(current []sqltypes.Value, aggregates []*AggregateParams) ([]sqltypes.Value, error) { result := sqltypes.CopyRow(current) for _, aggr := range aggregates { @@ -282,10 +285,6 @@ func convertFields(fields []*querypb.Field, aggrs []*AggregateParams) []*querypb if aggr.Alias != "" { fields[aggr.Col].Name = aggr.Alias } - if aggr.isDistinct() { - // TODO: this should move to plan time - aggr.KeyCol = aggr.Col - } } return fields } diff --git a/go/vt/vtgate/engine/opcode/constants.go b/go/vt/vtgate/engine/opcode/constants.go index fa54b898ede..818a9e67db6 100644 --- a/go/vt/vtgate/engine/opcode/constants.go +++ b/go/vt/vtgate/engine/opcode/constants.go @@ -161,3 +161,21 @@ func (code AggregateOpcode) Type(typ *querypb.Type) (querypb.Type, bool) { panic(code.String()) // we have a unit test checking we never reach here } } + +func (code AggregateOpcode) NeedsComparableValues() bool { + switch code { + case AggregateCountDistinct, AggregateSumDistinct, AggregateMin, AggregateMax: + return true + default: + return false + } +} + +func (code AggregateOpcode) IsDistinct() bool { + switch code { + case AggregateCountDistinct, AggregateSumDistinct: + return true + default: + return false + } +} diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index c4edac39171..27f3d057979 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -120,6 +120,33 @@ func TestOrderedAggregateExecuteTruncate(t *testing.T) { utils.MustMatch(t, wantResult, result) } +func TestMinMaxFailsCorrectly(t *testing.T) { + fp := &fakePrimitive{ + results: []*sqltypes.Result{sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "col|weight_string(col)", + "varchar|varbinary", + ), + "a|A", + "A|A", + "b|B", + "C|C", + "c|C", + )}, + } + + aggr := NewAggregateParam(AggregateMax, 0, "") + aggr.WCol = 1 + oa := &ScalarAggregate{ + Aggregates: []*AggregateParams{aggr}, + TruncateColumnCount: 1, + Input: fp, + } + + _, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) + assert.ErrorContains(t, err, "min/max on types that are not comparable is not supported") +} + func TestOrderedAggregateStreamExecute(t *testing.T) { assert := assert.New(t) fields := sqltypes.MakeTestFields( diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index a40fad16b82..ce00afd9c02 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -287,7 +287,7 @@ func (a *Aggregator) planOffsets(ctx *plancontext.PlanningContext) error { } for idx, aggr := range a.Aggregations { - if !aggr.NeedWeightString(ctx) { + if !aggr.NeedsWeightString(ctx) { continue } offset, err := a.internalAddColumn(ctx, aeWrap(weightStringFor(aggr.Func.GetArg())), true) @@ -407,7 +407,7 @@ func (a *Aggregator) pushRemainingGroupingColumnsAndWeightStrings(ctx *planconte a.Grouping[idx].WSOffset = offset } for idx, aggr := range a.Aggregations { - if aggr.WSOffset != -1 || !aggr.NeedWeightString(ctx) { + if aggr.WSOffset != -1 || !aggr.NeedsWeightString(ctx) { continue } diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 423ef90f106..8f70df5c52e 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -109,13 +109,8 @@ type ( } ) -func (aggr Aggr) NeedWeightString(ctx *plancontext.PlanningContext) bool { - switch aggr.OpCode { - case opcode.AggregateCountDistinct, opcode.AggregateSumDistinct, opcode.AggregateMin, opcode.AggregateMax: - return ctx.SemTable.NeedsWeightString(aggr.Func.GetArg()) - default: - return false - } +func (aggr Aggr) NeedsWeightString(ctx *plancontext.PlanningContext) bool { + return aggr.OpCode.NeedsComparableValues() && ctx.SemTable.NeedsWeightString(aggr.Func.GetArg()) } func (aggr Aggr) GetCollation(ctx *plancontext.PlanningContext) collations.ID { From f6d57f0b3999cf306b4ca7b0d1bac831f295f250 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Fri, 30 Jun 2023 16:22:43 +0200 Subject: [PATCH 21/26] test: remove pattern not used Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/ordered_aggregate_test.go | 46 +++++++------------ .../vtgate/engine/scalar_aggregation_test.go | 13 ++---- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/go/vt/vtgate/engine/ordered_aggregate_test.go b/go/vt/vtgate/engine/ordered_aggregate_test.go index 27f3d057979..539e81b4c59 100644 --- a/go/vt/vtgate/engine/ordered_aggregate_test.go +++ b/go/vt/vtgate/engine/ordered_aggregate_test.go @@ -45,7 +45,6 @@ func init() { } func TestOrderedAggregateExecute(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|count(*)", "varbinary|decimal", @@ -68,7 +67,7 @@ func TestOrderedAggregateExecute(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, @@ -80,7 +79,6 @@ func TestOrderedAggregateExecute(t *testing.T) { } func TestOrderedAggregateExecuteTruncate(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -106,7 +104,7 @@ func TestOrderedAggregateExecuteTruncate(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -148,7 +146,6 @@ func TestMinMaxFailsCorrectly(t *testing.T) { } func TestOrderedAggregateStreamExecute(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|count(*)", "varbinary|decimal", @@ -175,7 +172,7 @@ func TestOrderedAggregateStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - assert.NoError(err) + assert.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( fields, @@ -189,7 +186,6 @@ func TestOrderedAggregateStreamExecute(t *testing.T) { } func TestOrderedAggregateStreamExecuteTruncate(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -216,7 +212,7 @@ func TestOrderedAggregateStreamExecuteTruncate(t *testing.T) { results = append(results, qr) return nil }) - assert.NoError(err) + assert.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( sqltypes.MakeTestFields( @@ -233,7 +229,6 @@ func TestOrderedAggregateStreamExecuteTruncate(t *testing.T) { } func TestOrderedAggregateGetFields(t *testing.T) { - assert := assert.New(t) input := sqltypes.MakeTestResult( sqltypes.MakeTestFields( "col|count(*)", @@ -245,8 +240,8 @@ func TestOrderedAggregateGetFields(t *testing.T) { oa := &OrderedAggregate{Input: fp} got, err := oa.GetFields(context.Background(), nil, nil) - assert.NoError(err) - assert.Equal(got, input) + assert.NoError(t, err) + assert.Equal(t, got, input) } func TestOrderedAggregateInputFail(t *testing.T) { @@ -271,7 +266,6 @@ func TestOrderedAggregateInputFail(t *testing.T) { } func TestOrderedAggregateExecuteCountDistinct(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -320,7 +314,7 @@ func TestOrderedAggregateExecuteCountDistinct(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -341,7 +335,6 @@ func TestOrderedAggregateExecuteCountDistinct(t *testing.T) { } func TestOrderedAggregateStreamCountDistinct(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -396,7 +389,7 @@ func TestOrderedAggregateStreamCountDistinct(t *testing.T) { results = append(results, qr) return nil }) - assert.NoError(err) + assert.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( sqltypes.MakeTestFields( @@ -425,7 +418,6 @@ func TestOrderedAggregateStreamCountDistinct(t *testing.T) { } func TestOrderedAggregateSumDistinctGood(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -475,7 +467,7 @@ func TestOrderedAggregateSumDistinctGood(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -494,7 +486,7 @@ func TestOrderedAggregateSumDistinctGood(t *testing.T) { ) want := fmt.Sprintf("%v", wantResult.Rows) got := fmt.Sprintf("%v", result.Rows) - assert.Equal(want, got) + assert.Equal(t, want, got) } func TestOrderedAggregateSumDistinctTolerateError(t *testing.T) { @@ -608,7 +600,6 @@ func TestOrderedAggregateMergeFail(t *testing.T) { } func TestMerge(t *testing.T) { - assert := assert.New(t) oa := &OrderedAggregate{ Aggregates: []*AggregateParams{ NewAggregateParam(AggregateSum, 1, ""), @@ -626,14 +617,14 @@ func TestMerge(t *testing.T) { ) merged, _, err := merge(fields, r.Rows[0], r.Rows[1], nil, oa.Aggregates) - assert.NoError(err) + assert.NoError(t, err) want := sqltypes.MakeTestResult(fields, "1|5|6.0|2|bc").Rows[0] - assert.Equal(want, merged) + assert.Equal(t, want, merged) // swap and retry merged, _, err = merge(fields, r.Rows[1], r.Rows[0], nil, oa.Aggregates) - assert.NoError(err) - assert.Equal(want, merged) + assert.NoError(t, err) + assert.Equal(t, want, merged) } func TestOrderedAggregateExecuteGtid(t *testing.T) { @@ -921,7 +912,6 @@ func TestMultiDistinct(t *testing.T) { } func TestOrderedAggregateCollate(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|count(*)", "varchar|decimal", @@ -949,7 +939,7 @@ func TestOrderedAggregateCollate(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, @@ -962,7 +952,6 @@ func TestOrderedAggregateCollate(t *testing.T) { } func TestOrderedAggregateCollateAS(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|count(*)", "varchar|decimal", @@ -988,7 +977,7 @@ func TestOrderedAggregateCollateAS(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, @@ -1002,7 +991,6 @@ func TestOrderedAggregateCollateAS(t *testing.T) { } func TestOrderedAggregateCollateKS(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|count(*)", "varchar|decimal", @@ -1029,7 +1017,7 @@ func TestOrderedAggregateCollateKS(t *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, diff --git a/go/vt/vtgate/engine/scalar_aggregation_test.go b/go/vt/vtgate/engine/scalar_aggregation_test.go index 273a6228c8f..033d69c9271 100644 --- a/go/vt/vtgate/engine/scalar_aggregation_test.go +++ b/go/vt/vtgate/engine/scalar_aggregation_test.go @@ -69,7 +69,6 @@ func TestEmptyRows(outer *testing.T) { for _, test := range testCases { outer.Run(test.opcode.String(), func(t *testing.T) { - assert := assert.New(t) fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -91,7 +90,7 @@ func TestEmptyRows(outer *testing.T) { } result, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, false) - assert.NoError(err) + assert.NoError(t, err) wantResult := sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -106,7 +105,6 @@ func TestEmptyRows(outer *testing.T) { } func TestScalarAggregateStreamExecute(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|weight_string(col)", "uint64|varbinary", @@ -135,17 +133,16 @@ func TestScalarAggregateStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - assert.NoError(err) + assert.NoError(t, err) // one for the fields, and one for the actual aggregation result require.EqualValues(t, 2, len(results), "number of results") got := fmt.Sprintf("%v", results[1].Rows) - assert.Equal("[[DECIMAL(4)]]", got) + assert.Equal(t, "[[DECIMAL(4)]]", got) } // TestScalarAggregateExecuteTruncate checks if truncate works func TestScalarAggregateExecuteTruncate(t *testing.T) { - assert := assert.New(t) fields := sqltypes.MakeTestFields( "col|weight_string(col)", "uint64|varbinary", @@ -169,8 +166,8 @@ func TestScalarAggregateExecuteTruncate(t *testing.T) { } qr, err := oa.TryExecute(context.Background(), &noopVCursor{}, nil, true) - assert.NoError(err) - assert.Equal("[[DECIMAL(4)]]", fmt.Sprintf("%v", qr.Rows)) + assert.NoError(t, err) + assert.Equal(t, "[[DECIMAL(4)]]", fmt.Sprintf("%v", qr.Rows)) } // TestScalarGroupConcatWithAggrOnEngine tests group_concat with full aggregation on engine. From 0f9d320979b9ce8fa9191908a363d4826204ac4e Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 1 Jul 2023 09:35:55 +0200 Subject: [PATCH 22/26] spread table id through derived tables Signed-off-by: Andres Taylor --- .../planbuilder/operators/aggregator.go | 7 + go/vt/vtgate/planbuilder/operators/delete.go | 2 +- go/vt/vtgate/planbuilder/operators/derived.go | 4 + go/vt/vtgate/planbuilder/operators/helpers.go | 10 +- .../planbuilder/operators/projection.go | 7 + .../planbuilder/operators/querygraph.go | 4 +- .../planbuilder/operators/route_planning.go | 4 +- go/vt/vtgate/planbuilder/operators/table.go | 2 +- go/vt/vtgate/planbuilder/operators/update.go | 2 +- go/vt/vtgate/planbuilder/operators/vindex.go | 2 +- .../planbuilder/testdata/from_cases.json | 8 +- .../testdata/memory_sort_cases.json | 57 ++--- .../testdata/postprocess_cases.json | 4 +- .../planbuilder/testdata/reference_cases.json | 4 +- .../planbuilder/testdata/select_cases.json | 12 +- .../planbuilder/testdata/tpcc_cases.json | 4 +- .../planbuilder/testdata/tpch_cases.json | 225 +++++++++++------- 17 files changed, 205 insertions(+), 153 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index ce00afd9c02..abd9c261e18 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -449,4 +449,11 @@ func (a *Aggregator) SplitAggregatorBelowRoute(input []ops.Operator) *Aggregator return newOp } +func (a *Aggregator) introducesTableID() semantics.TableSet { + if a.TableID == nil { + return semantics.EmptyTableSet() + } + return *a.TableID +} + var _ ops.Operator = (*Aggregator)(nil) diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index af1db82ad80..c24ab9f5065 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -37,7 +37,7 @@ type Delete struct { } // Introduces implements the PhysicalOperator interface -func (d *Delete) Introduces() semantics.TableSet { +func (d *Delete) introducesTableID() semantics.TableSet { return d.QTable.ID } diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go index 3ac8b5029a2..e18240b395b 100644 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ b/go/vt/vtgate/planbuilder/operators/derived.go @@ -247,3 +247,7 @@ func (d *Derived) setQP(qp *QueryProjection) { func (d *Derived) ShortDescription() string { return d.Alias } + +func (d *Derived) introducesTableID() semantics.TableSet { + return d.TableId +} diff --git a/go/vt/vtgate/planbuilder/operators/helpers.go b/go/vt/vtgate/planbuilder/operators/helpers.go index 1c472acf413..21be634d7d8 100644 --- a/go/vt/vtgate/planbuilder/operators/helpers.go +++ b/go/vt/vtgate/planbuilder/operators/helpers.go @@ -67,15 +67,15 @@ func Clone(op ops.Operator) ops.Operator { return op.Clone(clones) } -// TableIDIntroducer is used to signal that this operator introduces data from a new source -type TableIDIntroducer interface { - Introduces() semantics.TableSet +// tableIDIntroducer is used to signal that this operator introduces data from a new source +type tableIDIntroducer interface { + introducesTableID() semantics.TableSet } func TableID(op ops.Operator) (result semantics.TableSet) { _ = rewrite.Visit(op, func(this ops.Operator) error { - if tbl, ok := this.(TableIDIntroducer); ok { - result = result.Merge(tbl.Introduces()) + if tbl, ok := this.(tableIDIntroducer); ok { + result = result.Merge(tbl.introducesTableID()) } return nil }) diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index 37b6cc0c7f4..ff5a43a8c35 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -342,3 +342,10 @@ func (p *Projection) planOffsets(ctx *plancontext.PlanningContext) error { return nil } + +func (p *Projection) introducesTableID() semantics.TableSet { + if p.TableID == nil { + return semantics.EmptyTableSet() + } + return *p.TableID +} diff --git a/go/vt/vtgate/planbuilder/operators/querygraph.go b/go/vt/vtgate/planbuilder/operators/querygraph.go index f9b05914e77..5d06da0e9b5 100644 --- a/go/vt/vtgate/planbuilder/operators/querygraph.go +++ b/go/vt/vtgate/planbuilder/operators/querygraph.go @@ -65,8 +65,8 @@ type ( var _ ops.Operator = (*QueryGraph)(nil) -// Introduces implements the TableIDIntroducer interface -func (qg *QueryGraph) Introduces() semantics.TableSet { +// Introduces implements the tableIDIntroducer interface +func (qg *QueryGraph) introducesTableID() semantics.TableSet { var ts semantics.TableSet for _, table := range qg.Tables { ts = ts.Merge(table.ID) diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index f38dcbab3a8..7db75f5df07 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -500,11 +500,11 @@ func findColumnVindex(ctx *plancontext.PlanningContext, a ops.Operator, exp sqlp deps := ctx.SemTable.RecursiveDeps(expr) _ = rewrite.Visit(a, func(rel ops.Operator) error { - to, isTableOp := rel.(TableIDIntroducer) + to, isTableOp := rel.(tableIDIntroducer) if !isTableOp { return nil } - id := to.Introduces() + id := to.introducesTableID() if deps.IsSolvedBy(id) { tableInfo, err := ctx.SemTable.TableInfoFor(id) if err != nil { diff --git a/go/vt/vtgate/planbuilder/operators/table.go b/go/vt/vtgate/planbuilder/operators/table.go index 081d70cc4df..2409789d0bf 100644 --- a/go/vt/vtgate/planbuilder/operators/table.go +++ b/go/vt/vtgate/planbuilder/operators/table.go @@ -54,7 +54,7 @@ func (to *Table) Clone([]ops.Operator) ops.Operator { } // Introduces implements the PhysicalOperator interface -func (to *Table) Introduces() semantics.TableSet { +func (to *Table) introducesTableID() semantics.TableSet { return to.QTable.ID } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 91273062c17..0627f07734e 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -38,7 +38,7 @@ type Update struct { } // Introduces implements the PhysicalOperator interface -func (u *Update) Introduces() semantics.TableSet { +func (u *Update) introducesTableID() semantics.TableSet { return u.QTable.ID } diff --git a/go/vt/vtgate/planbuilder/operators/vindex.go b/go/vt/vtgate/planbuilder/operators/vindex.go index a3c16fea0c4..e39d245bb50 100644 --- a/go/vt/vtgate/planbuilder/operators/vindex.go +++ b/go/vt/vtgate/planbuilder/operators/vindex.go @@ -52,7 +52,7 @@ type ( const VindexUnsupported = "WHERE clause for vindex function must be of the form id = or id in(,...)" // Introduces implements the Operator interface -func (v *Vindex) Introduces() semantics.TableSet { +func (v *Vindex) introducesTableID() semantics.TableSet { return v.Solved } diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.json b/go/vt/vtgate/planbuilder/testdata/from_cases.json index d6c12fd9b77..ecec6a803ea 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.json @@ -2273,8 +2273,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select ref.col from ref, (select aa from `user` where 1 != 1) as `user` where 1 != 1", - "Query": "select ref.col from ref, (select aa from `user` where `user`.id = 1) as `user`", + "FieldQuery": "select ref.col from (select aa from `user` where 1 != 1) as `user`, ref where 1 != 1", + "Query": "select ref.col from (select aa from `user` where `user`.id = 1) as `user`, ref", "Table": "`user`, ref", "Values": [ "INT64(1)" @@ -2534,8 +2534,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select t.id from user_extra, (select id from `user` where 1 != 1) as t where 1 != 1", - "Query": "select t.id from user_extra, (select id from `user` where id = 5) as t where t.id = user_extra.user_id", + "FieldQuery": "select t.id from (select id from `user` where 1 != 1) as t, user_extra where 1 != 1", + "Query": "select t.id from (select id from `user` where id = 5) as t, user_extra where t.id = user_extra.user_id", "Table": "`user`, user_extra", "Values": [ "INT64(5)" diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json index 4dd03fa2608..944e5cd617c 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.json @@ -501,40 +501,33 @@ "QueryType": "SELECT", "Original": "select id from (select user.id, user.col from user join user_extra) as t order by id", "Instructions": { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "(0|1) ASC", - "ResultColumns": 1, + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0", + "TableName": "`user`_user_extra", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1", - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from (select `user`.id, `user`.col from `user` where 1 != 1) as t where 1 != 1", - "Query": "select id, weight_string(id) from (select `user`.id, `user`.col from `user`) as t", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user_extra where 1 != 1", - "Query": "select 1 from user_extra", - "Table": "user_extra" - } - ] + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select id, weight_string(id) from (select `user`.id, `user`.col from `user` where 1 != 1) as t where 1 != 1", + "OrderBy": "(0|1) ASC", + "Query": "select id, weight_string(id) from (select `user`.id, `user`.col from `user`) as t order by id asc", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from user_extra where 1 != 1", + "Query": "select 1 from user_extra", + "Table": "user_extra" } ] }, diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index ca521685cd6..4908a29d134 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -1860,8 +1860,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select * from `user` as u, (select user_id from user_extra where 1 != 1) as eu where 1 != 1", - "Query": "select * from `user` as u, (select user_id from user_extra where user_id = 5) as eu where u.id = 5 and u.id = eu.user_id order by eu.user_id asc", + "FieldQuery": "select * from (select user_id from user_extra where 1 != 1) as eu, `user` as u where 1 != 1", + "Query": "select * from (select user_id from user_extra where user_id = 5) as eu, `user` as u where u.id = 5 and u.id = eu.user_id order by eu.user_id asc", "Table": "`user`, user_extra", "Values": [ "INT64(5)" diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index ac5338ecd3a..289e23b0b07 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -238,8 +238,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, (select aa from `user` where 1 != 1) as `user` where 1 != 1", - "Query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, (select aa from `user` where `user`.id = 1) as `user`", + "FieldQuery": "select ambiguous_ref_with_source.col from (select aa from `user` where 1 != 1) as `user`, ambiguous_ref_with_source where 1 != 1", + "Query": "select ambiguous_ref_with_source.col from (select aa from `user` where `user`.id = 1) as `user`, ambiguous_ref_with_source", "Table": "`user`, ambiguous_ref_with_source", "Values": [ "INT64(1)" diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.json b/go/vt/vtgate/planbuilder/testdata/select_cases.json index cf730f526ad..08227bc5a15 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.json @@ -7459,8 +7459,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music, (select max(id) as maxt from music where 1 != 1) as other where 1 != 1", - "Query": "select music.id from music, (select max(id) as maxt from music where music.user_id = 5) as other where other.maxt = music.id", + "FieldQuery": "select music.id from (select max(id) as maxt from music where 1 != 1) as other, music where 1 != 1", + "Query": "select music.id from (select max(id) as maxt from music where music.user_id = 5) as other, music where other.maxt = music.id", "Table": "music", "Values": [ "INT64(5)" @@ -7500,8 +7500,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music, (select id from music where 1 != 1) as other where 1 != 1", - "Query": "select music.id from music, (select id from music where music.user_id = 5) as other where other.id = music.id", + "FieldQuery": "select music.id from (select id from music where 1 != 1) as other, music where 1 != 1", + "Query": "select music.id from (select id from music where music.user_id = 5) as other, music where other.id = music.id", "Table": "music", "Values": [ "INT64(5)" @@ -7541,8 +7541,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.id from music, (select id from music where 1 != 1) as other where 1 != 1", - "Query": "select music.id from music, (select id from music where music.user_id in ::__vals) as other where other.id = music.id", + "FieldQuery": "select music.id from (select id from music where 1 != 1) as other, music where 1 != 1", + "Query": "select music.id from (select id from music where music.user_id in ::__vals) as other, music where other.id = music.id", "Table": "music", "Values": [ "(INT64(5), INT64(6), INT64(7))" diff --git a/go/vt/vtgate/planbuilder/testdata/tpcc_cases.json b/go/vt/vtgate/planbuilder/testdata/tpcc_cases.json index a36319cb322..51be1f7522c 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpcc_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpcc_cases.json @@ -1633,8 +1633,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select o.o_id, o.o_d_id from orders1 as o, (select o_c_id, o_w_id, o_d_id, count(distinct o_w_id), o_id from orders1 where 1 != 1 group by o_c_id, o_d_id, o_w_id) as t where 1 != 1", - "Query": "select o.o_id, o.o_d_id from orders1 as o, (select o_c_id, o_w_id, o_d_id, count(distinct o_w_id), o_id from orders1 where o_w_id = 1 and o_id > 2100 and o_id < 11153 group by o_c_id, o_d_id, o_w_id having count(distinct o_id) > 1 limit 1) as t where t.o_w_id = o.o_w_id and t.o_d_id = o.o_d_id and t.o_c_id = o.o_c_id limit 1", + "FieldQuery": "select o.o_id, o.o_d_id from (select o_c_id, o_w_id, o_d_id, count(distinct o_w_id), o_id from orders1 where 1 != 1 group by o_c_id, o_d_id, o_w_id) as t, orders1 as o where 1 != 1", + "Query": "select o.o_id, o.o_d_id from (select o_c_id, o_w_id, o_d_id, count(distinct o_w_id), o_id from orders1 where o_w_id = 1 and o_id > 2100 and o_id < 11153 group by o_c_id, o_d_id, o_w_id having count(distinct o_id) > 1 limit 1) as t, orders1 as o where t.o_w_id = o.o_w_id and t.o_d_id = o.o_d_id and t.o_c_id = o.o_c_id limit 1", "Table": "orders1", "Values": [ "INT64(1)" diff --git a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json index 3e97f2c01fa..eb24df0b0b8 100644 --- a/go/vt/vtgate/planbuilder/testdata/tpch_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/tpch_cases.json @@ -553,131 +553,172 @@ "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "sum(3) AS revenue", - "GroupBy": "(0|6), (1|5), (2|4)", + "GroupBy": "(0|4), (1|5), (2|6)", "ResultColumns": 4, "Inputs": [ { "OperatorType": "Projection", "Expressions": [ - "[COLUMN 4] as supp_nation", - "[COLUMN 5] as cust_nation", - "[COLUMN 6] as l_year", - "(((([COLUMN 10] * COALESCE([COLUMN 11], INT64(1))) * COALESCE([COLUMN 12], INT64(1))) * COALESCE([COLUMN 13], INT64(1))) * COALESCE([COLUMN 14], INT64(1))) * COALESCE([COLUMN 15], INT64(1)) as revenue", - "[COLUMN 9]", - "[COLUMN 8]", - "[COLUMN 7]" + "[COLUMN 2] as supp_nation", + "[COLUMN 3] as cust_nation", + "[COLUMN 4] as l_year", + "[COLUMN 0] * [COLUMN 1] as revenue", + "[COLUMN 5] as weight_string(supp_nation)", + "[COLUMN 6] as weight_string(cust_nation)", + "[COLUMN 7] as weight_string(l_year)" ], "Inputs": [ { - "OperatorType": "Sort", - "Variant": "Memory", - "OrderBy": "(0|16) ASC, (1|17) ASC, (2|18) ASC", + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,R:1,L:2,L:5,R:2,L:6", + "JoinVars": { + "n1_n_name": 4, + "o_custkey": 3 + }, + "TableName": "lineitem_orders_supplier_nation_customer_nation", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:2,R:0,L:3,L:4,L:8,R:1,L:9,L:13,R:2,L:14,L:15,L:16,L:17,L:18,R:3,R:4,L:19,R:5,L:20", - "JoinVars": { - "n1_n_name": 1, - "o_custkey": 0 - }, - "TableName": "lineitem_orders_supplier_nation_customer_nation", + "OperatorType": "Projection", + "Expressions": [ + "[COLUMN 0] * [COLUMN 1] as revenue", + "[COLUMN 2] as supp_nation", + "[COLUMN 3] as l_year", + "[COLUMN 4] as orders.o_custkey", + "[COLUMN 5] as n1.n_name", + "[COLUMN 6] as weight_string(supp_nation)", + "[COLUMN 7] as weight_string(l_year)" + ], "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,R:0,L:3,L:4,L:6,L:7,L:8,R:1,L:9,L:11,L:12,L:13,R:2,L:14,L:15,L:16,R:3,R:4,R:5,L:17", + "JoinColumnIndexes": "L:0,R:0,R:1,L:1,L:2,L:3,R:2,L:5", "JoinVars": { - "l_suppkey": 2 + "l_suppkey": 4 }, "TableName": "lineitem_orders_supplier_nation", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,L:4,L:5,L:2,L:0,L:1,L:1,L:4,L:8,L:9,L:10,L:10,L:11,L:6,R:1,L:12", - "JoinVars": { - "l_orderkey": 3 - }, - "TableName": "lineitem_orders", + "OperatorType": "Projection", + "Expressions": [ + "[COLUMN 0] * [COLUMN 1] as revenue", + "[COLUMN 2] as l_year", + "[COLUMN 3] as orders.o_custkey", + "[COLUMN 4] as n1.n_name", + "[COLUMN 5] as lineitem.l_suppkey", + "[COLUMN 6] as weight_string(l_year)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(shipping.`lineitem.l_orderkey`), weight_string(shipping.`lineitem.l_suppkey`), weight_string(shipping.`orders.o_custkey`), weight_string(shipping.`n1.n_name`), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where 1 != 1 group by shipping.`lineitem.l_orderkey`, weight_string(shipping.`lineitem.l_orderkey`), shipping.`lineitem.l_suppkey`, weight_string(shipping.`lineitem.l_suppkey`), shipping.`orders.o_custkey`, weight_string(shipping.`orders.o_custkey`), shipping.`n1.n_name`, weight_string(shipping.`n1.n_name`), l_year, weight_string(l_year)", - "Query": "select shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, sum(volume) as revenue, weight_string(shipping.`lineitem.l_orderkey`), weight_string(shipping.`lineitem.l_suppkey`), weight_string(shipping.`orders.o_custkey`), weight_string(shipping.`n1.n_name`), weight_string(extract(year from l_shipdate)), weight_string(extract(year from l_shipdate)) from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31') group by shipping.`lineitem.l_orderkey`, weight_string(shipping.`lineitem.l_orderkey`), shipping.`lineitem.l_suppkey`, weight_string(shipping.`lineitem.l_suppkey`), shipping.`orders.o_custkey`, weight_string(shipping.`orders.o_custkey`), shipping.`n1.n_name`, weight_string(shipping.`n1.n_name`), l_year, weight_string(l_year)", - "Table": "lineitem" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,L:1,L:2,L:3,L:4,L:6", + "JoinVars": { + "l_orderkey": 5 }, - "FieldQuery": "select 1, count(*) from orders where 1 != 1 group by 1", - "Query": "select 1, count(*) from orders where o_orderkey = :l_orderkey group by 1", - "Table": "orders", - "Values": [ - ":l_orderkey" - ], - "Vindex": "hash" + "TableName": "lineitem_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(volume) as revenue, l_year, shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, weight_string(l_year), supp_nation, weight_string(supp_nation), cust_nation, weight_string(cust_nation) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, orders.o_custkey as `orders.o_custkey`, lineitem.l_suppkey as `lineitem.l_suppkey`, lineitem.l_orderkey as `lineitem.l_orderkey` from lineitem where 1 != 1) as shipping where 1 != 1 group by l_year, shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, weight_string(l_year)", + "OrderBy": "(7|8) ASC, (9|10) ASC, (1|6) ASC", + "Query": "select sum(volume) as revenue, l_year, shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, weight_string(l_year), supp_nation, weight_string(supp_nation), cust_nation, weight_string(cust_nation) from (select extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume, orders.o_custkey as `orders.o_custkey`, lineitem.l_suppkey as `lineitem.l_suppkey`, lineitem.l_orderkey as `lineitem.l_orderkey` from lineitem where l_shipdate between date('1995-01-01') and date('1996-12-31')) as shipping group by l_year, shipping.`orders.o_custkey`, shipping.`n1.n_name`, shipping.`lineitem.l_suppkey`, shipping.`lineitem.l_orderkey`, weight_string(l_year) order by supp_nation asc, cust_nation asc, l_year asc", + "Table": "lineitem" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*) from orders where 1 != 1 group by .0", + "Query": "select count(*) from orders where o_orderkey = :l_orderkey group by .0", + "Table": "orders", + "Values": [ + ":l_orderkey" + ], + "Vindex": "hash" + } + ] } ] }, { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:0,R:2,L:1,R:1,R:3", - "JoinVars": { - "s_nationkey": 0 - }, - "TableName": "supplier_nation", + "OperatorType": "Projection", + "Expressions": [ + "[COLUMN 0] * [COLUMN 1] as count(*)", + "[COLUMN 2] as supp_nation", + "[COLUMN 3] as weight_string(supp_nation)" + ], "Inputs": [ { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true - }, - "FieldQuery": "select shipping.`supplier.s_nationkey`, count(*), weight_string(shipping.`supplier.s_nationkey`) from supplier where 1 != 1 group by shipping.`supplier.s_nationkey`, weight_string(shipping.`supplier.s_nationkey`)", - "Query": "select shipping.`supplier.s_nationkey`, count(*), weight_string(shipping.`supplier.s_nationkey`) from supplier where s_suppkey = :l_suppkey group by shipping.`supplier.s_nationkey`, weight_string(shipping.`supplier.s_nationkey`)", - "Table": "supplier", - "Values": [ - ":l_suppkey" - ], - "Vindex": "hash" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "main", - "Sharded": true + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", + "JoinVars": { + "s_nationkey": 1 }, - "FieldQuery": "select n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where 1 != 1 group by supp_nation, weight_string(supp_nation)", - "Query": "select n1.n_name as supp_nation, count(*), weight_string(n1.n_name), weight_string(n1.n_name) from nation as n1 where n1.n_nationkey = :s_nationkey group by supp_nation, weight_string(supp_nation)", - "Table": "nation", - "Values": [ - ":s_nationkey" - ], - "Vindex": "hash" + "TableName": "supplier_nation", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), shipping.`supplier.s_nationkey` from (select supplier.s_nationkey as `supplier.s_nationkey` from supplier where 1 != 1) as shipping where 1 != 1 group by shipping.`supplier.s_nationkey`", + "Query": "select count(*), shipping.`supplier.s_nationkey` from (select supplier.s_nationkey as `supplier.s_nationkey` from supplier where s_suppkey = :l_suppkey) as shipping group by shipping.`supplier.s_nationkey`", + "Table": "supplier", + "Values": [ + ":l_suppkey" + ], + "Vindex": "hash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), supp_nation, weight_string(supp_nation) from (select n1.n_name as supp_nation from nation as n1 where 1 != 1) as shipping where 1 != 1 group by supp_nation, weight_string(supp_nation)", + "Query": "select count(*), supp_nation, weight_string(supp_nation) from (select n1.n_name as supp_nation from nation as n1 where n1.n_nationkey = :s_nationkey) as shipping group by supp_nation, weight_string(supp_nation)", + "Table": "nation", + "Values": [ + ":s_nationkey" + ], + "Vindex": "hash" + } + ] } ] } ] - }, + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + "[COLUMN 0] * [COLUMN 1] as count(*)", + "[COLUMN 2] as cust_nation", + "[COLUMN 3] as weight_string(cust_nation)" + ], + "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "R:0,R:0,R:2,L:1,R:1,R:3", + "JoinColumnIndexes": "L:0,R:0,R:1,R:2", "JoinVars": { - "c_nationkey": 0 + "c_nationkey": 1 }, "TableName": "customer_nation", "Inputs": [ @@ -688,8 +729,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select shipping.`customer.c_nationkey`, count(*), weight_string(shipping.`customer.c_nationkey`) from customer where 1 != 1 group by shipping.`customer.c_nationkey`, weight_string(shipping.`customer.c_nationkey`)", - "Query": "select shipping.`customer.c_nationkey`, count(*), weight_string(shipping.`customer.c_nationkey`) from customer where c_custkey = :o_custkey group by shipping.`customer.c_nationkey`, weight_string(shipping.`customer.c_nationkey`)", + "FieldQuery": "select count(*), shipping.`customer.c_nationkey` from (select customer.c_nationkey as `customer.c_nationkey` from customer where 1 != 1) as shipping where 1 != 1 group by shipping.`customer.c_nationkey`", + "Query": "select count(*), shipping.`customer.c_nationkey` from (select customer.c_nationkey as `customer.c_nationkey` from customer where c_custkey = :o_custkey) as shipping group by shipping.`customer.c_nationkey`", "Table": "customer", "Values": [ ":o_custkey" @@ -703,8 +744,8 @@ "Name": "main", "Sharded": true }, - "FieldQuery": "select n2.n_name as cust_nation, count(*), weight_string(n2.n_name), weight_string(n2.n_name) from nation as n2 where 1 != 1 group by cust_nation, weight_string(cust_nation)", - "Query": "select n2.n_name as cust_nation, count(*), weight_string(n2.n_name), weight_string(n2.n_name) from nation as n2 where (:n1_n_name = 'FRANCE' and n2.n_name = 'GERMANY' or :n1_n_name = 'GERMANY' and n2.n_name = 'FRANCE') and n2.n_nationkey = :c_nationkey group by cust_nation, weight_string(cust_nation)", + "FieldQuery": "select count(*), cust_nation, weight_string(cust_nation) from (select n2.n_name as cust_nation from nation as n2 where 1 != 1) as shipping where 1 != 1 group by cust_nation, weight_string(cust_nation)", + "Query": "select count(*), cust_nation, weight_string(cust_nation) from (select n2.n_name as cust_nation from nation as n2 where (:n1_n_name = 'FRANCE' and n2.n_name = 'GERMANY' or :n1_n_name = 'GERMANY' and n2.n_name = 'FRANCE') and n2.n_nationkey = :c_nationkey) as shipping group by cust_nation, weight_string(cust_nation)", "Table": "nation", "Values": [ ":c_nationkey" From 8aeb0ddb61289424604deeb3b32137852562e7c5 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 1 Jul 2023 10:59:39 +0200 Subject: [PATCH 23/26] add ordering bottom up so the order can be re-used Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/operators/phases.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/planbuilder/operators/phases.go b/go/vt/vtgate/planbuilder/operators/phases.go index b1155903dd0..4a8c1ca0067 100644 --- a/go/vt/vtgate/planbuilder/operators/phases.go +++ b/go/vt/vtgate/planbuilder/operators/phases.go @@ -88,7 +88,7 @@ func addOrderBysForAggregations(ctx *plancontext.PlanningContext, root ops.Opera return in, rewrite.NewTree("added ordering before aggregation", in), nil } - return rewrite.TopDown(root, TableID, visitor, stopAtRoute) + return rewrite.BottomUp(root, TableID, visitor, stopAtRoute) } func needsOrdering(ctx *plancontext.PlanningContext, in *Aggregator) (bool, error) { From 14c7bceca40ea724537356a74d24771c8bf8452d Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Sat, 1 Jul 2023 12:52:17 +0200 Subject: [PATCH 24/26] unify Derived and Horizon into a single struct Signed-off-by: Andres Taylor --- .../planbuilder/operator_transformers.go | 9 +- .../planbuilder/operators/SQL_builder.go | 11 +- .../planbuilder/operators/aggregator.go | 2 +- go/vt/vtgate/planbuilder/operators/ast2op.go | 8 +- go/vt/vtgate/planbuilder/operators/derived.go | 253 ------------------ go/vt/vtgate/planbuilder/operators/horizon.go | 216 ++++++++++++--- .../operators/horizon_expanding.go | 19 +- .../planbuilder/operators/horizon_planning.go | 21 +- .../planbuilder/operators/offset_planning.go | 2 +- .../planbuilder/operators/route_planning.go | 15 +- go/vt/vtgate/planbuilder/operators/union.go | 2 +- .../planbuilder/testdata/union_cases.json | 4 +- 12 files changed, 224 insertions(+), 338 deletions(-) delete mode 100644 go/vt/vtgate/planbuilder/operators/derived.go diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 1b599ec2f78..307936d3c08 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -51,8 +51,6 @@ func transformToLogicalPlan(ctx *plancontext.PlanningContext, op ops.Operator, i return transformSubQueryPlan(ctx, op) case *operators.CorrelatedSubQueryOp: return transformCorrelatedSubQueryPlan(ctx, op) - case *operators.Derived: - return transformDerivedPlan(ctx, op) case *operators.Filter: return transformFilter(ctx, op) case *operators.Horizon: @@ -271,11 +269,14 @@ func transformFilter(ctx *plancontext.PlanningContext, op *operators.Filter) (lo } func transformHorizon(ctx *plancontext.PlanningContext, op *operators.Horizon, isRoot bool) (logicalPlan, error) { + if op.IsDerived() { + return transformDerivedPlan(ctx, op) + } source, err := transformToLogicalPlan(ctx, op.Source, isRoot) if err != nil { return nil, err } - switch node := op.Select.(type) { + switch node := op.Query.(type) { case *sqlparser.Select: hp := horizonPlanning{ sel: node, @@ -862,7 +863,7 @@ func getCollationsFor(ctx *plancontext.PlanningContext, n *operators.Union) []co return colls } -func transformDerivedPlan(ctx *plancontext.PlanningContext, op *operators.Derived) (logicalPlan, error) { +func transformDerivedPlan(ctx *plancontext.PlanningContext, op *operators.Horizon) (logicalPlan, error) { // transforming the inner part of the derived table into a logical plan // so that we can do horizon planning on the inner. If the logical plan // we've produced is a Route, we set its Select.From field to be an aliased diff --git a/go/vt/vtgate/planbuilder/operators/SQL_builder.go b/go/vt/vtgate/planbuilder/operators/SQL_builder.go index 07fa5fbbd9d..573f101471c 100644 --- a/go/vt/vtgate/planbuilder/operators/SQL_builder.go +++ b/go/vt/vtgate/planbuilder/operators/SQL_builder.go @@ -246,7 +246,7 @@ func (ts *tableSorter) Swap(i, j int) { } func (h *Horizon) toSQL(qb *queryBuilder) error { - err := stripDownQuery(h.Select, qb.sel) + err := stripDownQuery(h.Query, qb.sel) if err != nil { return err } @@ -318,9 +318,10 @@ func buildQuery(op ops.Operator, qb *queryBuilder) error { return buildApplyJoin(op, qb) case *Filter: return buildFilter(op, qb) - case *Derived: - return buildDerived(op, qb) case *Horizon: + if op.TableId != nil { + return buildDerived(op, qb) + } return buildHorizon(op, qb) case *Limit: return buildLimit(op, qb) @@ -457,7 +458,7 @@ func buildFilter(op *Filter, qb *queryBuilder) error { return nil } -func buildDerived(op *Derived, qb *queryBuilder) error { +func buildDerived(op *Horizon, qb *queryBuilder) error { err := buildQuery(op.Source, qb) if err != nil { return err @@ -486,7 +487,7 @@ func buildHorizon(op *Horizon, qb *queryBuilder) error { return err } - err = stripDownQuery(op.Select, qb.sel) + err = stripDownQuery(op.Query, qb.sel) if err != nil { return err } diff --git a/go/vt/vtgate/planbuilder/operators/aggregator.go b/go/vt/vtgate/planbuilder/operators/aggregator.go index abd9c261e18..36602e024d4 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregator.go +++ b/go/vt/vtgate/planbuilder/operators/aggregator.go @@ -203,7 +203,7 @@ func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser } func (a *Aggregator) GetColumns() ([]*sqlparser.AliasedExpr, error) { - if _, isSourceDerived := a.Source.(*Derived); isSourceDerived { + if _, isSourceDerived := a.Source.(*Horizon); isSourceDerived { return a.Columns, nil } diff --git a/go/vt/vtgate/planbuilder/operators/ast2op.go b/go/vt/vtgate/planbuilder/operators/ast2op.go index 6b368f78fce..59223f0e631 100644 --- a/go/vt/vtgate/planbuilder/operators/ast2op.go +++ b/go/vt/vtgate/planbuilder/operators/ast2op.go @@ -81,7 +81,7 @@ func createOperatorFromSelect(ctx *plancontext.PlanningContext, sel *sqlparser.S return &Horizon{ Source: op, - Select: sel, + Query: sel, }, nil } @@ -104,7 +104,7 @@ func createOperatorFromUnion(ctx *plancontext.PlanningContext, node *sqlparser.U Distinct: node.Distinct, Sources: []ops.Operator{opLHS, opRHS}, } - return &Horizon{Source: union, Select: node}, nil + return &Horizon{Source: union, Query: node}, nil } func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlparser.Update) (ops.Operator, error) { @@ -606,8 +606,8 @@ func getOperatorFromAliasedTableExpr(ctx *plancontext.PlanningContext, tableExpr return nil, err } - return &Derived{ - TableId: tableID, + return &Horizon{ + TableId: &tableID, Alias: tableExpr.As.String(), Source: inner, Query: stmt, diff --git a/go/vt/vtgate/planbuilder/operators/derived.go b/go/vt/vtgate/planbuilder/operators/derived.go deleted file mode 100644 index e18240b395b..00000000000 --- a/go/vt/vtgate/planbuilder/operators/derived.go +++ /dev/null @@ -1,253 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operators - -import ( - "golang.org/x/exp/slices" - - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" - "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - "vitess.io/vitess/go/vt/vtgate/semantics" -) - -type Derived struct { - Source ops.Operator - TableId semantics.TableSet - - // QP contains the QueryProjection for this op - QP *QueryProjection - - Query sqlparser.SelectStatement - Alias string - ColumnAliases sqlparser.Columns - - // Columns needed to feed other plans - Columns []*sqlparser.ColName - ColumnsOffset []int -} - -// Clone implements the Operator interface -func (d *Derived) Clone(inputs []ops.Operator) ops.Operator { - return &Derived{ - Source: inputs[0], - Query: d.Query, - Alias: d.Alias, - ColumnAliases: sqlparser.CloneColumns(d.ColumnAliases), - Columns: slices.Clone(d.Columns), - ColumnsOffset: slices.Clone(d.ColumnsOffset), - TableId: d.TableId, - QP: d.QP, - } -} - -// findOutputColumn returns the index on which the given name is found in the slice of -// *sqlparser.SelectExprs of the derivedTree. The *sqlparser.SelectExpr must be of type -// *sqlparser.AliasedExpr and match the given name. -// If name is not present but the query's select expressions contain a *sqlparser.StarExpr -// the function will return no error and an index equal to -1. -// If name is not present and the query does not have a *sqlparser.StarExpr, the function -// will return an unknown column error. -func (d *Derived) findOutputColumn(name *sqlparser.ColName) (int, error) { - hasStar := false - for j, exp := range sqlparser.GetFirstSelect(d.Query).SelectExprs { - switch exp := exp.(type) { - case *sqlparser.AliasedExpr: - if !exp.As.IsEmpty() && exp.As.Equal(name.Name) { - return j, nil - } - if exp.As.IsEmpty() { - col, ok := exp.Expr.(*sqlparser.ColName) - if !ok { - return 0, vterrors.VT12001("complex expression needs column alias: %s", sqlparser.String(exp)) - } - if name.Name.Equal(col.Name) { - return j, nil - } - } - case *sqlparser.StarExpr: - hasStar = true - } - } - - // we have found a star but no matching *sqlparser.AliasedExpr, thus we return -1 with no error. - if hasStar { - return -1, nil - } - return 0, vterrors.VT03014(name.Name.String(), "field list") -} - -// IsMergeable is not a great name for this function. Suggestions for a better one are welcome! -// This function will return false if the derived table inside it has to run on the vtgate side, and so can't be merged with subqueries -// This logic can also be used to check if this is a derived table that can be had on the left hand side of a vtgate join. -// Since vtgate joins are always nested loop joins, we can't execute them on the RHS -// if they do some things, like LIMIT or GROUP BY on wrong columns -func (d *Derived) IsMergeable(ctx *plancontext.PlanningContext) bool { - return isMergeable(ctx, d.Query, d) -} - -// Inputs implements the Operator interface -func (d *Derived) Inputs() []ops.Operator { - return []ops.Operator{d.Source} -} - -// SetInputs implements the Operator interface -func (d *Derived) SetInputs(ops []ops.Operator) { - d.Source = ops[0] -} - -func (d *Derived) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (ops.Operator, error) { - if _, isUNion := d.Source.(*Union); isUNion { - // If we have a derived table on top of a UNION, we can let the UNION do the expression rewriting - var err error - d.Source, err = d.Source.AddPredicate(ctx, expr) - return d, err - } - tableInfo, err := ctx.SemTable.TableInfoForExpr(expr) - if err != nil { - if err == semantics.ErrNotSingleTable { - return &Filter{ - Source: d, - Predicates: []sqlparser.Expr{expr}, - }, nil - } - return nil, err - } - - newExpr := semantics.RewriteDerivedTableExpression(expr, tableInfo) - if sqlparser.ContainsAggregation(newExpr) { - return &Filter{Source: d, Predicates: []sqlparser.Expr{expr}}, nil - } - d.Source, err = d.Source.AddPredicate(ctx, newExpr) - if err != nil { - return nil, err - } - return d, nil -} - -func (d *Derived) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { - col, ok := expr.Expr.(*sqlparser.ColName) - if !ok { - return nil, 0, vterrors.VT13001("cannot push non-colname expression to a derived table") - } - - identity := func(c *sqlparser.ColName) sqlparser.Expr { return c } - if offset, found := canReuseColumn(ctx, d.Columns, col, identity); found { - return d, offset, nil - } - - i, err := d.findOutputColumn(col) - if err != nil { - return nil, 0, err - } - var pos int - d.ColumnsOffset, pos = addToIntSlice(d.ColumnsOffset, i) - - d.Columns = append(d.Columns, col) - // add it to the source if we were not already passing it through - if i <= -1 { - newSrc, _, err := d.Source.AddColumn(ctx, aeWrap(sqlparser.NewColName(col.Name.String())), true, addToGroupBy) - if err != nil { - return nil, 0, err - } - d.Source = newSrc - } - return d, pos, nil -} - -// canReuseColumn is generic, so it can be used with slices of different types. -// We don't care about the actual type, as long as we know it's a sqlparser.Expr -func canReuseColumn[T any]( - ctx *plancontext.PlanningContext, - columns []T, - col sqlparser.Expr, - f func(T) sqlparser.Expr, -) (offset int, found bool) { - for offset, column := range columns { - if ctx.SemTable.EqualsExprWithDeps(col, f(column)) { - return offset, true - } - } - - return -} - -func (d *Derived) GetColumns() (exprs []*sqlparser.AliasedExpr, err error) { - for _, expr := range sqlparser.GetFirstSelect(d.Query).SelectExprs { - ae, ok := expr.(*sqlparser.AliasedExpr) - if !ok { - return nil, vterrors.VT09015() - } - exprs = append(exprs, ae) - } - return -} - -func (d *Derived) GetSelectExprs() (sqlparser.SelectExprs, error) { - return sqlparser.GetFirstSelect(d.Query).SelectExprs, nil -} - -func (d *Derived) GetOrdering() ([]ops.OrderBy, error) { - if d.QP == nil { - return nil, vterrors.VT13001("QP should already be here") - } - return d.QP.OrderExprs, nil -} - -func addToIntSlice(columnOffset []int, valToAdd int) ([]int, int) { - for idx, val := range columnOffset { - if val == valToAdd { - return columnOffset, idx - } - } - columnOffset = append(columnOffset, valToAdd) - return columnOffset, len(columnOffset) - 1 -} - -// TODO: REMOVE -func (d *Derived) selectStatement() sqlparser.SelectStatement { - return d.Query -} - -func (d *Derived) src() ops.Operator { - return d.Source -} - -func (d *Derived) getQP(ctx *plancontext.PlanningContext) (*QueryProjection, error) { - if d.QP != nil { - return d.QP, nil - } - qp, err := CreateQPFromSelectStatement(ctx, d.Query) - if err != nil { - return nil, err - } - d.QP = qp - return d.QP, nil -} - -func (d *Derived) setQP(qp *QueryProjection) { - d.QP = qp -} - -func (d *Derived) ShortDescription() string { - return d.Alias -} - -func (d *Derived) introducesTableID() semantics.TableSet { - return d.TableId -} diff --git a/go/vt/vtgate/planbuilder/operators/horizon.go b/go/vt/vtgate/planbuilder/operators/horizon.go index 719c0330a6c..d95e3244cff 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon.go +++ b/go/vt/vtgate/planbuilder/operators/horizon.go @@ -17,10 +17,13 @@ limitations under the License. package operators import ( + "golang.org/x/exp/slices" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" + "vitess.io/vitess/go/vt/vtgate/semantics" ) // Horizon is an operator that allows us to postpone planning things like SELECT/GROUP BY/ORDER BY/LIMIT until later. @@ -32,62 +35,180 @@ import ( // and some that have to be evaluated at the vtgate level. type Horizon struct { Source ops.Operator - Select sqlparser.SelectStatement - QP *QueryProjection + + // If this is a derived table, the two following fields will contain the tableID and name of it + TableId *semantics.TableSet + Alias string + + // QP contains the QueryProjection for this op + QP *QueryProjection + + Query sqlparser.SelectStatement + ColumnAliases sqlparser.Columns + + // Columns needed to feed other plans + Columns []*sqlparser.ColName + ColumnsOffset []int } -func (h *Horizon) AddColumn(*plancontext.PlanningContext, *sqlparser.AliasedExpr, bool, bool) (ops.Operator, int, error) { - return nil, 0, vterrors.VT13001("the Horizon operator cannot accept new columns") +// Clone implements the Operator interface +func (h *Horizon) Clone(inputs []ops.Operator) ops.Operator { + return &Horizon{ + Source: inputs[0], + Query: h.Query, + Alias: h.Alias, + ColumnAliases: sqlparser.CloneColumns(h.ColumnAliases), + Columns: slices.Clone(h.Columns), + ColumnsOffset: slices.Clone(h.ColumnsOffset), + TableId: h.TableId, + QP: h.QP, + } } -func (h *Horizon) GetColumns() (exprs []*sqlparser.AliasedExpr, err error) { - for _, expr := range sqlparser.GetFirstSelect(h.Select).SelectExprs { - ae, ok := expr.(*sqlparser.AliasedExpr) - if !ok { - return nil, vterrors.VT09015() +// findOutputColumn returns the index on which the given name is found in the slice of +// *sqlparser.SelectExprs of the derivedTree. The *sqlparser.SelectExpr must be of type +// *sqlparser.AliasedExpr and match the given name. +// If name is not present but the query's select expressions contain a *sqlparser.StarExpr +// the function will return no error and an index equal to -1. +// If name is not present and the query does not have a *sqlparser.StarExpr, the function +// will return an unknown column error. +func (h *Horizon) findOutputColumn(name *sqlparser.ColName) (int, error) { + hasStar := false + for j, exp := range sqlparser.GetFirstSelect(h.Query).SelectExprs { + switch exp := exp.(type) { + case *sqlparser.AliasedExpr: + if !exp.As.IsEmpty() && exp.As.Equal(name.Name) { + return j, nil + } + if exp.As.IsEmpty() { + col, ok := exp.Expr.(*sqlparser.ColName) + if !ok { + return 0, vterrors.VT12001("complex expression needs column alias: %s", sqlparser.String(exp)) + } + if name.Name.Equal(col.Name) { + return j, nil + } + } + case *sqlparser.StarExpr: + hasStar = true } - exprs = append(exprs, ae) } - return + + // we have found a star but no matching *sqlparser.AliasedExpr, thus we return -1 with no error. + if hasStar { + return -1, nil + } + return 0, vterrors.VT03014(name.Name.String(), "field list") } -func (h *Horizon) GetSelectExprs() (sqlparser.SelectExprs, error) { - return sqlparser.GetFirstSelect(h.Select).SelectExprs, nil +// IsMergeable is not a great name for this function. Suggestions for a better one are welcome! +// This function will return false if the derived table inside it has to run on the vtgate side, and so can't be merged with subqueries +// This logic can also be used to check if this is a derived table that can be had on the left hand side of a vtgate join. +// Since vtgate joins are always nested loop joins, we can't execute them on the RHS +// if they do some things, like LIMIT or GROUP BY on wrong columns +func (h *Horizon) IsMergeable(ctx *plancontext.PlanningContext) bool { + return isMergeable(ctx, h.Query, h) +} + +// Inputs implements the Operator interface +func (h *Horizon) Inputs() []ops.Operator { + return []ops.Operator{h.Source} } -var _ ops.Operator = (*Horizon)(nil) +// SetInputs implements the Operator interface +func (h *Horizon) SetInputs(ops []ops.Operator) { + h.Source = ops[0] +} func (h *Horizon) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (ops.Operator, error) { - newSrc, err := h.Source.AddPredicate(ctx, expr) + if _, isUNion := h.Source.(*Union); isUNion { + // If we have a derived table on top of a UNION, we can let the UNION do the expression rewriting + var err error + h.Source, err = h.Source.AddPredicate(ctx, expr) + return h, err + } + tableInfo, err := ctx.SemTable.TableInfoForExpr(expr) + if err != nil { + if err == semantics.ErrNotSingleTable { + return &Filter{ + Source: h, + Predicates: []sqlparser.Expr{expr}, + }, nil + } + return nil, err + } + + newExpr := semantics.RewriteDerivedTableExpression(expr, tableInfo) + if sqlparser.ContainsAggregation(newExpr) { + return &Filter{Source: h, Predicates: []sqlparser.Expr{expr}}, nil + } + h.Source, err = h.Source.AddPredicate(ctx, newExpr) if err != nil { return nil, err } - h.Source = newSrc return h, nil } -func (h *Horizon) Clone(inputs []ops.Operator) ops.Operator { - return &Horizon{ - Source: inputs[0], - Select: h.Select, +func (h *Horizon) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) { + col, ok := expr.Expr.(*sqlparser.ColName) + if !ok { + return nil, 0, vterrors.VT13001("cannot push non-colname expression to a derived table") } -} -func (h *Horizon) Inputs() []ops.Operator { - return []ops.Operator{h.Source} + identity := func(c *sqlparser.ColName) sqlparser.Expr { return c } + if offset, found := canReuseColumn(ctx, h.Columns, col, identity); found { + return h, offset, nil + } + + i, err := h.findOutputColumn(col) + if err != nil { + return nil, 0, err + } + var pos int + h.ColumnsOffset, pos = addToIntSlice(h.ColumnsOffset, i) + + h.Columns = append(h.Columns, col) + // add it to the source if we were not already passing it through + if i <= -1 { + newSrc, _, err := h.Source.AddColumn(ctx, aeWrap(sqlparser.NewColName(col.Name.String())), true, addToGroupBy) + if err != nil { + return nil, 0, err + } + h.Source = newSrc + } + return h, pos, nil } -// SetInputs implements the Operator interface -func (h *Horizon) SetInputs(ops []ops.Operator) { - h.Source = ops[0] +// canReuseColumn is generic, so it can be used with slices of different types. +// We don't care about the actual type, as long as we know it's a sqlparser.Expr +func canReuseColumn[T any]( + ctx *plancontext.PlanningContext, + columns []T, + col sqlparser.Expr, + f func(T) sqlparser.Expr, +) (offset int, found bool) { + for offset, column := range columns { + if ctx.SemTable.EqualsExprWithDeps(col, f(column)) { + return offset, true + } + } + + return } -func (h *Horizon) selectStatement() sqlparser.SelectStatement { - return h.Select +func (h *Horizon) GetColumns() (exprs []*sqlparser.AliasedExpr, err error) { + for _, expr := range sqlparser.GetFirstSelect(h.Query).SelectExprs { + ae, ok := expr.(*sqlparser.AliasedExpr) + if !ok { + return nil, vterrors.VT09015() + } + exprs = append(exprs, ae) + } + return } -func (h *Horizon) src() ops.Operator { - return h.Source +func (h *Horizon) GetSelectExprs() (sqlparser.SelectExprs, error) { + return sqlparser.GetFirstSelect(h.Query).SelectExprs, nil } func (h *Horizon) GetOrdering() ([]ops.OrderBy, error) { @@ -97,11 +218,30 @@ func (h *Horizon) GetOrdering() ([]ops.OrderBy, error) { return h.QP.OrderExprs, nil } +func addToIntSlice(columnOffset []int, valToAdd int) ([]int, int) { + for idx, val := range columnOffset { + if val == valToAdd { + return columnOffset, idx + } + } + columnOffset = append(columnOffset, valToAdd) + return columnOffset, len(columnOffset) - 1 +} + +// TODO: REMOVE +func (h *Horizon) selectStatement() sqlparser.SelectStatement { + return h.Query +} + +func (h *Horizon) src() ops.Operator { + return h.Source +} + func (h *Horizon) getQP(ctx *plancontext.PlanningContext) (*QueryProjection, error) { if h.QP != nil { return h.QP, nil } - qp, err := CreateQPFromSelectStatement(ctx, h.Select) + qp, err := CreateQPFromSelectStatement(ctx, h.Query) if err != nil { return nil, err } @@ -114,5 +254,17 @@ func (h *Horizon) setQP(qp *QueryProjection) { } func (h *Horizon) ShortDescription() string { - return "" + return h.Alias +} + +func (h *Horizon) introducesTableID() semantics.TableSet { + if h.TableId == nil { + return semantics.EmptyTableSet() + } + + return *h.TableId +} + +func (h *Horizon) IsDerived() bool { + return h.TableId != nil } diff --git a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go index a03c9a95904..f12015e7d7c 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_expanding.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_expanding.go @@ -26,7 +26,7 @@ import ( "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) -func expandHorizon(ctx *plancontext.PlanningContext, horizon horizonLike) (ops.Operator, *rewrite.ApplyResult, error) { +func expandHorizon(ctx *plancontext.PlanningContext, horizon *Horizon) (ops.Operator, *rewrite.ApplyResult, error) { sel, isSel := horizon.selectStatement().(*sqlparser.Select) if !isSel { return nil, nil, errHorizonNotPlanned() @@ -74,7 +74,7 @@ func expandHorizon(ctx *plancontext.PlanningContext, horizon horizonLike) (ops.O return op, rewrite.NewTree("expand horizon into smaller components", op), nil } -func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizonLike) (out ops.Operator, err error) { +func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon *Horizon) (out ops.Operator, err error) { qp, err := horizon.getQP(ctx) if err != nil { return nil, err @@ -85,11 +85,8 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizo if err != nil { return nil, err } - if derived, isDerived := horizon.(*Derived); isDerived { - id := derived.TableId - projX.TableID = &id - projX.Alias = derived.Alias - } + projX.TableID = horizon.TableId + projX.Alias = horizon.Alias out = projX return out, nil @@ -106,12 +103,8 @@ func createProjectionFromSelect(ctx *plancontext.PlanningContext, horizon horizo QP: qp, Grouping: qp.GetGrouping(), Aggregations: aggregations, - } - - if derived, isDerived := horizon.(*Derived); isDerived { - id := derived.TableId - a.TableID = &id - a.Alias = derived.Alias + TableID: horizon.TableId, + Alias: horizon.Alias, } if complexAggr { diff --git a/go/vt/vtgate/planbuilder/operators/horizon_planning.go b/go/vt/vtgate/planbuilder/operators/horizon_planning.go index 4ff4cfeb7dc..dd6d5bc8898 100644 --- a/go/vt/vtgate/planbuilder/operators/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/operators/horizon_planning.go @@ -33,14 +33,6 @@ type ( cols []ProjExpr names []*sqlparser.AliasedExpr } - - // horizonLike should be removed. we should use Horizon for both these cases - horizonLike interface { - ops.Operator - selectStatement() sqlparser.SelectStatement - src() ops.Operator - getQP(ctx *plancontext.PlanningContext) (*QueryProjection, error) - } ) func errHorizonNotPlanned() error { @@ -120,7 +112,7 @@ func planHorizons(ctx *plancontext.PlanningContext, root ops.Operator) (op ops.O func optimizeHorizonPlanning(ctx *plancontext.PlanningContext, root ops.Operator) (ops.Operator, error) { visitor := func(in ops.Operator, _ semantics.TableSet, isRoot bool) (ops.Operator, *rewrite.ApplyResult, error) { switch in := in.(type) { - case horizonLike: + case *Horizon: return pushOrExpandHorizon(ctx, in) case *Projection: return tryPushingDownProjection(ctx, in) @@ -151,12 +143,11 @@ func optimizeHorizonPlanning(ctx *plancontext.PlanningContext, root ops.Operator return newOp, nil } -func pushOrExpandHorizon(ctx *plancontext.PlanningContext, in horizonLike) (ops.Operator, *rewrite.ApplyResult, error) { - if derived, ok := in.(*Derived); ok { - if len(derived.ColumnAliases) > 0 { - return nil, nil, errHorizonNotPlanned() - } +func pushOrExpandHorizon(ctx *plancontext.PlanningContext, in *Horizon) (ops.Operator, *rewrite.ApplyResult, error) { + if len(in.ColumnAliases) > 0 { + return nil, nil, errHorizonNotPlanned() } + rb, isRoute := in.src().(*Route) if isRoute && rb.IsSingleShard() { return rewrite.Swap(in, rb, "push horizon into route") @@ -628,7 +619,7 @@ func makeSureOutputIsCorrect(ctx *plancontext.PlanningContext, oldHorizon ops.Op horizon := oldHorizon.(*Horizon) - sel := sqlparser.GetFirstSelect(horizon.Select) + sel := sqlparser.GetFirstSelect(horizon.Query) if len(sel.SelectExprs) == len(cols) { return output, nil diff --git a/go/vt/vtgate/planbuilder/operators/offset_planning.go b/go/vt/vtgate/planbuilder/operators/offset_planning.go index 42ec0fe606c..b94252d1af6 100644 --- a/go/vt/vtgate/planbuilder/operators/offset_planning.go +++ b/go/vt/vtgate/planbuilder/operators/offset_planning.go @@ -38,7 +38,7 @@ func planOffsets(ctx *plancontext.PlanningContext, root ops.Operator) (ops.Opera visitor := func(in ops.Operator, _ semantics.TableSet, _ bool) (ops.Operator, *rewrite.ApplyResult, error) { var err error switch op := in.(type) { - case *Derived, *Horizon: + case *Horizon: return nil, nil, vterrors.VT13001(fmt.Sprintf("should not see %T here", in)) case offsettable: err = op.planOffsets(ctx) diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index 7db75f5df07..ac08b1fef62 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -52,15 +52,16 @@ func transformToPhysical(ctx *plancontext.PlanningContext, in ops.Operator) (ops return optimizeQueryGraph(ctx, op) case *Join: return optimizeJoin(ctx, op) - case *Derived: - return pushDownDerived(ctx, op) + case *Horizon: + if op.TableId != nil { + return pushDownDerived(ctx, op) + } case *SubQuery: return optimizeSubQuery(ctx, op, ts) case *Filter: return pushDownFilter(op) - default: - return operator, rewrite.SameTree, nil } + return operator, rewrite.SameTree, nil }) if err != nil { @@ -79,7 +80,7 @@ func pushDownFilter(op *Filter) (ops.Operator, *rewrite.ApplyResult, error) { return op, rewrite.SameTree, nil } -func pushDownDerived(ctx *plancontext.PlanningContext, op *Derived) (ops.Operator, *rewrite.ApplyResult, error) { +func pushDownDerived(ctx *plancontext.PlanningContext, op *Horizon) (ops.Operator, *rewrite.ApplyResult, error) { innerRoute, ok := op.Source.(*Route) if !ok { return op, rewrite.SameTree, nil @@ -389,9 +390,9 @@ func requiresSwitchingSides(ctx *plancontext.PlanningContext, op ops.Operator) b required := false _ = rewrite.Visit(op, func(current ops.Operator) error { - derived, isDerived := current.(*Derived) + horizon, isHorizon := current.(*Horizon) - if isDerived && !derived.IsMergeable(ctx) { + if isHorizon && horizon.IsDerived() && !horizon.IsMergeable(ctx) { required = true return io.EOF } diff --git a/go/vt/vtgate/planbuilder/operators/union.go b/go/vt/vtgate/planbuilder/operators/union.go index 7a68007ff63..6568521383d 100644 --- a/go/vt/vtgate/planbuilder/operators/union.go +++ b/go/vt/vtgate/planbuilder/operators/union.go @@ -144,7 +144,7 @@ func (u *Union) GetSelectFor(source int) (*sqlparser.Select, error) { for { switch op := src.(type) { case *Horizon: - return sqlparser.GetFirstSelect(op.Select), nil + return sqlparser.GetFirstSelect(op.Query), nil case *Route: src = op.Source default: diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index ef76413b5c7..ed836bf207b 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -2361,7 +2361,7 @@ "Sharded": false }, "FieldQuery": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", - "Query": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.COLUMN_NAME = 'primary' union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.COLUMN_NAME = 'primary') as kcu", + "Query": "select kcu.COLUMN_NAME from (select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and COLUMN_NAME = 'primary' union select kcu.COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and COLUMN_NAME = 'primary') as kcu", "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", "Table": "information_schema.key_column_usage" @@ -2411,7 +2411,7 @@ "Sharded": false }, "FieldQuery": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1 union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where 1 != 1) as kcu where 1 != 1", - "Query": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary' union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and kcu.CONSTRAINT_NAME = 'primary') as kcu", + "Query": "select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from (select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name /* VARCHAR */ and CONSTRAINT_NAME = 'primary' union select kcu.CONSTRAINT_CATALOG, kcu.CONSTRAINT_SCHEMA, kcu.CONSTRAINT_NAME, kcu.TABLE_CATALOG, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, kcu.POSITION_IN_UNIQUE_CONSTRAINT, kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME from information_schema.key_column_usage as kcu where kcu.table_schema = :__vtschemaname /* VARCHAR */ and kcu.table_name = :kcu_table_name1 /* VARCHAR */ and CONSTRAINT_NAME = 'primary') as kcu", "SysTableTableName": "[kcu_table_name1:VARCHAR(\"music\"), kcu_table_name:VARCHAR(\"user_extra\")]", "SysTableTableSchema": "[VARCHAR(\"user\"), VARCHAR(\"user\")]", "Table": "information_schema.key_column_usage" From fb615426b188010e42f500c1e3a8e85de60ffd45 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 5 Jul 2023 10:01:03 +0200 Subject: [PATCH 25/26] refactor: aggregation-pushing Change the aggregation type when pushed down and not after the fact. Signed-off-by: Andres Taylor --- .../vtgate/planbuilder/operators/aggregation_pushing.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index 63a4b820284..a778bb694b1 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -39,16 +39,10 @@ func tryPushingDownAggregator(ctx *plancontext.PlanningContext, aggregator *Aggr case *ApplyJoin: if ctx.DelegateAggregation { output, applyResult, err = pushDownAggregationThroughJoin(ctx, aggregator, src) - if applyResult != rewrite.SameTree && aggregator.Original { - aggregator.aggregateTheAggregates() - } } case *Filter: if ctx.DelegateAggregation { output, applyResult, err = pushDownAggregationThroughFilter(ctx, aggregator, src) - if applyResult != rewrite.SameTree && aggregator.Original { - aggregator.aggregateTheAggregates() - } } default: return aggregator, rewrite.SameTree, nil @@ -183,7 +177,7 @@ withNextColumn: // by splitting one and pushing under a join, we can get rid of this one return aggregator.Source, rewrite.NewTree("push aggregation under filter - remove original", aggregator), nil } - + aggregator.aggregateTheAggregates() return aggregator, rewrite.NewTree("push aggregation under filter - keep original", aggregator), nil } @@ -331,6 +325,7 @@ func pushDownAggregationThroughJoin(ctx *plancontext.PlanningContext, rootAggr * return output, rewrite.NewTree("push Aggregation under join - keep original", rootAggr), nil } + rootAggr.aggregateTheAggregates() rootAggr.Source = output return rootAggr, rewrite.NewTree("push Aggregation under join", rootAggr), nil } From 15453dc2d735fbbd4be06b50a4cdbfafc6e3409f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 5 Jul 2023 10:15:35 +0200 Subject: [PATCH 26/26] add support handling sum(distinct x) and count(distinct x) on top of joins Signed-off-by: Andres Taylor --- .../operators/aggregation_pushing.go | 2 + .../planbuilder/testdata/aggr_cases.json | 88 ++++++++----------- 2 files changed, 41 insertions(+), 49 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go index a778bb694b1..cb506aac959 100644 --- a/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go +++ b/go/vt/vtgate/planbuilder/operators/aggregation_pushing.go @@ -508,6 +508,8 @@ func (ab *aggBuilder) handleAggr(ctx *plancontext.PlanningContext, aggr Aggr) er case opcode.AggregateGtid: // this is only used for SHOW GTID queries that will never contain joins return vterrors.VT13001("cannot do join with vgtid") + case opcode.AggregateSumDistinct, opcode.AggregateCountDistinct: + return errAbortAggrPushing default: return errHorizonNotPlanned() } diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json index 2966ab8a86a..aa54fa01776 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.json @@ -3829,59 +3829,34 @@ "ResultColumns": 2, "Inputs": [ { - "OperatorType": "Projection", - "Expressions": [ - "[COLUMN 0] as textcol1", - "[COLUMN 1] as val2", - "[COLUMN 2]" - ], + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,L:1,L:2", + "JoinVars": { + "u2_val2": 3 + }, + "TableName": "`user`_`user`_music", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:2,L:3,L:5", + "JoinColumnIndexes": "L:0,L:1,L:2,R:0", "JoinVars": { - "u2_val2": 0 + "u_val2": 1 }, - "TableName": "`user`_`user`_music", + "TableName": "`user`_`user`", "Inputs": [ { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "R:0,R:0,L:2,L:0,R:1,L:1", - "JoinVars": { - "u_val2": 0 + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true }, - "TableName": "`user`_`user`", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "Scatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select u.val2, weight_string(u.val2), u.textcol1 from `user` as u where 1 != 1 group by u.val2, weight_string(u.val2), u.textcol1", - "OrderBy": "2 ASC COLLATE latin1_swedish_ci, (0|1) ASC", - "Query": "select u.val2, weight_string(u.val2), u.textcol1 from `user` as u group by u.val2, weight_string(u.val2), u.textcol1 order by u.textcol1 asc, u.val2 asc", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "EqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select u2.val2, weight_string(u2.val2) from `user` as u2 where 1 != 1 group by u2.val2, weight_string(u2.val2)", - "Query": "select u2.val2, weight_string(u2.val2) from `user` as u2 where u2.id = :u_val2 group by u2.val2, weight_string(u2.val2)", - "Table": "`user`", - "Values": [ - ":u_val2" - ], - "Vindex": "user_index" - } - ] + "FieldQuery": "select u.textcol1, u.val2, weight_string(u.val2) from `user` as u where 1 != 1", + "OrderBy": "0 ASC COLLATE latin1_swedish_ci", + "Query": "select u.textcol1, u.val2, weight_string(u.val2) from `user` as u order by u.textcol1 asc", + "Table": "`user`" }, { "OperatorType": "Route", @@ -3890,15 +3865,30 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from music as m where 1 != 1", - "Query": "select 1 from music as m where m.id = :u2_val2", - "Table": "music", + "FieldQuery": "select u2.val2 from `user` as u2 where 1 != 1", + "Query": "select u2.val2 from `user` as u2 where u2.id = :u_val2", + "Table": "`user`", "Values": [ - ":u2_val2" + ":u_val2" ], - "Vindex": "music_user_map" + "Vindex": "user_index" } ] + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music as m where 1 != 1", + "Query": "select 1 from music as m where m.id = :u2_val2", + "Table": "music", + "Values": [ + ":u2_val2" + ], + "Vindex": "music_user_map" } ] }