Query: TPC prune unused concrete tables and columns (#27993)

- Add TpcTablesExpression which holds all union all tables so we can prune unused subqueries
- Also allow pruning of unused column inside each subquery
- Assign unique aliases in inner subqueries across whole SelectExpression
- Apply predicate on discriminator column directly to subqueries by removing unnecessary subqueries

Resolves #27967
Resolves #27957

src/EFCore.Relational/Query/Internal/TpcTablesExpression.cs

@@ -0,0 +1,150 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
+
+namespace Microsoft.EntityFrameworkCore.Query.Internal;
+
+/// <summary>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </summary>
+public class TpcTablesExpression : TableExpressionBase
+{
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public TpcTablesExpression(
+        string? alias, IEntityType entityType, IReadOnlyList<SelectExpression> subSelectExpressions)
+        : base(alias)
+    {
+        EntityType = entityType;
+        SelectExpressions = subSelectExpressions;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    private TpcTablesExpression(
+        string? alias,
+        IEntityType entityType,
+        IReadOnlyList<SelectExpression> subSelectExpressions,
+        IEnumerable<IAnnotation>? annotations)
+        : base(alias, annotations)
+    {
+        EntityType = entityType;
+        SelectExpressions = subSelectExpressions;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    [NotNull]
+    public override string? Alias
+    {
+        get => base.Alias!;
+        internal set => base.Alias = value;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public virtual IEntityType EntityType { get; }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public virtual IReadOnlyList<SelectExpression> SelectExpressions { get; }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public virtual TpcTablesExpression Prune(IReadOnlyList<string> discriminatorValues, IReadOnlyCollection<string>? referencedColumns)
+    {
+        var subSelectExpressions = discriminatorValues.Count == 0
+            ? new List<SelectExpression> { SelectExpressions[0] }
+            : SelectExpressions.Where(se =>
+                discriminatorValues.Contains((string)((SqlConstantExpression)se.Projection[^1].Expression).Value!)).ToList();
+
+        Check.DebugAssert(subSelectExpressions.Count > 0, "TPC must have at least 1 table selected.");
+
+        if (referencedColumns != null)
+        {
+            foreach (var se in subSelectExpressions)
+            {
+                se.Prune(referencedColumns);
+            }
+        }
+
+        return new TpcTablesExpression(Alias, EntityType, subSelectExpressions, GetAnnotations());
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    // This is implementation detail hence visitors are not supposed to see inside unless they really need to.
+    protected override Expression VisitChildren(ExpressionVisitor visitor) => this;
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    protected override void Print(ExpressionPrinter expressionPrinter)
+    {
+        expressionPrinter.AppendLine("(");
+        using (expressionPrinter.Indent())
+        {
+            expressionPrinter.VisitCollection(SelectExpressions, e => e.AppendLine().AppendLine("UNION ALL"));
+        }
+        expressionPrinter.AppendLine()
+            .AppendLine(") AS " + Alias);
+        PrintAnnotations(expressionPrinter);
+    }
+
+    /// <inheritdoc />
+    public override bool Equals(object? obj)
+        => obj != null
+            && (ReferenceEquals(this, obj)
+                || obj is TpcTablesExpression tpcTablesExpression
+                && Equals(tpcTablesExpression));
+
+    private bool Equals(TpcTablesExpression tpcTablesExpression)
+    {
+        if (!base.Equals(tpcTablesExpression)
+            || EntityType != tpcTablesExpression.EntityType)
+        {
+            return false;
+        }
+
+        return SelectExpressions.SequenceEqual(tpcTablesExpression.SelectExpressions);
+    }
+
+    /// <inheritdoc />
+    public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), EntityType);
+}

src/EFCore.Relational/Query/QuerySqlGenerator.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using Microsoft.EntityFrameworkCore.Query.Internal;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
 using Microsoft.EntityFrameworkCore.Storage.Internal;
 
@@ -132,11 +133,50 @@ private static bool IsNonComposedSetOperation(SelectExpression selectExpression)
             && selectExpression.Projection.Count == setOperation.Source1.Projection.Count
             && selectExpression.Projection.Select(
                     (pe, index) => pe.Expression is ColumnExpression column
-                        && string.Equals(column.TableAlias, setOperation.Alias, StringComparison.OrdinalIgnoreCase)
+                        && string.Equals(column.TableAlias, setOperation.Alias, StringComparison.Ordinal)
                         && string.Equals(
-                            column.Name, setOperation.Source1.Projection[index].Alias, StringComparison.OrdinalIgnoreCase))
+                            column.Name, setOperation.Source1.Projection[index].Alias, StringComparison.Ordinal))
+                .All(e => e);
+
+    private static bool IsNonComposedTpc(SelectExpression selectExpression)
+        => selectExpression.Offset == null
+            && selectExpression.Limit == null
+            && !selectExpression.IsDistinct
+            && selectExpression.Predicate == null
+            && selectExpression.Having == null
+            && selectExpression.Orderings.Count == 0
+            && selectExpression.GroupBy.Count == 0
+            && selectExpression.Tables.Count == 1
+            && selectExpression.Tables[0] is TpcTablesExpression tpcTablesExpression
+            && selectExpression.Projection.Count == tpcTablesExpression.SelectExpressions[0].Projection.Count
+            && selectExpression.Projection.Select(
+                    (pe, index) => pe.Expression is ColumnExpression column
+                        && string.Equals(column.TableAlias, tpcTablesExpression.Alias, StringComparison.Ordinal)
+                        && string.Equals(
+                            column.Name, tpcTablesExpression.SelectExpressions[0].Projection[index].Alias, StringComparison.Ordinal))
                 .All(e => e);
 
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression extensionExpression)
+    {
+        if (extensionExpression is TpcTablesExpression tpcTablesExpression)
+        {
+            _relationalCommandBuilder.AppendLine("(");
+            using (_relationalCommandBuilder.Indent())
+            {
+                GenerateList(tpcTablesExpression.SelectExpressions, e => Visit(e), e => e.AppendLine().AppendLine("UNION ALL"));
+            }
+            _relationalCommandBuilder.AppendLine()
+                .Append(")")
+                .Append(AliasSeparator)
+                .Append(_sqlGenerationHelper.DelimitIdentifier(tpcTablesExpression.Alias));
+
+            return tpcTablesExpression;
+        }
+
+        return base.VisitExtension(extensionExpression);
+    }
+
     /// <inheritdoc />
     protected override Expression VisitSelect(SelectExpression selectExpression)
     {
@@ -150,6 +190,30 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
 
         IDisposable? subQueryIndent = null;
 
+        if (IsNonComposedTpc(selectExpression))
+        {
+            var tpcTablesExpression = (TpcTablesExpression)selectExpression.Tables[0];
+            if (selectExpression.Alias != null)
+            {
+                _relationalCommandBuilder.AppendLine("(");
+                subQueryIndent = _relationalCommandBuilder.Indent();
+            }
+
+            GenerateList(tpcTablesExpression.SelectExpressions, e => Visit(e), e => e.AppendLine().AppendLine("UNION ALL"));
+
+            if (selectExpression.Alias != null)
+            {
+                subQueryIndent!.Dispose();
+
+                _relationalCommandBuilder.AppendLine()
+                    .Append(")")
+                    .Append(AliasSeparator)
+                    .Append(_sqlGenerationHelper.DelimitIdentifier(selectExpression.Alias));
+            }
+
+            return selectExpression;
+        }
+
         if (selectExpression.Alias != null)
         {
             _relationalCommandBuilder.AppendLine("(");

src/EFCore.Relational/Query/RelationalQueryTranslationPostprocessor.cs

@@ -76,6 +76,16 @@ private sealed class SelectExpressionMutableVerifyingExpressionVisitor : Express
                 return shapedQueryExpression;
             }
 
+            if (expression is TpcTablesExpression tpcTablesExpression)
+            {
+                foreach (var se in tpcTablesExpression.SelectExpressions)
+                {
+                    Visit(se);
+                }
+
+                return expression;
+            }
+
             return base.Visit(expression);
         }
     }
@@ -142,6 +152,15 @@ public Expression EntryPoint(Expression expression)
             public override Expression? Visit(Expression? expression)
             {
                 var visitedExpression = base.Visit(expression);
+                if (visitedExpression is TpcTablesExpression tpcTablesExpression)
+                {
+                    // We need to look inside SelectExpressions in TPC for aliases
+                    foreach (var selectExpression in tpcTablesExpression.SelectExpressions)
+                    {
+                        Visit(selectExpression);
+                    }
+                }
+
                 if (visitedExpression is TableExpressionBase tableExpressionBase
                     && !_visitedTableExpressionBases.Contains(tableExpressionBase)
                     && tableExpressionBase.Alias != null)

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.Helper.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics.CodeAnalysis;
+using Microsoft.EntityFrameworkCore.Query.Internal;
 
 namespace Microsoft.EntityFrameworkCore.Query.SqlExpressions;
 
@@ -404,6 +405,17 @@ public AliasUniquifier(HashSet<string> usedAliases)
                 _visitedSelectExpressions.Add(innerSelectExpression);
             }
 
+            if (expression is TpcTablesExpression tpcTablesExpression)
+            {
+                // We uniquify aliases in inner selectexpressions too
+                foreach (var selectExpression in tpcTablesExpression.SelectExpressions)
+                {
+                    Visit(selectExpression);
+                }
+
+                return expression;
+            }
+
             return base.Visit(expression);
         }
     }
@@ -812,6 +824,10 @@ private sealed class CloningExpressionVisitor : ExpressionVisitor
                 var newProjections = selectExpression._projection.Select(Visit).ToList<ProjectionExpression>();
 
                 var newTables = selectExpression._tables.Select(Visit).ToList<TableExpressionBase>();
+                var tpcTableMap = selectExpression._tables.Zip(newTables)
+                    .Where(e => e.First is TpcTablesExpression)
+                    .ToDictionary(e => (TpcTablesExpression)e.First, e => (TpcTablesExpression)e.Second);
+
                 // Since we are cloning we need to generate new table references
                 // In other cases (like VisitChildren), we just reuse the same table references and update the SelectExpression inside it.
                 // We initially assign old SelectExpression in table references and later update it once we construct clone
@@ -845,6 +861,10 @@ private sealed class CloningExpressionVisitor : ExpressionVisitor
                 newSelectExpression._mutable = selectExpression._mutable;
 
                 newSelectExpression._tptLeftJoinTables.AddRange(selectExpression._tptLeftJoinTables);
+                foreach (var kvp in selectExpression._tpcDiscriminatorValues)
+                {
+                    newSelectExpression._tpcDiscriminatorValues[tpcTableMap[kvp.Key]] = kvp.Value;
+                }
                 // Since identifiers are ColumnExpression, they are not visited since they don't contain SelectExpression inside it.
                 newSelectExpression._identifier.AddRange(selectExpression._identifier);
                 newSelectExpression._childIdentifiers.AddRange(selectExpression._childIdentifiers);
@@ -862,6 +882,19 @@ private sealed class CloningExpressionVisitor : ExpressionVisitor
                 return newSelectExpression;
             }
 
+            if (expression is TpcTablesExpression tpcTablesExpression)
+            {
+                // Deep clone
+                var subSelectExpressions = tpcTablesExpression.SelectExpressions.Select(Visit).ToList<SelectExpression>();
+                var newTpcTable = new TpcTablesExpression(tpcTablesExpression.Alias, tpcTablesExpression.EntityType, subSelectExpressions);
+                foreach (var annotation in tpcTablesExpression.GetAnnotations())
+                {
+                    newTpcTable.AddAnnotation(annotation.Name, annotation.Value);
+                }
+
+                return newTpcTable;
+            }
+
             return expression is IClonableTableExpressionBase cloneable ? cloneable.Clone() : base.Visit(expression);
         }
     }