Skip to content

Commit

Permalink
Improved injecting variable filter values via top-level nested loop
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarkMpn
MarkMpn committed Aug 2, 2024
1 parent 3efdbed commit 29207c8
Show file tree
Hide file tree
Showing 8 changed files with 37 additions and 64 deletions.
39 changes: 8 additions & 31 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,37 +224,15 @@ public void MergeStatsFrom(BaseDataNode other)
/// <param name="targetEntityAlias">The alias of the root entity that the FetchXML query is targetting</param>
/// <param name="items">The child items of the root entity in the FetchXML query</param>
/// <param name="filter">The FetchXML version of the <paramref name="criteria"/> that is generated by this method</param>
/// <param name="nestedLoop">A nested loop node that needs to be added as a parent of the FetchXML node to calculate the filter values at runtime</param>
/// <returns><c>true</c> if the <paramref name="criteria"/> can be translated to FetchXML, or <c>false</c> otherwise</returns>
protected bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpression, out filter filter, out NestedLoopNode nestedLoop)
protected bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpression, out filter filter)
{
nestedLoop = new NestedLoopNode
{
LeftSource = new ComputeScalarNode
{
Source = new ConstantScanNode
{
Values =
{
new Dictionary<string, ScalarExpression>()
}
}
},
OuterReferences = new Dictionary<string, string>()
};

if (!TranslateFetchXMLCriteria(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpression, out var condition, out filter, nestedLoop))
{
nestedLoop = null;
if (!TranslateFetchXMLCriteria(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpression, out var condition, out filter))
return false;
}

if (condition != null)
filter = new filter { Items = new object[] { condition } };

if (nestedLoop.OuterReferences.Count == 0)
nestedLoop = null;

return true;
}

Expand All @@ -272,9 +250,8 @@ protected bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSou
/// <param name="items">The child items of the root entity in the FetchXML query</param>
/// <param name="filter">The FetchXML version of the <paramref name="criteria"/> that is generated by this method when it covers multiple conditions</param>
/// <param name="condition">The FetchXML version of the <paramref name="criteria"/> that is generated by this method when it is for a single condition only</param>
/// <param name="nestedLoop">A nested loop node that needs to be added as a parent of the FetchXML node to calculate the filter values at runtime</param>
/// <returns><c>true</c> if the <paramref name="criteria"/> can be translated to FetchXML, or <c>false</c> otherwise</returns>
private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpression, out condition condition, out filter filter, NestedLoopNode nestedLoop)
private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpression, out condition condition, out filter filter)
{
condition = null;
filter = null;
Expand Down Expand Up @@ -318,7 +295,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc

if (criteria is BooleanParenthesisExpression paren)
{
return TranslateFetchXMLCriteria(context, dataSource, paren.Expression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpression, out condition, out filter, nestedLoop);
return TranslateFetchXMLCriteria(context, dataSource, paren.Expression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpression, out condition, out filter);
}

if (criteria is DistinctPredicate distinct)
Expand Down Expand Up @@ -399,8 +376,8 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
// Need to evaluate other expressions at runtime, but can then inject the result into the FetchXML as a parameter
var exprName = context.GetExpressionName();
var referenceName = "@" + context.GetExpressionName();
((ComputeScalarNode)nestedLoop.LeftSource).Columns[exprName] = expr;
nestedLoop.OuterReferences[exprName] = referenceName;
((ComputeScalarNode)context.GlobalCalculations.LeftSource).Columns[exprName] = expr;
context.GlobalCalculations.OuterReferences[exprName] = referenceName;
variable = new VariableReference { Name = referenceName };
}
else
Expand Down Expand Up @@ -558,8 +535,8 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
// Need to evaluate other expressions at runtime, but can then inject the result into the FetchXML as a parameter
var exprName = context.GetExpressionName();
var referenceName = "@" + context.GetExpressionName();
((ComputeScalarNode)nestedLoop.LeftSource).Columns[exprName] = expr;
nestedLoop.OuterReferences[exprName] = referenceName;
((ComputeScalarNode)context.GlobalCalculations.LeftSource).Columns[exprName] = expr;
context.GlobalCalculations.OuterReferences[exprName] = referenceName;
values = new[] { new VariableReference { Name = referenceName } };
}
else if (field2 == null)
Expand Down
5 changes: 5 additions & 0 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ public void Dispose()
/// <returns>The node that should be used in place of this node</returns>
public virtual IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext context, IList<OptimizerHint> hints)
{
context.ResetGlobalCalculations();

if (Source is IDataExecutionPlanNodeInternal dataNode)
Source = dataNode.FoldQuery(context, hints);
else if (Source is IDataReaderExecutionPlanNode dataSetNode)
Expand All @@ -168,6 +170,9 @@ public virtual IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext
BypassCustomPluginExecution = GetBypassPluginExecution(context, hints);
ContinueOnError = GetContinueOnError(context, hints);

if (Source is IDataExecutionPlanNodeInternal source)
Source = context.InsertGlobalCalculations(this, source);

return new[] { this };
}

Expand Down
4 changes: 4 additions & 0 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConditionalNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,12 @@ public string Execute(NodeExecutionContext context)

public IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext context, IList<OptimizerHint> hints)
{
context.ResetGlobalCalculations();

Source = Source?.FoldQuery(context, hints);

Source = context.InsertGlobalCalculations(this, Source);

return new[] { this };
}

Expand Down
30 changes: 7 additions & 23 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1515,10 +1515,7 @@ private bool FoldFiltersToDataSources(NodeCompilationContext context, IList<Opti
ignoreAliasesByNode[fetchXml],
fetchXml,
subqueryExpressions,
out var fetchFilter,
out var fetchLoop);

nestedLoop = CombineLoops(nestedLoop, fetchLoop);
out var fetchFilter);

if (fetchFilter != null)
{
Expand Down Expand Up @@ -1872,14 +1869,14 @@ public override void AddRequiredColumns(NodeCompilationContext context, IList<st
Source.AddRequiredColumns(context, requiredColumns);
}

private BooleanExpression ExtractFetchXMLFilters(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, FetchXmlScan fetchXmlScan, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, out filter filter, out NestedLoopNode nestedLoop)
private BooleanExpression ExtractFetchXMLFilters(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, FetchXmlScan fetchXmlScan, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, out filter filter)
{
var targetEntityName = fetchXmlScan.Entity.name;
var targetEntityAlias = fetchXmlScan.Alias;
var items = fetchXmlScan.Entity.Items;

var subqueryConditions = new HashSet<BooleanExpression>();
var result = ExtractFetchXMLFilters(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, subqueryConditions, out filter, out nestedLoop);
var result = ExtractFetchXMLFilters(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, subqueryConditions, out filter);

if (result == criteria)
return result;
Expand Down Expand Up @@ -1925,41 +1922,28 @@ private BooleanExpression ExtractFetchXMLFilters(NodeCompilationContext context,
return result;
}

private BooleanExpression ExtractFetchXMLFilters(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpressions, out filter filter, out NestedLoopNode nestedLoop)
private BooleanExpression ExtractFetchXMLFilters(NodeCompilationContext context, DataSource dataSource, BooleanExpression criteria, INodeSchema schema, string allowedPrefix, HashSet<string> barredPrefixes, string targetEntityName, string targetEntityAlias, object[] items, Dictionary<BooleanExpression, ConvertedSubquery> subqueryExpressions, HashSet<BooleanExpression> replacedSubqueryExpressions, out filter filter)
{
if (TranslateFetchXMLCriteria(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out filter, out nestedLoop))
if (TranslateFetchXMLCriteria(context, dataSource, criteria, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out filter))
return null;

nestedLoop = null;

if (!(criteria is BooleanBinaryExpression bin))
return criteria;

if (bin.BinaryExpressionType != BooleanBinaryExpressionType.And)
return criteria;

bin.FirstExpression = ExtractFetchXMLFilters(context, dataSource, bin.FirstExpression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out var lhsFilter, out var lhsLoop);
bin.SecondExpression = ExtractFetchXMLFilters(context, dataSource, bin.SecondExpression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out var rhsFilter, out var rhsLoop);
bin.FirstExpression = ExtractFetchXMLFilters(context, dataSource, bin.FirstExpression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out var lhsFilter);
bin.SecondExpression = ExtractFetchXMLFilters(context, dataSource, bin.SecondExpression, schema, allowedPrefix, barredPrefixes, targetEntityName, targetEntityAlias, items, subqueryExpressions, replacedSubqueryExpressions, out var rhsFilter);

filter = (lhsFilter != null && rhsFilter != null) ? new filter { Items = new object[] { lhsFilter, rhsFilter } } : lhsFilter ?? rhsFilter;
nestedLoop = CombineLoops(lhsLoop, rhsLoop);

if (bin.FirstExpression != null && bin.SecondExpression != null)
return bin;

return bin.FirstExpression ?? bin.SecondExpression;
}

private NestedLoopNode CombineLoops(NestedLoopNode loop1, NestedLoopNode loop2)
{
if (loop1 == null || loop2 == null)
return loop1 ?? loop2;

loop1.RightSource = loop2;

return loop1;
}

protected BooleanExpression ExtractMetadataFilters(NodeCompilationContext context, BooleanExpression criteria, MetadataQueryNode meta, out MetadataFilterExpression entityFilter, out MetadataFilterExpression attributeFilter, out MetadataFilterExpression relationshipFilter, out MetadataFilterExpression keyFilter)
{
if (TranslateMetadataCriteria(context, criteria, meta, out entityFilter, out attributeFilter, out relationshipFilter, out keyFilter))
Expand Down
9 changes: 1 addition & 8 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ private bool FoldFetchXmlJoin(NodeCompilationContext context, IList<OptimizerHin
// in the new link entity or we must be using an inner join so we can use a post-filter node
var additionalCriteria = AdditionalJoinCriteria;

if (TranslateFetchXMLCriteria(context, dataSource, additionalCriteria, rightSchema, rightFetch.Alias, null, rightEntity.name, rightFetch.Alias, rightEntity.Items, null, null, out var filter, out var dynamicValuesLoop))
if (TranslateFetchXMLCriteria(context, dataSource, additionalCriteria, rightSchema, rightFetch.Alias, null, rightEntity.name, rightFetch.Alias, rightEntity.Items, null, null, out var filter))
{
rightEntity.AddItem(filter);
additionalCriteria = null;
Expand Down Expand Up @@ -514,13 +514,6 @@ private bool FoldFetchXmlJoin(NodeCompilationContext context, IList<OptimizerHin
if (additionalCriteria != null)
folded = new FilterNode { Filter = additionalCriteria, Source = leftFetch }.FoldQuery(context, hints);

// Insert the loop to calculate the dynamic filter values into the plan
if (dynamicValuesLoop != null)
{
dynamicValuesLoop.RightSource = folded;
folded = dynamicValuesLoop;
}

// We might have previously folded a sort to the FetchXML that is no longer valid as we require custom paging
if (leftFetch.RequiresCustomPaging(context.DataSources))
leftFetch.RemoveSorts();
Expand Down
4 changes: 4 additions & 0 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/GoToNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,11 @@ public string Execute(NodeExecutionContext context)
public IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext context, IList<OptimizerHint> hints)
{
if (Source != null)
{
context.ResetGlobalCalculations();
Source = Source.FoldQuery(context, hints);
Source = context.InsertGlobalCalculations(this, Source);
}

_condition = Condition?.Compile(new ExpressionCompilationContext(context, null, null));

Expand Down
6 changes: 4 additions & 2 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ public override object Clone()
JoinType = JoinType,
LeftSource = (IDataExecutionPlanNodeInternal)LeftSource.Clone(),
OuterReferences = OuterReferences,
RightSource = (IDataExecutionPlanNodeInternal)RightSource.Clone(),
RightSource = (IDataExecutionPlanNodeInternal)RightSource?.Clone(),
SemiJoin = SemiJoin,
OutputLeftSchema = OutputLeftSchema,
OutputRightSchema = OutputRightSchema
Expand All @@ -428,7 +428,9 @@ public override object Clone()
clone.DefinedValues.Add(kvp);

clone.LeftSource.Parent = clone;
clone.RightSource.Parent = clone;

if (clone.RightSource != null)
clone.RightSource.Parent = clone;

return clone;
}
Expand Down
4 changes: 4 additions & 0 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ public override IEnumerable<IExecutionPlanNode> GetSources()

public IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext context, IList<OptimizerHint> hints)
{
context.ResetGlobalCalculations();

Source = Source.FoldQuery(context, hints);
Source.Parent = this;

Expand All @@ -100,6 +102,8 @@ public IRootExecutionPlanNodeInternal[] FoldQuery(NodeCompilationContext context
Source.Parent = this;
}

Source = context.InsertGlobalCalculations(this, Source);

return new[] { this };
}

Expand Down

0 comments on commit 29207c8

Please sign in to comment.