From 6a3919562833b92a46529d3ca0e6f5bc92839663 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Fri, 6 Sep 2024 08:29:00 +0100 Subject: [PATCH 01/10] Fixed swapping order of inputs on hash join Fixes #542 --- MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs index 2ad72b88..69cf18f2 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashJoinNode.cs @@ -169,6 +169,8 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext LeftAttribute = RightAttribute; RightAttribute = leftAttr; + (_leftKeyAccessors, _rightKeyAccessors) = (_rightKeyAccessors, _leftKeyAccessors); + if (JoinType == QualifiedJoinType.LeftOuter) JoinType = QualifiedJoinType.RightOuter; else if (JoinType == QualifiedJoinType.RightOuter) From f40e04865bb6a4049a3366de4aefff80ba110bfd Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:14:22 +0100 Subject: [PATCH 02/10] Do not attempt to use CreateMultiple or UpdateMultiple for virtual tables Fixes #541 --- MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs | 2 +- MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs index 1c15ba0a..491d19e2 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs @@ -313,7 +313,7 @@ protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource if (!req.Requests.All(r => r is CreateRequest)) return base.ExecuteMultiple(dataSource, org, meta, req); - if (meta.DataProviderId == DataProviders.ElasticDataProvider || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "CreateMultiple")) + if (meta.DataProviderId == DataProviders.ElasticDataProvider || meta.DataProviderId == null && dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "CreateMultiple")) { // Elastic tables can use CreateMultiple for better performance than ExecuteMultiple var entities = new EntityCollection { EntityName = meta.LogicalName }; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs index 1bd11dca..1e09788c 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs @@ -599,7 +599,7 @@ protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource if (!req.Requests.All(r => r is UpdateRequest)) return base.ExecuteMultiple(dataSource, org, meta, req); - if (meta.DataProviderId == DataProviders.ElasticDataProvider || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "UpdateMultiple")) + if (meta.DataProviderId == DataProviders.ElasticDataProvider || meta.DataProviderId == null && dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "UpdateMultiple")) { // Elastic tables can use UpdateMultiple for better performance than ExecuteMultiple var entities = new EntityCollection { EntityName = meta.LogicalName }; From 0619008f6e1dd62ac2e1b72ff5553a433c0acd14 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:27:58 +0100 Subject: [PATCH 03/10] Fixed folding subquery alias to constant scan Fixes #546 --- .../AdoProviderTests.cs | 25 +++++++++++++++++++ .../ExecutionPlan/AliasNode.cs | 4 +-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs index 59a901a2..94ad21d9 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs @@ -2441,5 +2441,30 @@ SELECT CAST(CAST(10.6496 AS float) AS int) AS trunc1, } } } + + [TestMethod] + public void QueryDerivedTableWithContradiction() + { + // https://github.com/MarkMpn/Sql4Cds/issues/546 + using (var con = new Sql4CdsConnection(_localDataSources)) + using (var cmd = con.CreateCommand()) + { + cmd.CommandTimeout = 0; + + cmd.CommandText = @" +SELECT * +FROM (SELECT a.accountid + FROM account AS a + WHERE 1 != 1) AS sub"; + + using (var reader = cmd.ExecuteReader()) + { + var schema = reader.GetSchemaTable(); + + if (reader.Read()) + Assert.Fail(); + } + } + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs index 614b34b8..3784f724 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs @@ -131,7 +131,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext { // Remove any unused columns var unusedColumns = constant.Schema.Keys - .Where(sourceCol => !ColumnSet.Any(col => col.SourceColumn.SplitMultiPartIdentifier().Last().EscapeIdentifier() == sourceCol)) + .Where(sourceCol => !ColumnSet.Any(col => (String.IsNullOrEmpty(constant.Alias) && col.SourceColumn == sourceCol) || (!String.IsNullOrEmpty(constant.Alias) && col.SourceColumn == constant.Alias.EscapeIdentifier() + "." + sourceCol))) .ToList(); foreach (var col in unusedColumns) @@ -145,7 +145,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext // Copy/rename any columns using the new aliases foreach (var col in ColumnSet) { - var sourceColumn = col.SourceColumn.SplitMultiPartIdentifier().Last(); + var sourceColumn = constant.Alias == null ? col.SourceColumn : col.SourceColumn.SplitMultiPartIdentifier().Last(); if (String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != col.SourceColumn || !String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != constant.Alias.EscapeIdentifier() + "." + col.SourceColumn) From 7c24e8817ef2b33a65ff541aa0686f9abe119e22 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:55:45 +0100 Subject: [PATCH 04/10] Use outer references when checking for not-null filters to apply to outer joins Fixes #547 --- .../ExecutionPlanTests.cs | 39 +++++++++++++++++++ .../ExecutionPlan/FilterNode.cs | 17 +++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs index 04f9910e..8885bd5c 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs @@ -8433,5 +8433,44 @@ FROM metadata.alternate_key AS ak CollectionAssert.AreEqual(new[] { "EntityLogicalName", "LogicalName", "EntityKeyIndexStatus" }, meta.Query.KeyQuery.Properties.PropertyNames); Assert.AreEqual(0, meta.Query.KeyQuery.Criteria.Conditions.Count); } + + [TestMethod] + public void OuterApplyOuterReference() + { + // https://github.com/MarkMpn/Sql4Cds/issues/547 + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +SELECT * +FROM ( + SELECT a.accountid, + a.name + FROM account a) AS q1 +FULL OUTER JOIN ( + SELECT c.contactid, + c.parentcustomerid, + c.fullname + FROM contact c) AS q2 + ON q1.accountid = q2.parentcustomerid +OUTER APPLY ( + SELECT CASE WHEN q1.accountid = q2.parentcustomerid THEN 1 ELSE 0 END AS [flag] +) AS q3 +WHERE q3.flag = 0"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var apply = AssertNode(select.Source); + var join = AssertNode(apply.LeftSource); + var fetch1 = AssertNode(join.LeftSource); + var sort = AssertNode(join.RightSource); + var fetch2 = AssertNode(sort.Source); + var alias = AssertNode(apply.RightSource); + var filter = AssertNode(alias.Source); + var compute = AssertNode(filter.Source); + var constant = AssertNode(compute.Source); + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs index f106ba60..abef92d6 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs @@ -790,7 +790,22 @@ private void ConvertOuterJoinsWithNonNullFiltersToInnerJoins(NodeCompilationCont if (outerSource != null) { - var outerSchema = outerSource.GetSchema(context); + // If we are enforcing a non-null constraint on the outer source, we can convert the join to an inner join + // To get the schema of the outer source, we need to include any outer references that are used in the join condition + var outerContext = context; + + if (join.JoinType == QualifiedJoinType.LeftOuter && join is NestedLoopNode loop && loop.OuterReferences != null && loop.OuterReferences.Count > 0) + { + var leftSchema = join.LeftSource.GetSchema(context); + + var innerParameterTypes = context.ParameterTypes + .Concat(loop.OuterReferences.Select(or => new KeyValuePair(or.Value, leftSchema.Schema[or.Key].Type))) + .ToDictionary(p => p.Key, p => p.Value, StringComparer.OrdinalIgnoreCase); + + outerContext = new NodeCompilationContext(context, innerParameterTypes); + } + + var outerSchema = outerSource.GetSchema(outerContext); if (notNullColumns.Any(col => outerSchema.ContainsColumn(col, out _))) join.JoinType = QualifiedJoinType.Inner; From 10289928a1a50575f6a0ce591d7cf18a8c8f1c7b Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sat, 14 Sep 2024 20:00:04 +0100 Subject: [PATCH 05/10] Do not fold full outer join to FetchXML/metadata queries --- MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs index 1d57ed63..843af188 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs @@ -124,7 +124,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext IDataExecutionPlanNodeInternal folded = null; - if (LeftAttributes.Count == 1 && ComparisonType == BooleanComparisonType.Equals) + if (LeftAttributes.Count == 1 && ComparisonType == BooleanComparisonType.Equals && JoinType != QualifiedJoinType.FullOuter) { var leftFilter = JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.LeftOuter ? LeftSource as FilterNode : null; var rightFilter = JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.RightOuter ? RightSource as FilterNode : null; From fd94f86c812c62a62619ddf27500a5a858a4e7ed Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sat, 14 Sep 2024 20:13:58 +0100 Subject: [PATCH 06/10] Do not fold join conditions from nested loop to outer source Fixes #548 --- .../ExecutionPlanTests.cs | 33 +++++++++++++++++++ .../ExecutionPlan/NestedLoopNode.cs | 9 +++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs index 8885bd5c..49c37418 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs @@ -8463,6 +8463,7 @@ OUTER APPLY ( var select = AssertNode(plans[0]); var apply = AssertNode(select.Source); + Assert.AreEqual(QualifiedJoinType.Inner, apply.JoinType); var join = AssertNode(apply.LeftSource); var fetch1 = AssertNode(join.LeftSource); var sort = AssertNode(join.RightSource); @@ -8472,5 +8473,37 @@ OUTER APPLY ( var compute = AssertNode(filter.Source); var constant = AssertNode(compute.Source); } + + [TestMethod] + public void FilterOnOuterApply() + { + // https://github.com/MarkMpn/Sql4Cds/issues/548 + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +SELECT * +FROM ( + SELECT a.accountid, + a.name + FROM account a) AS q1 +OUTER APPLY ( + SELECT IIF(q1.name = 'Test1', 1, 0) AS [flag1], + IIF(q1.name = 'Test2', 1, 0) AS [flag2] +) AS q2 +WHERE q2.flag1 = 1 OR q2.flag2 = 1"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var apply = AssertNode(select.Source); + Assert.AreEqual(QualifiedJoinType.LeftOuter, apply.JoinType); + Assert.AreEqual("q2.flag1 = 1 OR q2.flag2 = 1", apply.JoinCondition.ToSql()); + var fetch = AssertNode(apply.LeftSource); + var alias = AssertNode(apply.RightSource); + var compute = AssertNode(alias.Source); + var constant = AssertNode(compute.Source); + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs index b6f7f656..91436897 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs @@ -193,8 +193,9 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext var joinColumns = JoinCondition.GetColumns().ToList(); var hasLeftColumn = joinColumns.Any(c => leftSchema.ContainsColumn(c, out _)); var hasRightColumn = joinColumns.Any(c => rightSchema.ContainsColumn(c, out _)); + var foldedFilter = false; - if (!hasLeftColumn) + if (!hasLeftColumn && JoinType == QualifiedJoinType.Inner) { // Join condition doesn't reference columns from the left source, so we can remove it from the join // and apply it as a filter to the right source. Inner source will often have a table spool - add the filter @@ -206,17 +207,19 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext RightSource = RightSource.FoldQuery(innerContext, hints); RightSource.Parent = this; + foldedFilter = true; } - if (!hasRightColumn) + if (!hasRightColumn && (JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.LeftOuter)) { // Join condition doesn't reference columns from the right source, so we can remove it from the join // and apply it as a filter to the left source LeftSource = new FilterNode { Source = LeftSource, Filter = JoinCondition }.FoldQuery(context, hints); LeftSource.Parent = this; + foldedFilter = true; } - if (!hasLeftColumn || !hasRightColumn) + if (foldedFilter) JoinCondition = null; } From 3aa9eec69e7d968e1b9a9ae4e633a316da1e8b09 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sat, 14 Sep 2024 20:19:18 +0100 Subject: [PATCH 07/10] Fixed test for filter on outer join nested loop --- MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs | 6 ++++-- MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs index 49c37418..bf2d91eb 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs @@ -8497,9 +8497,11 @@ SELECT IIF(q1.name = 'Test1', 1, 0) AS [flag1], Assert.AreEqual(1, plans.Length); var select = AssertNode(plans[0]); - var apply = AssertNode(select.Source); + var filter = AssertNode(select.Source); + Assert.AreEqual("q2.flag1 = 1 OR q2.flag2 = 1", filter.Filter.ToSql()); + var apply = AssertNode(filter.Source); Assert.AreEqual(QualifiedJoinType.LeftOuter, apply.JoinType); - Assert.AreEqual("q2.flag1 = 1 OR q2.flag2 = 1", apply.JoinCondition.ToSql()); + Assert.IsNull(apply.JoinCondition); var fetch = AssertNode(apply.LeftSource); var alias = AssertNode(apply.RightSource); var compute = AssertNode(alias.Source); diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs index abef92d6..5ba5a272 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs @@ -356,7 +356,7 @@ private bool FoldFiltersToNestedLoopCondition(NodeCompilationContext context, IL if (Filter == null) return false; - if (!(Source is NestedLoopNode loop)) + if (!(Source is NestedLoopNode loop) || loop.JoinType != QualifiedJoinType.Inner) return false; // Can't move the filter to the loop condition if we're using any of the defined values created by the loop From 0ea9df957a62375cef309b62c71abce0d411ffe2 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sun, 15 Sep 2024 16:30:56 +0100 Subject: [PATCH 08/10] Standardised checking for missing records on UPDATE/DELETE --- MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs | 4 +++- MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs index 6054d3fa..d4812fa8 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs @@ -275,7 +275,9 @@ protected override bool FilterErrors(NodeExecutionContext context, OrganizationR { // Ignore errors trying to delete records that don't exist - record may have been deleted by another // process in parallel. - return fault.ErrorCode != -2147220891 && fault.ErrorCode != -2147185406 && fault.ErrorCode != -2147220969 && fault.ErrorCode != 404; + return fault.ErrorCode != -2147185406 && // IsvAbortedNotFound + fault.ErrorCode != -2147220969 && // ObjectDoesNotExist + fault.ErrorCode != 404; // Elastic tables } protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource, IOrganizationService org, EntityMetadata meta, ExecuteMultipleRequest req) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs index 1e09788c..55cf3e06 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs @@ -591,7 +591,9 @@ protected override bool FilterErrors(NodeExecutionContext context, OrganizationR { // Ignore errors trying to update records that don't exist - record may have been deleted by another // process in parallel. - return fault.ErrorCode != -2147220969 && fault.ErrorCode != -2147185406 && fault.ErrorCode != -2147220969 && fault.ErrorCode != 404; + return fault.ErrorCode != -2147185406 && // IsvAbortedNotFound + fault.ErrorCode != -2147220969 && // ObjectDoesNotExist + fault.ErrorCode != 404; // Elastic tables } protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource, IOrganizationService org, EntityMetadata meta, ExecuteMultipleRequest req) From 20104f933ecf56656fd68485173d3238017373fe Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Sun, 15 Sep 2024 16:31:34 +0100 Subject: [PATCH 09/10] Added manual retry logic for service protection limits #544 --- .../ExecutionPlan/BaseDmlNode.cs | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs index ee5d4b9d..56f08f65 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs @@ -691,24 +691,44 @@ protected void ExecuteDmlOperation(DataSource dataSource, IQueryExecutionOptions else options.Progress(progress, $"{operationNames.InProgressUppercase} {newCount - threadCount + 1:N0}-{newCount:N0} of {entities.Count:N0} {GetDisplayName(0, meta)} ({progress:P0}, {threadCount:N0} threads)..."); - try + while (true) { - var response = dataSource.Execute(threadLocalState.Service, request); - Interlocked.Increment(ref count); - - responseHandler?.Invoke(response); - } - catch (FaultException ex) - { - if (FilterErrors(context, request, ex.Detail)) + try { - if (ContinueOnError) - fault = fault ?? ex.Detail; - else - throw; + var response = dataSource.Execute(threadLocalState.Service, request); + Interlocked.Increment(ref count); + + responseHandler?.Invoke(response); + break; } + catch (FaultException ex) + { + if (ex.Detail.ErrorCode == 429 || // Virtual/elastic tables + ex.Detail.ErrorCode == -2147015902 || // Number of requests exceeded the limit of 6000 over time window of 300 seconds. + ex.Detail.ErrorCode == -2147015903 || // Combined execution time of incoming requests exceeded limit of 1,200,000 milliseconds over time window of 300 seconds. Decrease number of concurrent requests or reduce the duration of requests and try again later. + ex.Detail.ErrorCode == -2147015898) // Number of concurrent requests exceeded the limit of 52. + { + // In case throttling isn't handled by normal retry logic in the service client + var retryAfterSeconds = 2; + + if (ex.Detail.ErrorDetails.TryGetValue("Retry-After", out var retryAfter) && (retryAfter is int || retryAfter is string s && Int32.TryParse(s, out _))) + retryAfterSeconds = Convert.ToInt32(retryAfter); - Interlocked.Increment(ref errorCount); + Thread.Sleep(retryAfterSeconds * 1000); + continue; + } + + if (FilterErrors(context, request, ex.Detail)) + { + if (ContinueOnError) + fault = fault ?? ex.Detail; + else + throw; + } + + Interlocked.Increment(ref errorCount); + break; + } } } else From 992352ff86938ce90872bcea1e249950969042f3 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:08:29 +0100 Subject: [PATCH 10/10] Fixed error with scalar subqueries without alias --- .../ExecutionPlanTests.cs | 27 +++++++++++++++++++ .../ExecutionPlan/FetchXmlScan.cs | 3 +++ 2 files changed, 30 insertions(+) diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs index bf2d91eb..d7bd7638 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs @@ -8507,5 +8507,32 @@ SELECT IIF(q1.name = 'Test1', 1, 0) AS [flag1], var compute = AssertNode(alias.Source); var constant = AssertNode(compute.Source); } + + [TestMethod] + public void ScalarSubqueryWithoutAlias() + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +SELECT a.accountid, +(SELECT fullname FROM contact WHERE contactid = a.primarycontactid) +FROM account a"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + AssertFetchXml(fetch, @" + + + + + + + +"); + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs index 950fb43a..ae754431 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs @@ -2144,6 +2144,9 @@ public void AddAliases(List columnSet, INodeSchema schema, IAttrib .Select(g => g.Single()) .Where(c => { + if (c.Alias == null) + return false; // Don't fold null aliases, e.g. scalar subqueries + var parts = c.SourceColumn.SplitMultiPartIdentifier(); if (parts.Length > 1 && aliasStars.Contains(parts[0]))