From 34d858e272fd5601e95b2ca0ecc5db945ab199c7 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:54:36 +0000 Subject: [PATCH 1/2] Include join alias when converting from Fetch XML to SQL --- .../FetchXml2SqlTests.cs | 16 ++++++++-------- MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs | 12 +++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs index b1e3bd43..a3238ab1 100644 --- a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs +++ b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs @@ -67,7 +67,7 @@ public void Joins() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name AS account_name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid", NormalizeWhitespace(converted)); } [TestMethod] @@ -93,7 +93,7 @@ public void JoinFilter() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid AND account.name = 'data8' WHERE contact.firstname = 'Mark'", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name AS account_name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid AND account.name = 'data8' WHERE contact.firstname = 'Mark'", NormalizeWhitespace(converted)); } [TestMethod] @@ -113,7 +113,7 @@ public void JoinAlias() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, a.name FROM contact INNER JOIN account AS a ON contact.parentcustomerid = a.accountid", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, a.name AS a_name FROM contact INNER JOIN account AS a ON contact.parentcustomerid = a.accountid", NormalizeWhitespace(converted)); } [TestMethod] @@ -136,7 +136,7 @@ public void JoinAliasFilter() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, a.name FROM contact INNER JOIN account AS a ON contact.parentcustomerid = a.accountid WHERE a.name = 'data8'", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, a.name AS a_name FROM contact INNER JOIN account AS a ON contact.parentcustomerid = a.accountid WHERE a.name = 'data8'", NormalizeWhitespace(converted)); } [TestMethod] @@ -228,7 +228,7 @@ public void JoinOrder() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid ORDER BY account.name ASC, contact.firstname ASC", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name AS account_name FROM contact INNER JOIN account ON contact.parentcustomerid = account.accountid ORDER BY account.name ASC, contact.firstname ASC", NormalizeWhitespace(converted)); } [TestMethod] @@ -527,7 +527,7 @@ public void ArchiveJoins() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions(), out _); - Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name FROM archive.contact INNER JOIN archive.account ON contact.parentcustomerid = account.accountid", NormalizeWhitespace(converted)); + Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name AS account_name FROM archive.contact INNER JOIN archive.account ON contact.parentcustomerid = account.accountid", NormalizeWhitespace(converted)); } [TestMethod] @@ -725,7 +725,7 @@ public void MatchFirstRowUsingCrossApply() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); Assert.AreEqual(NormalizeWhitespace(@" - SELECT contact.fullname, account.accountid, account.name FROM contact CROSS APPLY ( SELECT TOP 1 account.accountid, account.name FROM account WHERE contact.contactid = account.primarycontactid ) AS account"), NormalizeWhitespace(converted)); + SELECT contact.fullname, account.accountid AS account_accountid, account.name AS account_name FROM contact CROSS APPLY ( SELECT TOP 1 account.accountid, account.name FROM account WHERE contact.contactid = account.primarycontactid ) AS account"), NormalizeWhitespace(converted)); } [TestMethod] @@ -777,7 +777,7 @@ public void ColumnComparisonCrossTable() var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); Assert.AreEqual(NormalizeWhitespace(@" - SELECT contact.contactid, contact.fullname, acct.name FROM contact LEFT OUTER JOIN account AS acct ON contact.parentcustomerid = acct.accountid WHERE contact.fullname = acct.name"), NormalizeWhitespace(converted)); + SELECT contact.contactid, contact.fullname, acct.name AS acct_name FROM contact LEFT OUTER JOIN account AS acct ON contact.parentcustomerid = acct.accountid WHERE contact.fullname = acct.name"), NormalizeWhitespace(converted)); } [TestMethod] diff --git a/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs b/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs index ae371932..ab842ace 100644 --- a/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs +++ b/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs @@ -63,7 +63,7 @@ public static string Convert(IOrganizationService org, IAttributeMetadataCache m // SELECT (columns from first table) var entity = fetch.Items.OfType().SingleOrDefault(); - AddSelectElements(query, entity.Items, entity?.name); + AddSelectElements(query, entity.Items, entity?.name, true); if (query.SelectElements.Count == 0) query.SelectElements.Add(new SelectStarExpression()); @@ -256,7 +256,7 @@ public static string Convert(IOrganizationService org, IAttributeMetadataCache m /// The SQL query to append to the SELECT clause of /// The FetchXML items to process /// The name or alias of the table being processed - private static void AddSelectElements(QuerySpecification query, object[] items, string prefix) + private static void AddSelectElements(QuerySpecification query, object[] items, string prefix, bool isRoot) { if (items == null) return; @@ -347,6 +347,8 @@ private static void AddSelectElements(QuerySpecification query, object[] items, // Apply alias if (!String.IsNullOrEmpty(attr.alias) && (attr.aggregateSpecified || attr.alias != attr.name)) element.ColumnName = new IdentifierOrValueExpression { Identifier = new Identifier { Value = attr.alias } }; + else if (!isRoot) + element.ColumnName = new IdentifierOrValueExpression { Identifier = new Identifier { Value = $"{prefix}_{attr.name}" } }; query.SelectElements.Add(element); @@ -464,8 +466,8 @@ private static TableReference BuildJoins(IOrganizationService org, IAttributeMet } else if (link.linktype == "matchfirstrowusingcrossapply") { - AddSelectElements(subquery, link.Items, link.alias ?? link.name); - AddSelectElements(query, link.Items, link.alias ?? link.name); + AddSelectElements(subquery, link.Items, link.alias ?? link.name, true); + AddSelectElements(query, link.Items, link.alias ?? link.name, false); } subquery.FromClause = new FromClause @@ -624,7 +626,7 @@ private static TableReference BuildJoins(IOrganizationService org, IAttributeMet }; // Update the SELECT clause - AddSelectElements(query, link.Items, link.alias ?? link.name); + AddSelectElements(query, link.Items, link.alias ?? link.name, false); // Handle any filters within the as additional join criteria var filter = GetFilter(org, metadata, link.Items, link.alias ?? link.name, aliasToLogicalName, options, ctes, parameters, ref requiresTimeZone, ref usesToday); From 69c51e13369259fdb27530a60388d0beaf1dd790 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:46:59 +0000 Subject: [PATCH 2/2] Improved SQL -> FetchXML conversion of datetime filters Fixes #576 --- .../ExecutionPlan/BaseDataNode.cs | 44 +++++++++---------- .../ExecutionPlan/FetchXmlScan.cs | 14 ++++++ 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs index 9fece06f..682fd1c2 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs @@ -975,40 +975,38 @@ private bool TranslateFetchXMLCriteriaWithVirtualAttributes(NodeCompilationConte if (attribute is DateTimeAttributeMetadata && literals != null && (op == @operator.eq || op == @operator.ne || op == @operator.neq || op == @operator.gt || op == @operator.ge || op == @operator.lt || op == @operator.le || op == @operator.@in || op == @operator.notin)) { + var ecc = new ExpressionCompilationContext(context, null, null); + var eec = new ExpressionExecutionContext(ecc); + for (var i = 0; i < literals.Length; i++) { if (!(literals[i] is Literal lit)) continue; - try - { - DateTime dt; + lit.GetType(ecc, out var sourceType); + var targetType = attribute.GetAttributeSqlType(dataSource, false); - if (lit is StringLiteral) - dt = SqlDateTime.Parse(lit.Value).Value; - else if (lit is IntegerLiteral || lit is NumericLiteral || lit is RealLiteral) - dt = new DateTime(1900, 1, 1).AddDays(Double.Parse(lit.Value, CultureInfo.InvariantCulture)); - else - throw new NotSupportedQueryFragmentException(Sql4CdsError.DateTimeParseError(lit)); + if (!SqlTypeConverter.CanChangeTypeImplicit(sourceType, targetType)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.DateTimeParseError(lit)); - DateTimeOffset dto; + var sqlValue = (INullable)lit.Compile(ecc)(eec); + var conversion = SqlTypeConverter.GetConversion(sourceType, targetType); + var datetimeValue = (SqlDateTime)conversion(sqlValue, eec); + var dt = datetimeValue.Value; - if (context.Options.UseLocalTimeZone) - dto = new DateTimeOffset(dt, TimeZoneInfo.Local.GetUtcOffset(dt)); - else - dto = new DateTimeOffset(dt, TimeSpan.Zero); + DateTimeOffset dto; - var formatted = dto.ToString("yyyy-MM-ddTHH':'mm':'ss.FFFzzz"); + if (context.Options.UseLocalTimeZone) + dto = new DateTimeOffset(dt, TimeZoneInfo.Local.GetUtcOffset(dt)); + else + dto = new DateTimeOffset(dt, TimeSpan.Zero); - if (literals.Length == 1) - value = formatted; + var formatted = dto.ToString("yyyy-MM-ddTHH':'mm':'ss.FFFzzz"); - values[i].Value = formatted; - } - catch (FormatException) - { - throw new NotSupportedQueryFragmentException(Sql4CdsError.DateTimeParseError(lit)); - } + if (literals.Length == 1) + value = formatted; + + values[i].Value = formatted; } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs index 5f9af58e..48bba560 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs @@ -515,8 +515,22 @@ private void VerifyFilterValueTypes(string entityName, object[] items, DataSourc break; } + // Apply the standard SQL string -> target type conversion to each value - it will throw an exception if any values are in the incorrect format. var conversion = SqlTypeConverter.GetConversion(DataTypeHelpers.NVarChar(Int32.MaxValue, dataSource.DefaultCollation, CollationLabel.CoercibleDefault), attrType); + // Special case: we convert datetime values to a different format to be understood by Dataverse which is different + // from the SQL formats in BaseDataNode.TranslateFetchXMLCriteriaWithVirtualAttributes + if (attrType.IsSameAs(DataTypeHelpers.DateTime)) + { + conversion = (value, ctx) => + { + if (value is SqlString str && DateTimeOffset.TryParseExact(str.Value, "yyyy-MM-ddTHH:mm:ss.FFFzzz", CultureInfo.InvariantCulture, DateTimeStyles.None, out var dt)) + return new SqlDateTimeOffset(dt); + + return conversion(value, ctx); + }; + } + if (condition.value != null) conversion(dataSource.DefaultCollation.ToSqlString(condition.value), context);