diff --git a/AzureDataStudioExtension/CHANGELOG.md b/AzureDataStudioExtension/CHANGELOG.md index 903b2dfa..cebc68cb 100644 --- a/AzureDataStudioExtension/CHANGELOG.md +++ b/AzureDataStudioExtension/CHANGELOG.md @@ -1,5 +1,23 @@ # Change Log +## [v9.1.0](https://github.com/MarkMpn/Sql4Cds/releases/tag/v9.1.0) - 2024-06-10 + +Enabled access to recycle bin records via the `bin` schema +Enabled `INSERT`, `UPDATE` and `DELETE` statements on `principalobjectaccess` table +Enabled use of subqueries within `ON` clause of `JOIN` statements +Added support for `___pid` virtual column for lookups to elastic tables +Improved folding of queries using index spools +Improved primary key calculation when using joins on non-key columns +Apply column order setting to parameters for stored procedures and table-valued functions +Fixed error with DeleteMultiple requests +Fixed paging error with `DISTINCT` queries causing results to be limited to 50,000 records +Fixed paging errors when sorting by optionset values causing some results to be skipped +Fixed errors when using joins inside `[NOT] EXISTS` subqueries +Fixed incorrect results when applying aliases to `___name` and `___type` virtual columns +Fixed max length calculation for string columns +Fixed display of error messages +Fixed "invalid program" errors when combining type conversions with `AND` or `OR` + ## [v9.0.1](https://github.com/MarkMpn/Sql4Cds/releases/tag/v9.0.1) - 2024-05-08 Fixed `NullReferenceException` errors when: diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/ExecutionPlanObjectSource.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/ExecutionPlanObjectSource.cs new file mode 100644 index 00000000..8b7c6f37 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/ExecutionPlanObjectSource.cs @@ -0,0 +1,35 @@ +using System.Text; +using MarkMpn.Sql4Cds.Engine; +using MarkMpn.Sql4Cds.Engine.ExecutionPlan; +using Microsoft.VisualStudio.DebuggerVisualizers; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide +{ + public class ExecutionPlanObjectSource : VisualizerObjectSource + { + public override void GetData(object target, Stream outgoingData) + { + if (target is IRootExecutionPlanNode root) + { + WritePlan(outgoingData, root); + } + else if (target is IExecutionPlanNode node) + { + WritePlan(outgoingData, new UnknownRootNode { Source = node }); + } + else if (target is Sql4CdsCommand cmd) + { + if (cmd.Plan != null) + WritePlan(outgoingData, cmd.Plan.First()); + else + WritePlan(outgoingData, cmd.GeneratePlan(false).First()); + } + } + + private void WritePlan(Stream outgoingData, IRootExecutionPlanNode source) + { + var json = ExecutionPlanSerializer.Serialize(source); + SerializeAsJson(outgoingData, new SerializedPlan { Plan = json }); + } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide.csproj b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide.csproj new file mode 100644 index 00000000..77abd04b --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide.csproj @@ -0,0 +1,19 @@ + + + + net6.0;net462 + enable + enable + latest + + + + + + + + + + + + diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/SerializedPlan.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/SerializedPlan.cs new file mode 100644 index 00000000..19024ed2 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/SerializedPlan.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide +{ + public class SerializedPlan + { + public string Plan { get; set; } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/UnknownRootNode.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/UnknownRootNode.cs new file mode 100644 index 00000000..7cc59105 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide/UnknownRootNode.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MarkMpn.Sql4Cds.Engine.ExecutionPlan; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide +{ + internal class UnknownRootNode : IRootExecutionPlanNode + { + public string Sql { get; set; } + public int Index { get; set; } + public int Length { get; set; } + public int LineNumber { get; set; } + + public IExecutionPlanNode Parent => null; + + public int ExecutionCount => 0; + + public TimeSpan Duration => TimeSpan.Zero; + + public IExecutionPlanNode Source { get; set; } + + public IEnumerable GetSources() + { + yield return Source; + } + + public override string ToString() + { + return "< Unknown Root >"; + } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/.vsextension/string-resources.json b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/.vsextension/string-resources.json new file mode 100644 index 00000000..6fb565e4 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/.vsextension/string-resources.json @@ -0,0 +1,3 @@ +{ + "MarkMpn.Sql4Cds.DebugVisualizer.DisplayName": "SQL 4 CDS Visualizer" +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExecutionPlanDebuggerVisualizerProvider.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExecutionPlanDebuggerVisualizerProvider.cs new file mode 100644 index 00000000..d9e9a85e --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExecutionPlanDebuggerVisualizerProvider.cs @@ -0,0 +1,87 @@ +using System.Diagnostics; +using MarkMpn.Sql4Cds.Engine; +using Microsoft; +using Microsoft.VisualStudio.Extensibility; +using Microsoft.VisualStudio.Extensibility.Commands; +using Microsoft.VisualStudio.Extensibility.DebuggerVisualizers; +using Microsoft.VisualStudio.Extensibility.Shell; +using Microsoft.VisualStudio.Extensibility.VSSdkCompatibility; +using Microsoft.VisualStudio.RpcContracts.RemoteUI; +using Microsoft.VisualStudio.Shell; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide +{ + /// + /// Debugger visualizer provider for . + /// + [VisualStudioContribution] + internal class ExecutionPlanDebuggerVisualizerProvider : DebuggerVisualizerProvider + { + private const string DisplayName = "MarkMpn.Sql4Cds.DebugVisualizer.DisplayName"; + + public override DebuggerVisualizerProviderConfiguration DebuggerVisualizerProviderConfiguration => new( + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.AdaptiveIndexSpoolNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.AliasNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.AssertNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.AssignVariablesNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.BulkDeleteJobNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ComputeScalarNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ConcatenateNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ConditionalNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ConstantScanNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ContinueBreakNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.DeclareVariablesNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.DeleteNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.DistinctNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ExecuteAsNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ExecuteMesageNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.FetchXmlScan, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.FilterNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.GlobalOptionSetQueryNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.GotoLabelNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.GoToNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.HashJoinNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.HashMatchAggregateNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.IndexSpoolNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.InsertNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.MergeJoinNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.MetadataQueryNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.NestedLoopNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.OffsetFetchNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.OpenJsonNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.PartitionedAggregateNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.PrintNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.RaiseErrorNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.RetrieveTotalRecordCountNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.RevertNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.SelectNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.SortNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.SqlNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.StreamAggregateNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.SystemFunctionNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.TableSpoolNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.ThrowNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.TopNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.BeginTryNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.EndTryNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.BeginCatchNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.EndCatchNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.UpdateNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.WaitForNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.ExecutionPlan.XmlWriterNode, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral"), + new VisualizerTargetType($"%{DisplayName}%", "MarkMpn.Sql4Cds.Engine.Sql4CdsCommand, MarkMpn.Sql4Cds.Engine, Version=0.0.0.0, Culture=neutral")) + { + VisualizerObjectSourceType = new("MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide.ExecutionPlanObjectSource, MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide") + }; + + public override async Task CreateVisualizerAsync(VisualizerTarget visualizerTarget, CancellationToken cancellationToken) + { + var serializedPlan = await visualizerTarget.ObjectSource.RequestDataAsync(null, cancellationToken); + var plan = ExecutionPlanSerializer.Deserialize(serializedPlan.Plan); + + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + var wrapper = new WpfControlWrapper(new QueryPlanUserControl(plan)); + return wrapper; + } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExtensionEntrypoint.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExtensionEntrypoint.cs new file mode 100644 index 00000000..614ca611 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/ExtensionEntrypoint.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.Extensibility; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide +{ + /// + /// Extension entrypoint for the VisualStudio.Extensibility extension. + /// + [VisualStudioContribution] + internal class ExtensionEntrypoint : Extension + { + /// + public override ExtensionConfiguration ExtensionConfiguration => new() + { + RequiresInProcessHosting = true + }; + + /// + protected override void InitializeServices(IServiceCollection serviceCollection) + { + base.InitializeServices(serviceCollection); + + // You can configure dependency injection here by adding services to the serviceCollection. + } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/Images/IconSmall.png b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/Images/IconSmall.png new file mode 100644 index 00000000..62e059bc Binary files /dev/null and b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/Images/IconSmall.png differ diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide.csproj b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide.csproj new file mode 100644 index 00000000..decc789f --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide.csproj @@ -0,0 +1,48 @@ + + + net472 + enable + enable + latest + true + + + + + + + + + + + + + MSBuild:Compile + + + + + + + + + runtime + + + + + + + + + + + + + + + + + + + diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml new file mode 100644 index 00000000..ac13c802 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml.cs new file mode 100644 index 00000000..bec02be5 --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/QueryPlanUserControl.xaml.cs @@ -0,0 +1,58 @@ +using MarkMpn.Sql4Cds.Engine; +using MarkMpn.Sql4Cds.Engine.ExecutionPlan; +using Microsoft.VisualStudio.Extensibility.DebuggerVisualizers; +using Microsoft.VisualStudio.PlatformUI; +//using Microsoft.Web.WebView2.Core; +using System.Buffers; +using System.Diagnostics; +using System.Text; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Forms; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide +{ + public partial class QueryPlanUserControl : System.Windows.Controls.UserControl + { + private readonly IRootExecutionPlanNode _plan; + + public QueryPlanUserControl(IRootExecutionPlanNode plan) + { + InitializeComponent(); + DataContext = this; + + _plan = plan; + this.Loaded += QueryPlanUserControl_Loaded; + } + + private void QueryPlanUserControl_Loaded(object sender, RoutedEventArgs e) + { + // Create the interop host control. + var host = new System.Windows.Forms.Integration.WindowsFormsHost(); + + // Create the control. + var control = new MarkMpn.Sql4Cds.Controls.ExecutionPlanView { Plan = _plan }; + control.Dock = DockStyle.Fill; + var propertyGrid = new PropertyGrid(); + propertyGrid.Dock = DockStyle.Fill; + control.NodeSelected += (s, e) => + { + if (control.Selected == null) + propertyGrid.SelectedObject = null; + else + propertyGrid.SelectedObject = new ExecutionPlanNodeTypeDescriptor(control.Selected, true, null); + }; + var splitter = new SplitContainer { Dock = DockStyle.Fill }; + splitter.Panel1.Controls.Add(control); + splitter.Panel2.Controls.Add(propertyGrid); + splitter.FixedPanel = FixedPanel.Panel2; + + // Assign the MaskedTextBox control as the host control's child. + host.Child = splitter; + + // Add the interop host control to the Grid + // control's collection of child controls. + this.grid.Children.Add(host); + } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/SerializedPlan.cs b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/SerializedPlan.cs new file mode 100644 index 00000000..2521ea6c --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/SerializedPlan.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide +{ + public class SerializedPlan + { + public string Plan { get; set; } + } +} diff --git a/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/source.extension.vsixmanifest b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/source.extension.vsixmanifest new file mode 100644 index 00000000..656133fa --- /dev/null +++ b/MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide/source.extension.vsixmanifest @@ -0,0 +1,39 @@ + + + + + SQL 4 CDS Visualizer + View SQL 4 CDS query plan directly inside Visual Studio. + false + https://github.com/MarkMpn.Sql4Cds + https://github.com/MarkMpn/Sql4Cds/blob/master/README.md + Images\IconSmall.png + SQL 4 CDS,SQL4CDS, Visualizer, Query Plan + + + + amd64 + + + arm64 + + + amd64 + + + arm64 + + + amd64 + + + arm64 + + + + + + + + + \ No newline at end of file diff --git a/MarkMpn.Sql4Cds.Engine.Tests/CteTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/CteTests.cs index f1c51508..31eb0aff 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/CteTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/CteTests.cs @@ -597,6 +597,9 @@ UNION ALL + + + "); } diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanNodeTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanNodeTests.cs index 09c12097..86dbaf33 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanNodeTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanNodeTests.cs @@ -1044,5 +1044,232 @@ public void NestedLoopJoinLeftOuterTest() Assert.AreEqual("Joe", ((SqlString)results[2]["f.firstname"]).Value); Assert.IsTrue(((SqlString)results[2]["l.lastname"]).IsNull); } + + [TestMethod] + public void FetchXmlSingleTablePrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.AreEqual("account.accountid", schema.PrimaryKey); + } + + [TestMethod] + public void FetchXmlChildTablePrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" }, + new FetchXml.FetchLinkEntityType + { + name = "contact", + from = "parentcustomerid", + to = "accountid", + alias = "contact", + linktype = "inner", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "contactid" }, + new FetchXml.FetchAttributeType { name = "fullname" } + } + } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.AreEqual("contact.contactid", schema.PrimaryKey); + } + + [TestMethod] + public void FetchXmlChildTableOuterJoinPrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" }, + new FetchXml.FetchLinkEntityType + { + name = "contact", + from = "parentcustomerid", + to = "accountid", + alias = "contact", + linktype = "outer", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "contactid" }, + new FetchXml.FetchAttributeType { name = "fullname" } + } + } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.IsNull(schema.PrimaryKey); + } + + [TestMethod] + public void FetchXmlParentTablePrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" }, + new FetchXml.FetchLinkEntityType + { + name = "contact", + from = "contactid", + to = "primarycontactid", + alias = "contact", + linktype = "inner", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "contactid" }, + new FetchXml.FetchAttributeType { name = "fullname" } + } + } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.AreEqual("account.accountid", schema.PrimaryKey); + } + + [TestMethod] + public void FetchXmlParentTableOuterJoinPrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" }, + new FetchXml.FetchLinkEntityType + { + name = "contact", + from = "contactid", + to = "primarycontactid", + alias = "contact", + linktype = "inner", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "contactid" }, + new FetchXml.FetchAttributeType { name = "fullname" } + } + } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.AreEqual("account.accountid", schema.PrimaryKey); + } + + [TestMethod] + public void FetchXmlChildTableFreeTextJoinPrimaryKey() + { + var fetch = new FetchXmlScan + { + DataSource = _localDataSource.Name, + Alias = "account", + FetchXml = new FetchXml.FetchType + { + Items = new object[] + { + new FetchXml.FetchEntityType + { + name = "account", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "accountid" }, + new FetchXml.FetchAttributeType { name = "name" }, + new FetchXml.FetchLinkEntityType + { + name = "contact", + from = "fullname", + to = "name", + alias = "contact", + linktype = "inner", + Items = new object[] + { + new FetchXml.FetchAttributeType { name = "contactid" }, + new FetchXml.FetchAttributeType { name = "fullname" } + } + } + } + } + } + } + }; + var schema = fetch.GetSchema(new NodeCompilationContext(_localDataSources, new StubOptions(), null, null)); + Assert.IsNull(schema.PrimaryKey); + } } } diff --git a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs index 854836db..230e7084 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs @@ -2823,8 +2823,7 @@ ORDER BY firstname "); var innerAlias = AssertNode(loop.RightSource); var innerTop = AssertNode(innerAlias.Source); - var innerSort = AssertNode(innerTop.Source); - var innerIndexSpool = AssertNode(innerSort.Source); + var innerIndexSpool = AssertNode(innerTop.Source); Assert.AreEqual("contact.parentcustomerid", innerIndexSpool.KeyColumn); Assert.AreEqual("@Expr1", innerIndexSpool.SeekValue); var innerFetch = AssertNode(innerIndexSpool.Source); @@ -2837,6 +2836,7 @@ ORDER BY firstname + "); } @@ -3714,7 +3714,7 @@ GROUP BY firstname }, }; - var result = select.Execute(new NodeExecutionContext(_localDataSources, this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var result = select.Execute(new NodeExecutionContext(_localDataSources, this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(result); @@ -4192,7 +4192,7 @@ DECLARE @test int Assert.AreEqual(1, selectConstantScan.Values.Count); var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -4246,7 +4246,7 @@ public void SetVariableInDeclaration() Assert.AreEqual(1, selectConstantScan.Values.Count); var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -4304,7 +4304,7 @@ DECLARE @test varchar(3) var plans = planBuilder.Build(query, null, out _); var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -4338,7 +4338,7 @@ DECLARE @test varchar(3) var plans = planBuilder.Build(query, null, out _); var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -4430,7 +4430,7 @@ DECLARE @test varchar(3) }; var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -4464,7 +4464,7 @@ DECLARE @test varchar var plans = planBuilder.Build(query, null, out _); var parameterTypes = new Dictionary(); - var parameterValues = new Dictionary(); + var parameterValues = new Dictionary(); foreach (var plan in plans) { @@ -5330,13 +5330,14 @@ public void DistinctOrderByOptionSet() Assert.AreEqual(1, plans.Length); var select = AssertNode(plans[0]); - var fetch = AssertNode(select.Source); + var sort = AssertNode(select.Source); + Assert.AreEqual("new_customentity.new_optionsetvalue", sort.Sorts[0].ToSql()); + var fetch = AssertNode(sort.Source); AssertFetchXml(fetch, @" - + - "); } @@ -6780,10 +6781,13 @@ AND [union. all].logicalname IN ('createdon') var sort = AssertNode(select.Source); - var join1 = AssertNode(sort.Source); + var filter1 = AssertNode(sort.Source); + Assert.AreEqual("[union. all].logicalname = a2.logicalname", filter1.Filter.ToSql()); + + var join1 = AssertNode(filter1.Source); Assert.AreEqual("a2.entitylogicalname", join1.LeftAttribute.ToSql()); Assert.AreEqual("[union. all].eln", join1.RightAttribute.ToSql()); - Assert.AreEqual("[union. all].logicalname = a2.logicalname", join1.AdditionalJoinCriteria.ToSql()); + Assert.IsNull(join1.AdditionalJoinCriteria); var mq1 = AssertNode(join1.LeftSource); Assert.AreEqual("french", mq1.DataSource); @@ -7210,33 +7214,36 @@ public void OrderByOptionSetName() } [TestMethod] - public void OrderByOptionSetValue() + public void OrderByOptionSetValueWithUseRawOrderBy() { - var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + using (_localDataSource.SetUseRawOrderByReliable(true)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); - var query = @"SELECT new_customentityid FROM new_customentity ORDER BY new_optionsetvalue"; + var query = @"SELECT new_customentityid FROM new_customentity ORDER BY new_optionsetvalue"; - var plans = planBuilder.Build(query, null, out _); + var plans = planBuilder.Build(query, null, out _); - Assert.AreEqual(1, plans.Length); + Assert.AreEqual(1, plans.Length); - var select = AssertNode(plans[0]); - var fetch = AssertNode(select.Source); - AssertFetchXml(fetch, @" - - - - - - "); + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + AssertFetchXml(fetch, @" + + + + + + "); + } } [TestMethod] - public void OrderByOptionSetValueAndName() + public void OrderByOptionSetValueWithoutUseRawOrderBy() { var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); - var query = @"SELECT new_customentityid FROM new_customentity ORDER BY new_optionsetvalue, new_optionsetvaluename"; + var query = @"SELECT new_customentityid FROM new_customentity ORDER BY new_optionsetvalue"; var plans = planBuilder.Build(query, null, out _); @@ -7244,20 +7251,47 @@ public void OrderByOptionSetValueAndName() var select = AssertNode(plans[0]); var sort = AssertNode(select.Source); - Assert.AreEqual(1, sort.PresortedCount); - Assert.AreEqual(2, sort.Sorts.Count); - Assert.AreEqual("new_customentity.new_optionsetvaluename", sort.Sorts[1].Expression.ToSql()); + Assert.AreEqual("new_customentity.new_optionsetvalue", sort.Sorts[0].Expression.ToSql()); var fetch = AssertNode(sort.Source); AssertFetchXml(fetch, @" - + - "); } + [TestMethod] + public void OrderByOptionSetValueAndName() + { + using (_localDataSource.SetUseRawOrderByReliable(true)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @"SELECT new_customentityid FROM new_customentity ORDER BY new_optionsetvalue, new_optionsetvaluename"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var sort = AssertNode(select.Source); + Assert.AreEqual(1, sort.PresortedCount); + Assert.AreEqual(2, sort.Sorts.Count); + Assert.AreEqual("new_customentity.new_optionsetvaluename", sort.Sorts[1].Expression.ToSql()); + var fetch = AssertNode(sort.Source); + AssertFetchXml(fetch, @" + + + + + + + "); + } + } + [TestMethod] public void ExistsOrInAndColumnComparisonOrderByEntityName() { @@ -7396,5 +7430,408 @@ ORDER BY "); } } + + [TestMethod] + public void DistinctUsesCustomPaging() + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select distinct +account.name, contact.firstname +from account +left outer join contact ON account.accountid = contact.parentcustomerid"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + + + +"); + Assert.IsTrue(fetch.UsingCustomPaging); + } + + [TestMethod] + public void NotExistWithJoin() + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select top 10 a2.name +from account a2 +where not exists ( + select top 10 a.accountid + from account a + inner join contact c on c.parentcustomerid = a.accountid + where a.accountid = a2.accountid +) +"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var top = AssertNode(select.Source); + var filter = AssertNode(top.Source); + var join = AssertNode(filter.Source); + var fetch1 = AssertNode(join.LeftSource); + var fetch2 = AssertNode(join.RightSource); + + Assert.AreEqual("a2", fetch1.Alias); + AssertFetchXml(fetch1, @" + + + + + + +"); + Assert.IsFalse(fetch1.UsingCustomPaging); + + Assert.AreEqual("Expr2", fetch2.Alias); + AssertFetchXml(fetch2, @" + + + + + + + +"); + Assert.IsTrue(fetch2.UsingCustomPaging); + Assert.AreEqual(1, fetch2.ColumnMappings.Count); + Assert.AreEqual("Expr2.accountid", fetch2.ColumnMappings[0].OutputColumn); + Assert.AreEqual("a.accountid", fetch2.ColumnMappings[0].SourceColumn); + + Assert.AreEqual("a2.accountid", join.LeftAttribute.ToSql()); + Assert.AreEqual("Expr2.accountid", join.RightAttribute.ToSql()); + Assert.AreEqual(QualifiedJoinType.LeftOuter, join.JoinType); + Assert.IsTrue(join.SemiJoin); + Assert.AreEqual(1, join.DefinedValues.Count); + Assert.AreEqual("Expr2.accountid", join.DefinedValues["Expr3"]); + + Assert.AreEqual("Expr3 IS NULL", filter.Filter.ToSql()); + + Assert.AreEqual("10", top.Top.ToSql()); + + Assert.AreEqual(1, select.ColumnSet.Count); + Assert.AreEqual("a2.name", select.ColumnSet[0].SourceColumn); + Assert.AreEqual("name", select.ColumnSet[0].OutputColumn); + } + + [TestMethod] + public void ScalarSubquery() + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select top 10 * from ( +select fullname, (select name from account where accountid = parentcustomerid) from contact +) a"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + +"); + } + + [TestMethod] + public void SubqueryInJoinCriteriaRHS() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +inner join contact ON account.accountid = contact.parentcustomerid AND contact.firstname IN (SELECT new_name FROM new_customentity)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + + +"); + } + } + + [TestMethod] + public void SubqueryInJoinCriteriaRHSCorrelatedExists() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +inner join contact ON account.accountid = contact.parentcustomerid AND EXISTS(SELECT * FROM new_customentity WHERE new_name = contact.firstname)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + + +"); + } + } + + [TestMethod] + public void SubqueryInJoinCriteriaLHS() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +inner join contact ON account.accountid = contact.parentcustomerid AND account.name IN (SELECT new_name FROM new_customentity)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + + +"); + } + } + + [TestMethod] + public void SubqueryInJoinCriteriaLHSCorrelatedExists() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +inner join contact ON account.accountid = contact.parentcustomerid AND EXISTS(SELECT * FROM new_customentity WHERE new_name = account.name)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + + + + +"); + } + } + + [TestMethod] + public void SubqueryInJoinCriteriaLHSAndRHSInnerJoin() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +inner join contact ON account.accountid = contact.parentcustomerid AND contact.fullname IN (SELECT new_name FROM new_customentity WHERE account.turnover = new_customentity.new_decimalprop)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var filter = AssertNode(select.Source); + Assert.AreEqual("Expr2 IS NOT NULL", filter.Filter.ToSql()); + var loop = AssertNode(filter.Source); + Assert.AreEqual(QualifiedJoinType.LeftOuter, loop.JoinType); + Assert.IsTrue(loop.SemiJoin); + Assert.AreEqual("contact.fullname = new_customentity.new_name", loop.JoinCondition.ToSql()); + Assert.AreEqual(1, loop.OuterReferences.Count); + Assert.AreEqual("@Expr1", loop.OuterReferences["account.turnover"]); + Assert.AreEqual("new_customentity.new_name", loop.DefinedValues["Expr2"]); + var fetch1 = AssertNode(loop.LeftSource); + AssertFetchXml(fetch1, @" + + + + + + + +"); + var spool = AssertNode(loop.RightSource); + Assert.AreEqual("new_customentity.new_decimalprop", spool.KeyColumn); + Assert.AreEqual("@Expr1", spool.SeekValue); + var fetch2 = AssertNode(spool.Source); + AssertFetchXml(fetch2, @" + + + + + + + + +"); + } + } + + [TestMethod] + public void SubqueryInJoinCriteriaLHSAndRHSOuterJoin() + { + using (_localDataSource.EnableJoinOperator(JoinOperator.In)) + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select +* +from account +left outer join contact ON account.accountid = contact.parentcustomerid AND contact.fullname IN (SELECT new_name FROM new_customentity WHERE account.turnover = new_customentity.new_decimalprop)"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var loop1 = AssertNode(select.Source); + Assert.AreEqual(QualifiedJoinType.LeftOuter, loop1.JoinType); + Assert.AreEqual("account.accountid = contact.parentcustomerid\r\nAND Expr3 IS NOT NULL", loop1.JoinCondition.ToSql()); + Assert.IsFalse(loop1.SemiJoin); + Assert.AreEqual(1, loop1.OuterReferences.Count); + Assert.AreEqual("@Expr1", loop1.OuterReferences["account.turnover"]); + var fetch1 = AssertNode(loop1.LeftSource); + AssertFetchXml(fetch1, @" + + + + +"); + var merge = AssertNode(loop1.RightSource); + Assert.AreEqual(1, merge.DefinedValues.Count); + Assert.AreEqual("Expr2.new_name", merge.DefinedValues["Expr3"]); + var tableSpool = AssertNode(merge.LeftSource); + var fetch2 = AssertNode(tableSpool.Source); + AssertFetchXml(fetch2, @" + + + + + +"); + var sort = AssertNode(merge.RightSource); + var distinct = AssertNode(sort.Source); + var alias = AssertNode(distinct.Source); + var spool = AssertNode(alias.Source); + Assert.AreEqual("new_customentity.new_decimalprop", spool.KeyColumn); + Assert.AreEqual("@Expr1", spool.SeekValue); + var fetch3 = AssertNode(spool.Source); + AssertFetchXml(fetch3, @" + + + + + + + + +"); + + } + } + + [TestMethod] + public void VirtualAttributeAliases() + { + var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); + + var query = @" +select statecodename [state], parentcustomerid x, parentcustomeridname from contact"; + + var plans = planBuilder.Build(query, null, out _); + + Assert.AreEqual(1, plans.Length); + + var select = AssertNode(plans[0]); + var fetch = AssertNode(select.Source); + + AssertFetchXml(fetch, @" + + + + + +"); + } } } diff --git a/MarkMpn.Sql4Cds.Engine.Tests/FakeXrmDataSource.cs b/MarkMpn.Sql4Cds.Engine.Tests/FakeXrmDataSource.cs index 52395a19..5ce9a63b 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/FakeXrmDataSource.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/FakeXrmDataSource.cs @@ -29,6 +29,7 @@ public void Dispose() private bool _columnComparisonAvailable = true; private bool _orderByEntityNameAvailable = false; + private bool _useRawOrderByReliable = false; private List _joinOperators; public FakeXrmDataSource() @@ -42,6 +43,8 @@ public FakeXrmDataSource() public override List JoinOperatorsAvailable => _joinOperators; + public override bool UseRawOrderByReliable => _useRawOrderByReliable; + public IDisposable SetColumnComparison(bool enable) { var original = _columnComparisonAvailable; @@ -54,6 +57,12 @@ public IDisposable SetOrderByEntityName(bool enable) return new Reset(this, x => x._orderByEntityNameAvailable = enable, x => x._orderByEntityNameAvailable = original); } + public IDisposable SetUseRawOrderByReliable(bool enable) + { + var original = _useRawOrderByReliable; + return new Reset(this, x => x._useRawOrderByReliable = enable, x => x._useRawOrderByReliable = original); + } + public IDisposable EnableJoinOperator(JoinOperator op) { var add = !JoinOperatorsAvailable.Contains(op); diff --git a/MarkMpn.Sql4Cds.Engine.Tests/Sql2FetchXmlTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/Sql2FetchXmlTests.cs index 8e601557..1f94a70a 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/Sql2FetchXmlTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/Sql2FetchXmlTests.cs @@ -538,11 +538,13 @@ public void InvalidSortOnLinkEntity() + - + + "); @@ -702,7 +704,7 @@ public void SelectArithmetic() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -752,7 +754,7 @@ public void WhereComparingTwoFields() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -797,7 +799,7 @@ public void WhereComparingExpression() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -838,7 +840,7 @@ public void BackToFrontLikeExpression() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -873,7 +875,7 @@ public void UpdateFieldToField() } }; - ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); + ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); Assert.AreEqual("Carrington", _context.Data["contact"][guid]["firstname"]); } @@ -905,7 +907,7 @@ public void UpdateFieldToExpression() } }; - ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); + ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); Assert.AreEqual("Hello Carrington", _context.Data["contact"][guid]["firstname"]); } @@ -941,7 +943,7 @@ public void UpdateReplace() } }; - ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); + ((UpdateNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); Assert.AreEqual("--CDS--", _context.Data["contact"][guid]["firstname"]); } @@ -972,7 +974,7 @@ public void StringFunctions() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1012,7 +1014,7 @@ public void SelectExpression() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1062,7 +1064,7 @@ public void SelectExpressionNullValues() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1107,7 +1109,7 @@ public void OrderByExpression() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1152,7 +1154,7 @@ public void OrderByAliasedField() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1196,7 +1198,7 @@ public void OrderByCalculatedField() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1240,7 +1242,7 @@ public void OrderByCalculatedFieldByIndex() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1282,7 +1284,7 @@ public void DateCalculations() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1370,7 +1372,7 @@ public void CustomFilterAggregateHavingProjectionSortAndTop() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1433,7 +1435,7 @@ public void FilterCaseInsensitive() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1490,7 +1492,7 @@ public void GroupCaseInsensitive() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1543,7 +1545,7 @@ public void AggregateExpressionsWithoutGrouping() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1598,7 +1600,7 @@ public void AggregateQueryProducesAlternative() } }; - var dataReader = alternativeQuery.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = alternativeQuery.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1654,7 +1656,7 @@ public void GuidEntityReferenceInequality() } }; - var dataReader = select.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = select.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -1706,7 +1708,7 @@ public void UpdateGuidToEntityReference() } }; - update.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); + update.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), out _, out _); Assert.AreEqual(new EntityReference("contact", contact1), _context.Data["account"][account1].GetAttributeValue("primarycontactid")); Assert.AreEqual(new EntityReference("contact", contact2), _context.Data["account"][account2].GetAttributeValue("primarycontactid")); @@ -1721,9 +1723,8 @@ public void CompareDateFields() var queries = planBuilder.Build(query, null, out _); AssertFetchXml(queries, @" - + - @@ -1731,7 +1732,6 @@ public void CompareDateFields() - "); @@ -1981,7 +1981,7 @@ public void ImplicitTypeConversion() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2014,7 +2014,7 @@ public void ImplicitTypeConversionComparison() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2038,7 +2038,7 @@ public void GlobalOptionSet() Assert.AreEqual("globaloptionset.name = 'test'", filterNode.Filter.ToSql()); var optionsetNode = (GlobalOptionSetQueryNode)filterNode.Source; - var dataReader = selectNode.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = selectNode.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2059,7 +2059,7 @@ public void EntityDetails() var sortNode = (SortNode)selectNode.Source; var metadataNode = (MetadataQueryNode)sortNode.Source; - var dataReader = selectNode.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = selectNode.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2077,7 +2077,7 @@ public void AttributeDetails() var planBuilder = new ExecutionPlanBuilder(_localDataSources.Values, this); var queries = planBuilder.Build(query, null, out _); - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2135,7 +2135,7 @@ public void OptionSetNameSelect() CollectionAssert.AreEqual(new[] { "new_optionsetvalue", "new_optionsetvaluename" }, select.ColumnSet.Select(c => c.OutputColumn).ToList()); - var dataReader = select.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = select.Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2459,7 +2459,7 @@ public void CharIndex() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2489,7 +2489,7 @@ public void CastDateTimeToDate() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); @@ -2527,7 +2527,7 @@ public void GroupByPrimaryFunction() } }; - var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); + var dataReader = ((SelectNode)queries[0]).Execute(new NodeExecutionContext(GetDataSources(_context), this, new Dictionary(), new Dictionary(), null), CommandBehavior.Default); var dataTable = new DataTable(); dataTable.Load(dataReader); diff --git a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsCommand.cs b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsCommand.cs index f491f2b6..6987088f 100644 --- a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsCommand.cs +++ b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsCommand.cs @@ -29,6 +29,13 @@ public class Sql4CdsCommand : DbCommand private bool _cancelledManually; private string _lastDatabase; + static Sql4CdsCommand() + { + // Ensure the FetchXmlScan class is loaded - avoids multithreading issues when using the custom debug visualizer + // on the command object. + new FetchXmlScan(); + } + public Sql4CdsCommand(Sql4CdsConnection connection) : this(connection, string.Empty) { } diff --git a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsConnection.cs b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsConnection.cs index 15578c28..167f66ac 100644 --- a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsConnection.cs +++ b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsConnection.cs @@ -29,7 +29,7 @@ public class Sql4CdsConnection : DbConnection private readonly IDictionary _dataSources; private readonly ChangeDatabaseOptionsWrapper _options; private readonly Dictionary _globalVariableTypes; - private readonly Dictionary _globalVariableValues; + private readonly Dictionary _globalVariableValues; private readonly TelemetryClient _ai; /// @@ -74,7 +74,7 @@ public Sql4CdsConnection(IDictionary dataSources) ["@@VERSION"] = DataTypeHelpers.NVarChar(Int32.MaxValue, _dataSources[_options.PrimaryDataSource].DefaultCollation, CollationLabel.CoercibleDefault), ["@@ERROR"] = DataTypeHelpers.Int, }; - _globalVariableValues = new Dictionary(StringComparer.OrdinalIgnoreCase) + _globalVariableValues = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["@@IDENTITY"] = SqlEntityReference.Null, ["@@ROWCOUNT"] = (SqlInt32)0, @@ -291,7 +291,7 @@ public ColumnOrdering ColumnOrdering internal Dictionary GlobalVariableTypes => _globalVariableTypes; - internal Dictionary GlobalVariableValues => _globalVariableValues; + internal Dictionary GlobalVariableValues => _globalVariableValues; internal TelemetryClient TelemetryClient => _ai; diff --git a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsDataReader.cs b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsDataReader.cs index 3354c182..a88c27d0 100644 --- a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsDataReader.cs +++ b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsDataReader.cs @@ -22,7 +22,7 @@ class Sql4CdsDataReader : DbDataReader private readonly IQueryExecutionOptions _options; private readonly CommandBehavior _behavior; private readonly Dictionary _parameterTypes; - private readonly Dictionary _parameterValues; + private readonly Dictionary _parameterValues; private Dictionary _labelIndexes; private int _recordsAffected; private int _instructionPointer; @@ -57,7 +57,7 @@ public Sql4CdsDataReader(Sql4CdsCommand command, IQueryExecutionOptions options, Close(); } - internal Dictionary ParameterValues => _parameterValues; + internal Dictionary ParameterValues => _parameterValues; private Dictionary LabelIndexes { @@ -75,7 +75,7 @@ private Dictionary LabelIndexes } } - private bool ExecuteWithExceptionHandling(Dictionary parameterTypes, Dictionary parameterValues) + private bool ExecuteWithExceptionHandling(Dictionary parameterTypes, Dictionary parameterValues) { while (true) { @@ -109,7 +109,7 @@ private bool ExecuteWithExceptionHandling(Dictionary } } - private bool Execute(Dictionary parameterTypes, Dictionary parameterValues) + private bool Execute(Dictionary parameterTypes, Dictionary parameterValues) { IRootExecutionPlanNode logNode = null; var context = new NodeExecutionContext(_connection.DataSources, _options, parameterTypes, parameterValues, msg => _connection.OnInfoMessage(logNode, msg)); diff --git a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsParameterCollection.cs b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsParameterCollection.cs index c5ea90eb..705df781 100644 --- a/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsParameterCollection.cs +++ b/MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsParameterCollection.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Data.Common; +using System.Data.SqlTypes; using System.Linq; using System.Text; using Microsoft.SqlServer.TransactSql.ScriptDom; @@ -50,11 +51,11 @@ internal Dictionary GetParameterTypes() .ToDictionary(param => param.FullParameterName, param => param.GetDataType(), StringComparer.OrdinalIgnoreCase); } - internal Dictionary GetParameterValues() + internal Dictionary GetParameterValues() { return _parameters .Cast() - .ToDictionary(param => param.FullParameterName, param => (object) param.GetValue(), StringComparer.OrdinalIgnoreCase); + .ToDictionary(param => param.FullParameterName, param => param.GetValue(), StringComparer.OrdinalIgnoreCase); } public override bool Contains(string value) diff --git a/MarkMpn.Sql4Cds.Engine/AttributeMetadataCache.cs b/MarkMpn.Sql4Cds.Engine/AttributeMetadataCache.cs index f13e619b..095e7104 100644 --- a/MarkMpn.Sql4Cds.Engine/AttributeMetadataCache.cs +++ b/MarkMpn.Sql4Cds.Engine/AttributeMetadataCache.cs @@ -2,6 +2,7 @@ using Microsoft.Xrm.Sdk.Messages; using Microsoft.Xrm.Sdk.Metadata; using Microsoft.Xrm.Sdk.Metadata.Query; +using Microsoft.Xrm.Sdk.Query; using System; using System.Collections.Generic; using System.Linq; @@ -21,6 +22,7 @@ public class AttributeMetadataCache : IAttributeMetadataCache private readonly IDictionary _minimalMetadata; private readonly ISet _minimalLoading; private readonly IDictionary _invalidEntities; + private readonly Lazy _recycleBinEntities; /// /// Creates a new @@ -34,6 +36,46 @@ public AttributeMetadataCache(IOrganizationService org) _minimalMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase); _minimalLoading = new HashSet(StringComparer.OrdinalIgnoreCase); _invalidEntities = new Dictionary(StringComparer.OrdinalIgnoreCase); + _recycleBinEntities = new Lazy(() => + { + // Check the recyclebinconfig entity exists + try + { + _ = this["recyclebinconfig"]; + } + catch + { + return null; + } + + // https://learn.microsoft.com/en-us/power-apps/developer/data-platform/restore-deleted-records?tabs=sdk#detect-which-tables-are-enabled-for-recycle-bin + var qry = new FetchExpression(@" + + + + + + + + + + + + "); + + var resp = _org.RetrieveMultiple(qry); + return resp.Entities + .Select(e => e.GetAttributeValue("entity.logicalname").Value as string) + .ToArray(); + }); } /// @@ -118,6 +160,7 @@ public bool TryGetValue(string logicalName, out EntityMetadata metadata) return false; } + /// public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata) { if (_metadata.TryGetValue(logicalName, out metadata)) @@ -202,9 +245,11 @@ public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata) } return false; - } + /// + public string[] RecycleBinEntities => _recycleBinEntities.Value; + public event EventHandler MetadataLoading; protected void OnMetadataLoading(MetadataLoadingEventArgs args) diff --git a/MarkMpn.Sql4Cds.Engine/DataSource.cs b/MarkMpn.Sql4Cds.Engine/DataSource.cs index bd546a85..739b8976 100644 --- a/MarkMpn.Sql4Cds.Engine/DataSource.cs +++ b/MarkMpn.Sql4Cds.Engine/DataSource.cs @@ -137,6 +137,11 @@ public DataSource() /// public virtual bool OrderByEntityNameAvailable { get; } + /// + /// Indicates if the server can reliably page results when using + /// + public virtual bool UseRawOrderByReliable { get; } + /// /// Returns a list of join operators that are supported by the server /// diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs index 9b6b45ec..614b34b8 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.SqlServer.TransactSql.ScriptDom; using Microsoft.Xrm.Sdk; +using Newtonsoft.Json; namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan { @@ -21,6 +22,9 @@ class AliasNode : BaseDataNode, ISingleSourceExecutionPlanNode /// The alias to use for the subquery public AliasNode(SelectNode select, Identifier identifier, NodeCompilationContext context) { + if (select == null) + return; + ColumnSet.AddRange(select.ColumnSet); Source = select.Source; Alias = identifier.Value; @@ -74,6 +78,7 @@ private AliasNode() /// The schema that shold be used for expanding "*" columns /// [Browsable(false)] + [JsonIgnore] public INodeSchema LogicalSourceSchema { get; set; } public override void AddRequiredColumns(NodeCompilationContext context, IList requiredColumns) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AssignVariablesNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AssignVariablesNode.cs index 7f65da59..af59ef61 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AssignVariablesNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AssignVariablesNode.cs @@ -54,7 +54,7 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect foreach (var entity in entities) { foreach (var variable in Variables) - context.ParameterValues[variable.VariableName] = valueAccessors[variable.VariableName](entity); + context.ParameterValues[variable.VariableName] = (INullable)valueAccessors[variable.VariableName](entity); count++; } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs index a8910590..496bc6b4 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs @@ -851,15 +851,12 @@ private bool TranslateFetchXMLCriteriaWithVirtualAttributes(NodeCompilationConte var attribute = meta.Attributes.SingleOrDefault(a => a.LogicalName.Equals(attrName, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); string attributeSuffix = null; - if (attribute == null && (attrName.EndsWith("name", StringComparison.OrdinalIgnoreCase) || attrName.EndsWith("type", StringComparison.OrdinalIgnoreCase))) + if (attribute == null) { - attribute = meta.Attributes.SingleOrDefault(a => a.LogicalName.Equals(attrName.Substring(0, attrName.Length - 4), StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); + attribute = meta.FindBaseAttributeFromVirtualAttribute(attrName, out attributeSuffix); if (attribute != null) - { - attributeSuffix = attrName.Substring(attrName.Length - 4).ToLower(); attrName = attribute.LogicalName; - } } // Can't fold LIKE queries for non-string fields - the server will try to convert the value to the type of @@ -1084,7 +1081,7 @@ private bool TranslateFetchXMLCriteriaWithVirtualAttributes(NodeCompilationConte if (attribute is LookupAttributeMetadata lookupAttr) { - // Check the real name of the underlying virtual attribute. We use the consistent suffixes "name" and "type" but + // Check the real name of the underlying virtual attribute. We use the consistent suffixes "name", "type" and "pid" but // it's not always the same under the hood. if (attributeSuffix == "name") { @@ -1097,21 +1094,29 @@ private bool TranslateFetchXMLCriteriaWithVirtualAttributes(NodeCompilationConte .OrderBy(a => a.LogicalName == attrName + "name" ? 0 : 1) .FirstOrDefault(); } - else + else if (attributeSuffix == "type") { attribute = meta.Attributes .SingleOrDefault(a => a.AttributeOf == attrName && a.AttributeType == AttributeTypeCode.EntityName); } + else + { + attribute = meta.Attributes + .OfType() + .Where(a => a.AttributeOf == attrName && a.AttributeType == AttributeTypeCode.String && a.YomiOf == null) + .OrderBy(a => a.LogicalName == attrName + "pid" ? 0 : 1) + .FirstOrDefault(); + } if (attribute != null) { attrName = attribute.LogicalName; - if (attributeSuffix == "name") + if (attributeSuffix == "name" || attributeSuffix == "pid") { virtualAttributeHandled = true; } - else + else if (attributeSuffix == "type") { // Type attributes can only handle a limited set of operators if (op == @operator.@null || op == @operator.notnull || op == @operator.eq || op == @operator.ne || op == @operator.@in || op == @operator.notin) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs index 8f7624a4..711558a1 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs @@ -400,6 +400,9 @@ protected Dictionary> CompileColumnMappings(DataSou if (!attributes.TryGetValue(destAttributeName, out var attr) || attr.AttributeOf != null) continue; + if (metadata.LogicalName == "principalobjectaccess" && (attr.LogicalName == "objecttypecode" || attr.LogicalName == "principaltypecode")) + continue; + var sourceSqlType = schema.Schema[sourceColumnName].Type; var destType = attr.GetAttributeType(); var destSqlType = attr.IsPrimaryId == true ? DataTypeHelpers.UniqueIdentifier : attr.GetAttributeSqlType(dataSource, true); @@ -425,8 +428,10 @@ protected Dictionary> CompileColumnMappings(DataSou expr = Expression.Convert(expr, sourceSqlType.ToNetType(out _)); Expression convertedExpr; + var lookupAttr = attr as LookupAttributeMetadata; - if (attr is LookupAttributeMetadata lookupAttr && lookupAttr.AttributeType != AttributeTypeCode.PartyList && metadata.IsIntersect != true) + if (lookupAttr != null && lookupAttr.AttributeType != AttributeTypeCode.PartyList && metadata.IsIntersect != true || + metadata.LogicalName == "principalobjectaccess" && (attr.LogicalName == "objectid" || attr.LogicalName == "principalid")) { if (sourceSqlType.IsSameAs(DataTypeHelpers.EntityReference)) { @@ -443,13 +448,18 @@ protected Dictionary> CompileColumnMappings(DataSou { Expression targetExpr; - if (lookupAttr.Targets.Length == 1) + if (lookupAttr != null && lookupAttr.Targets.Length == 1) { targetExpr = Expression.Constant(lookupAttr.Targets[0]); } else { - var sourceTargetColumnName = mappings[destAttributeName + "type"]; + var typeColName = destAttributeName + "type"; + + if (metadata.LogicalName == "principalobjectaccess" && (attr.LogicalName == "objectid" || attr.LogicalName == "principalid")) + typeColName = destAttributeName.Replace("id", "typecode"); + + var sourceTargetColumnName = mappings[typeColName]; var sourceTargetType = schema.Schema[sourceTargetColumnName].Type; targetExpr = Expression.Property(entityParam, typeof(Entity).GetCustomAttribute().MemberName, Expression.Constant(sourceTargetColumnName)); @@ -478,6 +488,15 @@ protected Dictionary> CompileColumnMappings(DataSou ); } + if (lookupAttr != null && lookupAttr.Targets.Any(t => dataSource.Metadata[t].DataProviderId == DataProviders.ElasticDataProvider) && mappings.TryGetValue(destAttributeName + "pid", out var partitionIdColumn)) + { + var partitionIdExpr = (Expression)Expression.Property(entityParam, typeof(Entity).GetCustomAttribute().MemberName, Expression.Constant(partitionIdColumn)); + partitionIdExpr = Expression.Convert(partitionIdExpr, schema.Schema[partitionIdColumn].Type.ToNetType(out _)); + partitionIdExpr = SqlTypeConverter.Convert(partitionIdExpr, schema.Schema[partitionIdColumn].Type, DataTypeHelpers.NVarChar(100, dataSource.DefaultCollation, CollationLabel.Implicit)); + partitionIdExpr = SqlTypeConverter.Convert(partitionIdExpr, typeof(string)); + convertedExpr = Expr.Call(() => CreateElasticEntityReference(Expr.Arg(), Expr.Arg(), Expr.Arg()), convertedExpr, partitionIdExpr, Expression.Constant(dataSource.Metadata)); + } + destType = typeof(EntityReference); } else @@ -546,6 +565,24 @@ private static string ObjectTypeCodeToLogicalName(SqlInt32 otc, IAttributeMetada return attributeMetadataCache[otc.Value].LogicalName; } + private static EntityReference CreateElasticEntityReference(EntityReference entityReference, string partitionId, IAttributeMetadataCache metadata) + { + if (entityReference == null) + return null; + + if (partitionId == null) + return entityReference; + + var meta = metadata[entityReference.LogicalName]; + var keys = new KeyAttributeCollection + { + [meta.PrimaryIdAttribute] = entityReference.Id, + ["partitionid"] = partitionId + }; + + return new EntityReference(entityReference.LogicalName, keys); + } + /// /// Provides values to include in log messages /// diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs index cc8e64b2..0e5f7f5d 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs @@ -192,7 +192,7 @@ protected virtual INodeSchema GetSchema(NodeCompilationContext context, bool inc foreach (var definedValue in DefinedValues) { innerSchema.ContainsColumn(definedValue.Value, out var innerColumn); - schema[definedValue.Key] = innerSchema.Schema[innerColumn]; + schema[definedValue.Key] = innerSchema.Schema[innerColumn].Invisible().Calculated(); } _lastLeftSchema = outerSchema; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseNode.cs index 14e2b18a..1df13123 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseNode.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using Microsoft.SqlServer.TransactSql.ScriptDom; using Microsoft.Xrm.Sdk.Metadata; +using Newtonsoft.Json; namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan { @@ -15,6 +16,7 @@ abstract class BaseNode : IExecutionPlanNode /// The parent of this node /// [Browsable(false)] + [JsonIgnore] public IExecutionPlanNode Parent { get; set; } /// diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs index ecbc11eb..6054d3fa 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs @@ -142,6 +142,11 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect primaryKey = relationship.Entity1IntersectAttribute; secondaryKey = relationship.Entity2IntersectAttribute; } + else if (meta.LogicalName == "principalobjectaccess") + { + primaryKey = "objectid"; + secondaryKey = "principalid"; + } else if (meta.DataProviderId == DataProviders.ElasticDataProvider) { secondaryKey = "partitionid"; @@ -154,6 +159,12 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect if (secondaryKey != null) fullMappings[secondaryKey] = SecondaryIdSource; + + if (meta.LogicalName == "principalobjectaccess") + { + fullMappings["objecttypecode"] = PrimaryIdSource.Replace("id", "typecode"); + fullMappings["principaltypecode"] = SecondaryIdSource.Replace("id", "typecode"); + } var attributeAccessors = CompileColumnMappings(dataSource, LogicalName, fullMappings, schema, dateTimeKind, entities); primaryIdAccessor = attributeAccessors[primaryKey]; @@ -204,6 +215,15 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect private OrganizationRequest CreateDeleteRequest(EntityMetadata meta, Entity entity, Func primaryIdAccessor, Func secondaryIdAccessor) { + if (meta.LogicalName == "principalobjectaccess") + { + return new RevokeAccessRequest + { + Target = (EntityReference)primaryIdAccessor(entity), + Revokee = (EntityReference)secondaryIdAccessor(entity) + }; + } + var id = (Guid)primaryIdAccessor(entity); // Special case messages for intersect entities @@ -263,7 +283,9 @@ protected override ExecuteMultipleResponse ExecuteMultiple(DataSource dataSource if (!req.Requests.All(r => r is DeleteRequest)) return base.ExecuteMultiple(dataSource, org, meta, req); - if (meta.DataProviderId == DataProviders.ElasticDataProvider || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "DeleteMultiple")) + if (meta.DataProviderId == DataProviders.ElasticDataProvider + // DeleteMultiple is only supported on elastic tables, even if other tables do define the message + /* || dataSource.MessageCache.IsMessageAvailable(meta.LogicalName, "DeleteMultiple")*/) { // Elastic tables can use DeleteMultiple for better performance than ExecuteMultiple var entities = new EntityReferenceCollection(); diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs index 85901cf8..cb3dcd9c 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs @@ -531,19 +531,13 @@ public static ExecuteMessageNode FromMessage(SchemaObjectFunctionTableReference // Check the number and type of input parameters matches var expectedInputParameters = new List(); - var pagingInfoPosition = -1; for (var i = 0; i < message.InputParameters.Count; i++) { if (message.InputParameters[i].Type == typeof(PagingInfo)) - { - pagingInfoPosition = i; node.PagingParameter = message.InputParameters[i].Name; - } else - { expectedInputParameters.Add(message.InputParameters[i]); - } } // Check we have the right number of parameters @@ -556,6 +550,14 @@ public static ExecuteMessageNode FromMessage(SchemaObjectFunctionTableReference if (expectedInputParameters.Count < tvf.Parameters.Count) throw new NotSupportedQueryFragmentException(Sql4CdsError.TooManyArguments(tvf.SchemaObject, false)); + expectedInputParameters.Sort((x, y) => + { + if (context.Options.ColumnOrdering == ColumnOrdering.Alphabetical) + return String.Compare(x.Name, y.Name, StringComparison.OrdinalIgnoreCase); + else + return x.Position.CompareTo(y.Position); + }); + // Add the parameter values to the node, including any required type conversions for (var i = 0; i < expectedInputParameters.Count; i++) { @@ -612,21 +614,23 @@ public static ExecuteMessageNode FromMessage(ExecutableProcedureReference sproc, // Check the number and type of input parameters matches var expectedInputParameters = new List(); - var pagingInfoPosition = -1; for (var i = 0; i < message.InputParameters.Count; i++) { if (message.InputParameters[i].Type == typeof(PagingInfo)) - { - pagingInfoPosition = i; node.PagingParameter = message.InputParameters[i].Name; - } else - { expectedInputParameters.Add(message.InputParameters[i]); - } } + expectedInputParameters.Sort((x, y) => + { + if (context.Options.ColumnOrdering == ColumnOrdering.Alphabetical) + return String.Compare(x.Name, y.Name, StringComparison.OrdinalIgnoreCase); + else + return x.Position.CompareTo(y.Position); + }); + // Add the parameter values to the node, including any required type conversions var usedParamName = false; @@ -645,15 +649,10 @@ public static ExecuteMessageNode FromMessage(ExecutableProcedureReference sproc, if (sproc.Parameters[i].Variable == null) { - var paramIndex = i; - - if (pagingInfoPosition != -1 && i >= pagingInfoPosition) - paramIndex++; - - if (paramIndex >= message.InputParameters.Count) + if (i >= expectedInputParameters.Count) throw new NotSupportedQueryFragmentException(Sql4CdsError.TooManyArguments(sproc.ProcedureReference.ProcedureReference.Name, true)); - targetParamName = message.InputParameters[paramIndex].Name; + targetParamName = expectedInputParameters[i].Name; } else { diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs index 7ad9ecba..908cc754 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs @@ -61,11 +61,10 @@ public IntermediateExpression(Expression converted, ParameterExpression[] parame public static Type GetType(this TSqlFragment expr, ExpressionCompilationContext context, out DataTypeReference sqlType) { ToExpression(expr, context, false, out _, out sqlType, out var cacheKey); - var details = _cache.GetOrAdd(cacheKey, __ => + var details = _intermediateCache.GetOrAdd(cacheKey, __ => { var converted = ToExpression(expr, context, true, out var parameters, out _, out _); - var compiled = Expression.Lambda>(Expr.Box(converted), parameters).Compile(); - return new CompiledExpression(expr, converted, compiled); + return new IntermediateExpression(converted, parameters); }); return details.Converted.Type; } @@ -82,8 +81,9 @@ public static Func Compile(this TSqlFragment var details = _cache.GetOrAdd(cacheKey, __ => { var converted = ToExpression(expr, context, true, out var parameters, out _, out _); - var compiled = Expression.Lambda>(Expr.Box(converted), parameters).Compile(); - return new CompiledExpression(expr, converted, compiled); + var folded = FoldLambdas(converted, parameters); + var compiled = Expression.Lambda>(Expr.Box(folded), parameters).Compile(); + return new CompiledExpression(expr, folded, compiled); }); return eec => details.Compiled(eec, expr); } @@ -100,8 +100,9 @@ public static Func Compile(this BooleanExpress var details = _boolCache.GetOrAdd(cacheKey, __ => { var converted = ToExpression(b, context, true, out var parameters, out _, out _); - var compiled = Expression.Lambda>(Expression.IsTrue(converted), parameters).Compile(); - return new CompiledExpression(b, converted, compiled); + var folded = FoldLambdas(converted, parameters); + var compiled = Expression.Lambda>(Expression.IsTrue(folded), parameters).Compile(); + return new CompiledExpression(b, folded, compiled); }); return eec => details.Compiled(eec, b); @@ -652,14 +653,29 @@ private static Expression ToExpression(BooleanBinaryExpression bin, ExpressionCo var lhs = bin.InvokeSubExpression(x => x.FirstExpression, x => x.FirstExpression, context, contextParam, exprParam, createExpression, out _, out var lhsCacheKey); var rhs = bin.InvokeSubExpression(x => x.SecondExpression, x => x.SecondExpression, context, contextParam, exprParam, createExpression, out _, out var rhsCacheKey); + if (createExpression) + { + lhs = Expression.IsTrue(lhs); + rhs = Expression.IsTrue(rhs); + } + + Expression binary; + if (bin.BinaryExpressionType == BooleanBinaryExpressionType.And) { cacheKey = lhsCacheKey + " AND " + rhsCacheKey; - return createExpression ? Expression.AndAlso(lhs, rhs) : null; + binary = createExpression ? Expression.AndAlso(lhs, rhs) : null; } + else + { + cacheKey = lhsCacheKey + " OR " + rhsCacheKey; + binary = createExpression ? Expression.OrElse(lhs, rhs) : null; + } + + if (createExpression) + binary = Expression.Convert(binary, typeof(SqlBoolean)); - cacheKey = lhsCacheKey + " OR " + rhsCacheKey; - return createExpression ? Expression.OrElse(lhs, rhs) : null; + return binary; } private static Expression ToExpression(BooleanParenthesisExpression paren, ExpressionCompilationContext context, ParameterExpression contextParam, ParameterExpression exprParam, bool createExpression, out DataTypeReference sqlType, out string cacheKey) @@ -813,6 +829,7 @@ rhsSqlType is SqlDataTypeReferenceWithCollation rhsSql && expr = Expression.Divide(lhs, rhs); expr = Expression.TryCatch(expr, Expression.Catch(typeof(DivideByZeroException), Expression.Throw(Expression.New(typeof(QueryExecutionException).GetConstructor(new[] { typeof(Sql4CdsError) }), Expr.Call(() => Sql4CdsError.DivideByZero())), expr.Type))); + //expr = Expression.Invoke(Expression.Lambda(expr)); break; case BinaryExpressionType.Modulo: @@ -1888,7 +1905,7 @@ private static Expression ToExpression(this BooleanNotExpression not, Expression { var value = not.InvokeSubExpression(x => x.Expression, x => x.Expression, context, contextParam, exprParam, createExpression, out sqlType, out cacheKey); cacheKey = "NOT " + cacheKey; - return createExpression ? Expression.Not(value) : null; + return createExpression ? Expression.Convert(Expression.IsFalse(value), typeof(SqlBoolean)) : null; } private static readonly Dictionary _typeMapping = new Dictionary @@ -2015,10 +2032,10 @@ private static Expression ToExpression(this ConvertCall convert, ExpressionCompi sqlType = convert.DataType; - return Convert(context, value, valueType, valueCacheKey, sqlType, style, styleType, styleCacheKey, convert, out cacheKey); + return Convert(context, value, valueType, valueCacheKey, sqlType, style, styleType, styleCacheKey, convert, "CONVERT", out cacheKey); } - private static Expression Convert(ExpressionCompilationContext context, Expression value, DataTypeReference valueType, string valueCacheKey, DataTypeReference sqlType, Expression style, DataTypeReference styleType, string styleCacheKey, TSqlFragment expr, out string cacheKey) + private static Expression Convert(ExpressionCompilationContext context, Expression value, DataTypeReference valueType, string valueCacheKey, DataTypeReference sqlType, Expression style, DataTypeReference styleType, string styleCacheKey, TSqlFragment expr, string cacheKeyRoot, out string cacheKey) { if (sqlType is SqlDataTypeReference sqlTargetType && sqlTargetType.SqlDataTypeOption.IsStringType()) @@ -2042,7 +2059,7 @@ private static Expression Convert(ExpressionCompilationContext context, Expressi }; } - cacheKey = $"CONVERT({GetTypeKey(sqlType, true)}, {valueCacheKey}"; + cacheKey = $"{cacheKeyRoot}({GetTypeKey(sqlType, true)}, {valueCacheKey}"; if (styleCacheKey != null) cacheKey += ", " + styleCacheKey; cacheKey += ")"; @@ -2055,7 +2072,7 @@ private static Expression ToExpression(this CastCall cast, ExpressionCompilation var value = cast.InvokeSubExpression(x => x.Parameter, x => x.Parameter, context, contextParam, exprParam, createExpression, out var valueType, out var valueCacheKey); sqlType = cast.DataType; - return Convert(context, value, valueType, valueCacheKey, sqlType, null, null, null, cast, out cacheKey); + return Convert(context, value, valueType, valueCacheKey, sqlType, null, null, null, cast, "CAST", out cacheKey); } private static readonly Regex _containsParser = new Regex("^\\S+( OR \\S+)*$", RegexOptions.IgnoreCase | RegexOptions.Compiled); @@ -2499,5 +2516,51 @@ private static string GetTypeKey(DataTypeReference type, bool includeStringLengt throw new ArgumentOutOfRangeException(nameof(type)); } + + private static Expression FoldLambdas(Expression expression, ParameterExpression[] parameters) + { + var visitor = new LambdaVisitor(parameters); + return visitor.Visit(expression); + } + + class LambdaVisitor : ExpressionVisitor + { + private ParameterExpression[] _parameters; + private Dictionary _parameterRewrites; + + public LambdaVisitor(ParameterExpression[] parameters) + { + _parameters = parameters; + _parameterRewrites = new Dictionary(); + } + + protected override Expression VisitInvocation(InvocationExpression node) + { + if (!(node.Expression is LambdaExpression lambda)) + return base.VisitInvocation(node); + + if (node.Arguments.Count != 2) + return base.VisitInvocation(node); + + if (node.Arguments[0].Type != typeof(ExpressionExecutionContext)) + return base.VisitInvocation(node); + + if (!typeof(TSqlFragment).IsAssignableFrom(node.Arguments[1].Type)) + return base.VisitInvocation(node); + + _parameterRewrites[lambda.Parameters[0]] = Visit(node.Arguments[0]); + _parameterRewrites[lambda.Parameters[1]] = Visit(node.Arguments[1]); + + return Visit(lambda.Body); + } + + protected override Expression VisitParameter(ParameterExpression node) + { + if (_parameterRewrites.TryGetValue(node, out var replacement)) + return replacement; + + return base.VisitParameter(node); + } + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs index 38298c67..bcb9b473 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs @@ -20,7 +20,7 @@ namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan { - class FetchXmlScan : BaseDataNode, IFetchXmlExecutionPlanNode + class FetchXmlScan : BaseDataNode, IFetchXmlExecutionPlanNode, IExecutionPlanNodeWarning { class ParameterizedCondition { @@ -44,7 +44,7 @@ public ParameterizedCondition(filter filter, condition condition, conditionValue }; } - public void SetValue(object value, IQueryExecutionOptions options) + public void SetValue(INullable value, IQueryExecutionOptions options) { if (value == null || (value is INullable nullable && nullable.IsNull)) { @@ -58,28 +58,7 @@ public void SetValue(object value, IQueryExecutionOptions options) if (!_filter.Items.Contains(_condition)) _filter.Items = _filter.Items.Except(new[] { _contradiction }).Concat(new[] { _condition }).ToArray(); - var formatted = value.ToString(); - - if (value is SqlDate d) - value = (SqlDateTime)d; - else if (value is SqlDateTime2 dt2) - value = (SqlDateTime)dt2; - else if (value is SqlTime t) - value = (SqlDateTime)t; - else if (value is SqlDateTimeOffset dto) - value = (SqlDateTime)dto; - - if (value is SqlDateTime dt) - { - DateTimeOffset dto; - - if (options.UseLocalTimeZone) - dto = new DateTimeOffset(dt.Value, TimeZoneInfo.Local.GetUtcOffset(dt.Value)); - else - dto = new DateTimeOffset(dt.Value, TimeSpan.Zero); - - formatted = dto.ToString("yyyy-MM-ddTHH':'mm':'ss.FFFzzz"); - } + var formatted = FetchXmlScan.FormatConditionValue(value, options); if (_value != null) _value.Value = formatted; @@ -111,7 +90,8 @@ public InvalidPagingException(string message) : base(message) private bool _resetPage; private string _startingPage; private List> _pagingFields; - private List _lastPageValues; + private List _lastPageValues; + private bool _missingPagingCookie; public FetchXmlScan() { @@ -213,16 +193,18 @@ public FetchXmlScan() [Description("A list of additional columns that should be included in the schema")] public List ColumnMappings { get; } = new List(); + [Browsable(false)] + public string Warning => _missingPagingCookie && RowsOut == 50_000 ? "Using legacy paging - results may be incomplete" : null; + public bool RequiresCustomPaging(IDictionary dataSources) { - // Custom paging is required if we have links to child entities, as standard Dataverse paging is applied at - // the top-level entity only. - // Custom paging can't be used with distinct queries, as the primary key fields required to implement the paging - // may not be included. - // It also can't be used with aggregate queries as doing so would affect the aggregae behaviour. - if (FetchXml.distinct) + // Never need to do paging if we're enforcing a TOP constraint + if (FetchXml.top != null) return false; + // Custom paging is required if we have links to child entities, as standard Dataverse paging is applied at + // the top-level entity only. + // Custom paging can't be used with aggregate queries as doing so would affect the aggregate behaviour. if (FetchXml.aggregate) return false; @@ -231,14 +213,18 @@ public bool RequiresCustomPaging(IDictionary dataSources) foreach (var linkEntity in Entity.GetLinkEntities()) { - // Link entities used for filtering do not require paging + // Link entities used for filtering do not require custom paging if (linkEntity.linktype == "exists" || linkEntity.linktype == "in") continue; + // Sorts on link entities always require custom paging + if (linkEntity.Items?.OfType().Any() == true) + return true; + if (HasSingleRecordFilter(linkEntity, dataSource.Metadata[linkEntity.name].PrimaryIdAttribute)) continue; - // Parental lookups do not require paging + // Parental lookups do not require custom paging if (linkEntity.from == dataSource.Metadata[linkEntity.name].PrimaryIdAttribute) continue; @@ -330,7 +316,7 @@ protected override IEnumerable ExecuteInternal(NodeExecutionContext cont } else { - _lastPageValues = new List(); + _lastPageValues = new List(); } var req = new RetrieveMultipleRequest { Query = new FetchExpression(Serialize(FetchXml)) }; @@ -422,11 +408,12 @@ protected override IEnumerable ExecuteInternal(NodeExecutionContext cont FetchXml.page = (Int32.Parse(FetchXml.page, CultureInfo.InvariantCulture) + 1).ToString(); FetchXml.pagingcookie = res.PagingCookie; + _missingPagingCookie = String.IsNullOrEmpty(res.PagingCookie); } else { pagingFilter = new filter { type = filterType.or }; - AddPagingFilters(pagingFilter); + AddPagingFilters(pagingFilter, context.Options); Entity.AddItem(pagingFilter); } @@ -504,7 +491,7 @@ private void VerifyFilterValueTypes(string entityName, object[] items, DataSourc VerifyFilterValueTypes(linkEntity.name, linkEntity.Items, dataSource); } - private void AddPagingFilters(filter filter) + private void AddPagingFilters(filter filter, IQueryExecutionOptions options) { for (var i = 0; i < _pagingFields.Count; i++) { @@ -516,16 +503,44 @@ private void AddPagingFilters(filter filter) for (var j = 0; j < i; j++) { var subParts = _pagingFields[j].Key.ToColumnReference().MultiPartIdentifier.Identifiers; - subFilter.AddItem(new condition { entityname = j == 0 ? null : subParts[0].Value, attribute = subParts[1].Value, @operator = _lastPageValues[j].IsNull ? @operator.@null : @operator.eq, value = _lastPageValues[j].IsNull ? null : _lastPageValues[j].Id.ToString() }); + subFilter.AddItem(new condition { entityname = subParts[0].Value == Alias ? null : subParts[0].Value, attribute = subParts[1].Value, @operator = _lastPageValues[j].IsNull ? @operator.@null : @operator.eq, value = _lastPageValues[j].IsNull ? null : FormatConditionValue(_lastPageValues[j], options) }); } var parts = _pagingFields[i].Key.ToColumnReference().MultiPartIdentifier.Identifiers; - subFilter.AddItem(new condition { entityname = i == 0 ? null : parts[0].Value, attribute = parts[1].Value, @operator = @operator.gt, value = _lastPageValues[i].Id.ToString() }); + subFilter.AddItem(new condition { entityname = parts[0].Value == Alias ? null : parts[0].Value, attribute = parts[1].Value, @operator = @operator.gt, value = FormatConditionValue(_lastPageValues[i], options) }); filter.AddItem(subFilter); } } + private static string FormatConditionValue(INullable value, IQueryExecutionOptions options) + { + var formatted = value.ToString(); + + if (value is SqlDate d) + value = (SqlDateTime)d; + else if (value is SqlDateTime2 dt2) + value = (SqlDateTime)dt2; + else if (value is SqlTime t) + value = (SqlDateTime)t; + else if (value is SqlDateTimeOffset dto) + value = (SqlDateTime)dto; + + if (value is SqlDateTime dt) + { + DateTimeOffset dto; + + if (options.UseLocalTimeZone) + dto = new DateTimeOffset(dt.Value, TimeZoneInfo.Local.GetUtcOffset(dt.Value)); + else + dto = new DateTimeOffset(dt.Value, TimeSpan.Zero); + + formatted = dto.ToString("yyyy-MM-ddTHH':'mm':'ss.FFFzzz"); + } + + return formatted; + } + /// /// Updates the with current parameter values /// @@ -687,6 +702,8 @@ private void OnRetrievedEntity(Entity entity, INodeSchema schema, IQueryExecutio var typeSuffix = AddSuffix(attribute.Key, "type"); var nameSuffix = AddSuffix(attribute.Key, "name"); + // NOTE: pid for elastic lookup values is exposed as a separate column in the returned entity already + if (!entity.Contains(typeSuffix)) entity[typeSuffix] = ((EntityReference)attribute.Value).LogicalName; @@ -729,7 +746,7 @@ private void OnRetrievedEntity(Entity entity, INodeSchema schema, IQueryExecutio _lastPageValues.Clear(); foreach (var pagingField in _pagingFields) - _lastPageValues.Add((SqlEntityReference)entity[pagingField.Value]); + _lastPageValues.Add((INullable)entity[pagingField.Value]); } } @@ -972,6 +989,10 @@ internal void ResetSchemaCache() internal FetchAttributeType AddAttribute(string colName, Func predicate, IAttributeMetadataCache metadata, out bool added, out FetchLinkEntityType linkEntity) { + var mapping = ColumnMappings.FirstOrDefault(m => m.OutputColumn == colName); + if (mapping != null) + colName = mapping.SourceColumn; + var parts = colName.SplitMultiPartIdentifier(); if (parts.Length == 1) @@ -988,11 +1009,9 @@ internal FetchAttributeType AddAttribute(string colName, Func a.LogicalName.Equals(attr.name, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); - if (meta == null && (attr.name.EndsWith("name", StringComparison.OrdinalIgnoreCase) || attr.name.EndsWith("type", StringComparison.OrdinalIgnoreCase))) - { - var logicalName = attr.name.Substring(0, attr.name.Length - 4); - meta = metadata[Entity.name].Attributes.SingleOrDefault(a => a.LogicalName.Equals(logicalName, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); - } + + if (meta == null) + meta = metadata[Entity.name].FindBaseAttributeFromVirtualAttribute(attr.name, out _); if (meta != null) attr.name = meta.LogicalName; @@ -1021,11 +1040,8 @@ internal FetchAttributeType AddAttribute(string colName, Func a.LogicalName.Equals(attr.name, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); - if (meta == null && (attr.name.EndsWith("name", StringComparison.OrdinalIgnoreCase) || attr.name.EndsWith("type", StringComparison.OrdinalIgnoreCase))) - { - var logicalName = attr.name.Substring(0, attr.name.Length - 4); - meta = metadata[linkEntity.name].Attributes.SingleOrDefault(a => a.LogicalName.Equals(logicalName, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); - } + if (meta == null) + meta = metadata[linkEntity.name].FindBaseAttributeFromVirtualAttribute(attr.name, out _); if (meta != null) attr.name = meta.LogicalName; @@ -1201,7 +1217,7 @@ private void AddSchemaAttributes(NodeCompilationContext context, DataSource data if (linkEntity.from != childMeta.PrimaryIdAttribute) { - if (linkEntity.linktype == "inner") + if (linkEntity.linktype == "inner" && linkEntity.to == meta.PrimaryIdAttribute && primaryKey == $"{alias}.{meta.PrimaryIdAttribute.EscapeIdentifier()}") primaryKey = $"{linkEntity.alias}.{childMeta.PrimaryIdAttribute.EscapeIdentifier()}"; else primaryKey = null; @@ -1285,6 +1301,9 @@ private void AddSchemaAttribute(DataSource dataSource, ColumnList schema, Dictio if (lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList) AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "type"), (attrMetadata.LogicalName + "type").EscapeIdentifier(), DataTypeHelpers.NVarChar(MetadataExtensions.EntityLogicalNameMaxLength, dataSource.DefaultCollation, CollationLabel.Implicit), notNull); ; + + if (lookup.Targets != null && lookup.Targets.Any(logicalName => dataSource.Metadata[logicalName].DataProviderId == DataProviders.ElasticDataProvider)) + AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "pid"), (attrMetadata.LogicalName + "pid").EscapeIdentifier(), DataTypeHelpers.NVarChar(100, dataSource.DefaultCollation, CollationLabel.Implicit), false); } } @@ -1840,16 +1859,27 @@ public override void AddRequiredColumns(NodeCompilationContext context, IList>(); + _lastPageValues = new List(); - _pagingFields = new List>(); - _lastPageValues = new List(); + if (FetchXml.distinct) + { + // Distinct queries should already be sorted by each attribute being returned + AddAllDistinctAttributes(Entity, dataSource); - // Ensure the primary key of each entity is included - AddPrimaryIdAttribute(Entity, dataSource); + foreach (var linkEntity in Entity.GetLinkEntities().Where(le => le.linktype != "exists" && le.linktype != "in" && !HasSingleRecordFilter(le, dataSource.Metadata[le.name].PrimaryIdAttribute))) + AddAllDistinctAttributes(linkEntity, dataSource); + } + else + { + RemoveSorts(); + + // Ensure the primary key of each entity is included + AddPrimaryIdAttribute(Entity, dataSource); - foreach (var linkEntity in Entity.GetLinkEntities().Where(le => le.linktype != "exists" && le.linktype != "in" && !HasSingleRecordFilter(le, dataSource.Metadata[le.name].PrimaryIdAttribute))) - AddPrimaryIdAttribute(linkEntity, dataSource); + foreach (var linkEntity in Entity.GetLinkEntities().Where(le => le.linktype != "exists" && le.linktype != "in" && !HasSingleRecordFilter(le, dataSource.Metadata[le.name].PrimaryIdAttribute))) + AddPrimaryIdAttribute(linkEntity, dataSource); + } } NormalizeAttributes(context.DataSources); @@ -1890,6 +1920,36 @@ private object[] AddPrimaryIdAttribute(object[] items, string alias, EntityMetad return items; } + private void AddAllDistinctAttributes(FetchEntityType entity, DataSource dataSource) + { + AddAllDistinctAttributes(entity.Items, Alias, dataSource.Metadata[entity.name]); + } + + private void AddAllDistinctAttributes(FetchLinkEntityType linkEntity, DataSource dataSource) + { + AddAllDistinctAttributes(linkEntity.Items, linkEntity.alias, dataSource.Metadata[linkEntity.name]); + } + + private void AddAllDistinctAttributes(object[] items, string alias, EntityMetadata metadata) + { + if (items == null) + return; + + // If we have the primary key, we don't need to worry about any other attributes + var allAttrs = items.OfType().Any(); + var primaryIdAttr = items.OfType().SingleOrDefault(a => a.name == metadata.PrimaryIdAttribute); + + if (allAttrs || primaryIdAttr != null) + { + _pagingFields.Add(new KeyValuePair(alias + "." + metadata.PrimaryIdAttribute, alias + "." + (primaryIdAttr?.alias ?? metadata.PrimaryIdAttribute))); + } + else + { + foreach (var attr in items.OfType()) + _pagingFields.Add(new KeyValuePair(alias + "." + attr.name, alias + "." + (attr.alias ?? attr.name))); + } + } + private bool HasAttribute(object[] items) { if (items == null) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs index fec84f72..f3e25326 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FilterNode.cs @@ -138,6 +138,14 @@ private void AddNotNullColumn(NodeSchema schema, ScalarExpression expr) public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext context, IList hints) { + // Swap filter to come after sort + if (Source is SortNode sort) + { + Source = sort.Source; + sort.Source = this; + return sort.FoldQuery(context, hints); + } + Filter = FoldNotIsNullToIsNotNull(Filter); // If we have a filter which implies a non-null value for a column that is generated by an outer join, @@ -156,6 +164,9 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext foldedFilters |= FoldTableSpoolToIndexSpool(context, hints); foldedFilters |= ExpandFiltersOnColumnComparisons(context); foldedFilters |= FoldFiltersToDataSources(context, hints, subqueryConditions); + foldedFilters |= FoldFiltersToInnerJoinSources(context, hints); + foldedFilters |= FoldFiltersToSpoolSource(context, hints); + foldedFilters |= FoldFiltersToNestedLoopCondition(context, hints); foreach (var addedLink in addedLinks) { @@ -181,6 +192,129 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext return this; } + private bool FoldFiltersToNestedLoopCondition(NodeCompilationContext context, IList hints) + { + if (Filter == null) + return false; + + if (!(Source is NestedLoopNode loop)) + return false; + + // Can't move the filter to the loop condition if we're using any of the defined values created by the loop + if (Filter.GetColumns().Any(c => loop.DefinedValues.ContainsKey(c))) + return false; + + if (loop.JoinCondition == null) + { + loop.JoinCondition = Filter; + } + else + { + loop.JoinCondition = new BooleanBinaryExpression + { + FirstExpression = loop.JoinCondition, + BinaryExpressionType = BooleanBinaryExpressionType.And, + SecondExpression = Filter + }; + } + + Filter = null; + return true; + } + + private bool FoldFiltersToSpoolSource(NodeCompilationContext context, IList hints) + { + if (Filter == null) + return false; + + if (!(Source is TableSpoolNode spool)) + return false; + + var usesVariables = Filter.GetVariables().Any(); + + if (usesVariables) + return false; + + spool.Source = new FilterNode + { + Source = spool.Source, + Filter = Filter + }; + + Filter = null; + + return true; + } + + private bool FoldFiltersToInnerJoinSources(NodeCompilationContext context, IList hints) + { + if (Filter == null) + return false; + + if (!(Source is BaseJoinNode join) || join.JoinType != QualifiedJoinType.Inner) + return false; + + var folded = false; + var leftSchema = join.LeftSource.GetSchema(context); + Filter = ExtractChildFilters(Filter, leftSchema, col => leftSchema.ContainsColumn(col, out _), out var leftFilter); + + if (leftFilter != null) + { + join.LeftSource = new FilterNode + { + Source = join.LeftSource, + Filter = leftFilter + }.FoldQuery(context, hints); + join.LeftSource.Parent = join; + + folded = true; + } + + if (Filter == null) + return true; + + var rightContext = context; + + if (join is NestedLoopNode loop && loop.OuterReferences != null) + { + var innerParameterTypes = context.ParameterTypes + .Concat(loop.OuterReferences.Select(or => new KeyValuePair(or.Value, leftSchema.Schema[or.Key].Type))) + .ToDictionary(p => p.Key, p => p.Value, StringComparer.OrdinalIgnoreCase); + + rightContext = new NodeCompilationContext(context, innerParameterTypes); + } + + var rightSchema = join.RightSource.GetSchema(rightContext); + Filter = ExtractChildFilters(Filter, rightSchema, col => rightSchema.ContainsColumn(col, out _) || join.DefinedValues.ContainsKey(col), out var rightFilter); + + if (rightFilter != null) + { + if (join.DefinedValues.Count > 0) + { + var rewrite = new RewriteVisitor(join.DefinedValues.ToDictionary(kvp => (ScalarExpression)kvp.Key.ToColumnReference(), kvp => (ScalarExpression)kvp.Value.ToColumnReference())); + rightFilter.Accept(rewrite); + } + + join.RightSource = new FilterNode + { + Source = join.RightSource, + Filter = rightFilter + }.FoldQuery(rightContext, hints); + join.RightSource.Parent = join; + + folded = true; + } + + if (folded) + { + // Re-fold the join + Source = Source.FoldQuery(context, hints); + Source.Parent = this; + } + + return folded; + } + private bool CheckStartupExpression() { // We only need to apply the filter expression to individual rows if it references any fields diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs index c80364fc..f7683d8a 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs @@ -51,6 +51,20 @@ abstract class FoldableJoinNode : BaseJoinNode public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext context, IList hints) { + // For inner joins, additional join criteria are eqivalent to doing the join without them and then applying the filter + // We've already got logic in the Filter node for efficiently folding those queries, so split them out and let it do + // what it can + if (JoinType == QualifiedJoinType.Inner && AdditionalJoinCriteria != null) + { + var filter = new FilterNode + { + Source = this, + Filter = AdditionalJoinCriteria + }; + AdditionalJoinCriteria = null; + return filter.FoldQuery(context, hints); + } + LeftSource = LeftSource.FoldQuery(context, hints); LeftSource.Parent = this; RightSource = RightSource.FoldQuery(context, hints); @@ -281,6 +295,10 @@ private bool FoldFetchXmlJoin(NodeCompilationContext context, IList(); + if (JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.RightOuter) + unnecessaryNotNullColumns.Add(LeftAttribute.GetColumnName()); + if (JoinType == QualifiedJoinType.Inner || JoinType == QualifiedJoinType.LeftOuter) + unnecessaryNotNullColumns.Add(RightAttribute.GetColumnName()); + + if (unnecessaryNotNullColumns != null) + { + var finalSchema = leftFetch.GetSchema(context); + + foreach (var col in unnecessaryNotNullColumns) + { + if (!finalSchema.ContainsColumn(col, out var normalizedCol)) + continue; + + var parts = normalizedCol.SplitMultiPartIdentifier(); + + if (parts[0] == leftFetch.Alias) + { + foreach (var entityFilter in leftFetch.Entity.Items.OfType()) + { + if (entityFilter.type != filterType.and) + continue; + + foreach (var condition in entityFilter.Items.OfType().ToList()) + { + if (condition.entityname == null && condition.attribute == parts[1] && condition.@operator == @operator.notnull) + entityFilter.Items = entityFilter.Items.Except(new[] { condition }).ToArray(); + } + } + } + else + { + foreach (var entityFilter in leftFetch.Entity.Items.OfType()) + { + if (entityFilter.type != filterType.and) + continue; + + foreach (var condition in entityFilter.Items.OfType().ToList()) + { + if (condition.entityname == parts[0] && condition.attribute == parts[1] && condition.@operator == @operator.notnull) + entityFilter.Items = entityFilter.Items.Except(new[] { condition }).ToArray(); + } + } + + var link = leftFetch.Entity.FindLinkEntity(parts[0]); + + if (link?.Items != null) + { + foreach (var linkFilter in link.Items.OfType()) + { + if (linkFilter.type != filterType.and) + continue; + + foreach (var condition in linkFilter.Items.OfType().ToList()) + { + if (condition.attribute == parts[1] && condition.@operator == @operator.notnull) + linkFilter.Items = linkFilter.Items.Except(new[] { condition }).ToArray(); + } + } + } + } + } + + // Re-fold the FetchXML node to remove any filters that are now blank + leftFetch.FoldQuery(context, hints); + } + return true; } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashMatchAggregateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashMatchAggregateNode.cs index 6cc3b057..7c85b61b 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashMatchAggregateNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/HashMatchAggregateNode.cs @@ -279,7 +279,7 @@ Source is FetchXmlScan fetch && var metadata = context.DataSources[fetchXml.DataSource].Metadata; // Aggregates are not supported on archive data - if (fetchXml.FetchXml.DataSource != null) + if (fetchXml.FetchXml.DataSource == "retained") canUseFetchXmlAggregate = false; // FetchXML is translated to QueryExpression for virtual entities, which doesn't support aggregates diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/IndexSpoolNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/IndexSpoolNode.cs index 856bb3af..2f451740 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/IndexSpoolNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/IndexSpoolNode.cs @@ -80,14 +80,28 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext if (KeyColumn != null && SeekValue != null) { // Index and seek values must be the same type - var indexType = Source.GetSchema(context).Schema[KeyColumn].Type; + var indexCol = Source.GetSchema(context).Schema[KeyColumn]; var seekType = context.ParameterTypes[SeekValue]; - if (!SqlTypeConverter.CanMakeConsistentTypes(indexType, seekType, context.PrimaryDataSource, null, null, out var consistentType)) - throw new QueryExecutionException($"No type conversion available for {indexType.ToSql()} and {seekType.ToSql()}"); + if (!SqlTypeConverter.CanMakeConsistentTypes(indexCol.Type, seekType, context.PrimaryDataSource, null, null, out var consistentType)) + throw new QueryExecutionException(Sql4CdsError.TypeClash(null, indexCol.Type, seekType)); - _keySelector = SqlTypeConverter.GetConversion(indexType, consistentType); + _keySelector = SqlTypeConverter.GetConversion(indexCol.Type, consistentType); _seekSelector = SqlTypeConverter.GetConversion(seekType, consistentType); + + if (!WithStack && indexCol.IsNullable) + { + // Try to fold a NOT NULL filter into the source - we'll never match a null value with an equality operator + Source = new FilterNode + { + Source = Source, + Filter = new BooleanIsNullExpression + { + Expression = KeyColumn.ToColumnReference(), + IsNot = true + } + }.FoldQuery(context, hints); + } } if (WithStack) @@ -168,7 +182,18 @@ private IDataExecutionPlanNodeInternal FoldCTEToFetchXml(NodeCompilationContext // Check for any other filters or link-entities if (spooledRecursiveFetchXml.Entity.GetLinkEntities().Any() || spooledRecursiveFetchXml.Entity.Items != null && spooledRecursiveFetchXml.Entity.Items.OfType().Any()) - return this; + { + // We might have added a not-null filter on the key column, so ignore that + var filters = spooledRecursiveFetchXml.Entity.Items.OfType().ToArray(); + if (filters.Length != 1 || + filters[0].Items.Length != 1 || + !(filters[0].Items[0] is condition condition) || + condition.attribute != adaptiveSpool.KeyColumn.SplitMultiPartIdentifier().Last() || + condition.@operator != @operator.notnull) + { + return this; + } + } // Check there are no extra calculated columns if (initialDepthCompute.Columns.Count != 1 || incrementDepthCompute.Columns.Count != 1) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs index d15f6499..1c15ba0a 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/InsertNode.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Data.SqlTypes; using System.Linq; using System.ServiceModel; using System.Threading; @@ -130,7 +131,7 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect attributeAccessors.TryGetValue(meta.PrimaryIdAttribute, out primaryIdAccessor); } - // Check again that the update is allowed. Don't count any UI interaction in the execution time + // Check again that the insert is allowed. Don't count any UI interaction in the execution time var confirmArgs = new ConfirmDmlStatementEventArgs(entities.Count, meta, BypassCustomPluginExecution); if (context.Options.CancellationToken.IsCancellationRequested) confirmArgs.Cancel = true; @@ -155,7 +156,7 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect context, out recordsAffected, out message, - LogicalName == "listmember" || meta.IsIntersect == true ? null : (Action) ((r) => SetIdentity(r, context.ParameterValues)) + r => SetIdentity(r, context.ParameterValues) ); } } @@ -177,19 +178,13 @@ private OrganizationRequest CreateInsertRequest(EntityMetadata meta, Entity enti // Special cases for intersect entities if (LogicalName == "listmember") { - var listId = (Guid?)attributeAccessors["listid"](entity); - var entityId = (Guid?)attributeAccessors["entityid"](entity); - - if (listId == null) - throw new QueryExecutionException(Sql4CdsError.NotNullInsert(new Identifier { Value = "listid" }, new Identifier { Value = "listmember" }, "Insert")); - - if (entityId == null) - throw new QueryExecutionException(Sql4CdsError.NotNullInsert(new Identifier { Value = "entityid" }, new Identifier { Value = "listmember" }, "Insert")); + var listId = GetNotNull("listid", entity, attributeAccessors); + var entityId = GetNotNull("entityid", entity, attributeAccessors); return new AddMemberListRequest { - ListId = listId.Value, - EntityId = entityId.Value + ListId = listId, + EntityId = entityId }; } @@ -199,20 +194,32 @@ private OrganizationRequest CreateInsertRequest(EntityMetadata meta, Entity enti // the relationship that this is the intersect entity for var relationship = meta.ManyToManyRelationships.Single(); - var e1 = (Guid?)attributeAccessors[relationship.Entity1IntersectAttribute](entity); - var e2 = (Guid?)attributeAccessors[relationship.Entity2IntersectAttribute](entity); - - if (e1 == null) - throw new QueryExecutionException(Sql4CdsError.NotNullInsert(new Identifier { Value = relationship.Entity1IntersectAttribute }, new Identifier { Value = meta.LogicalName }, "Insert")); - - if (e2 == null) - throw new QueryExecutionException(Sql4CdsError.NotNullInsert(new Identifier { Value = relationship.Entity2IntersectAttribute }, new Identifier { Value = meta.LogicalName }, "Insert")); + var e1 = GetNotNull(relationship.Entity1IntersectAttribute, entity, attributeAccessors); + var e2 = GetNotNull(relationship.Entity2IntersectAttribute, entity, attributeAccessors); return new AssociateRequest { - Target = new EntityReference(relationship.Entity1LogicalName, e1.Value), + Target = new EntityReference(relationship.Entity1LogicalName, e1), Relationship = new Relationship(relationship.SchemaName) { PrimaryEntityRole = EntityRole.Referencing }, - RelatedEntities = new EntityReferenceCollection { new EntityReference(relationship.Entity2LogicalName, e2.Value) } + RelatedEntities = new EntityReferenceCollection { new EntityReference(relationship.Entity2LogicalName, e2) } + }; + } + + if (LogicalName == "principalobjectaccess") + { + // Insert into principalobjectaccess is equivalent to a share + var objectId = GetNotNull("objectid", entity, attributeAccessors); + var principalId = GetNotNull("principalid", entity, attributeAccessors); + var accessRightsMask = GetNotNull("accessrightsmask", entity, attributeAccessors); + + return new GrantAccessRequest + { + Target = objectId, + PrincipalAccess = new PrincipalAccess + { + Principal = principalId, + AccessMask = (AccessRights)accessRightsMask + } }; } @@ -239,6 +246,16 @@ private OrganizationRequest CreateInsertRequest(EntityMetadata meta, Entity enti return new CreateRequest { Target = insert }; } + private T GetNotNull(string attribute, Entity entity, Dictionary> attributeAccessors) + { + var value = attributeAccessors[attribute](entity); + + if (value == null) + throw new QueryExecutionException(Sql4CdsError.NotNullInsert(new Identifier { Value = attribute }, new Identifier { Value = LogicalName }, "Insert")); + + return (T)value; + } + protected override bool FilterErrors(NodeExecutionContext context, OrganizationRequest request, OrganizationServiceFault fault) { if (IgnoreDuplicateKey) @@ -276,10 +293,10 @@ protected override bool FilterErrors(NodeExecutionContext context, OrganizationR return true; } - private void SetIdentity(OrganizationResponse response, IDictionary parameterValues) + private void SetIdentity(OrganizationResponse response, IDictionary parameterValues) { - var create = (CreateResponse)response; - parameterValues["@@IDENTITY"] = new SqlEntityReference(DataSource, LogicalName, create.id); + if (response is CreateResponse create) + parameterValues["@@IDENTITY"] = new SqlEntityReference(DataSource, LogicalName, create.id); } protected override void RenameSourceColumns(IDictionary columnRenamings) diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MergeJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MergeJoinNode.cs index 76f1a87e..71baaf06 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MergeJoinNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MergeJoinNode.cs @@ -17,6 +17,9 @@ namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan /// class MergeJoinNode : FoldableJoinNode { + private SortNode _leftSort; + private SortNode _rightSort; + [Description("Many to Many")] [Category("Merge Join")] public bool ManyToMany { get; private set; } @@ -233,6 +236,19 @@ private bool Done(bool hasLeft, bool hasRight) public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext context, IList hints) { + // If we've previously added sort nodes to either input, remove them before trying to fold the query again + if (LeftSource == _leftSort) + { + LeftSource = _leftSort.Source; + _leftSort = null; + } + + if (RightSource == _rightSort) + { + RightSource = _rightSort.Source; + _rightSort = null; + } + var folded = base.FoldQuery(context, hints); if (folded != this) @@ -289,7 +305,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext return this; // Can't fold the join down into the FetchXML, so add a sort and try to fold that in instead - LeftSource = new SortNode + _leftSort = new SortNode { Source = LeftSource, Sorts = @@ -300,10 +316,11 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext SortOrder = SortOrder.Ascending } } - }.FoldQuery(context, hints); + }; + LeftSource = _leftSort.FoldQuery(context, hints); LeftSource.Parent = this; - RightSource = new SortNode + _rightSort = new SortNode { Source = RightSource, Sorts = @@ -314,24 +331,21 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext SortOrder = SortOrder.Ascending } } - }.FoldQuery(context, hints); + }; + RightSource = _rightSort.FoldQuery(context, hints); RightSource.Parent = this; // If we couldn't fold the sorts, it's probably faster to use a hash join instead if we only want partial results - var leftSort = LeftSource as SortNode; - var rightSort = RightSource as SortNode; - - if (leftSort == null && rightSort == null) + if (LeftSource != _leftSort && RightSource != _rightSort) return this; - hashJoin.LeftSource = leftSort?.Source ?? LeftSource; - hashJoin.RightSource = rightSort?.Source ?? RightSource; - + hashJoin.LeftSource = (LeftSource == _leftSort) ? _leftSort.Source : LeftSource; + hashJoin.RightSource = (RightSource == _rightSort) ? _rightSort.Source : RightSource; var foldedHashJoin = hashJoin.FoldQuery(context, hints); if (Parent is TopNode || - leftSort != null && rightSort != null) + LeftSource == _leftSort && RightSource == _rightSort) return foldedHashJoin; LeftSource.Parent = this; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs index de391dc0..069d9c91 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NestedLoopNode.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Data.SqlTypes; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -71,14 +72,14 @@ protected override IEnumerable ExecuteInternal(NodeExecutionContext cont if (OuterReferences != null) { if (innerParameters == null) - innerParameters = new Dictionary(); + innerParameters = new Dictionary(); else - innerParameters = new Dictionary(innerParameters); + innerParameters = new Dictionary(innerParameters); foreach (var kvp in OuterReferences) { left.Attributes.TryGetValue(kvp.Key, out var outerValue); - innerParameters[kvp.Value] = outerValue; + innerParameters[kvp.Value] = (INullable)outerValue; } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NodeSchema.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NodeSchema.cs index d2e5b7d2..769db722 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NodeSchema.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/NodeSchema.cs @@ -157,6 +157,16 @@ public static IColumnDefinition Null(this IColumnDefinition col) { return new ColumnDefinition(col.Type, true, col.IsCalculated, col.IsVisible); } + + public static IColumnDefinition Invisible(this IColumnDefinition col) + { + return new ColumnDefinition(col.Type, col.IsNullable, col.IsCalculated, false); + } + + public static IColumnDefinition Calculated(this IColumnDefinition col) + { + return new ColumnDefinition(col.Type, col.IsNullable, true, col.IsVisible); + } } /// diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/PartitionedAggregateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/PartitionedAggregateNode.cs index 9041e45d..bbc31602 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/PartitionedAggregateNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/PartitionedAggregateNode.cs @@ -187,7 +187,7 @@ protected override IEnumerable ExecuteInternal(NodeExecutionContext cont Parent = this }; - var partitionParameterValues = new Dictionary + var partitionParameterValues = new Dictionary { ["@PartitionStart"] = minKey, ["@PartitionEnd"] = maxKey diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/QueryExecutionException.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/QueryExecutionException.cs index 5b8375bf..f73ad6fa 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/QueryExecutionException.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/QueryExecutionException.cs @@ -93,6 +93,7 @@ private static int FaultCodeToSqlError(OrganizationServiceFault fault) case -2147086332: case -2147187954: return 547; case 409: // Elastic tables use HTTP status codes instead of the standard web service error codes + case -2147220950: case -2147220937: return 2627; } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectDataReader.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectDataReader.cs index ff5b55ab..1ceef794 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectDataReader.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectDataReader.cs @@ -17,12 +17,12 @@ class SelectDataReader : DbDataReader private readonly IDisposable _timer; private readonly INodeSchema _schema; private readonly IEnumerator _source; - private readonly IDictionary _parameterValues; + private readonly IDictionary _parameterValues; private Entity _row; private bool _closed; private int _rowCount; - public SelectDataReader(List columnSet, IDisposable timer, INodeSchema schema, IEnumerable source, IDictionary parameterValues) + public SelectDataReader(List columnSet, IDisposable timer, INodeSchema schema, IEnumerable source, IDictionary parameterValues) { _columnSet = columnSet; _timer = timer; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs index 653d867f..d9060ce9 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs @@ -11,6 +11,7 @@ using Microsoft.SqlServer.TransactSql.ScriptDom; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata.Query; +using Newtonsoft.Json; namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan { @@ -40,6 +41,7 @@ class SelectNode : BaseNode, ISingleSourceExecutionPlanNode, IDataReaderExecutio /// The schema that should be used for expanding "*" columns /// [Browsable(false)] + [JsonIgnore] public INodeSchema LogicalSourceSchema { get; set; } [Browsable(false)] @@ -167,6 +169,7 @@ internal static void FoldFetchXmlColumns(IDataExecutionPlanNode source, List + // 4. virtual ___name or ___type attributes if (!hasStar) { var aliasedColumns = columnSet @@ -178,9 +181,36 @@ internal static void FoldFetchXmlColumns(IDataExecutionPlanNode source, List c.SourceColumn, StringComparer.OrdinalIgnoreCase) - .Where(g => g.Count() == 1) // Don't fold aliases if there are multiple aliases for the same source column + .Select(c => + { + // Check which underlying attribute the data is coming from, handling virtual attributes + var parts = c.SourceColumn.SplitMultiPartIdentifier(); + var entityName = fetchXml.Entity.name; + var attrName = parts.Last(); + + if (parts.Length > 1 && !parts[0].Equals(fetchXml.Alias)) + entityName = fetchXml.Entity.FindLinkEntity(parts[0])?.name; + + if (entityName == null) + return null; + + var metadata = dataSource.Metadata; + var meta = metadata[entityName].Attributes.SingleOrDefault(a => a.LogicalName.Equals(attrName, StringComparison.OrdinalIgnoreCase) && a.AttributeOf == null); + var isVirtual = false; + if (meta == null) + { + meta = metadata[entityName].FindBaseAttributeFromVirtualAttribute(attrName, out _); + if (meta != null) + isVirtual = true; + } + + return new { c.Mapping, c.SourceColumn, c.Alias, meta?.LogicalName, IsVirtual = isVirtual }; + }) + .Where(c => c?.LogicalName != null) // Ignore attributes we can't find in the metadata + .GroupBy(c => c.LogicalName, StringComparer.OrdinalIgnoreCase) + .Where(g => g.Count() == 1) // Ignore attributes that appear multiple times, either as physical or virtual attributes .Select(g => g.Single()) + .Where(c => c.IsVirtual == false) // Ignore virtual attributes .GroupBy(c => c.Alias, StringComparer.OrdinalIgnoreCase) .Where(g => g.Count() == 1) // Don't fold aliases if there are multiple columns using the same alias .Select(g => g.Single()) @@ -205,7 +235,7 @@ internal static void FoldFetchXmlColumns(IDataExecutionPlanNode source, List { var attr = fetchXml.AddAttribute(c.SourceColumn, null, dataSource.Metadata, out _, out var linkEntity); - return new { Mapping = c.Mapping, SourceColumn = c.SourceColumn, Alias = c.Alias, Attr = attr, LinkEntity = linkEntity }; + return new { c.Mapping, c.SourceColumn, c.Alias, Attr = attr, LinkEntity = linkEntity }; }) .Where(c => { diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SortNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SortNode.cs index 0859317b..938f53f5 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SortNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SortNode.cs @@ -241,7 +241,7 @@ private IDataExecutionPlanNodeInternal FoldSorts(NodeCompilationContext context) } // Allow folding sorts around filters and Compute Scalar (so long as sort is not on a calculated field) - // Can fold to the outer input of a nested loop join + // Can fold to the outer input of a nested loop join and sources of spools var source = Source; var fetchXml = Source as FetchXmlScan; @@ -253,6 +253,10 @@ private IDataExecutionPlanNodeInternal FoldSorts(NodeCompilationContext context) source = computeScalar.Source; else if (source is NestedLoopNode nestedLoop) source = nestedLoop.LeftSource; + else if (source is TableSpoolNode tableSpool) + source = tableSpool.Source; + else if (source is IndexSpoolNode indexSpool) + source = indexSpool.Source; else break; @@ -383,7 +387,7 @@ private IDataExecutionPlanNodeInternal FoldSorts(NodeCompilationContext context) if (attribute is EnumAttributeMetadata || attribute is BooleanAttributeMetadata) { - if (useRawOrderBy == false) + if (useRawOrderBy == false || !dataSource.UseRawOrderByReliable) return this; useRawOrderBy = true; @@ -433,7 +437,7 @@ private IDataExecutionPlanNodeInternal FoldSorts(NodeCompilationContext context) if (attribute is EnumAttributeMetadata || attribute is BooleanAttributeMetadata) { - if (useRawOrderBy == false) + if (useRawOrderBy == false || !dataSource.UseRawOrderByReliable) return this; useRawOrderBy = true; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SqlTypeConverter.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SqlTypeConverter.cs index 12b861c2..d7d09477 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SqlTypeConverter.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/SqlTypeConverter.cs @@ -662,6 +662,7 @@ public static Expression Convert(Expression expr, DataTypeReference from, DataTy var catchOverflowExceptionBlock = Expression.Catch(typeof(OverflowException), Expression.Throw(Expression.New(typeof(QueryExecutionException).GetConstructor(new[] { typeof(Sql4CdsError) }), overflowError), targetType)); expr = Expression.TryCatch(expr, catchFormatExceptionBlock, catchOverflowExceptionBlock); + //expr = Expression.Invoke(Expression.Lambda(expr)); } if (toSqlType == null) @@ -1374,6 +1375,7 @@ private static Func CompileConversion(Type sourceType, Type dest var block = Expression.Block(destType, variables, body); var catchBlock = Expression.Catch(typeof(ArgumentException), block); parsedValue = Expression.TryCatch(parsedValue, catchBlock); + //parsedValue = Expression.Invoke(Expression.Lambda(parsedValue)); expression = Expression.Condition(nullCheck, nullValue, parsedValue); } @@ -1395,6 +1397,7 @@ private static Func CompileConversion(Type sourceType, Type dest var block = Expression.Block(destType, variables, body); var catchBlock = Expression.Catch(typeof(ArgumentException), block); expression = Expression.TryCatch(expression, catchBlock); + //expression = Expression.Invoke(Expression.Lambda(expression)); } else if (expression.Type == typeof(SqlInt32) && destType == typeof(OptionSetValue)) { diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs index 67f55cb8..1bd11dca 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/UpdateNode.cs @@ -268,6 +268,39 @@ public override void Execute(NodeExecutionContext context, out int recordsAffect }); } } + else if (meta.LogicalName == "principalobjectaccess") + { + var objectIdPrev = preImage.GetAttributeValue("objectid"); + var principalIdPrev = preImage.GetAttributeValue("principalid"); + var accessMaskPrev = (AccessRights)preImage.GetAttributeValue("accessrightsmask"); + var objectIdNew = update.GetAttributeValue("objectid") ?? objectIdPrev; + var principalIdNew = update.GetAttributeValue("principalid") ?? principalIdPrev; + var accessMaskNew = (AccessRights?)update.GetAttributeValue("accessrightsmask") ?? accessMaskPrev; + + // Check if we need to remove any previous share permissions + if (!objectIdPrev.Equals(objectIdNew) || !principalIdPrev.Equals(principalIdNew) || (accessMaskNew & accessMaskPrev) != accessMaskPrev) + { + requests.Add(new RevokeAccessRequest + { + Target = objectIdPrev, + Revokee = principalIdPrev + }); + } + + // Check if we need to add any new share permissions + if (accessMaskNew != AccessRights.None) + { + requests.Add(new GrantAccessRequest + { + Target = objectIdNew, + PrincipalAccess = new PrincipalAccess + { + Principal = principalIdNew, + AccessMask = accessMaskNew + } + }); + } + } else { var updateRequest = new UpdateRequest { Target = update }; diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs index a30ececd..6cb58de2 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs @@ -1393,7 +1393,7 @@ private InsertNode ConvertInsertSpecification(NamedTableReference target, IList< Source = source }; - ValidateDMLSchema(target); + ValidateDMLSchema(target, false); // Validate the entity name EntityMetadata metadata; @@ -1410,6 +1410,7 @@ private InsertNode ConvertInsertSpecification(NamedTableReference target, IList< var attributes = metadata.Attributes.ToDictionary(attr => attr.LogicalName, StringComparer.OrdinalIgnoreCase); var attributeNames = new HashSet(StringComparer.OrdinalIgnoreCase); var virtualTypeAttributes = new HashSet(StringComparer.OrdinalIgnoreCase); + var virtualPidAttributes = new HashSet(StringComparer.OrdinalIgnoreCase); var schema = sourceColumns == null ? null : ((IDataExecutionPlanNodeInternal)source).GetSchema(_nodeContext); // Check all target columns are valid for create @@ -1430,6 +1431,18 @@ attr is LookupAttributeMetadata lookupAttr && continue; } + // Could be a virtual ___pid attribute + if (colName.EndsWith("pid", StringComparison.OrdinalIgnoreCase) && + attributes.TryGetValue(colName.Substring(0, colName.Length - 3), out attr) && + attr is LookupAttributeMetadata elasticLookupAttr && + elasticLookupAttr.Targets.Any(t => dataSource.Metadata[t].DataProviderId == DataProviders.ElasticDataProvider)) + { + if (!virtualPidAttributes.Add(colName)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.DuplicateInsertUpdateColumn(col)); + + continue; + } + if (!attributes.TryGetValue(colName, out attr)) throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidColumnName(col)); @@ -1448,6 +1461,19 @@ attr is LookupAttributeMetadata lookupAttr && if (attr.LogicalName != relationship.Entity1IntersectAttribute && attr.LogicalName != relationship.Entity2IntersectAttribute) throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) { Suggestion = $"Only the {relationship.Entity1IntersectAttribute} and {relationship.Entity2IntersectAttribute} columns can be used when inserting values into the {metadata.LogicalName} table" }; } + else if (metadata.LogicalName == "principalobjectaccess") + { + if (attr.LogicalName == "objecttypecode" || attr.LogicalName == "principaltypecode") + { + if (!virtualTypeAttributes.Add(colName)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.DuplicateInsertUpdateColumn(col)); + + continue; + } + + if (attr.LogicalName != "objectid" && attr.LogicalName != "principalid" && attr.LogicalName != "accessrightsmask") + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) { Suggestion = "Only the objectid, principalid and accessrightsmask columns can be used when inserting values into the principalobjectaccess table" }; + } else { if (attr.IsValidForCreate == false) @@ -1471,6 +1497,15 @@ attr is LookupAttributeMetadata lookupAttr && if (!attributeNames.Contains(relationship.Entity2IntersectAttribute)) throw new NotSupportedQueryFragmentException(Sql4CdsError.NotNullInsert(new Identifier { Value = relationship.Entity2IntersectAttribute }, new Identifier { Value = metadata.LogicalName }, "Insert", target)) { Suggestion = $"Inserting values into the {metadata.LogicalName} table requires the {relationship.Entity2IntersectAttribute} column to be set" }; } + else if (metadata.LogicalName == "principalobjectaccess") + { + if (!attributeNames.Contains("objectid")) + throw new NotSupportedQueryFragmentException(Sql4CdsError.NotNullInsert(new Identifier { Value = "objectid" }, new Identifier { Value = metadata.LogicalName }, "Insert", target)) { Suggestion = $"Inserting values into the {metadata.LogicalName} table requires the objectid column to be set" }; + if (!attributeNames.Contains("principalid")) + throw new NotSupportedQueryFragmentException(Sql4CdsError.NotNullInsert(new Identifier { Value = "principalid" }, new Identifier { Value = metadata.LogicalName }, "Insert", target)) { Suggestion = $"Inserting values into the {metadata.LogicalName} table requires the principalid column to be set" }; + if (!attributeNames.Contains("accessrightsmask")) + throw new NotSupportedQueryFragmentException(Sql4CdsError.NotNullInsert(new Identifier { Value = "accessrightsmask" }, new Identifier { Value = metadata.LogicalName }, "Insert", target)) { Suggestion = $"Inserting values into the {metadata.LogicalName} table requires the accessrightsmask column to be set" }; + } if (sourceColumns == null) { @@ -1499,6 +1534,11 @@ attr is LookupAttributeMetadata lookupAttr && targetName = colName; targetType = DataTypeHelpers.NVarChar(MetadataExtensions.EntityLogicalNameMaxLength, dataSource.DefaultCollation, CollationLabel.CoercibleDefault); } + else if (virtualPidAttributes.Contains(colName)) + { + targetName = colName; + targetType = DataTypeHelpers.NVarChar(100, dataSource.DefaultCollation, CollationLabel.CoercibleDefault); + } else { var attr = attributes[colName]; @@ -1531,24 +1571,51 @@ attr is LookupAttributeMetadata lookupAttr && if (attributeNames.Contains(targetAttrName)) { - var targetLookupAttribute = attributes[targetAttrName] as LookupAttributeMetadata; - - if (targetLookupAttribute == null) - continue; - - if (targetLookupAttribute.Targets.Length > 1 && - !virtualTypeAttributes.Contains(targetAttrName + "type") && - targetLookupAttribute.AttributeType != AttributeTypeCode.PartyList && - (schema == null || (node.ColumnMappings[targetAttrName].ToColumnReference().GetType(GetExpressionContext(schema, _nodeContext), out var lookupType) != typeof(SqlEntityReference) && lookupType != DataTypeHelpers.ImplicitIntForNullLiteral))) + // Special case for principalobjectaccess table + if (metadata.LogicalName == "principalobjectaccess") { - // Special case: not required for listmember.entityid - if (metadata.LogicalName == "listmember" && targetLookupAttribute.LogicalName == "entityid") + if ((targetAttrName == "objectid" || targetAttrName == "principalid") && + !virtualTypeAttributes.Contains(targetAttrName.Replace("id", "typecode")) && + (schema == null || (node.ColumnMappings[targetAttrName].ToColumnReference().GetType(GetExpressionContext(schema, _nodeContext), out var lookupType) != typeof(SqlEntityReference) && lookupType != DataTypeHelpers.ImplicitIntForNullLiteral))) + { + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) + { + Suggestion = $"Inserting values into a polymorphic lookup field requires setting the associated type column as well\r\nAdd a value for the {targetAttrName.Replace("id", "typecode")} column" + }; + } + } + else + { + var targetLookupAttribute = attributes[targetAttrName] as LookupAttributeMetadata; + + if (targetLookupAttribute == null) continue; - throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) + if (targetLookupAttribute.Targets.Length > 1 && + !virtualTypeAttributes.Contains(targetAttrName + "type") && + targetLookupAttribute.AttributeType != AttributeTypeCode.PartyList && + (schema == null || (node.ColumnMappings[targetAttrName].ToColumnReference().GetType(GetExpressionContext(schema, _nodeContext), out var lookupType) != typeof(SqlEntityReference) && lookupType != DataTypeHelpers.ImplicitIntForNullLiteral))) { - Suggestion = $"Inserting values into a polymorphic lookup field requires setting the associated type column as well\r\nAdd a value for the {targetLookupAttribute.LogicalName}type column and set it to one of the following values:\r\n{String.Join("\r\n", targetLookupAttribute.Targets.Select(t => $"* {t}"))}" - }; + // Special case: not required for listmember.entityid + if (metadata.LogicalName != "listmember" || targetLookupAttribute.LogicalName != "entityid") + { + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) + { + Suggestion = $"Inserting values into a polymorphic lookup field requires setting the associated type column as well\r\nAdd a value for the {targetLookupAttribute.LogicalName}type column and set it to one of the following values:\r\n{String.Join("\r\n", targetLookupAttribute.Targets.Select(t => $"* {t}"))}" + }; + } + } + + // If the lookup references an elastic table we also need the pid column + if (targetLookupAttribute.Targets.Any(t => dataSource.Metadata[t].DataProviderId == DataProviders.ElasticDataProvider) && + !virtualPidAttributes.Contains(targetAttrName + "pid") && + (schema == null || (node.ColumnMappings[targetAttrName].ToColumnReference().GetType(GetExpressionContext(schema, _nodeContext), out var elasticLookupType) != null && elasticLookupType != DataTypeHelpers.ImplicitIntForNullLiteral))) + { + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) + { + Suggestion = $"Inserting values into an elastic lookup field requires setting the associated pid column as well\r\nAdd a value for the {targetLookupAttribute.LogicalName}pid column" + }; + } } } else if (virtualTypeAttributes.Contains(targetAttrName)) @@ -1563,12 +1630,24 @@ attr is LookupAttributeMetadata lookupAttr && }; } } + else if (virtualPidAttributes.Contains(targetAttrName)) + { + var idAttrName = targetAttrName.Substring(0, targetAttrName.Length - 3); + + if (!attributeNames.Contains(idAttrName)) + { + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(col)) + { + Suggestion = $"Inserting values into an elastic lookup field requires setting the associated ID column as well\r\nAdd a value for the {idAttrName} column" + }; + } + } } return node; } - private void ValidateDMLSchema(NamedTableReference target) + private void ValidateDMLSchema(NamedTableReference target, bool allowBin) { if (String.IsNullOrEmpty(target.SchemaObject.SchemaIdentifier?.Value)) return; @@ -1582,6 +1661,14 @@ private void ValidateDMLSchema(NamedTableReference target) if (target.SchemaObject.SchemaIdentifier.Value.Equals("metadata", StringComparison.OrdinalIgnoreCase)) throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidObjectName(target.SchemaObject)) { Suggestion = "Metadata tables are read-only" }; + if (target.SchemaObject.SchemaIdentifier.Value.Equals("bin", StringComparison.OrdinalIgnoreCase)) + { + if (allowBin) + return; + + throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidObjectName(target.SchemaObject)) { Suggestion = "Recycle bin tables are valid for SELECT and DELETE only" }; + } + throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidObjectName(target.SchemaObject)) { Suggestion = "All data tables are in the 'dbo' schema" }; } @@ -1612,8 +1699,6 @@ private DeleteNode ConvertDeleteStatement(DeleteSpecification delete, IList k))}" }; + ValidateDMLSchema(deleteTarget.Target, true); + var targetAlias = deleteTarget.TargetAliasName ?? deleteTarget.TargetEntityName; var targetLogicalName = deleteTarget.TargetEntityName; @@ -1651,6 +1738,7 @@ private DeleteNode ConvertDeleteStatement(DeleteSpecification delete, IList(); if (targetMetadata.LogicalName == "listmember") { @@ -1668,27 +1756,134 @@ private DeleteNode ConvertDeleteStatement(DeleteSpecification delete, IList k))}" }; + ValidateDMLSchema(updateTarget.Target, false); + var targetAlias = updateTarget.TargetAliasName ?? updateTarget.TargetEntityName; var targetLogicalName = updateTarget.TargetEntityName; @@ -1867,6 +2062,7 @@ private UpdateNode ConvertUpdateStatement(UpdateSpecification update, IList attr.LogicalName, StringComparer.OrdinalIgnoreCase); var attributeNames = new HashSet(StringComparer.OrdinalIgnoreCase); var virtualTypeAttributes = new HashSet(StringComparer.OrdinalIgnoreCase); + var virtualPidAttributes = new HashSet(StringComparer.OrdinalIgnoreCase); var existingAttributes = new HashSet(StringComparer.OrdinalIgnoreCase); var useStateTransitions = !hints.OfType().Any(h => h.Hints.Any(s => s.Value.Equals("DISABLE_STATE_TRANSITIONS", StringComparison.OrdinalIgnoreCase))); var stateTransitions = useStateTransitions ? StateTransitionLoader.LoadStateTransitions(targetMetadata) : null; @@ -1928,6 +2124,20 @@ attr is LookupAttributeMetadata lookupAttr && if (!virtualTypeAttributes.Add(targetAttrName)) throw new NotSupportedQueryFragmentException(Sql4CdsError.DuplicateInsertUpdateColumn(assignment.Column)); } + else if (targetMetadata.LogicalName == "principalobjectaccess" && + targetAttrName.EndsWith("typecode", StringComparison.OrdinalIgnoreCase)) + { + if (!virtualTypeAttributes.Add(targetAttrName)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.DuplicateInsertUpdateColumn(assignment.Column)); + } + else if (targetAttrName.EndsWith("pid", StringComparison.OrdinalIgnoreCase) && + attributes.TryGetValue(targetAttrName.Substring(0, targetAttrName.Length - 3), out attr) && + attr is LookupAttributeMetadata elasticLookupAttr && + elasticLookupAttr.Targets.Any(t => dataSource.Metadata[t].DataProviderId == DataProviders.ElasticDataProvider)) + { + if (!virtualPidAttributes.Add(targetAttrName)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.DuplicateInsertUpdateColumn(assignment.Column)); + } else { if (!attributes.TryGetValue(targetAttrName, out attr)) @@ -1946,6 +2156,11 @@ attr is LookupAttributeMetadata lookupAttr && if (attr.LogicalName != "listid" && attr.LogicalName != "entityid") throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(assignment.Column)) { Suggestion = "Only the listid and entityid columns can be used when updating values in the listmember table" }; } + else if (targetMetadata.LogicalName == "principalobjectaccess") + { + if (attr.LogicalName != "objectid" && attr.LogicalName != "principalid" && attr.LogicalName != "accessrightsmask") + throw new NotSupportedQueryFragmentException(Sql4CdsError.ReadOnlyColumn(assignment.Column)) { Suggestion = "Only the objectid, principalid and accessrightsmask columns can be used when updating values in the principalobjectaccess table" }; + } else { if (attr.IsValidForUpdate == false) @@ -1984,6 +2199,14 @@ attr is LookupAttributeMetadata lookupAttr && existingAttributes.Add("listid"); existingAttributes.Add("entityid"); } + else if (targetLogicalName == "principalobjectaccess") + { + existingAttributes.Add("objectid"); + existingAttributes.Add("objecttypecode"); + existingAttributes.Add("principalid"); + existingAttributes.Add("principaltypecode"); + existingAttributes.Add("accessrightsmask"); + } foreach (var existingAttribute in existingAttributes) { @@ -2013,7 +2236,7 @@ attr is LookupAttributeMetadata lookupAttr && var source = ConvertSelectStatement(selectStatement); // Add UPDATE - var updateNode = ConvertSetClause(update.SetClauses, existingAttributes, dataSource, source, targetLogicalName, targetAlias, attributeNames, virtualTypeAttributes, hints); + var updateNode = ConvertSetClause(update.SetClauses, existingAttributes, dataSource, source, targetLogicalName, targetAlias, attributeNames, virtualTypeAttributes, virtualPidAttributes, hints); updateNode.StateTransitions = stateTransitions; return updateNode; @@ -2041,7 +2264,7 @@ private void CopyDmlHintsToSelectStatement(IList hints, SelectSta } } - private UpdateNode ConvertSetClause(IList setClauses, HashSet existingAttributes, DataSource dataSource, IExecutionPlanNodeInternal node, string targetLogicalName, string targetAlias, HashSet attributeNames, HashSet virtualTypeAttributes, IList queryHints) + private UpdateNode ConvertSetClause(IList setClauses, HashSet existingAttributes, DataSource dataSource, IExecutionPlanNodeInternal node, string targetLogicalName, string targetAlias, HashSet attributeNames, HashSet virtualTypeAttributes, HashSet virtualPidAttributes, IList queryHints) { var targetMetadata = dataSource.Metadata[targetLogicalName]; var attributes = targetMetadata.Attributes.ToDictionary(attr => attr.LogicalName, StringComparer.OrdinalIgnoreCase); @@ -2073,8 +2296,11 @@ private UpdateNode ConvertSetClause(IList setClauses, HashSet { targetType = DataTypeHelpers.NVarChar(MetadataExtensions.EntityLogicalNameMaxLength, dataSource.DefaultCollation, CollationLabel.CoercibleDefault); - var targetAttribute = attributes[targetAttrName.Substring(0, targetAttrName.Length - 4)]; - targetAttrName = targetAttribute.LogicalName + targetAttrName.Substring(targetAttrName.Length - 4, 4).ToLower(); + if (targetMetadata.LogicalName != "principalobjectaccess") + { + var targetAttribute = attributes[targetAttrName.Substring(0, targetAttrName.Length - 4)]; + targetAttrName = targetAttribute.LogicalName + targetAttrName.Substring(targetAttrName.Length - 4, 4).ToLower(); + } } else { @@ -2165,11 +2391,25 @@ private UpdateNode ConvertSetClause(IList setClauses, HashSet Suggestion = $"Add a SET clause for the {targetLookupAttribute.LogicalName}type column and set it to one of the following values:\r\n{String.Join("\r\n", targetLookupAttribute.Targets.Select(t => $"* {t}"))}" }; } + + // If the lookup references an elastic table we also need the pid column + if (targetLookupAttribute.Targets.Any(t => dataSource.Metadata[t].DataProviderId == DataProviders.ElasticDataProvider) && + !virtualPidAttributes.Contains(targetAttrName + "pid") && + (!sourceTypes.TryGetValue(targetAttrName, out var elasticLookupType) || elasticLookupType != DataTypeHelpers.ImplicitIntForNullLiteral)) + { + throw new NotSupportedQueryFragmentException("Updating an elastic lookup field requires setting the associated pid column as well", assignment.Column) + { + Suggestion = $"Add a SET clause for the {targetLookupAttribute.LogicalName}pid column" + }; + } } else if (virtualTypeAttributes.Contains(targetAttrName)) { var idAttrName = targetAttrName.Substring(0, targetAttrName.Length - 4); + if (targetMetadata.LogicalName == "principalobjectaccess") + idAttrName = targetAttrName.Substring(0, targetAttrName.Length - 8) + "id"; + if (!attributeNames.Contains(idAttrName)) { throw new NotSupportedQueryFragmentException("Updating a polymorphic type field requires setting the associated ID column as well", assignment.Column) @@ -2178,6 +2418,18 @@ private UpdateNode ConvertSetClause(IList setClauses, HashSet }; } } + else if (virtualPidAttributes.Contains(targetAttrName)) + { + var idAttrName = targetAttrName.Substring(0, targetAttrName.Length - 3); + + if (!attributeNames.Contains(idAttrName)) + { + throw new NotSupportedQueryFragmentException("Updating an elastic lookup field requires setting the associated ID column as well", assignment.Column) + { + Suggestion = $"Add a SET clause for the {idAttrName} column" + }; + } + } } return update; @@ -2623,7 +2875,7 @@ private void ConvertForXmlClause(SelectNode selectNode, XmlForClause forXml, Nod }); } - private IDataExecutionPlanNodeInternal ConvertInSubqueries(IDataExecutionPlanNodeInternal source, IList hints, TSqlFragment query, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences) + private IDataExecutionPlanNodeInternal ConvertInSubqueries(IDataExecutionPlanNodeInternal source, IList hints, TSqlFragment query, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences) { var visitor = new InSubqueryVisitor(); query.Accept(visitor); @@ -2631,141 +2883,143 @@ private IDataExecutionPlanNodeInternal ConvertInSubqueries(IDataExecutionPlanNod if (visitor.InSubqueries.Count == 0) return source; + foreach (var inSubquery in visitor.InSubqueries) + source = ConvertInSubquery(source, hints, query, inSubquery, context, outerSchema, outerReferences); + + return source; + } + + private IDataExecutionPlanNodeInternal ConvertInSubquery(IDataExecutionPlanNodeInternal source, IList hints, TSqlFragment query, InPredicate inSubquery, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences) + { var computeScalar = source as ComputeScalarNode; - var rewrites = new Dictionary(); var schema = source.GetSchema(context); - foreach (var inSubquery in visitor.InSubqueries) - { - // Validate the LHS expression - inSubquery.Expression.GetType(GetExpressionContext(schema, context), out _); + // Validate the LHS expression + inSubquery.Expression.GetType(GetExpressionContext(schema, context), out _); - // Each query of the format "col1 IN (SELECT col2 FROM source)" becomes a left outer join: - // LEFT JOIN source ON col1 = col2 - // and the result is col2 IS NOT NULL + // Each query of the format "col1 IN (SELECT col2 FROM source)" becomes a left outer join: + // LEFT JOIN source ON col1 = col2 + // and the result is col2 IS NOT NULL - // Ensure the left hand side is a column - if (!(inSubquery.Expression is ColumnReferenceExpression lhsCol)) - { - if (computeScalar == null) - { - computeScalar = new ComputeScalarNode { Source = source }; - source = computeScalar; - } - - var alias = context.GetExpressionName(); - computeScalar.Columns[alias] = inSubquery.Expression.Clone(); - lhsCol = alias.ToColumnReference(); - } - else + // Ensure the left hand side is a column + if (!(inSubquery.Expression is ColumnReferenceExpression lhsCol)) + { + if (computeScalar == null) { - // Normalize the LHS column - if (schema.ContainsColumn(lhsCol.GetColumnName(), out var lhsColNormalized)) - lhsCol = lhsColNormalized.ToColumnReference(); + computeScalar = new ComputeScalarNode { Source = source }; + source = computeScalar; } - var parameters = context.ParameterTypes == null ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary(context.ParameterTypes, StringComparer.OrdinalIgnoreCase); - var innerContext = new NodeCompilationContext(context, parameters); - var references = new Dictionary(); - var innerQuery = ConvertSelectStatement(inSubquery.Subquery.QueryExpression, hints, schema, references, innerContext); + var alias = context.GetExpressionName(); + computeScalar.Columns[alias] = inSubquery.Expression.Clone(); + lhsCol = alias.ToColumnReference(); + } + else + { + // Normalize the LHS column + if (schema.ContainsColumn(lhsCol.GetColumnName(), out var lhsColNormalized)) + lhsCol = lhsColNormalized.ToColumnReference(); + } - // Scalar subquery must return exactly one column and one row - if (innerQuery.ColumnSet.Count != 1) - throw new NotSupportedQueryFragmentException(Sql4CdsError.MultiColumnScalarSubquery(inSubquery.Subquery)); + var parameters = context.ParameterTypes == null ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary(context.ParameterTypes, StringComparer.OrdinalIgnoreCase); + var innerContext = new NodeCompilationContext(context, parameters); + var references = new Dictionary(); + var innerQuery = ConvertSelectStatement(inSubquery.Subquery.QueryExpression, hints, schema, references, innerContext); - // Create the join - BaseJoinNode join; - var testColumn = innerQuery.ColumnSet[0].SourceColumn; + // Scalar subquery must return exactly one column and one row + if (innerQuery.ColumnSet.Count != 1) + throw new NotSupportedQueryFragmentException(Sql4CdsError.MultiColumnScalarSubquery(inSubquery.Subquery)); - if (references.Count == 0) + // Create the join + BaseJoinNode join; + var testColumn = innerQuery.ColumnSet[0].SourceColumn; + + if (references.Count == 0) + { + if (UseMergeJoin(source, innerQuery.Source, context, references, testColumn, lhsCol.GetColumnName(), true, true, out var outputCol, out var merge)) { - if (UseMergeJoin(source, innerQuery.Source, context, references, testColumn, lhsCol.GetColumnName(), true, true, out var outputCol, out var merge)) - { - testColumn = outputCol; - join = merge; - } - else + testColumn = outputCol; + join = merge; + } + else + { + // We need the inner list to be distinct to avoid creating duplicates during the join + var innerSchema = innerQuery.Source.GetSchema(new NodeCompilationContext(DataSources, Options, parameters, Log)); + if (innerQuery.ColumnSet[0].SourceColumn != innerSchema.PrimaryKey && !(innerQuery.Source is DistinctNode)) { - // We need the inner list to be distinct to avoid creating duplicates during the join - var innerSchema = innerQuery.Source.GetSchema(new NodeCompilationContext(DataSources, Options, parameters, Log)); - if (innerQuery.ColumnSet[0].SourceColumn != innerSchema.PrimaryKey && !(innerQuery.Source is DistinctNode)) - { - innerQuery.Source = new DistinctNode - { - Source = innerQuery.Source, - Columns = { innerQuery.ColumnSet[0].SourceColumn } - }; - } - - // This isn't a correlated subquery, so we can use a foldable join type. Alias the results so there's no conflict with the - // same table being used inside the IN subquery and elsewhere - var alias = new AliasNode(innerQuery, new Identifier { Value = context.GetExpressionName() }, context); - - testColumn = $"{alias.Alias}.{alias.ColumnSet[0].OutputColumn}"; - join = new HashJoinNode + innerQuery.Source = new DistinctNode { - LeftSource = source, - LeftAttribute = lhsCol.Clone(), - RightSource = alias, - RightAttribute = new ColumnReferenceExpression { MultiPartIdentifier = new MultiPartIdentifier { Identifiers = { new Identifier { Value = alias.Alias }, new Identifier { Value = alias.ColumnSet[0].OutputColumn } } } } + Source = innerQuery.Source, + Columns = { innerQuery.ColumnSet[0].SourceColumn } }; } - if (!join.SemiJoin) - { - // Convert the join to a semi join to ensure requests for wildcard columns aren't folded to the IN subquery - var definedValue = context.GetExpressionName(); - join.SemiJoin = true; - join.OutputRightSchema = false; - join.DefinedValues[definedValue] = testColumn; - testColumn = definedValue; - } - } - else - { - // We need to use nested loops for correlated subqueries - // TODO: We could use a hash join where there is a simple correlation, but followed by a distinct node to eliminate duplicates - // We could also move the correlation criteria out of the subquery and into the join condition. We would then make one request to - // get all the related records and spool that in memory to get the relevant results in the nested loop. Need to understand how - // many rows are likely from the outer query to work out if this is going to be more efficient or not. - if (innerQuery.Source is ISingleSourceExecutionPlanNode loopRightSourceSimple) - InsertCorrelatedSubquerySpool(loopRightSourceSimple, source, hints, context, references.Values.ToArray()); - - var definedValue = context.GetExpressionName(); + // This isn't a correlated subquery, so we can use a foldable join type. Alias the results so there's no conflict with the + // same table being used inside the IN subquery and elsewhere + var alias = new AliasNode(innerQuery, new Identifier { Value = context.GetExpressionName() }, context); - join = new NestedLoopNode + testColumn = $"{alias.Alias}.{alias.ColumnSet[0].OutputColumn}"; + join = new HashJoinNode { LeftSource = source, - RightSource = innerQuery.Source, - OuterReferences = references, - JoinCondition = new BooleanComparisonExpression - { - FirstExpression = lhsCol, - ComparisonType = BooleanComparisonType.Equals, - SecondExpression = innerQuery.ColumnSet[0].SourceColumn.ToColumnReference() - }, - SemiJoin = true, - OutputRightSchema = false, - DefinedValues = { [definedValue] = innerQuery.ColumnSet[0].SourceColumn } + LeftAttribute = lhsCol.Clone(), + RightSource = alias, + RightAttribute = new ColumnReferenceExpression { MultiPartIdentifier = new MultiPartIdentifier { Identifiers = { new Identifier { Value = alias.Alias }, new Identifier { Value = alias.ColumnSet[0].OutputColumn } } } } }; + } + if (!join.SemiJoin) + { + // Convert the join to a semi join to ensure requests for wildcard columns aren't folded to the IN subquery + var definedValue = context.GetExpressionName(); + join.SemiJoin = true; + join.OutputRightSchema = false; + join.DefinedValues[definedValue] = testColumn; testColumn = definedValue; } + } + else + { + // We need to use nested loops for correlated subqueries + // TODO: We could use a hash join where there is a simple correlation, but followed by a distinct node to eliminate duplicates + // We could also move the correlation criteria out of the subquery and into the join condition. We would then make one request to + // get all the related records and spool that in memory to get the relevant results in the nested loop. Need to understand how + // many rows are likely from the outer query to work out if this is going to be more efficient or not. + if (innerQuery.Source is ISingleSourceExecutionPlanNode loopRightSourceSimple) + InsertCorrelatedSubquerySpool(loopRightSourceSimple, source, hints, context, references.Values.ToArray()); - join.JoinType = QualifiedJoinType.LeftOuter; + var definedValue = context.GetExpressionName(); - rewrites[inSubquery] = new BooleanIsNullExpression + join = new NestedLoopNode { - IsNot = !inSubquery.NotDefined, - Expression = testColumn.ToColumnReference() + LeftSource = source, + RightSource = innerQuery.Source, + OuterReferences = references, + JoinCondition = new BooleanComparisonExpression + { + FirstExpression = lhsCol, + ComparisonType = BooleanComparisonType.Equals, + SecondExpression = innerQuery.ColumnSet[0].SourceColumn.ToColumnReference() + }, + SemiJoin = true, + OutputRightSchema = false, + DefinedValues = { [definedValue] = innerQuery.ColumnSet[0].SourceColumn } }; - source = join; + testColumn = definedValue; } + join.JoinType = QualifiedJoinType.LeftOuter; + + var rewrites = new Dictionary(); + rewrites[inSubquery] = new BooleanIsNullExpression + { + IsNot = !inSubquery.NotDefined, + Expression = testColumn.ToColumnReference() + }; query.Accept(new BooleanRewriteVisitor(rewrites)); - return source; + return join; } private IDataExecutionPlanNodeInternal ConvertExistsSubqueries(IDataExecutionPlanNodeInternal source, IList hints, TSqlFragment query, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences) @@ -2776,138 +3030,140 @@ private IDataExecutionPlanNodeInternal ConvertExistsSubqueries(IDataExecutionPla if (visitor.ExistsSubqueries.Count == 0) return source; - var rewrites = new Dictionary(); - var schema = source.GetSchema(context); - foreach (var existsSubquery in visitor.ExistsSubqueries) - { - // Each query of the format "EXISTS (SELECT * FROM source)" becomes a outer semi join - var parameters = context.ParameterTypes == null ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary(context.ParameterTypes, StringComparer.OrdinalIgnoreCase); - var innerContext = new NodeCompilationContext(context, parameters); - var references = new Dictionary(); - var innerQuery = ConvertSelectStatement(existsSubquery.Subquery.QueryExpression, hints, schema, references, innerContext); - var innerSchema = innerQuery.Source.GetSchema(new NodeCompilationContext(DataSources, Options, parameters, Log)); - var innerSchemaPrimaryKey = innerSchema.PrimaryKey; - - // Create the join - BaseJoinNode join; - string testColumn; - if (references.Count == 0) - { - // We only need one record to check for EXISTS - if (!(innerQuery.Source is TopNode) && !(innerQuery.Source is OffsetFetchNode)) - { - innerQuery.Source = new TopNode - { - Source = innerQuery.Source, - Top = new IntegerLiteral { Value = "1" } - }; - } + source = ConvertExistsSubquery(source, hints, query, existsSubquery, context, outerSchema, outerReferences); - // We need a non-null value to use - if (innerSchemaPrimaryKey == null) - { - innerSchemaPrimaryKey = context.GetExpressionName(); - - if (!(innerQuery.Source is ComputeScalarNode computeScalar)) - { - computeScalar = new ComputeScalarNode { Source = innerQuery.Source }; - innerQuery.Source = computeScalar; - } + return source; + } - computeScalar.Columns[innerSchemaPrimaryKey] = new IntegerLiteral { Value = "1" }; - } + private IDataExecutionPlanNodeInternal ConvertExistsSubquery(IDataExecutionPlanNodeInternal source, IList hints, TSqlFragment query, ExistsPredicate existsSubquery, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences) + { + var schema = source.GetSchema(context); - // We can spool the results for reuse each time - innerQuery.Source = new TableSpoolNode + // Each query of the format "EXISTS (SELECT * FROM source)" becomes a outer semi join + var parameters = context.ParameterTypes == null ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary(context.ParameterTypes, StringComparer.OrdinalIgnoreCase); + var innerContext = new NodeCompilationContext(context, parameters); + var references = new Dictionary(); + var innerQuery = ConvertSelectStatement(existsSubquery.Subquery.QueryExpression, hints, schema, references, innerContext); + var innerSchema = innerQuery.Source.GetSchema(new NodeCompilationContext(DataSources, Options, parameters, Log)); + var innerSchemaPrimaryKey = innerSchema.PrimaryKey; + + // Create the join + BaseJoinNode join; + string testColumn; + if (references.Count == 0) + { + // We only need one record to check for EXISTS + if (!(innerQuery.Source is TopNode) && !(innerQuery.Source is OffsetFetchNode)) + { + innerQuery.Source = new TopNode { Source = innerQuery.Source, - SpoolType = SpoolType.Lazy + Top = new IntegerLiteral { Value = "1" } }; + } - testColumn = context.GetExpressionName(); + // We need a non-null value to use + if (innerSchemaPrimaryKey == null) + { + innerSchemaPrimaryKey = context.GetExpressionName(); - join = new NestedLoopNode + if (!(innerQuery.Source is ComputeScalarNode computeScalar)) { - LeftSource = source, - RightSource = innerQuery.Source, - JoinType = QualifiedJoinType.LeftOuter, - SemiJoin = true, - OutputRightSchema = false, - OuterReferences = references, - DefinedValues = - { - [testColumn] = innerSchemaPrimaryKey - } - }; - } - else if (UseMergeJoin(source, innerQuery.Source, context, references, null, null, true, false, out testColumn, out var merge)) - { - join = merge; + computeScalar = new ComputeScalarNode { Source = innerQuery.Source }; + innerQuery.Source = computeScalar; + } + + computeScalar.Columns[innerSchemaPrimaryKey] = new IntegerLiteral { Value = "1" }; } - else + + // We can spool the results for reuse each time + innerQuery.Source = new TableSpoolNode { - // We need to use nested loops for correlated subqueries - // TODO: We could use a hash join where there is a simple correlation, but followed by a distinct node to eliminate duplicates - // We could also move the correlation criteria out of the subquery and into the join condition. We would then make one request to - // get all the related records and spool that in memory to get the relevant results in the nested loop. Need to understand how - // many rows are likely from the outer query to work out if this is going to be more efficient or not. - if (innerQuery.Source is ISingleSourceExecutionPlanNode loopRightSourceSimple) - InsertCorrelatedSubquerySpool(loopRightSourceSimple, source, hints, context, references.Values.ToArray()); + Source = innerQuery.Source, + SpoolType = SpoolType.Lazy + }; + + testColumn = context.GetExpressionName(); - // We only need one record to check for EXISTS - if (!(innerQuery.Source is TopNode) && !(innerQuery.Source is OffsetFetchNode)) + join = new NestedLoopNode + { + LeftSource = source, + RightSource = innerQuery.Source, + JoinType = QualifiedJoinType.LeftOuter, + SemiJoin = true, + OutputRightSchema = false, + OuterReferences = references, + DefinedValues = { - innerQuery.Source = new TopNode - { - Source = innerQuery.Source, - Top = new IntegerLiteral { Value = "1" } - }; + [testColumn] = innerSchemaPrimaryKey } + }; + } + else if (UseMergeJoin(source, innerQuery.Source, context, references, null, null, true, false, out testColumn, out var merge)) + { + join = merge; + } + else + { + // We need to use nested loops for correlated subqueries + // TODO: We could use a hash join where there is a simple correlation, but followed by a distinct node to eliminate duplicates + // We could also move the correlation criteria out of the subquery and into the join condition. We would then make one request to + // get all the related records and spool that in memory to get the relevant results in the nested loop. Need to understand how + // many rows are likely from the outer query to work out if this is going to be more efficient or not. + if (innerQuery.Source is ISingleSourceExecutionPlanNode loopRightSourceSimple) + InsertCorrelatedSubquerySpool(loopRightSourceSimple, source, hints, context, references.Values.ToArray()); - // We need a non-null value to use - if (innerSchemaPrimaryKey == null) + // We only need one record to check for EXISTS + if (!(innerQuery.Source is TopNode) && !(innerQuery.Source is OffsetFetchNode)) + { + innerQuery.Source = new TopNode { - innerSchemaPrimaryKey = context.GetExpressionName(); - - if (!(innerQuery.Source is ComputeScalarNode computeScalar)) - { - computeScalar = new ComputeScalarNode { Source = innerQuery.Source }; - innerQuery.Source = computeScalar; - } - - computeScalar.Columns[innerSchemaPrimaryKey] = new IntegerLiteral { Value = "1" }; - } + Source = innerQuery.Source, + Top = new IntegerLiteral { Value = "1" } + }; + } - var definedValue = context.GetExpressionName(); + // We need a non-null value to use + if (innerSchemaPrimaryKey == null) + { + innerSchemaPrimaryKey = context.GetExpressionName(); - join = new NestedLoopNode + if (!(innerQuery.Source is ComputeScalarNode computeScalar)) { - LeftSource = source, - RightSource = innerQuery.Source, - OuterReferences = references, - SemiJoin = true, - OutputRightSchema = false, - DefinedValues = { [definedValue] = innerSchemaPrimaryKey } - }; + computeScalar = new ComputeScalarNode { Source = innerQuery.Source }; + innerQuery.Source = computeScalar; + } - testColumn = definedValue; + computeScalar.Columns[innerSchemaPrimaryKey] = new IntegerLiteral { Value = "1" }; } - join.JoinType = QualifiedJoinType.LeftOuter; + var definedValue = context.GetExpressionName(); - rewrites[existsSubquery] = new BooleanIsNullExpression + join = new NestedLoopNode { - IsNot = true, - Expression = testColumn.ToColumnReference() + LeftSource = source, + RightSource = innerQuery.Source, + OuterReferences = references, + SemiJoin = true, + OutputRightSchema = false, + DefinedValues = { [definedValue] = innerSchemaPrimaryKey } }; - source = join; + testColumn = definedValue; } + join.JoinType = QualifiedJoinType.LeftOuter; + + var rewrites = new Dictionary(); + rewrites[existsSubquery] = new BooleanIsNullExpression + { + IsNot = true, + Expression = testColumn.ToColumnReference() + }; query.Accept(new BooleanRewriteVisitor(rewrites)); - return source; + return join; } private IDataExecutionPlanNodeInternal ConvertHavingClause(IDataExecutionPlanNodeInternal source, IList hints, HavingClause havingClause, NodeCompilationContext context, INodeSchema outerSchema, IDictionary outerReferences, TSqlFragment query, INodeSchema nonAggregateSchema) @@ -3774,7 +4030,7 @@ private ColumnReferenceExpression ConvertScalarSubqueries(TSqlFragment expressio query.Accept(new RewriteVisitor(rewrites)); if (expression is ScalarExpression scalar && rewrites.ContainsKey(scalar)) - return new ColumnReferenceExpression { MultiPartIdentifier = new MultiPartIdentifier { Identifiers = { new Identifier { Value = rewrites[scalar] } } } }; + return rewrites[scalar].ToColumnReference(); return null; } @@ -3881,7 +4137,7 @@ private bool UseMergeJoin(IDataExecutionPlanNodeInternal node, IDataExecutionPla else if (semiJoin && alias == null) { var select = new SelectNode { Source = subNode }; - select.ColumnSet.Add(new SelectColumn { SourceColumn = subqueryCol, OutputColumn = subqueryCol.SplitMultiPartIdentifier().Last() }); + select.ColumnSet.Add(new SelectColumn { SourceColumn = innerKey, OutputColumn = innerKey.SplitMultiPartIdentifier().Last() }); alias = new AliasNode(select, new Identifier { Value = context.GetExpressionName() }, context); subAlias = alias.Alias; } @@ -4192,7 +4448,8 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe if (!String.IsNullOrEmpty(table.SchemaObject.SchemaIdentifier?.Value) && !table.SchemaObject.SchemaIdentifier.Value.Equals("dbo", StringComparison.OrdinalIgnoreCase) && - !table.SchemaObject.SchemaIdentifier.Value.Equals("archive", StringComparison.OrdinalIgnoreCase)) + !table.SchemaObject.SchemaIdentifier.Value.Equals("archive", StringComparison.OrdinalIgnoreCase) && + !(table.SchemaObject.SchemaIdentifier.Value.Equals("bin", StringComparison.OrdinalIgnoreCase) && dataSource.Metadata.RecycleBinEntities != null)) throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidObjectName(table.SchemaObject)); // Validate the entity name @@ -4244,6 +4501,14 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe fetchXmlScan.FetchXml.DataSource = "retained"; } + // Check if this should be using the recycle bin table + else if (table.SchemaObject.SchemaIdentifier?.Value.Equals("bin", StringComparison.OrdinalIgnoreCase) == true) + { + if (!dataSource.Metadata.RecycleBinEntities.Contains(meta.LogicalName)) + throw new NotSupportedQueryFragmentException(Sql4CdsError.InvalidObjectName(table.SchemaObject)) { Suggestion = "Ensure restoring of deleted records is enabled for this table - see https://learn.microsoft.com/en-us/power-platform/admin/restore-deleted-table-records?WT.mc_id=DX-MVP-5004203" }; + + fetchXmlScan.FetchXml.DataSource = "bin"; + } return fetchXmlScan; } @@ -4375,9 +4640,147 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe }; } - // Validate the join condition - var joinSchema = joinNode.GetSchema(context); - join.SearchCondition.GetType(GetExpressionContext(joinSchema, context), out _); + BooleanExpression additionalCriteria = (joinNode as FoldableJoinNode)?.AdditionalJoinCriteria ?? (joinNode as NestedLoopNode)?.JoinCondition; + + if (additionalCriteria != null) + { + var where = new WhereClause { SearchCondition = additionalCriteria }; + + if (join.QualifiedJoinType == QualifiedJoinType.Inner) + { + // Move any additional criteria to a filter node on the join results + var result = (IDataExecutionPlanNodeInternal)joinNode; + result = ConvertInSubqueries(result, hints, where, context, outerSchema, outerReferences); + result = ConvertExistsSubqueries(result, hints, where, context, outerSchema, outerReferences); + + if (where.SearchCondition != null) + { + var joinSchema = result.GetSchema(context); + where.SearchCondition.GetType(GetExpressionContext(joinSchema, context), out _); + + result = new FilterNode + { + Source = result, + Filter = where.SearchCondition + }; + } + + if (joinNode is FoldableJoinNode foldable) + foldable.AdditionalJoinCriteria = null; + else if (joinNode is NestedLoopNode nestedLoop) + nestedLoop.JoinCondition = null; + + return result; + } + else + { + // Convert any subqueries in the join criteria. There may be multiple subqueries, and each one could reference data from + // just the LHS, just the RHS, or both. + // Subqueries that only reference the LHS data should be added to that path. Any others should be added to the RHS path, + // including any required data from the LHS via an outer reference. + var inSubqueries = new InSubqueryVisitor(); + where.Accept(inSubqueries); + + var inSubqueryConversions = inSubqueries.InSubqueries + .Select(subquery => + { + var cols = subquery.GetColumns(); + var hasLhsCols = cols.Any(c => lhsSchema.ContainsColumn(c, out _)); + var hasRhsCols = cols.Any(c => rhsSchema.ContainsColumn(c, out _)); + + return new + { + Subquery = subquery, + HasLHSCols = hasLhsCols, + HasRHSCols = hasRhsCols + }; + }) + .ToList(); + + var existsSubqueries = new ExistsSubqueryVisitor(); + where.Accept(existsSubqueries); + + var existsSubqueryConversions = existsSubqueries.ExistsSubqueries + .Select(subquery => + { + var cols = subquery.GetColumns(); + var hasLhsCols = cols.Any(c => lhsSchema.ContainsColumn(c, out _)); + var hasRhsCols = cols.Any(c => rhsSchema.ContainsColumn(c, out _)); + + return new + { + Subquery = subquery, + HasLHSCols = hasLhsCols, + HasRHSCols = hasRhsCols + }; + }) + .ToList(); + + if (joinNode is FoldableJoinNode foldable && (inSubqueryConversions.Any(s => s.HasLHSCols && s.HasRHSCols) || existsSubqueryConversions.Any(s => s.HasLHSCols && s.HasRHSCols))) + { + // We're currently using a hash- or merge-join, but we need to apply a subquery that requires data from both + // sides of the join. Replace the join with a nested loop so we can apply the required outer references + joinNode = new NestedLoopNode + { + LeftSource = foldable.LeftSource, + RightSource = new TableSpoolNode { Source = foldable.RightSource }, + JoinType = foldable.JoinType + }; + + // Need to add the original join condition back in + where.SearchCondition = new BooleanBinaryExpression + { + FirstExpression = new BooleanComparisonExpression + { + FirstExpression = foldable.LeftAttribute, + ComparisonType = BooleanComparisonType.Equals, + SecondExpression = foldable.RightAttribute + }, + BinaryExpressionType = BooleanBinaryExpressionType.And, + SecondExpression = where.SearchCondition + }; + } + + var nestedLoopJoinNode = joinNode as NestedLoopNode; + + if (nestedLoopJoinNode != null && nestedLoopJoinNode.OuterReferences == null) + nestedLoopJoinNode.OuterReferences = new Dictionary(); + + foreach (var subquery in inSubqueryConversions) + { + if (subquery.HasLHSCols && subquery.HasRHSCols) + CaptureOuterReferences(lhsSchema, joinNode.RightSource, subquery.Subquery, context, nestedLoopJoinNode.OuterReferences); + + if (!subquery.HasRHSCols) + joinNode.LeftSource = ConvertInSubquery(joinNode.LeftSource, hints, where, subquery.Subquery, context, outerSchema, outerReferences); + else + joinNode.RightSource = ConvertInSubquery(joinNode.RightSource, hints, where, subquery.Subquery, context, outerSchema, outerReferences); + } + + foreach (var subquery in existsSubqueryConversions) + { + if (subquery.HasLHSCols && subquery.HasRHSCols) + CaptureOuterReferences(lhsSchema, joinNode.RightSource, subquery.Subquery, context, nestedLoopJoinNode.OuterReferences); + + if (!subquery.HasRHSCols) + joinNode.LeftSource = ConvertExistsSubquery(joinNode.LeftSource, hints, where, subquery.Subquery, context, outerSchema, outerReferences); + else + joinNode.RightSource = ConvertExistsSubquery(joinNode.RightSource, hints, where, subquery.Subquery, context, outerSchema, outerReferences); + } + + // Validate the remaining join condition + if (where.SearchCondition != null) + { + var joinSchema = joinNode.GetSchema(context); + where.SearchCondition.GetType(GetExpressionContext(joinSchema, context), out _); + } + + if (nestedLoopJoinNode != null) + nestedLoopJoinNode.JoinCondition = where.SearchCondition; + else + ((FoldableJoinNode)joinNode).AdditionalJoinCriteria = where.SearchCondition; + } + } return joinNode; } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlanSerializer.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlanSerializer.cs new file mode 100644 index 00000000..2966b96a --- /dev/null +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlanSerializer.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; +using MarkMpn.Sql4Cds.Engine.ExecutionPlan; +using Newtonsoft.Json; + +namespace MarkMpn.Sql4Cds.Engine +{ + public static class ExecutionPlanSerializer + { + public static string Serialize(IRootExecutionPlanNode plan) + { + var settings = new JsonSerializerSettings + { + TypeNameHandling = TypeNameHandling.Auto + }; + return JsonConvert.SerializeObject(plan, typeof(IRootExecutionPlanNode), settings); + } + + public static IRootExecutionPlanNode Deserialize(string plan) + { + var settings = new JsonSerializerSettings + { + TypeNameHandling = TypeNameHandling.Auto + }; + return JsonConvert.DeserializeObject(plan, settings); + } + } +} diff --git a/MarkMpn.Sql4Cds.Engine/IAttributeMetadataCache.cs b/MarkMpn.Sql4Cds.Engine/IAttributeMetadataCache.cs index 47a68cee..007293cd 100644 --- a/MarkMpn.Sql4Cds.Engine/IAttributeMetadataCache.cs +++ b/MarkMpn.Sql4Cds.Engine/IAttributeMetadataCache.cs @@ -83,5 +83,10 @@ public interface IAttributeMetadataCache /// /// bool TryGetMinimalData(string logicalName, out EntityMetadata metadata); + + /// + /// Returns a list of entity logical names that are enabled for recycle bin access + /// + string[] RecycleBinEntities { get; } } } diff --git a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.nuspec b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.nuspec index a35c3f19..df6bd2c9 100644 --- a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.nuspec +++ b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.nuspec @@ -11,17 +11,21 @@ https://markcarrington.dev/sql4cds-icon/ Convert SQL queries to FetchXml and execute them against Dataverse / D365 Convert SQL queries to FetchXml and execute them against Dataverse / D365 - Fixed NullReferenceException errors when: -- executing a conditional SELECT query -- retrieving results from a Fetch XML query using IN or EXISTS -- handling an error returned from TDS Endpoint -- handling internal errors such as UPDATE without WHERE - -Standardised errors on: -- JSON path errors -- DML statement cancellation - -Fixed filtering audit data on changedata attribute + Enabled access to recycle bin records via the `bin` schema +Enabled `INSERT`, `UPDATE` and `DELETE` statements on `principalobjectaccess` table +Enabled use of subqueries within `ON` clause of `JOIN` statements +Added support for `___pid` virtual column for lookups to elastic tables +Improved folding of queries using index spools +Improved primary key calculation when using joins on non-key columns +Apply column order setting to parameters for stored procedures and table-valued functions +Fixed error with DeleteMultiple requests +Fixed paging error with `DISTINCT` queries causing results to be limited to 50,000 records +Fixed paging errors when sorting by optionset values causing some results to be skipped +Fixed errors when using joins inside `[NOT] EXISTS` subqueries +Fixed incorrect results when applying aliases to `___name` and `___type` virtual columns +Fixed max length calculation for string columns +Added debug visualizer to inspect query plans within Visual Studio +Fixed "invalid program" errors when combining type conversions with `AND` or `OR` in .NET Core applications Copyright © 2020 Mark Carrington en-GB diff --git a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems index 78cdd812..a0b5cfce 100644 --- a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems +++ b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems @@ -28,6 +28,7 @@ + diff --git a/MarkMpn.Sql4Cds.Engine/MetaMetadataCache.cs b/MarkMpn.Sql4Cds.Engine/MetaMetadataCache.cs index dbbd6ef3..5b7ad405 100644 --- a/MarkMpn.Sql4Cds.Engine/MetaMetadataCache.cs +++ b/MarkMpn.Sql4Cds.Engine/MetaMetadataCache.cs @@ -301,5 +301,8 @@ public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata) return _inner.TryGetMinimalData(logicalName, out metadata); } + + /// + public string[] RecycleBinEntities => _inner.RecycleBinEntities; } } diff --git a/MarkMpn.Sql4Cds.Engine/MetadataExtensions.cs b/MarkMpn.Sql4Cds.Engine/MetadataExtensions.cs index a6036bb2..f0c4e262 100644 --- a/MarkMpn.Sql4Cds.Engine/MetadataExtensions.cs +++ b/MarkMpn.Sql4Cds.Engine/MetadataExtensions.cs @@ -15,6 +15,20 @@ static class MetadataExtensions { public static int EntityLogicalNameMaxLength { get; } = 64; + public static string[] VirtualLookupAttributeSuffixes { get; } = new[] { "name", "type", "pid" }; + + public static AttributeMetadata FindBaseAttributeFromVirtualAttribute(this EntityMetadata entity, string virtualAttributeLogicalName, out string suffix) + { + var matchingSuffix = VirtualLookupAttributeSuffixes.SingleOrDefault(s => virtualAttributeLogicalName.EndsWith(s, StringComparison.OrdinalIgnoreCase)); + suffix = matchingSuffix; + + if (suffix == null) + return null; + + return entity.Attributes + .SingleOrDefault(a => a.LogicalName.Equals(virtualAttributeLogicalName.Substring(0, virtualAttributeLogicalName.Length - matchingSuffix.Length), StringComparison.OrdinalIgnoreCase)); + } + public static Type GetAttributeType(this AttributeMetadata attrMetadata) { if (attrMetadata is MultiSelectPicklistAttributeMetadata) @@ -163,11 +177,11 @@ public static DataTypeReference GetAttributeSqlType(this AttributeMetadata attrM if (attrMetadata is StringAttributeMetadata str) { - // MaxLength validation is applied on write, but existing values could be up to DatabaseLength - var maxLengthSetting = write || str.DatabaseLength == null || str.DatabaseLength == 0 ? str.MaxLength : str.DatabaseLength; + // MaxLength validation is applied on write, but existing values could be up to DatabaseLength / 2 + maxLength = str.MaxLength ?? maxLength; - if (maxLengthSetting != null) - maxLength = maxLengthSetting.Value; + if (!write && str.DatabaseLength != null && str.DatabaseLength.Value / 2 > maxLength) + maxLength = str.DatabaseLength.Value / 2; } return DataTypeHelpers.NVarChar(maxLength, dataSource.DefaultCollation, CollationLabel.Implicit); diff --git a/MarkMpn.Sql4Cds.Engine/NodeContext.cs b/MarkMpn.Sql4Cds.Engine/NodeContext.cs index 22034f62..68b6c98d 100644 --- a/MarkMpn.Sql4Cds.Engine/NodeContext.cs +++ b/MarkMpn.Sql4Cds.Engine/NodeContext.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data.SqlTypes; using System.Text; using MarkMpn.Sql4Cds.Engine.ExecutionPlan; using Microsoft.SqlServer.TransactSql.ScriptDom; @@ -105,7 +106,7 @@ public NodeExecutionContext( IDictionary dataSources, IQueryExecutionOptions options, IDictionary parameterTypes, - IDictionary parameterValues, + IDictionary parameterValues, Action log) : base(dataSources, options, parameterTypes, log) { @@ -115,7 +116,7 @@ public NodeExecutionContext( /// /// Returns the current value of each parameter /// - public IDictionary ParameterValues { get; } + public IDictionary ParameterValues { get; } public Sql4CdsError Error { get; set; } } @@ -191,7 +192,7 @@ public ExpressionExecutionContext( IDictionary dataSources, IQueryExecutionOptions options, IDictionary parameterTypes, - IDictionary parameterValues, + IDictionary parameterValues, Action log, Entity entity) : base(dataSources, options, parameterTypes, parameterValues, log) diff --git a/MarkMpn.Sql4Cds.Engine/Visitors/ExistsSubqueryVisitor.cs b/MarkMpn.Sql4Cds.Engine/Visitors/ExistsSubqueryVisitor.cs index 69db392c..602e6ab6 100644 --- a/MarkMpn.Sql4Cds.Engine/Visitors/ExistsSubqueryVisitor.cs +++ b/MarkMpn.Sql4Cds.Engine/Visitors/ExistsSubqueryVisitor.cs @@ -21,5 +21,10 @@ public override void ExplicitVisit(ScalarSubquery node) { // Do not recurse into subqueries } + + public override void ExplicitVisit(FromClause node) + { + // Do not recurse into data sources + } } } diff --git a/MarkMpn.Sql4Cds.Engine/Visitors/InSubqueryVisitor.cs b/MarkMpn.Sql4Cds.Engine/Visitors/InSubqueryVisitor.cs index 3dfd2970..3a620a97 100644 --- a/MarkMpn.Sql4Cds.Engine/Visitors/InSubqueryVisitor.cs +++ b/MarkMpn.Sql4Cds.Engine/Visitors/InSubqueryVisitor.cs @@ -21,5 +21,10 @@ public override void ExplicitVisit(ScalarSubquery node) { // Do not recurse into subqueries } + + public override void ExplicitVisit(FromClause node) + { + // Do not recurse into data sources + } } } diff --git a/MarkMpn.Sql4Cds.Engine/Visitors/JoinConditionVisitor.cs b/MarkMpn.Sql4Cds.Engine/Visitors/JoinConditionVisitor.cs index a70870fd..7a904308 100644 --- a/MarkMpn.Sql4Cds.Engine/Visitors/JoinConditionVisitor.cs +++ b/MarkMpn.Sql4Cds.Engine/Visitors/JoinConditionVisitor.cs @@ -108,6 +108,7 @@ node.SecondExpression is ColumnReferenceExpression rhsCol && LhsKey = LhsExpression as ColumnReferenceExpression; RhsKey = RhsExpression as ColumnReferenceExpression; + JoinCondition = node; } } diff --git a/MarkMpn.Sql4Cds.Engine/Visitors/UpdateTargetVisitor.cs b/MarkMpn.Sql4Cds.Engine/Visitors/UpdateTargetVisitor.cs index 750389b5..3f839db8 100644 --- a/MarkMpn.Sql4Cds.Engine/Visitors/UpdateTargetVisitor.cs +++ b/MarkMpn.Sql4Cds.Engine/Visitors/UpdateTargetVisitor.cs @@ -24,6 +24,8 @@ public UpdateTargetVisitor(SchemaObjectName search, string primaryDataSource) public string TargetDataSource { get; private set; } + public string TargetSchema { get; private set; } + public string TargetEntityName { get; private set; } public string TargetAliasName { get; private set; } @@ -41,6 +43,7 @@ public override void ExplicitVisit(NamedTableReference node) _ambiguous = _foundAlias; TargetDataSource = node.SchemaObject.DatabaseIdentifier?.Value ?? PrimaryDataSource; + TargetSchema = node.SchemaObject.SchemaIdentifier?.Value; TargetEntityName = node.SchemaObject.BaseIdentifier.Value; TargetAliasName = node.Alias.Value; Target = node; @@ -55,6 +58,7 @@ public override void ExplicitVisit(NamedTableReference node) _ambiguous = true; TargetDataSource = node.SchemaObject.DatabaseIdentifier?.Value ?? PrimaryDataSource; + TargetSchema = node.SchemaObject.SchemaIdentifier?.Value; TargetEntityName = node.SchemaObject.BaseIdentifier.Value; TargetAliasName = node.Alias?.Value ?? _search.BaseIdentifier.Value; Target = node; diff --git a/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/Autocomplete.cs b/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/Autocomplete.cs index bc34f655..5781055a 100644 --- a/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/Autocomplete.cs +++ b/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/Autocomplete.cs @@ -12,6 +12,7 @@ using Microsoft.VisualStudio.LanguageServer.Protocol; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; using static MarkMpn.Sql4Cds.LanguageServer.Autocomplete.FunctionMetadata; namespace MarkMpn.Sql4Cds.LanguageServer.Autocomplete @@ -23,16 +24,19 @@ public class Autocomplete { private readonly IDictionary _dataSources; private readonly string _primaryDataSource; + private readonly ColumnOrdering _columnOrdering; /// /// Creates a new /// /// The list of entities available to use in the query /// The cache of metadata about each entity - public Autocomplete(IDictionary dataSources, string primaryDataSource) + /// The order that columns are passed to table-valued functions or stored procedures + public Autocomplete(IDictionary dataSources, string primaryDataSource, ColumnOrdering columnOrdering) { _dataSources = dataSources; _primaryDataSource = primaryDataSource; + _columnOrdering = columnOrdering; } /// @@ -433,7 +437,7 @@ public IEnumerable GetSuggestions(string text, int pos) if (tables.TryGetValue(targetTable, out var tableName)) { if (TryParseTableName(tableName, out var instanceName, out _, out tableName) && _dataSources.TryGetValue(instanceName, out var instance) && instance.Metadata.TryGetMinimalData(tableName, out var metadata)) - return FilterList(metadata.Attributes.Where(a => a.IsValidForUpdate != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(a => a.IsValidForUpdate != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true, instance.Metadata)).OrderBy(a => a), currentWord); } } @@ -456,14 +460,14 @@ public IEnumerable GetSuggestions(string text, int pos) if (instance.Messages.TryGetValue(messageName, out var message) && message.IsValidAsTableValuedFunction()) { - return FilterList(GetMessageOutputAttributes(message, instance).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false)).OrderBy(a => a), currentWord); + return FilterList(GetMessageOutputAttributes(message, instance).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false, instance.Metadata)).OrderBy(a => a), currentWord); } } else { // Table if (instance.Metadata.TryGetMinimalData((schemaName == "metadata" ? "metadata." : "") + tableName, out var metadata)) - return FilterList(metadata.Attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false, instance.Metadata)).OrderBy(a => a), currentWord); } } } @@ -489,11 +493,15 @@ public IEnumerable GetSuggestions(string text, int pos) var relationship = metadata.ManyToManyRelationships.Single(); attributeFilter = a => a.LogicalName == relationship.Entity1IntersectAttribute || a.LogicalName == relationship.Entity2IntersectAttribute; } + else if (metadata.LogicalName == "principalobjectaccess") + { + attributeFilter = a => a.LogicalName == "objectid" || a.LogicalName == "objecttypecode" || a.LogicalName == "principalid" || a.LogicalName == "principaltypecode" || a.LogicalName == "accessrightsmask"; + } else { attributeFilter = a => a.IsValidForCreate != false && a.AttributeOf == null; } - return FilterList(metadata.Attributes.Where(attributeFilter).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(attributeFilter).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true, instance.Metadata)).OrderBy(a => a), currentWord); } } else @@ -505,6 +513,7 @@ public IEnumerable GetSuggestions(string text, int pos) // * variables var items = new List(); var attributes = new List(); + var instance = default(AutocompleteDataSource); foreach (var table in tables) { @@ -514,7 +523,7 @@ public IEnumerable GetSuggestions(string text, int pos) var messageName = table.Value.Substring(0, table.Value.Length - 1); if (TryParseTableName(messageName, out var instanceName, out var schemaName, out var tableName) && - _dataSources.TryGetValue(instanceName, out var instance) && + _dataSources.TryGetValue(instanceName, out instance) && (string.IsNullOrEmpty(schemaName) || schemaName == "dbo") && instance.Messages.TryGetValue(messageName, out var message) && message.IsValidAsTableValuedFunction()) @@ -528,7 +537,7 @@ public IEnumerable GetSuggestions(string text, int pos) else { // Table - if (TryParseTableName(table.Value, out var instanceName, out var schemaName, out var tableName) && _dataSources.TryGetValue(instanceName, out var instance)) + if (TryParseTableName(table.Value, out var instanceName, out var schemaName, out var tableName) && _dataSources.TryGetValue(instanceName, out instance)) { var entity = instance.Entities.SingleOrDefault(e => e.LogicalName == tableName && @@ -547,6 +556,12 @@ public IEnumerable GetSuggestions(string text, int pos) schemaName == "archive" && (e.IsArchivalEnabled == true || e.IsRetentionEnabled == true) ) + || + ( + schemaName == "bin" && + instance.Metadata.RecycleBinEntities != null && + instance.Metadata.RecycleBinEntities.Contains(e.LogicalName) + ) ) ); @@ -559,7 +574,7 @@ public IEnumerable GetSuggestions(string text, int pos) } } - items.AddRange(attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).GroupBy(x => x.LogicalName).Where(g => g.Count() == 1).SelectMany(g => AttributeAutocompleteItem.CreateList(g.Single(), currentLength, false))); + items.AddRange(attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).GroupBy(x => x.LogicalName).Where(g => g.Count() == 1).SelectMany(g => AttributeAutocompleteItem.CreateList(g.Single(), currentLength, false, instance.Metadata))); items.AddRange(typeof(SqlFunctions).GetMethods(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public).Select(m => new FunctionAutocompleteItem(m, currentLength))); @@ -569,7 +584,6 @@ public IEnumerable GetSuggestions(string text, int pos) { // Check if there are any applicable filter operator functions that match the type of the current attribute var identifiers = prevPrevWord.Split('.'); - var instance = default(AutocompleteDataSource); var entity = default(EntityMetadata); var attribute = default(AttributeMetadata); @@ -761,11 +775,12 @@ private IEnumerable AutocompleteTableName(string currentWor // Show TVF list if (fromClause && ds.Messages != null) - list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsTableValuedFunction()).Select(x => new TVFAutocompleteItem(x, currentLength))); + list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsTableValuedFunction()).Select(x => new TVFAutocompleteItem(x, _columnOrdering, currentLength))); } } else if (TryParseTableName(currentWord, out var instanceName, out var schemaName, out var tableName, out var parts, out var lastPartLength)) { + _dataSources.TryGetValue(instanceName, out var instance); var lastPart = tableName; if (parts == 1) @@ -777,18 +792,18 @@ private IEnumerable AutocompleteTableName(string currentWor if (parts == 1 || (parts == 2 && _dataSources.ContainsKey(schemaName))) { // Could be a schema name - if ("dbo".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("dbo", lastPartLength)); + var schemaNames = (IEnumerable)new[] { "dbo", "archive", "metadata" }; + if (instance?.Metadata?.RecycleBinEntities != null) + schemaNames = schemaNames.Append("bin"); - if ("archive".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("archive", lastPartLength)); + schemaNames = schemaNames.Where(s => s.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - if ("metadata".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("metadata", lastPartLength)); + foreach (var schema in schemaNames) + list.Add(new SchemaAutocompleteItem(schema, lastPartLength)); } // Could be a table name - if (_dataSources.TryGetValue(instanceName, out var instance) && instance.Entities != null) + if (instance?.Entities != null) { IEnumerable entities; IEnumerable messages = Array.Empty(); @@ -814,6 +829,11 @@ private IEnumerable AutocompleteTableName(string currentWor messages = instance.Messages.GetAllMessages(); } } + else if (schemaName.Equals("bin", StringComparison.OrdinalIgnoreCase)) + { + // Suggest tables that are enabled for the recycle bin + entities = instance.Entities.Where(e => instance.Metadata.RecycleBinEntities != null && instance.Metadata.RecycleBinEntities.Contains(e.LogicalName)); + } else { entities = Array.Empty(); @@ -825,7 +845,7 @@ private IEnumerable AutocompleteTableName(string currentWor if (fromClause) { messages = messages.Where(e => e.IsValidAsTableValuedFunction() && e.Name.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - list.AddRange(messages.Select(e => new TVFAutocompleteItem(e, lastPartLength))); + list.AddRange(messages.Select(e => new TVFAutocompleteItem(e, _columnOrdering, lastPartLength))); } } } @@ -846,10 +866,11 @@ private IEnumerable AutocompleteSprocName(string currentWor list.AddRange(_dataSources.Values.Select(x => new InstanceAutocompleteItem(x, currentLength))); if (_dataSources.TryGetValue(_primaryDataSource, out var ds) && ds.Messages != null) - list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(x => new SprocAutocompleteItem(x, currentLength))); + list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(x => new SprocAutocompleteItem(x, _columnOrdering, currentLength))); } else if (TryParseTableName(currentWord, out var instanceName, out var schemaName, out var tableName, out var parts, out var lastPartLength)) { + _dataSources.TryGetValue(instanceName, out var instance); var lastPart = tableName; if (parts == 1) @@ -861,19 +882,19 @@ private IEnumerable AutocompleteSprocName(string currentWor if (parts == 1 || parts == 2) { // Could be a schema name - if ("dbo".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("dbo", lastPartLength)); + var schemaNames = (IEnumerable)new[] { "dbo", "archive", "metadata" }; + if (instance?.Metadata?.RecycleBinEntities != null) + schemaNames = schemaNames.Append("bin"); - if ("archive".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("archive", lastPartLength)); + schemaNames = schemaNames.Where(s => s.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - if ("metadata".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("metadata", lastPartLength)); + foreach (var schema in schemaNames) + list.Add(new SchemaAutocompleteItem(schema, lastPartLength)); } // Could be a sproc name - if (schemaName.Equals("dbo", StringComparison.OrdinalIgnoreCase) && _dataSources.TryGetValue(instanceName, out var instance) && instance.Messages != null) - list.AddRange(instance.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(e => new SprocAutocompleteItem(e, lastPartLength))); + if (schemaName.Equals("dbo", StringComparison.OrdinalIgnoreCase) && instance?.Messages != null) + list.AddRange(instance.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(e => new SprocAutocompleteItem(e, _columnOrdering, lastPartLength))); } list.Sort(); @@ -1232,7 +1253,7 @@ public int CompareTo(object obj) public static string GetSqlTypeName(Type type) { if (type == typeof(string)) - return "NNVARCHAR(MAX)"; + return "NVARCHAR(MAX)"; if (type == typeof(int)) return "INT"; @@ -1494,7 +1515,10 @@ public override string ToolTipTitle public override string ToolTipText { - get => Text == "metadata" ? "Schema containing the metadata information" : "Schema containing the data tables"; + get => Text == "metadata" ? "Schema containing the metadata information" : + Text == "archive" ? "Schema containing long-term retention tables" : + Text == "bin" ? "Schema containing recycle bin tables" : + "Schema containing the data tables"; set => base.ToolTipText = value; } } @@ -1536,10 +1560,12 @@ public override string GetTextForReplace() class TVFAutocompleteItem : SqlAutocompleteItem { private readonly Message _message; + private readonly ColumnOrdering _columnOrdering; - public TVFAutocompleteItem(Message message, int replaceLength) : base(message.Name, replaceLength, CompletionItemKind.Function) + public TVFAutocompleteItem(Message message, ColumnOrdering columnOrdering, int replaceLength) : base(message.Name, replaceLength, CompletionItemKind.Function) { _message = message; + _columnOrdering = columnOrdering; } public TVFAutocompleteItem(Message message, string alias, int replaceLength) : base(alias, replaceLength, CompletionItemKind.Function) @@ -1555,7 +1581,18 @@ public override string ToolTipTitle public override string ToolTipText { - get => _message.Name + "(" + string.Join(", ", _message.InputParameters.Select(p => p.Name + " " + GetSqlTypeName(p.Type))) + ")"; + get + { + var parameters = _message.InputParameters + .Where(p => p.Type != typeof(PagingInfo)); + + if (_columnOrdering == ColumnOrdering.Alphabetical) + parameters = parameters.OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase); + else + parameters = parameters.OrderBy(p => p.Position); + + return _message.Name + "(" + String.Join(", ", parameters.Select(p => p.Name + " " + GetSqlTypeName(p.Type))) + ")"; + } set => base.ToolTipText = value; } @@ -1568,10 +1605,12 @@ public override string GetTextForReplace() class SprocAutocompleteItem : SqlAutocompleteItem { private readonly Message _message; + private readonly ColumnOrdering _columnOrdering; - public SprocAutocompleteItem(Message message, int replaceLength) : base(message.Name, replaceLength, CompletionItemKind.Method) + public SprocAutocompleteItem(Message message, ColumnOrdering columnOrdering, int replaceLength) : base(message.Name, replaceLength, CompletionItemKind.Method) { _message = message; + _columnOrdering = columnOrdering; } public override string ToolTipTitle @@ -1582,7 +1621,18 @@ public override string ToolTipTitle public override string ToolTipText { - get => _message.Name + " " + string.Join(", ", _message.InputParameters.Select(p => (p.Optional ? "[" : "") + "@" + p.Name + " = " + GetSqlTypeName(p.Type) + (p.Optional ? "]" : ""))) + (_message.OutputParameters.Count == 0 ? "" : (_message.InputParameters.Count == 0 ? "" : ",") + " " + string.Join(", ", _message.OutputParameters.Select(p => "[@" + p.Name + " = " + GetSqlTypeName(p.Type) + " OUTPUT]"))); + get + { + var parameters = _message.InputParameters + .Where(p => p.Type != typeof(PagingInfo)); + + if (_columnOrdering == ColumnOrdering.Alphabetical) + parameters = parameters.OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase); + else + parameters = parameters.OrderBy(p => p.Position); + + return _message.Name + " " + String.Join(", ", parameters.Select(p => (p.Optional ? "[" : "") + "@" + p.Name + " = " + GetSqlTypeName(p.Type) + (p.Optional ? "]" : ""))) + (_message.OutputParameters.Count == 0 ? "" : ((_message.InputParameters.Count == 0 ? "" : ",") + " " + String.Join(", ", _message.OutputParameters.Select(p => "[@" + p.Name + " = " + GetSqlTypeName(p.Type) + " OUTPUT]")))); + } set => base.ToolTipText = value; } } @@ -1674,15 +1724,21 @@ public AttributeAutocompleteItem(AttributeMetadata attribute, int replaceLength, _virtualSuffix = virtualSuffix; } - public static IEnumerable CreateList(AttributeMetadata attribute, int replaceLength, bool writeable) + public static IEnumerable CreateList(AttributeMetadata attribute, int replaceLength, bool writeable, IAttributeMetadataCache metadata) { yield return new AttributeAutocompleteItem(attribute, replaceLength); if (!writeable && (attribute is EnumAttributeMetadata || attribute is BooleanAttributeMetadata || attribute is LookupAttributeMetadata)) yield return new AttributeAutocompleteItem(attribute, replaceLength, "name"); - if (attribute is LookupAttributeMetadata lookup && lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList && (lookup.EntityLogicalName != "listmember" || lookup.LogicalName != "entityid")) - yield return new AttributeAutocompleteItem(attribute, replaceLength, "type"); + if (attribute is LookupAttributeMetadata lookup) + { + if (lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList && (lookup.EntityLogicalName != "listmember" || lookup.LogicalName != "entityid")) + yield return new AttributeAutocompleteItem(attribute, replaceLength, "type"); + + if (lookup.Targets != null && metadata.TryGetMinimalData(lookup.EntityLogicalName, out var entity) && entity.Attributes.Any(a => a.LogicalName == attribute.LogicalName + "pid")) + yield return new AttributeAutocompleteItem(attribute, replaceLength, "pid"); + } } public override string ToolTipTitle @@ -1701,6 +1757,8 @@ public override string ToolTipText description += $"\r\n\r\nThis attribute holds the display name of the {_attribute.LogicalName} field"; else if (_virtualSuffix == "type") description += $"\r\n\r\nThis attribute holds the logical name of the type of record referenced by the {_attribute.LogicalName} field"; + else if (_virtualSuffix == "pid") + description += $"\r\n\r\nThis attribute holds the partition id of the record referenced by the {_attribute.LogicalName} field"; else if (_attribute.AttributeType == AttributeTypeCode.Picklist) description += "\r\n\r\nThis attribute holds the underlying integer value of the field"; else if (_attribute.AttributeType == AttributeTypeCode.Lookup || _attribute.AttributeType == AttributeTypeCode.Customer || _attribute.AttributeType == AttributeTypeCode.Owner) diff --git a/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/AutocompleteHandler.cs b/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/AutocompleteHandler.cs index 747fa6cc..ed8a137c 100644 --- a/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/AutocompleteHandler.cs +++ b/MarkMpn.Sql4Cds.LanguageServer/Autocomplete/AutocompleteHandler.cs @@ -57,7 +57,7 @@ public void Initialize(JsonRpc lsp) Messages = c.Value.MessageCache }); } - var ac = new Autocomplete(acds, con.DataSource.Name); + var ac = new Autocomplete(acds, con.DataSource.Name, con.Connection.ColumnOrdering); var suggestions = ac.GetSuggestions(doc, pos); return suggestions .Select(s => new CompletionItem @@ -98,7 +98,7 @@ public Hover HandleHover(TextDocumentPositionParams request) Messages = c.Value.MessageCache }); } - var ac = new Autocomplete(acds, con.DataSource.Name); + var ac = new Autocomplete(acds, con.DataSource.Name, con.Connection.ColumnOrdering); var suggestions = ac.GetSuggestions(doc, pos); var exactSuggestions = suggestions.Where(suggestion => suggestion.Text.Length <= wordEnd.Index && doc.Substring(wordEnd.Index - suggestion.CompareText.Length, suggestion.CompareText.Length).Equals(suggestion.CompareText, StringComparison.OrdinalIgnoreCase)).ToList(); diff --git a/MarkMpn.Sql4Cds.LanguageServer/Connection/CachedMetadata.cs b/MarkMpn.Sql4Cds.LanguageServer/Connection/CachedMetadata.cs index d26921b3..c948a10a 100644 --- a/MarkMpn.Sql4Cds.LanguageServer/Connection/CachedMetadata.cs +++ b/MarkMpn.Sql4Cds.LanguageServer/Connection/CachedMetadata.cs @@ -91,6 +91,8 @@ public bool TryGetValue(string logicalName, out EntityMetadata metadata) return _defaultCache.TryGetValue(logicalName, out metadata); } + public string[] RecycleBinEntities => _defaultCache.RecycleBinEntities; + public EntityMetadata[] GetAutocompleteEntities() { if (_cacheUnavailable && _autocompleteCache == null) diff --git a/MarkMpn.Sql4Cds.LanguageServer/ObjectExplorer/ObjectExplorerHandler.cs b/MarkMpn.Sql4Cds.LanguageServer/ObjectExplorer/ObjectExplorerHandler.cs index d27b8067..9bf7985c 100644 --- a/MarkMpn.Sql4Cds.LanguageServer/ObjectExplorer/ObjectExplorerHandler.cs +++ b/MarkMpn.Sql4Cds.LanguageServer/ObjectExplorer/ObjectExplorerHandler.cs @@ -113,6 +113,17 @@ public bool HandleExpand(ExpandParams request) }); } + if (session.DataSource.Metadata.RecycleBinEntities != null) + { + nodes.Add(new NodeInfo + { + IsLeaf = false, + Label = "Recycle Bin", + NodePath = request.NodePath + "/Bin", + NodeType = "Folder", + }); + } + nodes.Add(new NodeInfo { IsLeaf = false, @@ -205,6 +216,26 @@ public bool HandleExpand(ExpandParams request) } } } + else if (url.AbsolutePath == "/Bin") + { + foreach (var entity in session.DataSource.Metadata.RecycleBinEntities) + { + nodes.Add(new NodeInfo + { + IsLeaf = false, + Label = entity, + NodePath = request.NodePath + "/" + entity, + NodeType = "Table", + Metadata = new ObjectMetadata + { + Urn = request.NodePath + "/" + entity, + MetadataType = MetadataType.Table, + Schema = "bin", + Name = entity + } + }); + } + } else if (url.AbsolutePath.StartsWith("/Tables/") && url.AbsolutePath.Split('/').Length == 3) { var tableName = url.AbsolutePath.Split('/')[2]; @@ -375,6 +406,18 @@ public bool HandleExpand(ExpandParams request) return true; } + private bool HasEntity(DataSourceWithInfo dataSource, string logicalName) + { + try + { + return dataSource.Metadata[logicalName] != null; + } + catch + { + return false; + } + } + private bool HandleRefresh(RefreshParams args) { return HandleExpand(args); diff --git a/MarkMpn.Sql4Cds.LanguageServer/QueryExecution/QueryExecutionHandler.cs b/MarkMpn.Sql4Cds.LanguageServer/QueryExecution/QueryExecutionHandler.cs index 5e4c6a5f..15aec57f 100644 --- a/MarkMpn.Sql4Cds.LanguageServer/QueryExecution/QueryExecutionHandler.cs +++ b/MarkMpn.Sql4Cds.LanguageServer/QueryExecution/QueryExecutionHandler.cs @@ -451,6 +451,18 @@ private async Task ExecuteAsync(Connection.Session session, ExecuteRequestParams IsError = true } }); + + await _lsp.NotifyAsync(MessageEvent.Type, new MessageParams + { + OwnerUri = request.OwnerUri, + Message = new ResultMessage + { + BatchId = batchSummary.Id, + Time = DateTime.UtcNow.ToString("o"), + Message = sql4CdsError.Message, + IsError = true + } + }); } } @@ -574,7 +586,7 @@ private string GetErrorMessage(Exception error, Sql4CdsException rootException) if (error is AggregateException aggregateException) msg = String.Join("\r\n", aggregateException.InnerExceptions.Select(ex => GetErrorMessage(ex, rootException)).Where(m => !String.IsNullOrEmpty(m))); - else if (rootException != null && rootException.Message == error.Message) + else if (rootException != null && rootException.Errors.Any(err => err.Message == error.Message)) msg = ""; else msg = error.Message; diff --git a/MarkMpn.Sql4Cds.Tests/AutocompleteTests.cs b/MarkMpn.Sql4Cds.Tests/AutocompleteTests.cs index 95331961..2ae7d9b4 100644 --- a/MarkMpn.Sql4Cds.Tests/AutocompleteTests.cs +++ b/MarkMpn.Sql4Cds.Tests/AutocompleteTests.cs @@ -38,7 +38,7 @@ public AutocompleteTests() Messages = new StubMessageCache() } }; - _autocomplete = new Autocomplete(dataSources, "local"); + _autocomplete = new Autocomplete(dataSources, "local", ColumnOrdering.Strict); } [TestMethod] diff --git a/MarkMpn.Sql4Cds.XTB/Autocomplete.cs b/MarkMpn.Sql4Cds.XTB/Autocomplete.cs index 8e55a28c..a1d13cdf 100644 --- a/MarkMpn.Sql4Cds.XTB/Autocomplete.cs +++ b/MarkMpn.Sql4Cds.XTB/Autocomplete.cs @@ -11,6 +11,7 @@ using Microsoft.Crm.Sdk.Messages; using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; using static MarkMpn.Sql4Cds.XTB.FunctionMetadata; namespace MarkMpn.Sql4Cds.XTB @@ -22,16 +23,19 @@ public class Autocomplete { private readonly IDictionary _dataSources; private readonly string _primaryDataSource; + private readonly ColumnOrdering _columnOrdering; /// /// Creates a new /// /// The list of entities available to use in the query /// The cache of metadata about each entity - public Autocomplete(IDictionary dataSources, string primaryDataSource) + /// The order that columns are passed to table-valued functions or stored procedures + public Autocomplete(IDictionary dataSources, string primaryDataSource, ColumnOrdering columnOrdering) { _dataSources = dataSources; _primaryDataSource = primaryDataSource; + _columnOrdering = columnOrdering; } /// @@ -432,7 +436,7 @@ public IEnumerable GetSuggestions(string text, int pos) if (tables.TryGetValue(targetTable, out var tableName)) { if (TryParseTableName(tableName, out var instanceName, out _, out tableName) && _dataSources.TryGetValue(instanceName, out var instance) && instance.Metadata.TryGetMinimalData(tableName, out var metadata)) - return FilterList(metadata.Attributes.Where(a => a.IsValidForUpdate != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(a => a.IsValidForUpdate != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true, instance.Metadata)).OrderBy(a => a), currentWord); } } @@ -455,14 +459,14 @@ public IEnumerable GetSuggestions(string text, int pos) if (instance.Messages.TryGetValue(messageName, out var message) && message.IsValidAsTableValuedFunction()) { - return FilterList(GetMessageOutputAttributes(message, instance).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false)).OrderBy(a => a), currentWord); + return FilterList(GetMessageOutputAttributes(message, instance).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false, instance.Metadata)).OrderBy(a => a), currentWord); } } else { // Table if (instance.Metadata.TryGetMinimalData((schemaName == "metadata" ? "metadata." : "") + tableName, out var metadata)) - return FilterList(metadata.Attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, false, instance.Metadata)).OrderBy(a => a), currentWord); } } } @@ -488,11 +492,15 @@ public IEnumerable GetSuggestions(string text, int pos) var relationship = metadata.ManyToManyRelationships.Single(); attributeFilter = a => a.LogicalName == relationship.Entity1IntersectAttribute || a.LogicalName == relationship.Entity2IntersectAttribute; } + else if (metadata.LogicalName == "principalobjectaccess") + { + attributeFilter = a => a.LogicalName == "objectid" || a.LogicalName == "objecttypecode" || a.LogicalName == "principalid" || a.LogicalName == "principaltypecode" || a.LogicalName == "accessrightsmask"; + } else { attributeFilter = a => a.IsValidForCreate != false && a.AttributeOf == null; } - return FilterList(metadata.Attributes.Where(attributeFilter).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true)).OrderBy(a => a), currentWord); + return FilterList(metadata.Attributes.Where(attributeFilter).SelectMany(a => AttributeAutocompleteItem.CreateList(a, currentLength, true, instance.Metadata)).OrderBy(a => a), currentWord); } } else @@ -504,6 +512,7 @@ public IEnumerable GetSuggestions(string text, int pos) // * variables var items = new List(); var attributes = new List(); + var instance = default(AutocompleteDataSource); foreach (var table in tables) { @@ -513,7 +522,7 @@ public IEnumerable GetSuggestions(string text, int pos) var messageName = table.Value.Substring(0, table.Value.Length - 1); if (TryParseTableName(messageName, out var instanceName, out var schemaName, out var tableName) && - _dataSources.TryGetValue(instanceName, out var instance) && + _dataSources.TryGetValue(instanceName, out instance) && (String.IsNullOrEmpty(schemaName) || schemaName == "dbo") && instance.Messages.TryGetValue(messageName, out var message) && message.IsValidAsTableValuedFunction()) @@ -527,7 +536,7 @@ public IEnumerable GetSuggestions(string text, int pos) else { // Table - if (TryParseTableName(table.Value, out var instanceName, out var schemaName, out var tableName) && _dataSources.TryGetValue(instanceName, out var instance)) + if (TryParseTableName(table.Value, out var instanceName, out var schemaName, out var tableName) && _dataSources.TryGetValue(instanceName, out instance)) { var entity = instance.Entities.SingleOrDefault(e => e.LogicalName == tableName && @@ -546,6 +555,12 @@ public IEnumerable GetSuggestions(string text, int pos) schemaName == "archive" && (e.IsRetentionEnabled == true || e.IsArchivalEnabled == true) ) + || + ( + schemaName == "bin" && + instance.Metadata.RecycleBinEntities != null && + instance.Metadata.RecycleBinEntities.Contains(e.LogicalName) + ) ) ); @@ -558,7 +573,7 @@ public IEnumerable GetSuggestions(string text, int pos) } } - items.AddRange(attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).GroupBy(x => x.LogicalName).Where(g => g.Count() == 1).SelectMany(g => AttributeAutocompleteItem.CreateList(g.Single(), currentLength, false))); + items.AddRange(attributes.Where(a => a.IsValidForRead != false && a.AttributeOf == null).GroupBy(x => x.LogicalName).Where(g => g.Count() == 1).SelectMany(g => AttributeAutocompleteItem.CreateList(g.Single(), currentLength, false, instance.Metadata))); items.AddRange(typeof(FunctionMetadata.SqlFunctions).GetMethods(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public).Select(m => new FunctionAutocompleteItem(m, currentLength))); @@ -568,7 +583,6 @@ public IEnumerable GetSuggestions(string text, int pos) { // Check if there are any applicable filter operator functions that match the type of the current attribute var identifiers = prevPrevWord.Split('.'); - var instance = default(AutocompleteDataSource); var entity = default(EntityMetadata); var attribute = default(AttributeMetadata); @@ -760,11 +774,12 @@ private IEnumerable AutocompleteTableName(string currentWor // Show TVF list if (fromClause && ds.Messages != null) - list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsTableValuedFunction()).Select(x => new TVFAutocompleteItem(x, currentLength))); + list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsTableValuedFunction()).Select(x => new TVFAutocompleteItem(x, _columnOrdering, currentLength))); } } else if (TryParseTableName(currentWord, out var instanceName, out var schemaName, out var tableName, out var parts, out var lastPartLength)) { + _dataSources.TryGetValue(instanceName, out var instance); var lastPart = tableName; if (parts == 1) @@ -776,18 +791,18 @@ private IEnumerable AutocompleteTableName(string currentWor if (parts == 1 || (parts == 2 && _dataSources.ContainsKey(schemaName))) { // Could be a schema name - if ("dbo".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("dbo", lastPartLength)); + var schemaNames = (IEnumerable)new[] { "dbo", "archive", "metadata" }; + if (instance?.Metadata?.RecycleBinEntities != null) + schemaNames = schemaNames.Append("bin"); - if ("archive".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("archive", lastPartLength)); + schemaNames = schemaNames.Where(s => s.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - if ("metadata".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("metadata", lastPartLength)); + foreach (var schema in schemaNames) + list.Add(new SchemaAutocompleteItem(schema, lastPartLength)); } // Could be a table name - if (_dataSources.TryGetValue(instanceName, out var instance) && instance.Entities != null) + if (instance?.Entities != null) { IEnumerable entities; IEnumerable messages = Array.Empty(); @@ -813,6 +828,11 @@ private IEnumerable AutocompleteTableName(string currentWor messages = instance.Messages.GetAllMessages(); } } + else if (schemaName.Equals("bin", StringComparison.OrdinalIgnoreCase)) + { + // Suggest tables that are enabled for the recycle bin + entities = instance.Entities.Where(e => instance.Metadata.RecycleBinEntities != null && instance.Metadata.RecycleBinEntities.Contains(e.LogicalName)); + } else { entities = Array.Empty(); @@ -824,7 +844,7 @@ private IEnumerable AutocompleteTableName(string currentWor if (fromClause) { messages = messages.Where(e => e.IsValidAsTableValuedFunction() && e.Name.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - list.AddRange(messages.Select(e => new TVFAutocompleteItem(e, lastPartLength))); + list.AddRange(messages.Select(e => new TVFAutocompleteItem(e, _columnOrdering, lastPartLength))); } } } @@ -845,10 +865,11 @@ private IEnumerable AutocompleteSprocName(string currentWor list.AddRange(_dataSources.Values.Select(x => new InstanceAutocompleteItem(x, currentLength))); if (_dataSources.TryGetValue(_primaryDataSource, out var ds) && ds.Messages != null) - list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(x => new SprocAutocompleteItem(x, currentLength))); + list.AddRange(ds.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(x => new SprocAutocompleteItem(x, _columnOrdering, currentLength))); } else if (TryParseTableName(currentWord, out var instanceName, out var schemaName, out var tableName, out var parts, out var lastPartLength)) { + _dataSources.TryGetValue(instanceName, out var instance); var lastPart = tableName; if (parts == 1) @@ -860,19 +881,19 @@ private IEnumerable AutocompleteSprocName(string currentWor if (parts == 1 || parts == 2) { // Could be a schema name - if ("dbo".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("dbo", lastPartLength)); + var schemaNames = (IEnumerable)new[] { "dbo", "archive", "metadata" }; + if (instance?.Metadata?.RecycleBinEntities != null) + schemaNames = schemaNames.Append("bin"); - if ("archive".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("archive", lastPartLength)); + schemaNames = schemaNames.Where(s => s.StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)); - if ("metadata".StartsWith(lastPart, StringComparison.OrdinalIgnoreCase)) - list.Add(new SchemaAutocompleteItem("metadata", lastPartLength)); + foreach (var schema in schemaNames) + list.Add(new SchemaAutocompleteItem(schema, lastPartLength)); } // Could be a sproc name - if (schemaName.Equals("dbo", StringComparison.OrdinalIgnoreCase) && _dataSources.TryGetValue(instanceName, out var instance) && instance.Messages != null) - list.AddRange(instance.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(e => new SprocAutocompleteItem(e, lastPartLength))); + if (schemaName.Equals("dbo", StringComparison.OrdinalIgnoreCase) && instance?.Messages != null) + list.AddRange(instance.Messages.GetAllMessages().Where(x => x.IsValidAsStoredProcedure()).Select(e => new SprocAutocompleteItem(e, _columnOrdering, lastPartLength))); } list.Sort(); @@ -1152,6 +1173,7 @@ private static int GetIconIndex(AttributeMetadata a) case AttributeTypeCode.String: case AttributeTypeCode.Virtual: + case AttributeTypeCode.EntityName: return 13; case AttributeTypeCode.Uniqueidentifier: @@ -1213,7 +1235,7 @@ public int CompareTo(object obj) public static string GetSqlTypeName(Type type) { if (type == typeof(string)) - return "NNVARCHAR(MAX)"; + return "NVARCHAR(MAX)"; if (type == typeof(int)) return "INT"; @@ -1475,7 +1497,10 @@ public override string ToolTipTitle public override string ToolTipText { - get => Text == "metadata" ? "Schema containing the metadata information" : "Schema containing the data tables"; + get => Text == "metadata" ? "Schema containing the metadata information" : + Text == "archive" ? "Schema containing long-term retention tables" : + Text == "bin" ? "Schema containing recycle bin tables" : + "Schema containing the data tables"; set => base.ToolTipText = value; } } @@ -1517,10 +1542,12 @@ public override string GetTextForReplace() class TVFAutocompleteItem : SqlAutocompleteItem { private readonly Message _message; + private readonly ColumnOrdering _columnOrdering; - public TVFAutocompleteItem(Message message, int replaceLength) : base(message.Name, replaceLength, 25) + public TVFAutocompleteItem(Message message, ColumnOrdering columnOrdering, int replaceLength) : base(message.Name, replaceLength, 25) { _message = message; + _columnOrdering = columnOrdering; } public TVFAutocompleteItem(Message message, string alias, int replaceLength) : base(alias, replaceLength, 25) @@ -1536,7 +1563,18 @@ public override string ToolTipTitle public override string ToolTipText { - get => _message.Name + "(" + String.Join(", ", _message.InputParameters.Select(p => p.Name + " " + GetSqlTypeName(p.Type))) + ")"; + get + { + var parameters = _message.InputParameters + .Where(p => p.Type != typeof(PagingInfo)); + + if (_columnOrdering == ColumnOrdering.Alphabetical) + parameters = parameters.OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase); + else + parameters = parameters.OrderBy(p => p.Position); + + return _message.Name + "(" + String.Join(", ", parameters.Select(p => p.Name + " " + GetSqlTypeName(p.Type))) + ")"; + } set => base.ToolTipText = value; } @@ -1549,10 +1587,12 @@ public override string GetTextForReplace() class SprocAutocompleteItem : SqlAutocompleteItem { private readonly Message _message; + private readonly ColumnOrdering _columnOrdering; - public SprocAutocompleteItem(Message message, int replaceLength) : base(message.Name, replaceLength, 26) + public SprocAutocompleteItem(Message message, ColumnOrdering columnOrdering, int replaceLength) : base(message.Name, replaceLength, 26) { _message = message; + _columnOrdering = columnOrdering; } public override string ToolTipTitle @@ -1563,7 +1603,18 @@ public override string ToolTipTitle public override string ToolTipText { - get => _message.Name + " " + String.Join(", ", _message.InputParameters.Select(p => (p.Optional ? "[" : "") + "@" + p.Name + " = " + GetSqlTypeName(p.Type) + (p.Optional ? "]" : ""))) + (_message.OutputParameters.Count == 0 ? "" : ((_message.InputParameters.Count == 0 ? "" : ",") + " " + String.Join(", ", _message.OutputParameters.Select(p => "[@" + p.Name + " = " + GetSqlTypeName(p.Type) + " OUTPUT]")))); + get + { + var parameters = _message.InputParameters + .Where(p => p.Type != typeof(PagingInfo)); + + if (_columnOrdering == ColumnOrdering.Alphabetical) + parameters = parameters.OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase); + else + parameters = parameters.OrderBy(p => p.Position); + + return _message.Name + " " + String.Join(", ", parameters.Select(p => (p.Optional ? "[" : "") + "@" + p.Name + " = " + GetSqlTypeName(p.Type) + (p.Optional ? "]" : ""))) + (_message.OutputParameters.Count == 0 ? "" : ((_message.InputParameters.Count == 0 ? "" : ",") + " " + String.Join(", ", _message.OutputParameters.Select(p => "[@" + p.Name + " = " + GetSqlTypeName(p.Type) + " OUTPUT]")))); + } set => base.ToolTipText = value; } } @@ -1655,15 +1706,21 @@ public AttributeAutocompleteItem(AttributeMetadata attribute, int replaceLength, _virtualSuffix = virtualSuffix; } - public static IEnumerable CreateList(AttributeMetadata attribute, int replaceLength, bool writeable) + public static IEnumerable CreateList(AttributeMetadata attribute, int replaceLength, bool writeable, IAttributeMetadataCache metadata) { yield return new AttributeAutocompleteItem(attribute, replaceLength); if (!writeable && (attribute is EnumAttributeMetadata || attribute is BooleanAttributeMetadata || attribute is LookupAttributeMetadata)) yield return new AttributeAutocompleteItem(attribute, replaceLength, "name"); - if (attribute is LookupAttributeMetadata lookup && lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList && (lookup.EntityLogicalName != "listmember" || lookup.LogicalName != "entityid")) - yield return new AttributeAutocompleteItem(attribute, replaceLength, "type"); + if (attribute is LookupAttributeMetadata lookup) + { + if (lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList && (lookup.EntityLogicalName != "listmember" || lookup.LogicalName != "entityid")) + yield return new AttributeAutocompleteItem(attribute, replaceLength, "type"); + + if (lookup.Targets != null && metadata.TryGetMinimalData(lookup.EntityLogicalName, out var entity) && entity.Attributes.Any(a => a.LogicalName == attribute.LogicalName + "pid")) + yield return new AttributeAutocompleteItem(attribute, replaceLength, "pid"); + } } public override string ToolTipTitle @@ -1682,6 +1739,8 @@ public override string ToolTipText description += $"\r\n\r\nThis attribute holds the display name of the {_attribute.LogicalName} field"; else if (_virtualSuffix == "type") description += $"\r\n\r\nThis attribute holds the logical name of the type of record referenced by the {_attribute.LogicalName} field"; + else if (_virtualSuffix == "pid") + description += $"\r\n\r\nThis attribute holds the partition id of the record referenced by the {_attribute.LogicalName} field"; else if (_attribute.AttributeType == AttributeTypeCode.Picklist) description += "\r\n\r\nThis attribute holds the underlying integer value of the field"; else if (_attribute.AttributeType == AttributeTypeCode.Lookup || _attribute.AttributeType == AttributeTypeCode.Customer || _attribute.AttributeType == AttributeTypeCode.Owner) diff --git a/MarkMpn.Sql4Cds.XTB/FetchXml2SqlSettingsForm.cs b/MarkMpn.Sql4Cds.XTB/FetchXml2SqlSettingsForm.cs index 260bb2db..d45f8fe5 100644 --- a/MarkMpn.Sql4Cds.XTB/FetchXml2SqlSettingsForm.cs +++ b/MarkMpn.Sql4Cds.XTB/FetchXml2SqlSettingsForm.cs @@ -208,6 +208,8 @@ public bool TryGetValue(string logicalName, out EntityMetadata metadata) { return _cache.TryGetValue(logicalName, out metadata); } + + public string[] RecycleBinEntities => throw new NotImplementedException(); } } } diff --git a/MarkMpn.Sql4Cds.XTB/ObjectExplorer.cs b/MarkMpn.Sql4Cds.XTB/ObjectExplorer.cs index acad9aa4..3e28166c 100644 --- a/MarkMpn.Sql4Cds.XTB/ObjectExplorer.cs +++ b/MarkMpn.Sql4Cds.XTB/ObjectExplorer.cs @@ -12,6 +12,8 @@ using Microsoft.Xrm.Sdk.Metadata.Query; using Microsoft.Xrm.Tooling.Connector; using System.Threading.Tasks; +using Microsoft.Xrm.Sdk.Query; +using Microsoft.Xrm.Sdk; namespace MarkMpn.Sql4Cds.XTB { @@ -72,13 +74,25 @@ public IEnumerable GetImages() return imageList.Images.OfType(); } - private TreeNode[] LoadEntities(TreeNode parent, bool archival) + enum EntityType + { + Regular, + Archive, + RecycleBin + } + + private TreeNode[] LoadEntities(TreeNode parent, EntityType entityType) { var connection = GetService(parent); var metadata = EntityCache.GetEntities(connection.MetadataCacheLoader, connection.ServiceClient); + var recycleBinEntities = _dataSources[connection.ConnectionName].Metadata.RecycleBinEntities; return metadata - .Where(e => !archival || e.IsArchivalEnabled == true || e.IsRetentionEnabled == true) + .Where(e => + entityType == EntityType.Regular || + (entityType == EntityType.Archive && (e.IsArchivalEnabled == true || e.IsRetentionEnabled == true)) || + (entityType == EntityType.RecycleBin && recycleBinEntities.Contains(e.LogicalName)) + ) .OrderBy(e => e.LogicalName) .Select(e => { @@ -89,14 +103,28 @@ private TreeNode[] LoadEntities(TreeNode parent, bool archival) SetIcon(attrsNode, "Folder"); AddVirtualChildNodes(attrsNode, LoadAttributes); - if (!archival) + if (entityType != EntityType.Archive) { var relsNode = node.Nodes.Add("Relationships"); SetIcon(relsNode, "Folder"); AddVirtualChildNodes(relsNode, LoadRelationships); } - parent.Tag = archival ? "archive" : "dbo"; + switch (entityType) + { + case EntityType.Regular: + parent.Tag = "dbo"; + break; + + case EntityType.Archive: + parent.Tag = "archive"; + break; + + case EntityType.RecycleBin: + parent.Tag = "bin"; + break; + } + return node; }) .ToArray(); @@ -142,13 +170,20 @@ private void AddConnectionChildNodes(ConnectionDetail con, CrmServiceClient svc, { var entitiesNode = conNode.Nodes.Add("Entities"); SetIcon(entitiesNode, "Folder"); - AddVirtualChildNodes(entitiesNode, parent => LoadEntities(parent, false)); + AddVirtualChildNodes(entitiesNode, parent => LoadEntities(parent, EntityType.Regular)); if (new Uri(con.OrganizationServiceUrl).Host.EndsWith(".dynamics.com")) { var archivalNode = conNode.Nodes.Add("Long Term Retention"); SetIcon(archivalNode, "Folder"); - AddVirtualChildNodes(archivalNode, parent => LoadEntities(parent, true)); + AddVirtualChildNodes(archivalNode, parent => LoadEntities(parent, EntityType.Archive)); + } + + if (_dataSources[con.ConnectionName].Metadata.RecycleBinEntities != null) + { + var recycleBinNode = conNode.Nodes.Add("Recycle Bin"); + SetIcon(recycleBinNode, "Folder"); + AddVirtualChildNodes(recycleBinNode, parent => LoadEntities(parent, EntityType.RecycleBin)); } var metadataNode = conNode.Nodes.Add("Metadata"); diff --git a/MarkMpn.Sql4Cds.XTB/SharedMetadataCache.cs b/MarkMpn.Sql4Cds.XTB/SharedMetadataCache.cs index 3ba43d00..af3a4c68 100644 --- a/MarkMpn.Sql4Cds.XTB/SharedMetadataCache.cs +++ b/MarkMpn.Sql4Cds.XTB/SharedMetadataCache.cs @@ -11,6 +11,9 @@ namespace MarkMpn.Sql4Cds.XTB { + /// + /// An implementation that uses the metadata cache provided by XrmToolBox where possible + /// class SharedMetadataCache : IAttributeMetadataCache { private readonly ConnectionDetail _connection; @@ -28,6 +31,7 @@ public SharedMetadataCache(ConnectionDetail connection, IOrganizationService org _innerCache = new AttributeMetadataCache(org); } + /// public EntityMetadata this[string name] { get @@ -44,6 +48,7 @@ public EntityMetadata this[string name] } } + /// public EntityMetadata this[int otc] { get @@ -60,6 +65,7 @@ public EntityMetadata this[int otc] } } + /// public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata) { if (!CacheReady) @@ -70,6 +76,7 @@ public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata) return _entitiesByName.TryGetValue(logicalName, out metadata); } + /// public bool TryGetValue(string logicalName, out EntityMetadata metadata) { if (!CacheReady) @@ -80,6 +87,9 @@ public bool TryGetValue(string logicalName, out EntityMetadata metadata) return _entitiesByName.TryGetValue(logicalName, out metadata); } + /// + public string[] RecycleBinEntities => _innerCache.RecycleBinEntities; + private bool CacheReady { get diff --git a/MarkMpn.Sql4Cds.XTB/SqlQueryControl.cs b/MarkMpn.Sql4Cds.XTB/SqlQueryControl.cs index 4027cd7b..a5bbb5c6 100644 --- a/MarkMpn.Sql4Cds.XTB/SqlQueryControl.cs +++ b/MarkMpn.Sql4Cds.XTB/SqlQueryControl.cs @@ -437,7 +437,7 @@ private Scintilla CreateSqlEditor() }) .ToDictionary(ds => ds.Name, StringComparer.OrdinalIgnoreCase); - var suggestions = new Autocomplete(autocompleteDataSources, _con.ConnectionName).GetSuggestions(text, wordEnd.Index - 1).ToList(); + var suggestions = new Autocomplete(autocompleteDataSources, _con.ConnectionName, Settings.Instance.ColumnOrdering).GetSuggestions(text, wordEnd.Index - 1).ToList(); var exactSuggestions = suggestions.Where(suggestion => suggestion.Text.Length <= wordEnd.Index && text.Substring(wordEnd.Index - suggestion.CompareText.Length, suggestion.CompareText.Length).Equals(suggestion.CompareText, StringComparison.OrdinalIgnoreCase)).ToList(); if (exactSuggestions.Count == 1) @@ -553,7 +553,7 @@ public IEnumerator GetEnumerator() }) .ToDictionary(ds => ds.Name, StringComparer.OrdinalIgnoreCase); - var suggestions = new Autocomplete(autocompleteDataSources, _control.Connection.ConnectionName).GetSuggestions(text, pos).ToList(); + var suggestions = new Autocomplete(autocompleteDataSources, _control.Connection.ConnectionName, Settings.Instance.ColumnOrdering).GetSuggestions(text, pos).ToList(); if (suggestions.Count == 0) yield break; diff --git a/MarkMpn.Sql4Cds.sln b/MarkMpn.Sql4Cds.sln index 575e0e24..8ad7ddb9 100644 --- a/MarkMpn.Sql4Cds.sln +++ b/MarkMpn.Sql4Cds.sln @@ -48,6 +48,15 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarkMpn.Sql4Cds.SSMS.20", " EndProject Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MarkMpn.Sql4Cds.SSMS.20.Setup", "MarkMpn.Sql4Cds.SSMS.20.Setup\MarkMpn.Sql4Cds.SSMS.20.Setup.wixproj", "{FE9EF004-BD77-49DA-85B5-D51DAEB76274}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DebuggerVisualizer", "DebuggerVisualizer", "{395CF6D2-5DDE-4A42-81A8-B1C4FF20C736}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide", "MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide\MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide.csproj", "{17EE9908-59A4-4E81-85DD-2144B3BB4441}" + ProjectSection(ProjectDependencies) = postProject + {65FD7411-5B94-443B-A4C3-A66082E0B0AE} = {65FD7411-5B94-443B-A4C3-A66082E0B0AE} + EndProjectSection +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide", "MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide\MarkMpn.Sql4Cds.DebugVisualizer.DebugeeSide.csproj", "{65FD7411-5B94-443B-A4C3-A66082E0B0AE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -232,10 +241,38 @@ Global {FE9EF004-BD77-49DA-85B5-D51DAEB76274}.Release|arm64.Build.0 = Release|x86 {FE9EF004-BD77-49DA-85B5-D51DAEB76274}.Release|x86.ActiveCfg = Release|x86 {FE9EF004-BD77-49DA-85B5-D51DAEB76274}.Release|x86.Build.0 = Release|x86 + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|Any CPU.Build.0 = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|arm64.ActiveCfg = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|arm64.Build.0 = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|x86.ActiveCfg = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Debug|x86.Build.0 = Debug|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|Any CPU.ActiveCfg = Release|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|Any CPU.Build.0 = Release|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|arm64.ActiveCfg = Release|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|arm64.Build.0 = Release|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|x86.ActiveCfg = Release|Any CPU + {17EE9908-59A4-4E81-85DD-2144B3BB4441}.Release|x86.Build.0 = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|arm64.ActiveCfg = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|arm64.Build.0 = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|x86.ActiveCfg = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Debug|x86.Build.0 = Debug|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|Any CPU.Build.0 = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|arm64.ActiveCfg = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|arm64.Build.0 = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|x86.ActiveCfg = Release|Any CPU + {65FD7411-5B94-443B-A4C3-A66082E0B0AE}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {17EE9908-59A4-4E81-85DD-2144B3BB4441} = {395CF6D2-5DDE-4A42-81A8-B1C4FF20C736} + {65FD7411-5B94-443B-A4C3-A66082E0B0AE} = {395CF6D2-5DDE-4A42-81A8-B1C4FF20C736} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {5E960561-7FE2-4022-964B-57E990767108} EndGlobalSection diff --git a/MarkMpn.Sql4Cds/MarkMpn.SQL4CDS.nuspec b/MarkMpn.Sql4Cds/MarkMpn.SQL4CDS.nuspec index 54eebf0a..febcd782 100644 --- a/MarkMpn.Sql4Cds/MarkMpn.SQL4CDS.nuspec +++ b/MarkMpn.Sql4Cds/MarkMpn.SQL4CDS.nuspec @@ -23,17 +23,19 @@ plugins or integrations by writing familiar SQL and converting it. Queries can also run using the preview TDS Endpoint. A wide range of SQL functionality is also built in to allow running queries that aren't directly supported by either FetchXML or the TDS Endpoint. Convert SQL queries to FetchXML and execute them against Dataverse / D365 - Fixed NullReferenceException errors when: -- executing a conditional SELECT query -- retrieving results from a Fetch XML query using IN or EXISTS -- handling an error returned from TDS Endpoint -- handling internal errors such as UPDATE without WHERE - -Standardised errors on: -- JSON path errors -- DML statement cancellation - -Fixed filtering audit data on changedata attribute + Enabled access to recycle bin records via the `bin` schema +Enabled `INSERT`, `UPDATE` and `DELETE` statements on `principalobjectaccess` table +Enabled use of subqueries within `ON` clause of `JOIN` statements +Added support for `___pid` virtual column for lookups to elastic tables +Improved folding of queries using index spools +Improved primary key calculation when using joins on non-key columns +Apply column order setting to parameters for stored procedures and table-valued functions +Fixed error with DeleteMultiple requests +Fixed paging error with `DISTINCT` queries causing results to be limited to 50,000 records +Fixed paging errors when sorting by optionset values causing some results to be skipped +Fixed errors when using joins inside `[NOT] EXISTS` subqueries +Fixed incorrect results when applying aliases to `___name` and `___type` virtual columns +Fixed max length calculation for string columns Copyright © 2019 Mark Carrington en-GB diff --git a/build.yml b/build.yml index db7ef236..5f2aafff 100644 --- a/build.yml +++ b/build.yml @@ -125,6 +125,13 @@ steps: targetPath: 'MarkMpn.Sql4Cds.SSMS.20.Setup\bin\$(buildConfiguration)\MarkMpn.Sql4Cds.SSMS.20.Setup.msi' artifact: 'SSMS20Installer' publishLocation: 'pipeline' + +- task: PublishPipelineArtifact@1 + displayName: Publish Debug Visualizer installer to pipeline + inputs: + targetPath: 'MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide\bin\$(buildConfiguration)\net472\MarkMpn.Sql4Cds.DebugVisualizer.DebuggerSide.vsix' + artifact: 'DebugVisualizer' + publishLocation: 'pipeline' - task: PublishPipelineArtifact@1 displayName: Publish version file to pipeline