diff --git a/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs index bad282d1..dd5d110b 100644 --- a/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs +++ b/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs @@ -1514,5 +1514,98 @@ public void ComplexFetchXmlAlias() } } } + + [TestMethod] + public void OpenJsonDefaultSchema() + { + using (var con = new Sql4CdsConnection(_localDataSource)) + using (var cmd = con.CreateCommand()) + { + cmd.CommandText = @" +DECLARE @json NVARCHAR(MAX) + +SET @json='{""name"":""John"",""surname"":""Doe"",""age"":45,""skills"":[""SQL"",""C#"",""MVC""]}'; + +SELECT * +FROM OPENJSON(@json);"; + + using (var reader = cmd.ExecuteReader()) + { + Assert.AreEqual("key", reader.GetName(0)); + Assert.AreEqual("value", reader.GetName(1)); + Assert.AreEqual("type", reader.GetName(2)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("name", reader.GetString(0)); + Assert.AreEqual("John", reader.GetString(1)); + Assert.AreEqual(1, reader.GetInt32(2)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("surname", reader.GetString(0)); + Assert.AreEqual("Doe", reader.GetString(1)); + Assert.AreEqual(1, reader.GetInt32(2)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("age", reader.GetString(0)); + Assert.AreEqual("45", reader.GetString(1)); + Assert.AreEqual(2, reader.GetInt32(2)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("skills", reader.GetString(0)); + Assert.AreEqual("[\r\n \"SQL\",\r\n \"C#\",\r\n \"MVC\"\r\n]", reader.GetString(1)); + Assert.AreEqual(4, reader.GetInt32(2)); + + Assert.IsFalse(reader.Read()); + } + } + } + + [TestMethod] + public void OpenJsonDefaultSchemaWithPath() + { + using (var con = new Sql4CdsConnection(_localDataSource)) + using (var cmd = con.CreateCommand()) + { + cmd.CommandText = @" +DECLARE @json NVARCHAR(4000) = N'{ + ""path"": { + ""to"":{ + ""sub-object"":[""en-GB"", ""en-UK"",""de-AT"",""es-AR"",""sr-Cyrl""] + } + } + }'; + +SELECT [key], value +FROM OPENJSON(@json,'$.path.to.""sub-object""')"; + + using (var reader = cmd.ExecuteReader()) + { + Assert.AreEqual("key", reader.GetName(0)); + Assert.AreEqual("value", reader.GetName(1)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("0", reader.GetString(0)); + Assert.AreEqual("en-GB", reader.GetString(1)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("1", reader.GetString(0)); + Assert.AreEqual("en-UK", reader.GetString(1)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("2", reader.GetString(0)); + Assert.AreEqual("de-AT", reader.GetString(1)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("3", reader.GetString(0)); + Assert.AreEqual("es-AR", reader.GetString(1)); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual("4", reader.GetString(0)); + Assert.AreEqual("sr-Cyrl", reader.GetString(1)); + + Assert.IsFalse(reader.Read()); + } + } + } } } diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/OpenJsonNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/OpenJsonNode.cs new file mode 100644 index 00000000..757bb13a --- /dev/null +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/OpenJsonNode.cs @@ -0,0 +1,275 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data.SqlTypes; +using System.Linq; +using System.Text; +using Microsoft.SqlServer.TransactSql.ScriptDom; +using Microsoft.Xrm.Sdk; +using Newtonsoft.Json.Linq; + +namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan +{ + class OpenJsonNode : BaseDataNode + { + private Func _jsonExpression; + private Func _pathExpression; + private Collation _jsonCollation; + + private static readonly Collation _keyCollation; + + static OpenJsonNode() + { + _keyCollation = new Collation(null, 1033, SqlCompareOptions.BinarySort2, null); + } + + private OpenJsonNode() + { + } + + public OpenJsonNode(OpenJsonTableReference tvf) + { + Alias = tvf.Alias?.Value; + Json = tvf.Variable.Clone(); + Path = tvf.RowPattern?.Clone(); + + // TODO: Check expressions are string types and add conversions if not + } + + /// + /// The alias for the data source + /// + [Category("Open JSON")] + [Description("The alias for the data source")] + public string Alias { get; set; } + + /// + /// The expression that provides the JSON to parse + /// + [Category("Open JSON")] + [Description("The expression that provides the JSON to parse")] + public ScalarExpression Json { get; set; } + + /// + /// The expression that defines the JSON path to the object or array to parse + /// + [Category("Open JSON")] + [Description("The expression that defines the JSON path to the object or array to parse")] + public ScalarExpression Path { get; set; } + + public override void AddRequiredColumns(NodeCompilationContext context, IList requiredColumns) + { + } + + public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext context, IList hints) + { + var ecc = new ExpressionCompilationContext(context, null, null); + _jsonExpression = Json.Compile(ecc); + _pathExpression = Path?.Compile(ecc); + + Json.GetType(ecc, out var jsonType); + _jsonCollation = (jsonType as SqlDataTypeReferenceWithCollation)?.Collation ?? context.PrimaryDataSource.DefaultCollation; + + return this; + } + + public override INodeSchema GetSchema(NodeCompilationContext context) + { + var columns = new ColumnList(); + var aliases = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + columns.Add(PrefixWithAlias("key", aliases), new ColumnDefinition(DataTypeHelpers.NVarChar(4000, _keyCollation, CollationLabel.Implicit), false, false)); + columns.Add(PrefixWithAlias("value", aliases), new ColumnDefinition(DataTypeHelpers.NVarChar(Int32.MaxValue, _jsonCollation, CollationLabel.Implicit), true, false)); + columns.Add(PrefixWithAlias("type", aliases), new ColumnDefinition(DataTypeHelpers.Int, false, false)); + + var schema = new NodeSchema( + columns, + aliases, + columns.First().Key, + null + ); + + return schema; + } + + private string PrefixWithAlias(string name, IDictionary> aliases) + { + name = name.EscapeIdentifier(); + + if (Alias == null) + return name; + + var fullName = Alias.EscapeIdentifier() + "." + name; + + if (aliases != null) + { + if (!aliases.TryGetValue(name, out var alias)) + { + alias = new List(); + aliases[name] = alias; + } + + ((List)alias).Add(fullName); + } + + return fullName; + } + + public override IEnumerable GetSources() + { + return Array.Empty(); + } + + protected override RowCountEstimate EstimateRowsOutInternal(NodeCompilationContext context) + { + return new RowCountEstimate(10); + } + + protected override IEnumerable ExecuteInternal(NodeExecutionContext context) + { + var eec = new ExpressionExecutionContext(context); + + var json = (SqlString) _jsonExpression(eec); + + if (json.IsNull || json.Value.Length == 0) + yield break; + + string path; + + if (_pathExpression != null) + { + var pathValue = (SqlString)_pathExpression(eec); + + if (pathValue.IsNull) + yield break; + + path = pathValue.Value; + } + else + { + path = "$"; + } + + JsonPath jpath; + JToken jsonDoc; + JToken jtoken; + + try + { + jpath = new JsonPath(path); + jsonDoc = JToken.Parse(json.Value); + jtoken = jpath.Evaluate(jsonDoc); + } + catch (Newtonsoft.Json.JsonException ex) + { + throw new QueryExecutionException(ex.Message, ex); + } + + if (jtoken == null) + { + if (jpath.Mode == JsonPathMode.Lax) + yield break; + else + throw new QueryExecutionException("Property does not exist"); + } + + var keyCol = PrefixWithAlias("key", null); + var valueCol = PrefixWithAlias("value", null); + var typeCol = PrefixWithAlias("type", null); + + if (jtoken.Type == JTokenType.Object) + { + foreach (var prop in ((JObject)jtoken).Properties()) + { + var key = _keyCollation.ToSqlString(prop.Name); + var value = GetValue(prop.Value); + var type = GetType(prop.Value); + + yield return new Entity + { + [keyCol] = key, + [valueCol] = value, + [typeCol] = type + }; + } + } + else if (jtoken.Type == JTokenType.Array) + { + for (var i = 0; i < ((JArray)jtoken).Count; i++) + { + var subToken = ((JArray)jtoken)[i]; + var key = _keyCollation.ToSqlString(i.ToString()); + var value = GetValue(subToken); + var type = GetType(subToken); + + yield return new Entity + { + [keyCol] = key, + [valueCol] = value, + [typeCol] = type + }; + } + } + else + { + if (jpath.Mode == JsonPathMode.Lax) + yield break; + else + throw new QueryExecutionException("Not an object or array"); + } + } + + private SqlString GetValue(JToken token) + { + string str; + + if (token is JContainer) + str = token.ToString(); + else + str = token.Value(); + + return _jsonCollation.ToSqlString(str); + } + + private SqlInt32 GetType(JToken token) + { + switch (token.Type) + { + case JTokenType.Null: + return 0; + + case JTokenType.String: + return 1; + + case JTokenType.Integer: + case JTokenType.Float: + return 2; + + case JTokenType.Boolean: + return 3; + + case JTokenType.Array: + return 4; + + case JTokenType.Object: + return 5; + + default: + throw new QueryExecutionException($"Unexpected token type: {token.Type}"); + } + } + + public override object Clone() + { + return new OpenJsonNode + { + Alias = Alias, + Json = Json.Clone(), + Path = Path.Clone(), + _jsonExpression = _jsonExpression, + _pathExpression = _pathExpression, + _jsonCollation = _jsonCollation + }; + } + } +} diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs index b8bb44f8..3796c6e0 100644 --- a/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs +++ b/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs @@ -4100,6 +4100,48 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe return loop; } + if (reference is OpenJsonTableReference openJson) + { + // Capture any references to data from an outer query + CaptureOuterReferences(outerSchema, null, openJson, context, outerReferences); + + // Convert any scalar subqueries in the parameters to its own execution plan, and capture the references from those plans + // as parameters to be passed to the function + IDataExecutionPlanNodeInternal source = new ConstantScanNode { Values = { new Dictionary() } }; + var computeScalar = new ComputeScalarNode { Source = source }; + + ConvertScalarSubqueries(openJson.Variable, hints, ref source, computeScalar, context, openJson); + + if (openJson.RowPattern != null) + ConvertScalarSubqueries(openJson.RowPattern, hints, ref source, computeScalar, context, openJson); + + if (source is ConstantScanNode) + source = null; + else if (computeScalar.Columns.Count > 0) + source = computeScalar; + + var scalarSubquerySchema = source?.GetSchema(context); + var scalarSubqueryReferences = new Dictionary(); + CaptureOuterReferences(scalarSubquerySchema, null, openJson, context, scalarSubqueryReferences); + + var execute = new OpenJsonNode(openJson); + + if (source == null) + return execute; + + // If we've got any subquery parameters we need to use a loop to pass them to the function + var loop = new NestedLoopNode + { + LeftSource = source, + RightSource = execute, + JoinType = QualifiedJoinType.Inner, + OuterReferences = scalarSubqueryReferences, + OutputLeftSchema = false, + }; + + return loop; + } + throw new NotSupportedQueryFragmentException("Unhandled table reference", reference); } diff --git a/MarkMpn.Sql4Cds.Engine/ExpressionFunctions.cs b/MarkMpn.Sql4Cds.Engine/ExpressionFunctions.cs index 610b7266..b7210305 100644 --- a/MarkMpn.Sql4Cds.Engine/ExpressionFunctions.cs +++ b/MarkMpn.Sql4Cds.Engine/ExpressionFunctions.cs @@ -58,21 +58,16 @@ public static SqlString Json_Value(SqlString json, SqlString jpath) return SqlString.Null; var path = jpath.Value; - var lax = !path.StartsWith("strict ", StringComparison.OrdinalIgnoreCase); - - if (path.StartsWith("strict ", StringComparison.OrdinalIgnoreCase)) - path = path.Substring(7); - else if (path.StartsWith("lax ", StringComparison.OrdinalIgnoreCase)) - path = path.Substring(4); - + try { + var jsonPath = new JsonPath(path); var jsonDoc = JToken.Parse(json.Value); - var jtoken = jsonDoc.SelectToken(path); + var jtoken = jsonPath.Evaluate(jsonDoc); if (jtoken == null) { - if (lax) + if (jsonPath.Mode == JsonPathMode.Lax) return SqlString.Null; else throw new QueryExecutionException("Property does not exist"); @@ -80,7 +75,7 @@ public static SqlString Json_Value(SqlString json, SqlString jpath) if (jtoken.Type == JTokenType.Object || jtoken.Type == JTokenType.Array) { - if (lax) + if (jsonPath.Mode == JsonPathMode.Lax) return SqlString.Null; else throw new QueryExecutionException("Not a scalar value"); @@ -93,7 +88,7 @@ public static SqlString Json_Value(SqlString json, SqlString jpath) if (value.Length > 4000) { - if (lax) + if (jsonPath.Mode == JsonPathMode.Lax) return SqlString.Null; else throw new QueryExecutionException("Value too long"); @@ -120,16 +115,11 @@ public static SqlBoolean Json_Path_Exists(SqlString json, SqlString jpath) var path = jpath.Value; - if (path.StartsWith("strict ", StringComparison.OrdinalIgnoreCase)) - path = path.Substring(7); - else if (path.StartsWith("lax ", StringComparison.OrdinalIgnoreCase)) - path = path.Substring(4); - try { - + var jsonPath = new JsonPath(path); var jsonDoc = JToken.Parse(json.Value); - var jtoken = jsonDoc.SelectToken(path); + var jtoken = jsonPath.Evaluate(jsonDoc); return jtoken != null; } diff --git a/MarkMpn.Sql4Cds.Engine/JsonPath.cs b/MarkMpn.Sql4Cds.Engine/JsonPath.cs new file mode 100644 index 00000000..a8e1b5d1 --- /dev/null +++ b/MarkMpn.Sql4Cds.Engine/JsonPath.cs @@ -0,0 +1,287 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json.Linq; + +namespace MarkMpn.Sql4Cds.Engine +{ + /// + /// Handles navigating JSON documents using a JSON Path expression + /// + /// + /// The JSON Path syntax supported by SQL Server is subtely different to that implemented by + /// JSON.NET, so to keep compatibility with the various T-SQL samples we need to use this class + /// instead of the built-in method. + /// https://learn.microsoft.com/en-us/sql/relational-databases/json/json-path-expressions-sql-server?view=sql-server-ver16 + /// + class JsonPath + { + private readonly JsonPathPart[] _parts; + private readonly JsonPathMode _mode; + + public JsonPath(string expression) + { + _parts = Parse(expression, out _mode); + } + + /// + /// Returns the mode the path should be evaluated in + /// + public JsonPathMode Mode { get; } + + /// + /// Finds the token matching the path + /// + /// The token to start matching from + /// The token matching the path, or null if no match is found + public JToken Evaluate(JToken token) + { + foreach (var part in _parts) + { + token = part.Match(token); + + if (token == null) + return null; + } + + return token; + } + + private static JsonPathPart[] Parse(string expression, out JsonPathMode mode) + { + mode = JsonPathMode.Lax; + + if (expression.StartsWith("lax ")) + { + expression = expression.Substring(4); + } + else if (expression.StartsWith("strict ")) + { + expression = expression.Substring(7); + mode = JsonPathMode.Strict; + } + + var parts = new List(); + + for (var i = 0; i < expression.Length; i++) + { + if (i == 0 && expression[i] == '$') + { + parts.Add(new ContextJsonPathPart()); + } + else if (expression[i] == '.') + { + // Start of a property key + i++; + + if (i == expression.Length) + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - missing property name after '.' at end of '{expression}'"); + + if (expression[i] == '"') + { + // Start of a quoted property key + i++; + var start = i; + + while (i < expression.Length) + { + if (expression[i] == '\\') + i += 2; + else if (expression[i] == '"') + break; + else + i++; + } + + if (i < expression.Length && expression[i] == '"') + { + var propertyName = expression.Substring(start, i - start).Replace("\\\"", "\"").Replace("\\\\", "\\"); + parts.Add(new PropertyJsonPathPart(propertyName)); + } + } + else if (expression[i] >= 'a' && expression[i] <= 'z' || + expression[i] >= 'A' && expression[i] <= 'Z') + { + // Start of an unquoted property key + var end = expression.IndexOfAny(new[] { '.', '[' }, i); + + if (end == -1) + end = expression.Length; + + var propertyName = expression.Substring(i, end - i); + + if (propertyName == "sql:identity()") + { + if (end == expression.Length) + parts.Add(new ArrayIndexJsonPathPart()); + else + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - sql:identity() function must be the final token of the path'"); + } + + parts.Add(new PropertyJsonPathPart(propertyName)); + i = end - 1; + } + else + { + // Error + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - invalid property name at index {i} of '{expression}'"); + } + } + else if (expression[i] == '[') + { + // Start of an array indexer + var end = expression.IndexOf(']', i); + + if (end == -1) + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - missing closing bracket for indexer at index {i} of '{expression}'"); + + var indexStr = expression.Substring(i + 1, end - i - 1); + + if (!UInt32.TryParse(indexStr, out var index)) + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - invalid indexer at index {i} of '{expression}'"); + + parts.Add(new ArrayElementJsonPathPart(index)); + i = end; + } + else + { + // Error + throw new Newtonsoft.Json.JsonException($"Invalid JSON path - invalid token at index {i} of '{expression}'"); + } + } + + return parts.ToArray(); + } + + public override string ToString() + { + return _mode.ToString().ToLowerInvariant() + " " + String.Join("", _parts.Select(p => p.ToString())); + } + + /// + /// Represents a single token within the path + /// + abstract class JsonPathPart + { + /// + /// Extracts the required child token from the current context token + /// + /// The current context token + /// The child token that is matched by this part of the path + public abstract JToken Match(JToken token); + } + + /// + /// Handles the $ sign representing the context item + /// + class ContextJsonPathPart : JsonPathPart + { + public override JToken Match(JToken token) + { + return token; + } + + public override string ToString() + { + return "$"; + } + } + + /// + /// Handles a property (key) name + /// + class PropertyJsonPathPart : JsonPathPart + { + /// + /// Creates a new + /// + /// The key of the property to extract + public PropertyJsonPathPart(string propertyName) + { + PropertyName = propertyName; + } + + /// + /// Returns the key of the property to extract + /// + public string PropertyName { get; } + + public override JToken Match(JToken token) + { + if (!(token is JObject obj)) + return null; + + var prop = obj.Property(PropertyName); + return prop?.Value; + } + + public override string ToString() + { + if ((PropertyName[0] >= 'a' && PropertyName[0] <= 'z' || PropertyName[0] >= 'A' && PropertyName[0] <= 'Z') + && PropertyName.All(ch => ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9')) + return "." + PropertyName; + + return ".\"" + PropertyName.Replace("\\", "\\\\").Replace("\"", "\\\"") + "\""; + } + } + + /// + /// Handles an array element indexer + /// + class ArrayElementJsonPathPart : JsonPathPart + { + /// + /// Creates a new + /// + /// The index of the array to extract + public ArrayElementJsonPathPart(uint index) + { + Index = (int)index; + } + + /// + /// The index of the array to extract + /// + public int Index { get; } + + public override JToken Match(JToken token) + { + if (!(token is JArray arr) || arr.Count <= Index) + return null; + + return arr[Index]; + } + + public override string ToString() + { + return $"[{Index}]"; + } + } + + /// + /// Handles the sql:identity() function to return the index of an element within it's containing array + /// + class ArrayIndexJsonPathPart : JsonPathPart + { + public override JToken Match(JToken token) + { + if (!(token.Parent is JArray arr)) + return null; + + var index = arr.IndexOf(token); + return new JValue(index); + } + + public override string ToString() + { + return ".sql:identity()"; + } + } + } + + enum JsonPathMode + { + Lax, + Strict + } +}