Change to NJsonSchema lib (#415)

* Change to NJsonSchema lib * Catch Exception when Parse JsonSchema * Add meta schema as embedded resource * Update FunctionalTest to ignore meta-schema.json * Add multiple times test for validate
microsoft · Aug 5, 2022 · 2dcf005 · 2dcf005
1 parent e93614b
commit 2dcf005
Show file tree

Hide file tree

Showing 17 changed files with 298 additions and 34 deletions.
diff --git a/data/Templates/Json/Schema/meta-schema.json b/data/Templates/Json/Schema/meta-schema.json
@@ -0,0 +1,150 @@
+{
+    "id": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "positiveInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "positiveIntegerDefault0": {
+            "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ]
+        },
+        "simpleTypes": {
+            "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "minItems": 1,
+            "uniqueItems": true
+        }
+    },
+    "type": "object",
+    "properties": {
+        "id": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": {},
+        "multipleOf": {
+            "type": "number",
+            "minimum": 0,
+            "exclusiveMinimum": true
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "boolean",
+            "default": false
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "boolean",
+            "default": false
+        },
+        "maxLength": { "$ref": "#/definitions/positiveInteger" },
+        "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": {
+            "anyOf": [
+                { "type": "boolean" },
+                { "$ref": "#" }
+            ],
+            "default": {}
+        },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": {}
+        },
+        "maxItems": { "$ref": "#/definitions/positiveInteger" },
+        "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "maxProperties": { "$ref": "#/definitions/positiveInteger" },
+        "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": {
+            "anyOf": [
+                { "type": "boolean" },
+                { "$ref": "#" }
+            ],
+            "default": {}
+        },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "enum": {
+            "type": "array",
+            "minItems": 1,
+            "uniqueItems": true
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "dependencies": {
+        "exclusiveMaximum": [ "maximum" ],
+        "exclusiveMinimum": [ "minimum" ]
+    },
+    "default": {}
+}
diff --git a/...crosoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs b/...crosoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs
@@ -12,7 +12,7 @@
 using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Models.Json;
 using Newtonsoft.Json.Linq;
-using Newtonsoft.Json.Schema;
+using NJsonSchema;
 using Xunit;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids
@@ -23,7 +23,6 @@ public class TemplateLocalFileSystemTests
         public void GivenAValidTemplateDirectory_WhenGetTemplate_CorrectResultsShouldBeReturned()
         {
             var templateLocalFileSystem = new TemplateLocalFileSystem(TestConstants.Hl7v2TemplateDirectory, DataType.Hl7v2);
-            var context = new Context(CultureInfo.InvariantCulture);
 
             // Template exists
             Assert.NotNull(templateLocalFileSystem.GetTemplate("ADT_A01"));
@@ -49,11 +48,10 @@ public void GivenAValidTemplateDirectory_WhenGetTemplateWithContext_CorrectResul
         }
 
         [Fact]
-        public void GivenAValidTemplateDirectory_WhenGetJsonSchemaTemplate_CorrectResultsShouldBeReturned()
+        public async void GivenAValidTemplateDirectory_WhenGetJsonSchemaTemplate_CorrectResultsShouldBeReturned()
         {
             var templateLocalFileSystem = new TemplateLocalFileSystem(Path.Join(TestConstants.TestTemplateDirectory, @"ValidValidateTemplates"), DataType.Hl7v2);
             var testSchemaPath = "Schemas/TestSchema.schema.json";
-            var context = new Context(CultureInfo.InvariantCulture);
 
             var schemaTemplate = templateLocalFileSystem.GetTemplate(testSchemaPath);
 
@@ -64,8 +62,8 @@ public void GivenAValidTemplateDirectory_WhenGetJsonSchemaTemplate_CorrectResult
             Assert.NotNull(jSchemaDocument);
             Assert.NotNull(jSchemaDocument.Schema);
 
-            JSchema expectedJSchema = JSchema.Parse(File.ReadAllText(Path.Join(TestConstants.TestTemplateDirectory, @"ValidValidateTemplates", testSchemaPath)));
-            Assert.True(JToken.DeepEquals(JToken.Parse(jSchemaDocument.Schema.ToString()), JToken.Parse(expectedJSchema.ToString())));
+            JsonSchema expectedJSchema = await JsonSchema.FromJsonAsync(File.ReadAllText(Path.Join(TestConstants.TestTemplateDirectory, @"ValidValidateTemplates", testSchemaPath)));
+            Assert.True(JToken.DeepEquals(JToken.Parse(jSchemaDocument.Schema.ToJson()), JToken.Parse(expectedJSchema.ToJson())));
         }
 
         [Fact]

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/ValidateTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/ValidateTests.cs
@@ -88,6 +88,43 @@ public void GivenValidValidateMatchedTemplateContent_WhenParseAndRender_CorrectR
             Assert.True(JToken.DeepEquals(expectedObject, actualObject));
         }
 
+        [Theory]
+        [MemberData(nameof(GetValidValidateMatchedTemplateContents))]
+        public void GivenValidValidateMatchedTemplateContent_WhenRenderMultipleTimes_CorrectResultShouldBeReturned(string templateContent, string expectedResult)
+        {
+            int repeatCount = 100000;
+
+            // Template should be parsed correctly
+            var template = TemplateUtility.ParseLiquidTemplate(TemplateName, templateContent);
+
+            var templateFolder = Path.Join(TestConstants.TestTemplateDirectory, @"ValidValidateTemplates");
+            var parser = new JsonDataParser();
+            var inputContent = "{\"id\": \"0\",\"valueReference\" : \"testReference\",\"resourceType\" : \"Patient\",\"name\":{\"valueString\" : \"valueString\"}}";
+
+            var templateProvider = new TemplateProvider(templateFolder, DataType.Json);
+            var jsonData = parser.Parse(inputContent);
+            var dictionary = new Dictionary<string, object> { { "msg", jsonData } };
+            var context = new Context(
+                environments: new List<Hash> { Hash.FromDictionary(dictionary) },
+                outerScope: new Hash(),
+                registers: Hash.FromDictionary(new Dictionary<string, object>() { { "file_system", templateProvider.GetTemplateFileSystem() } }),
+                errorsOutputMode: ErrorsOutputMode.Rethrow,
+                maxIterations: 0,
+                timeout: 0,
+                formatProvider: CultureInfo.InvariantCulture);
+            context.AddFilters(typeof(Filters));
+
+            string result = string.Empty;
+            for (int i = 0; i < repeatCount; i++)
+            {
+                result = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));
+            }
+
+            var expectedObject = JObject.Parse(expectedResult);
+            var actualObject = JObject.Parse(result);
+            Assert.True(JToken.DeepEquals(expectedObject, actualObject));
+        }
+
         [Theory]
         [MemberData(nameof(GetValidValidateMatchedTemplateWithSchemaContents))]
         public void GivenValidValidateMatchedTemplateContent_WithJsonContext_WhenParseAndRender_InvolvedSchemaShouldBeReturned(string templateContent, List<string> expectSchemaFiles)
@@ -120,7 +157,7 @@ public void GivenValidValidateMatchedTemplateContent_WithJsonContext_WhenParseAn
             for (int i = 0; i < expectSchemaFiles.Count; i++)
             {
                 var expectedSchemaObject = JObject.Parse(File.ReadAllText(Path.Join(TestConstants.TestTemplateDirectory, expectSchemaFiles[i])));
-                var actualSchemaObject = JObject.Parse(context.ValidateSchemas[i].ToString());
+                var actualSchemaObject = JObject.Parse(context.ValidateSchemas[i].ToJson());
                 Assert.True(JToken.DeepEquals(expectedSchemaObject, actualSchemaObject));
             }
         }

diff --git a/...sts/TestData/TestTemplates/InvalidValidateTemplates/Schemas/InvalidTestSchema.schema.json b/...sts/TestData/TestTemplates/InvalidValidateTemplates/Schemas/InvalidTestSchema.schema.json
@@ -1,3 +1,3 @@
 {
-    "type":  "invalidType"
+    "type": "InvalidType"
 }
diff --git a/...er.UnitTests/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSchema.schema.json b/...er.UnitTests/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSchema.schema.json
@@ -1,4 +1,5 @@
 {
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "title": "Patient customized schema",
     "type": "object",
     "properties": {

diff --git a/...r.UnitTests/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSchema2.schema.json b/...r.UnitTests/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSchema2.schema.json
@@ -1,4 +1,5 @@
 {
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "title": "Patient customized schema",
     "type": "object",
     "properties": {

diff --git a/...s/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSubPropertySchema.schema.json b/...s/TestData/TestTemplates/ValidValidateTemplates/Schemas/TestSubPropertySchema.schema.json
@@ -1,4 +1,5 @@
 {
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "title": "Patient customized schema",
     "type": "object",
     "properties": {

diff --git a/...er.UnitTests/TestData/TestTemplates/ValidValidateTemplates/ValidUnmatchedTemplate3.liquid b/...er.UnitTests/TestData/TestTemplates/ValidValidateTemplates/ValidUnmatchedTemplate3.liquid
@@ -1,4 +1,4 @@
-{% validate "Schemas/TestSchema3.schema.json" -%}
+{% validate "Schemas/TestSchema2.schema.json" -%}
 {
     "resourceType": "{{ msg.resourceType }}"
 }

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Utilities/TemplateUtilityTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Utilities/TemplateUtilityTests.cs
@@ -118,5 +118,26 @@ public void GivenInvalidCcdaTemplateContents_WhenParseTemplates_ExceptionsShould
             exception = Assert.Throws<TemplateLoadException>(() => TemplateUtility.ParseTemplates(templates));
             Assert.Equal(FhirConverterErrorCode.InvalidCodeMapping, exception.FhirConverterErrorCode);
         }
+
+        [Fact]
+        public void GivenInvalidJsonSchemaTemplateContents_WhenParseTemplates_ExceptionsShouldBeThrown()
+        {
+            // Invalid schema content
+            var templates = new Dictionary<string, string> { { "InvalidSchema.schema.json", @"{""type"": ""InvalidType"" }" } };
+            var exception = Assert.Throws<TemplateLoadException>(() => TemplateUtility.ParseTemplates(templates));
+            Assert.Equal(FhirConverterErrorCode.InvalidJsonSchema, exception.FhirConverterErrorCode);
+
+            // Invalid JSON
+            templates = new Dictionary<string, string> { { "InvalidSchema.schema.json", @"{""a""" } };
+            exception = Assert.Throws<TemplateLoadException>(() => TemplateUtility.ParseTemplates(templates));
+            Assert.Equal(FhirConverterErrorCode.InvalidJsonSchema, exception.FhirConverterErrorCode);
+
+            // Null or empty schema
+            templates = new Dictionary<string, string> { { "InvalidSchema.schema.json", string.Empty } };
+            exception = Assert.Throws<TemplateLoadException>(() => TemplateUtility.ParseTemplates(templates));
+
+            templates = new Dictionary<string, string> { { "InvalidSchema.schema.json", null} };
+            exception = Assert.Throws<TemplateLoadException>(() => TemplateUtility.ParseTemplates(templates));
+        }
     }
 }
diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/Validate.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/Validate.cs
@@ -5,6 +5,7 @@
 
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using DotLiquid;
 using DotLiquid.Exceptions;
@@ -13,7 +14,7 @@
 using Microsoft.Health.Fhir.Liquid.Converter.Models.Json;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using Newtonsoft.Json.Schema;
+using NJsonSchema;
 using RenderException = Microsoft.Health.Fhir.Liquid.Converter.Exceptions.RenderException;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.DotLiquids
@@ -50,7 +51,7 @@ public override void Initialize(string tagName, string markup, List<string> toke
 
         public override void Render(Context context, TextWriter result)
         {
-            JSchema validateSchema = LoadValidateSchema(context);
+            JsonSchema validateSchema = LoadValidateSchema(context);
 
             if (context is JSchemaContext jSchemaContext)
             {
@@ -76,15 +77,17 @@ public override void Render(Context context, TextWriter result)
                 throw new RenderException(FhirConverterErrorCode.InvalidValidateBlockContent, string.Format(Resources.InvalidValidateContentBlock, ex.Message));
             }
 
-            if (!validateObject.IsValid(validateSchema, out IList<string> messages))
+            var errors = validateSchema.Validate(validateObject);
+
+            if (errors.Any())
             {
-                throw new RenderException(FhirConverterErrorCode.UnmatchedValidateBlockContent, string.Format(Resources.UnMatchedValidateBlockContent, _schemaFileName, string.Join(";", messages)));
+                throw new RenderException(FhirConverterErrorCode.UnmatchedValidateBlockContent, string.Format(Resources.UnMatchedValidateBlockContent, _schemaFileName, string.Join(";", errors)));
             }
 
             result.Write(validateContent);
         }
 
-        private JSchema LoadValidateSchema(Context context)
+        private JsonSchema LoadValidateSchema(Context context)
         {
             if (!(context.Registers["file_system"] is IFhirConverterTemplateFileSystem fileSystem))
             {

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Microsoft.Health.Fhir.Liquid.Converter.csproj b/src/Microsoft.Health.Fhir.Liquid.Converter/Microsoft.Health.Fhir.Liquid.Converter.csproj
@@ -13,7 +13,7 @@
     <PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.9" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.9" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="Newtonsoft.Json.Schema" Version="3.0.14" />
+    <PackageReference Include="NJsonSchema" Version="10.7.2" />
   </ItemGroup>
 
   <ItemGroup>
@@ -28,6 +28,10 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+	<EmbeddedResource  Include="..\..\data\Templates\Json\Schema\meta-schema.json" Link="meta-schema.json" />
+  </ItemGroup>
+
   <ItemGroup>
     <Compile Update="Resources.Designer.cs">
       <DesignTime>True</DesignTime>

diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Models/Json/JSchemaContext.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Models/Json/JSchemaContext.cs
@@ -6,7 +6,7 @@
 using System;
 using System.Collections.Generic;
 using DotLiquid;
-using Newtonsoft.Json.Schema;
+using NJsonSchema;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Models.Json
 {
@@ -15,9 +15,9 @@ public class JSchemaContext : Context
         public JSchemaContext(List<Hash> environments, Hash outerScope, Hash registers, ErrorsOutputMode errorsOutputMode, int maxIterations, int timeout, IFormatProvider formatProvider)
              : base(environments, outerScope, registers, errorsOutputMode, maxIterations, timeout, formatProvider)
         {
-            ValidateSchemas = new List<JSchema>();
+            ValidateSchemas = new List<JsonSchema>();
         }
 
-        public List<JSchema> ValidateSchemas { get; set; }
+        public List<JsonSchema> ValidateSchemas { get; set; }
     }
 }