From 3c54c01cc91fc10b0900f545f1bbdb780ada31b9 Mon Sep 17 00:00:00 2001 From: mmsmits Date: Fri, 8 Dec 2023 14:18:13 +0100 Subject: [PATCH 1/3] Move manifest tests to newest format. Remove "current validator". --- firely-validator-api-tests.props | 1 + .../FhirTestCases | 2 +- .../FhirTests/CurrentValidator.cs | 72 - .../{WipValidator.cs => DotNetValidator.cs} | 48 +- .../FhirTests/ITestValidator.cs | 4 +- .../FhirTests/Manifest.cs | 117 +- .../FhirTests/TestCaseRunner.cs | 142 +- .../FhirTests/ValidationManifestTest.cs | 20 +- ...idation.Compilation.Tests.Shared.projitems | 1766 +---------------- 9 files changed, 248 insertions(+), 1924 deletions(-) delete mode 100644 test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/CurrentValidator.cs rename test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/{WipValidator.cs => DotNetValidator.cs} (74%) diff --git a/firely-validator-api-tests.props b/firely-validator-api-tests.props index a84cbccc..2d4bff02 100644 --- a/firely-validator-api-tests.props +++ b/firely-validator-api-tests.props @@ -30,6 +30,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTestCases b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTestCases index b785dbc1..d3a31ee4 160000 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTestCases +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTestCases @@ -1 +1 @@ -Subproject commit b785dbc13974310cc8b7981def72b0e7cdb2c92b +Subproject commit d3a31ee4f594339de15bec3ee96d9ca5f8911064 diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/CurrentValidator.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/CurrentValidator.cs deleted file mode 100644 index b04bcb67..00000000 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/CurrentValidator.cs +++ /dev/null @@ -1,72 +0,0 @@ -using Hl7.Fhir.ElementModel; -using Hl7.Fhir.Model; -using Hl7.Fhir.Specification.Source; -using System; -using System.Diagnostics; -using System.Linq; - -namespace Firely.Fhir.Validation.Compilation.Tests -{ - internal class CurrentValidator : ITestValidator - { - private static readonly IResourceResolver BASE_RESOLVER = new CachedResolver(ZipSource.CreateValidationSource()); - - private readonly string[] _unsupportedTests = new[] - { - // these tests are not FHIR resources, but CDA resource. We cannot handle at the moment. - "cda/example", "cda/example-no-styles", - - // do not run an Empty testcase - ValidationManifestDataSourceAttribute.EMPTY_TESTCASE_NAME - }; - - private readonly Stopwatch _stopWatch; - - public static ITestValidator Create() => new CurrentValidator(); - - public string Name => "Current"; - - public string[] UnvalidatableTests => _unsupportedTests; - - public CurrentValidator(Stopwatch? stopwatch = null) - { - _stopWatch = stopwatch ?? new(); - } - - public ExpectedResult? GetExpectedResults(IValidatorEnginesResults engine) => engine.FirelySDKCurrent; - public void SetExpectedResults(IValidatorEnginesResults engine, ExpectedResult result) => engine.FirelySDKCurrent = result; - - /// - /// Validation engine of the current Firely SDK (2.x) - /// - public OperationOutcome Validate(ITypedElement instance, IResourceResolver? resolver, string? profile = null) - { - // This code needs the new shims, and no longer compiles since the old validator has been removed from the SDK. - throw new NotImplementedException(); - - //var extendeResolver = resolver is null ? BASE_RESOLVER : new SnapshotSource(new MultiResolver(BASE_RESOLVER, resolver)); - - //var settings = new ValidationSettings - //{ - // GenerateSnapshot = true, - // GenerateSnapshotSettings = SnapshotGeneratorSettings.CreateDefault(), - // ResourceResolver = extendeResolver, - // TerminologyService = new LocalTerminologyService(extendeResolver.AsAsync()), - //}; - - //var validator = new Hl7.Fhir.Validation.Validator(settings); - - //_stopWatch.Start(); - - //#if STU3 - // var outcome = profile is null ? validator.Validate(instance) : validator.Validate(instance, profile); - //#else - // var outcome = profile is null ? validator.Validate(instance, ModelInfo.ModelInspector) : validator.Validate(instance, ModelInfo.ModelInspector, profile); - //#endif - // _stopWatch.Stop(); - // return outcome.RemoveDuplicateMessages(); - } - - public bool CannotValidateTest(TestCase c) => UnvalidatableTests.Contains(c.Name) && ModelInfo.CheckMinorVersionCompatibility(c.Version ?? "5.0"); - } -} diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/WipValidator.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/DotNetValidator.cs similarity index 74% rename from test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/WipValidator.cs rename to test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/DotNetValidator.cs index af4e2ba6..9d00e70e 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/WipValidator.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/DotNetValidator.cs @@ -1,5 +1,6 @@ using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; using Hl7.Fhir.Specification.Source; using Hl7.Fhir.Specification.Terminology; using Hl7.Fhir.Support; @@ -8,11 +9,13 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; using static Hl7.Fhir.Model.OperationOutcome; namespace Firely.Fhir.Validation.Compilation.Tests { - internal class WipValidator : ITestValidator + internal class DotNetValidator : ITestValidator { private readonly string[] _unsupportedTests = new[] { @@ -25,21 +28,54 @@ internal class WipValidator : ITestValidator private static readonly IResourceResolver BASE_RESOLVER = new CachedResolver(new StructureDefinitionCorrectionsResolver(ZipSource.CreateValidationSource())); private static readonly IElementSchemaResolver SCHEMA_RESOLVER = StructureDefinitionToElementSchemaResolver.CreatedCached(BASE_RESOLVER.AsAsync()); private readonly Stopwatch _stopWatch; + private readonly JsonSerializerOptions _serializerOptions; - public WipValidator(Stopwatch? stopwatch = null) + public DotNetValidator(Stopwatch? stopwatch = null) { _stopWatch = stopwatch ?? new(); + _serializerOptions = new JsonSerializerOptions().ForFhir(ModelInfo.ModelInspector).Pretty(); } - public static ITestValidator Create() => new WipValidator(); + public static ITestValidator Create() => new DotNetValidator(); - public string Name => "Wip"; + public string Name => "dotnet-current"; public string[] UnvalidatableTests => _unsupportedTests; public bool CannotValidateTest(TestCase c) => UnvalidatableTests.Contains(c.Name); - public ExpectedResult? GetExpectedResults(IValidatorEnginesResults engine) => engine.FirelySDKWip; - public void SetExpectedResults(IValidatorEnginesResults engine, ExpectedResult result) => engine.FirelySDKWip = result; + + public OperationOutcome? GetExpectedOperationOutcome(IValidatorEnginesResults engine) + { + var json = engine.FirelyDotNet?.Outcome; + if (json is null) + { + return null; + } + else + { + try + { + return JsonSerializer.Deserialize(json.ToJsonString(), _serializerOptions); + } + catch (DeserializationFailedException e) + { + return e.PartialResult as OperationOutcome; + } + } + } + + public void SetOperationOutcome(IValidatorEnginesResults engine, OperationOutcome outcome) + { + var oo = JsonSerializer.Serialize(outcome, _serializerOptions); + if (engine.FirelyDotNet is null) + { + engine.FirelyDotNet = new ExpectedOutcome() { Outcome = JsonNode.Parse(oo)?.AsObject() }; + } + else + { + engine.FirelyDotNet.Outcome = JsonNode.Parse(oo)?.AsObject(); + } + } /// /// Validator engine based in this solution: the work in progress (wip) validator diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ITestValidator.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ITestValidator.cs index 7aaf3b34..ea52eeac 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ITestValidator.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ITestValidator.cs @@ -28,8 +28,8 @@ internal interface ITestValidator OperationOutcome Validate(ITypedElement instance, IResourceResolver? resolver, string? profile = null); - ExpectedResult? GetExpectedResults(IValidatorEnginesResults engine); + OperationOutcome? GetExpectedOperationOutcome(IValidatorEnginesResults engine); - void SetExpectedResults(IValidatorEnginesResults engine, ExpectedResult result); + void SetOperationOutcome(IValidatorEnginesResults engine, OperationOutcome outcome); } } diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/Manifest.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/Manifest.cs index 4081dd39..c348a3e9 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/Manifest.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/Manifest.cs @@ -5,17 +5,26 @@ */ using System.Collections.Generic; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace Firely.Fhir.Validation.Compilation.Tests { public interface IValidatorEnginesResults { - public ExpectedResult? Java { get; set; } + public ExpectedOutcome? Java { get; set; } - public ExpectedResult? FirelySDKCurrent { get; set; } + public ExpectedOutcome? FirelyDotNet { get; set; } + } - public ExpectedResult? FirelySDKWip { get; set; } + public class ExpectedOutcome + { + [JsonPropertyName("outcome")] + public JsonObject? Outcome { get; set; } + [JsonPropertyName("comment")] + public string? Comment { get; set; } + [JsonPropertyName("explanation")] + public string? Explanation { get; set; } } public class ExpectedResult @@ -50,6 +59,9 @@ public class Profile : IValidatorEnginesResults [JsonPropertyName("source")] public string? Source { get; set; } + [JsonPropertyName("packages")] + public string[]? Packages { get; set; } + // Is this needed? Should be a part of ExpectedResult [JsonPropertyName("errorCount")] public int? ErrorCount { get; set; } @@ -61,13 +73,10 @@ public class Profile : IValidatorEnginesResults public List? Supporting { get; set; } [JsonPropertyName("java")] - public ExpectedResult? Java { get; set; } + public ExpectedOutcome? Java { get; set; } - [JsonPropertyName("firely-sdk-current")] - public ExpectedResult? FirelySDKCurrent { get; set; } - - [JsonPropertyName("firely-sdk-wip")] - public ExpectedResult? FirelySDKWip { get; set; } + [JsonPropertyName("firely-dotnet")] + public ExpectedOutcome? FirelyDotNet { get; set; } } public class BundleParameter @@ -90,17 +99,17 @@ public class Logical [JsonPropertyName("expressions")] public string[]? Expressions { get; set; } - [JsonPropertyName("explanation")] - public string? Explanation { get; set; } + [JsonPropertyName("format")] + public string? Format { get; set; } [JsonPropertyName("java")] - public ExpectedResult? Java { get; set; } + public ExpectedOutcome? Java { get; set; } [JsonPropertyName("firely-sdk-current")] public ExpectedResult? FirelySDKCurrent { get; set; } - [JsonPropertyName("firely-sdk-wip")] - public ExpectedResult? FirelySDKWip { get; set; } + [JsonPropertyName("firely-dotnet")] + public ExpectedOutcome? FirelyDotNet { get; set; } } public class TestCase : IValidatorEnginesResults @@ -111,15 +120,54 @@ public class TestCase : IValidatorEnginesResults [JsonPropertyName("file")] public string? FileName { get; set; } + [JsonPropertyName("default-version")] + public bool? DefaultVersion { get; set; } + [JsonPropertyName("description")] public string? Description { get; set; } + [JsonPropertyName("documentation")] + public string? Documentation { get; set; } + + [JsonPropertyName("noHtmlInMarkDown")] + public bool? NoHtmlInMarkdown { get; set; } + + [JsonPropertyName("for-publication")] + public string? ForPublication { get; set; } + [JsonPropertyName("fetcher")] public string? Fetcher { get; set; } + [JsonPropertyName("module")] + public string? Module { get; set; } + + [JsonPropertyName("allow-comments")] + public bool? AllowComments { get; set; } + + [JsonPropertyName("no-tx")] + public bool? NoTx { get; set; } + + [JsonPropertyName("ips")] + public string? Ips { get; set; } + + [JsonPropertyName("close-up")] + public bool? CloseUp { get; set; } + + [JsonPropertyName("packages")] + public List? Packages { get; set; } + [JsonPropertyName("version")] public string? Version { get; set; } + [JsonPropertyName("supporting")] + public List? Supporting { get; set; } + + [JsonPropertyName("validateReference")] + public string? ValidateReference { get; set; } + + [JsonPropertyName("validateContains")] + public string? ValidateContains { get; set; } + // Is this needed? [JsonPropertyName("x-errors")] public string[]? XErrors { get; set; } @@ -146,11 +194,6 @@ public class TestCase : IValidatorEnginesResults [JsonPropertyName("language")] public string? Language { get; set; } - [JsonPropertyName("packages")] - public List? Packages { get; set; } - - [JsonPropertyName("supporting")] - public List? Supporting { get; set; } [JsonPropertyName("error")] public string? Error { get; set; } @@ -182,9 +225,6 @@ public class TestCase : IValidatorEnginesResults [JsonPropertyName("crumb-trail")] public bool? CrumbTrail { get; set; } - [JsonPropertyName("profile")] - public Profile? Profile { get; set; } - [JsonPropertyName("explanation")] public string? Explanation { get; set; } @@ -192,13 +232,16 @@ public class TestCase : IValidatorEnginesResults public Logical? Logical { get; set; } [JsonPropertyName("java")] - public ExpectedResult? Java { get; set; } + public ExpectedOutcome? Java { get; set; } [JsonPropertyName("firely-sdk-current")] public ExpectedResult? FirelySDKCurrent { get; set; } - [JsonPropertyName("firely-sdk-wip")] - public ExpectedResult? FirelySDKWip { get; set; } + [JsonPropertyName("firely-dotnet")] + public ExpectedOutcome? FirelyDotNet { get; set; } + + [JsonPropertyName("profile")] + public Profile? Profile { get; set; } } public class Manifest @@ -206,7 +249,31 @@ public class Manifest [JsonPropertyName("documentation")] public List? Documentation { get; set; } + [JsonPropertyName("modules")] + public Modules? Modules { get; set; } + [JsonPropertyName("test-cases")] public List? TestCases { get; set; } } + + public class Modules + { + [JsonPropertyName("(default)")] + public string? Default { get; set; } + + [JsonPropertyName("cda")] + public string? Cda { get; set; } + + [JsonPropertyName("cdshooks")] + public string? CdsHooks { get; set; } + + [JsonPropertyName("shc")] + public string? Hhc { get; set; } + + [JsonPropertyName("notes:")] + public string? Notes { get; set; } + + [JsonPropertyName("json5")] + public string? Json5 { get; set; } + } } diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs index b9d5a6f0..63384196 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs @@ -1,4 +1,5 @@ -using FluentAssertions; +using Firely.Fhir.Packages; +using FluentAssertions; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -28,7 +29,14 @@ public enum AssertionOptions internal class TestCaseRunner { - private static readonly StructureDefinitionSummaryProvider SDPROVIDER = new(new CachedResolver(ZipSource.CreateValidationSource())); + private readonly StructureDefinitionSummaryProvider _sdprovider; + private readonly string _testPath; + + internal TestCaseRunner(string testpath) + { + _testPath = testpath; + _sdprovider = new(new CachedResolver(new MultiResolver(ZipSource.CreateValidationSource(), new DirectorySource(contentDirectory: _testPath)))); + } public (OperationOutcome, OperationOutcome?) RunTestCase(TestCase testCase, ITestValidator engine, string baseDirectory, AssertionOptions options = AssertionOptions.OutputTextAssertion) => RunTestCaseAsync(testCase, engine, baseDirectory, options); @@ -38,29 +46,71 @@ internal class TestCaseRunner if (engine.CannotValidateTest(testCase)) return (new OperationOutcome(), default(OperationOutcome)); var absolutePath = Path.GetFullPath(baseDirectory); - var testResource = parseResource(Path.Combine(absolutePath, testCase.FileName!)); - OperationOutcome? outcomeWithProfile = null; - if (testCase.Profile?.Source is { } source) + OperationOutcome outcome; + ITypedElement? testResource = null; + try { - var profileResource = parseResource(Path.Combine(absolutePath, source)); - var profileUri = profileResource?.InstanceType == "StructureDefinition" ? profileResource.Children("url").SingleOrDefault()?.Value as string : null; - - Assert.IsNotNull(profileUri, $"Could not find url in profile {source}"); - - var supportingFiles = (testCase.Profile.Supporting ?? Enumerable.Empty()) - .Concat(testCase.Supporting ?? Enumerable.Empty()) - .Concat(testCase.Profiles ?? Enumerable.Empty()) - .Concat(new[] { source }); - var resolver = buildTestContextResolver(absolutePath, supportingFiles); - outcomeWithProfile = engine.Validate(testResource, resolver, profileUri); - assertResult(engine.GetExpectedResults(testCase.Profile), outcomeWithProfile, options); + testResource = parseResource(Path.Combine(absolutePath, testCase.FileName!)); + var supportFiles = (testCase.Supporting ?? Enumerable.Empty()).Concat(testCase.Profiles ?? Enumerable.Empty()); + var contextResolver = buildTestContextResolver(absolutePath, supportFiles, testCase.Packages); + + outcome = engine.Validate(testResource, contextResolver, null); + assertResult(engine.GetExpectedOperationOutcome(testCase), outcome, options); + } + catch (Exception e) + { + if (e is InvalidOperationException || e is FormatException) + outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message } } }; + else if (e is FileNotFoundException) + { + //file is not found, so we can't run the test + outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" } } }; + } + else + throw; } - var supportFiles = (testCase.Supporting ?? Enumerable.Empty()).Concat(testCase.Profiles ?? Enumerable.Empty()); - var contextResolver = buildTestContextResolver(absolutePath, supportFiles); - OperationOutcome outcome = engine.Validate(testResource, contextResolver, null); - assertResult(engine.GetExpectedResults(testCase), outcome, options); + OperationOutcome? outcomeWithProfile = null; + if (testResource is not null && testCase.Profile?.Source is { } source) + { + try + { + string? profileUri; + + if (source.StartsWith("http://")) //this is a canonical + { + profileUri = source; + } + else //we think this is a reference to a local file + { + var profileResource = parseResource(Path.Combine(absolutePath, source)); + profileUri = profileResource?.InstanceType == "StructureDefinition" ? profileResource.Children("url").SingleOrDefault()?.Value as string : null; + } + + Assert.IsNotNull(profileUri, $"Could not find url in profile {source}"); + + var supportingFiles = (testCase.Profile.Supporting ?? Enumerable.Empty()) + .Concat(testCase.Supporting ?? Enumerable.Empty()) + .Concat(testCase.Profiles ?? Enumerable.Empty()) + .Concat(new[] { source }); + var resolver = buildTestContextResolver(absolutePath, supportingFiles, testCase.Profile.Packages); + outcomeWithProfile = engine.Validate(testResource, resolver, profileUri); + assertResult(engine.GetExpectedOperationOutcome(testCase.Profile), outcomeWithProfile, options); + } + catch (Exception e) + { + if (e is System.InvalidOperationException || e is FormatException) + outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message } } }; + else if (e is System.IO.FileNotFoundException) + { + //file is not found, so we can't run the test + outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" } } }; + } + else + throw; + } + } return (outcome, outcomeWithProfile); } @@ -82,10 +132,10 @@ public void AddOrEditValidatorResults(string manifestFileName, IEnumerable supportingFiles) + private static IResourceResolver? buildTestContextResolver(string baseDirectory, IEnumerable supportingFiles, IEnumerable? packages = null) { + if (!supportingFiles.Any() && (packages is null || !packages.Any())) + { + return null; + } + var resolver = new MultiResolver(); + if (supportingFiles.Any()) { - // build a resolver made only for this test var testContextResolver = new DirectorySource( - baseDirectory, - new DirectorySourceSettings { Includes = supportingFiles.ToArray(), IncludeSubDirectories = true } - ); - return testContextResolver; + baseDirectory, + new DirectorySourceSettings { Includes = supportingFiles.ToArray(), IncludeSubDirectories = true } + ); + resolver.AddSource(testContextResolver); } - return null; + if (packages?.Any() == true) + { + var packageResolver = new FhirPackageSource(ModelInfo.ModelInspector, packages.ToArray()); + resolver.AddSource(packageResolver); + } + + return resolver; } - private static void assertResult(ExpectedResult? result, OperationOutcome outcome, AssertionOptions options) + private static void assertResult(OperationOutcome? result, OperationOutcome outcome, AssertionOptions options) { + if (options.HasFlag(AssertionOptions.NoAssertion)) return; // no assertion asked outcome.RemoveDuplicateMessages(); result.Should().NotBeNull("There should be an expected result"); - Assert.AreEqual(result!.ErrorCount ?? 0, outcome.Errors + outcome.Fatals, errorsWarnings(result, outcome)); - Assert.AreEqual(result.WarningCount ?? 0, outcome.Warnings, errorsWarnings(result, outcome)); + Assert.AreEqual(result?.Fatals ?? 0, outcome.Fatals, errorsWarnings(result, outcome)); + Assert.AreEqual(result?.Errors ?? 0, outcome.Errors, errorsWarnings(result, outcome)); + Assert.AreEqual(result?.Warnings ?? 0, outcome.Warnings, errorsWarnings(result, outcome)); if (options.HasFlag(AssertionOptions.OutputTextAssertion)) { - outcome.Issue.Select(i => i.ToString()).ToList().Should().BeEquivalentTo(result.Output ?? new()); + outcome.Issue.Select(i => i.ToString()).ToList().Should().BeEquivalentTo(result?.Issue.Select(i => i.ToString()).ToList() ?? new()); } - static string errorsWarnings(ExpectedResult expected, OperationOutcome actual) => - $"Errors: {actual.Errors + actual.Fatals} (expected {expected.ErrorCount}), " + - $"Warnings: {actual.Warnings} (expected {expected.WarningCount}) - {actual}"; + static string errorsWarnings(OperationOutcome? expected, OperationOutcome actual) => + $"Fatals: {actual.Fatals} (expected {expected?.Fatals ?? 0}), " + + $"Errors: {actual.Errors + actual.Fatals} (expected {expected?.Errors ?? 0}), " + + $"Warnings: {actual.Warnings} (expected {expected?.Warnings ?? 0}) - {actual}"; } private ITypedElement parseResource(string fileName) { var resourceText = File.ReadAllText(fileName); return fileName.EndsWith(".xml") - ? FhirXmlNode.Parse(resourceText).ToTypedElement(SDPROVIDER) - : FhirJsonNode.Parse(resourceText).ToTypedElement(SDPROVIDER); + ? FhirXmlNode.Parse(resourceText).ToTypedElement(_sdprovider) + : FhirJsonNode.Parse(resourceText).ToTypedElement(_sdprovider); } } diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs index 6baf5fff..9f6b160d 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs @@ -21,7 +21,7 @@ public class ValidationManifestTest private const string TEST_CASES_BASE_PATH = TESTPROJECT_BASE_PATH + @"FhirTestCases\validator"; private const string TEST_CASES_MANIFEST = TEST_CASES_BASE_PATH + @"\manifest.json"; private const string DOC_COMPOSITION_TEST_CASES_MANIFEST = TESTPROJECT_BASE_PATH + @"TestData\DocumentComposition\manifest.json"; - private readonly TestCaseRunner _runner = new(); + private readonly TestCaseRunner _runner = new(TEST_CASES_BASE_PATH); /// @@ -32,7 +32,7 @@ public class ValidationManifestTest [DataTestMethod] [ValidationManifestDataSource(TEST_CASES_MANIFEST, singleTest: "mr-m-simple-nossystem")] public void RunSingleTest(TestCase testCase, string baseDirectory) - => _runner.RunTestCase(testCase, WipValidator.Create(), baseDirectory); + => _runner.RunTestCase(testCase, DotNetValidator.Create(), baseDirectory); /// /// Running the testcases from the repo https://github.com/FHIR/fhir-test-cases, using the Firely SDK expectation. @@ -41,23 +41,14 @@ public void RunSingleTest(TestCase testCase, string baseDirectory) /// the single testcase to run /// the base directory of the testcase [DataTestMethod] -#if R5 - [Ignore("TODO: Make this work for R5 as well.")] -#endif [ValidationManifestDataSource(TEST_CASES_MANIFEST)] public void RunFirelySdkWipTests(TestCase testCase, string baseDirectory) - => _runner.RunTestCase(testCase, WipValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); - - [DataTestMethod] - [Ignore("Until we have ported the old SDK-style validator, this test cannot be run anymore")] - [ValidationManifestDataSource(TEST_CASES_MANIFEST)] - public void RunFirelySdkCurrentTests(TestCase testCase, string baseDirectory) - => _runner.RunTestCase(testCase, CurrentValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); + => _runner.RunTestCase(testCase, DotNetValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); [DataTestMethod] [ValidationManifestDataSource(DOC_COMPOSITION_TEST_CASES_MANIFEST)] public void OldExamples(TestCase testCase, string baseDirectory) - => _runner.RunTestCase(testCase, WipValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); + => _runner.RunTestCase(testCase, DotNetValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); /// @@ -72,9 +63,8 @@ public void OldExamples(TestCase testCase, string baseDirectory) /// that method /// [TestMethod] - [Ignore] public void AddFirelySdkValidatorResults() - => _runner.AddOrEditValidatorResults(TEST_CASES_MANIFEST, new[] { CurrentValidator.Create(), WipValidator.Create() }); + => _runner.AddOrEditValidatorResults(TEST_CASES_MANIFEST, new[] { DotNetValidator.Create() }); [TestMethod] public void RoundTripTest() diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/Firely.Fhir.Validation.Compilation.Tests.Shared.projitems b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/Firely.Fhir.Validation.Compilation.Tests.Shared.projitems index e472bdf7..d5b00859 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/Firely.Fhir.Validation.Compilation.Tests.Shared.projitems +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/Firely.Fhir.Validation.Compilation.Tests.Shared.projitems @@ -9,1768 +9,7 @@ Firely.Fhir.Validation.Compilation.Tests.Shared - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Designer - - - Designer - - - Designer - - - Designer - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1794,14 +33,13 @@ - - + From d34810d1adeda60dd10ac8687e72c14468873ad2 Mon Sep 17 00:00:00 2001 From: mmsmits Date: Mon, 11 Dec 2023 13:29:47 +0100 Subject: [PATCH 2/3] minor improvements --- .../FhirTests/TestCaseRunner.cs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs index 77f9a0ef..4bf6ba90 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/TestCaseRunner.cs @@ -58,18 +58,16 @@ internal TestCaseRunner(string testpath) outcome = engine.Validate(testResource, contextResolver, null); assertResult(engine.GetExpectedOperationOutcome(testCase), outcome, options); } - catch (Exception e) + catch (Exception e) when (e is InvalidOperationException || e is FormatException) { - if (e is InvalidOperationException || e is FormatException) - outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message } } }; - else if (e is FileNotFoundException) - { - //file is not found, so we can't run the test - outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" } } }; - } - else - throw; + outcome = new OperationOutcome() { Issue = [new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message }] }; } + catch (Exception e) when (e is FileNotFoundException) + { + //file is not found, so we can't run the test + outcome = new OperationOutcome() { Issue = [new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" }] }; + } + OperationOutcome? outcomeWithProfile = null; if (testResource is not null && testCase.Profile?.Source is { } source) @@ -101,11 +99,11 @@ internal TestCaseRunner(string testpath) catch (Exception e) { if (e is System.InvalidOperationException || e is FormatException) - outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message } } }; + outcome = new OperationOutcome() { Issue = [new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.Invalid, Diagnostics = e.Message }] }; else if (e is System.IO.FileNotFoundException) { //file is not found, so we can't run the test - outcome = new OperationOutcome() { Issue = new List() { new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" } } }; + outcome = new OperationOutcome() { Issue = [new() { Severity = OperationOutcome.IssueSeverity.Fatal, Code = OperationOutcome.IssueType.NotFound, Diagnostics = $"File not found: {e.Message}" }] }; } else throw; @@ -153,7 +151,7 @@ public void AddOrEditValidatorResults(string manifestFileName, IEnumerable supportingFiles, IEnumerable? packages = null) + private static MultiResolver? buildTestContextResolver(string baseDirectory, IEnumerable supportingFiles, IEnumerable? packages = null) { if (!supportingFiles.Any() && (packages is null || !packages.Any())) { From 4b6b3dbd3a397e80ee649021509eb92495e37be0 Mon Sep 17 00:00:00 2001 From: mmsmits Date: Mon, 11 Dec 2023 13:57:08 +0100 Subject: [PATCH 3/3] old examples are not tested anymore --- .../FhirTests/ValidationManifestTest.cs | 6 +----- .../TestData/DocumentComposition/manifest.json | 16 +--------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs index a303347b..10c95e91 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/FhirTests/ValidationManifestTest.cs @@ -45,11 +45,6 @@ public void RunSingleTest(TestCase testCase, string baseDirectory) public void RunFirelySdkTests(TestCase testCase, string baseDirectory) => _runner.RunTestCase(testCase, DotNetValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); - [DataTestMethod] - [ValidationManifestDataSource(DOC_COMPOSITION_TEST_CASES_MANIFEST)] - public void OldExamples(TestCase testCase, string baseDirectory) - => _runner.RunTestCase(testCase, DotNetValidator.Create(), baseDirectory, AssertionOptions.OutputTextAssertion); - /// /// Not really an unit test, but a way to generate Firely SDK results in an existing manifest. @@ -61,6 +56,7 @@ public void OldExamples(TestCase testCase, string baseDirectory) /// - The method `ClassCleanup` will gather all the testcases and serialize those to disk. The filename can be altered in /// that method /// + [Ignore("This test is only used to generate the Firely SDK results in the manifest. See the method for more info")] [TestMethod] public void AddFirelySdkValidatorResults() => _runner.AddOrEditValidatorResults(TEST_CASES_MANIFEST, new[] { DotNetValidator.Create() }); diff --git a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/TestData/DocumentComposition/manifest.json b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/TestData/DocumentComposition/manifest.json index a8a56707..62dd281f 100644 --- a/test/Firely.Fhir.Validation.Compilation.Tests.Shared/TestData/DocumentComposition/manifest.json +++ b/test/Firely.Fhir.Validation.Compilation.Tests.Shared/TestData/DocumentComposition/manifest.json @@ -16,21 +16,7 @@ "Weight.observation.xml", "Hippocrates.practitioner.xml" ], - "java": { - "errorCount": 0, - "warningCount": 0, - "output": [] - }, - "firely-sdk-current": { - "errorCount": 0, - "warningCount": 0, - "output": [] - }, - "firely-sdk-wip": { - "errorCount": 0, - "warningCount": 0, - "output": [] - } + "java": {} } ] } \ No newline at end of file