Finish validator functional test upgrade

src/Sarif.Multitool.FunctionalTests/Rules/EndTimeMustNotBeBeforeStartTimeTests.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Xunit;
+
+namespace Microsoft.CodeAnalysis.Sarif.Multitool.Rules
+{
+    public class EndTimeMustNotBeBeforeStartTimeTests : ValidationSkimmerTestsBase<EndTimeMustNotBeBeforeStartTime>
+    {
+        [Fact(DisplayName = nameof(EndTimeMustNotBeBeforeStartTime_ReportsInvalidSarif))]
+        public void EndTimeMustNotBeBeforeStartTime_ReportsInvalidSarif() => Verify("Invalid.sarif");
+
+        [Fact(DisplayName = nameof(EndTimeMustNotBeBeforeStartTime_AcceptsValidSarif))]
+        public void EndTimeMustNotBeBeforeStartTime_AcceptsValidSarif() => Verify("Valid.sarif");
+    }
+}

src/Sarif.Multitool.FunctionalTests/Rules/UriBaseIdRequiresRelativeUriTests.cs

@@ -7,118 +7,10 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool.Rules
 {
     public class UriBaseIdRequiresRelativeUriTests : ValidationSkimmerTestsBase<UriBaseIdRequiresRelativeUri>
     {
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_UrisAreRelative))]
-        public void UriBaseIdRequiresRelativeUri_UrisAreRelative()
-        {
-            Verify("UrisAreRelative.sarif");
-        }
+        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_ReportsInvalidSarif))]
+        public void UriBaseIdRequiresRelativeUri_ReportsInvalidSarif() => Verify("Invalid.sarif");
 
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInAnalysisTarget))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInAnalysisTarget()
-        {
-            Verify("AbsoluteUriInAnalysisTarget.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInCodeFlow))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInCodeFlow()
-        {
-            Verify("AbsoluteUriInCodeFlow.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInConfigurationNotification))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInConfigurationNotification()
-        {
-            Verify("AbsoluteUriInConfigurationNotification.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInConversionAnalysisToolLogFiles))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInConversionAnalysisToolLogFiles()
-        {
-            Verify("AbsoluteUriInConversionAnalysisToolLogFiles.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInFileChange))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInFileChange()
-        {
-            Verify("AbsoluteUriInFileChange.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInFilesDictionary))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInFilesDictionary()
-        {
-            Verify("AbsoluteUriInFilesDictionary.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationAttachment))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationAttachment()
-        {
-            Verify("AbsoluteUriInInvocationAttachment.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationExecutableLocation))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationExecutableLocation()
-        {
-            Verify("AbsoluteUriInInvocationExecutableLocation.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationResponseFile))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationResponseFile()
-        {
-            Verify("AbsoluteUriInInvocationResponseFile.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationStandardStreams))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInInvocationStandardStreams()
-        {
-            Verify("AbsoluteUriInInvocationStandardStreams.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInRelatedLocation))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInRelatedLocation()
-        {
-            Verify("AbsoluteUriInRelatedLocation.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInResultAttachment))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInResultAttachment()
-        {
-            Verify("AbsoluteUriInResultAttachment.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInResultConversionProvenance))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInResultConversionProvenance()
-        {
-            Verify("AbsoluteUriInResultConversionProvenance.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInResultGraphNode))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInResultGraphNode()
-        {
-            Verify("AbsoluteUriInResultGraphNode.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInResultLocation))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInResultLocation()
-        {
-            Verify("AbsoluteUriInResultLocation.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInRunGraphNode))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInRunGraphNode()
-        {
-            Verify("AbsoluteUriInRunGraphNode.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInStackFrame))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInStackFrame()
-        {
-            Verify("AbsoluteUriInStackFrame.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AbsoluteUriInToolNotification))]
-        public void UriBaseIdRequiresRelativeUri_AbsoluteUriInToolNotification()
-        {
-            Verify("AbsoluteUriInToolNotification.sarif");
-        }
+        [Fact(DisplayName = nameof(UriBaseIdRequiresRelativeUri_AcceptsValidSarif))]
+        public void UriBaseIdRequiresRelativeUri_AcceptsValidSarif() => Verify("Valid.sarif");
     }
 }

src/Sarif.Multitool.FunctionalTests/Rules/ValidationSkimmerTestsBase.cs

@@ -29,24 +29,6 @@ protected void Verify(string testFileName)
             Verify(testFileName, disablePrereleaseCompatibilityTransform: false);
         }
 
-        // For the moment, we support two different test designs.
-        //
-        // The new, preferred design (all new tests should be written this way):
-        //
-        // The test file itself contains a custom property that summarizes the expected
-        // results of running the rule on the test file.
-        //
-        // The old, deprecated design:
-        //
-        // To each test file there exists a corresponding file whose name ends in
-        // "_Expected.sarif" that contains the expected results of running the rule
-        // on the test file. We perform a "selective compare" of the expected and
-        // actual validation log file contents.
-        //
-        // As we migrate from the old to the new design, if the custom property exists,
-        // we use the new design, if the "expected" file exists, we use the old design,
-        // and if both the custom property and the "expected" file exist, we execute
-        // both the new and the old style tests.
         protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTransform)
         {
             string targetPath = Path.Combine(_testDirectory, testFileName);
@@ -61,6 +43,9 @@ protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTr
 
             SarifLog inputLog = JsonConvert.DeserializeObject<SarifLog>(inputLogContents);
 
+            bool expectedResultsArePresent = inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults);
+            expectedResultsArePresent.Should().Be(true);
+
             var skimmer = new TSkimmer();
 
             using (var logger = new SarifLogger(
@@ -92,33 +77,9 @@ protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTr
             }
 
             string actualLogContents = File.ReadAllText(actualFilePath);
-            bool resultsWereVerified = false;
-
-            string expectedFilePath = MakeExpectedFilePath(_testDirectory, testFileName);
-            if (File.Exists(expectedFilePath))
-            {
-                // The "expected" file exists. Use the old, deprecated verification method.
-                string expectedLogContents = File.ReadAllText(expectedFilePath);
-                expectedLogContents = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(expectedLogContents, formatting: Formatting.Indented);
-
-                // We can't just compare the text of the log files because properties
-                // like start time, and absolute paths, will differ from run to run.
-                // Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
-                // we perform a selective compare of just the elements we care about.
-                SelectiveCompare(actualLogContents, expectedLogContents);
-                resultsWereVerified = true;
-            }
-
-            if (inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults))
-            {
-                // The custom property exists. Use the new, preferred verification method.
-                SarifLog outputLog = JsonConvert.DeserializeObject<SarifLog>(actualLogContents);
-                Verify(outputLog.Runs[0], expectedResults);
-                resultsWereVerified = true;
-            }
+            SarifLog outputLog = JsonConvert.DeserializeObject<SarifLog>(actualLogContents);
 
-            // Temporary: make sure at least one of the verification methods executed.
-            resultsWereVerified.Should().BeTrue();
+            Verify(outputLog.Runs[0], expectedResults);
         }
 
         // Every validation message begins with a placeholder "{0}: " that specifies the