Fix #1138: Add validation rule: contextRegion requires region (#1142)

Also:

- Enhance the "new-style" verification so that we no longer require the file "Invalid_Expected.sarif". Each file can now contain a property that specifies the expected locations of all the validation results.

src/Sarif.Multitool.FunctionalTests/Rules/ContextRegionRequiresRegionTests.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Xunit;
+
+namespace Microsoft.CodeAnalysis.Sarif.Multitool.Rules
+{
+    public class ContextRegionRequiresRegionTests : ValidationSkimmerTestsBase<ContextRegionRequiresRegion>
+    {
+        [Fact(DisplayName = nameof(ContextRegionRequiresRegion_ReportsInvalidSarif))]
+        public void ContextRegionRequiresRegion_ReportsInvalidSarif()
+        {
+            // We need to disable compatibility transformations for any files that require
+            // a malformed schema or malformed JSON, as this code fixes those things up
+            Verify("Invalid.sarif", disablePrereleaseCompatibilityTransform: true);
+        }
+
+        [Fact(DisplayName = nameof(ContextRegionRequiresRegion_AcceptsValidSarif))]
+        public void ContextRegionRequiresRegion_AcceptsValidSarif()
+        {
+            Verify("Valid.sarif");
+        }
+    }
+}

src/Sarif.Multitool.FunctionalTests/Rules/DoNotUseFriendlyNameAsRuleIdTests.cs

@@ -8,15 +8,9 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool.Rules
     public class DoNotUseFriendlyNameAsRuleIdTests : ValidationSkimmerTestsBase<DoNotUseFriendlyNameAsRuleId>
     {
         [Fact(DisplayName = nameof(DoNotUseFriendlyNameAsRuleId_ReportsInvalidSarif))]
-        public void DoNotUseFriendlyNameAsRuleId_ReportsInvalidSarif()
-        {
-            Verify("Invalid.sarif");
-        }
+        public void DoNotUseFriendlyNameAsRuleId_ReportsInvalidSarif() => Verify("Invalid.sarif");
 
         [Fact(DisplayName = nameof(DoNotUseFriendlyNameAsRuleId_AcceptsValidSarif))]
-        public void DoNotUseFriendlyNameAsRuleId_AcceptsValidSarif()
-        {
-            Verify("Valid.sarif");
-        }
+        public void DoNotUseFriendlyNameAsRuleId_AcceptsValidSarif() => Verify("Valid.sarif");
     }
 }

src/Sarif.Multitool.FunctionalTests/Rules/UrisMustBeValidTests.cs

@@ -18,59 +18,14 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool.Rules
 
     public class UrisMustBeValidTests : ValidationSkimmerTestsBase<UrisMustBeValid>
     {
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidFileLocationUri))]
-        public void UrisMustBeValid_InvalidFileLocationUri()
-        {
-            Verify("InvalidFileLocationUri.sarif");
-        }
+        [Fact(DisplayName = nameof(UrisMustBeValid_ReportsInvalidSarif))]
+        public void UrisMustBeValid_ReportsInvalidSarif() =>
+            // We need to disable compatibility transformations for any files that require
+            // a malformed schema or malformed JSON, as this code fixes those things up
+            Verify("Invalid.sarif", disablePrereleaseCompatibilityTransform: true);
 
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidResultWorkItemUri))]
-        public void UrisMustBeValid_InvalidResultWorkItemUri()
-        {
-            Verify("InvalidResultWorkItemUri.sarif");
-        }
+        [Fact(DisplayName = nameof(UrisMustBeValid_AcceptsValidSarif))]
 
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidRuleHelpUri))]
-        public void UrisMustBeValid_InvalidRuleHelpUri()
-        {
-            Verify("InvalidRuleHelpUri.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidSarifLogSchemaUri))]
-        public void UrisMustBeValid_InvalidSarifLogSchemaUri()
-        {
-            Verify("InvalidSarifLogSchemaUri.sarif", disablePrereleaseCompatibilityTransform: true);
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidToolDownloadUri))]
-        public void UrisMustBeValid_InvalidToolDownloadUri()
-        {
-            Verify("InvalidToolDownloadUri.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidUriInFilePropertyName))]
-        public void UrisMustBeValid_InvalidUriInFilePropertyName()
-        {
-            Verify("InvalidUriInFilePropertyName.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidUriInOriginalUriBaseIds))]
-        public void UrisMustBeValid_InvalidUriInOriginalUriBaseIds()
-        {
-            Verify("InvalidUriInOriginalUriBaseIds.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_InvalidVersionControlDetailsUri))]
-        public void UrisMustBeValid_InvalidVersionControlDetailsUri()
-        {
-            Verify("InvalidVersionControlDetailsUri.sarif");
-        }
-
-        [Fact(DisplayName = nameof(UrisMustBeValid_ValidUris))]
-
-        public void UrisMustBeValid_ValidUris()
-        {
-            Verify("ValidUris.sarif");
-        }
+        public void UrisMustBeValid_AcceptsValidSarif() => Verify("Valid.sarif");
     }
 }

src/Sarif.Multitool.FunctionalTests/Rules/ValidationSkimmerTestsBase.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.IO;
+using System.Linq;
 using FluentAssertions;
 using Microsoft.CodeAnalysis.Sarif.Driver;
 using Microsoft.CodeAnalysis.Sarif.Writers;
@@ -120,9 +121,17 @@ protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTr
             resultsWereVerified.Should().BeTrue();
         }
 
+        // Every validation message begins with a placeholder "{0}: " that specifies the
+        // result location, for example, "runs[0].results[0].locations[0].physicalLocation".
+        // Verify that those detected result locations match the expected locations.
         private void Verify(Run run, ExpectedValidationResults expectedResults)
         {
             run.Results.Count.Should().Be(expectedResults.ResultCount);
+
+            string[] detectedResultLocations = run.Results.Select(r => r.Message.Arguments[0]).OrderBy(loc => loc).ToArray();
+            string[] expectedResultLocations = expectedResults.ResultLocationPointers.OrderBy(loc => loc).ToArray();
+
+            detectedResultLocations.Should().ContainInOrder(expectedResultLocations);
         }
     }
 }

src/Sarif.Multitool.FunctionalTests/TestData/ContextRegionRequiresRegion/Invalid.sarif

@@ -0,0 +1,37 @@
+{
+  "$schema": "http://json.schemastore.org/sarif-2.0.0",
+  "version": "2.0.0",
+  "runs": [
+    {
+      "tool": {
+        "name": "CodeScanner"
+      },
+      "results": [
+        {
+          "ruleId": "TST0001",
+          "level": "error",
+          "locations": [
+            {
+              "physicalLocation": {
+                "fileLocation": {
+                  "uri": "src/test.c"
+                },
+                "contextRegion": {
+                  "startLine": 2,
+                  "endLine": 4
+                }
+              }
+            }
+          ]
+        }
+      ],
+      "properties": {
+        "expectedResults": {
+          "resultLocationPointers": [
+            "runs[0].results[0].locations[0].physicalLocation"
+          ]
+        }
+      }
+    }
+  ]
+}

src/Sarif.Multitool.FunctionalTests/TestData/ContextRegionRequiresRegion/Valid.sarif

@@ -0,0 +1,38 @@
+{
+  "$schema": "http://json.schemastore.org/sarif-2.0.0",
+  "version": "2.0.0",
+  "runs": [
+    {
+      "tool": {
+        "name": "CodeScanner"
+      },
+      "results": [
+        {
+          "ruleId": "TST0001",
+          "level": "error",
+          "locations": [
+            {
+              "physicalLocation": {
+                "fileLocation": {
+                  "uri": "src/test.c"
+                },
+                "region": {
+                  "startLine": 3
+                },
+                "contextRegion": {
+                  "startLine": 2,
+                  "endLine": 4
+                }
+              }
+            }
+          ]
+        }
+      ],
+      "properties": {
+        "expectedResults": {
+          "resultLocationPointers": []
+        }
+      }
+    }
+  ]
+}

src/Sarif.Multitool.FunctionalTests/TestData/DoNotUseFriendlyNameAsRuleId/Invalid.sarif

@@ -22,6 +22,14 @@
             }
           }
         }
+      },
+      "properties": {
+        "expectedResults": {
+          "resultLocationPointers": [
+            "runs[0].resources.rules.RULE0001",
+            "runs[0].resources.rules['RULE0002-1']"
+          ]
+        }
       }
     }
   ]

src/Sarif.Multitool.FunctionalTests/TestData/DoNotUseFriendlyNameAsRuleId/Valid.sarif

@@ -22,6 +22,11 @@
             }
           }
         }
+      },
+      "properties": {
+        "expectedResults": {
+          "resultLocationPointers": []
+        }
       }
     }
   ]