Sanitize json body (#12707)

* sanitize body

* PR feedback

* update to master

* no var

doc/dev/Track2TestFramework.md

@@ -1,17 +1,18 @@
 # Acquiring TestFramework
 
-To start using test framework import `sdk\core\Azure.Core\tests\TestFramework.props` into test `.csproj`:
+To start using Test Framework add a project reference using the alias `AzureCoreTestFramework` into your test `.csproj`:
 
 ``` xml
 <Project Sdk="Microsoft.NET.Sdk">
 
 ...
-  <Import Project="..\..\..\core\Azure.Core\tests\TestFramework.props" />
+   <ProjectReference Include="$(AzureCoreTestFramework)" />
 ...
 
 </Project>
 
 ```
+As an example, see the [Template](https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/template/Azure.Template/tests/Azure.Template.Tests.csproj#L15) project.
 
 # Sync-async tests
 
@@ -182,13 +183,19 @@ __NOTE:__ recordings are copied from `netcoreapp2.1` directory by default, make
 
 ## Sanitizing
 
-Secrets that are part of requests, responses, headers or connections strings should be sanitized before saving the record. Common headers like `Authentication` are sanitized automatically but if custom logic is required `RecordedTest.Sanitizer` should be used as extension point.
+Secrets that are part of requests, responses, headers, or connections strings should be sanitized before saving the record. Common headers like `Authentication` are sanitized automatically but if custom logic is required and/or if request or response body need to be sanitied, the `RecordedTest.Sanitizer` should be used as extension point.
 
 For example:
 
 ``` C#
     public class ConfigurationRecordedTestSanitizer : RecordedTestSanitizer
     {
+        public ConfigurationRecordedTestSanitizer()
+            base()
+        {
+            JsonPathSanitizers.Add("$..secret");
+        }
+
         public override void SanitizeConnectionString(ConnectionString connectionString)
         {
             const string secretKey = "secret";
@@ -209,6 +216,8 @@ For example:
     }
 ```
 
+**Note:** `JsonPathSanitizers` takes [Json Path](https://www.newtonsoft.com/json/help/html/QueryJsonSelectToken.htm) format strings that will be validated against the body. If a match exists, the value will be sanitized.
+
 ## Matching
 
 When tests are ran in replay mode HTTP method, uri and headers are used to match request to response. Some headers change on every request and are not controlled by the client code and should be ignored during the matching. Common headers like `Date`, `x-ms-date`, `x-ms-client-request-id`, `User-Agent`, `Request-Id` are ignored by default but if more headers need to be ignored use `Recording.Matcher` extensions point.

sdk/core/Azure.Core.TestFramework/src/Azure.Core.TestFramework.csproj

@@ -5,6 +5,7 @@
 
   <ItemGroup>
     <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="nunit" />
     <PackageReference Include="NUnit3TestAdapter" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />

sdk/core/Azure.Core.TestFramework/src/RecordedTestSanitizer.cs

@@ -5,12 +5,16 @@
 using System.Globalization;
 using System.Linq;
 using System.Text;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace Azure.Core.TestFramework
 {
     public class RecordedTestSanitizer
     {
         public const string SanitizeValue = "Sanitized";
+        public List<string> JsonPathSanitizers { get; } = new List<string>();
+
         private static readonly string[] s_sanitizeValueArray = { SanitizeValue };
 
         private static readonly string[] s_sanitizedHeaders = { "Authorization" };
@@ -33,7 +37,24 @@ public virtual void SanitizeHeaders(IDictionary<string, string[]> headers)
 
         public virtual string SanitizeTextBody(string contentType, string body)
         {
-            return body;
+            if (JsonPathSanitizers.Count == 0)
+                return body;
+            try
+            {
+                var jsonO = JObject.Parse(body);
+                foreach (string jsonPath in JsonPathSanitizers)
+                {
+                    foreach (JToken token in jsonO.SelectTokens(jsonPath))
+                    {
+                        token.Replace(JToken.FromObject(SanitizeValue));
+                    }
+                }
+                return JsonConvert.SerializeObject(jsonO);
+            }
+            catch
+            {
+                return body;
+            }
         }
 
         public virtual byte[] SanitizeBody(string contentType, byte[] body)

sdk/core/Azure.Core/tests/RecordSessionTests.cs

@@ -206,6 +206,35 @@ public void RecordingSessionSanitizeSanitizesVariables()
             Assert.AreEqual("Totally not a SANITIZED", session.Variables["B"]);
         }
 
+        [TestCase("*", "invalid json", "invalid json")]
+        [TestCase("$..secret",
+                "{\"secret\":\"I should be sanitized\",\"level\":{\"key\":\"value\",\"secret\":\"I should be sanitized\"}}",
+                "{\"secret\":\"Sanitized\",\"level\":{\"key\":\"value\",\"secret\":\"Sanitized\"}}")]
+        public void RecordingSessionSanitizeTextBody(string jsonPath, string body, string expected)
+        {
+            var sanitizer = new RecordedTestSanitizer();
+            sanitizer.JsonPathSanitizers.Add(jsonPath);
+
+            string response = sanitizer.SanitizeTextBody(default, body);
+
+            Assert.AreEqual(expected, response);
+        }
+
+        [Test]
+        public void RecordingSessionSanitizeTextBodyMultipleValues()
+        {
+            var sanitizer = new RecordedTestSanitizer();
+            sanitizer.JsonPathSanitizers.Add("$..secret");
+            sanitizer.JsonPathSanitizers.Add("$..topSecret");
+
+            var body = "{\"secret\":\"I should be sanitized\",\"key\":\"value\",\"topSecret\":\"I should be sanitized\"}";
+            var expected = "{\"secret\":\"Sanitized\",\"key\":\"value\",\"topSecret\":\"Sanitized\"}";
+
+            string response = sanitizer.SanitizeTextBody(default, body);
+
+            Assert.AreEqual(expected, response);
+        }
+
         [Test]
         public void SavingRecordingSanitizesValues()
         {

sdk/formrecognizer/Azure.AI.FormRecognizer/tests/FormTrainingClient/FormTrainingClientLiveTests.cs

@@ -220,21 +220,19 @@ public async Task CopyModel()
         }
 
         [Test]
-        [Ignore("Tracked by issue: https://github.com/Azure/azure-sdk-for-net/issues/12193")]
-        public async Task CopyModelError()
+        public void CopyModelError()
         {
             var sourceClient = CreateInstrumentedFormTrainingClient();
             var targetClient = CreateInstrumentedFormTrainingClient();
             var resourceID = TestEnvironment.TargetResourceId;
             var region = TestEnvironment.TargetResourceRegion;
 
-            CopyAuthorization targetAuth = await targetClient.GetCopyAuthorizationAsync(resourceID, region);
+            CopyAuthorization targetAuth = CopyAuthorization.FromJson("{\"modelId\":\"328c3b7d - a563 - 4ba2 - 8c2f - 2f26d664486a\",\"accessToken\":\"5b5685e4 - 2f24 - 4423 - ab18 - 000000000000\",\"expirationDateTimeTicks\":1591932653,\"resourceId\":\"resourceId\",\"resourceRegion\":\"westcentralus\"}");
 
             Assert.ThrowsAsync<RequestFailedException>(async () => await sourceClient.StartCopyModelAsync("00000000-0000-0000-0000-000000000000", targetAuth));
         }
 
         [Test]
-        [Ignore("Tracked by issue: https://github.com/Azure/azure-sdk-for-net/issues/12193")]
         public async Task GetCopyAuthorization()
         {
             var targetClient = CreateInstrumentedFormTrainingClient();
@@ -251,7 +249,6 @@ public async Task GetCopyAuthorization()
         }
 
         [Test]
-        [Ignore("Tracked by issue: https://github.com/Azure/azure-sdk-for-net/issues/12193")]
         public async Task SerializeDeserializeCopyAuthorizationAsync()
         {
             var targetClient = CreateInstrumentedFormTrainingClient();

sdk/formrecognizer/Azure.AI.FormRecognizer/tests/Infrastructure/FormRecognizerRecordedTestSanitizer.cs

@@ -10,6 +10,12 @@ public class FormRecognizerRecordedTestSanitizer : RecordedTestSanitizer
     {
         private const string SanitizedSasUri = "https://sanitized.blob.core.windows.net";
 
+        public FormRecognizerRecordedTestSanitizer()
+            : base()
+        {
+            JsonPathSanitizers.Add("$..accessToken");
+        }
+
         public override void SanitizeHeaders(IDictionary<string, string[]> headers)
         {
             if (headers.ContainsKey(Constants.AuthorizationHeader))