Playback and Record for Azure Storage Blob (#4971)

* Initial recording for Spock unit tests

* Updated client construction to use playback client

* Ignore sleeps in playback

* Tests cleanup after each run

* Each test keeps track of its increment count instead of the global

* Initial testing records (WIP)

* Fixing more playback and record edge cases, added new pipeline exception tracing

* More cleaning up of edge cases

* More edge cases fixed

* Octet-stream response types work properly

* Merged TestCommon into APISpec, removed JUnit dependency in blobs testing

* Added redaction for sensitive information in UserDelegationKey responses

* Added @requires annotations to live tests only

* Added playback records and added test ignore for live tests only

* Set dummy values for playback in pipeline

* Updating failing test records

* Fixed playback issue with AAD

* Cleaning up failing File tests

* Updating playback records

* Updated tests.yml for storage

* Fixed tests and playback, added more testing configurations

* Removing files that were deleted before merge

* Fixed checkstyle rule issues

* Updating playback records

* Responses to comments

* Fixed spacing issue

* Fix unit test issues

* Variable name change to try to fix test

* Added SLF4J testing dependency and ignoring tests that fail live run

* Ignore more failing tests

* Minor change to attempt to fix CI being stuck

* Fixed not enough bytes issue

* Fixed test issue

sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/PlaybackClient.java

@@ -7,13 +7,14 @@
 import com.azure.core.http.HttpRequest;
 import com.azure.core.http.HttpResponse;
 import com.azure.core.http.ProxyOptions;
+import com.azure.core.implementation.http.UrlBuilder;
 import com.azure.core.test.models.NetworkCallRecord;
 import com.azure.core.test.models.RecordedData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.azure.core.util.logging.ClientLogger;
+import reactor.core.Exceptions;
 import reactor.core.publisher.Mono;
 
-import java.net.URI;
+import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
@@ -25,14 +26,14 @@
  * HTTP client that plays back {@link NetworkCallRecord NetworkCallRecords}.
  */
 public final class PlaybackClient implements HttpClient {
-    private final Logger logger = LoggerFactory.getLogger(PlaybackClient.class);
+    private final ClientLogger logger = new ClientLogger(PlaybackClient.class);
     private final AtomicInteger count = new AtomicInteger(0);
     private final Map<String, String> textReplacementRules;
     private final RecordedData recordedData;
 
     /**
-     * Creates a PlaybackClient that replays network calls from {@code recordedData} and replaces
-     * {@link NetworkCallRecord#response() response text} for any rules specified in {@code textReplacementRules}.
+     * Creates a PlaybackClient that replays network calls from {@code recordedData} and replaces {@link
+     * NetworkCallRecord#response() response text} for any rules specified in {@code textReplacementRules}.
      *
      * @param recordedData The data to playback.
      * @param textReplacementRules A set of rules to replace text in network call responses.
@@ -88,14 +89,16 @@ private Mono<HttpResponse> playbackHttpResponse(final HttpRequest request) {
         count.incrementAndGet();
 
         if (networkCallRecord == null) {
-            if (logger.isWarnEnabled()) {
-                logger.warn("NOT FOUND - Method: {} URL: {}", incomingMethod, incomingUrl);
-                logger.warn("Records requested: {}.", count);
-            }
+            logger.warning("NOT FOUND - Method: {} URL: {}", incomingMethod, incomingUrl);
+            logger.warning("Records requested: {}.", count);
 
             return Mono.error(new IllegalStateException("==> Unexpected request: " + incomingMethod + " " + incomingUrl));
         }
 
+        if (networkCallRecord.exception() != null) {
+            throw logger.logExceptionAsWarning(Exceptions.propagate(networkCallRecord.exception().get()));
+        }
+
         int recordStatusCode = Integer.parseInt(networkCallRecord.response().get("StatusCode"));
         HttpHeaders headers = new HttpHeaders();
 
@@ -112,7 +115,7 @@ private Mono<HttpResponse> playbackHttpResponse(final HttpRequest request) {
         }
 
         String rawBody = networkCallRecord.response().get("Body");
-        byte[] bytes = new byte[0];
+        byte[] bytes = null;
 
         if (rawBody != null) {
             for (Map.Entry<String, String> rule : textReplacementRules.entrySet()) {
@@ -121,7 +124,20 @@ private Mono<HttpResponse> playbackHttpResponse(final HttpRequest request) {
                 }
             }
 
-            bytes = rawBody.getBytes(StandardCharsets.UTF_8);
+            String contentType = networkCallRecord.response().get("Content-Type");
+
+            // octet-stream's are written to disk using Arrays.toString() which creates an output such as "[12, -1]".
+            if (contentType != null && contentType.equalsIgnoreCase("application/octet-stream")) {
+                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+                for (String piece : rawBody.substring(1, rawBody.length() - 1).split(", ")) {
+                    outputStream.write(Byte.parseByte(piece));
+                }
+
+                bytes = outputStream.toByteArray();
+            } else {
+                bytes = rawBody.getBytes(StandardCharsets.UTF_8);
+            }
+
             if (bytes.length > 0) {
                 headers.put("Content-Length", String.valueOf(bytes.length));
             }
@@ -141,7 +157,12 @@ private String applyReplacementRule(String text) {
     }
 
     private static String removeHost(String url) {
-        URI uri = URI.create(url);
-        return String.format("%s?%s", uri.getPath(), uri.getQuery());
+        UrlBuilder urlBuilder = UrlBuilder.parse(url);
+
+        if (urlBuilder.query().containsKey("sig")) {
+            urlBuilder.setQueryParameter("sig", "REDACTED");
+        }
+
+        return String.format("%s%s", urlBuilder.path(), urlBuilder.queryString());
     }
 }

sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/NetworkCallError.java

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.core.test.models;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.net.UnknownHostException;
+
+/**
+ * This class represents a caught throwable during a network call. It is used to serialize exceptions that were thrown
+ * during the pipeline and deserialize them back into their actual throwable class when running in playback mode.
+ */
+public class NetworkCallError {
+    @JsonProperty("Throwable")
+    private Throwable throwable;
+
+    @JsonProperty("ClassName")
+    private String className;
+
+    /**
+     * Empty constructor used by deserialization.
+     */
+    public NetworkCallError() {
+    }
+
+    /**
+     * Constructs the class setting the throwable and its class name.
+     *
+     * @param throwable Throwable thrown during a network call.
+     */
+    public NetworkCallError(Throwable throwable) {
+        this.throwable = throwable;
+        this.className = throwable.getClass().getName();
+    }
+
+    /**
+     * @return the thrown throwable as the class it was thrown as by converting is using its class name.
+     */
+    public Throwable get() {
+        switch (className) {
+            case "java.lang.NullPointerException":
+                return new NullPointerException(throwable.getMessage());
+
+            case "java.lang.IndexOutOfBoundsException":
+                return new IndexOutOfBoundsException(throwable.getMessage());
+
+            case "java.net.UnknownHostException":
+                return new UnknownHostException(throwable.getMessage());
+
+            default:
+                return throwable;
+        }
+    }
+
+    /**
+     * Sets the throwable that was thrown during a network call.
+     *
+     * @param throwable Throwable that was thrown.
+     */
+    public void throwable(Throwable throwable) {
+        this.throwable = throwable;
+    }
+
+    /**
+     * Sets the name of the class of the throwable. This is used during deserialization the construct the throwable
+     * as the actual class that was thrown.
+     *
+     * @param className Class name of the throwable.
+     */
+    public void className(String className) {
+        this.className = className;
+    }
+}

sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/NetworkCallRecord.java

@@ -23,6 +23,9 @@ public class NetworkCallRecord {
     @JsonProperty("Response")
     private Map<String, String> response;
 
+    @JsonProperty("Exception")
+    private NetworkCallError exception;
+
     /**
      * Gets the HTTP method for with this network call
      *
@@ -96,4 +99,22 @@ public Map<String, String> response() {
     public void response(Map<String, String> response) {
         this.response = response;
     }
+
+    /**
+     * Gets the throwable thrown during evaluation of the network call.
+     *
+     * @return Throwable thrown during the network call.
+     */
+    public NetworkCallError exception() {
+        return exception;
+    }
+
+    /**
+     * Sets the throwable thrown during evaluation of the network call.
+     *
+     * @param exception Throwable thrown during the network call.
+     */
+    public void exception(NetworkCallError exception) {
+        this.exception = exception;
+    }
 }