feat: Change display progress to unknown progress instead of percenta…

…ge as percentages were unreliable
Endava · Jun 8, 2024 · ac0befb · ac0befb
1 parent 567bd71
commit ac0befb
Show file tree

Hide file tree

Showing 9 changed files with 59 additions and 110 deletions.
diff --git a/src/main/java/com/endava/cats/command/CatsCommand.java b/src/main/java/com/endava/cats/command/CatsCommand.java
@@ -356,21 +356,10 @@ private void fuzzPath(Map.Entry<String, PathItem> pathItemEntry, OpenAPI openAPI
                 .collect(Collectors.toSet());
 
         List<Fuzzer> fuzzersToRun = filterArguments.filterOutFuzzersNotMatchingHttpMethods(allHttpMethodsFromFuzzingData);
-        int totalToRun = this.computeTotalsToRun(fuzzersToRun, filteredFuzzingData);
-        testCaseListener.setTotalRunsPerPath(pathItemEntry.getKey(), totalToRun);
         this.runFuzzers(filteredFuzzingData, fuzzersToRun);
         this.runFuzzers(filteredFuzzingData, filterArguments.getSecondPhaseFuzzers());
     }
 
-    private int computeTotalsToRun(List<Fuzzer> fuzzersToRun, List<FuzzingData> filteredFuzzingData) {
-        int total = 0;
-        for (FuzzingData data : filteredFuzzingData) {
-            total = total + (int) fuzzersToRun.stream().filter(fuzzer -> !fuzzer.skipForHttpMethods().contains(data.getMethod())).count();
-        }
-
-        return total;
-    }
-
     private void runFuzzers(List<FuzzingData> fuzzingDataListWithHttpMethodsFiltered, List<Fuzzer> configuredFuzzers) {
         /*We only run the fuzzers supplied and exclude those that do not apply for certain HTTP methods*/
 
@@ -384,7 +373,9 @@ private void runFuzzers(List<FuzzingData> fuzzingDataListWithHttpMethodsFiltered
             filteredData.forEach(data -> {
                 logger.start("Starting Fuzzer {}, http method {}, path {}", ansi().fgGreen().a(fuzzer.toString()).reset(), data.getMethod(), data.getPath());
                 logger.debug("Fuzzing payload: {}", data.getPayload());
-                testCaseListener.beforeFuzz(fuzzer.getClass(), data.getContractPath(), data.getMethod().name());
+                if (!(fuzzer instanceof FunctionalFuzzer)) {
+                    testCaseListener.beforeFuzz(fuzzer.getClass(), data.getContractPath(), data.getMethod().name());
+                }
                 fuzzer.fuzz(data);
                 if (!(fuzzer instanceof FunctionalFuzzer)) {
                     testCaseListener.afterFuzz(data.getContractPath(), data.getMethod().name());

diff --git a/src/main/java/com/endava/cats/command/ListCommand.java b/src/main/java/com/endava/cats/command/ListCommand.java
@@ -15,10 +15,10 @@
 import com.endava.cats.fuzzer.special.mutators.api.Mutator;
 import com.endava.cats.generator.format.api.OpenAPIFormat;
 import com.endava.cats.http.HttpMethod;
-import com.endava.cats.util.JsonUtils;
 import com.endava.cats.model.FuzzingData;
 import com.endava.cats.openapi.OpenApiUtils;
 import com.endava.cats.util.ConsoleUtils;
+import com.endava.cats.util.JsonUtils;
 import com.endava.cats.util.VersionProvider;
 import io.github.ludovicianul.prettylogger.PrettyLogger;
 import io.github.ludovicianul.prettylogger.PrettyLoggerFactory;
@@ -196,7 +196,7 @@ void listPath(OpenAPI openAPI, String path) {
         } else {
             logger.noFormat(path);
             for (PathDetailsEntry.OperationDetails operation : pathDetailsEntry.getOperations()) {
-                logger.noFormat("  ");
+                logger.noFormat(ConsoleUtils.SEPARATOR);
                 logger.noFormat(" ◼ Operation: " + operation.getOperationId());
                 logger.noFormat(" ◼ HTTP Method: " + operation.getHttpMethod());
                 logger.noFormat(" ◼ Response Codes: " + operation.getResponses());

diff --git a/src/main/java/com/endava/cats/fuzzer/special/CustomFuzzerUtil.java b/src/main/java/com/endava/cats/fuzzer/special/CustomFuzzerUtil.java
@@ -8,14 +8,14 @@
 import com.endava.cats.http.ResponseCodeFamilyDynamic;
 import com.endava.cats.io.ServiceCaller;
 import com.endava.cats.io.ServiceData;
-import com.endava.cats.util.JsonUtils;
 import com.endava.cats.model.CatsHeader;
 import com.endava.cats.model.CatsResponse;
 import com.endava.cats.model.FuzzingData;
 import com.endava.cats.report.TestCaseListener;
 import com.endava.cats.strategy.FuzzingStrategy;
 import com.endava.cats.util.CatsDSLWords;
 import com.endava.cats.util.CatsUtil;
+import com.endava.cats.util.JsonUtils;
 import com.endava.cats.util.WordUtils;
 import io.github.ludovicianul.prettylogger.PrettyLogger;
 import io.github.ludovicianul.prettylogger.PrettyLoggerFactory;
@@ -40,7 +40,6 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static com.endava.cats.util.JsonUtils.NOT_SET;
 import static com.endava.cats.util.CatsDSLWords.CATS_BODY_FUZZ;
 import static com.endava.cats.util.CatsDSLWords.CATS_HEADERS;
 import static com.endava.cats.util.CatsDSLWords.CHECKS;
@@ -50,6 +49,7 @@
 import static com.endava.cats.util.CatsDSLWords.OUTPUT;
 import static com.endava.cats.util.CatsDSLWords.RESERVED_WORDS;
 import static com.endava.cats.util.CatsDSLWords.VERIFY;
+import static com.endava.cats.util.JsonUtils.NOT_SET;
 
 /**
  * Common methods used by the FunctionalFuzzer and SecurityFuzzer.
@@ -332,7 +332,7 @@ private boolean isCatsRemove(Map.Entry<String, Object> keyValue) {
      * @param fuzzer The custom fuzzer used for generating individual test cases.
      */
     public void executeTestCases(FuzzingData data, String key, Object value, CustomFuzzerBase fuzzer) {
-        testCaseListener.notifySummaryObservers(data.getContractPath(), data.getMethod().name(), 0d);
+        testCaseListener.notifySummaryObservers(data.getContractPath());
         log.debug("Path [{}] for method [{}] has the following custom data [{}]", data.getContractPath(), data.getMethod(), value);
         boolean isValidOneOf = this.isValidOneOf(data, (Map<String, Object>) value);
         List<String> missingKeywords = this.getMissingKeywords(fuzzer, (Map<String, Object>) value);

diff --git a/src/main/java/com/endava/cats/fuzzer/special/FunctionalFuzzer.java b/src/main/java/com/endava/cats/fuzzer/special/FunctionalFuzzer.java
@@ -22,7 +22,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
  * Executes functional tests written in Cats DSL.
@@ -78,9 +77,6 @@ public void executeCustomFuzzerTests() {
         logger.debug("Executing {} functional tests.", executions.size());
         Collections.sort(executions);
 
-        executions.stream().collect(Collectors.groupingBy(customFuzzerExecution -> customFuzzerExecution.getFuzzingData().getContractPath(), Collectors.counting()))
-                .forEach((s, aLong) -> testCaseListener.setTotalRunsPerPath(s, aLong.intValue()));
-
         for (Map.Entry<String, Map<String, Object>> entry : filesArguments.getCustomFuzzerDetails().entrySet()) {
             executions.stream().filter(customFuzzerExecution -> customFuzzerExecution.getFuzzingData().getContractPath().equalsIgnoreCase(entry.getKey()))
                     .forEach(customFuzzerExecution -> {

diff --git a/src/main/java/com/endava/cats/fuzzer/special/RandomFuzzer.java b/src/main/java/com/endava/cats/fuzzer/special/RandomFuzzer.java
@@ -11,14 +11,14 @@
 import com.endava.cats.fuzzer.special.mutators.api.CustomMutatorConfig;
 import com.endava.cats.fuzzer.special.mutators.api.CustomMutatorKeywords;
 import com.endava.cats.fuzzer.special.mutators.api.Mutator;
-import com.endava.cats.util.JsonUtils;
 import com.endava.cats.model.CatsHeader;
 import com.endava.cats.model.CatsResponse;
 import com.endava.cats.model.FuzzingData;
 import com.endava.cats.report.ExecutionStatisticsListener;
 import com.endava.cats.report.TestCaseListener;
 import com.endava.cats.util.CatsUtil;
 import com.endava.cats.util.ConsoleUtils;
+import com.endava.cats.util.JsonUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -95,7 +95,7 @@ public void fuzz(FuzzingData data) {
         boolean shouldStop = false;
         Set<String> allCatsFields = data.getAllFieldsByHttpMethod();
 
-        testCaseListener.startUnknownProgress(data);
+        testCaseListener.updateUnknownProgress(data);
 
         while (!shouldStop) {
             String targetField = CatsUtil.selectRandom(allCatsFields);

diff --git a/src/main/java/com/endava/cats/fuzzer/special/TemplateFuzzer.java b/src/main/java/com/endava/cats/fuzzer/special/TemplateFuzzer.java
@@ -7,15 +7,15 @@
 import com.endava.cats.generator.simple.StringGenerator;
 import com.endava.cats.generator.simple.UnicodeGenerator;
 import com.endava.cats.io.ServiceCaller;
-import com.endava.cats.util.JsonUtils;
 import com.endava.cats.model.CatsHeader;
 import com.endava.cats.model.CatsRequest;
 import com.endava.cats.model.CatsResponse;
 import com.endava.cats.model.FuzzingData;
-import com.endava.cats.util.KeyValuePair;
 import com.endava.cats.report.TestCaseListener;
 import com.endava.cats.strategy.FuzzingStrategy;
 import com.endava.cats.util.ConsoleUtils;
+import com.endava.cats.util.JsonUtils;
+import com.endava.cats.util.KeyValuePair;
 import com.jayway.jsonpath.JsonPathException;
 import io.github.ludovicianul.prettylogger.PrettyLogger;
 import io.github.ludovicianul.prettylogger.PrettyLoggerFactory;
@@ -65,7 +65,7 @@ public TemplateFuzzer(ServiceCaller sc, TestCaseListener lr, UserArguments ua, M
 
     @Override
     public void fuzz(FuzzingData data) {
-        testCaseListener.startUnknownProgress(data);
+        testCaseListener.updateUnknownProgress(data);
         for (String targetField : Optional.ofNullable(data.getTargetFields()).orElse(Collections.emptySet())) {
             int payloadSize = this.getPayloadSize(data, targetField);
 

diff --git a/src/main/java/com/endava/cats/report/TestCaseListener.java b/src/main/java/com/endava/cats/report/TestCaseListener.java
@@ -61,7 +61,7 @@
 @ApplicationScoped
 @DryRun
 public class TestCaseListener {
-    private final Iterator<String> cycle = Iterators.cycle("\\", "|", "/", "-");
+    private static final Iterator<Character> cycle = Iterators.cycle('\\', '\\', '\\', '|', '|', '|', '/', '/', '/', '-', '-', '-');
     private static final String DEFAULT = "*******";
     static final String ID = "id";
     private static final String FUZZER_KEY = "fuzzerKey";
@@ -91,8 +91,7 @@ public class TestCaseListener {
     @ConfigProperty(name = "app.timestamp", defaultValue = "1-1-1")
     String appBuildTime;
 
-    private final Map<String, Double> runPerPathListener = new HashMap<>();
-    private final Map<String, Integer> runTotals = new HashMap<>();
+    private final Deque<String> runPerPathListener = new ArrayDeque<>();
 
     /**
      * Constructs a TestCaseListener with the provided dependencies and configuration.
@@ -136,7 +135,7 @@ public void beforeFuzz(Class<?> fuzzer, String path, String method) {
         String clazz = ConsoleUtils.removeTrimSanitize(fuzzer.getSimpleName()).replaceAll("[a-z]", "");
         MDC.put(FUZZER, ConsoleUtils.centerWithAnsiColor(clazz, getKeyDefault().length(), Ansi.Color.MAGENTA));
         MDC.put(FUZZER_KEY, ConsoleUtils.removeTrimSanitize(fuzzer.getSimpleName()));
-        this.notifySummaryObservers(path, method, 1);
+        this.notifySummaryObservers(path);
 
     }
 
@@ -147,8 +146,7 @@ public void beforeFuzz(Class<?> fuzzer, String path, String method) {
      * @param httpMethod the HTTP method for which fuzzing has been completed
      */
     public void afterFuzz(String path, String httpMethod) {
-        double chunkSize = 100d / runTotals.getOrDefault(path, 1) + 0.01;
-        this.notifySummaryObservers(path, httpMethod, chunkSize);
+        this.notifySummaryObservers(path);
 
         MDC.put(FUZZER, this.getKeyDefault());
         MDC.put(FUZZER_KEY, this.getKeyDefault());
@@ -203,16 +201,6 @@ private void startTestCase() {
         testCaseMap.get(testId).setTestId("Test " + testId);
     }
 
-    /**
-     * Sets the total number of runs to be executed for a specific path.
-     *
-     * @param path         the path for which the total runs are being set
-     * @param totalToBeRun the total number of runs to be executed for the specified path
-     */
-    public void setTotalRunsPerPath(String path, Integer totalToBeRun) {
-        this.runTotals.put(path, totalToBeRun);
-    }
-
     /**
      * Adds a scenario to the test case and logs it using the provided logger.
      *
@@ -326,49 +314,38 @@ private void keepExecutionDetails(CatsTestCase testCase) {
      * Notifies summary observers about the progress of a specific path and HTTP method during the testing session.
      * If configured to display summaries in the console, this method renders the progress dynamically.
      *
-     * @param path      the path for which the progress is being reported
-     * @param method    the HTTP method associated with the path
-     * @param chunkSize the chunk size representing the progress
+     * @param path the path for which the progress is being reported
      */
-    public void notifySummaryObservers(String path, String method, double chunkSize) {
-        if (reportingArguments.isSummaryInConsole()) {
-            double percentage = runPerPathListener.getOrDefault(path, 0d) + chunkSize;
-            String printPath = path + "  " + (percentage >= 100 ? executionStatisticsListener.resultAsStringPerPath(path) : method);
+    public void notifySummaryObservers(String path) {
+        if (!reportingArguments.isSummaryInConsole()) {
+            return;
+        }
+        String printPath = path + ConsoleUtils.SEPARATOR + executionStatisticsListener.resultAsStringPerPath(path);
 
-            if (runPerPathListener.get(path) != null) {
-                ConsoleUtils.renderSameRow(printPath, percentage);
-            } else {
-                ConsoleUtils.renderNewRow(printPath, percentage);
-            }
-            runPerPathListener.merge(path, chunkSize, Double::sum);
+        if (runPerPathListener.contains(path)) {
+            ConsoleUtils.renderSameRow(printPath, cycle.next());
+        } else {
+            this.markPreviousPathAsDone();
+            runPerPathListener.push(path);
+            ConsoleUtils.renderNewRow(printPath, cycle.next());
         }
     }
 
-    /**
-     * Use this to start a summary progress when the number of tests to run is not known.
-     *
-     * @param data the FuzzingData context
-     */
-    public void startUnknownProgress(FuzzingData data) {
-        if (!reportingArguments.isSummaryInConsole()) {
-            return;
+    private void markPreviousPathAsDone() {
+        String previousPath = runPerPathListener.peek();
+        if (previousPath != null) {
+            String toRenderPreviousPath = previousPath + ConsoleUtils.SEPARATOR + executionStatisticsListener.resultAsStringPerPath(previousPath);
+            ConsoleUtils.renderSameRow(toRenderPreviousPath, '✔');
         }
-        this.notifySummaryObservers(data.getContractPath(), data.getMethod().name(), 0d);
-        ConsoleUtils.renderSameRow(data.getPath() + "  " + data.getMethod(), cycle.next());
     }
 
     /**
-     * Updates the progress with a new character to signla progress.
+     * Updates the progress with a new character to signal progress.
      *
      * @param data the FuzzingData context
      */
     public void updateUnknownProgress(FuzzingData data) {
-        if (!reportingArguments.isSummaryInConsole()) {
-            return;
-        }
-        if (this.getCurrentTestCaseNumber() % 20 == 0) {
-            ConsoleUtils.renderSameRow(data.getPath() + "  " + data.getMethod(), cycle.next());
-        }
+        this.notifySummaryObservers(data.getContractPath());
     }
 
     /**
@@ -432,6 +409,7 @@ public void writeHelperFiles() {
      * Additionally, prints execution details using the associated logger.
      */
     public void endSession() {
+        markPreviousPathAsDone();
         reportingArguments.enableAdditionalLoggingIfSummary();
         testCaseExporter.writeSummary(testCaseSummaryDetails, executionStatisticsListener);
         testCaseExporter.writeHelperFiles();
@@ -571,7 +549,7 @@ void reportError(PrettyLogger logger, String message, Object... params) {
      */
     private void renderProgress(CatsResponse catsResponse) {
         if (reportingArguments.isPrintProgress()) {
-            ConsoleUtils.renderSameRow("+ " + catsResponse.getPath());
+            ConsoleUtils.renderSameRowAndMoveToNextLine("+ " + catsResponse.getPath());
         }
     }