Merge pull request #1498 from input-output-hk/retest_testrun

Add ability to retest failures in a test run
IntersectMBO · Nov 21, 2022 · 88084bd · 88084bd
2 parents 3074a75 + 9a1658b
commit 88084bd
Show file tree

Hide file tree

Showing 10 changed files with 112 additions and 23 deletions.
diff --git a/.github/regression.sh b/.github/regression.sh
@@ -69,8 +69,8 @@ nix develop --accept-flake-config $(node_override) --command bash -c '
 '
 retval="$?"
 
-# move html report to root dir
-mv .reports/testrun-report.html testrun-report.html
+# move reports to root dir
+mv .reports/testrun-report.* ./
 
 # create results archive
 "$REPODIR"/.buildkite/results.sh .

diff --git a/.github/workflows/regression-dbsync.yaml b/.github/workflows/regression-dbsync.yaml
@@ -4,16 +4,10 @@ on:
   workflow_dispatch:
     inputs:
       node_rev:
-        description: "cardano-node revision (default: HEAD)"
-        required: false
-      node_branch:
-        description: "cardano-node branch (default: master)"
+        description: "cardano-node revision (default: master HEAD)"
         required: false
       dbsync_rev:
-        description: "db-sync revision (default: HEAD)"
-        required: false
-      dbsync_branch:
-        description: "db-sync branch (default: master)"
+        description: "db-sync revision (default: master HEAD)"
         required: false
       cluster_era:
         type: choice
@@ -44,19 +38,31 @@ on:
         type: boolean
         default: false
         description: "Start cluster directly in target era"
+      testrun_name:
+        required: false
+        description: "Test run name (internal)"
+      skip_passed:
+        type: boolean
+        default: false
+        description: "Skip tests that already passed (internal)"
+
+run-name: ${{ inputs.testrun_name && 'Run:' || ''}} ${{ inputs.testrun_name }} ${{ (inputs.testrun_name && inputs.skip_passed) && ':repeat:' || '' }}
 
 jobs:
   regression_tests:
     # reusable workflow from local repo and same branch as this config
     uses: ./.github/workflows/regression_reusable.yaml
     with:
       node_rev: ${{ inputs.node_rev }}
-      node_branch: ${{ inputs.node_branch }}
       dbsync_rev: ${{ inputs.dbsync_rev }}
-      dbsync_branch: ${{ inputs.dbsync_branch }}
       cluster_era: ${{ inputs.cluster_era }}
       tx_era: ${{ inputs.tx_era }}
       enable_p2p: ${{ inputs.enable_p2p }}
       enable_dbsync: true
       skip_long: ${{ inputs.skip_long }}
       fast_cluster: ${{ inputs.fast_cluster }}
+      testrun_name: ${{ inputs.testrun_name }}
+      skip_passed: ${{ inputs.skip_passed }}
+    secrets:
+      TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+      TCACHE_URL: ${{ secrets.TCACHE_URL }}
diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml
@@ -4,10 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       node_rev:
-        description: "cardano-node revision (default: HEAD)"
-        required: false
-      node_branch:
-        description: "cardano-node branch (default: master)"
+        description: "cardano-node revision (default: master HEAD)"
         required: false
       cluster_era:
         type: choice
@@ -38,16 +35,29 @@ on:
         type: boolean
         default: false
         description: "Start cluster directly in target era"
+      testrun_name:
+        required: false
+        description: "Test run name (internal)"
+      skip_passed:
+        type: boolean
+        default: false
+        description: "Skip tests that already passed (internal)"
+
+run-name: ${{ inputs.testrun_name && 'Run:' || ''}} ${{ inputs.testrun_name }} ${{ (inputs.testrun_name && inputs.skip_passed) && ':repeat:' || '' }}
 
 jobs:
   regression_tests:
     # reusable workflow from local repo and same branch as this config
     uses: ./.github/workflows/regression_reusable.yaml
     with:
       node_rev: ${{ inputs.node_rev }}
-      node_branch: ${{ inputs.node_branch }}
       cluster_era: ${{ inputs.cluster_era }}
       tx_era: ${{ inputs.tx_era }}
       enable_p2p: ${{ inputs.enable_p2p }}
       skip_long: ${{ inputs.skip_long }}
       fast_cluster: ${{ inputs.fast_cluster }}
+      testrun_name: ${{ inputs.testrun_name }}
+      skip_passed: ${{ inputs.skip_passed }}
+    secrets:
+      TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+      TCACHE_URL: ${{ secrets.TCACHE_URL }}
diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml
@@ -51,6 +51,19 @@ on:
         required: false
         type: boolean
         default: false
+      testrun_name:
+        required: false
+        type: string
+        default: ""
+      skip_passed:
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      TCACHE_BASIC_AUTH:
+        required: false
+      TCACHE_URL:
+        required: false
 
 jobs:
   reusable_run:
@@ -77,13 +90,28 @@ jobs:
           echo "CI_ENABLE_DBSYNC=${{ inputs.enable_dbsync }}" >> .github_ci_env
           echo "CI_SKIP_LONG=${{ inputs.skip_long }}" >> .github_ci_env
           echo "CI_FAST_CLUSTER=${{ inputs.fast_cluster }}" >> .github_ci_env
+          echo "CI_TESTRUN_NAME=${{ inputs.testrun_name }}" >> .github_ci_env
+          echo "CI_SKIP_PASSED=${{ inputs.skip_passed }}" >> .github_ci_env
           if [ -e "${{ inputs.env-path }}" ]; then cat "${{ inputs.env-path }}" >> .github_ci_env; fi
       - name: Export env variables
         run: |
           cat .github_ci_env
           cat .github_ci_env >> $GITHUB_ENV
+      - name: Get previous test results
+        if: inputs.testrun_name && inputs.skip_passed
+        run: |
+          if [ -n "${{ secrets.TCACHE_BASIC_AUTH }}" ] && [ -n "${{ secrets.TCACHE_URL }}" ]; then
+            curl -s -u ${{ secrets.TCACHE_BASIC_AUTH }} "${{ secrets.TCACHE_URL }}/${{ inputs.testrun_name }}/pypassed" > deselected_tests.txt
+            echo "DESELECT_FROM_FILE=deselected_tests.txt" >> $GITHUB_ENV
+          fi
       - name: Run CLI regression tests
         run: .github/regression.sh
+      - name: Report test results
+        if: (success() || failure()) && inputs.testrun_name
+        run: |
+          if [ -n "${{ secrets.TCACHE_BASIC_AUTH }}" ] && [ -n "${{ secrets.TCACHE_URL }}" ]; then
+            curl -s -X PUT --fail-with-body -u ${{ secrets.TCACHE_BASIC_AUTH }} "${{ secrets.TCACHE_URL }}/${{ inputs.testrun_name }}/${{ github.run_number }}/import" -F "junitxml=@testrun-report.xml"
+          fi
       - name: Upload testing artifacts on failure
         uses: actions/upload-artifact@v3
         if: failure()
@@ -113,3 +141,5 @@ jobs:
             cli_coverage.json
             scheduling.log.xz
             errors_all.log
+            testrun-report.xml
+            deselected_tests.txt
diff --git a/Makefile b/Makefile
@@ -48,22 +48,42 @@ ifndef NO_ARTIFACTS
 endif
 
 ifndef PYTEST_ARGS
-	HTML_REPORT_ARGS := --html=$(REPORTS_DIR)/testrun-report.html --self-contained-html
+	TESTRUN_REPORT_ARGS := --html=$(REPORTS_DIR)/testrun-report.html --self-contained-html --junitxml=$(REPORTS_DIR)/testrun-report.xml
+endif
+
+ifdef DESELECT_FROM_FILE
+	DESELECT_FROM_FILE_ARGS := --deselect-from-file=$(DESELECT_FROM_FILE)
+endif
+
+CLEANUP := yes
+RUN_SKIPS := yes
+
+ifdef PYTEST_ARGS
+	CLEANUP := no
+	RUN_SKIPS := no
+endif
+
+ifdef DESELECT_FROM_FILE
+	RUN_SKIPS := no
 endif
 
 .dirs:
 	mkdir -p $(ARTIFACTS_DIR) $(COVERAGE_DIR) $(REPORTS_DIR)
 
 .run_tests:
-# First just skip all tests so Allure has a list of runable tests. Also delete artifacts from previous runs.
-# Run only if no pytest args were specified.
-ifndef PYTEST_ARGS
-	rm -f $(REPORTS_DIR)/{*-attachment.txt,*-result.json,*-container.json,testrun-report.html}
+# delete artifacts from previous runs
+ifeq ($(CLEANUP),yes)
+	rm -f $(REPORTS_DIR)/{*-attachment.txt,*-result.json,*-container.json,testrun-report.*}
 	rm -f $(COVERAGE_DIR)/cli_coverage_*
+endif
+
+# first just skip all tests so Allure has a list of all tests that were supposed to run
+ifeq ($(RUN_SKIPS),yes)
 	pytest -s cardano_node_tests $(MARKEXPR) --skipall --alluredir=$(REPORTS_DIR) >/dev/null
 endif
+
 # run tests for real and produce Allure results
-	pytest cardano_node_tests $(PYTEST_ARGS) $(CI_ARGS) $(MARKEXPR) -n $(TEST_THREADS) $(ARTIFACTS_ARGS) --cli-coverage-dir=$(COVERAGE_DIR) --alluredir=$(REPORTS_DIR) $(HTML_REPORT_ARGS)
+	pytest cardano_node_tests $(PYTEST_ARGS) $(CI_ARGS) $(MARKEXPR) $(DESELECT_FROM_FILE_ARGS) -n $(TEST_THREADS) $(ARTIFACTS_ARGS) --cli-coverage-dir=$(COVERAGE_DIR) --alluredir=$(REPORTS_DIR) $(TESTRUN_REPORT_ARGS)
 
 
 # run all tests, generate allure report

diff --git a/cardano_node_tests/tests/conftest.py b/cardano_node_tests/tests/conftest.py
@@ -74,6 +74,12 @@ def pytest_configure(config: Any) -> None:
     config._metadata["cardano-cli exe"] = shutil.which("cardano-cli") or ""
     config._metadata["cardano-node exe"] = shutil.which("cardano-node") or ""
 
+    testrun_name = os.environ.get("CI_TESTRUN_NAME")
+    if testrun_name:
+        skip_passed = os.environ.get("CI_SKIP_PASSED") == "true"
+        config._metadata["CI_TESTRUN_NAME"] = testrun_name
+        config._metadata["CI_SKIP_PASSED"] = str(skip_passed)
+
     network_magic = configuration.NETWORK_MAGIC_LOCAL
     if configuration.BOOTSTRAP_DIR:
         with open(

diff --git a/flake.nix b/flake.nix
@@ -26,6 +26,7 @@
             packageOverrides = self: _: {
               allure = self.callPackage ./nix/allure.nix { };
               pytest-allure = self.callPackage ./nix/pytest-allure.nix { };
+              pytest-select = self.callPackage ./nix/pytest-select.nix {};
               cardano-clusterlib = self.callPackage ./nix/cardano-clusterlib.nix { };
             };
           };
@@ -65,6 +66,7 @@
                   pytest-allure
                   pytest-html
                   pytest-order
+                  pytest-select
                   pytest_xdist
                   pyyaml
                   setuptools

diff --git a/nix/default.nix b/nix/default.nix
@@ -17,6 +17,7 @@ let
         packageOverrides = self: _: {
           allure = self.callPackage ./allure.nix {};
           pytest-allure = self.callPackage ./pytest-allure.nix {};
+          pytest-select = self.callPackage ./pytest-select.nix {};
           cardano-clusterlib = self.callPackage ./cardano-clusterlib.nix {};
         };
       };
@@ -31,6 +32,7 @@ let
           pytest-allure
           pytest-html
           pytest-order
+          pytest-select
           pytest_xdist
           pyyaml
           setuptools

diff --git a/nix/pytest-select.nix b/nix/pytest-select.nix
@@ -0,0 +1,12 @@
+{ stdenv, buildPythonPackage, fetchPypi, pytest }:
+
+buildPythonPackage rec {
+  pname = "pytest-select";
+  version = "0.1.2";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-FGqq8pQDT2TMAyZEjQTAP6U98ZckjJh0cS8hqRnOQU4=";
+  };
+  doCheck = false;
+  propagatedBuildInputs = [ pytest ];
+}
diff --git a/setup.cfg b/setup.cfg
@@ -48,6 +48,7 @@ install_requires =
     pytest-html
     pytest-metadata
     pytest-order
+    pytest-select
     pytest-xdist
     pyyaml
     requests