TT-1821 Fix processing skipped tests in flaky test detector (#15032)

* Bump flakeguard * Bump again * Include skipped field in summary * Bump * Bump flaky threshold to 100%
smartcontractkit · Oct 30, 2024 · 235c85c · 235c85c
1 parent 89183a8
commit 235c85c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -407,7 +407,7 @@ jobs:
       baseRef: ${{ github.base_ref }}
       headRef: ${{ github.head_ref }}
       runCount: '5'
-      runThreshold: '0.9'
+      runThreshold: '1'
       runWithRace: true
       findByTestFilesDiff: true
       findByAffectedPackages: false

diff --git a/.github/workflows/find-new-flaky-tests.yml b/.github/workflows/find-new-flaky-tests.yml
@@ -86,7 +86,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fc79d0ec518a24cb2bf016c8a913c62197080cc7
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@e23466e1b565345af550198c55128ab89ef857f2
 
       - name: Find new or updated test packages
         id: find-tests
@@ -219,7 +219,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fc79d0ec518a24cb2bf016c8a913c62197080cc7
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@e23466e1b565345af550198c55128ab89ef857f2
 
       - name: Run tests with flakeguard
         shell: bash
@@ -262,7 +262,7 @@ jobs:
             ALL_TESTS_COUNT=$(jq 'length' all_tests.json)
             echo "All tests count: $ALL_TESTS_COUNT"
             echo "all_tests_count=$ALL_TESTS_COUNT" >> "$GITHUB_OUTPUT"
-            jq -c 'map(select(.PassRatio < ($runThreshold | tonumber))) | map(.PassRatio |= (. * 100 | tostring + "%"))' all_tests.json --arg runThreshold '${{ inputs.runThreshold }}' > failed_tests.json
+            jq -c 'map(select(.PassRatio < ($runThreshold | tonumber) and .Skipped != true)) | map(.PassRatio |= (. * 100 | tostring + "%"))' all_tests.json --arg runThreshold '${{ inputs.runThreshold }}' > failed_tests.json
             FAILED_TESTS_COUNT=$(jq 'length' failed_tests.json)
             echo "Failed tests count: $FAILED_TESTS_COUNT"
             echo "failed_tests_count=$FAILED_TESTS_COUNT" >> "$GITHUB_OUTPUT"
@@ -283,14 +283,14 @@ jobs:
         if: ${{ fromJson(steps.set_test_results.outputs.failed_tests_count) > 0 }}
         shell: bash
         run: |
-          jq -r '["TestPackage", "TestName", "PassRatio", "RunCount"], ["---------", "---------", "---------", "---------"], (.[] | [.TestPackage, .TestName, .PassRatio, .Runs]) | @tsv' test_results/failed_tests.json | column -t -s$'\t' > test_results/failed_tests_ascii.txt
+          jq -r '["TestPackage", "TestName", "PassRatio", "RunCount", "Skipped"], ["---------", "---------", "---------", "---------", "---------"], (.[] | [.TestPackage, .TestName, .PassRatio, .Runs, .Skipped]) | @tsv' test_results/failed_tests.json | column -t -s$'\t' > test_results/failed_tests_ascii.txt
           cat test_results/failed_tests_ascii.txt
 
       - name: Create ASCII table with all test results
         if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) > 0 }}
         shell: bash
         run: |
-          jq -r '["TestPackage", "TestName", "PassRatio", "RunCount"], ["---------", "---------", "---------", "---------"], (.[] | [.TestPackage, .TestName, .PassRatio, .Runs]) | @tsv' test_results/all_tests.json | column -t -s$'\t' > test_results/all_tests_ascii.txt
+          jq -r '["TestPackage", "TestName", "PassRatio", "RunCount", "Skipped"], ["---------", "---------", "---------", "---------", "---------"], (.[] | [.TestPackage, .TestName, .PassRatio, .Runs, .Skipped]) | @tsv' test_results/all_tests.json | column -t -s$'\t' > test_results/all_tests_ascii.txt
           cat test_results/all_tests_ascii.txt
 
       - name: Create GitHub Summary