diff --git a/system-tests/__snapshots__/experimental_retries.spec.ts.js b/system-tests/__snapshots__/experimental_retries.spec.ts.js new file mode 100644 index 000000000000..615941fb13ca --- /dev/null +++ b/system-tests/__snapshots__/experimental_retries.spec.ts.js @@ -0,0 +1,1685 @@ +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / only runs the first attempt of the test if the test passes'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (always-passes.cy.js) │ + │ Searched: cypress/e2e/always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: always-passes.cy.js (1 of 1) + + + always passes + ✓ always passes + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ always-passes.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / retries up to the "passesRequired" limit if the config can be satisfied'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✓ deterministically runs pass/fail on this test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ deterministic-flaky.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / retries up to the "passesRequired" limit if the config can be satisfied (max attempts)'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✖(Attempt 7 of 10) deterministically runs pass/fail on this test + ✓(Attempt 8 of 10) deterministically runs pass/fail on this test + ✖(Attempt 9 of 10) deterministically runs pass/fail on this test + ✓(Attempt 10 of 10) deterministically runs pass/fail on this test + ✓ deterministically runs pass/fail on this test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ deterministic-flaky.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / fails / short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 6) deterministically runs pass/fail on this test + ✓(Attempt 2 of 6) deterministically runs pass/fail on this test + ✖(Attempt 3 of 6) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / fails / retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 7) deterministically runs pass/fail on this test + ✓(Attempt 2 of 7) deterministically runs pass/fail on this test + ✖(Attempt 3 of 7) deterministically runs pass/fail on this test + ✓(Attempt 4 of 7) deterministically runs pass/fail on this test + ✖(Attempt 5 of 7) deterministically runs pass/fail on this test + ✓(Attempt 6 of 7) deterministically runs pass/fail on this test + ✖(Attempt 7 of 7) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / passes / only runs the first attempt of the test if the test passes'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (always-passes.cy.js) │ + │ Searched: cypress/e2e/always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: always-passes.cy.js (1 of 1) + + + always passes + ✓ always passes + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ always-passes.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / fails / runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✖(Attempt 7 of 10) deterministically runs pass/fail on this test + ✓(Attempt 8 of 10) deterministically runs pass/fail on this test + ✖(Attempt 9 of 10) deterministically runs pass/fail on this test + ✓(Attempt 10 of 10) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + [object Object] +AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / fails / runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + [object Object] +AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / exercises experimental-retries suite to verify console reporter and final status code are correct.'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 8 found (can-retry-from-afterEach.retries.mochaEvents.cy.js, can-retry-from-before │ + │ Each.retries.mochaEvents.cy.js, cant-retry-from-before.retries.mochaEvents.cy.js, │ + │ does-not-serialize-dom-error.cy.js, simple-fail.retries.mochaEvents.cy.js, test-re │ + │ try-with-h...) │ + │ Searched: cypress/e2e/runner/experimental-retries/* │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-afterEach.retries.mochaEvents.cy.js (1 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓ test 1 + ✓ test 2 + ✓ test 3 + + suite 2 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓ test 1 + + suite 3 + ✓ test 1 + + + 5 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 5 │ + │ Passing: 5 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-afterEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-beforeEach.retries.mochaEvents.cy.js (2 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓ test 1 + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-beforeEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: cant-retry-from-before.retries.mochaEvents.cy.js (3 of 8) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: cant-retry-from-before.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: does-not-serialize-dom-error.cy.js (4 of 8) + + + ✖(Attempt 1 of 10) visits + ✖(Attempt 2 of 10) visits + ✖(Attempt 3 of 10) visits + ✖(Attempt 4 of 10) visits + ✖(Attempt 5 of 10) visits + ✖(Attempt 6 of 10) visits + ✖ visits + + 0 passing + 1 failing + + 1) visits: + AssertionError: Timed out retrying after 200ms: expected '' not to be 'visible' + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: does-not-serialize-dom-error.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: simple-fail.retries.mochaEvents.cy.js (5 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓ test 1 + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: simple-fail.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-hooks.retries.mochaEvents.cy.js (6 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓ test 1 + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-hooks.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-only.retries.mochaEvents.cy.js (7 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 2 + ✓(Attempt 2 of 10) test 2 + ✓(Attempt 3 of 10) test 2 + ✓(Attempt 4 of 10) test 2 + ✓(Attempt 5 of 10) test 2 + ✓(Attempt 6 of 10) test 2 + ✓ test 2 + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-only.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: three-tests-with-retry.retries.mochaEvents.cy.js (8 of 8) + + + suite 1 + ✓ test 1 + ✖(Attempt 1 of 10) test 2 + ✖(Attempt 2 of 10) test 2 + ✓(Attempt 3 of 10) test 2 + ✓(Attempt 4 of 10) test 2 + ✓(Attempt 5 of 10) test 2 + ✓(Attempt 6 of 10) test 2 + ✓(Attempt 7 of 10) test 2 + ✓ test 2 + ✓ test 3 + + + 3 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 3 │ + │ Passing: 3 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: three-tests-with-retry.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ can-retry-from-afterEach.retries.mo XX:XX 5 5 - - - │ + │ chaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ can-retry-from-beforeEach.retries.m XX:XX 1 1 - - - │ + │ ochaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ cant-retry-from-before.retries.moch XX:XX 1 - 1 - - │ + │ aEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ does-not-serialize-dom-error.cy.js XX:XX 1 - 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ simple-fail.retries.mochaEvents.cy. XX:XX 1 1 - - - │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ test-retry-with-hooks.retries.mocha XX:XX 1 1 - - - │ + │ Events.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ test-retry-with-only.retries.mochaE XX:XX 1 1 - - - │ + │ vents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ three-tests-with-retry.retries.moch XX:XX 3 3 - - - │ + │ aEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 2 of 8 failed (25%) XX:XX 14 12 2 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / exercises experimental-retries suite to verify console reporter and final status code are correct.'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 8 found (can-retry-from-afterEach.retries.mochaEvents.cy.js, can-retry-from-before │ + │ Each.retries.mochaEvents.cy.js, cant-retry-from-before.retries.mochaEvents.cy.js, │ + │ does-not-serialize-dom-error.cy.js, simple-fail.retries.mochaEvents.cy.js, test-re │ + │ try-with-h...) │ + │ Searched: cypress/e2e/runner/experimental-retries/* │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-afterEach.retries.mochaEvents.cy.js (1 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓(Attempt 8 of 10) test 1 + ✓(Attempt 9 of 10) test 1 + ✓(Attempt 10 of 10) test 1 + ✖ test 1 + ✓ test 2 + ✓ test 3 + + suite 2 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓(Attempt 8 of 10) test 1 + ✓(Attempt 9 of 10) test 1 + ✓(Attempt 10 of 10) test 1 + ✖ test 1 + + suite 3 + ✓ test 1 + + + 3 passing + 2 failing + + 1) suite 1 + test 1: + + [object Object] + Error + [stack trace lines] + + 2) suite 2 + test 1: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 5 │ + │ Passing: 3 │ + │ Failing: 2 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-afterEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-beforeEach.retries.mochaEvents.cy.js (2 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓(Attempt 8 of 10) test 1 + ✓(Attempt 9 of 10) test 1 + ✓(Attempt 10 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-beforeEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: cant-retry-from-before.retries.mochaEvents.cy.js (3 of 8) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: cant-retry-from-before.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: does-not-serialize-dom-error.cy.js (4 of 8) + + + ✖(Attempt 1 of 10) visits + ✖(Attempt 2 of 10) visits + ✖(Attempt 3 of 10) visits + ✖(Attempt 4 of 10) visits + ✖(Attempt 5 of 10) visits + ✖(Attempt 6 of 10) visits + ✖(Attempt 7 of 10) visits + ✖(Attempt 8 of 10) visits + ✖(Attempt 9 of 10) visits + ✖(Attempt 10 of 10) visits + ✖ visits + + 0 passing + 1 failing + + 1) visits: + AssertionError: Timed out retrying after 200ms: expected '' not to be 'visible' + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: does-not-serialize-dom-error.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: simple-fail.retries.mochaEvents.cy.js (5 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓(Attempt 8 of 10) test 1 + ✓(Attempt 9 of 10) test 1 + ✓(Attempt 10 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + [object Object] +Error: test 1 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: simple-fail.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-hooks.retries.mochaEvents.cy.js (6 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✓(Attempt 4 of 10) test 1 + ✓(Attempt 5 of 10) test 1 + ✓(Attempt 6 of 10) test 1 + ✓(Attempt 7 of 10) test 1 + ✓(Attempt 8 of 10) test 1 + ✓(Attempt 9 of 10) test 1 + ✓(Attempt 10 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + [object Object] +Error: test 1 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-hooks.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-only.retries.mochaEvents.cy.js (7 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 2 + ✓(Attempt 2 of 10) test 2 + ✓(Attempt 3 of 10) test 2 + ✓(Attempt 4 of 10) test 2 + ✓(Attempt 5 of 10) test 2 + ✓(Attempt 6 of 10) test 2 + ✓(Attempt 7 of 10) test 2 + ✓(Attempt 8 of 10) test 2 + ✓(Attempt 9 of 10) test 2 + ✓(Attempt 10 of 10) test 2 + ✖ test 2 + + + 0 passing + 1 failing + + 1) suite 1 + test 2: + [object Object] +Error: test 2 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-only.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: three-tests-with-retry.retries.mochaEvents.cy.js (8 of 8) + + + suite 1 + ✓ test 1 + ✖(Attempt 1 of 10) test 2 + ✖(Attempt 2 of 10) test 2 + ✓(Attempt 3 of 10) test 2 + ✓(Attempt 4 of 10) test 2 + ✓(Attempt 5 of 10) test 2 + ✓(Attempt 6 of 10) test 2 + ✓(Attempt 7 of 10) test 2 + ✓(Attempt 8 of 10) test 2 + ✓(Attempt 9 of 10) test 2 + ✓(Attempt 10 of 10) test 2 + ✖ test 2 + ✓ test 3 + + + 2 passing + 1 failing + + 1) suite 1 + test 2: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 3 │ + │ Passing: 2 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: three-tests-with-retry.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ can-retry-from-afterEach.retries.mo XX:XX 5 3 2 - - │ + │ chaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ can-retry-from-beforeEach.retries.m XX:XX 1 - 1 - - │ + │ ochaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ cant-retry-from-before.retries.moch XX:XX 1 - 1 - - │ + │ aEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ does-not-serialize-dom-error.cy.js XX:XX 1 - 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ simple-fail.retries.mochaEvents.cy. XX:XX 1 - 1 - - │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ test-retry-with-hooks.retries.mocha XX:XX 1 - 1 - - │ + │ Events.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ test-retry-with-only.retries.mochaE XX:XX 1 - 1 - - │ + │ vents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ three-tests-with-retry.retries.moch XX:XX 3 2 1 - - │ + │ aEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 8 of 8 failed (100%) XX:XX 14 5 9 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / exercises experimental-retries suite to verify console reporter and final status code are correct (stopIfAnyPassed=true).'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 8 found (can-retry-from-afterEach.retries.mochaEvents.cy.js, can-retry-from-before │ + │ Each.retries.mochaEvents.cy.js, cant-retry-from-before.retries.mochaEvents.cy.js, │ + │ does-not-serialize-dom-error.cy.js, simple-fail.retries.mochaEvents.cy.js, test-re │ + │ try-with-h...) │ + │ Searched: cypress/e2e/runner/experimental-retries/* │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-afterEach.retries.mochaEvents.cy.js (1 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖ test 1 + ✓ test 2 + ✓ test 3 + + suite 2 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖ test 1 + ✖(Attempt 2 of 10) test 1 + ✓(Attempt 3 of 10) test 1 + ✖ test 1 + + suite 3 + ✓ test 1 + + + 3 passing + 3 failing + + 1) suite 1 + test 1: + + [object Object] + Error + [stack trace lines] + + 2) suite 2 + test 1: + + [object Object] + Error + [stack trace lines] + + 3) suite 2 + test 1: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 5 │ + │ Passing: 3 │ + │ Failing: 2 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-afterEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: can-retry-from-beforeEach.retries.mochaEvents.cy.js (2 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: can-retry-from-beforeEach.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: cant-retry-from-before.retries.mochaEvents.cy.js (3 of 8) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: cant-retry-from-before.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: does-not-serialize-dom-error.cy.js (4 of 8) + + + ✖(Attempt 1 of 10) visits + ✖(Attempt 2 of 10) visits + ✖(Attempt 3 of 10) visits + ✖(Attempt 4 of 10) visits + ✖(Attempt 5 of 10) visits + ✖(Attempt 6 of 10) visits + ✖(Attempt 7 of 10) visits + ✖(Attempt 8 of 10) visits + ✖(Attempt 9 of 10) visits + ✖(Attempt 10 of 10) visits + ✖ visits + + 0 passing + 1 failing + + 1) visits: + AssertionError: Timed out retrying after 200ms: expected '' not to be 'visible' + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: does-not-serialize-dom-error.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: simple-fail.retries.mochaEvents.cy.js (5 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + [object Object] +Error: test 1 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: simple-fail.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-hooks.retries.mochaEvents.cy.js (6 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✓(Attempt 2 of 10) test 1 + ✖ test 1 + + + 0 passing + 1 failing + + 1) suite 1 + test 1: + [object Object] +Error: test 1 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-hooks.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: test-retry-with-only.retries.mochaEvents.cy.js (7 of 8) + + + suite 1 + ✖(Attempt 1 of 10) test 2 + ✓(Attempt 2 of 10) test 2 + ✖ test 2 + + + 0 passing + 1 failing + + 1) suite 1 + test 2: + [object Object] +Error: test 2 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: test-retry-with-only.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: three-tests-with-retry.retries.mochaEvents.cy.js (8 of 8) + + + suite 1 + ✓ test 1 + ✖(Attempt 1 of 10) test 2 + ✖(Attempt 2 of 10) test 2 + ✓(Attempt 3 of 10) test 2 + ✖ test 2 + ✓ test 3 + + + 2 passing + 1 failing + + 1) suite 1 + test 2: + + [object Object] + Error + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 3 │ + │ Passing: 2 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: three-tests-with-retry.retries.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ can-retry-from-afterEach.retries.mo XX:XX 5 3 2 - - │ + │ chaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ can-retry-from-beforeEach.retries.m XX:XX 1 - 1 - - │ + │ ochaEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ cant-retry-from-before.retries.moch XX:XX 1 - 1 - - │ + │ aEvents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ does-not-serialize-dom-error.cy.js XX:XX 1 - 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ simple-fail.retries.mochaEvents.cy. XX:XX 1 - 1 - - │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ test-retry-with-hooks.retries.mocha XX:XX 1 - 1 - - │ + │ Events.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ test-retry-with-only.retries.mochaE XX:XX 1 - 1 - - │ + │ vents.cy.js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ three-tests-with-retry.retries.moch XX:XX 3 2 1 - - │ + │ aEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 8 of 8 failed (100%) XX:XX 14 5 9 - - + + +` diff --git a/system-tests/projects/experimental-retries/cypress.config.js b/system-tests/projects/experimental-retries/cypress.config.js new file mode 100644 index 000000000000..2db5c69b464b --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress.config.js @@ -0,0 +1,14 @@ +module.exports = { + e2e: { + supportFile: false, + setupNodeEvents (on, config) { + // in the case the tests needed to be debugged: + + // on('before:browser:launch', (browser, launchOptions) => { + // launchOptions.args.push('--auto-open-devtools-for-tabs') + + // return launchOptions + // }) + }, + }, +} diff --git a/system-tests/projects/experimental-retries/cypress/e2e/always-fails.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/always-fails.cy.js new file mode 100644 index 000000000000..1728d87f56c8 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/always-fails.cy.js @@ -0,0 +1,5 @@ +describe('always fails', () => { + it('always fails', function () { + expect(true).to.be.false + }) +}) diff --git a/system-tests/projects/experimental-retries/cypress/e2e/always-passes.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/always-passes.cy.js new file mode 100644 index 000000000000..40154fb58d0a --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/always-passes.cy.js @@ -0,0 +1,5 @@ +describe('always passes', () => { + it('always passes', function () { + expect(true).to.be.true + }) +}) diff --git a/system-tests/projects/experimental-retries/cypress/e2e/deterministic-flaky.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/deterministic-flaky.cy.js new file mode 100644 index 000000000000..99996e5ebc2f --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/deterministic-flaky.cy.js @@ -0,0 +1,15 @@ +describe('deterministic flaky test', () => { + it('deterministically runs pass/fail on this test', function () { + // this means the test WILL behave as + // first attempt (FAIL) + // second attempt (PASS) + // third attempt (FAIL) + // fourth attempt (PASS) + // fifth attempt (FAIL)... + if (this.test.currentRetry() % 2) { + expect(true).to.be.true + } else { + expect(true).to.be.false + } + }) +}) diff --git a/system-tests/test/experimental_retries.spec.ts b/system-tests/test/experimental_retries.spec.ts new file mode 100644 index 000000000000..b62daa563537 --- /dev/null +++ b/system-tests/test/experimental_retries.spec.ts @@ -0,0 +1,262 @@ +import systemTests from '../lib/system-tests' + +describe('e2e retries.experimentalStrategy', () => { + systemTests.setup() + + describe('experimentalBurnIn=false', () => { + describe('"detect-flake-and-pass-on-threshold"', () => { + describe('passes', () => { + systemTests.it('only runs the first attempt of the test if the test passes', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'always-passes.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 10, + passesRequired: 3, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 3, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts)', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + describe('fails', () => { + systemTests.it('short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 5, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 6, + passesRequired: 4, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + /** + * exercised additionally in cy-in-cy tests to verify correct mocha snapshots and cypress reporter output: + * packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.cy.ts + * packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.cy.ts + */ + systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct.', { + project: 'detect-flake-and-pass-on-threshold', + browser: '!webkit', + spec: 'runner/experimental-retries/*', + snapshot: true, + expectedExitCode: 2, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + describe('"detect-flake-but-always-fail"', () => { + describe('passes', () => { + systemTests.it('only runs the first attempt of the test if the test passes', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'always-passes.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + describe('fails', () => { + systemTests.it('runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: true, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + /** + * exercised additionally in cy-in-cy tests to verify correct mocha snapshots and cypress reporter output: + * packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.cy.ts + * packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.cy.ts + */ + systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct.', { + project: 'detect-flake-but-always-fail', + browser: '!webkit', + spec: 'runner/experimental-retries/*', + snapshot: true, + // FIXME: this should be 8 + expectedExitCode: 9, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct (stopIfAnyPassed=true).', { + project: 'detect-flake-but-always-fail-stop-any-passed', + browser: '!webkit', + spec: 'runner/experimental-retries/*', + snapshot: true, + // FIXME: this should be 8 + expectedExitCode: 9, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: true, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + }) +})