From cfd69a0e057c512ed866ddc36e5e08013dc297cf Mon Sep 17 00:00:00 2001 From: Bill Glesias Date: Fri, 15 Sep 2023 15:37:28 -0400 Subject: [PATCH] chore: add sanity system tests to verify console reporter output for experimental retries logic --- .../experimental_retries.spec.ts.js | 511 ++++++++++++++++++ .../experimental-retries/cypress.config.js | 14 + .../cypress/e2e/tests/always-fails.cy.js | 5 + .../cypress/e2e/tests/always-passes.cy.js | 5 + .../e2e/tests/deterministic-flaky.cy.js | 15 + .../test/experimental_retries.spec.ts | 187 +++++++ 6 files changed, 737 insertions(+) create mode 100644 system-tests/__snapshots__/experimental_retries.spec.ts.js create mode 100644 system-tests/projects/experimental-retries/cypress.config.js create mode 100644 system-tests/projects/experimental-retries/cypress/e2e/tests/always-fails.cy.js create mode 100644 system-tests/projects/experimental-retries/cypress/e2e/tests/always-passes.cy.js create mode 100644 system-tests/projects/experimental-retries/cypress/e2e/tests/deterministic-flaky.cy.js create mode 100644 system-tests/test/experimental_retries.spec.ts diff --git a/system-tests/__snapshots__/experimental_retries.spec.ts.js b/system-tests/__snapshots__/experimental_retries.spec.ts.js new file mode 100644 index 000000000000..43edfc1139d2 --- /dev/null +++ b/system-tests/__snapshots__/experimental_retries.spec.ts.js @@ -0,0 +1,511 @@ +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / only runs the first attempt of the test if the test passes'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (always-passes.cy.js) │ + │ Searched: cypress/e2e/tests/always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: always-passes.cy.js (1 of 1) + + + always passes + ✓ always passes + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ always-passes.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / retries up to the "passesRequired" limit if the config can be satisfied'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✓ deterministically runs pass/fail on this test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ deterministic-flaky.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / passes / retries up to the "passesRequired" limit if the config can be satisfied (max attempts)'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✖(Attempt 7 of 10) deterministically runs pass/fail on this test + ✓(Attempt 8 of 10) deterministically runs pass/fail on this test + ✖(Attempt 9 of 10) deterministically runs pass/fail on this test + ✓(Attempt 10 of 10) deterministically runs pass/fail on this test + ✓ deterministically runs pass/fail on this test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ deterministic-flaky.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / fails / short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 6) deterministically runs pass/fail on this test + ✓(Attempt 2 of 6) deterministically runs pass/fail on this test + ✖(Attempt 3 of 6) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / fails / retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 7) deterministically runs pass/fail on this test + ✓(Attempt 2 of 7) deterministically runs pass/fail on this test + ✖(Attempt 3 of 7) deterministically runs pass/fail on this test + ✓(Attempt 4 of 7) deterministically runs pass/fail on this test + ✖(Attempt 5 of 7) deterministically runs pass/fail on this test + ✓(Attempt 6 of 7) deterministically runs pass/fail on this test + ✖(Attempt 7 of 7) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / passes / only runs the first attempt of the test if the test passes'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (always-passes.cy.js) │ + │ Searched: cypress/e2e/tests/always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: always-passes.cy.js (1 of 1) + + + always passes + ✓ always passes + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: always-passes.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ always-passes.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 1 1 - - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / fails / runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖(Attempt 3 of 10) deterministically runs pass/fail on this test + ✓(Attempt 4 of 10) deterministically runs pass/fail on this test + ✖(Attempt 5 of 10) deterministically runs pass/fail on this test + ✓(Attempt 6 of 10) deterministically runs pass/fail on this test + ✖(Attempt 7 of 10) deterministically runs pass/fail on this test + ✓(Attempt 8 of 10) deterministically runs pass/fail on this test + ✖(Attempt 9 of 10) deterministically runs pass/fail on this test + ✓(Attempt 10 of 10) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + [object Object] +AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / fails / runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (deterministic-flaky.cy.js) │ + │ Searched: cypress/e2e/tests/deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: deterministic-flaky.cy.js (1 of 1) + + + deterministic flaky test + ✖(Attempt 1 of 10) deterministically runs pass/fail on this test + ✓(Attempt 2 of 10) deterministically runs pass/fail on this test + ✖ deterministically runs pass/fail on this test + + + 0 passing + 1 failing + + 1) deterministic flaky test + deterministically runs pass/fail on this test: + [object Object] +AssertionError: expected true to be false + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: deterministic-flaky.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ deterministic-flaky.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - + + +` diff --git a/system-tests/projects/experimental-retries/cypress.config.js b/system-tests/projects/experimental-retries/cypress.config.js new file mode 100644 index 000000000000..2db5c69b464b --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress.config.js @@ -0,0 +1,14 @@ +module.exports = { + e2e: { + supportFile: false, + setupNodeEvents (on, config) { + // in the case the tests needed to be debugged: + + // on('before:browser:launch', (browser, launchOptions) => { + // launchOptions.args.push('--auto-open-devtools-for-tabs') + + // return launchOptions + // }) + }, + }, +} diff --git a/system-tests/projects/experimental-retries/cypress/e2e/tests/always-fails.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/tests/always-fails.cy.js new file mode 100644 index 000000000000..1728d87f56c8 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/tests/always-fails.cy.js @@ -0,0 +1,5 @@ +describe('always fails', () => { + it('always fails', function () { + expect(true).to.be.false + }) +}) diff --git a/system-tests/projects/experimental-retries/cypress/e2e/tests/always-passes.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/tests/always-passes.cy.js new file mode 100644 index 000000000000..40154fb58d0a --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/tests/always-passes.cy.js @@ -0,0 +1,5 @@ +describe('always passes', () => { + it('always passes', function () { + expect(true).to.be.true + }) +}) diff --git a/system-tests/projects/experimental-retries/cypress/e2e/tests/deterministic-flaky.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/tests/deterministic-flaky.cy.js new file mode 100644 index 000000000000..99996e5ebc2f --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/tests/deterministic-flaky.cy.js @@ -0,0 +1,15 @@ +describe('deterministic flaky test', () => { + it('deterministically runs pass/fail on this test', function () { + // this means the test WILL behave as + // first attempt (FAIL) + // second attempt (PASS) + // third attempt (FAIL) + // fourth attempt (PASS) + // fifth attempt (FAIL)... + if (this.test.currentRetry() % 2) { + expect(true).to.be.true + } else { + expect(true).to.be.false + } + }) +}) diff --git a/system-tests/test/experimental_retries.spec.ts b/system-tests/test/experimental_retries.spec.ts new file mode 100644 index 000000000000..dc4c79ac0485 --- /dev/null +++ b/system-tests/test/experimental_retries.spec.ts @@ -0,0 +1,187 @@ +import systemTests from '../lib/system-tests' + +describe('e2e retries.experimentalStrategy', () => { + systemTests.setup() + + describe('experimentalBurnIn=false', () => { + describe('"detect-flake-and-pass-on-threshold"', () => { + describe('passes', () => { + systemTests.it('only runs the first attempt of the test if the test passes', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/always-passes.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 10, + passesRequired: 3, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 3, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts)', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + describe('fails', () => { + systemTests.it('short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 5, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 6, + passesRequired: 4, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + }) + + describe('"detect-flake-but-always-fail"', () => { + describe('passes', () => { + systemTests.it('only runs the first attempt of the test if the test passes', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/always-passes.cy.js', + snapshot: true, + expectedExitCode: 0, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + + describe('fails', () => { + systemTests.it('runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + + systemTests.it('runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"', { + project: 'experimental-retries', + browser: '!webkit', + spec: 'tests/deterministic-flaky.cy.js', + snapshot: true, + expectedExitCode: 1, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: true, + }, + }, + screenshotOnRunFailure: false, + }, + }) + }) + }) + }) +})