chore: add sanity system tests to verify console reporter output for …

…experimental retries logic

system-tests/projects/experimental-retries/cypress.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  e2e: {
+    supportFile: false,
+    setupNodeEvents (on, config) {
+      // in the case the tests needed to be debugged:
+
+      // on('before:browser:launch', (browser, launchOptions) => {
+      //   launchOptions.args.push('--auto-open-devtools-for-tabs')
+
+      //   return launchOptions
+      // })
+    },
+  },
+}

system-tests/projects/experimental-retries/cypress/e2e/tests/always-fails.cy.js

@@ -0,0 +1,5 @@
+describe('always fails', () => {
+  it('always fails', function () {
+    expect(true).to.be.false
+  })
+})

system-tests/projects/experimental-retries/cypress/e2e/tests/always-passes.cy.js

@@ -0,0 +1,5 @@
+describe('always passes', () => {
+  it('always passes', function () {
+    expect(true).to.be.true
+  })
+})

system-tests/projects/experimental-retries/cypress/e2e/tests/deterministic-flaky.cy.js

@@ -0,0 +1,15 @@
+describe('deterministic flaky test', () => {
+  it('deterministically runs pass/fail on this test', function () {
+    // this means the test WILL behave as
+    // first attempt (FAIL)
+    // second attempt (PASS)
+    // third attempt (FAIL)
+    // fourth attempt (PASS)
+    // fifth attempt (FAIL)...
+    if (this.test.currentRetry() % 2) {
+      expect(true).to.be.true
+    } else {
+      expect(true).to.be.false
+    }
+  })
+})

system-tests/test/experimental_retries.spec.ts

@@ -0,0 +1,187 @@
+import systemTests from '../lib/system-tests'
+
+describe('e2e retries.experimentalStrategy', () => {
+  systemTests.setup()
+
+  describe('experimentalBurnIn=false', () => {
+    describe('"detect-flake-and-pass-on-threshold"', () => {
+      describe('passes', () => {
+        systemTests.it('only runs the first attempt of the test if the test passes', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/always-passes.cy.js',
+          snapshot: true,
+          expectedExitCode: 0,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+              experimentalOptions: {
+                maxRetries: 10,
+                passesRequired: 3,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+
+        systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 0,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+              experimentalOptions: {
+                maxRetries: 9,
+                passesRequired: 3,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+
+        systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts)', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 0,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+              experimentalOptions: {
+                maxRetries: 9,
+                passesRequired: 5,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+      })
+
+      describe('fails', () => {
+        systemTests.it('short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 1,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+              experimentalOptions: {
+                maxRetries: 5,
+                passesRequired: 5,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+
+        systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 1,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+              experimentalOptions: {
+                maxRetries: 6,
+                passesRequired: 4,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+      })
+    })
+
+    describe('"detect-flake-but-always-fail"', () => {
+      describe('passes', () => {
+        systemTests.it('only runs the first attempt of the test if the test passes', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/always-passes.cy.js',
+          snapshot: true,
+          expectedExitCode: 0,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-but-always-fail',
+              experimentalOptions: {
+                maxRetries: 9,
+                stopIfAnyPassed: false,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+      })
+
+      describe('fails', () => {
+        systemTests.it('runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 1,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-but-always-fail',
+              experimentalOptions: {
+                maxRetries: 9,
+                stopIfAnyPassed: false,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+
+        systemTests.it('runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"', {
+          project: 'experimental-retries',
+          browser: '!webkit',
+          spec: 'tests/deterministic-flaky.cy.js',
+          snapshot: true,
+          expectedExitCode: 1,
+          config: {
+            experimentalBurnIn: false,
+            retries: {
+              openMode: false,
+              runMode: true,
+              experimentalStrategy: 'detect-flake-but-always-fail',
+              experimentalOptions: {
+                maxRetries: 9,
+                stopIfAnyPassed: true,
+              },
+            },
+            screenshotOnRunFailure: false,
+          },
+        })
+      })
+    })
+  })
+})