feat: experimental retries

.circleci/workflows.yml

@@ -29,6 +29,8 @@ mainBuildFilters: &mainBuildFilters
         - develop
         - /^release\/\d+\.\d+\.\d+$/
         # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
+        - 'update-v8-snapshot-cache-on-develop'
+        - 'feature/experimental-retries'
         - 'publish-binary'
         - 'chore/update_electron25_and_node18'
 
@@ -41,17 +43,14 @@ macWorkflowFilters: &darwin-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/update_electron25_and_node18', << pipeline.git.branch >> ]
-    - matches:
-        pattern: /^release\/\d+\.\d+\.\d+$/
-        value: << pipeline.git.branch >>
-
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
 linuxArm64WorkflowFilters: &linux-arm64-workflow-filters
   when:
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
-    - equal: [ 'jordanpowell88/update-angular-tsconfig-path', << pipeline.git.branch >> ]
+    - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
     - equal: [ 'publish-binary', << pipeline.git.branch >> ]
     - equal: [ 'chore/update_electron25_and_node18', << pipeline.git.branch >> ]
     - matches:
@@ -76,7 +75,7 @@ windowsWorkflowFilters: &windows-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'chore/update_electron25_and_node18', << pipeline.git.branch >> ]
+    - equal: [ 'feature/experimental-retries', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>
@@ -146,7 +145,7 @@ commands:
           name: Set environment variable to determine whether or not to persist artifacts
           command: |
             echo "Setting SHOULD_PERSIST_ARTIFACTS variable"
-            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" && "$CIRCLE_BRANCH" != "chore/update_electron25_and_node18" ]]; then
+            echo 'if ! [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "publish-binary" && "$CIRCLE_BRANCH" != "update-v8-snapshot-cache-on-develop" && "$CIRCLE_BRANCH" != "feature/experimental-retries" ]]; then
                 export SHOULD_PERSIST_ARTIFACTS=true
             fi' >> "$BASH_ENV"
   # You must run `setup_should_persist_artifacts` command and be using bash before running this command

.github/workflows/update_v8_snapshot_cache.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - 'release/**'
+      - 'feature/experimental-retries'
   workflow_dispatch:
     inputs:
       branch:

cli/CHANGELOG.md

@@ -1,8 +1,20 @@
 <!-- See the ../guides/writing-the-cypress-changelog.md for details on writing the changelog. -->
-## 13.3.1
+## 13.4.0
 
 _Released 10/03/2023 (PENDING)_
 
+**Features:**
+
+- Users can now specify different retry options with the `experimentalRetries` configuration. These configuration options can be useful when detecting flake and figuring out under what conditions you would like to pass a retried test. The `retries` configuration option includes two new keys, `experimentalStrategy` and `experimentalOptions`. The first, `experimentalStrategy`, can be one of two options:
+    - `detect-flake-and-pass-on-threshold`
+    - `detect-flake-but-always-fail`
+
+  `detect-flake-and-pass-on-threshold` is similar to how retries work today. Users will be able to specify a retry limit and specify a required number of passes needed in order for that test to be marked as passed. Today, that limit is `1`, but users can set the limit to whatever they'd like with this strategy by specifying the `experimentalOptions.passesRequired` key in their global configuration. This can be useful if a test is retrying and is known to be flaky, but you want to make sure that the test can at least pass a certain threshold in order to be marked as passed overall, which might help in confidence that the test is just flaky and the failure is not considered legitimate. It's important to note that if the `passesRequired` can not be achieved by the remaining retries of the test, the test will stop retrying and the test will be marked as a failure.
+
+  The other, `detect-flake-but-always-fail`, will always mark a test that enters retries as failed, regardless of how many retried attempts pass. There is an option here, called `experimentalOptions.stopIfAnyPassed`, that will stop the test retries if any retry attempt passes. Without this option, you might want to see what the probability that your test might pass (similar to `detect-flake-and-pass-on-threshold`, but mark the test as failed as you might not want to mark any detected flaky test as a passed test.
+
+  Both of these strategies require the `experimentalOptions.maxRetries` option to be set, which will be the maximum number of retries a test might run. Addressed in [#27930](https://github.com/cypress-io/cypress/pull/27930).
+
 **Bugfixes:**
 
 - Fixed an issue where requests were correlated in the wrong order in the proxy. This could cause an issue where the wrong request is used for `cy.intercept` or assets (e.g. stylesheets or images) may not properly be available in Test Replay. Addressed in [#27892](https://github.com/cypress-io/cypress/pull/27892).

cli/types/cypress.d.ts

@@ -2851,6 +2851,30 @@ declare namespace Cypress {
     certs: PEMCert[] | PFXCert[]
   }
 
+  type RetryStrategyWithModeSpecs = RetryStrategy & {
+    openMode: boolean; // defaults to false
+    runMode: boolean; // defaults to true
+  }
+
+  type RetryStrategy =
+    | RetryStrategyDetectFlakeAndPassOnThresholdType
+    | RetryStrategyDetectFlakeButAlwaysFailType
+
+  interface RetryStrategyDetectFlakeAndPassOnThresholdType {
+    experimentalStrategy: "detect-flake-and-pass-on-threshold"
+    experimentalOptions?: {
+      maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0
+      passesRequired: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0 and <= maxRetries
+    }
+  }
+
+  interface RetryStrategyDetectFlakeButAlwaysFailType {
+    experimentalStrategy: "detect-flake-but-always-fail"
+    experimentalOptions?: {
+      maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0
+      stopIfAnyPassed: boolean; // defaults to false if experimentalOptions is not provided
+    }
+  }
   interface ResolvedConfigOptions<ComponentDevServerOpts = any> {
     /**
      * Url used as prefix for [cy.visit()](https://on.cypress.io/visit) or [cy.request()](https://on.cypress.io/request) command's url
@@ -3122,7 +3146,7 @@ declare namespace Cypress {
      * To enable test retries only in runMode, set e.g. `{ openMode: null, runMode: 2 }`
      * @default null
      */
-    retries: Nullable<number | { runMode?: Nullable<number>, openMode?: Nullable<number> }>
+    retries: Nullable<number | ({ runMode?: Nullable<number>, openMode?: Nullable<number> }) | RetryStrategyWithModeSpecs>
     /**
      * Enables including elements within the shadow DOM when using querying
      * commands (e.g. cy.get(), cy.find()). Can be set globally in cypress.config.{js,ts,mjs,cjs},

cli/types/tests/cypress-tests.ts

@@ -1159,6 +1159,39 @@ namespace CypressLocalStorageTests {
   cy.clearAllSessionStorage({ log: 'true' }) // $ExpectError
 }
 
+namespace CypressRetriesSpec {
+  Cypress.config('retries', {
+    openMode: 0,
+    runMode: 1
+  })
+
+  Cypress.config('retries', {
+    openMode: false,
+    runMode: false,
+    experimentalStrategy: "detect-flake-and-pass-on-threshold",
+    experimentalOptions: {
+      maxRetries: 2,
+      passesRequired: 2
+    }
+  })
+
+  Cypress.config('retries', {
+    openMode: false,
+    runMode: false,
+    experimentalStrategy: "detect-flake-but-always-fail",
+    experimentalOptions: {
+      maxRetries: 2,
+      stopIfAnyPassed: true
+    }
+  })
+
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { maxRetries: 2} }) // $ExpectError
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { maxRetries: 2} }) // $ExpectError
+
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { passesRequired: 2} }) // $ExpectError
+  Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { stopIfAnyPassed: true} }) // $ExpectError
+}
+
 namespace CypressTraversalTests {
   cy.wrap({}).prevUntil('a') // $ExpectType Chainable<JQuery<HTMLAnchorElement>>
   cy.wrap({}).prevUntil('#myItem') // $ExpectType Chainable<JQuery<HTMLElement>>