Skip to content

Commit

Permalink
chore: better validation messages for experimental retries (#28214)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MuazOthman
MuazOthman authored Dec 5, 2023
1 parent fff6ce9 commit 522835d
Show file tree
Hide file tree
Showing 4 changed files with 246 additions and 243 deletions.
265 changes: 85 additions & 180 deletions packages/config/__snapshots__/validation.spec.ts.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ exports['invalid retry object'] = {
'value': {
'fakeMode': 1,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls',
}

exports['not qualified url'] = {
Expand Down Expand Up @@ -263,233 +263,138 @@ exports['invalid upper bound'] = {
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'foo',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalStrategy',
'value': 'foo',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options (valid) 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': true,
'openMode': false,
'experimentalStrategy': 'bar',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-but-always-fail',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalStrategy',
'value': 'bar',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-but-always-fail" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-and-pass-on-threshold" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = {
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': -4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': -4,
'type': 'a positive whole number less than or equals to maxRetries',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 0,
'type': 'a positive whole number less than or equals to maxRetries',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 3.5,
'type': 'a positive whole number less than or equals to maxRetries',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'passesRequired': 3,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired that is greater than maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 5,
'type': 'a positive whole number less than or equals to maxRetries',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides stopIfAnyPassed option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'stopIfAnyPassed': true,
},
'maxRetries': 3,
'passesRequired': 2,
'stopIfAnyPassed': true,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, passesRequired',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides passesRequired option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 2,
},
'maxRetries': 3,
'passesRequired': 2,
'stopIfAnyPassed': true,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, stopIfAnyPassed',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides stopIfAnyPassed without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'stopIfAnyPassed': false,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (0 and 1 do not work) 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 2,
'stopIfAnyPassed': 1,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.stopIfAnyPassed',
'value': 1,
'type': 'a boolean',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}

exports['config/src/validation .isValidRetriesConfig returns error message for openMode as boolean without strategy 1'] = {
'key': 'mockConfigKey.openMode',
'value': true,
'type': 'a number since no experimental strategy is provided',
}

exports['config/src/validation .isValidRetriesConfig returns error message for runMode as boolean without strategy 1'] = {
'key': 'mockConfigKey.runMode',
'value': true,
'type': 'a number since no experimental strategy is provided',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides maxRetries without stopIfAnyPassed 1'] = {
'key': 'mockConfigKey.experimentalOptions.stopIfAnyPassed',
'type': 'is required when using the "detect-flake-but-always-fail" strategy',
}
Loading

5 comments on commit 522835d

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 522835d Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.6.1/linux-arm64/develop-522835d3858ad84316535bb244d6c7b8d7a3596e/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 522835d Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.6.1/linux-x64/develop-522835d3858ad84316535bb244d6c7b8d7a3596e/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 522835d Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.6.1/darwin-x64/develop-522835d3858ad84316535bb244d6c7b8d7a3596e/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 522835d Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the win32 x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.6.1/win32-x64/develop-522835d3858ad84316535bb244d6c7b8d7a3596e/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 522835d Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.6.1/darwin-arm64/develop-522835d3858ad84316535bb244d6c7b8d7a3596e/cypress.tgz

Please sign in to comment.