Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: better validation messages for experimental retries #28214

Merged
merged 8 commits into from
Dec 5, 2023
Merged

chore: better validation messages for experimental retries #28214

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fb8ec87
chore: better validation messages for experimental retries
MuazOthman Nov 1, 2023
1de22d4
Merge branch 'develop' into muaz/CYCLOUD-1765-better-retries-validation
MuazOthman Nov 14, 2023
be7c2ba
Validation for openMode/runMode w/o experiment
MuazOthman Nov 15, 2023
96f1cea
Fixed error message assertion
MuazOthman Nov 16, 2023
2fbca46
Fixed validation
MuazOthman Nov 17, 2023
0d84d44
Merge branch 'develop' into muaz/CYCLOUD-1765-better-retries-validation
MuazOthman Nov 21, 2023
850fc62
Merge branch 'develop' into muaz/CYCLOUD-1765-better-retries-validation
MuazOthman Nov 22, 2023
899ea43
Merge branch 'develop' into muaz/CYCLOUD-1765-better-retries-validation
MuazOthman Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 69 additions & 173 deletions packages/config/__snapshots__/validation.spec.ts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ exports['invalid retry object'] = {
'value': {
'fakeMode': 1,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls',
}

exports['not qualified url'] = {
Expand Down Expand Up @@ -263,11 +263,9 @@ exports['invalid upper bound'] = {
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'foo',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalStrategy',
'value': 'foo',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options (valid) 1'] = {
Expand All @@ -280,216 +278,114 @@ exports['config/src/validation .isValidRetriesConfig experimental options fails
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-but-always-fail',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-but-always-fail" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-and-pass-on-threshold" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = {
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': -4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': -4,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'passesRequired': 3,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired that is greater than maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 5,
'type': 'a positive whole number less than or equal to maxRetries',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides stopIfAnyPassed option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'stopIfAnyPassed': true,
},
'maxRetries': 3,
'stopIfAnyPassed': true,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, passesRequired',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides passesRequired option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 2,
},
'maxRetries': 3,
'passesRequired': 2,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, stopIfAnyPassed',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides stopIfAnyPassed without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'stopIfAnyPassed': false,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (0 and 1 do not work) 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 2,
'stopIfAnyPassed': 1,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.stopIfAnyPassed',
'value': 1,
'type': 'null or boolean',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options (valid) 1'] = {
'key': 'mockConfigKey.experimentalStrategy',
'value': 'bar',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}

exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}
Loading
Loading