Skip to content

Commit

Permalink
fix: test config overrides leak for .only execution (#18961)
Browse files Browse the repository at this point in the history
  • Loading branch information
@emilyrohrbough
emilyrohrbough authored Nov 22, 2021
1 parent de4fca3 commit b7002bf
Show file tree
Hide file tree
Showing 9 changed files with 158 additions and 1,457 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"npm-release": "node scripts/npm-release.js",
"prestart": "yarn ensure-deps",
"start": "cypress open --dev --global",
"stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src,__snapshots__ --exclude e2e.ts,cypress-tests.ts",
"stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src,__snapshots__ --exclude e2e.ts,cypress-tests.ts,*only_spec.js",
"stop-only-all": "yarn stop-only --folder packages",
"pretest": "yarn ensure-deps",
"test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{config,electron,extension,https-proxy,launcher,net-stubbing,network,proxy,rewriter,runner,runner-shared,socket}'\"",
Expand Down
2 changes: 1 addition & 1 deletion packages/config/lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ module.exports = {
options,

validate: (cfg, onErr) => {
debug('validating configuration')
debug('validating configuration', cfg)

return _.each(cfg, (value, key) => {
const validationFn = validationRules[key]
Expand Down
27 changes: 27 additions & 0 deletions packages/driver/cypress/integration/e2e/testConfigOverrides-describe-only_spec.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
const shouldNotExecute = () => {
throw new Error('Test Override validation should have failed & it block should not have executed.')
}

it('first should not run', () => {
shouldNotExecute()
})

describe('second should not run', () => {
it('test', () => {
shouldNotExecute()
})
})

describe('correctly applies overrides ', { retries: 1 }, () => {
// eslint-disable-next-line mocha/no-exclusive-tests
describe.only('when valid configuration', () => {
it('for describe.only', { baseUrl: null }, () => {
const config = Cypress.config()

expect(config.testConfigList).to.be.undefined
expect(config.unverifiedTestConfig).to.be.undefined
expect(config.baseUrl).to.be.null
expect(config.retries).to.eq(1)
})
})
})
25 changes: 25 additions & 0 deletions packages/driver/cypress/integration/e2e/testConfigOverrides-it-only_spec.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
const shouldNotExecute = () => {
throw new Error('Test Override validation should have failed & it block should not have executed.')
}

it('first should not run', () => {
shouldNotExecute()
})

describe('second should not run', () => {
it('test', () => {
shouldNotExecute()
})
})

describe('correctly applies overrides when valid configuration', { retries: 1 }, () => {
// eslint-disable-next-line mocha/no-exclusive-tests
it.only('for it.only', { baseUrl: null }, () => {
const config = Cypress.config()

expect(config.testConfigList).to.be.undefined
expect(config.unverifiedTestConfig).to.be.undefined
expect(config.baseUrl).to.be.null
expect(config.retries).to.eq(1)
})
})
32 changes: 17 additions & 15 deletions packages/driver/src/cy/testConfigOverrides.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,27 +109,29 @@ function mutateConfiguration (testConfig: ResolvedTestConfigOverride, config, en
// in order to resolve the test config upfront before test runs
// note: must return as an object to meet the dashboard recording API
export function getResolvedTestConfigOverride (test): ResolvedTestConfigOverride {
let curParent = test.parent
const testConfigList = [{
overrides: test._testConfig,
invocationDetails: test.invocationDetails,
}]

while (curParent) {
if (curParent._testConfig) {
testConfigList.unshift({
overrides: curParent._testConfig,
invocationDetails: curParent.invocationDetails,
})
let curr = test
let testConfigList: TestConfig[] = []

while (curr) {
if (curr._testConfig) {
if (curr._testConfig.testConfigList) {
// configuration for mocha function has already been processed
testConfigList = testConfigList.concat(curr._testConfig.testConfigList)
} else {
testConfigList.unshift({
overrides: curr._testConfig,
invocationDetails: curr.invocationDetails,
})
}
}

curParent = curParent.parent
curr = curr.parent
}

const testConfig = {
testConfigList: testConfigList.filter((opt) => opt.overrides !== undefined),
testConfigList: testConfigList.filter(({ overrides }) => overrides !== undefined),
// collect test overrides to send to the dashboard api when @packages/server is ran in record mode
unverifiedTestConfig: _.reduce(testConfigList, (acc, opts) => _.extend(acc, opts.overrides), {}),
unverifiedTestConfig: _.reduce(testConfigList, (acc, { overrides }) => _.extend(acc, overrides), {}),
}

return testConfig
Expand Down
2 changes: 1 addition & 1 deletion packages/driver/src/cypress/mocha.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ function overloadMochaFnForConfig (fnName, specWindow) {
overrideMochaFn(replacementFn)
}

const ui = (specWindow, _mocha, config) => {
const ui = (specWindow, _mocha) => {
// Override mocha.ui so that the pre-require event is emitted
// with the iframe's `window` reference, rather than the parent's.
_mocha.ui = function (name) {
Expand Down
120 changes: 16 additions & 104 deletions packages/runner/__snapshots__/retries.mochaEvents.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1007,19 +1007,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test:before:run",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"order": 1,
Expand Down Expand Up @@ -1057,19 +1046,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"order": 1,
Expand Down Expand Up @@ -1152,19 +1130,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"retry",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"order": 1,
Expand Down Expand Up @@ -1264,19 +1231,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test:after:run",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"order": 1,
Expand Down Expand Up @@ -1353,19 +1309,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test:before:run",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"title": "test 2",
Expand Down Expand Up @@ -1472,19 +1417,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"pass",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"title": "test 2",
Expand Down Expand Up @@ -1533,19 +1467,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test end",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"title": "test 2",
Expand Down Expand Up @@ -1608,19 +1531,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
"test:after:run",
{
"_testConfig": {
"testConfigList": [
{
"overrides": {
"testConfigList": [],
"unverifiedTestConfig": {}
},
"invocationDetails": "{Object 9}"
}
],
"unverifiedTestConfig": {
"testConfigList": [],
"unverifiedTestConfig": {}
}
"testConfigList": [],
"unverifiedTestConfig": {}
},
"id": "r3",
"title": "test 2",
Expand Down
Loading

3 comments on commit b7002bf

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on b7002bf Nov 22, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.1.0/circle-develop-b7002bfd52b8e9db5b761ec571bba0a0524f3555/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on b7002bf Nov 22, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AppVeyor has built the win32 x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.1.0/appveyor-develop-b7002bfd52b8e9db5b761ec571bba0a0524f3555/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on b7002bf Nov 22, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.1.0/circle-develop-b7002bfd52b8e9db5b761ec571bba0a0524f3555/cypress.tgz

Please sign in to comment.