Skip to content

Commit

Permalink
test: clean up skipped flaky tests (#25671)
Browse files Browse the repository at this point in the history
  • Loading branch information
flotwig authored Feb 22, 2023
1 parent e8ab1b5 commit 4a47081
Show file tree
Hide file tree
Showing 7 changed files with 14 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,8 @@ describe('GitDataSource', () => {
sinon.restore()
})

// TODO: Very flaky on CI, even with a bunch of retries it still fails more often than nought
// see: https://github.com/cypress-io/cypress/issues/25824
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
it.skip(`gets correct status for files on ${os.platform()}`, async function () {
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
this.retries(15)

const onBranchChange = sinon.stub()
const onGitInfoChange = sinon.stub()
const onError = sinon.stub()
Expand Down
1 change: 0 additions & 1 deletion packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ describe('Proxy Logging', () => {
.visit('/fixtures/empty.html')
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23420
it('intercept log has consoleProps with intercept info', (done) => {
cy.intercept('/some-url', 'stubbed response').as('alias')
.then(() => {
Expand Down
4 changes: 2 additions & 2 deletions packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => {
cy.get('a[data-cy="xhr-fetch-requests-onload"]').click()
})

// TODO: the xhr event is showing up twice in the log, which is wrong and causing flake. skipping until: https://github.com/cypress-io/cypress/issues/23840 is addressed.
// TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840
it.skip('verifies XHR requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => {
cy.origin('http://www.foobar.com:3500', () => {
// need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js
Expand All @@ -100,7 +100,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23437
// TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840
it.skip('verifies fetch requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => {
cy.origin('http://www.foobar.com:3500', () => {
// need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js
Expand Down
2 changes: 1 addition & 1 deletion packages/driver/cypress/e2e/e2e/origin/validation.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ describe('cy.origin', { browser: '!webkit' }, () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23451
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23451
it.skip('uses cy.origin twice', () => {
cy.visit('/fixtures/auth/index.html')
cy.get('[data-cy="login-idp"]').click()
Expand Down
9 changes: 5 additions & 4 deletions packages/launchpad/cypress/e2e/project-setup.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,8 @@ describe('Launchpad: Setup Project', () => {
})
})

describe('Command for package managers', () => {
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153
describe('Command for package managers', { retries: 15 }, () => {
it('makes the right command for yarn', () => {
scaffoldAndOpenProject('pristine-yarn')

Expand All @@ -578,7 +579,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.findByTestId('terminal-prompt-input').should('have.value', 'pnpm install -D react-scripts react-dom react')
cy.findByDisplayValue('pnpm install -D react-scripts react-dom react')
})

it('makes the right command for npm', () => {
Expand All @@ -590,7 +591,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.findByTestId('terminal-prompt-input').should('have.value', 'npm install -D react-scripts react-dom react')
cy.findByDisplayValue('npm install -D react-scripts react-dom react')
})
})

Expand Down Expand Up @@ -655,7 +656,7 @@ describe('Launchpad: Setup Project', () => {
verifyScaffoldedFiles('e2e')
})

// TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23418
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23418
it.skip('takes the user to first step of ct setup when switching from app', () => {
scaffoldAndOpenProject('pristine-with-e2e-testing')
cy.visitLaunchpad()
Expand Down
2 changes: 1 addition & 1 deletion packages/server/test/integration/cypress_spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -1791,7 +1791,7 @@ describe('lib/cypress', () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23149
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23149
it.skip('passes filtered options to Project#open and sets cli config', async function () {
const open = sinon.stub(ServerE2E.prototype, 'open').resolves([])

Expand Down
6 changes: 4 additions & 2 deletions system-tests/test/record_spec.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-console */
const _ = require('lodash')
const path = require('path')
const Promise = require('bluebird')
Expand Down Expand Up @@ -342,7 +343,7 @@ describe('e2e record', () => {

setupStubbedServer(routes)

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23152
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23152
it.skip('passes in parallel with group', function () {
this.retries(3)

Expand All @@ -366,6 +367,7 @@ describe('e2e record', () => {

// stagger the 2nd run
// starting up a bit
// NOTE: this is probably why this test flakes - despite waiting 3s, sometimes the second instance finishes first
Promise
.delay(3000)
.then(() => {
Expand All @@ -392,7 +394,7 @@ describe('e2e record', () => {
context('metadata', () => {
setupStubbedServer(createRoutes())

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23151
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23151
it.skip('sends Studio usage metadata', function () {
return systemTests.exec(this, {
key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',
Expand Down

5 comments on commit 4a47081

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 4a47081 Feb 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.7.0/linux-arm64/develop-4a47081c68d5d5c3a252485ee54cdc09517f1c47/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 4a47081 Feb 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.7.0/linux-x64/develop-4a47081c68d5d5c3a252485ee54cdc09517f1c47/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 4a47081 Feb 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.7.0/darwin-arm64/develop-4a47081c68d5d5c3a252485ee54cdc09517f1c47/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 4a47081 Feb 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.7.0/darwin-x64/develop-4a47081c68d5d5c3a252485ee54cdc09517f1c47/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 4a47081 Feb 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the win32 x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/12.7.0/win32-x64/develop-4a47081c68d5d5c3a252485ee54cdc09517f1c47/cypress.tgz

Please sign in to comment.