-
Notifications
You must be signed in to change notification settings - Fork 27.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add job to test flakiness of added/changed tests (#63943)
To help detect when newly added/changed assertions are flakey this adds a job to our build and test workflow to re-run them 3 times. If the changed test is an E2E test it will re-run in both development and production mode to ensure it's not flakey specifically in one of those modes. Test run with changed test can be seen here https://github.com/vercel/next.js/actions/runs/8511797725/job/23312158523?pr=63943 Closes NEXT-2973
- Loading branch information
Showing
3 changed files
with
158 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| // @ts-check | ||
| import fs from 'fs/promises' | ||
| import execa from 'execa' | ||
| import path from 'path' | ||
|
|
||
| async function main() { | ||
| let eventData = {} | ||
|
|
||
| /** @type import('execa').Options */ | ||
| const EXECA_OPTS = { shell: true } | ||
| /** @type import('execa').Options */ | ||
| const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' } | ||
|
|
||
| try { | ||
| eventData = | ||
| JSON.parse( | ||
| await fs.readFile(process.env.GITHUB_EVENT_PATH || '', 'utf8') | ||
| )['pull_request'] || {} | ||
| } catch (_) {} | ||
|
|
||
| // detect changed test files | ||
| const branchName = | ||
| eventData?.head?.ref || | ||
| process.env.GITHUB_REF_NAME || | ||
| (await execa('git rev-parse --abbrev-ref HEAD', EXECA_OPTS)).stdout | ||
|
|
||
| const remoteUrl = | ||
| eventData?.head?.repo?.full_name || | ||
| process.env.GITHUB_REPOSITORY || | ||
| (await execa('git remote get-url origin', EXECA_OPTS)).stdout | ||
|
|
||
| const isCanary = | ||
| branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js') | ||
|
|
||
| if (isCanary) { | ||
| console.error(`Skipping flake detection for canary`) | ||
| return | ||
| } | ||
|
|
||
| try { | ||
| await execa('git remote set-branches --add origin canary', EXECA_OPTS_STDIO) | ||
| await execa('git fetch origin canary --depth=20', EXECA_OPTS_STDIO) | ||
| } catch (err) { | ||
| console.error(await execa('git remote -v', EXECA_OPTS_STDIO)) | ||
| console.error(`Failed to fetch origin/canary`, err) | ||
| } | ||
|
|
||
| const changesResult = await execa( | ||
| `git diff origin/canary --name-only`, | ||
| EXECA_OPTS | ||
| ).catch((err) => { | ||
| console.error(err) | ||
| return { stdout: '', stderr: '' } | ||
| }) | ||
| console.error( | ||
| { | ||
| branchName, | ||
| remoteUrl, | ||
| isCanary, | ||
| }, | ||
| `\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}` | ||
| ) | ||
| const changedFiles = changesResult.stdout.split('\n') | ||
|
|
||
| // run each test 3 times in each test mode (if E2E) with no-retrying | ||
| // and if any fail it's flakey | ||
| const devTests = [] | ||
| const prodTests = [] | ||
|
|
||
| for (let file of changedFiles) { | ||
| // normalize slashes | ||
| file = file.replace(/\\/g, '/') | ||
| const fileExists = await fs | ||
| .access(path.join(process.cwd(), file), fs.constants.F_OK) | ||
| .then(() => true) | ||
| .catch(() => false) | ||
|
|
||
| if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) { | ||
| if (file.startsWith('test/e2e/')) { | ||
| devTests.push(file) | ||
| prodTests.push(file) | ||
| } else if (file.startsWith('test/prod')) { | ||
| prodTests.push(file) | ||
| } else if (file.startsWith('test/development')) { | ||
| devTests.push(file) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| console.log( | ||
| 'Detected tests:', | ||
| JSON.stringify( | ||
| { | ||
| devTests, | ||
| prodTests, | ||
| }, | ||
| null, | ||
| 2 | ||
| ) | ||
| ) | ||
|
|
||
| if (prodTests.length === 0 && devTests.length === 0) { | ||
| console.log(`No added/changed tests detected`) | ||
| return | ||
| } | ||
|
|
||
| const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0'] | ||
|
|
||
| async function invokeRunTests({ mode, testFiles }) { | ||
| await execa('node', [...RUN_TESTS_ARGS, ...testFiles], { | ||
| ...EXECA_OPTS_STDIO, | ||
| env: { | ||
| ...process.env, | ||
| NEXT_TEST_MODE: mode, | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| if (devTests.length > 0) { | ||
| for (let i = 0; i < 3; i++) { | ||
| console.log(`\n\nRun ${i + 1} for dev tests`) | ||
| await invokeRunTests({ | ||
| mode: 'dev', | ||
| testFiles: devTests, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| if (prodTests.length > 0) { | ||
| for (let i = 0; i < 3; i++) { | ||
| console.log(`\n\nRun ${i + 1} for production tests`) | ||
| await invokeRunTests({ | ||
| mode: 'start', | ||
| testFiles: prodTests, | ||
| }) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| main().catch((err) => { | ||
| console.error(err) | ||
| process.exit(1) | ||
| }) |