From 4d55a026d2d5d1920018c9b79a6d969b875fa11e Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 19 Dec 2023 12:57:03 +0100 Subject: [PATCH] feat: add --no-isolate flag to improve performance, add documentation about performance (#4777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ari Perkkiö Co-authored-by: Pascal Jufer --- docs/config/index.md | 15 ++++ docs/guide/cli.md | 1 + docs/guide/performance.md | 51 +++++++++++ packages/vitest/src/defaults.ts | 1 + packages/vitest/src/node/cli.ts | 1 + packages/vitest/src/node/core.ts | 1 + packages/vitest/src/node/plugins/index.ts | 10 +++ packages/vitest/src/types/config.ts | 9 ++ test/config/test/resolution.test.ts | 104 ++++++++++++++++++++++ test/run-once/vitest.config.ts | 2 +- test/stacktraces/fixtures/vite.config.ts | 6 +- 11 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 docs/guide/performance.md create mode 100644 test/config/test/resolution.test.ts diff --git a/docs/config/index.md b/docs/config/index.md index 953efe03c274..7150b84bb24a 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -2066,3 +2066,18 @@ Tells fake timers to clear "native" (i.e. not fake) timers by delegating to thei - **Version:** Since Vitest 1.1.0 Path to a [workspace](/guide/workspace) config file relative to [root](#root). + +### isolate + +- **Type:** `boolean` +- **Default:** `true` +- **CLI:** `--no-isolate`, `--isolate=false` +- **Version:** Since Vitest 1.1.0 + +Run tests in an isolated environment. This option has no effect on `vmThreads` pool. + +Disabling this option might improve [performance](/guide/performance) if your code doesn't rely on side effects (which is usually true for projects with `node` environment). + +::: note +You can disable isolation for specific pools by using [`poolOptions`](#pooloptions) property. +::: diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b133f157f6cd..72d0dc2f96fb 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -81,6 +81,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim | `--outputFile ` | Write test results to a file when the `--reporter=json` or `--reporter=junit` option is also specified
Via [cac's dot notation] you can specify individual outputs for multiple reporters | | `--coverage` | Enable coverage report | | `--run` | Do not watch | +| `--isolate` | Run every test file in isolation. To disable isolation, use --no-isolate (default: `true`) | | `--mode ` | Override Vite mode (default: `test`) | | `--workspace ` | Path to a workspace configuration file | | `--globals` | Inject APIs globally | diff --git a/docs/guide/performance.md b/docs/guide/performance.md new file mode 100644 index 000000000000..2462221053c6 --- /dev/null +++ b/docs/guide/performance.md @@ -0,0 +1,51 @@ +# Performance + +By default Vitest runs every test file in an isolated environment based on the [pool](/config/#pool): + +- `threads` pool runs every test file in a separate [`Worker`](https://nodejs.org/api/worker_threads.html#class-worker) +- `forks` pool runs every test file in a separate [forked child process](https://nodejs.org/api/child_process.html#child_processforkmodulepath-args-options) +- `vmThreads` pool runs every test file in a separate [VM context](https://nodejs.org/api/vm.html#vmcreatecontextcontextobject-options), but it uses workers for parallelism + +This greatly increases test times, which might not be desirable for projects that don't rely on side effects and properly cleanup their state (which is usually true for projects with `node` environment). In this case disabling isolation will improve the speed of your tests. To do that, you can provide `--no-isolate` flag to the CLI or set [`test.isolate`](/config/#isolate) property in the config to `false`. If you are using several pools at once with `poolMatchGlobs`, you can also disable isolation for a specific pool you are using. + +::: code-group +```bash [CLI] +vitest --no-isolate +``` +```ts [vitest.config.js] +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + isolate: false, + // you can also disable isolation only for specific pools + poolOptions: { + forks: { + isolate: false, + }, + }, + }, +}) +``` +::: + +::: note +If you are using `vmThreads` pool, you cannot disable isolation. Use `threads` pool instead to improve your tests performance. +::: + +For some projects, it might also be desirable to disable parallelism to improve startup time. To do that, provide `--no-file-parallelism` flag to the CLI or set [`test.fileParallelism`](/config/#fileParallelism) property in the config to `false`. + +::: code-group +```bash [CLI] +vitest --no-file-parallelism +``` +```ts [vitest.config.js] +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + fileParallelism: false, + }, +}) +``` +::: \ No newline at end of file diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index 2d1b2ad57e8a..cbc5570ffdd9 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -62,6 +62,7 @@ export const fakeTimersDefaults = { const config = { allowOnly: !isCI, + isolate: true, watch: !isCI, globals: false, environment: 'node' as const, diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index c760b643017b..1d28f29bf8ce 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -30,6 +30,7 @@ cli .option('--run', 'Disable watch mode') .option('--mode ', 'Override Vite mode (default: test)') .option('--workspace ', 'Path to a workspace configuration file') + .option('--isolate', 'Run every test file in isolation. To disable isolation, use --no-isolate (default: true)') .option('--globals', 'Inject apis globally') .option('--dom', 'Mock browser API with happy-dom') .option('--browser [options]', 'Run tests in the browser (default: false)') diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 09fa54bc6286..fcde2720a58d 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -281,6 +281,7 @@ export class Vitest { 'testNamePattern', 'passWithNoTests', 'bail', + 'isolate', ] as const const cliOverrides = overridesOptions.reduce((acc, name) => { diff --git a/packages/vitest/src/node/plugins/index.ts b/packages/vitest/src/node/plugins/index.ts index 6ffeabc06572..df0d95783ced 100644 --- a/packages/vitest/src/node/plugins/index.ts +++ b/packages/vitest/src/node/plugins/index.ts @@ -51,6 +51,13 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t ) testConfig.api = resolveApiServerConfig(testConfig) + testConfig.poolOptions ??= {} + testConfig.poolOptions.threads ??= {} + testConfig.poolOptions.forks ??= {} + // prefer --poolOptions.{name}.isolate CLI arguments over --isolate, but still respect config value + testConfig.poolOptions.threads.isolate = options.poolOptions?.threads?.isolate ?? options.isolate ?? testConfig.poolOptions.threads.isolate ?? viteConfig.test?.isolate + testConfig.poolOptions.forks.isolate = options.poolOptions?.forks?.isolate ?? options.isolate ?? testConfig.poolOptions.forks.isolate ?? viteConfig.test?.isolate + // store defines for globalThis to make them // reassignable when running in worker in src/runtime/setup.ts const defines: Record = deleteDefineConfig(viteConfig) @@ -91,6 +98,9 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t allow: resolveFsAllow(getRoot(), testConfig.config), }, }, + test: { + poolOptions: testConfig.poolOptions, + }, } // chokidar fsevents is unstable on macos when emitting "ready" event diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 88a69c5aa642..e3605e7c7633 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -280,6 +280,15 @@ export interface InlineConfig { */ environmentMatchGlobs?: [string, VitestEnvironment][] + /** + * Run tests in an isolated environment. This option has no effect on vmThreads pool. + * + * Disabling this option might improve performance if your code doesn't rely on side effects. + * + * @default true + */ + isolate?: boolean + /** * Pool used to run tests in. * diff --git a/test/config/test/resolution.test.ts b/test/config/test/resolution.test.ts new file mode 100644 index 000000000000..a222e7812692 --- /dev/null +++ b/test/config/test/resolution.test.ts @@ -0,0 +1,104 @@ +import type { UserConfig } from 'vitest' +import { describe, expect, it } from 'vitest' +import { createVitest } from 'vitest/node' + +async function config(cliOptions: UserConfig, configValue: UserConfig = {}) { + const vitest = await createVitest('test', { ...cliOptions, watch: false }, { test: configValue }) + return vitest.config +} + +describe('correctly defines isolated flags', async () => { + it('prefers CLI poolOptions flags over config', async () => { + const c = await config({ + isolate: true, + poolOptions: { + threads: { + isolate: false, + }, + forks: { + isolate: false, + }, + }, + }) + expect(c.poolOptions?.threads?.isolate).toBe(false) + expect(c.poolOptions?.forks?.isolate).toBe(false) + expect(c.isolate).toBe(true) + }) + + it('override CLI poolOptions flags over isolate', async () => { + const c = await config({ + isolate: false, + poolOptions: { + threads: { + isolate: true, + }, + forks: { + isolate: true, + }, + }, + }, { + poolOptions: { + threads: { + isolate: false, + }, + forks: { + isolate: false, + }, + }, + }) + expect(c.poolOptions?.threads?.isolate).toBe(true) + expect(c.poolOptions?.forks?.isolate).toBe(true) + expect(c.isolate).toBe(false) + }) + + it('override CLI isolate flag if poolOptions is not set via CLI', async () => { + const c = await config({ + isolate: true, + }, { + poolOptions: { + threads: { + isolate: false, + }, + forks: { + isolate: false, + }, + }, + }) + expect(c.poolOptions?.threads?.isolate).toBe(true) + expect(c.poolOptions?.forks?.isolate).toBe(true) + expect(c.isolate).toBe(true) + }) + + it('keeps user configured poolOptions if no CLI flag is provided', async () => { + const c = await config({}, { + poolOptions: { + threads: { + isolate: false, + }, + forks: { + isolate: false, + }, + }, + }) + expect(c.poolOptions?.threads?.isolate).toBe(false) + expect(c.poolOptions?.forks?.isolate).toBe(false) + expect(c.isolate).toBe(true) + }) + + it('isolate config value overrides poolOptions defaults', async () => { + const c = await config({}, { + isolate: false, + }) + expect(c.poolOptions?.threads?.isolate).toBe(false) + expect(c.poolOptions?.forks?.isolate).toBe(false) + expect(c.isolate).toBe(false) + }) + + it('if no isolation is defined in the config, fallback ot undefined', async () => { + const c = await config({}, {}) + expect(c.poolOptions?.threads?.isolate).toBe(undefined) + expect(c.poolOptions?.forks?.isolate).toBe(undefined) + // set in configDefaults, so it's always defined + expect(c.isolate).toBe(true) + }) +}) diff --git a/test/run-once/vitest.config.ts b/test/run-once/vitest.config.ts index 8d93d710efcb..9ca84b2b94b0 100644 --- a/test/run-once/vitest.config.ts +++ b/test/run-once/vitest.config.ts @@ -2,6 +2,6 @@ import { defineConfig } from 'vite' export default defineConfig({ test: { - poolOptions: { threads: { isolate: false } }, + isolate: false, }, }) diff --git a/test/stacktraces/fixtures/vite.config.ts b/test/stacktraces/fixtures/vite.config.ts index 54b0069ddff0..64d31c38f6d1 100644 --- a/test/stacktraces/fixtures/vite.config.ts +++ b/test/stacktraces/fixtures/vite.config.ts @@ -41,12 +41,8 @@ export default defineConfig({ }, }], test: { + isolate: false, pool: 'forks', - poolOptions: { - forks: { - isolate: false, - }, - }, include: ['**/*.{test,spec}.{imba,?(c|m)[jt]s?(x)}'], setupFiles: ['./setup.js'], },