From dde91067f0ddd7bd7199f3727fc19dd4ca1f7c07 Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Fri, 6 Oct 2023 21:28:28 -0400 Subject: [PATCH] fix(core): hash the same environment as the tasks are run with (#19487) --- docs/generated/devkit/TaskHasher.md | 40 +- .../linter/src/executors/eslint/hasher.ts | 7 +- .../command-line/affected/print-affected.ts | 12 +- packages/nx/src/daemon/client/client.ts | 5 +- .../nx/src/daemon/server/handle-hash-tasks.ts | 5 +- packages/nx/src/hasher/hash-task.ts | 6 +- packages/nx/src/hasher/set-hash-env.ts | 19 - packages/nx/src/hasher/task-hasher.spec.ts | 402 +++++++++--------- packages/nx/src/hasher/task-hasher.ts | 114 ++++- .../forked-process-task-runner.ts | 211 +-------- packages/nx/src/tasks-runner/run-command.ts | 36 +- packages/nx/src/tasks-runner/task-env.ts | 202 +++++++++ .../nx/src/tasks-runner/task-orchestrator.ts | 102 ++++- .../src/tasks-runner/tasks-schedule.spec.ts | 12 +- .../nx/src/tasks-runner/tasks-schedule.ts | 6 - 15 files changed, 691 insertions(+), 488 deletions(-) delete mode 100644 packages/nx/src/hasher/set-hash-env.ts create mode 100644 packages/nx/src/tasks-runner/task-env.ts diff --git a/docs/generated/devkit/TaskHasher.md b/docs/generated/devkit/TaskHasher.md index 7132bae2bb722..41216c7275e5b 100644 --- a/docs/generated/devkit/TaskHasher.md +++ b/docs/generated/devkit/TaskHasher.md @@ -15,7 +15,7 @@ **`Deprecated`** -use hashTask(task:Task, taskGraph: TaskGraph) +use hashTask(task:Task, taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 #### Parameters @@ -29,12 +29,30 @@ use hashTask(task:Task, taskGraph: TaskGraph) ▸ **hashTask**(`task`, `taskGraph`): `Promise`<[`Hash`](../../devkit/documents/Hash)\> +**`Deprecated`** + +use hashTask(task:Task, taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 + +#### Parameters + +| Name | Type | +| :---------- | :---------------------------------------------- | +| `task` | [`Task`](../../devkit/documents/Task) | +| `taskGraph` | [`TaskGraph`](../../devkit/documents/TaskGraph) | + +#### Returns + +`Promise`<[`Hash`](../../devkit/documents/Hash)\> + +▸ **hashTask**(`task`, `taskGraph`, `env`): `Promise`<[`Hash`](../../devkit/documents/Hash)\> + #### Parameters | Name | Type | | :---------- | :---------------------------------------------- | | `task` | [`Task`](../../devkit/documents/Task) | | `taskGraph` | [`TaskGraph`](../../devkit/documents/TaskGraph) | +| `env` | `ProcessEnv` | #### Returns @@ -48,7 +66,7 @@ use hashTask(task:Task, taskGraph: TaskGraph) **`Deprecated`** -use hashTasks(tasks:Task[], taskGraph: TaskGraph) +use hashTasks(tasks:Task[], taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 #### Parameters @@ -62,12 +80,30 @@ use hashTasks(tasks:Task[], taskGraph: TaskGraph) ▸ **hashTasks**(`tasks`, `taskGraph`): `Promise`<[`Hash`](../../devkit/documents/Hash)[]\> +**`Deprecated`** + +use hashTasks(tasks:Task[], taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 + +#### Parameters + +| Name | Type | +| :---------- | :---------------------------------------------- | +| `tasks` | [`Task`](../../devkit/documents/Task)[] | +| `taskGraph` | [`TaskGraph`](../../devkit/documents/TaskGraph) | + +#### Returns + +`Promise`<[`Hash`](../../devkit/documents/Hash)[]\> + +▸ **hashTasks**(`tasks`, `taskGraph`, `env`): `Promise`<[`Hash`](../../devkit/documents/Hash)[]\> + #### Parameters | Name | Type | | :---------- | :---------------------------------------------- | | `tasks` | [`Task`](../../devkit/documents/Task)[] | | `taskGraph` | [`TaskGraph`](../../devkit/documents/TaskGraph) | +| `env` | `ProcessEnv` | #### Returns diff --git a/packages/linter/src/executors/eslint/hasher.ts b/packages/linter/src/executors/eslint/hasher.ts index 517f118d8c35a..3f944f190eaec 100644 --- a/packages/linter/src/executors/eslint/hasher.ts +++ b/packages/linter/src/executors/eslint/hasher.ts @@ -15,9 +15,14 @@ export default async function run( projectGraph: ProjectGraph; taskGraph: TaskGraph; projectsConfigurations: ProjectsConfigurations; + env: NodeJS.ProcessEnv; } ): Promise { - const res = await context.hasher.hashTask(task, context.taskGraph); + const res = await context.hasher.hashTask( + task, + context.taskGraph, + context.env + ); if (task.overrides['hasTypeAwareRules'] === true) { return res; } diff --git a/packages/nx/src/command-line/affected/print-affected.ts b/packages/nx/src/command-line/affected/print-affected.ts index 62fe68980ed7f..17675c763cc63 100644 --- a/packages/nx/src/command-line/affected/print-affected.ts +++ b/packages/nx/src/command-line/affected/print-affected.ts @@ -15,6 +15,7 @@ import { hashTask } from '../../hasher/hash-task'; import { getPackageManagerCommand } from '../../utils/package-manager'; import { printAffectedDeprecationMessage } from './command-object'; import { logger, NX_PREFIX } from '../../utils/logger'; +import { getTaskSpecificEnv } from '../../tasks-runner/task-env'; /** * @deprecated Use showProjectsHandler, generateGraph, or affected (without the print-affected mode) instead. @@ -76,7 +77,16 @@ async function createTasks( const tasks = Object.values(taskGraph.tasks); await Promise.all( - tasks.map((t) => hashTask(hasher, projectGraph, taskGraph, t)) + tasks.map((t) => + hashTask( + hasher, + projectGraph, + taskGraph, + t, + // This loads dotenv files for the task + getTaskSpecificEnv(t) + ) + ) ); return tasks.map((task) => ({ diff --git a/packages/nx/src/daemon/client/client.ts b/packages/nx/src/daemon/client/client.ts index 17d173c683a00..ad83147df74e6 100644 --- a/packages/nx/src/daemon/client/client.ts +++ b/packages/nx/src/daemon/client/client.ts @@ -136,12 +136,13 @@ export class DaemonClient { hashTasks( runnerOptions: any, tasks: Task[], - taskGraph: TaskGraph + taskGraph: TaskGraph, + env: NodeJS.ProcessEnv ): Promise { return this.sendToDaemonViaQueue({ type: 'HASH_TASKS', runnerOptions, - env: process.env, + env, tasks, taskGraph, }); diff --git a/packages/nx/src/daemon/server/handle-hash-tasks.ts b/packages/nx/src/daemon/server/handle-hash-tasks.ts index fca80b0c822e9..7e0a85d634c38 100644 --- a/packages/nx/src/daemon/server/handle-hash-tasks.ts +++ b/packages/nx/src/daemon/server/handle-hash-tasks.ts @@ -2,7 +2,6 @@ import { Task, TaskGraph } from '../../config/task-graph'; import { getCachedSerializedProjectGraphPromise } from './project-graph-incremental-recomputation'; import { InProcessTaskHasher } from '../../hasher/task-hasher'; import { readNxJson } from '../../config/configuration'; -import { setHashEnv } from '../../hasher/set-hash-env'; /** * We use this not to recreated hasher for every hash operation @@ -17,8 +16,6 @@ export async function handleHashTasks(payload: { tasks: Task[]; taskGraph: TaskGraph; }) { - setHashEnv(payload.env); - const { projectGraph, allWorkspaceFiles, fileMap } = await getCachedSerializedProjectGraphPromise(); const nxJson = readNxJson(); @@ -34,7 +31,7 @@ export async function handleHashTasks(payload: { ); } const response = JSON.stringify( - await storedHasher.hashTasks(payload.tasks, payload.taskGraph) + await storedHasher.hashTasks(payload.tasks, payload.taskGraph, payload.env) ); return { response, diff --git a/packages/nx/src/hasher/hash-task.ts b/packages/nx/src/hasher/hash-task.ts index 4361a186149eb..e495a391d733b 100644 --- a/packages/nx/src/hasher/hash-task.ts +++ b/packages/nx/src/hasher/hash-task.ts @@ -45,7 +45,8 @@ export async function hashTask( hasher: TaskHasher, projectGraph: ProjectGraph, taskGraph: TaskGraph, - task: Task + task: Task, + env: NodeJS.ProcessEnv ) { const customHasher = await getCustomHasher(task, projectGraph); const projectsConfigurations = @@ -58,8 +59,9 @@ export async function hashTask( workspaceConfig: projectsConfigurations, // to make the change non-breaking. Remove after v18 projectsConfigurations, nxJsonConfiguration: readNxJson(), + env, } as any) - : hasher.hashTask(task, taskGraph)); + : hasher.hashTask(task, taskGraph, env)); task.hash = value; task.hashDetails = details; } diff --git a/packages/nx/src/hasher/set-hash-env.ts b/packages/nx/src/hasher/set-hash-env.ts deleted file mode 100644 index 513df4b79aa5e..0000000000000 --- a/packages/nx/src/hasher/set-hash-env.ts +++ /dev/null @@ -1,19 +0,0 @@ -// if using without the daemon, the hashEnv is always going to be the process.env. -// When using the daemon, we'll need to set the hashEnv with `setHashEnv` - -let hashEnv = process.env; - -/** - * Set the environment to be used by the hasher - * @param env - */ -export function setHashEnv(env: any) { - hashEnv = env; -} - -/** - * Get the environment used by the hasher - */ -export function getHashEnv() { - return hashEnv; -} diff --git a/packages/nx/src/hasher/task-hasher.spec.ts b/packages/nx/src/hasher/task-hasher.spec.ts index 9f2bd7d594f16..a0c5e9487b41c 100644 --- a/packages/nx/src/hasher/task-hasher.spec.ts +++ b/packages/nx/src/hasher/task-hasher.spec.ts @@ -8,7 +8,6 @@ import { filterUsingGlobPatterns, InProcessTaskHasher, } from './task-hasher'; -import { withEnvironmentVariables } from '../internal-testing-utils/with-environment'; describe('TaskHasher', () => { const packageJson = { @@ -51,90 +50,90 @@ describe('TaskHasher', () => { tempFs.cleanup(); }); - it('should create task hash', () => - withEnvironmentVariables({ TESTENV: 'env123' }, async () => { - const hasher = new InProcessTaskHasher( - { - parent: [{ file: '/file', hash: 'file.hash' }], - unrelated: [{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash' }], - }, - allWorkspaceFiles, - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: 'libs/parent', - targets: { - build: { - executor: 'nx:run-commands', - inputs: [ - 'default', - '^default', - { runtime: 'echo runtime123' }, - { env: 'TESTENV' }, - { env: 'NONEXISTENTENV' }, - { - input: 'default', - projects: ['unrelated', 'tag:some-tag'], - }, - ], - }, + it('should create task hash', async () => { + const hasher = new InProcessTaskHasher( + { + parent: [{ file: '/file', hash: 'file.hash' }], + unrelated: [{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash' }], + }, + allWorkspaceFiles, + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + targets: { + build: { + executor: 'nx:run-commands', + inputs: [ + 'default', + '^default', + { runtime: 'echo runtime123' }, + { env: 'TESTENV' }, + { env: 'NONEXISTENTENV' }, + { + input: 'default', + projects: ['unrelated', 'tag:some-tag'], + }, + ], }, }, }, - unrelated: { - name: 'unrelated', - type: 'lib', - data: { - root: 'libs/unrelated', - targets: { build: {} }, - }, - }, - tagged: { - name: 'tagged', - type: 'lib', - data: { - root: 'libs/tagged', - targets: { build: {} }, - tags: ['some-tag'], - }, + }, + unrelated: { + name: 'unrelated', + type: 'lib', + data: { + root: 'libs/unrelated', + targets: { build: {} }, }, }, - dependencies: { - parent: [], + tagged: { + name: 'tagged', + type: 'lib', + data: { + root: 'libs/tagged', + targets: { build: {} }, + tags: ['some-tag'], + }, }, - externalNodes: {}, }, + dependencies: { + parent: [], + }, + externalNodes: {}, + }, - {} as any, - { - runtimeCacheInputs: ['echo runtime456'], - } - ); + {} as any, + { + runtimeCacheInputs: ['echo runtime456'], + } + ); - const hash = await hasher.hashTask( - { - target: { project: 'parent', target: 'build' }, - id: 'parent-build', - overrides: { prop: 'prop-value' }, - }, - { - roots: ['parent-build'], - tasks: { - 'parent-build': { - id: 'parent-build', - target: { project: 'parent', target: 'build' }, - overrides: {}, - }, + const hash = await hasher.hashTask( + { + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }, + { + roots: ['parent-build'], + tasks: { + 'parent-build': { + id: 'parent-build', + target: { project: 'parent', target: 'build' }, + overrides: {}, }, - dependencies: {}, - } - ); + }, + dependencies: {}, + }, + { TESTENV: 'env123' } + ); - expect(hash).toMatchSnapshot(); - })); + expect(hash).toMatchSnapshot(); + }); it('should hash task where the project has dependencies', async () => { const hasher = new InProcessTaskHasher( @@ -200,7 +199,8 @@ describe('TaskHasher', () => { dependencies: { 'parent-build': ['child-build'], }, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -282,7 +282,8 @@ describe('TaskHasher', () => { dependencies: { 'parent-build': ['child-build'], }, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -349,7 +350,8 @@ describe('TaskHasher', () => { id: 'parent-test', overrides: { prop: 'prop-value' }, }, - taskGraph + taskGraph, + {} ); expect(test).toMatchSnapshot(); @@ -360,121 +362,119 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, }, - taskGraph + taskGraph, + {} ); expect(build).toMatchSnapshot(); }); it('should be able to handle multiple filesets per project', async () => { - await withEnvironmentVariables( - { MY_TEST_HASH_ENV: 'MY_TEST_HASH_ENV_VALUE' }, - async () => { - const hasher = new InProcessTaskHasher( - { - parent: [ - { file: 'libs/parent/filea.ts', hash: 'a.hash' }, - { file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' }, - ], - child: [ - { file: 'libs/child/fileb.ts', hash: 'b.hash' }, - { file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' }, - ], - }, - allWorkspaceFiles, - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: 'libs/parent', - targets: { - test: { - inputs: ['default', '^prod'], - executor: 'nx:run-commands', - }, - }, + const hasher = new InProcessTaskHasher( + { + parent: [ + { file: 'libs/parent/filea.ts', hash: 'a.hash' }, + { file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' }, + ], + child: [ + { file: 'libs/child/fileb.ts', hash: 'b.hash' }, + { file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + allWorkspaceFiles, + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + targets: { + test: { + inputs: ['default', '^prod'], + executor: 'nx:run-commands', }, }, - child: { - name: 'child', - type: 'lib', - data: { - root: 'libs/child', - namedInputs: { - prod: [ - '!{projectRoot}/**/*.spec.ts', - '{workspaceRoot}/global2', - { env: 'MY_TEST_HASH_ENV' }, - ], - }, - targets: { - test: { - inputs: ['default'], - executor: 'nx:run-commands', - }, - }, + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + namedInputs: { + prod: [ + '!{projectRoot}/**/*.spec.ts', + '{workspaceRoot}/global2', + { env: 'MY_TEST_HASH_ENV' }, + ], + }, + targets: { + test: { + inputs: ['default'], + executor: 'nx:run-commands', }, }, }, - externalNodes: {}, - dependencies: { - parent: [{ source: 'parent', target: 'child', type: 'static' }], - }, }, + }, + externalNodes: {}, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, - { - namedInputs: { - default: ['{projectRoot}/**/*', '{workspaceRoot}/global1'], - prod: ['!{projectRoot}/**/*.spec.ts'], - }, - } as any, - {} - ); - - const taskGraph = { - roots: ['child-test'], - tasks: { - 'parent-test': { - id: 'parent-test', - target: { project: 'parent', target: 'test' }, - overrides: {}, - }, - 'child-test': { - id: 'child-test', - target: { project: 'child', target: 'test' }, - overrides: {}, - }, - }, - dependencies: { - 'parent-test': ['child-test'], - }, - }; + { + namedInputs: { + default: ['{projectRoot}/**/*', '{workspaceRoot}/global1'], + prod: ['!{projectRoot}/**/*.spec.ts'], + }, + } as any, + {} + ); - const parentHash = await hasher.hashTask( - { - target: { project: 'parent', target: 'test' }, - id: 'parent-test', - overrides: { prop: 'prop-value' }, - }, - taskGraph - ); + const taskGraph = { + roots: ['child-test'], + tasks: { + 'parent-test': { + id: 'parent-test', + target: { project: 'parent', target: 'test' }, + overrides: {}, + }, + 'child-test': { + id: 'child-test', + target: { project: 'child', target: 'test' }, + overrides: {}, + }, + }, + dependencies: { + 'parent-test': ['child-test'], + }, + }; - expect(parentHash).toMatchSnapshot(); + const parentHash = await hasher.hashTask( + { + target: { project: 'parent', target: 'test' }, + id: 'parent-test', + overrides: { prop: 'prop-value' }, + }, + taskGraph, + { MY_TEST_HASH_ENV: 'MY_TEST_HASH_ENV_VALUE' } + ); - const childHash = await hasher.hashTask( - { - target: { project: 'child', target: 'test' }, - id: 'child-test', - overrides: { prop: 'prop-value' }, - }, - taskGraph - ); + expect(parentHash).toMatchSnapshot(); - expect(childHash).toMatchSnapshot(); - } + const childHash = await hasher.hashTask( + { + target: { project: 'child', target: 'test' }, + id: 'child-test', + overrides: { prop: 'prop-value' }, + }, + taskGraph, + { MY_TEST_HASH_ENV: 'MY_TEST_HASH_ENV_VALUE' } ); + + expect(childHash).toMatchSnapshot(); }); it('should use targetDefaults from nx.json', async () => { @@ -553,7 +553,8 @@ describe('TaskHasher', () => { dependencies: { 'parent-build': ['child-build'], }, - } + }, + {} ); expect(hash).toMatchSnapshot(); }); @@ -604,7 +605,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -672,7 +674,8 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, }, - taskGraph + taskGraph, + {} ); expect(tasksHash).toMatchSnapshot(); @@ -683,7 +686,8 @@ describe('TaskHasher', () => { id: 'child-build', overrides: { prop: 'prop-value' }, }, - taskGraph + taskGraph, + {} ); expect(hashb).toMatchSnapshot(); @@ -734,7 +738,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); fail('Should not be here'); } catch (e) { @@ -801,7 +806,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); }); @@ -865,7 +871,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); // note that the parent hash is based on parent source files only! @@ -920,7 +927,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -1014,7 +1022,8 @@ describe('TaskHasher', () => { target: { project: 'a', target: 'build' }, overrides: {}, }, - taskGraph + taskGraph, + {} ); const hashB1 = await hasher1.hashTask( { @@ -1022,7 +1031,8 @@ describe('TaskHasher', () => { target: { project: 'b', target: 'build' }, overrides: {}, }, - taskGraph + taskGraph, + {} ); const hashB2 = await hasher2.hashTask( @@ -1031,7 +1041,8 @@ describe('TaskHasher', () => { target: { project: 'b', target: 'build' }, overrides: {}, }, - taskGraph + taskGraph, + {} ); const hashA2 = await hasher2.hashTask( { @@ -1039,7 +1050,8 @@ describe('TaskHasher', () => { target: { project: 'a', target: 'build' }, overrides: {}, }, - taskGraph + taskGraph, + {} ); expect(hashA1).toEqual(hashA2); @@ -1145,7 +1157,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -1357,7 +1370,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash.details.nodes['AllExternalDependencies']).toEqual( @@ -1437,7 +1451,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -1515,7 +1530,8 @@ describe('TaskHasher', () => { }, }, dependencies: {}, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -1650,7 +1666,8 @@ describe('TaskHasher', () => { 'parent-build': ['child-build'], 'child-build': ['grandchild-build'], }, - } + }, + {} ); expect(hash).toMatchSnapshot(); @@ -1784,7 +1801,8 @@ describe('TaskHasher', () => { 'parent-build': ['child-build'], 'child-build': ['grandchild-build'], }, - } + }, + {} ); expect(hash).toMatchSnapshot(); diff --git a/packages/nx/src/hasher/task-hasher.ts b/packages/nx/src/hasher/task-hasher.ts index 1c8afed83bd52..9b53384e95e8f 100644 --- a/packages/nx/src/hasher/task-hasher.ts +++ b/packages/nx/src/hasher/task-hasher.ts @@ -16,7 +16,6 @@ import { createProjectRootMappings } from '../project-graph/utils/find-project-f import { findMatchingProjects } from '../utils/find-matching-projects'; import { hashArray, hashObject } from './file-hasher'; import { getOutputsForTargetAndConfiguration } from '../tasks-runner/utils'; -import { getHashEnv } from './set-hash-env'; import { workspaceRoot } from '../utils/workspace-root'; import { join, relative } from 'path'; import { normalizePath } from '../utils/path'; @@ -61,20 +60,38 @@ export interface Hash { export interface TaskHasher { /** - * @deprecated use hashTask(task:Task, taskGraph: TaskGraph) + * @deprecated use hashTask(task:Task, taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 * @param task */ hashTask(task: Task): Promise; + /** + * @deprecated use hashTask(task:Task, taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 + */ hashTask(task: Task, taskGraph: TaskGraph): Promise; + hashTask( + task: Task, + taskGraph: TaskGraph, + env: NodeJS.ProcessEnv + ): Promise; + /** - * @deprecated use hashTasks(tasks:Task[], taskGraph: TaskGraph) + * @deprecated use hashTasks(tasks:Task[], taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 * @param tasks */ hashTasks(tasks: Task[]): Promise; + /** + * @deprecated use hashTasks(tasks:Task[], taskGraph: TaskGraph, env: NodeJS.ProcessEnv) instead. This will be removed in v18 + */ hashTasks(tasks: Task[], taskGraph: TaskGraph): Promise; + + hashTasks( + tasks: Task[], + taskGraph: TaskGraph, + env: NodeJS.ProcessEnv + ): Promise; } export type Hasher = TaskHasher; @@ -85,13 +102,31 @@ export class DaemonBasedTaskHasher implements TaskHasher { private readonly runnerOptions: any ) {} - async hashTasks(tasks: Task[], taskGraph?: TaskGraph): Promise { - return this.daemonClient.hashTasks(this.runnerOptions, tasks, taskGraph); + async hashTasks( + tasks: Task[], + taskGraph?: TaskGraph, + env?: NodeJS.ProcessEnv + ): Promise { + return this.daemonClient.hashTasks( + this.runnerOptions, + tasks, + taskGraph, + env ?? process.env + ); } - async hashTask(task: Task, taskGraph?: TaskGraph): Promise { + async hashTask( + task: Task, + taskGraph?: TaskGraph, + env?: NodeJS.ProcessEnv + ): Promise { return ( - await this.daemonClient.hashTasks(this.runnerOptions, [task], taskGraph) + await this.daemonClient.hashTasks( + this.runnerOptions, + [task], + taskGraph, + env ?? process.env + ) )[0]; } } @@ -136,14 +171,27 @@ export class InProcessTaskHasher implements TaskHasher { ); } - async hashTasks(tasks: Task[], taskGraph?: TaskGraph): Promise { - return await Promise.all(tasks.map((t) => this.hashTask(t, taskGraph))); + async hashTasks( + tasks: Task[], + taskGraph?: TaskGraph, + env?: NodeJS.ProcessEnv + ): Promise { + return await Promise.all( + tasks.map((t) => this.hashTask(t, taskGraph, env)) + ); } - async hashTask(task: Task, taskGraph?: TaskGraph): Promise { - const res = await this.taskHasher.hashTask(task, taskGraph, [ - task.target.project, - ]); + async hashTask( + task: Task, + taskGraph?: TaskGraph, + env?: NodeJS.ProcessEnv + ): Promise { + const res = await this.taskHasher.hashTask( + task, + taskGraph, + env ?? process.env, + [task.target.project] + ); const command = this.hashCommand(task); return { value: hashArray([res.value, command]), @@ -215,6 +263,7 @@ class TaskHasherImpl { async hashTask( task: Task, taskGraph: TaskGraph, + env: NodeJS.ProcessEnv, visited: string[] ): Promise { return Promise.resolve().then(async () => { @@ -232,6 +281,7 @@ class TaskHasherImpl { depsOutputs, projectInputs, taskGraph, + env, visited ); @@ -252,6 +302,7 @@ class TaskHasherImpl { task: Task, namedInput: string, taskGraph: TaskGraph, + env: NodeJS.ProcessEnv, visited: string[] ): Promise { const projectNode = this.projectGraph.nodes[projectName]; @@ -273,6 +324,7 @@ class TaskHasherImpl { depsOutputs, [], taskGraph, + env, visited ); } @@ -285,22 +337,28 @@ class TaskHasherImpl { depsOutputs: ExpandedDepsOutput[], projectInputs: { input: string; projects: string[] }[], taskGraph: TaskGraph, + env: NodeJS.ProcessEnv, visited: string[] ) { const projectGraphDeps = this.projectGraph.dependencies[projectName] ?? []; // we don't want random order of dependencies to change the hash projectGraphDeps.sort((a, b) => a.target.localeCompare(b.target)); - const self = await this.hashSingleProjectInputs(projectName, selfInputs); + const self = await this.hashSingleProjectInputs( + projectName, + selfInputs, + env + ); const deps = await this.hashDepsInputs( task, depsInputs, projectGraphDeps, taskGraph, + env, visited ); const depsOut = await this.hashDepsOutputs(task, depsOutputs, taskGraph); - const projects = await this.hashProjectInputs(projectInputs); + const projects = await this.hashProjectInputs(projectInputs, env); return this.combinePartialHashes([ ...self, @@ -330,6 +388,7 @@ class TaskHasherImpl { inputs: { input: string }[], projectGraphDeps: ProjectGraphDependency[], taskGraph: TaskGraph, + env: NodeJS.ProcessEnv, visited: string[] ): Promise { return ( @@ -347,6 +406,7 @@ class TaskHasherImpl { task, input.input || 'default', taskGraph, + env, visited ); } else { @@ -578,7 +638,8 @@ class TaskHasherImpl { private async hashSingleProjectInputs( projectName: string, - inputs: ExpandedInput[] + inputs: ExpandedInput[], + env: NodeJS.ProcessEnv ): Promise { const filesets = extractPatternsFromFileSets(inputs); @@ -629,13 +690,16 @@ class TaskHasherImpl { ...this.legacyFilesetInputs.map((r) => r.fileset), ].map((fileset) => this.hashRootFileset(fileset)), ...[...notFilesets, ...this.legacyRuntimeInputs].map((r) => - r['runtime'] ? this.hashRuntime(r['runtime']) : this.hashEnv(r['env']) + r['runtime'] + ? this.hashRuntime(env, r['runtime']) + : this.hashEnv(env, r['env']) ), ]); } private async hashProjectInputs( - projectInputs: { input: string; projects: string[] }[] + projectInputs: { input: string; projects: string[] }[], + env: NodeJS.ProcessEnv ): Promise { const partialHashes: Promise[] = []; for (const input of projectInputs) { @@ -653,7 +717,7 @@ class TaskHasherImpl { namedInputs ); partialHashes.push( - this.hashSingleProjectInputs(project, expandedInput) + this.hashSingleProjectInputs(project, expandedInput, env) ); } } @@ -743,8 +807,10 @@ class TaskHasherImpl { return this.filesetHashes[mapKey]; } - private async hashRuntime(runtime: string): Promise { - const env = getHashEnv(); + private async hashRuntime( + env: NodeJS.ProcessEnv, + runtime: string + ): Promise { const env_key = JSON.stringify(env); const mapKey = `runtime:${runtime}-${env_key}`; if (!this.runtimeHashes[mapKey]) { @@ -777,8 +843,10 @@ class TaskHasherImpl { return this.runtimeHashes[mapKey]; } - private async hashEnv(envVarName: string): Promise { - let env = getHashEnv(); + private async hashEnv( + env: NodeJS.ProcessEnv, + envVarName: string + ): Promise { const value = hashArray([env[envVarName] ?? '']); return { details: { [`env:${envVarName}`]: value }, diff --git a/packages/nx/src/tasks-runner/forked-process-task-runner.ts b/packages/nx/src/tasks-runner/forked-process-task-runner.ts index 22a90dab04907..c7db7e75632bb 100644 --- a/packages/nx/src/tasks-runner/forked-process-task-runner.ts +++ b/packages/nx/src/tasks-runner/forked-process-task-runner.ts @@ -1,10 +1,7 @@ import { readFileSync, writeFileSync } from 'fs'; -import { config as loadDotEnvFile } from 'dotenv'; -import { expand } from 'dotenv-expand'; import { ChildProcess, fork, Serializable } from 'child_process'; import * as chalk from 'chalk'; import * as logTransformer from 'strong-log-transformer'; -import { workspaceRoot } from '../utils/workspace-root'; import { DefaultTasksRunnerOptions } from './default-tasks-runner'; import { output } from '../utils/output'; import { getCliPath, getPrintableCommandArgsForTask } from './utils'; @@ -22,7 +19,6 @@ import { Transform } from 'stream'; const workerPath = join(__dirname, './batch/run-batch.js'); export class ForkedProcessTaskRunner { - workspaceRoot = workspaceRoot; cliPath = getCliPath(); private readonly verbose = process.env.NX_VERBOSE_LOGGING === 'true'; @@ -35,7 +31,8 @@ export class ForkedProcessTaskRunner { // TODO: vsavkin delegate terminal output printing public forkProcessForBatch( { executorName, taskGraph: batchTaskGraph }: Batch, - fullTaskGraph: TaskGraph + fullTaskGraph: TaskGraph, + env: NodeJS.ProcessEnv ) { return new Promise((res, rej) => { try { @@ -56,7 +53,7 @@ export class ForkedProcessTaskRunner { const p = fork(workerPath, { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], - env: this.getEnvVariablesForProcess(), + env, }); this.processes.add(p); @@ -116,10 +113,12 @@ export class ForkedProcessTaskRunner { streamOutput, temporaryOutputPath, taskGraph, + env, }: { streamOutput: boolean; temporaryOutputPath: string; taskGraph: TaskGraph; + env: NodeJS.ProcessEnv; } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { @@ -132,14 +131,7 @@ export class ForkedProcessTaskRunner { const p = fork(this.cliPath, { stdio: ['inherit', 'pipe', 'pipe', 'ipc'], - env: this.getEnvVariablesForTask( - task, - process.env.FORCE_COLOR === undefined - ? 'true' - : process.env.FORCE_COLOR, - null, - null - ), + env, }); this.processes.add(p); @@ -217,10 +209,12 @@ export class ForkedProcessTaskRunner { streamOutput, temporaryOutputPath, taskGraph, + env, }: { streamOutput: boolean; temporaryOutputPath: string; taskGraph: TaskGraph; + env: NodeJS.ProcessEnv; } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { @@ -232,12 +226,7 @@ export class ForkedProcessTaskRunner { } const p = fork(this.cliPath, { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], - env: this.getEnvVariablesForTask( - task, - undefined, - temporaryOutputPath, - streamOutput - ), + env, }); this.processes.add(p); @@ -299,188 +288,6 @@ export class ForkedProcessTaskRunner { writeFileSync(outputPath, content); } - // region Environment Variables - private getEnvVariablesForProcess() { - return { - // User Process Env Variables override Dotenv Variables - ...process.env, - // Nx Env Variables overrides everything - ...this.getNxEnvVariablesForForkedProcess( - process.env.FORCE_COLOR === undefined ? 'true' : process.env.FORCE_COLOR - ), - }; - } - - private getEnvVariablesForTask( - task: Task, - forceColor: string, - outputPath: string, - streamOutput: boolean - ) { - // Unload any dot env files at the root of the workspace that were loaded on init of Nx. - const taskEnv = this.unloadDotEnvFiles({ ...process.env }); - - const res = { - // Start With Dotenv Variables - ...(process.env.NX_LOAD_DOT_ENV_FILES === 'true' - ? this.loadDotEnvFilesForTask(task, taskEnv) - : // If not loading dot env files, ensure env vars created by system are still loaded - taskEnv), - // Nx Env Variables overrides everything - ...this.getNxEnvVariablesForTask( - task, - forceColor, - outputPath, - streamOutput - ), - }; - - // we have to delete it because if we invoke Nx from within Nx, we need to reset those values - if (!outputPath) { - delete res.NX_TERMINAL_OUTPUT_PATH; - delete res.NX_STREAM_OUTPUT; - delete res.NX_PREFIX_OUTPUT; - } - delete res.NX_BASE; - delete res.NX_HEAD; - delete res.NX_SET_CLI; - return res; - } - - private getNxEnvVariablesForForkedProcess( - forceColor: string, - outputPath?: string, - streamOutput?: boolean - ) { - const env: NodeJS.ProcessEnv = { - FORCE_COLOR: forceColor, - NX_WORKSPACE_ROOT: this.workspaceRoot, - NX_SKIP_NX_CACHE: this.options.skipNxCache ? 'true' : undefined, - }; - - if (outputPath) { - env.NX_TERMINAL_OUTPUT_PATH = outputPath; - if (this.options.captureStderr) { - env.NX_TERMINAL_CAPTURE_STDERR = 'true'; - } - if (streamOutput) { - env.NX_STREAM_OUTPUT = 'true'; - } - } - return env; - } - - private getNxEnvVariablesForTask( - task: Task, - forceColor: string, - outputPath: string, - streamOutput: boolean - ) { - const env: NodeJS.ProcessEnv = { - NX_TASK_TARGET_PROJECT: task.target.project, - NX_TASK_TARGET_TARGET: task.target.target, - NX_TASK_TARGET_CONFIGURATION: task.target.configuration ?? undefined, - NX_TASK_HASH: task.hash, - // used when Nx is invoked via Lerna - LERNA_PACKAGE_NAME: task.target.project, - }; - - // TODO: remove this once we have a reasonable way to configure it - if (task.target.target === 'test') { - env.NX_TERMINAL_CAPTURE_STDERR = 'true'; - } - - return { - ...this.getNxEnvVariablesForForkedProcess( - forceColor, - outputPath, - streamOutput - ), - ...env, - }; - } - - private loadDotEnvFilesForTask( - task: Task, - environmentVariables: NodeJS.ProcessEnv - ) { - // Collect dot env files that may pertain to a task - const dotEnvFiles = [ - // Load DotEnv Files for a configuration in the project root - ...(task.target.configuration - ? [ - `${task.projectRoot}/.env.${task.target.target}.${task.target.configuration}`, - `${task.projectRoot}/.env.${task.target.configuration}`, - `${task.projectRoot}/.${task.target.target}.${task.target.configuration}.env`, - `${task.projectRoot}/.${task.target.configuration}.env`, - ] - : []), - - // Load DotEnv Files for a target in the project root - `${task.projectRoot}/.env.${task.target.target}`, - `${task.projectRoot}/.${task.target.target}.env`, - `${task.projectRoot}/.env.local`, - `${task.projectRoot}/.local.env`, - `${task.projectRoot}/.env`, - - // Load DotEnv Files for a configuration in the workspace root - ...(task.target.configuration - ? [ - `.env.${task.target.target}.${task.target.configuration}`, - `.env.${task.target.configuration}`, - `.${task.target.target}.${task.target.configuration}.env`, - `.${task.target.configuration}.env`, - ] - : []), - - // Load DotEnv Files for a target in the workspace root - `.env.${task.target.target}`, - `.${task.target.target}.env`, - - // Load base DotEnv Files at workspace root - `.local.env`, - `.env.local`, - `.env`, - ]; - - for (const file of dotEnvFiles) { - const myEnv = loadDotEnvFile({ - path: file, - processEnv: environmentVariables, - // Do not override existing env variables as we load - override: false, - }); - environmentVariables = { - ...expand({ - ...myEnv, - ignoreProcessEnv: true, // Do not override existing env variables as we load - }).parsed, - ...environmentVariables, - }; - } - - return environmentVariables; - } - - private unloadDotEnvFiles(environmentVariables: NodeJS.ProcessEnv) { - const unloadDotEnvFile = (filename: string) => { - let parsedDotEnvFile: NodeJS.ProcessEnv = {}; - loadDotEnvFile({ path: filename, processEnv: parsedDotEnvFile }); - Object.keys(parsedDotEnvFile).forEach((envVarKey) => { - if (environmentVariables[envVarKey] === parsedDotEnvFile[envVarKey]) { - delete environmentVariables[envVarKey]; - } - }); - }; - - for (const file of ['.env', '.local.env', '.env.local']) { - unloadDotEnvFile(file); - } - return environmentVariables; - } - - // endregion Environment Variables - private signalToCode(signal: string) { if (signal === 'SIGHUP') return 128 + 1; if (signal === 'SIGINT') return 128 + 2; diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index 4b8b3c07371d8..e41fd417ddcba 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -270,30 +270,56 @@ export async function invokeTasksRunner({ nxArgs, taskGraph, hasher: { - hashTask(task: Task, taskGraph_?: TaskGraph) { + hashTask(task: Task, taskGraph_?: TaskGraph, env?: NodeJS.ProcessEnv) { if (!taskGraph_) { output.warn({ - title: `TaskGraph is now required as an argument to hashTasks`, + title: `TaskGraph is now required as an argument to hashTask`, bodyLines: [ `The TaskGraph object can be retrieved from the context`, + 'This will result in an error in Nx 18', ], }); taskGraph_ = taskGraph; } - return hasher.hashTask(task, taskGraph_); + if (!env) { + output.warn({ + title: `The environment variables are now required as an argument to hashTask`, + bodyLines: [ + `Please pass the environment variables used when running the task`, + 'This will result in an error in Nx 18', + ], + }); + env = process.env; + } + return hasher.hashTask(task, taskGraph_, env); }, - hashTasks(task: Task[], taskGraph_?: TaskGraph) { + hashTasks( + task: Task[], + taskGraph_?: TaskGraph, + env?: NodeJS.ProcessEnv + ) { if (!taskGraph_) { output.warn({ title: `TaskGraph is now required as an argument to hashTasks`, bodyLines: [ `The TaskGraph object can be retrieved from the context`, + 'This will result in an error in Nx 18', ], }); taskGraph_ = taskGraph; } + if (!env) { + output.warn({ + title: `The environment variables are now required as an argument to hashTasks`, + bodyLines: [ + `Please pass the environment variables used when running the tasks`, + 'This will result in an error in Nx 18', + ], + }); + env = process.env; + } - return hasher.hashTasks(task, taskGraph_); + return hasher.hashTasks(task, taskGraph_, env); }, }, daemon: daemonClient, diff --git a/packages/nx/src/tasks-runner/task-env.ts b/packages/nx/src/tasks-runner/task-env.ts new file mode 100644 index 0000000000000..b46a727c0b8a9 --- /dev/null +++ b/packages/nx/src/tasks-runner/task-env.ts @@ -0,0 +1,202 @@ +import { Task } from '../config/task-graph'; +import { config as loadDotEnvFile } from 'dotenv'; +import { expand } from 'dotenv-expand'; +import { workspaceRoot } from '../utils/workspace-root'; + +export function getEnvVariablesForBatchProcess( + skipNxCache: boolean, + captureStderr: boolean +) { + return { + // User Process Env Variables override Dotenv Variables + ...process.env, + // Nx Env Variables overrides everything + ...getNxEnvVariablesForForkedProcess( + process.env.FORCE_COLOR === undefined ? 'true' : process.env.FORCE_COLOR, + skipNxCache, + captureStderr + ), + }; +} + +export function getTaskSpecificEnv(task: Task) { + // Unload any dot env files at the root of the workspace that were loaded on init of Nx. + const taskEnv = unloadDotEnvFiles({ ...process.env }); + return process.env.NX_LOAD_DOT_ENV_FILES === 'true' + ? loadDotEnvFilesForTask(task, taskEnv) + : // If not loading dot env files, ensure env vars created by system are still loaded + taskEnv; +} + +export function getEnvVariablesForTask( + task: Task, + taskSpecificEnv: NodeJS.ProcessEnv, + forceColor: string, + skipNxCache: boolean, + captureStderr: boolean, + outputPath: string, + streamOutput: boolean +) { + const res = { + // Start With Dotenv Variables + ...taskSpecificEnv, + // Nx Env Variables overrides everything + ...getNxEnvVariablesForTask( + task, + forceColor, + skipNxCache, + captureStderr, + outputPath, + streamOutput + ), + }; + + // we have to delete it because if we invoke Nx from within Nx, we need to reset those values + if (!outputPath) { + delete res.NX_TERMINAL_OUTPUT_PATH; + delete res.NX_STREAM_OUTPUT; + delete res.NX_PREFIX_OUTPUT; + } + delete res.NX_BASE; + delete res.NX_HEAD; + delete res.NX_SET_CLI; + return res; +} + +function getNxEnvVariablesForForkedProcess( + forceColor: string, + skipNxCache: boolean, + captureStderr: boolean, + outputPath?: string, + streamOutput?: boolean +) { + const env: NodeJS.ProcessEnv = { + FORCE_COLOR: forceColor, + NX_WORKSPACE_ROOT: workspaceRoot, + NX_SKIP_NX_CACHE: skipNxCache ? 'true' : undefined, + }; + + if (outputPath) { + env.NX_TERMINAL_OUTPUT_PATH = outputPath; + if (captureStderr) { + env.NX_TERMINAL_CAPTURE_STDERR = 'true'; + } + if (streamOutput) { + env.NX_STREAM_OUTPUT = 'true'; + } + } + return env; +} + +function getNxEnvVariablesForTask( + task: Task, + forceColor: string, + skipNxCache: boolean, + captureStderr: boolean, + outputPath: string, + streamOutput: boolean +) { + const env: NodeJS.ProcessEnv = { + NX_TASK_TARGET_PROJECT: task.target.project, + NX_TASK_TARGET_TARGET: task.target.target, + NX_TASK_TARGET_CONFIGURATION: task.target.configuration ?? undefined, + NX_TASK_HASH: task.hash, + // used when Nx is invoked via Lerna + LERNA_PACKAGE_NAME: task.target.project, + }; + + // TODO: remove this once we have a reasonable way to configure it + if (task.target.target === 'test') { + env.NX_TERMINAL_CAPTURE_STDERR = 'true'; + } + + return { + ...getNxEnvVariablesForForkedProcess( + forceColor, + skipNxCache, + captureStderr, + outputPath, + streamOutput + ), + ...env, + }; +} + +function loadDotEnvFilesForTask( + task: Task, + environmentVariables: NodeJS.ProcessEnv +) { + // Collect dot env files that may pertain to a task + const dotEnvFiles = [ + // Load DotEnv Files for a configuration in the project root + ...(task.target.configuration + ? [ + `${task.projectRoot}/.env.${task.target.target}.${task.target.configuration}`, + `${task.projectRoot}/.env.${task.target.configuration}`, + `${task.projectRoot}/.${task.target.target}.${task.target.configuration}.env`, + `${task.projectRoot}/.${task.target.configuration}.env`, + ] + : []), + + // Load DotEnv Files for a target in the project root + `${task.projectRoot}/.env.${task.target.target}`, + `${task.projectRoot}/.${task.target.target}.env`, + `${task.projectRoot}/.env.local`, + `${task.projectRoot}/.local.env`, + `${task.projectRoot}/.env`, + + // Load DotEnv Files for a configuration in the workspace root + ...(task.target.configuration + ? [ + `.env.${task.target.target}.${task.target.configuration}`, + `.env.${task.target.configuration}`, + `.${task.target.target}.${task.target.configuration}.env`, + `.${task.target.configuration}.env`, + ] + : []), + + // Load DotEnv Files for a target in the workspace root + `.env.${task.target.target}`, + `.${task.target.target}.env`, + + // Load base DotEnv Files at workspace root + `.local.env`, + `.env.local`, + `.env`, + ]; + + for (const file of dotEnvFiles) { + const myEnv = loadDotEnvFile({ + path: file, + processEnv: environmentVariables, + // Do not override existing env variables as we load + override: false, + }); + environmentVariables = { + ...expand({ + ...myEnv, + ignoreProcessEnv: true, // Do not override existing env variables as we load + }).parsed, + ...environmentVariables, + }; + } + + return environmentVariables; +} + +function unloadDotEnvFiles(environmentVariables: NodeJS.ProcessEnv) { + const unloadDotEnvFile = (filename: string) => { + let parsedDotEnvFile: NodeJS.ProcessEnv = {}; + loadDotEnvFile({ path: filename, processEnv: parsedDotEnvFile }); + Object.keys(parsedDotEnvFile).forEach((envVarKey) => { + if (environmentVariables[envVarKey] === parsedDotEnvFile[envVarKey]) { + delete environmentVariables[envVarKey]; + } + }); + }; + + for (const file of ['.env', '.local.env', '.env.local']) { + unloadDotEnvFile(file); + } + return environmentVariables; +} diff --git a/packages/nx/src/tasks-runner/task-orchestrator.ts b/packages/nx/src/tasks-runner/task-orchestrator.ts index 9aa2539ccddc5..ac5395059efb4 100644 --- a/packages/nx/src/tasks-runner/task-orchestrator.ts +++ b/packages/nx/src/tasks-runner/task-orchestrator.ts @@ -18,13 +18,18 @@ import { TaskMetadata } from './life-cycle'; import { ProjectGraph } from '../config/project-graph'; import { Task, TaskGraph } from '../config/task-graph'; import { DaemonClient } from '../daemon/client/client'; +import { hashTask } from '../hasher/hash-task'; +import { + getEnvVariablesForBatchProcess, + getEnvVariablesForTask, + getTaskSpecificEnv, +} from './task-env'; export class TaskOrchestrator { private cache = new Cache(this.options); private forkedProcessTaskRunner = new ForkedProcessTaskRunner(this.options); private tasksSchedule = new TasksSchedule( - this.hasher, this.projectGraph, this.taskGraph, this.options @@ -177,7 +182,11 @@ export class TaskOrchestrator { const taskEntries = Object.entries(batch.taskGraph.tasks); const tasks = taskEntries.map(([, task]) => task); - await this.preRunSteps(tasks, { groupId }); + const env = getEnvVariablesForBatchProcess( + this.options.skipNxCache, + this.options.captureStderr + ); + await this.preRunSteps(tasks, { groupId }, env); let results: { task: Task; @@ -192,10 +201,13 @@ export class TaskOrchestrator { results.map(({ task }) => task.id) ); - const batchResults = await this.runBatch({ - executorName: batch.executorName, - taskGraph: unrunTaskGraph, - }); + const batchResults = await this.runBatch( + { + executorName: batch.executorName, + taskGraph: unrunTaskGraph, + }, + env + ); results.push(...batchResults); } @@ -222,11 +234,12 @@ export class TaskOrchestrator { } } - private async runBatch(batch: Batch) { + private async runBatch(batch: Batch, env: NodeJS.ProcessEnv) { try { const results = await this.forkedProcessTaskRunner.forkProcessForBatch( batch, - this.taskGraph + this.taskGraph, + env ); const batchResultEntries = Object.entries(results); return batchResultEntries.map(([taskId, result]) => ({ @@ -255,9 +268,36 @@ export class TaskOrchestrator { task: Task, groupId: number ) { - await this.preRunSteps([task], { groupId }); + const taskSpecificEnv = getTaskSpecificEnv(task); + await this.preRunSteps([task], { groupId }, taskSpecificEnv); + + const pipeOutput = await this.pipeOutputCapture(task); + // obtain metadata + const temporaryOutputPath = this.cache.temporaryOutputPath(task); + const streamOutput = shouldStreamOutput(task, this.initiatingProject); + + const env = pipeOutput + ? getEnvVariablesForTask( + task, + taskSpecificEnv, + process.env.FORCE_COLOR === undefined + ? 'true' + : process.env.FORCE_COLOR, + this.options.skipNxCache, + this.options.captureStderr, + null, + null + ) + : getEnvVariablesForTask( + task, + taskSpecificEnv, + undefined, + this.options.skipNxCache, + this.options.captureStderr, + temporaryOutputPath, + streamOutput + ); - // hash the task here let results: { task: Task; status: TaskStatus; @@ -267,7 +307,13 @@ export class TaskOrchestrator { // the task wasn't cached if (results.length === 0) { // cache prep - const { code, terminalOutput } = await this.runTaskInForkedProcess(task); + const { code, terminalOutput } = await this.runTaskInForkedProcess( + task, + env, + pipeOutput, + temporaryOutputPath, + streamOutput + ); results.push({ task, @@ -278,14 +324,14 @@ export class TaskOrchestrator { await this.postRunSteps([task], results, doNotSkipCache, { groupId }); } - private async runTaskInForkedProcess(task: Task) { + private async runTaskInForkedProcess( + task: Task, + env: NodeJS.ProcessEnv, + pipeOutput: boolean, + temporaryOutputPath: string, + streamOutput: boolean + ) { try { - // obtain metadata - const temporaryOutputPath = this.cache.temporaryOutputPath(task); - const streamOutput = shouldStreamOutput(task, this.initiatingProject); - - const pipeOutput = await this.pipeOutputCapture(task); - // execution const { code, terminalOutput } = pipeOutput ? await this.forkedProcessTaskRunner.forkProcessPipeOutputCapture( @@ -294,6 +340,7 @@ export class TaskOrchestrator { temporaryOutputPath, streamOutput, taskGraph: this.taskGraph, + env, } ) : await this.forkedProcessTaskRunner.forkProcessDirectOutputCapture( @@ -302,6 +349,7 @@ export class TaskOrchestrator { temporaryOutputPath, streamOutput, taskGraph: this.taskGraph, + env, } ); @@ -319,7 +367,23 @@ export class TaskOrchestrator { // endregion Single Task // region Lifecycle - private async preRunSteps(tasks: Task[], metadata: TaskMetadata) { + private async preRunSteps( + tasks: Task[], + metadata: TaskMetadata, + env: NodeJS.ProcessEnv + ) { + for (const task of tasks) { + if (!task.hash) { + await hashTask( + this.hasher, + this.projectGraph, + this.taskGraph, + task, + env + ); + } + } + this.options.lifeCycle.startTasks(tasks, metadata); } diff --git a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts index 018ade6f23321..2839ca98525c1 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts @@ -114,16 +114,12 @@ describe('TasksSchedule', () => { version: '5', }; - const hasher = { - hashTask: () => 'hash', - } as any; - lifeCycle = { startTask: jest.fn(), endTask: jest.fn(), scheduleTask: jest.fn(), }; - taskSchedule = new TasksSchedule(hasher, projectGraph, taskGraph, { + taskSchedule = new TasksSchedule(projectGraph, taskGraph, { lifeCycle, }); }); @@ -329,16 +325,12 @@ describe('TasksSchedule', () => { version: '5', }; - const hasher = { - hashTask: () => 'hash', - } as any; - lifeCycle = { startTask: jest.fn(), endTask: jest.fn(), scheduleTask: jest.fn(), }; - taskSchedule = new TasksSchedule(hasher, projectGraph, taskGraph, { + taskSchedule = new TasksSchedule(projectGraph, taskGraph, { lifeCycle, }); }); diff --git a/packages/nx/src/tasks-runner/tasks-schedule.ts b/packages/nx/src/tasks-runner/tasks-schedule.ts index 2a010e42f420c..21d7b345b6a58 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.ts @@ -8,7 +8,6 @@ import { DefaultTasksRunnerOptions } from './default-tasks-runner'; import { TaskHasher } from '../hasher/task-hasher'; import { Task, TaskGraph } from '../config/task-graph'; import { ProjectGraph } from '../config/project-graph'; -import { hashTask } from '../hasher/hash-task'; import { findAllProjectNodeDependencies } from '../utils/project-graph-utils'; import { reverse } from '../project-graph/operators'; @@ -27,7 +26,6 @@ export class TasksSchedule { private scheduleRequestsExecutionChain = Promise.resolve(); constructor( - private readonly hasher: TaskHasher, private readonly projectGraph: ProjectGraph, private readonly taskGraph: TaskGraph, private readonly options: DefaultTasksRunnerOptions @@ -86,10 +84,6 @@ export class TasksSchedule { private async scheduleTask(taskId: string) { const task = this.taskGraph.tasks[taskId]; - if (!task.hash) { - await hashTask(this.hasher, this.projectGraph, this.taskGraph, task); - } - this.notScheduledTaskGraph = removeTasksFromTaskGraph( this.notScheduledTaskGraph, [taskId]