diff --git a/CHANGELOG.md b/CHANGELOG.md index 2108ab360d75..d640b362f7e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## Unreleased +- Add support for additional env var files (#9961) + + Fixes #9877. This PR adds a new middleware step to the CLI that looks for an `--include-env-files` flag and includes `.env.[file]` to the list of dotfiles to load. This PR also introduces functionality so that `.env.[file]` files are loaded based on `NODE_ENV`. + + Using the `--include-env-files` flag: + + ```bash + yarn rw exec myScript --include-env-files prod stripe-prod + # Alternatively you can specify the flag twice: + yarn rw exec myScript --include-env-files prod --include-env-files stripe-prod + ``` + + Using `NODE_ENV`: + + ``` + # loads .env.production + NODE_ENV=production yarn rw exec myScript + ``` + + These files are loaded in addition to `.env` and `.env.defaults` and more generally are additive. Subsequent dotfiles won't overwrite environment variables defined previous ones. As such, files loaded via NODE_ENV have lower priority than those loaded specifically via `--include-env-files`. + + Note that this feature is mainly for local scripting. Most deploy providers don't let you upload dotfiles and usually have their own way of determining environments. + - fix(render): reduce memory and handle server file This PR improves Render deploys by reducing memory consumption and fixing it so that it uses the server file if it's present. diff --git a/__fixtures__/fragment-test-project/.gitignore b/__fixtures__/fragment-test-project/.gitignore index 9b8149560d9b..31d9637ede81 100644 --- a/__fixtures__/fragment-test-project/.gitignore +++ b/__fixtures__/fragment-test-project/.gitignore @@ -1,6 +1,8 @@ .idea .DS_Store -.env +.env* +!.env.example +!.env.defaults .netlify .redwood/* !.redwood/README.md diff --git a/__fixtures__/test-project/.gitignore b/__fixtures__/test-project/.gitignore index 9b8149560d9b..31d9637ede81 100644 --- a/__fixtures__/test-project/.gitignore +++ b/__fixtures__/test-project/.gitignore @@ -1,6 +1,8 @@ .idea .DS_Store -.env +.env* +!.env.example +!.env.defaults .netlify .redwood/* !.redwood/README.md diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.base b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.base new file mode 100644 index 000000000000..ad03f36b034a --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.base @@ -0,0 +1,2 @@ +DATABASE_URL="postgresql://user:password@localhost:5432/mydb" +TEST_BASE=1 diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.collision b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.collision new file mode 100644 index 000000000000..2b93192fdbb6 --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.collision @@ -0,0 +1,2 @@ +DATABASE_URL="postgresql://user:password@localhost:5432/mycollisiondb" +TEST_COLLISION=1 diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.dev b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.dev new file mode 100644 index 000000000000..76c15133b8e0 --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.dev @@ -0,0 +1 @@ +DEV_DATABASE_URL="postgresql://user:password@localhost:5432/mydevdb" diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.prod b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.prod new file mode 100644 index 000000000000..e7a72b95961b --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.prod @@ -0,0 +1 @@ +PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb" diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.bazinga b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.bazinga new file mode 100644 index 000000000000..e603ee15cbbe --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.bazinga @@ -0,0 +1,2 @@ +PROD_DATABASE_URL="postgresql://user:password@localhost:5432/bazinga" +BAZINGA=1 diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.prod b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.prod new file mode 100644 index 000000000000..e7a72b95961b --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.prod @@ -0,0 +1 @@ +PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb" diff --git a/packages/cli/src/__tests__/__fixtures__/redwood-app-env-prod/.env.prod b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-prod/.env.prod new file mode 100644 index 000000000000..e7a72b95961b --- /dev/null +++ b/packages/cli/src/__tests__/__fixtures__/redwood-app-env-prod/.env.prod @@ -0,0 +1 @@ +PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb" diff --git a/packages/cli/src/__tests__/addAdditionalEnvFiles.test.js b/packages/cli/src/__tests__/addAdditionalEnvFiles.test.js new file mode 100644 index 000000000000..fc20fe13079b --- /dev/null +++ b/packages/cli/src/__tests__/addAdditionalEnvFiles.test.js @@ -0,0 +1,137 @@ +import path from 'path' + +import { afterEach, beforeAll, describe, expect, it, test } from 'vitest' + +import { addAdditionalEnvFiles } from '../middleware/addAdditionalEnvFiles' + +describe('addAdditionalEnvFiles', () => { + let originalProcessEnv + beforeAll(() => { + originalProcessEnv = { ...process.env } + }) + afterEach(() => { + process.env = { ...originalProcessEnv } + }) + + it("doesn't load .env files if there are none to load", () => { + const fn = addAdditionalEnvFiles(__dirname) + fn({}) + + expect(process.env).toEqual(originalProcessEnv) + }) + + it("doesn't load .env files if not instructed to", () => { + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-prod') + ) + fn({}) + + expect(process.env).toEqual(originalProcessEnv) + }) + + it('loads specified .env files', () => { + expect(process.env).not.toHaveProperty('PROD_DATABASE_URL') + + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-prod') + ) + fn({ includeEnvFiles: ['prod'] }) + + expect(process.env).toHaveProperty( + 'PROD_DATABASE_URL', + 'postgresql://user:password@localhost:5432/myproddb' + ) + }) + + test('process.env is reset between tests', () => { + expect(process.env).not.toHaveProperty('PROD_DATABASE_URL') + }) + + it('loads multiple .env files', () => { + expect(process.env).not.toHaveProperty('DEV_DATABASE_URL') + expect(process.env).not.toHaveProperty('PROD_DATABASE_URL') + + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-many') + ) + fn({ includeEnvFiles: ['dev', 'prod'] }) + + expect(process.env).toHaveProperty( + 'DEV_DATABASE_URL', + 'postgresql://user:password@localhost:5432/mydevdb' + ) + expect(process.env).toHaveProperty( + 'PROD_DATABASE_URL', + 'postgresql://user:password@localhost:5432/myproddb' + ) + }) + + it('is additive (i.e. only adds to process.env, not overwrites)', () => { + expect(process.env).not.toHaveProperty('DATABASE_URL') + expect(process.env).not.toHaveProperty('TEST_BASE') + expect(process.env).not.toHaveProperty('TEST_COLLISION') + + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-collision') + ) + fn({ includeEnvFiles: ['base', 'collision'] }) + + expect(process.env).toHaveProperty( + 'DATABASE_URL', + 'postgresql://user:password@localhost:5432/mydb' + ) + expect(process.env).toHaveProperty('TEST_BASE', '1') + expect(process.env).toHaveProperty('TEST_COLLISION', '1') + }) + + it('loads .env files based on NODE_ENV ', () => { + expect(process.env).not.toHaveProperty('PROD_DATABASE_URL') + expect(process.env).not.toHaveProperty('BAZINGA') + + process.env.NODE_ENV = 'bazinga' + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-node-env') + ) + fn({}) + + expect(process.env).toHaveProperty( + 'PROD_DATABASE_URL', + 'postgresql://user:password@localhost:5432/bazinga' + ) + expect(process.env).toHaveProperty('BAZINGA', '1') + }) + + it('loads .env files based on NODE_ENV last', () => { + expect(process.env).not.toHaveProperty('PROD_DATABASE_URL') + expect(process.env).not.toHaveProperty('BAZINGA') + + process.env.NODE_ENV = 'bazinga' + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-node-env') + ) + fn({ + includeEnvFiles: ['prod'], + }) + + expect(process.env).toHaveProperty( + 'PROD_DATABASE_URL', + 'postgresql://user:password@localhost:5432/myproddb' + ) + expect(process.env).toHaveProperty('BAZINGA', '1') + }) + + it("throws if it can't find a specified env file", () => { + const fn = addAdditionalEnvFiles( + path.join(__dirname, '__fixtures__/redwood-app-env-node-env') + ) + + try { + fn({ + includeEnvFiles: ['missing'], + }) + } catch (error) { + // Just testing that the error message reports the file it tried to load. + expect(error.message).toMatch(/\.env\.missing/) + } + }) +}) diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index ff8281269ee2..9490c763956e 100644 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -35,6 +35,7 @@ import * as upgradeCommand from './commands/upgrade' import { getPaths, findUp } from './lib' import { exitWithError } from './lib/exit' import * as updateCheck from './lib/updateCheck' +import { addAdditionalEnvFiles } from './middleware/addAdditionalEnvFiles' import { loadPlugins } from './plugin' import { startTelemetry, shutdownTelemetry } from './telemetry/index' @@ -104,6 +105,7 @@ process.env.RWJS_CWD = cwd // # Load .env, .env.defaults // // This should be done as early as possible, and the earliest we can do it is after setting `cwd`. +// Further down in middleware, we allow additional .env files to be loaded based on args. if (!process.env.REDWOOD_ENV_FILES_LOADED) { config({ @@ -181,6 +183,27 @@ async function runYargs() { .option('cwd', { describe: 'Working directory to use (where `redwood.toml` is located)', }) + .option('include-env-files', { + describe: 'Load additional .env files. These are incremental', + array: true, + }) + .example( + 'yarn rw exec MigrateUsers --include-env-files prod stripe-prod', + '"Run a script, and also include .env.prod and .env.stripe-prod"' + ) + .middleware([ + addAdditionalEnvFiles(cwd), + // Once we've loaded the additional .env files, remove the option from yargs. + // If we leave it in, it and its alias will be passed to scripts run via `yarn rw exec` like... + // + // ``` + // { args: { _: [ 'exec' ], 'include-env-files': [ 'prod' ], includeEnvFiles: [ 'prod' ], '$0': 'rw' } } + // ``` + (argv) => { + delete argv.includeEnvFiles + delete argv['include-env-files'] + }, + ]) .option('telemetry', { describe: 'Whether to send anonymous usage telemetry to RedwoodJS', boolean: true, diff --git a/packages/cli/src/middleware/addAdditionalEnvFiles.js b/packages/cli/src/middleware/addAdditionalEnvFiles.js new file mode 100644 index 000000000000..7186dfd78dcd --- /dev/null +++ b/packages/cli/src/middleware/addAdditionalEnvFiles.js @@ -0,0 +1,33 @@ +// @ts-check +import fs from 'fs' +import path from 'path' + +import { config } from 'dotenv' + +/** + * @param { string } cwd + * @returns {(yargs: import('yargs').Argv) => void} + */ +export const addAdditionalEnvFiles = (cwd) => (yargs) => { + // Allow for additional .env files to be included via --include-env + if ('includeEnvFiles' in yargs && Array.isArray(yargs.includeEnvFiles)) { + for (const suffix of yargs.includeEnvFiles) { + const envPath = path.join(cwd, `.env.${suffix}`) + if (!fs.existsSync(envPath)) { + throw new Error( + `Couldn't find an .env file at '${envPath}' - which was noted via --include-env` + ) + } + + config({ path: envPath }) + } + } + + // Support automatically matching a .env file based on NODE_ENV + if (process.env.NODE_ENV) { + const processBasedEnvPath = path.join(cwd, `.env.${process.env.NODE_ENV}`) + if (fs.existsSync(processBasedEnvPath)) { + config({ path: processBasedEnvPath }) + } + } +} diff --git a/packages/create-redwood-app/templates/js/gitignore.template b/packages/create-redwood-app/templates/js/gitignore.template index 9b8149560d9b..31d9637ede81 100644 --- a/packages/create-redwood-app/templates/js/gitignore.template +++ b/packages/create-redwood-app/templates/js/gitignore.template @@ -1,6 +1,8 @@ .idea .DS_Store -.env +.env* +!.env.example +!.env.defaults .netlify .redwood/* !.redwood/README.md diff --git a/packages/create-redwood-app/templates/ts/gitignore.template b/packages/create-redwood-app/templates/ts/gitignore.template index 9b8149560d9b..31d9637ede81 100644 --- a/packages/create-redwood-app/templates/ts/gitignore.template +++ b/packages/create-redwood-app/templates/ts/gitignore.template @@ -1,6 +1,8 @@ .idea .DS_Store -.env +.env* +!.env.example +!.env.defaults .netlify .redwood/* !.redwood/README.md diff --git a/tasks/server-tests/bothServer.test.mts b/tasks/server-tests/bothServer.test.mts index 4e6ec138c022..ab0fde7f26b5 100644 --- a/tasks/server-tests/bothServer.test.mts +++ b/tasks/server-tests/bothServer.test.mts @@ -20,6 +20,8 @@ describe('rw serve', () => { --version Show version number [boolean] --cwd Working directory to use (where \`redwood.toml\` is located) + --include-env-files Load additional .env files. These + are incremental [array] --telemetry Whether to send anonymous usage telemetry to RedwoodJS [boolean] --webPort, --web-port The port for the web server to @@ -65,6 +67,8 @@ describe('rw serve', () => { --version Show version number [boolean] --cwd Working directory to use (where \`redwood.toml\` is located) + --include-env-files Load additional .env files. These + are incremental [array] --telemetry Whether to send anonymous usage telemetry to RedwoodJS [boolean] --webPort, --web-port The port for the web server to