Add support for additional env var files (#9961)

Fixes #9877

Adds a new middleware step to the CLI booting up, which looks for
`--include-env` and includes `.env.[file]` to the list of files to look
at.

Also generally adds a .env var based on the `NODE_ENV` - I could take
this or leave it, but I was in the space.

![image](https://github.com/redwoodjs/redwood/assets/49038/267c9f8d-0b4d-4c83-8607-712b837e3cfc)

---------

Co-authored-by: Tobbe Lundberg <tobbe@tlundberg.com>
Co-authored-by: Dominic Saadi <dominiceliassaadi@gmail.com>

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 ## Unreleased
 
+- Add support for additional env var files (#9961)
+
+  Fixes #9877. This PR adds a new middleware step to the CLI that looks for an `--include-env-files` flag and includes `.env.[file]` to the list of dotfiles to load. This PR also introduces functionality so that `.env.[file]` files are loaded based on `NODE_ENV`.
+
+  Using the `--include-env-files` flag:
+
+  ```bash
+  yarn rw exec myScript --include-env-files prod stripe-prod
+  # Alternatively you can specify the flag twice:
+  yarn rw exec myScript --include-env-files prod --include-env-files stripe-prod
+  ```
+
+  Using `NODE_ENV`:
+
+  ```
+  # loads .env.production
+  NODE_ENV=production yarn rw exec myScript
+  ```
+
+  These files are loaded in addition to `.env` and `.env.defaults` and more generally are additive. Subsequent dotfiles won't overwrite environment variables defined previous ones. As such, files loaded via NODE_ENV have lower priority than those loaded specifically via `--include-env-files`.
+
+  Note that this feature is mainly for local scripting. Most deploy providers don't let you upload dotfiles and usually have their own way of determining environments.
+
 - fix(render): reduce memory and handle server file
 
   This PR improves Render deploys by reducing memory consumption and fixing it so that it uses the server file if it's present.

__fixtures__/fragment-test-project/.gitignore

@@ -1,6 +1,8 @@
 .idea
 .DS_Store
-.env
+.env*
+!.env.example
+!.env.defaults
 .netlify
 .redwood/*
 !.redwood/README.md

__fixtures__/test-project/.gitignore

@@ -1,6 +1,8 @@
 .idea
 .DS_Store
-.env
+.env*
+!.env.example
+!.env.defaults
 .netlify
 .redwood/*
 !.redwood/README.md

packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.base

@@ -0,0 +1,2 @@
+DATABASE_URL="postgresql://user:password@localhost:5432/mydb"
+TEST_BASE=1

packages/cli/src/__tests__/__fixtures__/redwood-app-env-collision/.env.collision

@@ -0,0 +1,2 @@
+DATABASE_URL="postgresql://user:password@localhost:5432/mycollisiondb"
+TEST_COLLISION=1

packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.dev

@@ -0,0 +1 @@
+DEV_DATABASE_URL="postgresql://user:password@localhost:5432/mydevdb"

packages/cli/src/__tests__/__fixtures__/redwood-app-env-many/.env.prod

@@ -0,0 +1 @@
+PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb"

packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.bazinga

@@ -0,0 +1,2 @@
+PROD_DATABASE_URL="postgresql://user:password@localhost:5432/bazinga"
+BAZINGA=1

packages/cli/src/__tests__/__fixtures__/redwood-app-env-node-env/.env.prod

@@ -0,0 +1 @@
+PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb"

packages/cli/src/__tests__/__fixtures__/redwood-app-env-prod/.env.prod

@@ -0,0 +1 @@
+PROD_DATABASE_URL="postgresql://user:password@localhost:5432/myproddb"

packages/cli/src/__tests__/addAdditionalEnvFiles.test.js

@@ -0,0 +1,137 @@
+import path from 'path'
+
+import { afterEach, beforeAll, describe, expect, it, test } from 'vitest'
+
+import { addAdditionalEnvFiles } from '../middleware/addAdditionalEnvFiles'
+
+describe('addAdditionalEnvFiles', () => {
+  let originalProcessEnv
+  beforeAll(() => {
+    originalProcessEnv = { ...process.env }
+  })
+  afterEach(() => {
+    process.env = { ...originalProcessEnv }
+  })
+
+  it("doesn't load .env files if there are none to load", () => {
+    const fn = addAdditionalEnvFiles(__dirname)
+    fn({})
+
+    expect(process.env).toEqual(originalProcessEnv)
+  })
+
+  it("doesn't load .env files if not instructed to", () => {
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-prod')
+    )
+    fn({})
+
+    expect(process.env).toEqual(originalProcessEnv)
+  })
+
+  it('loads specified .env files', () => {
+    expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
+
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-prod')
+    )
+    fn({ includeEnvFiles: ['prod'] })
+
+    expect(process.env).toHaveProperty(
+      'PROD_DATABASE_URL',
+      'postgresql://user:password@localhost:5432/myproddb'
+    )
+  })
+
+  test('process.env is reset between tests', () => {
+    expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
+  })
+
+  it('loads multiple .env files', () => {
+    expect(process.env).not.toHaveProperty('DEV_DATABASE_URL')
+    expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
+
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-many')
+    )
+    fn({ includeEnvFiles: ['dev', 'prod'] })
+
+    expect(process.env).toHaveProperty(
+      'DEV_DATABASE_URL',
+      'postgresql://user:password@localhost:5432/mydevdb'
+    )
+    expect(process.env).toHaveProperty(
+      'PROD_DATABASE_URL',
+      'postgresql://user:password@localhost:5432/myproddb'
+    )
+  })
+
+  it('is additive (i.e. only adds to process.env, not overwrites)', () => {
+    expect(process.env).not.toHaveProperty('DATABASE_URL')
+    expect(process.env).not.toHaveProperty('TEST_BASE')
+    expect(process.env).not.toHaveProperty('TEST_COLLISION')
+
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-collision')
+    )
+    fn({ includeEnvFiles: ['base', 'collision'] })
+
+    expect(process.env).toHaveProperty(
+      'DATABASE_URL',
+      'postgresql://user:password@localhost:5432/mydb'
+    )
+    expect(process.env).toHaveProperty('TEST_BASE', '1')
+    expect(process.env).toHaveProperty('TEST_COLLISION', '1')
+  })
+
+  it('loads .env files based on NODE_ENV ', () => {
+    expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
+    expect(process.env).not.toHaveProperty('BAZINGA')
+
+    process.env.NODE_ENV = 'bazinga'
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
+    )
+    fn({})
+
+    expect(process.env).toHaveProperty(
+      'PROD_DATABASE_URL',
+      'postgresql://user:password@localhost:5432/bazinga'
+    )
+    expect(process.env).toHaveProperty('BAZINGA', '1')
+  })
+
+  it('loads .env files based on NODE_ENV last', () => {
+    expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
+    expect(process.env).not.toHaveProperty('BAZINGA')
+
+    process.env.NODE_ENV = 'bazinga'
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
+    )
+    fn({
+      includeEnvFiles: ['prod'],
+    })
+
+    expect(process.env).toHaveProperty(
+      'PROD_DATABASE_URL',
+      'postgresql://user:password@localhost:5432/myproddb'
+    )
+    expect(process.env).toHaveProperty('BAZINGA', '1')
+  })
+
+  it("throws if it can't find a specified env file", () => {
+    const fn = addAdditionalEnvFiles(
+      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
+    )
+
+    try {
+      fn({
+        includeEnvFiles: ['missing'],
+      })
+    } catch (error) {
+      // Just testing that the error message reports the file it tried to load.
+      expect(error.message).toMatch(/\.env\.missing/)
+    }
+  })
+})

packages/cli/src/index.js

@@ -35,6 +35,7 @@ import * as upgradeCommand from './commands/upgrade'
 import { getPaths, findUp } from './lib'
 import { exitWithError } from './lib/exit'
 import * as updateCheck from './lib/updateCheck'
+import { addAdditionalEnvFiles } from './middleware/addAdditionalEnvFiles'
 import { loadPlugins } from './plugin'
 import { startTelemetry, shutdownTelemetry } from './telemetry/index'
 
@@ -104,6 +105,7 @@ process.env.RWJS_CWD = cwd
 // # Load .env, .env.defaults
 //
 // This should be done as early as possible, and the earliest we can do it is after setting `cwd`.
+// Further down in middleware, we allow additional .env files to be loaded based on args.
 
 if (!process.env.REDWOOD_ENV_FILES_LOADED) {
   config({
@@ -181,6 +183,27 @@ async function runYargs() {
     .option('cwd', {
       describe: 'Working directory to use (where `redwood.toml` is located)',
     })
+    .option('include-env-files', {
+      describe: 'Load additional .env files. These are incremental',
+      array: true,
+    })
+    .example(
+      'yarn rw exec MigrateUsers --include-env-files prod stripe-prod',
+      '"Run a script, and also include .env.prod and .env.stripe-prod"'
+    )
+    .middleware([
+      addAdditionalEnvFiles(cwd),
+      // Once we've loaded the additional .env files, remove the option from yargs.
+      // If we leave it in, it and its alias will be passed to scripts run via `yarn rw exec` like...
+      //
+      // ```
+      // { args: { _: [ 'exec' ], 'include-env-files': [ 'prod' ], includeEnvFiles: [ 'prod' ], '$0': 'rw' } }
+      // ```
+      (argv) => {
+        delete argv.includeEnvFiles
+        delete argv['include-env-files']
+      },
+    ])
     .option('telemetry', {
       describe: 'Whether to send anonymous usage telemetry to RedwoodJS',
       boolean: true,

packages/cli/src/middleware/addAdditionalEnvFiles.js

@@ -0,0 +1,33 @@
+// @ts-check
+import fs from 'fs'
+import path from 'path'
+
+import { config } from 'dotenv'
+
+/**
+ * @param { string } cwd
+ * @returns {(yargs: import('yargs').Argv) => void}
+ */
+export const addAdditionalEnvFiles = (cwd) => (yargs) => {
+  // Allow for additional .env files to be included via --include-env
+  if ('includeEnvFiles' in yargs && Array.isArray(yargs.includeEnvFiles)) {
+    for (const suffix of yargs.includeEnvFiles) {
+      const envPath = path.join(cwd, `.env.${suffix}`)
+      if (!fs.existsSync(envPath)) {
+        throw new Error(
+          `Couldn't find an .env file at '${envPath}' - which was noted via --include-env`
+        )
+      }
+
+      config({ path: envPath })
+    }
+  }
+
+  // Support automatically matching a .env file based on NODE_ENV
+  if (process.env.NODE_ENV) {
+    const processBasedEnvPath = path.join(cwd, `.env.${process.env.NODE_ENV}`)
+    if (fs.existsSync(processBasedEnvPath)) {
+      config({ path: processBasedEnvPath })
+    }
+  }
+}

packages/create-redwood-app/templates/js/gitignore.template

@@ -1,6 +1,8 @@
 .idea
 .DS_Store
-.env
+.env*
+!.env.example
+!.env.defaults
 .netlify
 .redwood/*
 !.redwood/README.md

packages/create-redwood-app/templates/ts/gitignore.template

@@ -1,6 +1,8 @@
 .idea
 .DS_Store
-.env
+.env*
+!.env.example
+!.env.defaults
 .netlify
 .redwood/*
 !.redwood/README.md

tasks/server-tests/bothServer.test.mts

@@ -20,6 +20,8 @@ describe('rw serve', () => {
             --version                             Show version number        [boolean]
             --cwd                                 Working directory to use (where
                                                   \`redwood.toml\` is located)
+            --include-env-files                   Load additional .env files. These
+                                                  are incremental              [array]
             --telemetry                           Whether to send anonymous usage
                                                   telemetry to RedwoodJS     [boolean]
             --webPort, --web-port                 The port for the web server to
@@ -65,6 +67,8 @@ describe('rw serve', () => {
               --version                             Show version number        [boolean]
               --cwd                                 Working directory to use (where
                                                     \`redwood.toml\` is located)
+              --include-env-files                   Load additional .env files. These
+                                                    are incremental              [array]
               --telemetry                           Whether to send anonymous usage
                                                     telemetry to RedwoodJS     [boolean]
               --webPort, --web-port                 The port for the web server to