Iterate on .env file loading

CHANGELOG.md

@@ -37,28 +37,27 @@
         DataDog/import-in-the-middle#57
       * This version does not support Node.js 18.19 or later
 
-- Add support for additional env var files (#9961)
+- Add support for additional env var files (#9961) and (#10093)
 
-  Fixes #9877. This PR adds a new middleware step to the CLI that looks for an `--include-env-files` flag and includes `.env.[file]` to the list of dotfiles to load. This PR also introduces functionality so that `.env.[file]` files are loaded based on `NODE_ENV`.
+  Fixes #9877. This PR adds CLI functionality to load more `.env` files via `NODE_ENV` and an `--add-env-files` flag.
 
-  Using the `--include-env-files` flag:
-
-  ```bash
-  yarn rw exec myScript --include-env-files prod stripe-prod
-  # Alternatively you can specify the flag twice:
-  yarn rw exec myScript --include-env-files prod --include-env-files stripe-prod
-  ```
-
-  Using `NODE_ENV`:
+  Env vars loaded via `NODE_ENV` override the values in `.env`; if there are conflicts, they win out:
 
   ```
-  # loads .env.production
+  # Loads '.env.production', which overwrites values in '.env'
   NODE_ENV=production yarn rw exec myScript
   ```
 
-  These files are loaded in addition to `.env` and `.env.defaults` and more generally are additive. Subsequent dotfiles won't overwrite environment variables defined previous ones. As such, files loaded via NODE_ENV have lower priority than those loaded specifically via `--include-env-files`.
+  Env vars loaded via `--add-env-files` only add to `process.env`; they will not override anything that was previously there:
+
+  ```bash
+  # Add new env vars defined in '.env.stripe' and '.env.nakama'
+  yarn rw exec myScript --add-env-files stripe nakama
+  # Or you can specify the flag twice:
+  yarn rw exec myScript --add-env-files stripe --add-env-files nakama
+  ```
 
-  Note that this feature is mainly for local scripting. Most deploy providers don't let you upload dotfiles and usually have their own way of determining environments.
+  Note that this feature is mainly for local scripting. Most deploy providers don't let you upload `.env` files (unless you're using baremetal) and usually have their own way of determining environments.
 
 ## v7.0.6
 

__fixtures__/test-project/web/package.json

@@ -23,7 +23,7 @@
     "@redwoodjs/vite": "7.0.0",
     "@types/react": "^18.2.55",
     "@types/react-dom": "^18.2.19",
-    "autoprefixer": "^10.4.17",
+    "autoprefixer": "^10.4.18",
     "postcss": "^8.4.35",
     "postcss-loader": "^8.1.1",
     "prettier-plugin-tailwindcss": "0.4.1",

packages/cli/src/__tests__/addAdditionalEnvFiles.test.js → packages/cli/src/__tests__/loadEnvFiles.test.js

@@ -2,9 +2,13 @@ import path from 'path'
 
 import { afterEach, beforeAll, describe, expect, it, test } from 'vitest'
 
-import { addAdditionalEnvFiles } from '../middleware/addAdditionalEnvFiles'
+import {
+  loadBasicEnvFiles,
+  loadNodeEnvDerivedEnvFile,
+  addUserSpecifiedEnvFiles,
+} from '../lib/loadEnvFiles'
 
-describe('addAdditionalEnvFiles', () => {
+describe('loadEnvFiles', () => {
   let originalProcessEnv
   beforeAll(() => {
     originalProcessEnv = { ...process.env }
@@ -14,28 +18,30 @@ describe('addAdditionalEnvFiles', () => {
   })
 
   it("doesn't load .env files if there are none to load", () => {
-    const fn = addAdditionalEnvFiles(__dirname)
-    fn({})
+    const cwd = __dirname
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, [])
 
     expect(process.env).toEqual(originalProcessEnv)
   })
 
   it("doesn't load .env files if not instructed to", () => {
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-prod')
-    )
-    fn({})
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-prod')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, [])
 
     expect(process.env).toEqual(originalProcessEnv)
   })
 
   it('loads specified .env files', () => {
     expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
 
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-prod')
-    )
-    fn({ includeEnvFiles: ['prod'] })
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-prod')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, ['prod'])
 
     expect(process.env).toHaveProperty(
       'PROD_DATABASE_URL',
@@ -51,10 +57,10 @@ describe('addAdditionalEnvFiles', () => {
     expect(process.env).not.toHaveProperty('DEV_DATABASE_URL')
     expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
 
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-many')
-    )
-    fn({ includeEnvFiles: ['dev', 'prod'] })
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-many')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, ['dev', 'prod'])
 
     expect(process.env).toHaveProperty(
       'DEV_DATABASE_URL',
@@ -71,10 +77,10 @@ describe('addAdditionalEnvFiles', () => {
     expect(process.env).not.toHaveProperty('TEST_BASE')
     expect(process.env).not.toHaveProperty('TEST_COLLISION')
 
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-collision')
-    )
-    fn({ includeEnvFiles: ['base', 'collision'] })
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-collision')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, ['base', 'collision'])
 
     expect(process.env).toHaveProperty(
       'DATABASE_URL',
@@ -89,10 +95,10 @@ describe('addAdditionalEnvFiles', () => {
     expect(process.env).not.toHaveProperty('BAZINGA')
 
     process.env.NODE_ENV = 'bazinga'
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
-    )
-    fn({})
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, [])
 
     expect(process.env).toHaveProperty(
       'PROD_DATABASE_URL',
@@ -101,34 +107,30 @@ describe('addAdditionalEnvFiles', () => {
     expect(process.env).toHaveProperty('BAZINGA', '1')
   })
 
-  it('loads .env files based on NODE_ENV last', () => {
+  it('loads .env files based on NODE_ENV before user-specified .env files', () => {
     expect(process.env).not.toHaveProperty('PROD_DATABASE_URL')
     expect(process.env).not.toHaveProperty('BAZINGA')
 
     process.env.NODE_ENV = 'bazinga'
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
-    )
-    fn({
-      includeEnvFiles: ['prod'],
-    })
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
+    loadBasicEnvFiles(cwd)
+    loadNodeEnvDerivedEnvFile(cwd)
+    addUserSpecifiedEnvFiles(cwd, ['prod'])
 
     expect(process.env).toHaveProperty(
       'PROD_DATABASE_URL',
-      'postgresql://user:password@localhost:5432/myproddb'
+      'postgresql://user:password@localhost:5432/bazinga'
     )
     expect(process.env).toHaveProperty('BAZINGA', '1')
   })
 
   it("throws if it can't find a specified env file", () => {
-    const fn = addAdditionalEnvFiles(
-      path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
-    )
+    const cwd = path.join(__dirname, '__fixtures__/redwood-app-env-node-env')
 
     try {
-      fn({
-        includeEnvFiles: ['missing'],
-      })
+      loadBasicEnvFiles(cwd)
+      loadNodeEnvDerivedEnvFile(cwd)
+      addUserSpecifiedEnvFiles(cwd, ['missing'])
     } catch (error) {
       // Just testing that the error message reports the file it tried to load.
       expect(error.message).toMatch(/\.env\.missing/)

packages/cli/src/index.js

@@ -3,7 +3,6 @@
 import path from 'path'
 
 import { trace, SpanStatusCode } from '@opentelemetry/api'
-import { config } from 'dotenv-defaults'
 import fs from 'fs-extra'
 import { hideBin, Parser } from 'yargs/helpers'
 import yargs from 'yargs/yargs'
@@ -32,10 +31,10 @@ import * as testCommand from './commands/test'
 import * as tstojsCommand from './commands/ts-to-js'
 import * as typeCheckCommand from './commands/type-check'
 import * as upgradeCommand from './commands/upgrade'
-import { getPaths, findUp } from './lib'
+import { findUp } from './lib'
 import { exitWithError } from './lib/exit'
+import { loadEnvFiles } from './lib/loadEnvFiles'
 import * as updateCheck from './lib/updateCheck'
-import { addAdditionalEnvFiles } from './middleware/addAdditionalEnvFiles'
 import { loadPlugins } from './plugin'
 import { startTelemetry, shutdownTelemetry } from './telemetry/index'
 
@@ -102,20 +101,10 @@ try {
 
 process.env.RWJS_CWD = cwd
 
-// # Load .env, .env.defaults
+// Load .env.* files.
 //
 // This should be done as early as possible, and the earliest we can do it is after setting `cwd`.
-// Further down in middleware, we allow additional .env files to be loaded based on args.
-
-if (!process.env.REDWOOD_ENV_FILES_LOADED) {
-  config({
-    path: path.join(getPaths().base, '.env'),
-    defaults: path.join(getPaths().base, '.env.defaults'),
-    multiline: true,
-  })
-
-  process.env.REDWOOD_ENV_FILES_LOADED = 'true'
-}
+loadEnvFiles()
 
 async function main() {
   // Start telemetry if it hasn't been disabled
@@ -174,6 +163,8 @@ async function runYargs() {
         // Likewise for `telemetry`.
         (argv) => {
           delete argv.cwd
+          delete argv.addEnvFiles
+          delete argv['add-env-files']
           delete argv.telemetry
         },
         telemetry && telemetryMiddleware,
@@ -183,27 +174,14 @@ async function runYargs() {
     .option('cwd', {
       describe: 'Working directory to use (where `redwood.toml` is located)',
     })
-    .option('include-env-files', {
+    .option('add-env-files', {
       describe: 'Load additional .env files. These are incremental',
       array: true,
     })
     .example(
-      'yarn rw exec MigrateUsers --include-env-files prod stripe-prod',
+      'yarn rw exec MigrateUsers --add-env-files prod stripe-prod',
       '"Run a script, and also include .env.prod and .env.stripe-prod"'
     )
-    .middleware([
-      addAdditionalEnvFiles(cwd),
-      // Once we've loaded the additional .env files, remove the option from yargs.
-      // If we leave it in, it and its alias will be passed to scripts run via `yarn rw exec` like...
-      //
-      // ```
-      // { args: { _: [ 'exec' ], 'include-env-files': [ 'prod' ], includeEnvFiles: [ 'prod' ], '$0': 'rw' } }
-      // ```
-      (argv) => {
-        delete argv.includeEnvFiles
-        delete argv['include-env-files']
-      },
-    ])
     .option('telemetry', {
       describe: 'Whether to send anonymous usage telemetry to RedwoodJS',
       boolean: true,

packages/cli/src/lib/loadEnvFiles.js

@@ -0,0 +1,84 @@
+// @ts-check
+
+import path from 'path'
+
+import { config as dotenvConfig } from 'dotenv'
+import { config as dotenvDefaultsConfig } from 'dotenv-defaults'
+import fs from 'fs-extra'
+import { hideBin, Parser } from 'yargs/helpers'
+
+import { getPaths } from '@redwoodjs/project-config'
+
+export function loadEnvFiles() {
+  if (process.env.REDWOOD_ENV_FILES_LOADED) {
+    return
+  }
+
+  const { base } = getPaths()
+
+  // These override.
+  loadBasicEnvFiles(base)
+  loadNodeEnvDerivedEnvFile(base)
+
+  // These are additive. I.e., They don't override existing env vars.
+  // defined in .env.defaults, .env, or .env.${NODE_ENV}
+  //
+  // Users have to opt-in to loading these files via `--add-env-files`.
+  const { addEnvFiles } = Parser(hideBin(process.argv), {
+    array: ['add-env-files'],
+    default: {
+      addEnvFiles: [],
+    },
+  })
+  if (addEnvFiles.length > 0) {
+    addUserSpecifiedEnvFiles(base, addEnvFiles)
+  }
+
+  process.env.REDWOOD_ENV_FILES_LOADED = 'true'
+}
+
+/**
+ * @param {string} cwd
+ */
+export function loadBasicEnvFiles(cwd) {
+  dotenvDefaultsConfig({
+    path: path.join(cwd, '.env'),
+    defaults: path.join(cwd, '.env.defaults'),
+    multiline: true,
+  })
+}
+
+/**
+ * @param {string} cwd
+ */
+export function loadNodeEnvDerivedEnvFile(cwd) {
+  if (!process.env.NODE_ENV) {
+    return
+  }
+
+  const nodeEnvDerivedEnvFilePath = path.join(
+    cwd,
+    `.env.${process.env.NODE_ENV}`
+  )
+  if (!fs.existsSync(nodeEnvDerivedEnvFilePath)) {
+    return
+  }
+
+  dotenvConfig({ path: nodeEnvDerivedEnvFilePath, override: true })
+}
+
+/**
+ * @param {string} cwd
+ */
+export function addUserSpecifiedEnvFiles(cwd, addEnvFiles) {
+  for (const suffix of addEnvFiles) {
+    const envPath = path.join(cwd, `.env.${suffix}`)
+    if (!fs.pathExistsSync(envPath)) {
+      throw new Error(
+        `Couldn't find an .env file at '${envPath}' as specified by '--add-env-files'`
+      )
+    }
+
+    dotenvConfig({ path: envPath })
+  }
+}