[Memory] Add option to reduce memory usage caused by duplicate string…

…s in webpack-sources (#66003) This PR adds a flag to Next.js to enable Webpack options to improve memory usage. See webpack/webpack-sources#155 for a full description of the changes and impact on memory. This PR adds a patch to `webpack-sources` temporarily that contains the fixes as the real changes are iterated on to merge upstream in the `webpack/webpack-sources` repository. After that is done, the patch will be reverted and the latest `webpack-sources` version will be updated in Next.js.
vercel · Aug 15, 2024 · 2fbfb0a · 2fbfb0a
1 parent fe5b0d5
commit 2fbfb0a
Show file tree

Hide file tree

Showing 8 changed files with 368 additions and 69 deletions.
diff --git a/package.json b/package.json
@@ -259,5 +259,10 @@
     "node": ">=18.17.0",
     "pnpm": "8.15.7"
   },
-  "packageManager": "pnpm@8.15.7"
+  "packageManager": "pnpm@8.15.7",
+  "pnpm": {
+    "patchedDependencies": {
+      "webpack-sources@3.2.3": "patches/webpack-sources@3.2.3.patch"
+    }
+  }
 }
diff --git a/packages/next/src/build/webpack-build/impl.ts b/packages/next/src/build/webpack-build/impl.ts
@@ -1,4 +1,5 @@
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
+import { stringBufferUtils } from 'next/dist/compiled/webpack-sources3'
 import { red } from '../../lib/picocolors'
 import formatWebpackMessages from '../../client/components/react-dev-overlay/internal/helpers/format-webpack-messages'
 import { nonNullable } from '../../lib/non-nullable'
@@ -185,6 +186,11 @@ export async function webpackBuildImpl(
   debug(`starting compiler`, compilerName)
   // We run client and server compilation separately to optimize for memory usage
   await runWebpackSpan.traceAsyncFn(async () => {
+    if (config.experimental.webpackMemoryOptimizations) {
+      stringBufferUtils.disableDualStringBufferCaching()
+      stringBufferUtils.enableStringInterning()
+    }
+
     // Run the server compilers first and then the client
     // compiler to track the boundary of server/client components.
     let clientResult: SingleCompilerResult | null = null
@@ -254,6 +260,9 @@ export async function webpackBuildImpl(
       }
     }
 
+    if (config.experimental.webpackMemoryOptimizations) {
+      stringBufferUtils.disableStringInterning()
+    }
     inputFileSystem?.purge?.()
 
     result = {

diff --git a/packages/next/src/compiled/webpack-sources3/index.js b/packages/next/src/compiled/webpack-sources3/index.js
diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts
@@ -361,6 +361,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
           .optional(),
         typedRoutes: z.boolean().optional(),
         webpackBuildWorker: z.boolean().optional(),
+        webpackMemoryOptimizations: z.boolean().optional(),
         turbo: z
           .object({
             loaders: z.record(z.string(), z.array(zTurboLoaderItem)).optional(),

diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts
@@ -9,8 +9,10 @@ import type {
 import type { SubresourceIntegrityAlgorithm } from '../build/webpack/plugins/subresource-integrity-plugin'
 import type { WEB_VITALS } from '../shared/lib/utils'
 import type { NextParsedUrlQuery } from './request-meta'
-import type { SizeLimit } from '../../types'
+import type { SizeLimit } from '../types'
 import type { SwrDelta } from './lib/revalidate'
+import type { SupportedTestRunners } from '../cli/next-test'
+import type { ExperimentalPPRConfig } from './lib/experimental/ppr'
 
 export type NextConfigComplete = Required<NextConfig> & {
   images: Required<ImageConfigComplete>
@@ -180,12 +182,26 @@ export interface NextJsWebpackConfig {
   ): any
 }
 
+/**
+ * Set of options for the react compiler next.js
+ * currently supports.
+ *
+ * This can be changed without breaking changes while supporting
+ * react compiler in the experimental phase.
+ */
+export interface ReactCompilerOptions {
+  compilationMode?: 'infer' | 'annotation' | 'all'
+  panicThreshold?: 'ALL_ERRORS' | 'CRITICAL_ERRORS' | 'NONE'
+}
+
 export interface ExperimentalConfig {
+  flyingShuttle?: boolean
   prerenderEarlyExit?: boolean
   linkNoTouchStart?: boolean
   caseSensitiveRoutes?: boolean
   appDocumentPreloading?: boolean
   preloadEntriesOnStart?: boolean
+  /** @default true */
   strictNextHead?: boolean
   clientRouterFilter?: boolean
   clientRouterFilterRedirects?: boolean
@@ -221,17 +237,7 @@ export interface ExperimentalConfig {
    * much as possible, even when this leads to many requests.
    */
   cssChunking?: 'strict' | 'loose'
-  /**
-   * @deprecated use config.cacheHandler instead
-   */
-  incrementalCacheHandlerPath?: string
-  /**
-   * @deprecated use config.cacheMaxMemorySize instead
-   *
-   */
-  isrMemoryCacheSize?: number
   disablePostcssPresetEnv?: boolean
-  swcMinify?: boolean
   cpus?: number
   memoryBasedWorkersCount?: boolean
   proxyTimeout?: number
@@ -366,6 +372,15 @@ export interface ExperimentalConfig {
    */
   webpackBuildWorker?: boolean
 
+  /**
+   * Enables optimizations to reduce memory usage in Webpack. This reduces the max size of the heap
+   * but may increase compile times slightly.
+   * Valid values are:
+   * - `false`: Disable Webpack memory optimizations (default).
+   * - `true`: Enables Webpack memory optimizations.
+   */
+  webpackMemoryOptimizations?: boolean
+
   /**
    *
    */
@@ -377,9 +392,10 @@ export interface ExperimentalConfig {
   clientTraceMetadata?: string[]
 
   /**
+   * Enables experimental Partial Prerendering feature of Next.js.
    * Using this feature will enable the `react@experimental` for the `app` directory.
    */
-  ppr?: boolean
+  ppr?: ExperimentalPPRConfig
 
   /**
    * Enables experimental taint APIs in React.
@@ -430,34 +446,65 @@ export interface ExperimentalConfig {
   useLightningcss?: boolean
 
   /**
-   * Certain methods calls like `useSearchParams()` can bail out of server-side rendering of **entire** pages to client-side rendering,
-   * if they are not wrapped in a suspense boundary.
+   * Enables early import feature for app router modules
+   */
+  useEarlyImport?: boolean
+
+  /**
+   * Enables `fetch` requests to be proxied to the experimental test proxy server
+   */
+  testProxy?: boolean
+
+  /**
+   * Set a default test runner to be used by `next experimental-test`.
+   */
+  defaultTestRunner?: SupportedTestRunners
+  /**
+   * Allow NODE_ENV=development even for `next build`.
+   */
+  allowDevelopmentBuild?: true
+  /**
+   * @deprecated use `config.bundlePagesRouterDependencies` instead
    *
-   * When this flag is set to `true`, Next.js will break the build instead of warning, to force the developer to add a suspense boundary above the method call.
+   */
+  bundlePagesExternals?: boolean
+  /**
+   * @deprecated use `config.serverExternalPackages` instead
    *
-   * @note This flag will be removed in Next.js 15.
-   * @default true
    */
-  missingSuspenseWithCSRBailout?: boolean
-
+  serverComponentsExternalPackages?: string[]
   /**
-   * Enables early import feature for app router modules
+   * Enable experimental React compiler optimization.
+   * Configuration accepts partial config object to the compiler, if provided
+   * compiler will be enabled.
    */
-  useEarlyImport?: boolean
+  reactCompiler?: boolean | ReactCompilerOptions
 
   /**
-   * Enables `fetch` requests to be proxied to the experimental text proxy server
+   * Enables `unstable_after`
    */
-  testProxy?: boolean
+  after?: boolean
 }
 
 export type ExportPathMap = {
   [path: string]: {
     page: string
     query?: NextParsedUrlQuery
+
+    /**
+     * @internal
+     */
     _isAppDir?: boolean
-    _isAppPrefetch?: boolean
+
+    /**
+     * @internal
+     */
     _isDynamicError?: boolean
+
+    /**
+     * @internal
+     */
+    _isRoutePPREnabled?: boolean
   }
 }
 
@@ -608,16 +655,6 @@ export interface NextConfig extends Record<string, any> {
   /** @see [Compression documentation](https://nextjs.org/docs/api-reference/next.config.js/compression) */
   compress?: boolean
 
-  /**
-   * The field should only be used when a Next.js project is not hosted on Vercel while using Vercel Speed Insights.
-   * Vercel provides zero-configuration insights for Next.js projects hosted on Vercel.
-   *
-   * @default ''
-   * @deprecated will be removed in next major version. Read more: https://nextjs.org/docs/messages/deprecated-analyticsid
-   * @see [how to fix deprecated analyticsId](https://nextjs.org/docs/messages/deprecated-analyticsid)
-   */
-  analyticsId?: string
-
   /** @see [Disabling x-powered-by](https://nextjs.org/docs/api-reference/next.config.js/disabling-x-powered-by) */
   poweredByHeader?: boolean
 
@@ -719,16 +756,6 @@ export interface NextConfig extends Record<string, any> {
    */
   httpAgentOptions?: { keepAlive?: boolean }
 
-  /**
-   * During a build, Next.js will automatically trace each page and its dependencies to determine all of the files
-   * that are needed for deploying a production version of your application.
-   *
-   * @see [Output File Tracing](https://nextjs.org/docs/advanced-features/output-file-tracing)
-   * @deprecated will be enabled by default and removed in Next.js 15
-   *
-   */
-  outputFileTracing?: boolean
-
   /**
    * Timeout after waiting to generate static pages in seconds
    *
@@ -744,14 +771,6 @@ export interface NextConfig extends Record<string, any> {
    */
   crossOrigin?: 'anonymous' | 'use-credentials'
 
-  /**
-   * Use [SWC compiler](https://swc.rs) to minify the generated JavaScript
-   * @deprecated will be enabled by default and removed in Next.js 15
-   *
-   * @see [SWC Minification](https://nextjs.org/docs/advanced-features/compiler#minification)
-   */
-  swcMinify?: boolean
-
   /**
    * Optionally enable compiler transforms
    *
@@ -861,7 +880,6 @@ export const defaultConfig: NextConfig = {
   pageExtensions: ['tsx', 'ts', 'jsx', 'js'],
   poweredByHeader: true,
   compress: true,
-  analyticsId: process.env.VERCEL_ANALYTICS_ID || '', // TODO: remove in the next major version
   images: imageConfigDefault,
   devIndicators: {
     buildActivity: true,
@@ -888,19 +906,18 @@ export const defaultConfig: NextConfig = {
   httpAgentOptions: {
     keepAlive: true,
   },
-  outputFileTracing: true,
   staticPageGenerationTimeout: 60,
-  swcMinify: true,
   output: !!process.env.NEXT_PRIVATE_STANDALONE ? 'standalone' : undefined,
   modularizeImports: undefined,
   experimental: {
-    prerenderEarlyExit: false,
+    flyingShuttle: false,
+    prerenderEarlyExit: true,
     serverMinification: true,
     serverSourceMaps: false,
     linkNoTouchStart: false,
     caseSensitiveRoutes: false,
     appDocumentPreloading: undefined,
-    preloadEntriesOnStart: undefined,
+    preloadEntriesOnStart: true,
     clientRouterFilter: true,
     clientRouterFilterRedirects: false,
     fetchCacheKeyPrefix: '',
@@ -953,13 +970,16 @@ export const defaultConfig: NextConfig = {
         process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
       ),
     webpackBuildWorker: undefined,
-    missingSuspenseWithCSRBailout: true,
+    webpackMemoryOptimizations: false,
     optimizeServerReact: true,
     useEarlyImport: false,
     staleTimes: {
-      dynamic: 30,
+      dynamic: 0,
       static: 300,
     },
+    allowDevelopmentBuild: undefined,
+    reactCompiler: undefined,
+    after: false,
   },
   bundlePagesRouterDependencies: false,
 }

diff --git a/packages/next/types/misc.d.ts b/packages/next/types/misc.d.ts
@@ -79,6 +79,11 @@ declare module 'next/dist/compiled/p-limit' {
   export = m
 }
 
+declare module 'next/dist/compiled/p-queue' {
+  import m from 'p-queue'
+  export = m
+}
+
 declare module 'next/dist/compiled/raw-body' {
   import m from 'raw-body'
   export = m
@@ -94,11 +99,6 @@ declare module 'next/dist/compiled/@hapi/accept' {
   export = m
 }
 
-declare module 'next/dist/compiled/get-orientation' {
-  import m from 'get-orientation'
-  export = m
-}
-
 declare module 'next/dist/compiled/acorn' {
   import m from 'acorn'
   export = m
@@ -446,3 +446,43 @@ declare module 'next/dist/compiled/zod' {
   import * as m from 'zod'
   export = m
 }
+
+declare module 'mini-css-extract-plugin'
+declare module 'next/dist/compiled/loader-utils3'
+
+declare module 'next/dist/compiled/webpack-sources3' {
+  interface StringBufferUtils {
+    disableDualStringBufferCaching: () => boolean
+    disableStringInterning: () => boolean
+    enableDualStringBufferCaching: () => boolean
+    enableStringInterning: () => boolean
+  }
+  export let stringBufferUtils: StringBufferUtils
+}
+
+declare module 'next/dist/compiled/webpack/webpack' {
+  import type webpackSources from 'webpack-sources1'
+  export function init(): void
+  export let BasicEvaluatedExpression: any
+  export let GraphHelpers: any
+  export let sources: typeof webpackSources
+  export let StringXor: any
+  export {
+    default as webpack,
+    Compiler,
+    Compilation,
+    Module,
+    Stats,
+    Template,
+    RuntimeModule,
+    RuntimeGlobals,
+    NormalModule,
+    ResolvePluginInstance,
+    ModuleFilenameHelpers,
+  } from 'webpack'
+  export type {
+    LoaderDefinitionFunction,
+    LoaderContext,
+    ModuleGraph,
+  } from 'webpack'
+}