Add experimental options for more parallelization in webpack builds (#…

…60177)

This PR introduces 2 experimental options for doing more work in the
webpack build in parallel instead of in serial. These options may
improve the performance of builds at the cost of more memory.

`parallelServerAndEdgeCompiles`: This option kicks off the builds for
both `server` and `edge-server` at the same time instead of waiting for
each to complete before the next one. In applications that have many
server and edge functions, this can increase performance by doing that
work in parallel. This can be used with `next build` or `next
experimental-compile`.

`parallelServerBuildTraces`: This option starts the server build traces
as soon as the server compile completes and runs it in the background
while the other compilations are happening. With this option enabled,
some unnecessary work may be done since ordinarily the client
compilation provides information that can reduce the amount of tracing
necessary. However, since it is in parallel with the other work, it may
still result in a faster build in total at the cost of more memory. This
option is already the default when using `next experimental-compile` but
can now be used when `next build` is used also.

---------

Co-authored-by: Delba de Oliveira <32464864+delbaoliveira@users.noreply.github.com>
Co-authored-by: JJ Kasper <jj@jjsweb.site>

errors/parallel-build-without-worker.mdx

@@ -0,0 +1,18 @@
+---
+title: Parallel Build Without Build Worker
+---
+
+## Why This Error Occurred
+
+The `experimental.parallelServerCompiles` and `experimental.parallelServerBuildTraces`
+options require that the `experimental.webpackBuildWorker` option is set to `true`. These
+options use workers to improve the parallelization of the build which may improve performance,
+but the build may use more memory at the same time.
+
+## Possible Ways to Fix It
+
+Build workers are enabled by default unless you have a custom webpack config. You can force enable the option by setting `experimental.webpackBuildWorker: true` in your `next.config.js` file, but some webpack configuration options may not be compatible.
+
+## Useful Links
+
+Also see https://nextjs.org/docs/messages/webpack-build-worker-opt-out

packages/next/src/build/index.ts

@@ -1395,6 +1395,12 @@ export default async function build(
         config.experimental.webpackBuildWorker ||
         (config.experimental.webpackBuildWorker === undefined &&
           !config.webpack)
+      const runServerAndEdgeInParallel =
+        config.experimental.parallelServerCompiles
+      const collectServerBuildTracesInParallel =
+        config.experimental.parallelServerBuildTraces ||
+        (config.experimental.parallelServerBuildTraces === undefined &&
+          isCompileMode)
 
       nextBuildSpan.setAttribute(
         'has-custom-webpack-config',
@@ -1410,45 +1416,67 @@ export default async function build(
           'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out'
         )
       }
+      if (
+        !useBuildWorker &&
+        (runServerAndEdgeInParallel || collectServerBuildTracesInParallel)
+      ) {
+        throw new Error(
+          'The "parallelServerBuildTraces" and "parallelServerCompiles" options may only be used when build workers can be used. Read more: https://nextjs.org/docs/messages/parallel-build-without-worker'
+        )
+      }
 
       Log.info('Creating an optimized production build ...')
 
       if (!isGenerateMode) {
-        if (isCompileMode && useBuildWorker) {
+        if (runServerAndEdgeInParallel || collectServerBuildTracesInParallel) {
           let durationInSeconds = 0
 
-          await webpackBuild(useBuildWorker, ['server']).then((res) => {
+          const serverBuildPromise = webpackBuild(useBuildWorker, [
+            'server',
+          ]).then((res) => {
             buildTraceContext = res.buildTraceContext
             durationInSeconds += res.duration
-            const buildTraceWorker = new Worker(
-              require.resolve('./collect-build-traces'),
-              {
-                numWorkers: 1,
-                exposedMethods: ['collectBuildTraces'],
-              }
-            ) as Worker & typeof import('./collect-build-traces')
 
-            buildTracesPromise = buildTraceWorker
-              .collectBuildTraces({
-                dir,
-                config,
-                distDir,
-                // Serialize Map as this is sent to the worker.
-                pageInfos: serializePageInfos(new Map()),
-                staticPages: [],
-                hasSsrAmpPages: false,
-                buildTraceContext,
-                outputFileTracingRoot,
-              })
-              .catch((err) => {
-                console.error(err)
-                process.exit(1)
-              })
+            if (collectServerBuildTracesInParallel) {
+              const buildTraceWorker = new Worker(
+                require.resolve('./collect-build-traces'),
+                {
+                  numWorkers: 1,
+                  exposedMethods: ['collectBuildTraces'],
+                }
+              ) as Worker & typeof import('./collect-build-traces')
+
+              buildTracesPromise = buildTraceWorker
+                .collectBuildTraces({
+                  dir,
+                  config,
+                  distDir,
+                  // Serialize Map as this is sent to the worker.
+                  pageInfos: serializePageInfos(new Map()),
+                  staticPages: [],
+                  hasSsrAmpPages: false,
+                  buildTraceContext,
+                  outputFileTracingRoot,
+                })
+                .catch((err) => {
+                  console.error(err)
+                  process.exit(1)
+                })
+            }
           })
+          if (!runServerAndEdgeInParallel) {
+            await serverBuildPromise
+          }
 
-          await webpackBuild(useBuildWorker, ['edge-server']).then((res) => {
+          const edgeBuildPromise = webpackBuild(useBuildWorker, [
+            'edge-server',
+          ]).then((res) => {
             durationInSeconds += res.duration
           })
+          if (runServerAndEdgeInParallel) {
+            await serverBuildPromise
+          }
+          await edgeBuildPromise
 
           await webpackBuild(useBuildWorker, ['client']).then((res) => {
             durationInSeconds += res.duration

packages/next/src/lib/turbopack-warning.ts

@@ -88,6 +88,8 @@ const supportedTurbopackNextConfigOptions = [
   'experimental.memoryBasedWorkersCount',
   'experimental.clientRouterFilterRedirects',
   'experimental.webpackBuildWorker',
+  'experimental.parallelServerCompiles',
+  'experimental.parallelServerBuildTraces',
   'experimental.appDocumentPreloading',
   'experimental.incrementalCacheHandlerPath',
   'experimental.amp',

packages/next/src/server/config-schema.ts

@@ -285,6 +285,8 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         outputFileTracingIncludes: z
           .record(z.string(), z.array(z.string()))
           .optional(),
+        parallelServerCompiles: z.boolean().optional(),
+        parallelServerBuildTraces: z.boolean().optional(),
         ppr: z.boolean().optional(),
         taint: z.boolean().optional(),
         proxyTimeout: z.number().gte(0).optional(),

packages/next/src/server/config-shared.ts

@@ -282,6 +282,35 @@ export interface ExperimentalConfig {
    */
   typedRoutes?: boolean
 
+  /**
+   * Runs the compilations for server and edge in parallel instead of in serial.
+   * This will make builds faster if there is enough server and edge functions
+   * in the application at the cost of more memory.
+   *
+   * NOTE: This option is only valid when the build process can use workers. See
+   * the documentation for `webpackBuildWorker` for more details.
+   */
+  parallelServerCompiles?: boolean
+
+  /**
+   * Runs the logic to collect build traces for the server routes in parallel
+   * with other work during the compilation. This will increase the speed of
+   * the build at the cost of more memory. This option may incur some additional
+   * work compared to if the option was disabled since the work is started
+   * before data from the client compilation is available to potentially reduce
+   * the amount of code that needs to be traced. Despite that, this may still
+   * result in faster builds for some applications.
+   *
+   * Valid values are:
+   * - `true`: Collect the server build traces in parallel.
+   * - `false`: Do not collect the server build traces in parallel.
+   * - `undefined`: Collect server build traces in parallel only in the `experimental-compile` mode.
+   *
+   * NOTE: This option is only valid when the build process can use workers. See
+   * the documentation for `webpackBuildWorker` for more details.
+   */
+  parallelServerBuildTraces?: boolean
+
   /**
    * Run the Webpack build in a separate process to optimize memory usage during build.
    * Valid values are:
@@ -811,6 +840,8 @@ export const defaultConfig: NextConfig = {
     typedRoutes: false,
     instrumentationHook: false,
     bundlePagesExternals: false,
+    parallelServerCompiles: false,
+    parallelServerBuildTraces: false,
     ppr:
       // TODO: remove once we've made PPR default
       // If we're testing, and the `__NEXT_EXPERIMENTAL_PPR` environment variable

test/e2e/app-dir/app/next.config.js

@@ -1,6 +1,8 @@
 module.exports = {
   experimental: {
     clientRouterFilterRedirects: true,
+    parallelServerCompiles: true,
+    parallelServerBuildTraces: true,
     webpackBuildWorker: true,
   },
   // output: 'standalone',

test/production/app-dir/typed-routes-with-webpack-worker/good-routes/next.config.js

@@ -2,6 +2,7 @@
 module.exports = {
   experimental: {
     typedRoutes: true,
+    parallelServerBuildTraces: true,
     webpackBuildWorker: true,
   },
 }