Update experimental trace handling to capture more deps

packages/next/src/build/index.ts

@@ -745,6 +745,12 @@ export default async function build(
       )
       NextBuildContext.buildId = buildId
 
+      if (config.experimental.flyingShuttle) {
+        await fs.mkdir(path.join(distDir, 'cache', 'shuttle'), {
+          recursive: true,
+        })
+      }
+
       const customRoutes: CustomRoutes = await nextBuildSpan
         .traceChild('load-custom-routes')
         .traceAsyncFn(() => loadCustomRoutes(config))

packages/next/src/build/webpack-config.ts

@@ -944,10 +944,25 @@ export default async function getBaseWebpackConfig(
                 }
               ),
           ],
+
+    ...(config.experimental.flyingShuttle
+      ? {
+          recordsPath: path.join(distDir, 'cache', 'shuttle', 'records.json'),
+        }
+      : {}),
+
     optimization: {
       emitOnErrors: !dev,
       checkWasmTypes: false,
       nodeEnv: false,
+
+      ...(config.experimental.flyingShuttle
+        ? {
+            moduleIds: 'deterministic',
+            portableRecords: true,
+          }
+        : {}),
+
       splitChunks: (():
         | Required<webpack.Configuration>['optimization']['splitChunks']
         | false => {
@@ -1180,6 +1195,19 @@ export default async function getBaseWebpackConfig(
       webassemblyModuleFilename: 'static/wasm/[modulehash].wasm',
       hashFunction: 'xxhash64',
       hashDigestLength: 16,
+
+      ...(config.experimental.flyingShuttle
+        ? {
+            // ensure we only use contenthash as it's more deterministic
+            filename: isNodeOrEdgeCompilation
+              ? dev || isEdgeServer
+                ? `[name].js`
+                : `../[name].js`
+              : `static/chunks/${isDevFallback ? 'fallback/' : ''}[name]${
+                  dev ? '' : appDir ? '-[contenthash]' : '-[contenthash]'
+                }.js`,
+          }
+        : {}),
     },
     performance: false,
     resolve: resolveConfig,
@@ -1789,7 +1817,7 @@ export default async function getBaseWebpackConfig(
           dev,
         }),
       (isClient || isEdgeServer) && new DropClientPage(),
-      isNodeServer &&
+      (isNodeServer || (config.experimental.flyingShuttle && isEdgeServer)) &&
         !dev &&
         new (require('./webpack/plugins/next-trace-entrypoints-plugin')
           .TraceEntryPointsPlugin as typeof import('./webpack/plugins/next-trace-entrypoints-plugin').TraceEntryPointsPlugin)(

packages/next/src/build/webpack/loaders/get-module-build-info.ts

@@ -40,6 +40,9 @@ export interface RouteMeta {
   absolutePagePath: string
   preferredRegion: string | string[] | undefined
   middlewareConfig: MiddlewareConfig
+  // references to other modules that this route needs
+  // e.g. related relates, not-found routes, etc
+  relatedModules?: string[]
 }
 
 export interface EdgeMiddlewareMeta {

packages/next/src/build/webpack/loaders/next-app-loader.ts

@@ -173,9 +173,11 @@ async function createTreeCodeFromPath(
     metadataResolver,
     pageExtensions,
     basePath,
+    buildInfo,
     collectedDeclarations,
   }: {
     page: string
+    buildInfo: ReturnType<typeof getModuleBuildInfo>
     resolveDir: DirResolver
     resolver: PathResolver
     metadataResolver: MetadataResolver
@@ -348,9 +350,15 @@ async function createTreeCodeFromPath(
         })
       )
 
-      const definedFilePaths = filePaths.filter(
-        ([, filePath]) => filePath !== undefined
-      ) as [ValueOf<typeof FILE_TYPES>, string][]
+      const definedFilePaths = filePaths.filter(([, filePath]) => {
+        if (filePath !== undefined) {
+          if (buildInfo.route?.relatedModules) {
+            buildInfo.route.relatedModules.push(filePath)
+          }
+          return true
+        }
+        return false
+      }) as [ValueOf<typeof FILE_TYPES>, string][]
 
       // Add default not found error as root not found if not present
       const hasNotFoundFile = definedFilePaths.some(
@@ -539,6 +547,7 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
     absolutePagePath: createAbsolutePath(appDir, pagePath),
     preferredRegion,
     middlewareConfig,
+    relatedModules: [],
   }
 
   const extensions = pageExtensions.map((extension) => `.${extension}`)
@@ -708,6 +717,7 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
     pageExtensions,
     basePath,
     collectedDeclarations,
+    buildInfo,
   })
 
   if (!treeCodeResult.rootLayout) {
@@ -757,6 +767,7 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
         pageExtensions,
         basePath,
         collectedDeclarations,
+        buildInfo,
       })
     }
   }

packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -435,6 +435,27 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                             }
                             entryModMap.set(absolutePath, entryMod)
                             entryNameMap.set(absolutePath, name)
+
+                            // attach related app route modules to ensure
+                            // we properly track them as dependencies
+                            // e.g. layouts, loading, etc
+                            if (moduleBuildInfo.route?.relatedModules) {
+                              let curAdditionalEntries =
+                                additionalEntries.get(name)
+
+                              if (!curAdditionalEntries) {
+                                curAdditionalEntries = new Map()
+                                additionalEntries.set(
+                                  name,
+                                  curAdditionalEntries
+                                )
+                              }
+
+                              for (const item of moduleBuildInfo.route
+                                ?.relatedModules) {
+                                curAdditionalEntries.set(item, entryMod)
+                              }
+                            }
                           }
                         }
 
@@ -494,29 +515,38 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
             }
 
             const entryPaths = Array.from(entryModMap.keys())
+            const entryPathDepMap = new Map<string, Set<string>>()
 
-            const collectDependencies = async (mod: any) => {
+            const collectDependencies = async (mod: any, parent: string) => {
               if (!mod || !mod.dependencies) return
 
               for (const dep of mod.dependencies) {
                 const depMod = getModuleFromDependency(compilation, dep)
 
                 if (depMod?.resource && !depModMap.get(depMod.resource)) {
                   if (this.flyingShuttle) {
+                    // ensure we associate this dep with the entry
+                    let curDepSet = entryPathDepMap.get(parent)
+
+                    if (!curDepSet) {
+                      curDepSet = new Set()
+                      entryPathDepMap.set(parent, curDepSet)
+                    }
+                    curDepSet.add(depMod.resource)
                     this.traceHashes.set(
                       depMod.resource,
                       await getOriginalHash(depMod.resource)
                     )
                   }
                   depModMap.set(depMod.resource, depMod)
-                  await collectDependencies(depMod)
+                  await collectDependencies(depMod, parent)
                 }
               }
             }
             const entriesToTrace = [...entryPaths]
 
             for (const entry of entryPaths) {
-              await collectDependencies(entryModMap.get(entry))
+              await collectDependencies(entryModMap.get(entry), entry)
               const entryName = entryNameMap.get(entry)!
               const curExtraEntries = additionalEntries.get(entryName)
 
@@ -629,7 +659,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                     this.tracingRoot,
                     entry
                   )
-
                   const curExtraEntries = additionalEntries.get(entryName)
                   const finalDeps = new Map<string, { bundled: boolean }>()
 
@@ -653,7 +682,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                         this.tracingRoot,
                         extraEntry
                       )
-                      finalDeps.set(extraEntry, { bundled: false })
+                      finalDeps.set(extraEntry, { bundled: true })
 
                       for (const [dep, info] of parentFilesMap
                         .get(normalizedExtraEntry)
@@ -664,6 +693,29 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
                       }
                     }
                   }
+
+                  // ensure we grab all associated dependencies
+                  if (this.flyingShuttle) {
+                    const curDepSet = entryPathDepMap.get(entry)
+
+                    for (const item of curDepSet || []) {
+                      if (!item.includes('?')) {
+                        if (!finalDeps.has(item)) {
+                          finalDeps.set(item, {
+                            bundled: true,
+                          })
+                        }
+                        for (const [dep, info] of parentFilesMap
+                          .get(nodePath.relative(this.tracingRoot, entry))
+                          ?.entries() || []) {
+                          finalDeps.set(nodePath.join(this.tracingRoot, dep), {
+                            bundled: info.ignored,
+                          })
+                        }
+                      }
+                    }
+                  }
+
                   this.entryTraces.set(entryName, finalDeps)
                 }
               })

test/e2e/app-dir/app/flying-shuttle.test.ts

@@ -14,7 +14,7 @@ describe('should output updated trace files', () => {
       nanoid: '4.0.1',
     },
     env: {
-      FLYING_SHUTTLE: 'true',
+      NEXT_PRIVATE_FLYING_SHUTTLE: 'true',
     },
   })
 
@@ -35,6 +35,12 @@ describe('should output updated trace files', () => {
     )
 
     expect(deploymentsTrace.fileHashes).toBeTruthy()
+
+    const deploymentsFileHashKeys = Object.keys(deploymentsTrace.fileHashes)
+    expect(
+      deploymentsFileHashKeys.filter((item) => item.includes('/layout')).length
+    ).toBe(3)
+
     expect(ssgTrace.fileHashes).toBeTruthy()
 
     // ensure all files have corresponding fileHashes

test/e2e/app-dir/app/next.config.js

@@ -7,7 +7,6 @@ module.exports = {
     parallelServerCompiles: true,
     parallelServerBuildTraces: true,
     webpackBuildWorker: true,
-    flyingShuttle: Boolean(process.env.FLYING_SHUTTLE),
   },
   // output: 'standalone',
   rewrites: async () => {