refactor: cleanup incremental cache implementation, reduce FS access

packages/next/src/build/utils.ts

@@ -1374,9 +1374,6 @@ export async function buildAppStaticPaths({
   const incrementalCache = new IncrementalCache({
     fs: nodeFs,
     dev: true,
-    // Enabled both for build as we're only writing this cache, not reading it.
-    pagesDir: true,
-    appDir: true,
     flushToDisk: isrFlushToDisk,
     serverDistDir: path.join(distDir, 'server'),
     fetchCacheKeyPrefix,

packages/next/src/export/helpers/create-incremental-cache.ts

@@ -1,5 +1,3 @@
-import type { NextEnabledDirectories } from '../../server/base-server'
-
 import path from 'path'
 import { IncrementalCache } from '../../server/lib/incremental-cache'
 import { hasNextSupport } from '../../telemetry/ci-info'
@@ -13,15 +11,13 @@ export async function createIncrementalCache({
   fetchCacheKeyPrefix,
   distDir,
   dir,
-  enabledDirectories,
   flushToDisk,
 }: {
   cacheHandler?: string
   cacheMaxMemorySize?: number
   fetchCacheKeyPrefix?: string
   distDir: string
   dir: string
-  enabledDirectories: NextEnabledDirectories
   flushToDisk?: boolean
 }) {
   // Custom cache handler overrides.
@@ -53,8 +49,6 @@ export async function createIncrementalCache({
       notFoundRoutes: [],
     }),
     fs: nodeFs,
-    pagesDir: enabledDirectories.pages,
-    appDir: enabledDirectories.app,
     serverDistDir: path.join(distDir, 'server'),
     CurCacheHandler: CacheHandler,
     minimalMode: hasNextSupport,

packages/next/src/export/index.ts

@@ -551,7 +551,6 @@ async function exportAppImpl(
           dir,
           distDir,
           outDir,
-          enabledDirectories,
           nextConfig,
           cacheHandler: nextConfig.cacheHandler,
           cacheMaxMemorySize: nextConfig.cacheMaxMemorySize,

packages/next/src/export/types.ts

@@ -55,7 +55,6 @@ export interface ExportPagesInput {
   fetchCache: boolean | undefined
   cacheHandler: string | undefined
   fetchCacheKeyPrefix: string | undefined
-  enabledDirectories: NextEnabledDirectories
   options: ExportAppOptions
 }
 

packages/next/src/export/worker.ts

@@ -328,7 +328,6 @@ export async function exportPages(
     cacheHandler,
     cacheMaxMemorySize,
     fetchCacheKeyPrefix,
-    enabledDirectories,
     pagesDataDir,
     renderOpts,
     nextConfig,
@@ -343,7 +342,6 @@ export async function exportPages(
     fetchCacheKeyPrefix,
     distDir,
     dir,
-    enabledDirectories: enabledDirectories,
     // skip writing to disk in minimal mode for now, pending some
     // changes to better support it
     flushToDisk: !hasNextSupport,

packages/next/src/server/base-server.ts

@@ -15,13 +15,15 @@ import type {
   RenderOptsPartial as AppRenderOptsPartial,
   ServerOnInstrumentationRequestError,
 } from './app-render/types'
-import type {
-  CachedAppPageValue,
-  CachedPageValue,
-  ServerComponentsHmrCache,
-  ResponseCacheBase,
-  ResponseCacheEntry,
-  ResponseGenerator,
+import {
+  type CachedAppPageValue,
+  type CachedPageValue,
+  type ServerComponentsHmrCache,
+  type ResponseCacheBase,
+  type ResponseCacheEntry,
+  type ResponseGenerator,
+  CachedRouteKind,
+  type CachedRedirectValue,
 } from './response-cache'
 import type { UrlWithParsedQuery } from 'url'
 import {
@@ -159,6 +161,8 @@ import {
 } from './after/builtin-request-context'
 import { ENCODED_TAGS } from './stream-utils/encodedTags'
 import { NextRequestHint } from './web/adapter'
+import { RouteKind } from './route-kind'
+import type { RouteModule } from './route-modules/route-module'
 
 export type FindComponentsResult = {
   components: LoadComponentsReturnType
@@ -2043,7 +2047,10 @@ export default abstract class Server<
       )
     }
 
-    const { routeModule } = components
+    let routeModule: RouteModule | undefined
+    if (components.routeModule) {
+      routeModule = components.routeModule
+    }
 
     /**
      * If the route being rendered is an app page, and the ppr feature has been
@@ -2493,7 +2500,7 @@ export default abstract class Server<
               // Create the cache entry for the response.
               const cacheEntry: ResponseCacheEntry = {
                 value: {
-                  kind: 'ROUTE',
+                  kind: CachedRouteKind.APP_ROUTE,
                   status: response.status,
                   body: Buffer.from(await blob.arrayBuffer()),
                   headers,
@@ -2657,9 +2664,9 @@ export default abstract class Server<
       if (metadata.isRedirect) {
         return {
           value: {
-            kind: 'REDIRECT',
+            kind: CachedRouteKind.REDIRECT,
             props: metadata.pageData ?? metadata.flightData,
-          },
+          } satisfies CachedRedirectValue,
           revalidate: metadata.revalidate,
         }
       }
@@ -2673,7 +2680,7 @@ export default abstract class Server<
       if (isAppPath) {
         return {
           value: {
-            kind: 'APP_PAGE',
+            kind: CachedRouteKind.APP_PAGE,
             html: result,
             headers,
             rscData: metadata.flightData,
@@ -2686,7 +2693,7 @@ export default abstract class Server<
 
       return {
         value: {
-          kind: 'PAGE',
+          kind: CachedRouteKind.PAGES,
           html: result,
           pageData: metadata.pageData ?? metadata.flightData,
           headers,
@@ -2802,11 +2809,11 @@ export default abstract class Server<
 
             return {
               value: {
-                kind: 'PAGE',
+                kind: CachedRouteKind.PAGES,
                 html: RenderResult.fromStatic(html),
+                pageData: Object.create(null),
                 status: undefined,
                 headers: undefined,
-                pageData: {},
               },
             }
           }
@@ -2845,12 +2852,12 @@ export default abstract class Server<
         return {
           revalidate: 1,
           value: {
-            kind: 'PAGE',
+            kind: CachedRouteKind.PAGES,
             html: RenderResult.fromStatic(''),
             pageData: {},
             headers: undefined,
             status: undefined,
-          },
+          } satisfies CachedPageValue,
         }
       }
 
@@ -2871,7 +2878,11 @@ export default abstract class Server<
       ssgCacheKey,
       responseGenerator,
       {
-        routeKind: routeModule?.definition.kind,
+        routeKind:
+          // If the route module is not defined, we can assume it's a page being
+          // rendered and thus check isAppPath.
+          routeModule?.definition.kind ??
+          (isAppPath ? RouteKind.APP_PAGE : RouteKind.PAGES),
         incrementalCache,
         isOnDemandRevalidate,
         isPrefetch: req.headers.purpose === 'prefetch',
@@ -2892,7 +2903,7 @@ export default abstract class Server<
     }
 
     const didPostpone =
-      cacheEntry.value?.kind === 'APP_PAGE' &&
+      cacheEntry.value?.kind === CachedRouteKind.APP_PAGE &&
       typeof cacheEntry.value.postponed === 'string'
 
     if (
@@ -2924,7 +2935,7 @@ export default abstract class Server<
     const { value: cachedData } = cacheEntry
 
     // If the cache value is an image, we should error early.
-    if (cachedData?.kind === 'IMAGE') {
+    if (cachedData?.kind === CachedRouteKind.IMAGE) {
       throw new Error('invariant SSG should not return an image cache value')
     }
 
@@ -3007,7 +3018,7 @@ export default abstract class Server<
           value: {
             ...cacheEntry.value,
             kind:
-              cacheEntry.value?.kind === 'APP_PAGE'
+              cacheEntry.value?.kind === CachedRouteKind.APP_PAGE
                 ? 'PAGE'
                 : cacheEntry.value?.kind,
           },
@@ -3049,7 +3060,7 @@ export default abstract class Server<
       }
       await this.render404(req, res, { pathname, query }, false)
       return null
-    } else if (cachedData.kind === 'REDIRECT') {
+    } else if (cachedData.kind === CachedRouteKind.REDIRECT) {
       if (cacheEntry.revalidate) {
         res.setHeader(
           'Cache-Control',
@@ -3073,7 +3084,7 @@ export default abstract class Server<
         await handleRedirect(cachedData.props)
         return null
       }
-    } else if (cachedData.kind === 'ROUTE') {
+    } else if (cachedData.kind === CachedRouteKind.APP_ROUTE) {
       const headers = { ...cachedData.headers }
 
       if (!(this.minimalMode && isSSG)) {
@@ -3089,7 +3100,7 @@ export default abstract class Server<
         })
       )
       return null
-    } else if (cachedData.kind === 'APP_PAGE') {
+    } else if (cachedData.kind === CachedRouteKind.APP_PAGE) {
       // If the request has a postponed state and it's a resume request we
       // should error.
       if (cachedData.postponed && minimalPostponed) {
@@ -3224,7 +3235,7 @@ export default abstract class Server<
             throw new Error('Invariant: expected a result to be returned')
           }
 
-          if (result.value?.kind !== 'APP_PAGE') {
+          if (result.value?.kind !== CachedRouteKind.APP_PAGE) {
             throw new Error(
               `Invariant: expected a page response, got ${result.value?.kind}`
             )

packages/next/src/server/image-optimizer.ts

@@ -14,9 +14,10 @@ import { hasMatch } from '../shared/lib/match-remote-pattern'
 import type { NextConfigComplete } from './config-shared'
 import { createRequestResponseMocks } from './lib/mock-request'
 import type { NextUrlWithParsedQuery } from './request-meta'
-import type {
-  IncrementalCacheEntry,
-  IncrementalCacheValue,
+import {
+  CachedRouteKind,
+  type IncrementalCacheEntry,
+  type IncrementalCacheValue,
 } from './response-cache'
 import { sendEtagResponse } from './send-payload'
 import { getContentType, getExtension } from './serve-static'
@@ -359,7 +360,7 @@ export class ImageOptimizerCache {
 
         return {
           value: {
-            kind: 'IMAGE',
+            kind: CachedRouteKind.IMAGE,
             etag,
             buffer,
             extension,
@@ -385,7 +386,7 @@ export class ImageOptimizerCache {
       revalidate?: number | false
     }
   ) {
-    if (value?.kind !== 'IMAGE') {
+    if (value?.kind !== CachedRouteKind.IMAGE) {
       throw new Error('invariant attempted to set non-image to image-cache')
     }
 

packages/next/src/server/lib/incremental-cache/fetch-cache.ts

@@ -1,5 +1,9 @@
 import type { CacheHandler, CacheHandlerContext, CacheHandlerValue } from './'
-import type { IncrementalCacheValue } from '../../response-cache'
+import {
+  CachedRouteKind,
+  IncrementalCacheKind,
+  type IncrementalCacheValue,
+} from '../../response-cache'
 
 import LRUCache from 'next/dist/compiled/lru-cache'
 import {
@@ -128,20 +132,22 @@ export default class FetchCache implements CacheHandler {
           length({ value }) {
             if (!value) {
               return 25
-            } else if (value.kind === 'REDIRECT') {
+            } else if (value.kind === CachedRouteKind.REDIRECT) {
               return JSON.stringify(value.props).length
-            } else if (value.kind === 'IMAGE') {
+            } else if (value.kind === CachedRouteKind.IMAGE) {
               throw new Error('invariant image should not be incremental-cache')
-            } else if (value.kind === 'FETCH') {
+            } else if (value.kind === CachedRouteKind.FETCH) {
               return JSON.stringify(value.data || '').length
-            } else if (value.kind === 'ROUTE') {
+            } else if (value.kind === CachedRouteKind.APP_ROUTE) {
               return value.body.length
             }
             // rough estimate of size of cache value
             return (
               value.html.length +
               (JSON.stringify(
-                value.kind === 'APP_PAGE' ? value.rscData : value.pageData
+                value.kind === CachedRouteKind.APP_PAGE
+                  ? value.rscData
+                  : value.pageData
               )?.length || 0)
             )
           },
@@ -203,10 +209,10 @@ export default class FetchCache implements CacheHandler {
   }
 
   public async get(...args: Parameters<CacheHandler['get']>) {
-    const [key, ctx = {}] = args
-    const { tags, softTags, kindHint, fetchIdx, fetchUrl } = ctx
+    const [key, ctx] = args
+    const { tags, softTags, kind: kindHint, fetchIdx, fetchUrl } = ctx
 
-    if (kindHint !== 'fetch') {
+    if (kindHint !== IncrementalCacheKind.FETCH) {
       return null
     }
 
@@ -223,7 +229,7 @@ export default class FetchCache implements CacheHandler {
     let data = memoryCache?.get(key)
 
     const hasFetchKindAndMatchingTags =
-      data?.value?.kind === 'FETCH' &&
+      data?.value?.kind === CachedRouteKind.FETCH &&
       this.hasMatchingTags(tags ?? [], data.value.tags ?? [])
 
     // Get data from fetch cache. Also check if new tags have been
@@ -274,13 +280,13 @@ export default class FetchCache implements CacheHandler {
 
         const cached: IncrementalCacheValue = await res.json()
 
-        if (!cached || cached.kind !== 'FETCH') {
+        if (!cached || cached.kind !== CachedRouteKind.FETCH) {
           DEBUG && console.log({ cached })
           throw new Error('invalid cache value')
         }
 
         // if new tags were specified, merge those tags to the existing tags
-        if (cached.kind === 'FETCH') {
+        if (cached.kind === CachedRouteKind.FETCH) {
           cached.tags ??= []
           for (const tag of tags ?? []) {
             if (!cached.tags.includes(tag)) {
@@ -331,11 +337,12 @@ export default class FetchCache implements CacheHandler {
   public async set(...args: Parameters<CacheHandler['set']>) {
     const [key, data, ctx] = args
 
-    const newValue = data?.kind === 'FETCH' ? data.data : undefined
+    const newValue =
+      data?.kind === CachedRouteKind.FETCH ? data.data : undefined
     const existingCache = memoryCache?.get(key)
     const existingValue = existingCache?.value
     if (
-      existingValue?.kind === 'FETCH' &&
+      existingValue?.kind === CachedRouteKind.FETCH &&
       Object.keys(existingValue.data).every(
         (field) =>
           JSON.stringify(