From 9f2a9ccfbfad35100f15e473c5593b4e78b710ab Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 15 Jul 2024 13:59:30 -0700 Subject: [PATCH 1/2] Add additional handling for experimental tracing --- .../flying-shuttle/detect-changed-entries.ts | 213 ++++++++ .../src/build/flying-shuttle/stitch-builds.ts | 504 ++++++++++++++++++ .../src/build/flying-shuttle/store-shuttle.ts | 73 +++ packages/next/src/build/index.ts | 177 ++++-- packages/next/src/build/webpack-config.ts | 47 +- .../webpack/plugins/build-manifest-plugin.ts | 34 +- .../webpack/plugins/define-env-plugin.ts | 2 + .../next/src/server/app-render/app-render.tsx | 6 +- .../server/app-render/required-scripts.tsx | 7 +- packages/next/src/server/get-page-files.ts | 4 + packages/next/src/shared/lib/router/router.ts | 120 +++-- .../dynamic-client/[category]/[id]/page.js | 6 + .../app-dir/app/components/button/button.js | 15 + .../app/components/button/button.module.css | 4 + test/e2e/app-dir/app/flying-shuttle.test.ts | 90 ++++ test/e2e/app-dir/app/next.config.js | 8 + test/e2e/app-dir/app/pages/index.js | 6 + test/e2e/app-dir/app/provide-paths.test.ts | 55 -- 18 files changed, 1202 insertions(+), 169 deletions(-) create mode 100644 packages/next/src/build/flying-shuttle/detect-changed-entries.ts create mode 100644 packages/next/src/build/flying-shuttle/stitch-builds.ts create mode 100644 packages/next/src/build/flying-shuttle/store-shuttle.ts create mode 100644 test/e2e/app-dir/app/components/button/button.js create mode 100644 test/e2e/app-dir/app/components/button/button.module.css delete mode 100644 test/e2e/app-dir/app/provide-paths.test.ts diff --git a/packages/next/src/build/flying-shuttle/detect-changed-entries.ts b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts new file mode 100644 index 0000000000000..a301f1cbc1dfd --- /dev/null +++ b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts @@ -0,0 +1,213 @@ +import fs from 'fs' +import path from 'path' +import crypto from 'crypto' +import { getPageFromPath } from '../entries' +import { Sema } from 'next/dist/compiled/async-sema' + +export interface DetectedEntriesResult { + app: string[] + pages: string[] +} + +let _hasShuttle: undefined | boolean = undefined +export async function hasShuttle(shuttleDir: string) { + if (typeof _hasShuttle === 'boolean') { + return _hasShuttle + } + _hasShuttle = await fs.promises + .access(path.join(shuttleDir, 'server')) + .then(() => true) + .catch(() => false) + + return _hasShuttle +} + +export async function detectChangedEntries({ + appPaths, + pagesPaths, + pageExtensions, + distDir, + shuttleDir, +}: { + appPaths?: string[] + pagesPaths?: string[] + pageExtensions: string[] + distDir: string + shuttleDir: string +}): Promise<{ + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult +}> { + const changedEntries: { + app: string[] + pages: string[] + } = { + app: [], + pages: [], + } + const unchangedEntries: typeof changedEntries = { + app: [], + pages: [], + } + + if (!(await hasShuttle(shuttleDir))) { + // no shuttle so consider everything changed + console.log('no shuttle can not detect changes') + return { + changed: { + pages: pagesPaths || [], + app: appPaths || [], + }, + unchanged: { + pages: [], + app: [], + }, + } + } + + const hashCache = new Map() + + async function computeHash(p: string): Promise { + let hash = hashCache.get(p) + if (hash) { + return hash + } + return new Promise((resolve, reject) => { + const hashInst = crypto.createHash('sha1') + const stream = fs.createReadStream(p) + stream.on('error', (err) => reject(err)) + stream.on('data', (chunk) => hashInst.update(chunk)) + stream.on('end', () => { + const digest = hashInst.digest('hex') + resolve(digest) + hashCache.set(p, digest) + }) + }) + } + + const hashSema = new Sema(16) + + async function detectChange({ + normalizedEntry, + entry, + type, + }: { + entry: string + normalizedEntry: string + type: keyof typeof changedEntries + }) { + const traceFile = path.join( + shuttleDir, + 'server', + type, + `${normalizedEntry}.js.nft.json` + ) + + const traceData: + | false + | { + fileHashes: Record + } = JSON.parse( + await fs.promises + .readFile(traceFile, 'utf8') + .catch(() => JSON.stringify(false)) + ) + let changed = false + + if (traceData) { + await Promise.all( + Object.keys(traceData.fileHashes).map(async (file) => { + if (changed) return + try { + await hashSema.acquire() + const originalTraceFile = path.join( + distDir, + 'server', + type, + path.relative(path.join(shuttleDir, 'server', type), traceFile) + ) + const absoluteFile = path.join( + path.dirname(originalTraceFile), + file + ) + + if (absoluteFile.startsWith(distDir)) { + return + } + + const prevHash = traceData.fileHashes[file] + const curHash = await computeHash(absoluteFile) + + if (prevHash !== curHash) { + console.error('detected change on', { + prevHash, + curHash, + file, + entry: normalizedEntry, + }) + changed = true + } + } finally { + hashSema.release() + } + }) + ) + } else { + console.error('missing trace data', traceFile, normalizedEntry) + changed = true + } + + if (changed || entry.match(/(_app|_document|_error)/)) { + changedEntries[type].push(entry) + } else { + unchangedEntries[type].push(entry) + } + } + + // collect page entries with default page extensions + console.error( + JSON.stringify( + { + appPaths, + pagePaths: pagesPaths, + }, + null, + 2 + ) + ) + // loop over entries and their dependency's hashes to find + // which changed + + // TODO: if _app or _document change it invalidates all pages + for (const entry of pagesPaths || []) { + let normalizedEntry = getPageFromPath(entry, pageExtensions) + + if (normalizedEntry === '/') { + normalizedEntry = '/index' + } + + await detectChange({ entry, normalizedEntry, type: 'pages' }) + } + + for (const entry of appPaths || []) { + const normalizedEntry = getPageFromPath(entry, pageExtensions) + await detectChange({ entry, normalizedEntry, type: 'app' }) + } + + console.error( + 'changed entries', + JSON.stringify( + { + changedEntries, + unchangedEntries, + }, + null, + 2 + ) + ) + + return { + changed: changedEntries, + unchanged: unchangedEntries, + } +} diff --git a/packages/next/src/build/flying-shuttle/stitch-builds.ts b/packages/next/src/build/flying-shuttle/stitch-builds.ts new file mode 100644 index 0000000000000..891fd54e82618 --- /dev/null +++ b/packages/next/src/build/flying-shuttle/stitch-builds.ts @@ -0,0 +1,504 @@ +import type { Rewrite, Redirect } from '../../lib/load-custom-routes' +import type { PagesManifest } from '../webpack/plugins/pages-manifest-plugin' + +import fs from 'fs' +import path from 'path' +import { getPageFromPath } from '../entries' +import { Sema } from 'next/dist/compiled/async-sema' +import { recursiveCopy } from '../../lib/recursive-copy' +import { getSortedRoutes } from '../../shared/lib/router/utils' +import { generateClientManifest } from '../webpack/plugins/build-manifest-plugin' +import { createClientRouterFilter } from '../../lib/create-client-router-filter' +import { + hasShuttle, + type DetectedEntriesResult, +} from './detect-changed-entries' +import { + APP_BUILD_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + APP_PATHS_MANIFEST, + AUTOMATIC_FONT_OPTIMIZATION_MANIFEST, + BUILD_MANIFEST, + CLIENT_REFERENCE_MANIFEST, + FUNCTIONS_CONFIG_MANIFEST, + MIDDLEWARE_BUILD_MANIFEST, + MIDDLEWARE_MANIFEST, + MIDDLEWARE_REACT_LOADABLE_MANIFEST, + NEXT_FONT_MANIFEST, + PAGES_MANIFEST, + REACT_LOADABLE_MANIFEST, + SERVER_REFERENCE_MANIFEST, + ROUTES_MANIFEST, +} from '../../shared/lib/constants' +import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' + +export async function stitchBuilds( + { + distDir, + shuttleDir, + buildId, + rewrites, + redirects, + allowedErrorRate, + encryptionKey, + edgePreviewProps, + }: { + buildId: string + distDir: string + shuttleDir: string + rewrites: { + beforeFiles: Rewrite[] + afterFiles: Rewrite[] + fallback: Rewrite[] + } + redirects: Redirect[] + allowedErrorRate?: number + encryptionKey: string + edgePreviewProps: Record + }, + entries: { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + pageExtensions: string[] + } +): Promise<{ + pagesManifest?: PagesManifest +}> { + if (!(await hasShuttle(shuttleDir))) { + // no shuttle directory nothing to stitch + return {} + } + + // we need to copy the chunks from the shuttle folder + // to the distDir (we copy all server split chunks currently) + await recursiveCopy( + path.join(shuttleDir, 'server'), + path.join(distDir, 'server'), + { + filter(item) { + // we copy page chunks separately to not copy stale entries + return !item.match(/^[/\\](pages|app)[/\\]/) + }, + overwrite: true, + } + ) + // copy static chunks (this includes stale chunks but won't be loaded) + // unless referenced + await recursiveCopy( + path.join(shuttleDir, 'static'), + path.join(distDir, 'static'), + { overwrite: true } + ) + + async function copyPageChunk(entry: string, type: 'app' | 'pages') { + // copy entry chunk and flight manifest stuff + // TODO: copy .map files? + const entryFile = path.join('server', type, `${entry}.js`) + + await fs.promises.mkdir(path.join(distDir, path.dirname(entryFile)), { + recursive: true, + }) + await fs.promises.copyFile( + path.join(shuttleDir, entryFile + '.nft.json'), + path.join(distDir, entryFile + '.nft.json') + ) + + if (type === 'app' && !entry.endsWith('/route')) { + const clientRefManifestFile = path.join( + 'server', + type, + `${entry}_${CLIENT_REFERENCE_MANIFEST}.js` + ) + await fs.promises.copyFile( + path.join(shuttleDir, clientRefManifestFile), + path.join(distDir, clientRefManifestFile) + ) + } + await fs.promises.copyFile( + path.join(shuttleDir, entryFile), + path.join(distDir, entryFile) + ) + } + const copySema = new Sema(8) + + // restore unchanged entries avoiding copying stale + // entries from the shuttle/previous build + for (const { type, curEntries } of [ + { type: 'app', curEntries: entries.unchanged.app }, + { type: 'pages', curEntries: entries.unchanged.pages }, + ] as Array<{ type: 'app' | 'pages'; curEntries: string[] }>) { + await Promise.all( + curEntries.map(async (entry) => { + try { + await copySema.acquire() + let normalizedEntry = getPageFromPath(entry, entries.pageExtensions) + if (normalizedEntry === '/') { + normalizedEntry = '/index' + } + await copyPageChunk(normalizedEntry, type) + } finally { + copySema.release() + } + }) + ) + } + // always attempt copying not-found chunk + await copyPageChunk('/_not-found/page', 'app').catch(() => {}) + + // merge dynamic/static routes in routes-manifest + const [restoreRoutesManifest, currentRoutesManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', ROUTES_MANIFEST), + path.join(distDir, ROUTES_MANIFEST), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const dynamicRouteMap: Record = {} + const combinedDynamicRoutes: Record[] = [ + ...currentRoutesManifest.dynamicRoutes, + ...restoreRoutesManifest.dynamicRoutes, + ] + for (const route of combinedDynamicRoutes) { + dynamicRouteMap[route.page] = route + } + + const mergedRoutesManifest = { + ...currentRoutesManifest, + dynamicRoutes: getSortedRoutes( + combinedDynamicRoutes.map((item) => item.page) + ).map((page) => dynamicRouteMap[page]), + staticRoutes: [ + ...currentRoutesManifest.staticRoutes, + ...restoreRoutesManifest.staticRoutes, + ], + } + await fs.promises.writeFile( + path.join(distDir, ROUTES_MANIFEST), + JSON.stringify(mergedRoutesManifest, null, 2) + ) + + // for build-manifest we use latest runtime files + // and only merge previous page chunk entries + // middleware-build-manifest.js (needs to be regenerated) + const [restoreBuildManifest, currentBuildManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', BUILD_MANIFEST), + path.join(distDir, BUILD_MANIFEST), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedBuildManifest = { + // we want to re-use original runtime + // chunks so we favor restored version + // over new + ...currentBuildManifest, + pages: { + ...restoreBuildManifest.pages, + ...currentBuildManifest.pages, + }, + } + + // _app and _error is unique per runtime + // so nest under each specific entry in build-manifest + const internalEntries = ['/_error', '/_app'] + + for (const entry of Object.keys(restoreBuildManifest.pages)) { + if (currentBuildManifest.pages[entry]) { + continue + } + for (const internalEntry of internalEntries) { + for (const chunk of restoreBuildManifest.pages[internalEntry]) { + if (!restoreBuildManifest.pages[entry].includes(chunk)) { + mergedBuildManifest.pages[entry].unshift(chunk) + } + } + } + } + + for (const entry of Object.keys(currentBuildManifest.pages)) { + for (const internalEntry of internalEntries) { + for (const chunk of currentBuildManifest.pages[internalEntry]) { + if (!currentBuildManifest.pages[entry].includes(chunk)) { + mergedBuildManifest.pages[entry].unshift(chunk) + } + } + } + } + + for (const key of internalEntries) { + mergedBuildManifest.pages[key] = [] + } + + /* + TODO: for rootMainFiles we need to add a map that allows + referencing previous runtimes e.g. + [ + { + entries: string[] + runtimeFiles: string[] + } + ] + then we update the lookup to iterate over the array + to find the runtime files for the specific entry + + for pages we need to ensure the react chunk and such + is broken out into it's own split chunk correctly so + we don't reference new runtime chunks in a previous build + */ + for (const entry of entries.unchanged.app || []) { + const normalizedEntry = getPageFromPath(entry, entries.pageExtensions) + mergedBuildManifest.rootMainFilesTree[normalizedEntry] = + restoreBuildManifest.rootMainFilesTree[normalizedEntry] || + restoreBuildManifest.rootMainFiles + } + + await fs.promises.writeFile( + path.join(distDir, BUILD_MANIFEST), + JSON.stringify(mergedBuildManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${MIDDLEWARE_BUILD_MANIFEST}.js`), + `self.__BUILD_MANIFEST=${JSON.stringify(mergedBuildManifest)}` + ) + await fs.promises.writeFile( + path.join(distDir, 'static', buildId, `_buildManifest.js`), + `self.__BUILD_MANIFEST = ${generateClientManifest( + mergedBuildManifest, + rewrites, + createClientRouterFilter( + [ + ...[ + // client filter always has all app paths + ...(entries.unchanged?.app || []), + ...(entries.changed?.app || []), + ].map((entry) => + normalizeAppPath(getPageFromPath(entry, entries.pageExtensions)) + ), + ...(entries.unchanged.pages.length + ? entries.changed?.pages || [] + : [] + ).map((item) => getPageFromPath(item, entries.pageExtensions)), + ], + redirects, + allowedErrorRate + ) + )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` + ) + + // for react-loadable-manifest we just merge directly + // prioritizing current manifest over previous, + // middleware-react-loadable-manifest (needs to be regenerated) + const [restoreLoadableManifest, currentLoadableManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', REACT_LOADABLE_MANIFEST), + path.join(distDir, REACT_LOADABLE_MANIFEST), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedLoadableManifest = { + ...restoreLoadableManifest, + ...currentLoadableManifest, + } + + await fs.promises.writeFile( + path.join(distDir, REACT_LOADABLE_MANIFEST), + JSON.stringify(mergedLoadableManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${MIDDLEWARE_REACT_LOADABLE_MANIFEST}.js`), + `self.__REACT_LOADABLE_MANIFEST=${JSON.stringify( + JSON.stringify(mergedLoadableManifest) + )}` + ) + + // for server/middleware-manifest we just merge the functions + // and middleware fields + const [restoreMiddlewareManifest, currentMiddlewareManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', MIDDLEWARE_MANIFEST), + path.join(distDir, 'server', MIDDLEWARE_MANIFEST), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedMiddlewareManifest = { + ...currentMiddlewareManifest, + functions: { + ...restoreMiddlewareManifest.functions, + ...currentMiddlewareManifest.functions, + }, + } + // update edge function env + const updatedEdgeEnv: Record = { + __NEXT_BUILD_ID: buildId, + NEXT_SERVER_ACTIONS_ENCRYPTION_KEY: encryptionKey, + ...edgePreviewProps, + } + if (mergedMiddlewareManifest.middleware['/']) { + Object.assign(mergedMiddlewareManifest.middleware['/'].env, updatedEdgeEnv) + } + for (const key of Object.keys(mergedMiddlewareManifest.functions)) { + Object.assign(mergedMiddlewareManifest.functions[key].env, updatedEdgeEnv) + } + + await fs.promises.writeFile( + path.join(distDir, 'server', MIDDLEWARE_MANIFEST), + JSON.stringify(mergedMiddlewareManifest, null, 2) + ) + + // for server/next-font-manifest we just merge nested + // page/app fields and regenerate server/next-font-manifest.js + const [restoreNextFontManifest, currentNextFontManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedNextFontManifest = { + ...currentNextFontManifest, + pages: { + ...restoreNextFontManifest.pages, + ...currentNextFontManifest.pages, + }, + app: { + ...restoreNextFontManifest.app, + ...currentNextFontManifest.app, + }, + } + + await fs.promises.writeFile( + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + JSON.stringify(mergedNextFontManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.js`), + `self.__NEXT_FONT_MANIFEST=${JSON.stringify( + JSON.stringify(mergedNextFontManifest) + )}` + ) + + // for server/font-manifest.json we just merge the arrays + for (const file of [AUTOMATIC_FONT_OPTIMIZATION_MANIFEST]) { + const [restoreFontManifest, currentFontManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', file), + path.join(distDir, 'server', file), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const mergedFontManifest = [...restoreFontManifest, ...currentFontManifest] + + await fs.promises.writeFile( + path.join(distDir, 'server', file), + JSON.stringify(mergedFontManifest, null, 2) + ) + } + + // for server/functions-config-manifest.json we just merge + // the functions field + const [restoreFunctionsConfigManifest, currentFunctionsConfigManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + path.join(distDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedFunctionsConfigManifest = { + ...currentFunctionsConfigManifest, + functions: { + ...restoreFunctionsConfigManifest.functions, + ...currentFunctionsConfigManifest.functions, + }, + } + await fs.promises.writeFile( + path.join(distDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + JSON.stringify(mergedFunctionsConfigManifest, null, 2) + ) + + // for server/pages-manifest.json and server/app-paths-manifest.json + // we just merge + for (const file of [APP_BUILD_MANIFEST, APP_PATH_ROUTES_MANIFEST]) { + const [restorePagesManifest, currentPagesManifest] = await Promise.all( + [path.join(shuttleDir, 'manifests', file), path.join(distDir, file)].map( + async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8')) + ) + ) + const mergedPagesManifest = { + ...restorePagesManifest, + ...currentPagesManifest, + + ...(file === APP_BUILD_MANIFEST + ? { + pages: { + ...restorePagesManifest.pages, + ...currentPagesManifest.pages, + }, + } + : {}), + } + await fs.promises.writeFile( + path.join(distDir, file), + JSON.stringify(mergedPagesManifest, null, 2) + ) + } + let mergedPagesManifest: PagesManifest | undefined + + for (const file of [PAGES_MANIFEST, APP_PATHS_MANIFEST]) { + const [restoreAppManifest, currentAppManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', file), + path.join(distDir, 'server', file), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const mergedAppManifest = { + ...restoreAppManifest, + ...currentAppManifest, + } + await fs.promises.writeFile( + path.join(distDir, 'server', file), + JSON.stringify(mergedAppManifest, null, 2) + ) + if (file === PAGES_MANIFEST) { + mergedPagesManifest = mergedAppManifest + } + } + + // for server/server-reference-manifest.json we merge + // and regenerate server/server-reference-manifest.js + const [restoreServerRefManifest, currentServerRefManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedServerRefManifest = { + ...currentServerRefManifest, + node: { + ...restoreServerRefManifest.node, + ...currentServerRefManifest.node, + }, + edge: { + ...restoreServerRefManifest.edge, + ...currentServerRefManifest.edge, + }, + } + await fs.promises.writeFile( + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + JSON.stringify(mergedServerRefManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.js`), + `self.__RSC_SERVER_MANIFEST=${JSON.stringify( + JSON.stringify(mergedServerRefManifest) + )}` + ) + + // TODO: inline env variables post build by find/replace + // in all the chunks for NEXT_PUBLIC_? + + return { + pagesManifest: mergedPagesManifest, + } +} diff --git a/packages/next/src/build/flying-shuttle/store-shuttle.ts b/packages/next/src/build/flying-shuttle/store-shuttle.ts new file mode 100644 index 0000000000000..f53050bd0b9c6 --- /dev/null +++ b/packages/next/src/build/flying-shuttle/store-shuttle.ts @@ -0,0 +1,73 @@ +import fs from 'fs' +import path from 'path' +import { + BUILD_MANIFEST, + APP_BUILD_MANIFEST, + REACT_LOADABLE_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + PAGES_MANIFEST, + ROUTES_MANIFEST, +} from '../../shared/lib/constants' +import { recursiveCopy } from '../../lib/recursive-copy' + +// we can create a new shuttle with the outputs before env values have +// been inlined, can be done after stitching takes place +export async function storeShuttle({ + distDir, + shuttleDir, +}: { + distDir: string + shuttleDir: string +}) { + await fs.promises.rm(shuttleDir, { force: true, recursive: true }) + await fs.promises.mkdir(shuttleDir, { recursive: true }) + + // copy all server entries + await recursiveCopy( + path.join(distDir, 'server'), + path.join(shuttleDir, 'server'), + { + filter(item) { + return !item.match(/\.(rsc|meta|html)$/) + }, + } + ) + + const pagesManifest = JSON.parse( + await fs.promises.readFile( + path.join(shuttleDir, 'server', PAGES_MANIFEST), + 'utf8' + ) + ) + // ensure manifest isn't modified to .html as it's before static gen + for (const key of Object.keys(pagesManifest)) { + pagesManifest[key] = pagesManifest[key].replace(/\.html$/, '.js') + } + await fs.promises.writeFile( + path.join(shuttleDir, 'server', PAGES_MANIFEST), + JSON.stringify(pagesManifest) + ) + + // copy static assets + await recursiveCopy( + path.join(distDir, 'static'), + path.join(shuttleDir, 'static') + ) + + // copy manifests not nested in {distDir}/server/ + await fs.promises.mkdir(path.join(shuttleDir, 'manifests'), { + recursive: true, + }) + + for (const item of [ + BUILD_MANIFEST, + ROUTES_MANIFEST, + APP_BUILD_MANIFEST, + REACT_LOADABLE_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + ]) { + const outputPath = path.join(shuttleDir, 'manifests', item) + await fs.promises.mkdir(path.dirname(outputPath), { recursive: true }) + await fs.promises.copyFile(path.join(distDir, item), outputPath) + } +} diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index dfb700ba86fb5..e059a0441a1f4 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -101,7 +101,7 @@ import { import type { EventBuildFeatureUsage } from '../telemetry/events' import { Telemetry } from '../telemetry/storage' import { getPageStaticInfo } from './analysis/get-page-static-info' -import { createPagesMapping, sortByPageExts } from './entries' +import { createPagesMapping, getPageFromPath, sortByPageExts } from './entries' import { PAGE_TYPES } from '../lib/page-types' import { generateBuildId } from './generate-build-id' import { isWriteable } from './is-writeable' @@ -187,6 +187,12 @@ import { checkIsAppPPREnabled, checkIsRoutePPREnabled, } from '../server/lib/experimental/ppr' +import { + detectChangedEntries, + type DetectedEntriesResult, +} from './flying-shuttle/detect-changed-entries' +import { storeShuttle } from './flying-shuttle/store-shuttle' +import { stitchBuilds } from './flying-shuttle/stitch-builds' interface ExperimentalBypassForInfo { experimentalBypassFor?: RouteHas[] @@ -743,8 +749,10 @@ export default async function build( ) NextBuildContext.buildId = buildId + const shuttleDir = path.join(distDir, 'cache', 'shuttle') + if (config.experimental.flyingShuttle) { - await fs.mkdir(path.join(distDir, 'cache', 'shuttle'), { + await fs.mkdir(shuttleDir, { recursive: true, }) } @@ -869,20 +877,32 @@ export default async function build( appDir ) - const providedPagePaths: string[] = JSON.parse( - process.env.NEXT_PROVIDED_PAGE_PATHS || '[]' - ) - let pagesPaths = - providedPagePaths.length > 0 - ? providedPagePaths - : !appDirOnly && pagesDir - ? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() => - recursiveReadDir(pagesDir, { - pathnameFilter: validFileMatcher.isPageFile, - }) - ) - : [] + !appDirOnly && pagesDir + ? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() => + recursiveReadDir(pagesDir, { + pathnameFilter: validFileMatcher.isPageFile, + }) + ) + : [] + + let changedPagePathsResult: + | undefined + | { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + } + + if (pagesPaths && config.experimental.flyingShuttle) { + changedPagePathsResult = await detectChangedEntries({ + pagesPaths, + pageExtensions: config.pageExtensions, + distDir, + shuttleDir, + }) + console.error({ changedPagePathsResult }) + pagesPaths = changedPagePathsResult.changed.pages + } const middlewareDetectionRegExp = new RegExp( `^${MIDDLEWARE_FILENAME}\\.(?:${config.pageExtensions.join('|')})$` @@ -943,27 +963,37 @@ export default async function build( let mappedAppPages: MappedPages | undefined let denormalizedAppPages: string[] | undefined + let changedAppPathsResult: + | undefined + | { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + } if (appDir) { - const providedAppPaths: string[] = JSON.parse( - process.env.NEXT_PROVIDED_APP_PATHS || '[]' - ) + let appPaths = await nextBuildSpan + .traceChild('collect-app-paths') + .traceAsyncFn(() => + recursiveReadDir(appDir, { + pathnameFilter: (absolutePath) => + validFileMatcher.isAppRouterPage(absolutePath) || + // For now we only collect the root /not-found page in the app + // directory as the 404 fallback + validFileMatcher.isRootNotFound(absolutePath), + ignorePartFilter: (part) => part.startsWith('_'), + }) + ) - let appPaths = - providedAppPaths.length > 0 - ? providedAppPaths - : await nextBuildSpan - .traceChild('collect-app-paths') - .traceAsyncFn(() => - recursiveReadDir(appDir, { - pathnameFilter: (absolutePath) => - validFileMatcher.isAppRouterPage(absolutePath) || - // For now we only collect the root /not-found page in the app - // directory as the 404 fallback - validFileMatcher.isRootNotFound(absolutePath), - ignorePartFilter: (part) => part.startsWith('_'), - }) - ) + if (appPaths && config.experimental.flyingShuttle) { + changedAppPathsResult = await detectChangedEntries({ + appPaths, + pageExtensions: config.pageExtensions, + distDir, + shuttleDir, + }) + console.error({ changedAppPathsResult }) + appPaths = changedAppPathsResult.changed.app + } mappedAppPages = await nextBuildSpan .traceChild('create-app-mapping') @@ -1105,7 +1135,7 @@ export default async function build( ) const routesManifestPath = path.join(distDir, ROUTES_MANIFEST) - const routesManifest: RoutesManifest = nextBuildSpan + let routesManifest: RoutesManifest = nextBuildSpan .traceChild('generate-routes-manifest') .traceFn(() => { const sortedRoutes = getSortedRoutes([ @@ -1168,19 +1198,41 @@ export default async function build( ), } } + let clientRouterFilters: + | undefined + | ReturnType if (config.experimental.clientRouterFilter) { const nonInternalRedirects = (config._originalRedirects || []).filter( (r: any) => !r.internal ) - const clientRouterFilters = createClientRouterFilter( - appPaths, + const filterPaths: string[] = [] + + if (config.experimental.flyingShuttle) { + filterPaths.push( + ...[ + // client filter always has all app paths + ...(changedAppPathsResult?.unchanged?.app || []), + ...(changedAppPathsResult?.changed?.app || []), + ].map((entry) => + normalizeAppPath(getPageFromPath(entry, config.pageExtensions)) + ), + ...(changedPagePathsResult?.unchanged.pages.length + ? changedPagePathsResult.changed?.pages || [] + : [] + ).map((item) => getPageFromPath(item, config.pageExtensions)) + ) + } else { + filterPaths.push(...appPaths) + } + + clientRouterFilters = createClientRouterFilter( + filterPaths, config.experimental.clientRouterFilterRedirects ? nonInternalRedirects : [], config.experimental.clientRouterFilterAllowedRate ) - NextBuildContext.clientRouterFilters = clientRouterFilters } @@ -1351,7 +1403,7 @@ export default async function build( env: process.env as Record, defineEnv: createDefineEnv({ isTurbopack: true, - clientRouterFilters: NextBuildContext.clientRouterFilters, + clientRouterFilters, config, dev, distDir, @@ -1764,7 +1816,7 @@ export default async function build( const appDynamicParamPaths = new Set() const appDefaultConfigs = new Map() const pageInfos: PageInfos = new Map() - const pagesManifest = await readManifest(pagesManifestPath) + let pagesManifest = await readManifest(pagesManifestPath) const buildManifest = await readManifest(buildManifestPath) const appBuildManifest = appDir ? await readManifest(appBuildManifestPath) @@ -2452,6 +2504,53 @@ export default async function build( path.join(distDir, SERVER_DIRECTORY, MIDDLEWARE_MANIFEST) ) + if (!isGenerateMode) { + if (config.experimental.flyingShuttle) { + console.log('stitching builds...') + const stitchResult = await stitchBuilds( + { + buildId, + distDir, + shuttleDir, + rewrites, + redirects, + edgePreviewProps: { + __NEXT_PREVIEW_MODE_ID: + NextBuildContext.previewProps!.previewModeId, + __NEXT_PREVIEW_MODE_ENCRYPTION_KEY: + NextBuildContext.previewProps!.previewModeEncryptionKey, + __NEXT_PREVIEW_MODE_SIGNING_KEY: + NextBuildContext.previewProps!.previewModeSigningKey, + }, + encryptionKey, + allowedErrorRate: + config.experimental.clientRouterFilterAllowedRate, + }, + { + changed: { + pages: changedPagePathsResult?.changed.pages || [], + app: changedAppPathsResult?.changed.app || [], + }, + unchanged: { + pages: changedPagePathsResult?.unchanged.pages || [], + app: changedAppPathsResult?.unchanged.app || [], + }, + pageExtensions: config.pageExtensions, + } + ) + // reload pagesManifest since it's been updated on disk + if (stitchResult.pagesManifest) { + pagesManifest = stitchResult.pagesManifest + } + + console.log('storing shuttle') + await storeShuttle({ + distDir, + shuttleDir, + }) + } + } + const finalPrerenderRoutes: { [route: string]: SsgRoute } = {} const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {} const tbdPrerenderRoutes: string[] = [] diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index fdacc235d3b1b..f2e1a709ffc74 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -945,24 +945,11 @@ export default async function getBaseWebpackConfig( ), ], - ...(config.experimental.flyingShuttle - ? { - recordsPath: path.join(distDir, 'cache', 'shuttle', 'records.json'), - } - : {}), - optimization: { emitOnErrors: !dev, checkWasmTypes: false, nodeEnv: false, - ...(config.experimental.flyingShuttle - ? { - moduleIds: 'deterministic', - portableRecords: true, - } - : {}), - splitChunks: ((): | Required['optimization']['splitChunks'] | false => { @@ -1019,7 +1006,7 @@ export default async function getBaseWebpackConfig( if (isNodeServer || isEdgeServer) { return { - filename: `${isEdgeServer ? 'edge-chunks/' : ''}[name].js`, + filename: `${isEdgeServer ? `edge-chunks${config.experimental.flyingShuttle ? `-${buildId}` : ''}/` : ''}[name].js`, chunks: 'all', minChunks: 2, } @@ -1107,6 +1094,7 @@ export default async function getBaseWebpackConfig( runtimeChunk: isClient ? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } : undefined, + minimize: !dev && (isClient || @@ -1199,13 +1187,28 @@ export default async function getBaseWebpackConfig( ...(config.experimental.flyingShuttle ? { // ensure we only use contenthash as it's more deterministic - filename: isNodeOrEdgeCompilation - ? dev || isEdgeServer - ? `[name].js` - : `../[name].js` - : `static/chunks/${isDevFallback ? 'fallback/' : ''}[name]${ - dev ? '' : '-[contenthash]' - }.js`, + filename: (p) => { + if (isNodeOrEdgeCompilation) { + // runtime chunk needs hash so it can be isolated + // across builds + const isRuntimeChunk = p.chunk?.name?.match( + /webpack-(api-runtime|runtime)/ + ) + return `${isEdgeServer ? '' : '../'}[name]${isRuntimeChunk ? `-${buildId}` : ''}.js` + } + // client filename + return `static/chunks/[name]-[contenthash].js` + }, + + path: isNodeServer + ? path.join(outputPath, `chunks-${buildId}`) + : outputPath, + + chunkFilename: isNodeOrEdgeCompilation + ? `[name].js` + : `static/chunks/[contenthash].js`, + + webassemblyModuleFilename: 'static/wasm/[contenthash].wasm', } : {}), }, @@ -1796,7 +1799,6 @@ export default async function getBaseWebpackConfig( }), getDefineEnvPlugin({ isTurbopack: false, - clientRouterFilters, config, dev, distDir, @@ -1894,6 +1896,7 @@ export default async function getBaseWebpackConfig( rewrites, isDevFallback, appDirEnabled: hasAppDir, + clientRouterFilters, }), new ProfilingPlugin({ runWebpackSpan, rootDir: dir }), config.optimizeFonts && diff --git a/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts b/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts index 5bdc80e627c74..1355c59905e1d 100644 --- a/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts +++ b/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts @@ -1,3 +1,4 @@ +import type { BloomFilter } from '../../../shared/lib/bloom-filter' import type { Rewrite, CustomRoutes } from '../../../lib/load-custom-routes' import devalue from 'next/dist/compiled/devalue' import { webpack, sources } from 'next/dist/compiled/webpack/webpack' @@ -17,6 +18,7 @@ import getRouteFromEntrypoint from '../../../server/get-route-from-entrypoint' import { ampFirstEntryNamesMap } from './next-drop-client-page-plugin' import { getSortedRoutes } from '../../../shared/lib/router/utils' import { spans } from './profiling-plugin' +import { Span } from '../../../trace' type DeepMutable = { -readonly [P in keyof T]: DeepMutable } @@ -91,13 +93,22 @@ export function normalizeRewritesForBuildManifest( // This function takes the asset map generated in BuildManifestPlugin and creates a // reduced version to send to the client. -function generateClientManifest( - compiler: any, - compilation: any, +export function generateClientManifest( assetMap: BuildManifest, - rewrites: CustomRoutes['rewrites'] + rewrites: CustomRoutes['rewrites'], + clientRouterFilters?: { + staticFilter: ReturnType + dynamicFilter: ReturnType + }, + compiler?: any, + compilation?: any ): string | undefined { - const compilationSpan = spans.get(compilation) || spans.get(compiler) + const compilationSpan = compilation + ? spans.get(compilation) + : compiler + ? spans.get(compiler) + : new Span({ name: 'client-manifest' }) + const genClientManifestSpan = compilationSpan?.traceChild( 'NextJsBuildManifest-generateClientManifest' ) @@ -105,6 +116,8 @@ function generateClientManifest( return genClientManifestSpan?.traceFn(() => { const clientManifest: ClientBuildManifest = { __rewrites: normalizeRewritesForBuildManifest(rewrites) as any, + __routerFilterStatic: clientRouterFilters?.staticFilter as any, + __routerFilterDynamic: clientRouterFilters?.dynamicFilter as any, } const appDependencies = new Set(assetMap.pages['/_app']) const sortedPageKeys = getSortedRoutes(Object.keys(assetMap.pages)) @@ -162,12 +175,14 @@ export default class BuildManifestPlugin { private rewrites: CustomRoutes['rewrites'] private isDevFallback: boolean private appDirEnabled: boolean + private clientRouterFilters?: Parameters[2] constructor(options: { buildId: string rewrites: CustomRoutes['rewrites'] isDevFallback?: boolean appDirEnabled: boolean + clientRouterFilters?: Parameters[2] }) { this.buildId = options.buildId this.isDevFallback = !!options.isDevFallback @@ -177,6 +192,7 @@ export default class BuildManifestPlugin { fallback: [], } this.appDirEnabled = options.appDirEnabled + this.clientRouterFilters = options.clientRouterFilters this.rewrites.beforeFiles = options.rewrites.beforeFiles.map(processRoute) this.rewrites.afterFiles = options.rewrites.afterFiles.map(processRoute) this.rewrites.fallback = options.rewrites.fallback.map(processRoute) @@ -195,6 +211,7 @@ export default class BuildManifestPlugin { ampDevFiles: [], lowPriorityFiles: [], rootMainFiles: [], + rootMainFilesTree: {}, pages: { '/_app': [] }, ampFirstPages: [], } @@ -308,10 +325,11 @@ export default class BuildManifestPlugin { assets[clientManifestPath] = new sources.RawSource( `self.__BUILD_MANIFEST = ${generateClientManifest( - compiler, - compilation, assetMap, - this.rewrites + this.rewrites, + this.clientRouterFilters, + compiler, + compilation )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` ) } diff --git a/packages/next/src/build/webpack/plugins/define-env-plugin.ts b/packages/next/src/build/webpack/plugins/define-env-plugin.ts index edd19f6105440..ea1b3452ddde6 100644 --- a/packages/next/src/build/webpack/plugins/define-env-plugin.ts +++ b/packages/next/src/build/webpack/plugins/define-env-plugin.ts @@ -197,6 +197,8 @@ export function getDefineEnv({ ? 5 * 60 // 5 minutes : config.experimental.staleTimes?.static ), + 'process.env.__NEXT_FLYING_SHUTTLE': + config.experimental.flyingShuttle ?? false, 'process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED': config.experimental.clientRouterFilter ?? true, 'process.env.__NEXT_CLIENT_ROUTER_S_FILTER': diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx index b7b8abed2a916..d779fa8ba933d 100644 --- a/packages/next/src/server/app-render/app-render.tsx +++ b/packages/next/src/server/app-render/app-render.tsx @@ -995,7 +995,8 @@ async function renderToHTMLOrFlightImpl( renderOpts.crossOrigin, subresourceIntegrityManifest, getAssetQueryString(ctx, true), - nonce + nonce, + renderOpts.page ) const RSCPayload = await getRSCPayload(tree, ctx, asNotFound) @@ -1345,7 +1346,8 @@ async function renderToHTMLOrFlightImpl( renderOpts.crossOrigin, subresourceIntegrityManifest, getAssetQueryString(ctx, false), - nonce + nonce, + '/_not-found/page' ) const errorRSCPayload = await getErrorRSCPayload(tree, ctx, errorType) diff --git a/packages/next/src/server/app-render/required-scripts.tsx b/packages/next/src/server/app-render/required-scripts.tsx index 113bd0ac183db..0bdc268afe3a5 100644 --- a/packages/next/src/server/app-render/required-scripts.tsx +++ b/packages/next/src/server/app-render/required-scripts.tsx @@ -9,7 +9,8 @@ export function getRequiredScripts( crossOrigin: undefined | '' | 'anonymous' | 'use-credentials', SRIManifest: undefined | Record, qs: string, - nonce: string | undefined + nonce: string | undefined, + pagePath: string ): [ () => void, { src: string; integrity?: string; crossOrigin?: string | undefined }, @@ -25,7 +26,9 @@ export function getRequiredScripts( crossOrigin, } - const files = buildManifest.rootMainFiles.map(encodeURIPath) + const files = ( + buildManifest.rootMainFilesTree?.[pagePath] || buildManifest.rootMainFiles + ).map(encodeURIPath) if (files.length === 0) { throw new Error( 'Invariant: missing bootstrap script. This is a bug in Next.js' diff --git a/packages/next/src/server/get-page-files.ts b/packages/next/src/server/get-page-files.ts index 229729b3e3974..9890b9f36e724 100644 --- a/packages/next/src/server/get-page-files.ts +++ b/packages/next/src/server/get-page-files.ts @@ -7,6 +7,10 @@ export type BuildManifest = { polyfillFiles: readonly string[] lowPriorityFiles: readonly string[] rootMainFiles: readonly string[] + // this is a separate field for flying shuttle to allow + // different root main files per entries/build (ideally temporary) + // until we can stitch the runtime chunks together safely + rootMainFilesTree: { [appRoute: string]: readonly string[] } pages: { '/_app': readonly string[] [page: string]: readonly string[] diff --git a/packages/next/src/shared/lib/router/router.ts b/packages/next/src/shared/lib/router/router.ts index 9265c94a74faf..fee787615db4d 100644 --- a/packages/next/src/shared/lib/router/router.ts +++ b/packages/next/src/shared/lib/router/router.ts @@ -765,45 +765,6 @@ export default class Router implements BaseRouter { ], } - if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) { - const { BloomFilter } = - require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter') - - type Filter = ReturnType< - import('../../lib/bloom-filter').BloomFilter['export'] - > - - const routerFilterSValue: Filter | false = process.env - .__NEXT_CLIENT_ROUTER_S_FILTER as any - - const staticFilterData: Filter | undefined = routerFilterSValue - ? routerFilterSValue - : undefined - - const routerFilterDValue: Filter | false = process.env - .__NEXT_CLIENT_ROUTER_D_FILTER as any - - const dynamicFilterData: Filter | undefined = routerFilterDValue - ? routerFilterDValue - : undefined - - if (staticFilterData?.numHashes) { - this._bfl_s = new BloomFilter( - staticFilterData.numItems, - staticFilterData.errorRate - ) - this._bfl_s.import(staticFilterData) - } - - if (dynamicFilterData?.numHashes) { - this._bfl_d = new BloomFilter( - dynamicFilterData.numItems, - dynamicFilterData.errorRate - ) - this._bfl_d.import(dynamicFilterData) - } - } - // Backwards compat for Router.router.events // TODO: Should be remove the following major version as it was never documented this.events = Router.events @@ -1060,10 +1021,86 @@ export default class Router implements BaseRouter { skipNavigate?: boolean ) { if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) { + if (!this._bfl_s && !this._bfl_d) { + const { BloomFilter } = + require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter') + + type Filter = ReturnType< + import('../../lib/bloom-filter').BloomFilter['export'] + > + let staticFilterData: Filter | undefined + let dynamicFilterData: Filter | undefined + + try { + ;({ + __routerFilterStatic: staticFilterData, + __routerFilterDynamic: dynamicFilterData, + } = (await getClientBuildManifest()) as any as { + __routerFilterStatic?: Filter + __routerFilterDynamic?: Filter + }) + } catch (err) { + // failed to load build manifest hard navigate + // to be safe + console.error(err) + if (skipNavigate) { + return true + } + handleHardNavigation({ + url: addBasePath( + addLocale(as, locale || this.locale, this.defaultLocale) + ), + router: this, + }) + return new Promise(() => {}) + } + + const routerFilterSValue: Filter | false = process.env + .__NEXT_CLIENT_ROUTER_S_FILTER as any + + if (!staticFilterData && routerFilterSValue) { + staticFilterData = routerFilterSValue ? routerFilterSValue : undefined + } + + const routerFilterDValue: Filter | false = process.env + .__NEXT_CLIENT_ROUTER_D_FILTER as any + + if (!dynamicFilterData && routerFilterDValue) { + dynamicFilterData = routerFilterDValue + ? routerFilterDValue + : undefined + } + + if (staticFilterData?.numHashes) { + this._bfl_s = new BloomFilter( + staticFilterData.numItems, + staticFilterData.errorRate + ) + this._bfl_s.import(staticFilterData) + } + + if (dynamicFilterData?.numHashes) { + this._bfl_d = new BloomFilter( + dynamicFilterData.numItems, + dynamicFilterData.errorRate + ) + this._bfl_d.import(dynamicFilterData) + } + } + let matchesBflStatic = false let matchesBflDynamic = false + const pathsToCheck: Array<{ as?: string; allowMatchCurrent?: boolean }> = + [{ as }, { as: resolvedAs }] + + if (process.env.__NEXT_FLYING_SHUTTLE) { + // if existing page changed we hard navigate to + // avoid runtime conflict with new page + // TODO: check buildManifest files instead? + pathsToCheck.push({ as: this.asPath, allowMatchCurrent: true }) + } - for (const curAs of [as, resolvedAs]) { + for (const { as: curAs, allowMatchCurrent } of pathsToCheck) { if (curAs) { const asNoSlash = removeTrailingSlash( new URL(curAs, 'http://n').pathname @@ -1073,8 +1110,9 @@ export default class Router implements BaseRouter { ) if ( + allowMatchCurrent || asNoSlash !== - removeTrailingSlash(new URL(this.asPath, 'http://n').pathname) + removeTrailingSlash(new URL(this.asPath, 'http://n').pathname) ) { matchesBflStatic = matchesBflStatic || diff --git a/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js b/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js index ca5cd77f0f571..d807ae8910cd7 100644 --- a/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js +++ b/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js @@ -1,6 +1,11 @@ 'use client' import { useSearchParams } from 'next/navigation' +import dynamic from 'next/dynamic' + +const Button = dynamic(() => + import('../../../../components/button/button').then((mod) => mod.Button) +) export default function IdPage({ children, params }) { return ( @@ -14,6 +19,7 @@ export default function IdPage({ children, params }) {

{JSON.stringify(Object.fromEntries(useSearchParams()))}

+ ) } diff --git a/test/e2e/app-dir/app/components/button/button.js b/test/e2e/app-dir/app/components/button/button.js new file mode 100644 index 0000000000000..b8836ebcc7274 --- /dev/null +++ b/test/e2e/app-dir/app/components/button/button.js @@ -0,0 +1,15 @@ +'use client' +import * as buttonStyle from './button.module.css' + +export function Button({ children }) { + return ( + + ) +} diff --git a/test/e2e/app-dir/app/components/button/button.module.css b/test/e2e/app-dir/app/components/button/button.module.css new file mode 100644 index 0000000000000..be1d0b06f1cd1 --- /dev/null +++ b/test/e2e/app-dir/app/components/button/button.module.css @@ -0,0 +1,4 @@ +.button { + background: #000; + color: #fff; +} diff --git a/test/e2e/app-dir/app/flying-shuttle.test.ts b/test/e2e/app-dir/app/flying-shuttle.test.ts index f07c1daecbd43..5711475f46c31 100644 --- a/test/e2e/app-dir/app/flying-shuttle.test.ts +++ b/test/e2e/app-dir/app/flying-shuttle.test.ts @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import { nextTestSetup, isNextStart } from 'e2e-utils' +import { retry } from 'next-test-utils' // This feature is only relevant to Webpack. ;(process.env.TURBOPACK ? describe.skip : describe)( @@ -70,5 +71,94 @@ import { nextTestSetup, isNextStart } from 'e2e-utils' } } }) + + async function checkAppPagesNavigation() { + for (const path of [ + '/', + '/blog/123', + '/dynamic-client/first/second', + '/dashboard', + '/dashboard/deployments/123', + ]) { + require('console').error('checking', path) + const res = await next.fetch(path) + expect(res.status).toBe(200) + + const browser = await next.browser(path) + // TODO: check for hydration success properly + await retry(async () => { + expect(await browser.eval('!!window.next.router')).toBe(true) + }) + const browserLogs = await browser.log() + expect( + browserLogs.some(({ message }) => message.includes('error')) + ).toBeFalse() + } + // TODO: check we hard navigate boundaries properly + } + + it('should only rebuild just a changed app route correctly', async () => { + await next.stop() + + const dataPath = 'app/dashboard/deployments/[id]/data.json' + const originalContent = await next.readFile(dataPath) + + try { + await next.patchFile(dataPath, JSON.stringify({ hello: 'again' })) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(dataPath, originalContent) + } + }) + + it('should only rebuild just a changed pages route correctly', async () => { + await next.stop() + + const pagePath = 'pages/index.js' + const originalContent = await next.readFile(pagePath) + + try { + await next.patchFile( + pagePath, + originalContent.replace( + 'hello from pages/index', + 'hello from pages/index!!' + ) + ) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(pagePath, originalContent) + } + }) + + it('should only rebuild a changed app and pages route correctly', async () => { + await next.stop() + + const pagePath = 'pages/index.js' + const originalPageContent = await next.readFile(pagePath) + const dataPath = 'app/dashboard/deployments/[id]/data.json' + const originalDataContent = await next.readFile(dataPath) + + try { + await next.patchFile( + pagePath, + originalPageContent.replace( + 'hello from pages/index', + 'hello from pages/index!!' + ) + ) + await next.patchFile(dataPath, JSON.stringify({ hello: 'again' })) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(pagePath, originalPageContent) + await next.patchFile(dataPath, originalDataContent) + } + }) } ) diff --git a/test/e2e/app-dir/app/next.config.js b/test/e2e/app-dir/app/next.config.js index 927e60e0ea5c6..6cb266cb54a28 100644 --- a/test/e2e/app-dir/app/next.config.js +++ b/test/e2e/app-dir/app/next.config.js @@ -8,6 +8,14 @@ module.exports = { parallelServerBuildTraces: true, webpackBuildWorker: true, }, + webpack(cfg) { + if (process.env.NEXT_PRIVATE_FLYING_SHUTTLE) { + // disable the webpack cache to make sure we're + // deterministic without + cfg.cache = false + } + return cfg + }, // output: 'standalone', rewrites: async () => { return { diff --git a/test/e2e/app-dir/app/pages/index.js b/test/e2e/app-dir/app/pages/index.js index b1037a470b719..7055a6eb21af3 100644 --- a/test/e2e/app-dir/app/pages/index.js +++ b/test/e2e/app-dir/app/pages/index.js @@ -1,7 +1,12 @@ import React from 'react' import Link from 'next/link' +import dynamic from 'next/dynamic' import styles from '../styles/shared.module.css' +const Button = dynamic(() => + import('../components/button/button').then((mod) => mod.Button) +) + export default function Page() { return ( <> @@ -10,6 +15,7 @@ export default function Page() {

Dashboard

{React.version}

+ ) } diff --git a/test/e2e/app-dir/app/provide-paths.test.ts b/test/e2e/app-dir/app/provide-paths.test.ts deleted file mode 100644 index ed5d38ad91b09..0000000000000 --- a/test/e2e/app-dir/app/provide-paths.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { nextTestSetup } from 'e2e-utils' -import glob from 'glob' -import path from 'path' - -describe('Provided page/app paths', () => { - const { next, isNextDev } = nextTestSetup({ - files: __dirname, - // Deployments are unable to inspect the `.next` directory. - skipDeployment: true, - dependencies: { - nanoid: '4.0.1', - }, - env: { - NEXT_PROVIDED_PAGE_PATHS: JSON.stringify(['/index.js', '/ssg.js']), - NEXT_PROVIDED_APP_PATHS: JSON.stringify([ - '/dashboard/page.js', - '/(newroot)/dashboard/another/page.js', - ]), - }, - }) - - if (isNextDev) { - it('should skip dev', () => {}) - return - } - - it('should only build the provided paths', async () => { - const appPaths = await glob.sync('**/*.js', { - cwd: path.join(next.testDir, '.next/server/app'), - }) - const pagePaths = await glob.sync('**/*.js', { - cwd: path.join(next.testDir, '.next/server/pages'), - }) - - expect(appPaths).toEqual([ - '_not-found/page_client-reference-manifest.js', - '_not-found/page.js', - '(newroot)/dashboard/another/page_client-reference-manifest.js', - '(newroot)/dashboard/another/page.js', - 'dashboard/page_client-reference-manifest.js', - 'dashboard/page.js', - ]) - expect(pagePaths).toEqual([ - '_app.js', - '_document.js', - '_error.js', - 'ssg.js', - ]) - - for (const pathname of ['/', '/ssg', '/dashboard', '/dashboard/another']) { - const res = await next.fetch(pathname) - expect(res.status).toBe(200) - } - }) -}) From 9af41a98dd638219b2fd431f24a67070ff344dae Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Jul 2024 13:46:00 -0700 Subject: [PATCH 2/2] apply suggestions from review Co-authored-by: Zack Tanner <1939140+ztanner@users.noreply.github.com> --- .../flying-shuttle/detect-changed-entries.ts | 146 +++++++++--------- .../src/build/flying-shuttle/stitch-builds.ts | 35 ++--- packages/next/src/build/index.ts | 6 +- 3 files changed, 90 insertions(+), 97 deletions(-) diff --git a/packages/next/src/build/flying-shuttle/detect-changed-entries.ts b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts index a301f1cbc1dfd..61c1b68e43ddc 100644 --- a/packages/next/src/build/flying-shuttle/detect-changed-entries.ts +++ b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts @@ -52,7 +52,7 @@ export async function detectChangedEntries({ if (!(await hasShuttle(shuttleDir))) { // no shuttle so consider everything changed - console.log('no shuttle can not detect changes') + console.log(`no shuttle. can't detect changes`) return { changed: { pages: pagesPaths || [], @@ -86,6 +86,7 @@ export async function detectChangedEntries({ } const hashSema = new Sema(16) + let globalEntryChanged = false async function detectChange({ normalizedEntry, @@ -102,90 +103,97 @@ export async function detectChangedEntries({ type, `${normalizedEntry}.js.nft.json` ) + let changed = true - const traceData: - | false - | { + // we don't need to check any further entry's dependencies if + // a global entry changed since that invalidates everything + if (!globalEntryChanged) { + try { + const traceData: { fileHashes: Record - } = JSON.parse( - await fs.promises - .readFile(traceFile, 'utf8') - .catch(() => JSON.stringify(false)) - ) - let changed = false - - if (traceData) { - await Promise.all( - Object.keys(traceData.fileHashes).map(async (file) => { - if (changed) return - try { - await hashSema.acquire() - const originalTraceFile = path.join( - distDir, - 'server', - type, - path.relative(path.join(shuttleDir, 'server', type), traceFile) - ) - const absoluteFile = path.join( - path.dirname(originalTraceFile), - file - ) - - if (absoluteFile.startsWith(distDir)) { - return - } - - const prevHash = traceData.fileHashes[file] - const curHash = await computeHash(absoluteFile) - - if (prevHash !== curHash) { - console.error('detected change on', { - prevHash, - curHash, - file, - entry: normalizedEntry, - }) - changed = true - } - } finally { - hashSema.release() + } = JSON.parse(await fs.promises.readFile(traceFile, 'utf8')) + + if (traceData) { + let changedDependency = false + await Promise.all( + Object.keys(traceData.fileHashes).map(async (file) => { + try { + if (changedDependency) return + await hashSema.acquire() + const relativeTraceFile = path.relative( + path.join(shuttleDir, 'server', type), + traceFile + ) + const originalTraceFile = path.join( + distDir, + 'server', + type, + relativeTraceFile + ) + const absoluteFile = path.join( + path.dirname(originalTraceFile), + file + ) + + if (absoluteFile.startsWith(distDir)) { + return + } + + const prevHash = traceData.fileHashes[file] + const curHash = await computeHash(absoluteFile) + + if (prevHash !== curHash) { + console.log('detected change on', { + prevHash, + curHash, + file, + entry: normalizedEntry, + }) + changedDependency = true + } + } finally { + hashSema.release() + } + }) + ) + + if (!changedDependency) { + changed = false } - }) - ) - } else { - console.error('missing trace data', traceFile, normalizedEntry) - changed = true + } else { + console.error('missing trace data', traceFile, normalizedEntry) + } + } catch (err) { + console.error(`Failed to detect change for ${entry}`, err) + } } - if (changed || entry.match(/(_app|_document|_error)/)) { + // we always rebuild global entries so we have a version + // that matches the newest build/runtime + const isGlobalEntry = /(_app|_document|_error)/.test(entry) + + if (changed || isGlobalEntry) { + // if a global entry changed all entries are changed + if (!globalEntryChanged && isGlobalEntry) { + console.log(`global entry ${entry} changed invalidating all entries`) + globalEntryChanged = true + // move unchanged to changed + changedEntries[type].push(...unchangedEntries[type]) + } changedEntries[type].push(entry) } else { unchangedEntries[type].push(entry) } } - // collect page entries with default page extensions - console.error( - JSON.stringify( - { - appPaths, - pagePaths: pagesPaths, - }, - null, - 2 - ) - ) - // loop over entries and their dependency's hashes to find - // which changed - - // TODO: if _app or _document change it invalidates all pages + // loop over entries and their dependency's hashes + // to detect which changed for (const entry of pagesPaths || []) { let normalizedEntry = getPageFromPath(entry, pageExtensions) if (normalizedEntry === '/') { normalizedEntry = '/index' } - await detectChange({ entry, normalizedEntry, type: 'pages' }) } @@ -194,7 +202,7 @@ export async function detectChangedEntries({ await detectChange({ entry, normalizedEntry, type: 'app' }) } - console.error( + console.log( 'changed entries', JSON.stringify( { diff --git a/packages/next/src/build/flying-shuttle/stitch-builds.ts b/packages/next/src/build/flying-shuttle/stitch-builds.ts index 891fd54e82618..c4d596c63971f 100644 --- a/packages/next/src/build/flying-shuttle/stitch-builds.ts +++ b/packages/next/src/build/flying-shuttle/stitch-builds.ts @@ -68,6 +68,12 @@ export async function stitchBuilds( // no shuttle directory nothing to stitch return {} } + // if a manifest is needed in the rest of the build + // we return it from here so it can be used without + // re-reading from disk after changing + const updatedManifests: { + pagesManifest?: PagesManifest + } = {} // we need to copy the chunks from the shuttle folder // to the distDir (we copy all server split chunks currently) @@ -227,22 +233,6 @@ export async function stitchBuilds( mergedBuildManifest.pages[key] = [] } - /* - TODO: for rootMainFiles we need to add a map that allows - referencing previous runtimes e.g. - [ - { - entries: string[] - runtimeFiles: string[] - } - ] - then we update the lookup to iterate over the array - to find the runtime files for the specific entry - - for pages we need to ensure the react chunk and such - is broken out into it's own split chunk correctly so - we don't reference new runtime chunks in a previous build - */ for (const entry of entries.unchanged.app || []) { const normalizedEntry = getPageFromPath(entry, entries.pageExtensions) mergedBuildManifest.rootMainFilesTree[normalizedEntry] = @@ -414,8 +404,6 @@ export async function stitchBuilds( JSON.stringify(mergedFunctionsConfigManifest, null, 2) ) - // for server/pages-manifest.json and server/app-paths-manifest.json - // we just merge for (const file of [APP_BUILD_MANIFEST, APP_PATH_ROUTES_MANIFEST]) { const [restorePagesManifest, currentPagesManifest] = await Promise.all( [path.join(shuttleDir, 'manifests', file), path.join(distDir, file)].map( @@ -440,7 +428,6 @@ export async function stitchBuilds( JSON.stringify(mergedPagesManifest, null, 2) ) } - let mergedPagesManifest: PagesManifest | undefined for (const file of [PAGES_MANIFEST, APP_PATHS_MANIFEST]) { const [restoreAppManifest, currentAppManifest] = await Promise.all( @@ -449,16 +436,16 @@ export async function stitchBuilds( path.join(distDir, 'server', file), ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) ) - const mergedAppManifest = { + const mergedManifest = { ...restoreAppManifest, ...currentAppManifest, } await fs.promises.writeFile( path.join(distDir, 'server', file), - JSON.stringify(mergedAppManifest, null, 2) + JSON.stringify(mergedManifest, null, 2) ) if (file === PAGES_MANIFEST) { - mergedPagesManifest = mergedAppManifest + updatedManifests.pagesManifest = mergedManifest } } @@ -498,7 +485,5 @@ export async function stitchBuilds( // TODO: inline env variables post build by find/replace // in all the chunks for NEXT_PUBLIC_? - return { - pagesManifest: mergedPagesManifest, - } + return updatedManifests } diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 3d6b3304f6075..fe356258db50b 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -869,7 +869,7 @@ export default async function build( distDir, shuttleDir, }) - console.error({ changedPagePathsResult }) + console.log({ changedPagePathsResult }) pagesPaths = changedPagePathsResult.changed.pages } @@ -960,7 +960,7 @@ export default async function build( distDir, shuttleDir, }) - console.error({ changedAppPathsResult }) + console.log({ changedAppPathsResult }) appPaths = changedAppPathsResult.changed.app } @@ -1104,7 +1104,7 @@ export default async function build( ) const routesManifestPath = path.join(distDir, ROUTES_MANIFEST) - let routesManifest: RoutesManifest = nextBuildSpan + const routesManifest: RoutesManifest = nextBuildSpan .traceChild('generate-routes-manifest') .traceFn(() => { const sortedRoutes = getSortedRoutes([