Update generating next-server dependencies (#28223)

* Update generating next-server dependencies

* Update tests

* ensure stale cache is not used

packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts

@@ -69,7 +69,9 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
       const traceOutputName = `${isWebpack5 ? '../' : ''}${
         entrypoint.name
       }.js.nft.json`
-      const traceOutputPath = nodePath.join(outputPath, traceOutputName)
+      const traceOutputPath = nodePath.dirname(
+        nodePath.join(outputPath, traceOutputName)
+      )
 
       assets[traceOutputName] = new sources.RawSource(
         JSON.stringify({

packages/next/package.json

@@ -50,7 +50,8 @@
   "scripts": {
     "dev": "taskr",
     "release": "taskr release",
-    "prepublish": "npm run release && yarn types",
+    "prepublish": "npm run release && yarn types && yarn trace-server",
+    "trace-server": "node ../../scripts/trace-next-server.js",
     "types": "tsc --declaration --emitDeclarationOnly --declarationDir dist",
     "typescript": "tsc --noEmit --declaration",
     "ncc-compiled": "ncc cache clean && taskr ncc",

packages/next/server/next-server.ts

@@ -87,7 +87,6 @@ import { FontManifest } from './font-utils'
 import { denormalizePagePath } from './denormalize-page-path'
 import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
 import * as Log from '../build/output/log'
-import { imageOptimizer } from './image-optimizer'
 import { detectDomainLocale } from '../shared/lib/i18n/detect-domain-locale'
 import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters'
 import { getUtils } from '../build/webpack/loaders/next-serverless-loader/utils'
@@ -700,16 +699,27 @@ export default class Server {
         match: route('/_next/image'),
         type: 'route',
         name: '_next/image catchall',
-        fn: (req, res, _params, parsedUrl) =>
-          imageOptimizer(
+        fn: (req, res, _params, parsedUrl) => {
+          if (this.minimalMode) {
+            res.statusCode = 400
+            res.end('Bad Request')
+            return {
+              finished: true,
+            }
+          }
+          const { imageOptimizer } =
+            require('./image-optimizer') as typeof import('./image-optimizer')
+
+          return imageOptimizer(
             server,
             req,
             res,
             parsedUrl,
             server.nextConfig,
             server.distDir,
             this.renderOpts.dev
-          ),
+          )
+        },
       },
       {
         match: route('/_next/:path*'),

packages/next/taskfile.js

@@ -1,9 +1,7 @@
-const fs = require('fs')
 // eslint-disable-next-line import/no-extraneous-dependencies
 const notifier = require('node-notifier')
 // eslint-disable-next-line import/no-extraneous-dependencies
-const { nodeFileTrace } = require('@vercel/nft')
-const { join, relative, basename, resolve } = require('path')
+const { relative, basename, resolve } = require('path')
 const { Module } = require('module')
 
 // Note:
@@ -764,45 +762,6 @@ export async function precompile(task, opts) {
   )
 }
 
-// eslint-disable-next-line camelcase
-export async function trace_next_server(task) {
-  const { TRACE_OUTPUT_VERSION } = require('next/dist/shared/lib/constants')
-  const root = join(__dirname, '../../')
-
-  const result = await nodeFileTrace(
-    [require.resolve('next/dist/server/next-server')],
-    {
-      base: root,
-      processCwd: __dirname,
-      ignore: [
-        'packages/next/dist/compiled/webpack/(bundle4|bundle5).js',
-        'node_modules/react/**/*.development.js',
-        'node_modules/react-dom/**/*.development.js',
-        'node_modules/use-subscription/**/*.development.js',
-        'node_modules/next/dist/next-server/server/lib/squoosh/**/*.wasm',
-        'packages/next/dist/pages/**/*',
-      ],
-    }
-  )
-
-  const tracedDeps = []
-
-  for (const file of result.fileList) {
-    if (result.reasons[file].type === 'initial') {
-      continue
-    }
-    tracedDeps.push(join(root, file))
-  }
-
-  fs.writeFileSync(
-    join(__dirname, 'dist/server/next-server.nft.json'),
-    JSON.stringify({
-      version: TRACE_OUTPUT_VERSION,
-      files: tracedDeps,
-    })
-  )
-}
-
 // eslint-disable-next-line camelcase
 export async function copy_ncced(task) {
   // we don't ncc every time we build since these won't change
@@ -1000,7 +959,7 @@ export async function telemetry(task, opts) {
 }
 
 export async function build(task, opts) {
-  await task.serial(['precompile', 'compile', 'trace_next_server'], opts)
+  await task.serial(['precompile', 'compile'], opts)
 }
 
 export default async function (task) {

scripts/trace-next-server.js

@@ -0,0 +1,132 @@
+const os = require('os')
+const path = require('path')
+const execa = require('execa')
+const fs = require('fs-extra')
+const prettyBytes = require('pretty-bytes')
+const gzipSize = require('next/dist/compiled/gzip-size')
+const { nodeFileTrace } = require('next/dist/compiled/@vercel/nft')
+
+const MAX_COMPRESSED_SIZE = 250 * 1000
+const MAX_UNCOMPRESSED_SIZE = 2.5 * 1000 * 1000
+
+// install next outside the monorepo for clean `node_modules`
+// to trace against which helps ensure minimal trace is
+// produced.
+// react and react-dom need to be traced specific to installed
+// version so isn't pre-traced
+async function main() {
+  const tmpdir = os.tmpdir()
+  await execa('yarn', ['pack'], {
+    cwd: path.join(__dirname, '../packages/next'),
+    stdio: ['ignore', 'inherit', 'inherit'],
+  })
+  const packagePath = path.join(
+    __dirname,
+    `../packages/next/next-v${
+      require('../packages/next/package.json').version
+    }.tgz`
+  )
+  const workDir = path.join(tmpdir, `trace-next-${Date.now()}`)
+  console.log('using workdir', workDir)
+  await fs.ensureDir(workDir)
+
+  await fs.writeFile(
+    path.join(workDir, 'package.json'),
+    JSON.stringify(
+      {
+        dependencies: {
+          next: packagePath,
+        },
+        private: true,
+      },
+      null,
+      2
+    )
+  )
+  await execa('yarn', ['install'], {
+    cwd: workDir,
+    stdio: ['ignore', 'inherit', 'inherit'],
+    env: {
+      ...process.env,
+      YARN_CACHE_FOLDER: path.join(workDir, '.yarn-cache'),
+    },
+  })
+
+  const nextServerPath = path.join(
+    workDir,
+    'node_modules/next/dist/server/next-server.js'
+  )
+
+  const traceLabel = `traced ${nextServerPath}`
+  console.time(traceLabel)
+
+  const result = await nodeFileTrace([nextServerPath], {
+    base: workDir,
+    processCwd: workDir,
+    ignore: [
+      'node_modules/next/dist/pages/**/*',
+      'node_modules/next/dist/server/image-optimizer.js',
+      'node_modules/next/dist/compiled/@ampproject/toolbox-optimizer/**/*',
+      'node_modules/next/dist/server/lib/squoosh/**/*.wasm',
+      'node_modules/next/dist/compiled/webpack/(bundle4|bundle5).js',
+      'node_modules/react/**/*.development.js',
+      'node_modules/react-dom/**/*.development.js',
+      'node_modules/use-subscription/**/*.development.js',
+      'node_modules/sharp/**/*',
+    ],
+  })
+
+  const tracedDeps = new Set()
+  let totalCompressedSize = 0
+  let totalUncompressedSize = 0
+
+  for (const file of result.fileList) {
+    if (result.reasons[file].type === 'initial') {
+      continue
+    }
+    tracedDeps.add(file)
+    const stat = await fs.stat(path.join(workDir, file))
+
+    if (stat.isFile()) {
+      const compressedSize = await gzipSize(path.join(workDir, file))
+      totalUncompressedSize += stat.size || 0
+      totalCompressedSize += compressedSize
+    } else {
+      console.log('not a file', file, stat.isDirectory())
+    }
+  }
+
+  console.log({
+    numberFiles: tracedDeps.size,
+    totalGzipSize: prettyBytes(totalCompressedSize),
+    totalUncompressedSize: prettyBytes(totalUncompressedSize),
+  })
+
+  await fs.writeFile(
+    path.join(
+      __dirname,
+      '../packages/next/dist/server/next-server.js.nft.json'
+    ),
+    JSON.stringify({
+      files: Array.from(tracedDeps),
+      version: 1,
+    })
+  )
+  await fs.unlink(packagePath)
+  await fs.remove(workDir)
+
+  console.timeEnd(traceLabel)
+
+  if (
+    totalCompressedSize > MAX_COMPRESSED_SIZE ||
+    totalUncompressedSize > MAX_UNCOMPRESSED_SIZE
+  ) {
+    throw new Error(
+      `Max traced size of next-server exceeded limits of ${MAX_COMPRESSED_SIZE} compressed or ${MAX_UNCOMPRESSED_SIZE} uncompressed`
+    )
+  }
+}
+
+main()
+  .then(() => console.log('done'))
+  .catch(console.error)

test/integration/required-server-files/next.config.js → test/integration/required-server-files/app/next.config.js

@@ -1,6 +1,12 @@
 module.exports = {
   // ensure incorrect target is overridden by env
   target: 'serverless',
+  experimental: {
+    nftTracing: true,
+  },
+  eslint: {
+    ignoreDuringBuilds: true,
+  },
   rewrites() {
     return [
       {