Add additional handling for experimental tracing (#67785)

This adds additional trace handling under the experimental trace mode.
Outside of the feature flag the bloom filter for app router paths used
in pages to hard navigate was updated to no longer be inlined by default
with webpack to avoid invalidating the main bundle for pages too often.
Instead the bloom filter data is stored in the `_buildManifest` file
which is only used in the pages router and is easier to update as it's
per-build already.

This tracing feature is still very experimental and should not be
leveraged in production or outside of testing.

---------

Co-authored-by: Zack Tanner <1939140+ztanner@users.noreply.github.com>

packages/next/src/build/flying-shuttle/detect-changed-entries.ts

@@ -0,0 +1,221 @@
+import fs from 'fs'
+import path from 'path'
+import crypto from 'crypto'
+import { getPageFromPath } from '../entries'
+import { Sema } from 'next/dist/compiled/async-sema'
+
+export interface DetectedEntriesResult {
+  app: string[]
+  pages: string[]
+}
+
+let _hasShuttle: undefined | boolean = undefined
+export async function hasShuttle(shuttleDir: string) {
+  if (typeof _hasShuttle === 'boolean') {
+    return _hasShuttle
+  }
+  _hasShuttle = await fs.promises
+    .access(path.join(shuttleDir, 'server'))
+    .then(() => true)
+    .catch(() => false)
+
+  return _hasShuttle
+}
+
+export async function detectChangedEntries({
+  appPaths,
+  pagesPaths,
+  pageExtensions,
+  distDir,
+  shuttleDir,
+}: {
+  appPaths?: string[]
+  pagesPaths?: string[]
+  pageExtensions: string[]
+  distDir: string
+  shuttleDir: string
+}): Promise<{
+  changed: DetectedEntriesResult
+  unchanged: DetectedEntriesResult
+}> {
+  const changedEntries: {
+    app: string[]
+    pages: string[]
+  } = {
+    app: [],
+    pages: [],
+  }
+  const unchangedEntries: typeof changedEntries = {
+    app: [],
+    pages: [],
+  }
+
+  if (!(await hasShuttle(shuttleDir))) {
+    // no shuttle so consider everything changed
+    console.log(`no shuttle. can't detect changes`)
+    return {
+      changed: {
+        pages: pagesPaths || [],
+        app: appPaths || [],
+      },
+      unchanged: {
+        pages: [],
+        app: [],
+      },
+    }
+  }
+
+  const hashCache = new Map<string, string>()
+
+  async function computeHash(p: string): Promise<string> {
+    let hash = hashCache.get(p)
+    if (hash) {
+      return hash
+    }
+    return new Promise((resolve, reject) => {
+      const hashInst = crypto.createHash('sha1')
+      const stream = fs.createReadStream(p)
+      stream.on('error', (err) => reject(err))
+      stream.on('data', (chunk) => hashInst.update(chunk))
+      stream.on('end', () => {
+        const digest = hashInst.digest('hex')
+        resolve(digest)
+        hashCache.set(p, digest)
+      })
+    })
+  }
+
+  const hashSema = new Sema(16)
+  let globalEntryChanged = false
+
+  async function detectChange({
+    normalizedEntry,
+    entry,
+    type,
+  }: {
+    entry: string
+    normalizedEntry: string
+    type: keyof typeof changedEntries
+  }) {
+    const traceFile = path.join(
+      shuttleDir,
+      'server',
+      type,
+      `${normalizedEntry}.js.nft.json`
+    )
+    let changed = true
+
+    // we don't need to check any further entry's dependencies if
+    // a global entry changed since that invalidates everything
+    if (!globalEntryChanged) {
+      try {
+        const traceData: {
+          fileHashes: Record<string, string>
+        } = JSON.parse(await fs.promises.readFile(traceFile, 'utf8'))
+
+        if (traceData) {
+          let changedDependency = false
+          await Promise.all(
+            Object.keys(traceData.fileHashes).map(async (file) => {
+              try {
+                if (changedDependency) return
+                await hashSema.acquire()
+                const relativeTraceFile = path.relative(
+                  path.join(shuttleDir, 'server', type),
+                  traceFile
+                )
+                const originalTraceFile = path.join(
+                  distDir,
+                  'server',
+                  type,
+                  relativeTraceFile
+                )
+                const absoluteFile = path.join(
+                  path.dirname(originalTraceFile),
+                  file
+                )
+
+                if (absoluteFile.startsWith(distDir)) {
+                  return
+                }
+
+                const prevHash = traceData.fileHashes[file]
+                const curHash = await computeHash(absoluteFile)
+
+                if (prevHash !== curHash) {
+                  console.log('detected change on', {
+                    prevHash,
+                    curHash,
+                    file,
+                    entry: normalizedEntry,
+                  })
+                  changedDependency = true
+                }
+              } finally {
+                hashSema.release()
+              }
+            })
+          )
+
+          if (!changedDependency) {
+            changed = false
+          }
+        } else {
+          console.error('missing trace data', traceFile, normalizedEntry)
+        }
+      } catch (err) {
+        console.error(`Failed to detect change for ${entry}`, err)
+      }
+    }
+
+    // we always rebuild global entries so we have a version
+    // that matches the newest build/runtime
+    const isGlobalEntry = /(_app|_document|_error)/.test(entry)
+
+    if (changed || isGlobalEntry) {
+      // if a global entry changed all entries are changed
+      if (!globalEntryChanged && isGlobalEntry) {
+        console.log(`global entry ${entry} changed invalidating all entries`)
+        globalEntryChanged = true
+        // move unchanged to changed
+        changedEntries[type].push(...unchangedEntries[type])
+      }
+      changedEntries[type].push(entry)
+    } else {
+      unchangedEntries[type].push(entry)
+    }
+  }
+
+  // loop over entries and their dependency's hashes
+  // to detect which changed
+  for (const entry of pagesPaths || []) {
+    let normalizedEntry = getPageFromPath(entry, pageExtensions)
+
+    if (normalizedEntry === '/') {
+      normalizedEntry = '/index'
+    }
+    await detectChange({ entry, normalizedEntry, type: 'pages' })
+  }
+
+  for (const entry of appPaths || []) {
+    const normalizedEntry = getPageFromPath(entry, pageExtensions)
+    await detectChange({ entry, normalizedEntry, type: 'app' })
+  }
+
+  console.log(
+    'changed entries',
+    JSON.stringify(
+      {
+        changedEntries,
+        unchangedEntries,
+      },
+      null,
+      2
+    )
+  )
+
+  return {
+    changed: changedEntries,
+    unchanged: unchangedEntries,
+  }
+}