Code frame and sourcemapped error support for Turbopack

packages/next-swc/crates/napi/src/next_api/project.rs

@@ -1,4 +1,4 @@
-use std::{io::Read, path::PathBuf, sync::Arc, time::Duration};
+use std::{path::PathBuf, sync::Arc, time::Duration};
 
 use anyhow::{anyhow, bail, Context, Result};
 use napi::{
@@ -613,14 +613,6 @@ pub fn project_update_info_subscribe(
     Ok(())
 }
 
-#[turbo_tasks::value]
-#[derive(Debug)]
-#[napi(object)]
-pub struct TracedSource {
-    pub frame: StackFrame,
-    pub source: String,
-}
-
 #[turbo_tasks::value]
 #[derive(Debug)]
 #[napi(object)]
@@ -635,7 +627,7 @@ pub struct StackFrame {
 pub async fn project_trace_source(
     #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
     frame: StackFrame,
-) -> napi::Result<Option<TracedSource>> {
+) -> napi::Result<Option<StackFrame>> {
     let turbo_tasks = project.turbo_tasks.clone();
     let traced_frame = turbo_tasks
         .run_once(async move {
@@ -677,10 +669,11 @@ pub async fn project_trace_source(
                 .root()
                 .join(chunk_path);
 
-            let generatable: Vc<Box<dyn GenerateSourceMap>> =
-                Vc::try_resolve_sidecast(project.container.get_versioned_content(path))
-                    .await?
-                    .context("Chunk cannot produce a sourcemap")?;
+            let Some(generatable): Option<Vc<Box<dyn GenerateSourceMap>>> =
+                Vc::try_resolve_sidecast(project.container.get_versioned_content(path)).await?
+            else {
+                return Ok(None);
+            };
 
             let map = generatable
                 .generate_source_map()
@@ -694,40 +687,50 @@ pub async fn project_trace_source(
                 .context("Unable to trace token from sourcemap")?;
 
             let Token::Original(token) = token else {
-                bail!("Expected token to be an OriginalToken")
+                return Ok(None);
             };
 
             let Some(source_file) = token.original_file.strip_prefix("/turbopack/[project]/")
             else {
                 bail!("Original file outside project")
             };
 
+            Ok(Some(StackFrame {
+                file: source_file.to_string(),
+                method_name: token.name,
+                line: token.original_line as u32,
+                column: Some(token.original_column as u32),
+            }))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+    Ok(traced_frame)
+}
+
+#[napi]
+pub async fn project_get_source_for_asset(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+    file_path: String,
+) -> napi::Result<Option<String>> {
+    let turbo_tasks = project.turbo_tasks.clone();
+    let source = turbo_tasks
+        .run_once(async move {
             let source_content = &*project
                 .container
                 .project()
                 .project_path()
-                .join(source_file.to_string())
+                .join(file_path.to_string())
                 .read()
                 .await?;
 
             let FileContent::Content(source_content) = source_content else {
-                bail!("No content for source file")
+                return Ok(None);
             };
 
-            let mut source = "".to_string();
-            Read::read_to_string(&mut source_content.read(), &mut source)?;
-
-            Ok(Some(TracedSource {
-                frame: StackFrame {
-                    file: source_file.to_string(),
-                    method_name: token.name,
-                    line: token.original_line as u32,
-                    column: Some(token.original_column as u32),
-                },
-                source,
-            }))
+            Ok(Some(source_content.content().to_str()?.to_string()))
         })
         .await
         .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
-    Ok(traced_frame)
+
+    Ok(source)
 }

packages/next/src/build/swc/index.ts

@@ -521,9 +521,9 @@ export interface HmrIdentifiers {
 
 export interface StackFrame {
   file: string
+  methodName: string | null
   line: number
-  column: number | undefined
-  methodName: string | undefined
+  column: number | null
 }
 
 export interface UpdateInfo {
@@ -547,13 +547,8 @@ export interface Project {
   hmrIdentifiersSubscribe(): AsyncIterableIterator<
     TurbopackResult<HmrIdentifiers>
   >
-  traceSource(stackFrame: StackFrame): Promise<
-    | {
-        frame: StackFrame
-        source: string
-      }
-    | undefined
-  >
+  getSourceForAsset(filePath: string): Promise<string | null>
+  traceSource(stackFrame: StackFrame): Promise<StackFrame | null>
   updateInfoSubscribe(): AsyncIterableIterator<TurbopackResult<UpdateInfo>>
 }
 
@@ -930,16 +925,14 @@ function bindingToApi(binding: any, _wasm: boolean) {
       return subscription
     }
 
-    traceSource(stackFrame: StackFrame): Promise<
-      | {
-          frame: StackFrame
-          source: string
-        }
-      | undefined
-    > {
+    traceSource(stackFrame: StackFrame): Promise<StackFrame | null> {
       return binding.projectTraceSource(this._nativeProject, stackFrame)
     }
 
+    getSourceForAsset(filePath: string): Promise<string | null> {
+      return binding.projectGetSourceForAsset(this._nativeProject, filePath)
+    }
+
     updateInfoSubscribe() {
       const subscription = subscribe<TurbopackResult<UpdateInfo>>(
         true,

packages/next/src/server/lib/router-utils/setup-dev-bundler.ts

@@ -1061,14 +1061,13 @@ async function startWatcher(opts: SetupOpts) {
     await writeOtherManifests()
     await writeFontManifest()
 
-    const middleware = getOverlayMiddleware(project)
-
+    const overlayMiddleware = getOverlayMiddleware(project)
     const turbopackHotReloader: NextJsHotReloaderInterface = {
       turbopackProject: project,
       activeWebpackConfigs: undefined,
       serverStats: null,
       edgeServerStats: null,
-      async run(req, _res, _parsedUrl) {
+      async run(req, res, _parsedUrl) {
         // intercept page chunks request and ensure them with turbopack
         if (req.url?.startsWith('/_next/static/chunks/pages/')) {
           const params = matchNextPageBundleRequest(req.url)
@@ -1090,12 +1089,7 @@ async function startWatcher(opts: SetupOpts) {
           }
         }
 
-        await new Promise<void>((resolve, reject) => {
-          middleware(req, _res, (err?: Error) => {
-            if (err) return reject(err)
-            resolve()
-          })
-        })
+        await overlayMiddleware(req, res)
 
         // Request was not finished.
         return { finished: undefined }

packages/react-dev-overlay/src/middleware-turbopack.ts

@@ -9,57 +9,88 @@ import { codeFrameColumns } from '@babel/code-frame'
 import { launchEditor } from './internal/helpers/launchEditor'
 
 interface Project {
-  traceSource(stackFrame: RustStackFrame): Promise<
-    | {
-        frame: RustStackFrame
-        source: string
-      }
-    | undefined
-  >
+  getSourceForAsset(filePath: string): Promise<string | null>
+  traceSource(stackFrame: RustStackFrame): Promise<RustStackFrame | null>
 }
 
 interface RustStackFrame {
   file: string
-  methodName: string | undefined
+  methodName: string | null
   line: number
-  column: number | undefined
+  column: number | null
 }
 
-export async function createOriginalStackFrame(
-  project: Project,
-  frame: StackFrame
-): Promise<OriginalStackFrameResponse | null> {
-  const source = await project.traceSource({
-    file: frame.file ?? '<unknown>',
+const currentSourcesByFile: Map<string, Promise<string | null>> = new Map()
+async function batchedTraceSource(project: Project, frame: StackFrame) {
+  const file = frame.file
+  if (!file) {
+    return
+  }
+
+  const rustStackFrame = {
+    file,
     methodName: frame.methodName,
     line: frame.lineNumber ?? 0,
-    column: frame.column ?? undefined,
-  })
+    column: frame.column,
+  }
 
-  if (!source) {
-    return null
+  const sourceFrame = await project.traceSource(rustStackFrame)
+  if (!sourceFrame) {
+    return
+  }
+
+  let source
+  // Don't show code frames for node_modules. These can also often be large bundled files.
+  if (!sourceFrame.file.includes('node_modules')) {
+    let sourcePromise = currentSourcesByFile.get(sourceFrame.file)
+    if (!sourcePromise) {
+      sourcePromise = new Promise((resolve) =>
+        // Batch reading sources content as this can be quite large, and stacks often reference the same files
+        setTimeout(resolve, 100)
+      ).then(() => project.getSourceForAsset(sourceFrame.file))
+      currentSourcesByFile.set(sourceFrame.file, sourcePromise)
+    }
+
+    source = await sourcePromise
+    currentSourcesByFile.delete(sourceFrame.file)
   }
 
   return {
-    originalStackFrame: {
-      file: source.frame.file,
-      lineNumber: source.frame.line,
-      column: source.frame.column ?? null,
-      methodName: source.frame.methodName ?? frame.methodName,
+    frame: {
+      file,
+      lineNumber: sourceFrame.line,
+      column: sourceFrame.column,
+      methodName: sourceFrame.methodName ?? frame.methodName,
       arguments: [],
     },
-    originalCodeFrame: source.frame.file.includes('node_modules')
-      ? null
-      : codeFrameColumns(
-          source.source,
-          {
-            start: {
-              line: source.frame.line,
-              column: source.frame.column ?? 0,
+    source: source ?? null,
+  }
+}
+
+export async function createOriginalStackFrame(
+  project: Project,
+  frame: StackFrame
+): Promise<OriginalStackFrameResponse | null> {
+  const traced = await batchedTraceSource(project, frame)
+  if (!traced) {
+    return null
+  }
+
+  return {
+    originalStackFrame: traced.frame,
+    originalCodeFrame:
+      traced.source === null || traced.frame.file.includes('node_modules')
+        ? null
+        : codeFrameColumns(
+            traced.source,
+            {
+              start: {
+                line: traced.frame.lineNumber,
+                column: traced.frame.column ?? 0,
+              },
             },
-          },
-          { forceColor: true }
-        ),
+            { forceColor: true }
+          ),
   }
 }
 
@@ -78,11 +109,7 @@ function stackFrameFromQuery(query: ParsedUrlQuery): StackFrame {
 }
 
 export function getOverlayMiddleware(project: Project) {
-  return async function (
-    req: IncomingMessage,
-    res: ServerResponse,
-    next: (error?: Error) => unknown
-  ) {
+  return async function (req: IncomingMessage, res: ServerResponse) {
     const { pathname, query } = url.parse(req.url!, true)
     if (pathname === '/__nextjs_original-stack-frame') {
       const frame = stackFrameFromQuery(query)
@@ -92,27 +119,31 @@ export function getOverlayMiddleware(project: Project) {
       } catch (e: any) {
         res.statusCode = 500
         res.write(e.message)
-        return res.end()
+        res.end()
+        return
       }
 
       if (originalStackFrame === null) {
         res.statusCode = 404
         res.write('Unable to resolve sourcemap')
-        return res.end()
+        res.end()
+        return
       }
 
       res.statusCode = 200
       res.setHeader('Content-Type', 'application/json')
       res.write(Buffer.from(JSON.stringify(originalStackFrame)))
-      return res.end()
+      res.end()
+      return
     } else if (pathname === '/__nextjs_launch-editor') {
       const frame = stackFrameFromQuery(query)
 
       const filePath = frame.file?.toString()
       if (filePath === undefined) {
         res.statusCode = 400
         res.write('Bad Request')
-        return res.end()
+        res.end()
+        return
       }
 
       const fileExists = await fs.access(filePath, FS.F_OK).then(
@@ -122,7 +153,8 @@ export function getOverlayMiddleware(project: Project) {
       if (!fileExists) {
         res.statusCode = 204
         res.write('No Content')
-        return res.end()
+        res.end()
+        return
       }
 
       try {
@@ -131,13 +163,12 @@ export function getOverlayMiddleware(project: Project) {
         console.log('Failed to launch editor:', err)
         res.statusCode = 500
         res.write('Internal Server Error')
-        return res.end()
+        res.end()
+        return
       }
 
       res.statusCode = 204
-      return res.end()
+      res.end()
     }
-
-    return next()
   }
 }