Turbopack: App Router build POC (#52036)

This PR adds proof-of-concept support for the App Router to `next build
--experimental-turbo`.

It introduces a new way to generate Next.js manifests in Turbopack.
Currently, in dev, we pass proxy objects in lieu of manifests, and rely
on the entries to know which chunks they need loaded on the client.
However, this can't work for builds because it requires control over
Next.js rendering, which is not compatible with a Next->Turbo approach.
We would need to modify Next.js to support these "lazy" entries. So for
now, we add well-known assets (`NextDynamicAsset`,
`NextServerComponentAsset`, `NextClientReferenceAsset`, etc.) to the
graph, which will get picked up when walking it during asset processing.
This lets us collect all possible entries before chunking.

This two-step process (collecting all entries, then chunking them) is
also a good first step towards production chunking.

## Turbopack updates

* vercel/turborepo#5494 <!-- Tobias Koppers - add
reporting of console messages -->
* vercel/turborepo#5448 <!-- Alex Kirszenberg -
Misc. changes to support App Router build -->

Cargo.toml

@@ -42,11 +42,11 @@ swc_core = { version = "0.79.13" }
 testing = { version = "0.33.20" }
 
 # Turbo crates
-turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230711.2" }
+turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230712.2" }
 # [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
-turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230711.2" }
+turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230712.2" }
 # [TODO]: need to refactor embed_directory! macro usage in next-core
-turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230711.2" }
+turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230712.2" }
 
 # General Deps
 
@@ -74,6 +74,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [
   "gzip",
   "tokio",
 ] }
+async-recursion = "1.0.2"
 async-trait = "0.1.64"
 atty = "0.2.14"
 chrono = "0.4.23"

packages/next-swc/crates/next-build/Cargo.toml

@@ -39,6 +39,7 @@ profile = []
 
 [dependencies]
 anyhow = { workspace = true }
+async-recursion = { workspace = true }
 clap = { workspace = true, features = ["derive", "env"], optional = true }
 console-subscriber = { workspace = true, optional = true }
 dunce = { workspace = true }
@@ -48,6 +49,10 @@ serde_json = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
+indoc = { workspace = true }
+indexmap = { workspace = true }
+mime_guess = "2.0.4"
+base64 = "0.21.0"
 
 turbopack-binding = { workspace = true, features = [
   "__turbo_tasks",

packages/next-swc/crates/next-build/src/lib.rs

@@ -1,10 +1,13 @@
+#![feature(type_alias_impl_trait)]
+
 use turbopack_binding::turbo::{
     tasks::{run_once, TransientInstance, TurboTasks},
     tasks_memory::MemoryBackend,
 };
 
 pub mod build_options;
 pub mod manifests;
+pub(crate) mod next_app;
 pub(crate) mod next_build;
 pub(crate) mod next_pages;