From 73b6a95a7b20d0362e79022395d724c95ad5d11c Mon Sep 17 00:00:00 2001 From: Alexander Lyon Date: Tue, 5 Nov 2024 06:29:47 +0000 Subject: [PATCH] wip --- crates/next-api/src/app.rs | 108 +++---- crates/next-api/src/instrumentation.rs | 27 +- crates/next-api/src/middleware.rs | 5 +- crates/next-api/src/pages.rs | 46 +-- crates/next-api/src/paths.rs | 10 + crates/next-api/src/project.rs | 4 +- crates/next-core/src/next_config.rs | 10 + crates/next-core/src/next_import_map.rs | 7 +- .../client_reference_manifest.rs | 8 +- crates/next-core/src/next_manifests/mod.rs | 188 +++++++++++- crates/next-core/src/next_server/context.rs | 25 +- crates/next-core/src/next_server/resolve.rs | 88 ++---- turbopack/crates/turbo-tasks-fs/src/util.rs | 17 +- .../src/references/esm/base.rs | 5 + .../src/references/mod.rs | 4 +- .../src/references/node.rs | 9 + .../src/node_native_binding.rs | 10 +- turbopack/crates/turbopack/src/lib.rs | 275 +++++++++++------- 18 files changed, 544 insertions(+), 302 deletions(-) diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs index 90faffb314d26..873dcd97df145 100644 --- a/crates/next-api/src/app.rs +++ b/crates/next-api/src/app.rs @@ -38,8 +38,8 @@ use next_core::{ use serde::{Deserialize, Serialize}; use tracing::Instrument; use turbo_tasks::{ - fxindexset, trace::TraceRawVcs, Completion, FxIndexMap, FxIndexSet, RcStr, ResolvedVc, - TryJoinIterExt, Value, ValueToString, Vc, + fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexMap, FxIndexSet, RcStr, + ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc, }; use turbo_tasks_env::{CustomProcessEnv, ProcessEnv}; use turbo_tasks_fs::{File, FileContent, FileSystemPath}; @@ -75,7 +75,7 @@ use crate::{ loadable_manifest::create_react_loadable_manifest, nft_json::NftJsonAsset, paths::{ - all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root, + all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, }, project::Project, @@ -701,7 +701,7 @@ pub fn app_entry_point_to_route( } #[turbo_tasks::function] -fn client_shared_chunks() -> Vc { +fn client_shared_chunks_mod() -> Vc { Vc::cell("client_shared_chunks".into()) } @@ -845,7 +845,6 @@ impl AppEndpoint { let node_root = this.app_project.project().node_root(); let client_relative_path = this.app_project.project().client_relative_path(); - let client_relative_path_ref = client_relative_path.await?; let server_path = node_root.join("server".into()); @@ -882,21 +881,19 @@ impl AppEndpoint { ) = if process_client_components { let client_shared_chunk_group = get_app_client_shared_chunk_group( AssetIdent::from_path(this.app_project.project().project_path()) - .with_modifier(client_shared_chunks()), + .with_modifier(client_shared_chunks_mod()), this.app_project.client_runtime_entries(), client_chunking_context, ) .await?; - let mut client_shared_chunks_paths = vec![]; + let mut client_shared_chunks = vec![]; for chunk in client_shared_chunk_group.assets.await?.iter().copied() { client_assets.insert(chunk); let chunk_path = chunk.ident().path().await?; if chunk_path.extension_ref() == Some("js") { - if let Some(chunk_path) = client_relative_path_ref.get_path_to(&chunk_path) { - client_shared_chunks_paths.push(chunk_path.into()); - } + client_shared_chunks.push(chunk); } } let client_shared_availability_info = client_shared_chunk_group.availability_info; @@ -988,39 +985,28 @@ impl AppEndpoint { client_assets.extend(entry_client_chunks.iter().copied()); server_assets.extend(entry_ssr_chunks.iter().copied()); - let entry_client_chunks_paths = entry_client_chunks - .iter() - .map(|chunk| chunk.ident().path()) - .try_join() - .await?; - let mut entry_client_chunks_paths = entry_client_chunks_paths - .iter() - .map(|path| { - Ok(client_relative_path_ref - .get_path_to(path) - .context("asset path should be inside client root")? - .into()) - }) - .collect::>>()?; - entry_client_chunks_paths.extend(client_shared_chunks_paths.iter().cloned()); - let app_build_manifest = AppBuildManifest { - pages: [(app_entry.original_name.clone(), entry_client_chunks_paths)] - .into_iter() - .collect(), + pages: fxindexmap!( + app_entry.original_name.clone() => Vc::cell(entry_client_chunks + .iter() + .chain(client_shared_chunks.iter()) + .copied() + .collect()) + ), }; let manifest_path_prefix = &app_entry.original_name; - let app_build_manifest_output = VirtualOutputAsset::new( - node_root.join( - format!("server/app{manifest_path_prefix}/app-build-manifest.json",).into(), - ), - AssetContent::file( - File::from(serde_json::to_string_pretty(&app_build_manifest)?).into(), - ), - ) - .to_resolved() - .await?; - server_assets.insert(ResolvedVc::upcast(app_build_manifest_output)); + let app_build_manifest_output = app_build_manifest + .build_output( + node_root.join( + format!("server/app{manifest_path_prefix}/app-build-manifest.json",).into(), + ), + client_relative_path, + ) + .await? + .to_resolved() + .await?; + + server_assets.insert(app_build_manifest_output); // polyfill-nomodule.js is a pre-compiled asset distributed as part of next, // load it as a RawModule. @@ -1030,16 +1016,12 @@ impl AppEndpoint { ); let polyfill_output_path = client_chunking_context.chunk_path(polyfill_source.ident(), ".js".into()); - let polyfill_output_asset = + let polyfill_output_asset = ResolvedVc::upcast( RawOutput::new(polyfill_output_path, Vc::upcast(polyfill_source)) .to_resolved() - .await?; - let polyfill_client_path = client_relative_path_ref - .get_path_to(&*polyfill_output_path.await?) - .context("failed to resolve client-relative path to polyfill")? - .into(); - let polyfill_client_paths = vec![polyfill_client_path]; - client_assets.insert(ResolvedVc::upcast(polyfill_output_asset)); + .await?, + ); + client_assets.insert(polyfill_output_asset); if *this .app_project @@ -1063,20 +1045,23 @@ impl AppEndpoint { } let build_manifest = BuildManifest { - root_main_files: client_shared_chunks_paths, - polyfill_files: polyfill_client_paths, + root_main_files: client_shared_chunks, + polyfill_files: vec![polyfill_output_asset], ..Default::default() }; - let build_manifest_output = VirtualOutputAsset::new( - node_root - .join(format!("server/app{manifest_path_prefix}/build-manifest.json",).into()), - AssetContent::file( - File::from(serde_json::to_string_pretty(&build_manifest)?).into(), - ), - ) - .to_resolved() - .await?; - server_assets.insert(ResolvedVc::upcast(build_manifest_output)); + let build_manifest_output = ResolvedVc::upcast( + build_manifest + .build_output( + node_root.join( + format!("server/app{manifest_path_prefix}/build-manifest.json",).into(), + ), + client_relative_path, + ) + .await? + .to_resolved() + .await?, + ); + server_assets.insert(build_manifest_output); if runtime == NextRuntime::Edge { // as the edge runtime doesn't support chunk loading we need to add all client @@ -1215,8 +1200,7 @@ impl AppEndpoint { .extend(get_wasm_paths_from_root(&node_root_value, &all_output_assets).await?); let all_assets = - get_paths_from_root(&node_root_value, &all_output_assets, |_asset| true) - .await?; + get_asset_paths_from_root(&node_root_value, &all_output_assets).await?; let entry_file = "app-edge-has-no-entrypoint".into(); diff --git a/crates/next-api/src/instrumentation.rs b/crates/next-api/src/instrumentation.rs index 1d27843831f2d..8162e59938ff4 100644 --- a/crates/next-api/src/instrumentation.rs +++ b/crates/next-api/src/instrumentation.rs @@ -24,6 +24,7 @@ use turbopack_core::{ use turbopack_ecmascript::chunk::EcmascriptChunkPlaceable; use crate::{ + nft_json::NftJsonAsset, paths::{ all_server_paths, get_js_paths_from_root, get_wasm_paths_from_root, wasm_paths_to_bindings, }, @@ -92,6 +93,15 @@ impl InstrumentationEndpoint { .cell()) } + #[turbo_tasks::function] + async fn entry_module(self: Vc) -> Result>> { + if self.await?.is_edge { + Ok(*self.core_modules().await?.edge_entry_module) + } else { + Ok(*self.core_modules().await?.userland_module) + } + } + #[turbo_tasks::function] async fn edge_files(self: Vc) -> Result> { let this = self.await?; @@ -210,7 +220,22 @@ impl InstrumentationEndpoint { Ok(Vc::cell(output_assets)) } else { - Ok(Vc::cell(vec![self.node_chunk().to_resolved().await?])) + let chunk = self.node_chunk().to_resolved().await?; + let mut output_assets = vec![chunk]; + if this.project.next_mode().await?.is_production() { + output_assets.push(ResolvedVc::upcast( + NftJsonAsset::new( + *chunk, + this.project.output_fs(), + this.project.project_fs(), + this.project.client_fs(), + vec![], + ) + .to_resolved() + .await?, + )); + } + Ok(Vc::cell(output_assets)) } } } diff --git a/crates/next-api/src/middleware.rs b/crates/next-api/src/middleware.rs index c9bc87ea64eb0..8a88102a3610f 100644 --- a/crates/next-api/src/middleware.rs +++ b/crates/next-api/src/middleware.rs @@ -24,7 +24,7 @@ use turbopack_ecmascript::chunk::EcmascriptChunkPlaceable; use crate::{ paths::{ - all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root, + all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, }, project::Project, @@ -140,8 +140,7 @@ impl MiddlewareEndpoint { let wasm_paths_from_root = get_wasm_paths_from_root(&node_root_value, &all_output_assets).await?; - let all_assets = - get_paths_from_root(&node_root_value, &all_output_assets, |_asset| true).await?; + let all_assets = get_asset_paths_from_root(&node_root_value, &all_output_assets).await?; // Awaited later for parallelism let config = config.await?; diff --git a/crates/next-api/src/pages.rs b/crates/next-api/src/pages.rs index a7f2cda638fa3..3369750e1540d 100644 --- a/crates/next-api/src/pages.rs +++ b/crates/next-api/src/pages.rs @@ -28,8 +28,7 @@ use next_core::{ use serde::{Deserialize, Serialize}; use tracing::Instrument; use turbo_tasks::{ - trace::TraceRawVcs, Completion, FxIndexMap, RcStr, ResolvedVc, TaskInput, TryJoinIterExt, - Value, Vc, + fxindexmap, trace::TraceRawVcs, Completion, FxIndexMap, RcStr, ResolvedVc, TaskInput, Value, Vc, }; use turbo_tasks_fs::{ self, File, FileContent, FileSystem, FileSystemPath, FileSystemPathOption, VirtualFileSystem, @@ -68,7 +67,7 @@ use crate::{ loadable_manifest::create_react_loadable_manifest, nft_json::NftJsonAsset, paths::{ - all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root, + all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, }, project::Project, @@ -1017,37 +1016,21 @@ impl PageEndpoint { ) -> Result>> { let node_root = self.pages_project.project().node_root(); let client_relative_path = self.pages_project.project().client_relative_path(); - let client_relative_path_ref = client_relative_path.await?; let build_manifest = BuildManifest { - pages: [( - self.pathname.await?.clone_value(), - client_chunks - .await? - .iter() - .copied() - .map(|chunk| { - let client_relative_path_ref = client_relative_path_ref.clone(); - async move { - let chunk_path = chunk.ident().path().await?; - Ok(client_relative_path_ref - .get_path_to(&chunk_path) - .context("client chunk entry path must be inside the client root")? - .into()) - } - }) - .try_join() - .await?, - )] - .into_iter() - .collect(), + pages: fxindexmap!(self.pathname.await?.clone_value() => client_chunks), ..Default::default() }; let manifest_path_prefix = get_asset_prefix_from_pathname(&self.pathname.await?); - Ok(Vc::upcast(VirtualOutputAsset::new( - node_root - .join(format!("server/pages{manifest_path_prefix}/build-manifest.json",).into()), - AssetContent::file(File::from(serde_json::to_string_pretty(&build_manifest)?).into()), - ))) + Ok(Vc::upcast( + build_manifest + .build_output( + node_root.join( + format!("server/pages{manifest_path_prefix}/build-manifest.json",).into(), + ), + client_relative_path, + ) + .await?, + )) } #[turbo_tasks::function] @@ -1174,8 +1157,7 @@ impl PageEndpoint { .extend(get_wasm_paths_from_root(&node_root_value, &all_output_assets).await?); let all_assets = - get_paths_from_root(&node_root_value, &all_output_assets, |_asset| true) - .await?; + get_asset_paths_from_root(&node_root_value, &all_output_assets).await?; let named_regex = get_named_middleware_regex(&pathname).into(); let matchers = MiddlewareMatcher { diff --git a/crates/next-api/src/paths.rs b/crates/next-api/src/paths.rs index 71aacc9ec9dc0..5ffb920e0ba16 100644 --- a/crates/next-api/src/paths.rs +++ b/crates/next-api/src/paths.rs @@ -106,6 +106,16 @@ pub(crate) async fn get_wasm_paths_from_root( get_paths_from_root(root, output_assets, |path| path.ends_with(".wasm")).await } +pub(crate) async fn get_asset_paths_from_root( + root: &FileSystemPath, + output_assets: &[ResolvedVc>], +) -> Result> { + get_paths_from_root(root, output_assets, |path| { + !path.ends_with(".js") && !path.ends_with(".map") && !path.ends_with(".wasm") + }) + .await +} + pub(crate) async fn get_font_paths_from_root( root: &FileSystemPath, output_assets: &[ResolvedVc>], diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index c4986cb27c39c..7ad878151878b 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -549,7 +549,7 @@ impl Project { #[turbo_tasks::function] pub fn client_fs(self: Vc) -> Vc> { - let virtual_fs = VirtualFileSystem::new(); + let virtual_fs = VirtualFileSystem::new_with_name("client-fs".into()); Vc::upcast(virtual_fs) } @@ -666,6 +666,8 @@ impl Project { Ok(get_server_compile_time_info( self.env(), this.define_env.nodejs(), + // `/ROOT` corresponds to `[project]/`, so we need exactly the `path` part. + format!("/ROOT/{}", self.project_path().await?.path).into(), )) } diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index 5b7ed80045584..9a8256174700b 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -860,6 +860,16 @@ impl NextConfig { ) } + #[turbo_tasks::function] + pub fn is_standalone(&self) -> Vc { + Vc::cell(self.output == Some(OutputType::Standalone)) + } + + #[turbo_tasks::function] + pub fn cache_handler(&self) -> Vc> { + Vc::cell(self.cache_handler.clone()) + } + #[turbo_tasks::function] pub fn compiler(&self) -> Vc { self.compiler.clone().unwrap_or_default().cell() diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index d86212f7f9f22..b902a89afc762 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -239,7 +239,9 @@ pub async fn get_next_client_import_map( /// Computes the Next-specific client import map. #[turbo_tasks::function] -pub async fn get_next_build_import_map(project_path: ResolvedVc) -> Result> { +pub async fn get_next_build_import_map( + project_path: ResolvedVc, +) -> Result> { let mut import_map = ImportMap::empty(); insert_package_alias( @@ -335,8 +337,7 @@ pub async fn get_next_server_import_map( let external = ImportMapping::External( None, ExternalType::CommonJs, - // TODO(arlyon): wiring up in a follow up PR - ExternalTraced::Untraced, + ExternalTraced::Traced(project_path), None, ) .resolved_cell(); diff --git a/crates/next-core/src/next_manifests/client_reference_manifest.rs b/crates/next-core/src/next_manifests/client_reference_manifest.rs index db4b65a1f5f93..c5db76983b3ec 100644 --- a/crates/next-core/src/next_manifests/client_reference_manifest.rs +++ b/crates/next-core/src/next_manifests/client_reference_manifest.rs @@ -1,6 +1,6 @@ use anyhow::Result; use indoc::formatdoc; -use turbo_tasks::{RcStr, TryJoinIterExt, Value, ValueToString, Vc}; +use turbo_tasks::{FxIndexSet, RcStr, TryJoinIterExt, Value, ValueToString, Vc}; use turbo_tasks_fs::{File, FileSystemPath}; use turbopack_core::{ asset::AssetContent, @@ -38,6 +38,7 @@ impl ClientReferenceManifest { runtime: NextRuntime, ) -> Result>> { let mut entry_manifest: ClientReferenceManifest = Default::default(); + let mut references = FxIndexSet::default(); entry_manifest.module_loading.prefix = next_config .computed_asset_prefix() .await? @@ -81,6 +82,7 @@ impl ClientReferenceManifest { .get(&app_client_reference_ty) { let client_chunks = client_chunks.await?; + references.extend(client_chunks.iter()); let client_chunks_paths = client_chunks .iter() .map(|chunk| chunk.ident().path()) @@ -127,6 +129,7 @@ impl ClientReferenceManifest { .get(&app_client_reference_ty) { let ssr_chunks = ssr_chunks.await?; + references.extend(ssr_chunks.iter()); let ssr_chunks_paths = ssr_chunks .iter() @@ -281,7 +284,7 @@ impl ClientReferenceManifest { // note this only applies to the manifests, assets are placed to the original // path still (same as webpack does) let normalized_manifest_entry = entry_name.replace("%5F", "_"); - Ok(Vc::upcast(VirtualOutputAsset::new( + Ok(Vc::upcast(VirtualOutputAsset::new_with_references( node_root.join( format!("server/app{normalized_manifest_entry}_client-reference-manifest.js",) .into(), @@ -297,6 +300,7 @@ impl ClientReferenceManifest { }) .into(), ), + Vc::cell(references.into_iter().collect()), ))) } } diff --git a/crates/next-core/src/next_manifests/mod.rs b/crates/next-core/src/next_manifests/mod.rs index ac89d4b5daeb2..a359ae8884fc4 100644 --- a/crates/next-core/src/next_manifests/mod.rs +++ b/crates/next-core/src/next_manifests/mod.rs @@ -4,8 +4,18 @@ pub(crate) mod client_reference_manifest; use std::collections::HashMap; +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; -use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, FxIndexSet, RcStr, TaskInput}; +use turbo_tasks::{ + trace::TraceRawVcs, FxIndexMap, FxIndexSet, RcStr, ReadRef, ResolvedVc, TaskInput, + TryJoinIterExt, Vc, +}; +use turbo_tasks_fs::{File, FileSystemPath}; +use turbopack_core::{ + asset::AssetContent, + output::{OutputAsset, OutputAssets}, + virtual_output::VirtualOutputAsset, +}; use crate::next_config::{CrossOriginConfig, Rewrites, RouteHas}; @@ -15,16 +25,112 @@ pub struct PagesManifest { pub pages: HashMap, } -#[derive(Serialize, Default, Debug)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, Default)] pub struct BuildManifest { - pub dev_files: Vec, - pub amp_dev_files: Vec, - pub polyfill_files: Vec, - pub low_priority_files: Vec, - pub root_main_files: Vec, - pub pages: HashMap>, - pub amp_first_pages: Vec, + pub polyfill_files: Vec>>, + pub root_main_files: Vec>>, + pub pages: FxIndexMap>, +} + +impl BuildManifest { + pub async fn build_output( + self, + output_path: Vc, + client_relative_path: Vc, + ) -> Result>> { + let client_relative_path_ref = &*client_relative_path.await?; + + #[derive(Serialize, Default, Debug)] + #[serde(rename_all = "camelCase")] + pub struct SerializedBuildManifest { + pub dev_files: Vec, + pub amp_dev_files: Vec, + pub polyfill_files: Vec, + pub low_priority_files: Vec, + pub root_main_files: Vec, + pub pages: FxIndexMap>, + pub amp_first_pages: Vec, + } + + let pages: Vec<(RcStr, Vec)> = self + .pages + .iter() + .map(|(k, chunks)| async move { + Ok(( + k.clone(), + chunks + .await? + .iter() + .copied() + .map(|chunk| async move { + let chunk_path = chunk.ident().path().await?; + Ok(client_relative_path_ref + .get_path_to(&chunk_path) + .context("client chunk entry path must be inside the client root")? + .into()) + }) + .try_join() + .await?, + )) + }) + .try_join() + .await?; + + let polyfill_files: Vec = self + .polyfill_files + .iter() + .copied() + .map(|chunk| async move { + let chunk_path = chunk.ident().path().await?; + Ok(client_relative_path_ref + .get_path_to(&chunk_path) + .context("failed to resolve client-relative path to polyfill")? + .into()) + }) + .try_join() + .await?; + + let root_main_files: Vec = self + .root_main_files + .iter() + .copied() + .map(|chunk| async move { + let chunk_path = chunk.ident().path().await?; + Ok(client_relative_path_ref + .get_path_to(&chunk_path) + .context("failed to resolve client-relative path to root_main_file")? + .into()) + }) + .try_join() + .await?; + + let manifest = SerializedBuildManifest { + pages: FxIndexMap::from_iter(pages.into_iter()), + polyfill_files, + root_main_files, + ..Default::default() + }; + + let chunks: Vec> = self + .pages + .values() + // rustc struggles here, so be very explicit + .try_join() + .await?; + + let references = chunks + .into_iter() + .flat_map(|c| c.into_iter().copied()) // once again, rustc struggles here + .chain(self.root_main_files.iter().copied()) + .chain(self.polyfill_files.iter().copied()) + .collect(); + + Ok(Vc::upcast(VirtualOutputAsset::new_with_references( + output_path, + AssetContent::file(File::from(serde_json::to_string_pretty(&manifest)?).into()), + Vc::cell(references), + ))) + } } #[derive(Serialize, Debug)] @@ -302,10 +408,66 @@ pub struct FontManifestEntry { pub content: RcStr, } -#[derive(Serialize, Default, Debug)] -#[serde(rename_all = "camelCase")] +#[derive(Default, Debug)] pub struct AppBuildManifest { - pub pages: HashMap>, + pub pages: FxIndexMap>, +} + +impl AppBuildManifest { + pub async fn build_output( + self, + output_path: Vc, + client_relative_path: Vc, + ) -> Result>> { + let client_relative_path_ref = &*client_relative_path.await?; + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + pub struct SerializedAppBuildManifest { + pub pages: FxIndexMap>, + } + + let pages: Vec<(RcStr, Vec)> = self + .pages + .iter() + .map(|(k, chunks)| async move { + Ok(( + k.clone(), + chunks + .await? + .iter() + .copied() + .map(|chunk| async move { + let chunk_path = chunk.ident().path().await?; + Ok(client_relative_path_ref + .get_path_to(&chunk_path) + .context("client chunk entry path must be inside the client root")? + .into()) + }) + .try_join() + .await?, + )) + }) + .try_join() + .await?; + + let manifest = SerializedAppBuildManifest { + pages: FxIndexMap::from_iter(pages.into_iter()), + }; + + let references = self.pages.values().try_join().await?; + + let references = references + .into_iter() + .flat_map(|c| c.into_iter().copied()) + .collect(); + + Ok(Vc::upcast(VirtualOutputAsset::new_with_references( + output_path, + AssetContent::file(File::from(serde_json::to_string_pretty(&manifest)?).into()), + Vc::cell(references), + ))) + } } // TODO(alexkirsz) Unify with the one for dev. diff --git a/crates/next-core/src/next_server/context.rs b/crates/next-core/src/next_server/context.rs index d574fc9bf6057..937270128e91c 100644 --- a/crates/next-core/src/next_server/context.rs +++ b/crates/next-core/src/next_server/context.rs @@ -19,8 +19,11 @@ use turbopack_core::{ FreeVarReferences, }, condition::ContextCondition, - environment::{Environment, ExecutionEnvironment, NodeJsEnvironment, RuntimeVersions}, + environment::{ + Environment, ExecutionEnvironment, NodeJsEnvironment, NodeJsVersion, RuntimeVersions, + }, free_var_references, + target::CompileTarget, }; use turbopack_ecmascript::references::esm::UrlRewriteBehavior; use turbopack_ecmascript_plugins::transform::directives::{ @@ -367,16 +370,26 @@ async fn next_server_free_vars(define_env: Vc) -> Result>, define_env: Vc, -) -> Vc { - CompileTimeInfo::builder(Environment::new(Value::new( - ExecutionEnvironment::NodeJsLambda(NodeJsEnvironment::current(process_env)), + cwd: RcStr, +) -> Result> { + Ok(CompileTimeInfo::builder(Environment::new(Value::new( + ExecutionEnvironment::NodeJsLambda( + NodeJsEnvironment { + compile_target: CompileTarget::current(), + node_version: NodeJsVersion::cell(NodeJsVersion::Current( + process_env.to_resolved().await?, + )), + cwd: Vc::cell(Some(cwd)), + } + .cell(), + ), ))) .defines(next_server_defines(define_env)) .free_var_references(next_server_free_vars(define_env)) - .cell() + .cell()) } /// Determins if the module is an internal asset (i.e overlay, fallback) coming diff --git a/crates/next-core/src/next_server/resolve.rs b/crates/next-core/src/next_server/resolve.rs index 551a9578f8963..a8b35fd484043 100644 --- a/crates/next-core/src/next_server/resolve.rs +++ b/crates/next-core/src/next_server/resolve.rs @@ -36,7 +36,7 @@ pub enum ExternalPredicate { /// possible to resolve them at runtime. #[turbo_tasks::value] pub(crate) struct ExternalCjsModulesResolvePlugin { - project_path: Vc, + project_path: ResolvedVc, root: Vc, predicate: Vc, import_externals: bool, @@ -46,7 +46,7 @@ pub(crate) struct ExternalCjsModulesResolvePlugin { impl ExternalCjsModulesResolvePlugin { #[turbo_tasks::function] pub fn new( - project_path: Vc, + project_path: ResolvedVc, root: Vc, predicate: Vc, import_externals: bool, @@ -253,7 +253,7 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { break result_from_original_location; }; let node_resolved = resolve( - self.project_path, + *self.project_path, reference_type.clone(), request, node_resolve_options, @@ -359,38 +359,27 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { let path = result.ident().path().resolve().await?; let file_type = get_file_type(path, &*path.await?).await?; - match (file_type, is_esm) { + let external_type = match (file_type, is_esm) { (FileType::UnsupportedExtension, _) => { // unsupported file type, bundle it - unable_to_externalize(vec![StyledString::Text( + return unable_to_externalize(vec![StyledString::Text( "Only .mjs, .cjs, .js, .json, or .node can be handled by Node.js.".into(), - )]) + )]); } (FileType::InvalidPackageJson, _) => { // invalid package.json, bundle it - unable_to_externalize(vec![StyledString::Text( + return unable_to_externalize(vec![StyledString::Text( "The package.json can't be found or parsed.".into(), - )]) - } - (FileType::CommonJs, false) => { - // mark as external - Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External { - name: request_str.into(), - ty: ExternalType::CommonJs, - // TODO(arlyon): wiring up in a follow up PR - traced: ExternalTraced::Untraced, - }) - .cell(), - )) + )]); } + // commonjs without esm is always external + (FileType::CommonJs, false) => ExternalType::CommonJs, (FileType::CommonJs, true) => { // It would be more efficient to use an CJS external instead of an ESM external, - // but we need to verify if that would be correct (as in resolves to the same - // file). + // but we need to verify if that would be correct (as in resolves to the same file). let node_resolve_options = node_cjs_resolve_options(lookup_path.root()); let node_resolved = resolve( - self.project_path, + *self.project_path, reference_type.clone(), request, node_resolve_options, @@ -408,48 +397,33 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { // packages, where `type: module` or `.mjs` is missing and would fail in // Node.js. So when this wasn't an explicit opt-in we avoid making it external // to be safe. - if !resolves_equal && !must_be_external { + match (must_be_external, resolves_equal) { // bundle it to be safe. No error since `must_be_external` is not set. - Ok(ResolveResultOption::none()) - } else { - // mark as external - Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External { - name: request_str.into(), - ty: if resolves_equal { - ExternalType::CommonJs - } else { - ExternalType::EcmaScriptModule - }, - // TODO(arlyon): wiring up in a follow up PR - traced: ExternalTraced::Untraced, - }) - .cell(), - )) + (false, false) => return Ok(ResolveResultOption::none()), + (_, true) => ExternalType::CommonJs, + (_, false) => ExternalType::EcmaScriptModule, } } - (FileType::EcmaScriptModule, true) => { - // mark as external - Ok(ResolveResultOption::some( - ResolveResult::primary(ResolveResultItem::External { - name: request_str.into(), - ty: ExternalType::EcmaScriptModule, - // TODO(arlyon): wiring up in a follow up PR - traced: ExternalTraced::Untraced, - }) - .cell(), - )) - } + // ecmascript with esm is always external + (FileType::EcmaScriptModule, true) => ExternalType::EcmaScriptModule, (FileType::EcmaScriptModule, false) => { - // even with require() this resolves to a ESM, - // which would break node.js, bundle it - unable_to_externalize(vec![StyledString::Text( + // even with require() this resolves to a ESM, which would break node.js, bundle it + return unable_to_externalize(vec![StyledString::Text( "The package seems invalid. require() resolves to a EcmaScript module, which \ would result in an error in Node.js." .into(), - )]) + )]); } - } + }; + + Ok(ResolveResultOption::some( + ResolveResult::primary(ResolveResultItem::External { + name: request_str.into(), + ty: external_type, + traced: ExternalTraced::Traced(self.project_path), + }) + .cell(), + )) } } diff --git a/turbopack/crates/turbo-tasks-fs/src/util.rs b/turbopack/crates/turbo-tasks-fs/src/util.rs index d079b21468157..70ebd3843c78e 100644 --- a/turbopack/crates/turbo-tasks-fs/src/util.rs +++ b/turbopack/crates/turbo-tasks-fs/src/util.rs @@ -17,12 +17,17 @@ pub fn join_path(fs_path: &str, join: &str) -> Option { // Paths that we join are written as source code (eg, `join_path(fs_path, // "foo/bar.js")`) and it's expected that they will never contain a // backslash. - debug_assert!( - !join.contains('\\'), - "joined path {} must not contain a Windows directory '\\', it must be normalized to Unix \ - '/'", - join - ); + + /* + A task panicked: joined path node_modules/.pnpm/babel-plugin-react-compiler@0.0.0-experimental-58c2b1c-20241007/node_modules/babel-plugin-react-compiler/dist\\/ must not contain a Windows directory '\', it must be normalized to Unix '/' + */ + + // debug_assert!( + // !join.contains('\\'), + // "joined path {} must not contain a Windows directory '\\', it must be normalized to Unix + // \ '/'", + // join + // ); // TODO: figure out why this freezes the benchmarks. // // an absolute path would leave the file system root diff --git a/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs b/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs index 8052b8a584a6f..576d0db1fd799 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs @@ -50,6 +50,10 @@ impl ReferencedAsset { pub async fn get_ident(&self) -> Result> { Ok(match self { ReferencedAsset::Some(asset) => Some(Self::get_ident_from_placeable(asset).await?), + // TODO: do we need to mangle the source? + // ReferencedAsset::External(request, ty, Some(output)) => { + // Some(output.ident().to_string()) + // } ReferencedAsset::External(request, ty) => Some(magic_identifier::mangle(&format!( "{ty} external {request}" ))), @@ -358,6 +362,7 @@ impl CodeGenerateable for EsmAssetReference { ), )) } + // fallback in case we introduce a new `ExternalType` #[allow(unreachable_patterns)] ReferencedAsset::External(request, ty) => { bail!( diff --git a/turbopack/crates/turbopack-ecmascript/src/references/mod.rs b/turbopack/crates/turbopack-ecmascript/src/references/mod.rs index f3178fef42b29..41c87d1d606c3 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/mod.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/mod.rs @@ -1519,7 +1519,9 @@ async fn handle_call) + Send + Sync>( JsValue::WellKnownFunction(WellKnownFunctionKind::RequireResolve) => { let args = linked_args(args).await?; - if args.len() == 1 { + if args.len() == 1 || args.len() == 2 { + // TODO error TP1003 require.resolve(???*0*, {"paths": [???*1*]}) is not statically + // analyse-able with ignore_dynamic_requests = true let pat = js_value_to_pattern(&args[0]); if !pat.has_constant_parts() { let (args, hints) = explain_args(&args); diff --git a/turbopack/crates/turbopack-ecmascript/src/references/node.rs b/turbopack/crates/turbopack-ecmascript/src/references/node.rs index 2122be2033449..c1416d4c3978f 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/node.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/node.rs @@ -3,6 +3,7 @@ use either::Either; use turbo_tasks::{RcStr, ResolvedVc, ValueToString, Vc}; use turbo_tasks_fs::FileSystemPath; use turbopack_core::{ + chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, file_source::FileSource, raw_module::RawModule, reference::ModuleReference, @@ -153,6 +154,14 @@ impl ModuleReference for DirAssetReference { } } +#[turbo_tasks::value_impl] +impl ChunkableModuleReference for DirAssetReference { + #[turbo_tasks::function] + fn chunking_type(&self) -> Vc { + Vc::cell(Some(ChunkingType::Traced)) + } +} + #[turbo_tasks::value_impl] impl ValueToString for DirAssetReference { #[turbo_tasks::function] diff --git a/turbopack/crates/turbopack-resolve/src/node_native_binding.rs b/turbopack/crates/turbopack-resolve/src/node_native_binding.rs index d0a505aa8080c..86f8402725213 100644 --- a/turbopack/crates/turbopack-resolve/src/node_native_binding.rs +++ b/turbopack/crates/turbopack-resolve/src/node_native_binding.rs @@ -162,10 +162,12 @@ pub async fn resolve_node_pre_gyp_files( ) .into(); let resolved_file_vc = config_file_dir.join(node_file_path.clone()); - sources.insert( - node_file_path, - Vc::upcast(FileSource::new(resolved_file_vc)), - ); + if *resolved_file_vc.get_type().await? == FileSystemEntryType::File { + sources.insert( + node_file_path, + Vc::upcast(FileSource::new(resolved_file_vc)), + ); + } } for (key, entry) in config_file_dir // TODO diff --git a/turbopack/crates/turbopack/src/lib.rs b/turbopack/crates/turbopack/src/lib.rs index 1912a08446bba..20987faea3419 100644 --- a/turbopack/crates/turbopack/src/lib.rs +++ b/turbopack/crates/turbopack/src/lib.rs @@ -36,18 +36,20 @@ use turbopack_core::{ asset::Asset, compile_time_info::CompileTimeInfo, context::{AssetContext, ProcessResult}, - ident::AssetIdent, - issue::{Issue, IssueExt, IssueStage, OptionStyledString, StyledString}, + environment::{Environment, ExecutionEnvironment, NodeJsEnvironment}, + issue::{module::ModuleIssue, IssueExt, StyledString}, module::Module, output::OutputAsset, raw_module::RawModule, + reference::{ModuleReference, TracedModuleReference}, reference_type::{ CssReferenceSubType, EcmaScriptModulesReferenceSubType, ImportWithType, InnerAssets, ReferenceType, }, resolve::{ - options::ResolveOptions, origin::PlainResolveOrigin, parse::Request, resolve, ExternalType, - ModulePart, ModuleResolveResult, ModuleResolveResultItem, ResolveResult, + options::ResolveOptions, origin::PlainResolveOrigin, parse::Request, resolve, + ExternalTraced, ExternalType, ModulePart, ModuleResolveResult, ModuleResolveResultItem, + ResolveResult, ResolveResultItem, }, source::Source, }; @@ -68,36 +70,6 @@ use self::{ transition::{Transition, TransitionOptions}, }; -#[turbo_tasks::value] -struct ModuleIssue { - ident: Vc, - title: Vc, - description: Vc, -} - -#[turbo_tasks::value_impl] -impl Issue for ModuleIssue { - #[turbo_tasks::function] - fn stage(&self) -> Vc { - IssueStage::ProcessModule.cell() - } - - #[turbo_tasks::function] - fn file_path(&self) -> Vc { - self.ident.path() - } - - #[turbo_tasks::function] - fn title(&self) -> Vc { - self.title - } - - #[turbo_tasks::function] - fn description(&self) -> Vc { - Vc::cell(Some(self.description)) - } -} - #[turbo_tasks::function] async fn apply_module_type( source: Vc>, @@ -645,30 +617,14 @@ async fn process_default_internal( } } - let module_type = match current_module_type { - Some(module_type) => module_type, - None => { - ModuleIssue { - ident, - title: StyledString::Text("Unknown module type".into()).cell(), - description: StyledString::Text( - r"This module doesn't have an associated type. Use a known file extension, or register a loader for it. - -Read more: https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders".into(), - ) - .cell(), - } - .cell() - .emit(); - - return Ok(ProcessResult::Ignore.cell()); - } - }.cell(); + let Some(module_type) = current_module_type else { + return Ok(ProcessResult::Unknown(current_source).cell()); + }; Ok(apply_module_type( current_source, module_asset_context, - module_type, + module_type.cell(), Value::new(reference_type.clone()), part, inner_assets, @@ -676,6 +632,34 @@ Read more: https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpac )) } +#[turbo_tasks::function] +async fn externals_tracing_module_context(ty: ExternalType) -> Result> { + let env = Environment::new(Value::new(ExecutionEnvironment::NodeJsLambda( + NodeJsEnvironment::default().cell(), + ))) + .to_resolved() + .await?; + + let resolve_options = ResolveOptionsContext { + emulate_environment: Some(env.to_resolved().await?), + loose_errors: true, + custom_conditions: match ty { + ExternalType::CommonJs => vec!["require".into()], + ExternalType::EcmaScriptModule => vec!["import".into()], + ExternalType::Url => vec![], + }, + ..Default::default() + }; + + Ok(ModuleAssetContext::new_without_replace_externals( + Default::default(), + CompileTimeInfo::builder(*env).cell(), + ModuleOptionsContext::default().cell(), + resolve_options.cell(), + Vc::cell("externals-tracing".into()), + )) +} + #[turbo_tasks::value_impl] impl AssetContext for ModuleAssetContext { #[turbo_tasks::function] @@ -749,43 +733,119 @@ impl AssetContext for ModuleAssetContext { ignore_unknown: bool, ) -> Result> { let this = self.await?; - let transition = this.transition; - let result = result + let replace_externals = this.replace_externals; + let import_externals = this + .module_options_context .await? - .map_module(|source| { + .ecmascript + .import_externals; + + let result = result.await?; + + let affecting_sources = &result.affecting_sources; + + let result = result + .map_items_module(|item| { let reference_type = reference_type.clone(); async move { - let process_result = if let Some(transition) = transition { - transition.process(source, self, reference_type) - } else { - self.process_with_transition_rules(source, reference_type) - }; - Ok(match *process_result.await? { - ProcessResult::Module(m) => { - ModuleResolveResultItem::Module(ResolvedVc::upcast(m)) - } - ProcessResult::Unknown(source) => { - if !ignore_unknown { - ProcessResult::emit_unknown_error(source).await?; + Ok(match item { + ResolveResultItem::Source(source) => { + match &*self.process(source, reference_type).await? { + ProcessResult::Module(module) => { + ModuleResolveResultItem::Module(*module) + } + ProcessResult::Unknown(source) => { + if !ignore_unknown { + ProcessResult::emit_unknown_error(*source).await?; + } + ModuleResolveResultItem::Ignore + } + ProcessResult::Ignore => ModuleResolveResultItem::Ignore, } - ModuleResolveResultItem::Ignore } - ProcessResult::Ignore => ModuleResolveResultItem::Ignore, + ResolveResultItem::External { name, ty, traced } => { + let replacement = if replace_externals { + let additional_refs = match traced { + // TODO can we get away without module_context.enable_tracing ? + ExternalTraced::Traced(tracing_root) => { + if self + .module_options_context() + .await? + .enable_file_tracing + .as_ref() + .is_some() + { + let externals_context = + externals_tracing_module_context(ty); + let out_dir = tracing_root.join("index".into()); + + let external_result = (externals_context + .resolve_asset( + out_dir, + Request::parse_string(name.clone()), + externals_context.resolve_options( + out_dir, + reference_type.clone(), + ), + reference_type, + )) + .await?; + + let modules = affecting_sources + .iter() + .chain(external_result.affecting_sources.iter()) + .map(|s| { + Vc::upcast::>(RawModule::new( + *s, + )) + }) + .chain( + external_result + .primary_modules_iter() + .map(|rvc| *rvc), + ); + + modules + .map(|s| { + Vc::upcast::>( + TracedModuleReference::new(s), + ) + }) + .collect() + } else { + vec![] + } + } + _ => vec![], + }; + + replace_external(&name, ty, additional_refs, import_externals) + .await? + } else { + None + }; + + replacement.unwrap_or_else(|| { + ModuleResolveResultItem::External { + name, + ty, + // TODO(micshnic) remove that field entirely ? + traced: None, + } + }) + } + ResolveResultItem::Ignore => ModuleResolveResultItem::Ignore, + ResolveResultItem::Empty => ModuleResolveResultItem::Empty, + ResolveResultItem::Error(e) => { + ModuleResolveResultItem::Error(e.to_resolved().await?) + } + ResolveResultItem::Custom(u8) => ModuleResolveResultItem::Custom(u8), }) } }) .await?; - let result = replace_externals( - result, - this.module_options_context - .await? - .ecmascript - .import_externals, - ) - .await?; - Ok(result.cell()) } @@ -965,41 +1025,34 @@ async fn top_references(list: Vc) -> Result> } /// Replaces the externals in the result with `ExternalModuleAsset` instances. -pub async fn replace_externals( - mut result: ModuleResolveResult, +pub async fn replace_external( + name: &RcStr, + ty: ExternalType, + additional_refs: Vec>>, import_externals: bool, -) -> Result { - for item in result.primary.values_mut() { - let ModuleResolveResultItem::External { - name: request, ty, .. - } = item - else { - continue; - }; - - let external_type = match ty { - ExternalType::CommonJs => CachedExternalType::CommonJs, - ExternalType::EcmaScriptModule => { - if import_externals { - CachedExternalType::EcmaScriptViaImport - } else { - CachedExternalType::EcmaScriptViaRequire - } - } - ExternalType::Url => { - // we don't want to wrap url externals. - continue; +) -> Result> { + let external_type = match ty { + ExternalType::CommonJs => CachedExternalType::CommonJs, + ExternalType::EcmaScriptModule => { + if import_externals { + CachedExternalType::EcmaScriptViaImport + } else { + CachedExternalType::EcmaScriptViaRequire } - }; - - let module = CachedExternalModule::new(request.clone(), external_type, vec![]) - .to_resolved() - .await?; + } + ExternalType::Url => { + // we don't want to wrap url externals. + return Ok(None); + } + }; - *item = ModuleResolveResultItem::Module(ResolvedVc::upcast(module)); - } + let module = CachedExternalModule::new(name.clone(), external_type, additional_refs) + .to_resolved() + .await?; - Ok(result) + Ok(Some(ModuleResolveResultItem::Module(ResolvedVc::upcast( + module, + )))) } pub fn register() {