From 393b2adb0c80b2ab752efa125dc6215e6f6fbb75 Mon Sep 17 00:00:00 2001 From: Alex Kirszenberg Date: Tue, 18 Apr 2023 14:51:22 +0200 Subject: [PATCH] Chunking Context Refactor pt. 3: Address PR comments from pt. 2 (vercel/turbo#4601) See https://github.com/vercel/turbo/pull/4546 for the first attempt. This causes Next.js tests to fail. ~~Currently figuring out what's wrong.~~ This surfaced an existing issue in our chunk optimization logic, where chunks of different chunking contexts (SSR and RSC) would be merged together, leading to invalid module ids. link WEB-891 --- .../src/chunk/containment_tree.rs | 220 ++++++++++++ crates/turbopack-core/src/chunk/mod.rs | 89 +---- crates/turbopack-core/src/chunk/optimize.rs | 314 +----------------- crates/turbopack-core/src/introspect/asset.rs | 9 +- crates/turbopack-core/src/reference/mod.rs | 3 +- crates/turbopack-css/src/chunk/mod.rs | 40 ++- crates/turbopack-dev/Cargo.toml | 1 + crates/turbopack-dev/src/chunking_context.rs | 128 +++---- crates/turbopack-dev/src/css/mod.rs | 1 + .../src/css}/optimize.rs | 79 ++--- crates/turbopack-dev/src/ecmascript/chunk.rs | 10 +- crates/turbopack-dev/src/ecmascript/mod.rs | 1 + .../src/ecmascript}/optimize.rs | 108 +++--- crates/turbopack-dev/src/lib.rs | 2 + crates/turbopack-ecmascript/src/chunk/mod.rs | 49 +-- ...absolute-uri-import_input_index_ec11e8.js} | 8 +- ...absolute-uri-import_input_index_fa9a30.js} | 4 +- ...lute-uri-import_input_index_fa9a30.js.map} | 0 ...ts_snapshot_css_css_input_index_37a138.js} | 4 +- ...napshot_css_css_input_index_37a138.js.map} | 0 ...ts_snapshot_css_css_input_index_469b46.js} | 10 +- 21 files changed, 474 insertions(+), 606 deletions(-) create mode 100644 crates/turbopack-core/src/chunk/containment_tree.rs create mode 100644 crates/turbopack-dev/src/css/mod.rs rename crates/{turbopack-css/src/chunk => turbopack-dev/src/css}/optimize.rs (52%) rename crates/{turbopack-ecmascript/src/chunk => turbopack-dev/src/ecmascript}/optimize.rs (83%) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js} (77%) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js} (99%) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map} (100%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js => crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js} (97%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js.map => crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js.map} (100%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js => crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js} (83%) diff --git a/crates/turbopack-core/src/chunk/containment_tree.rs b/crates/turbopack-core/src/chunk/containment_tree.rs new file mode 100644 index 0000000000000..2960716365854 --- /dev/null +++ b/crates/turbopack-core/src/chunk/containment_tree.rs @@ -0,0 +1,220 @@ +use std::{cell::RefCell, mem::take, rc::Rc}; + +use anyhow::Result; +use indexmap::{IndexMap, IndexSet}; +use turbo_tasks::TryJoinIterExt; +use turbo_tasks_fs::{FileSystemPathOptionVc, FileSystemPathVc}; + +#[derive(Default)] +pub struct ContainmentTree { + pub path: Option, + pub chunks: Option>, + pub children: Vec>, +} + +impl ContainmentTree { + pub async fn build( + chunks: impl Iterator, + ) -> Result> { + async fn resolve( + chunks: impl Iterator, + ) -> Result, T)>> { + chunks + .map(|(common_parent, chunk)| async move { + let common_parent = if let Some(common_parent) = *common_parent.await? { + Some(common_parent.resolve().await?) + } else { + None + }; + Ok((common_parent, chunk)) + }) + .try_join() + .await + } + + async fn expand_common_parents( + common_parents: &mut IndexSet, + ) -> Result<()> { + // This is mutated while iterating, so we need to loop with index + let mut i = 0; + while i < common_parents.len() { + let current = common_parents[i]; + let parent = current.parent().resolve().await?; + common_parents.insert(parent); + i += 1; + } + Ok(()) + } + + async fn compute_relationships( + common_parents: &IndexSet, + ) -> Result, FileSystemPathVc)>> { + common_parents + .iter() + .map(|&key| { + let common_parents = &common_parents; + async move { + let mut current = key; + loop { + let parent = current.parent().resolve().await?; + if parent == current { + return Ok((None, key)); + } + if common_parents.contains(&parent) { + // Can't insert here into the parent tree, since we want the order + // of children to be deterministic + return Ok((Some(parent), key)); + } + current = parent; + } + } + }) + .try_join() + .await + } + + // Temp structure which uses Rc and RefCell + struct Node { + path: FileSystemPathVc, + chunks: Vec, + children: Vec>>>, + } + + fn create_node_tree( + common_parents: IndexSet, + ) -> IndexMap>>> { + let mut trees = IndexMap::>>>::new(); + for common_parent in common_parents { + trees.insert( + common_parent, + Rc::new(RefCell::new(Node { + path: common_parent, + chunks: Vec::new(), + children: Vec::new(), + })), + ); + } + trees + } + + fn add_chunks_to_tree( + trees: &mut IndexMap>>>, + chunks: Vec<(Option, T)>, + ) -> Vec { + let mut orphan_chunks = Vec::new(); + for (common_parent, chunk) in chunks { + if let Some(common_parent) = common_parent { + trees + .get_mut(&common_parent) + .unwrap() + .borrow_mut() + .chunks + .push(chunk); + } else { + orphan_chunks.push(chunk); + } + } + orphan_chunks + } + + fn treeify( + relationships: Vec<(Option, FileSystemPathVc)>, + trees: &mut IndexMap>>>, + ) -> Vec>>> { + relationships + .into_iter() + .flat_map(|(parent, key)| { + let tree = trees.get(&key).unwrap().clone(); + if let Some(parent) = parent { + trees.get(&parent).unwrap().borrow_mut().children.push(tree); + None + } else { + Some(tree) + } + }) + .collect::>() + } + + fn skip_unnessary_nodes(trees: &mut IndexMap>>>) { + for tree in trees.values_mut() { + let mut tree = tree.borrow_mut(); + if tree.chunks.is_empty() && tree.children.len() == 1 { + let child = tree.children.pop().unwrap(); + let mut child = child.borrow_mut(); + tree.path = child.path; + tree.chunks.append(&mut child.chunks); + tree.children.append(&mut child.children); + } + } + } + + // Convert function to the real data structure + fn node_to_common_parent_tree(node: Rc>>) -> ContainmentTree { + let mut node = node.borrow_mut(); + let children = take(&mut node.children) + .into_iter() + .map(node_to_common_parent_tree) + .collect(); + // TODO keyed cell: this would benefit from keying the cell by node.path + let chunks = Some(take(&mut node.chunks)); + ContainmentTree { + path: Some(node.path), + chunks, + children, + } + } + + fn convert_into_common_parent_tree( + roots: Vec>>>, + orphan_chunks: Vec, + ) -> Vec> { + roots + .into_iter() + .map(node_to_common_parent_tree) + .chain(orphan_chunks.into_iter().map(|chunk| ContainmentTree { + path: None, + chunks: Some(vec![chunk]), + children: Vec::new(), + })) + .collect::>() + } + + // resolve all paths + let chunks = resolve(chunks).await?; + // compute all unique common_parents + let mut common_parents = chunks + .iter() + .filter_map(|&(path, _)| path) + .collect::>(); + // expand all common parents to include all their parents + expand_common_parents(&mut common_parents).await?; + // compute parent -> child relationships between common_parents + let relationships = compute_relationships(&common_parents).await?; + + // create the tree nodes + let mut trees = create_node_tree(common_parents); + + // add chunks to nodes + let orphan_chunks = add_chunks_to_tree(&mut trees, chunks); + + // nest each tree by relationship, compute the roots + let roots = treeify(relationships, &mut trees); + + // optimize tree by removing unnecessary nodes + skip_unnessary_nodes(&mut trees); + + // do conversion + let roots = convert_into_common_parent_tree(roots, orphan_chunks); + + // top level nesting + Ok(if roots.len() == 1 { + roots.into_iter().next().unwrap() + } else { + ContainmentTree { + path: None, + chunks: None, + children: roots, + } + }) + } +} diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 82adad54306aa..607512451355c 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -1,6 +1,7 @@ pub mod availability_info; pub mod available_assets; pub(crate) mod chunking_context; +pub(crate) mod containment_tree; pub(crate) mod evaluate; pub mod optimize; @@ -17,7 +18,7 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::{ debug::ValueDebugFormat, graph::{GraphTraversal, GraphTraversalResult, ReverseTopological, Visit, VisitControlFlow}, - primitives::{BoolVc, StringVc}, + primitives::StringVc, trace::TraceRawVcs, TryJoinIterExt, Value, ValueToString, ValueToStringVc, }; @@ -28,7 +29,6 @@ use self::availability_info::AvailabilityInfo; pub use self::{ chunking_context::{ChunkingContext, ChunkingContextVc}, evaluate::{EvaluatableAsset, EvaluatableAssetVc, EvaluatableAssets, EvaluatableAssetsVc}, - optimize::optimize, }; use crate::{ asset::{Asset, AssetVc, AssetsVc}, @@ -98,13 +98,17 @@ pub trait ChunkableAsset: Asset { #[turbo_tasks::value(transparent)] pub struct Chunks(Vec); +#[turbo_tasks::value_impl] +impl ChunksVc { + /// Creates a new empty [ChunksVc]. + #[turbo_tasks::function] + pub fn empty() -> ChunksVc { + Self::cell(vec![]) + } +} + /// A chunk is one type of asset. /// It usually contains multiple chunk items. -/// There is an optional trait [ParallelChunkReference] that -/// [AssetReference]s from a [Chunk] can implement. -/// If they implement that and [ParallelChunkReference::is_loaded_in_parallel] -/// returns true, all referenced assets (if they are [Chunk]s) are placed in the -/// same chunk group. #[turbo_tasks::value_trait] pub trait Chunk: Asset { fn chunking_context(&self) -> ChunkingContextVc; @@ -115,6 +119,11 @@ pub trait Chunk: Asset { fn path(&self) -> FileSystemPathVc { self.ident().path() } + /// Returns a list of chunks that should be loaded in parallel to this + /// chunk. + fn parallel_chunks(&self) -> ChunksVc { + ChunksVc::empty() + } } /// Aggregated information about a chunk content that can be used by the runtime @@ -132,12 +141,6 @@ pub trait OutputChunk: Asset { fn runtime_info(&self) -> OutputChunkRuntimeInfoVc; } -/// see [Chunk] for explanation -#[turbo_tasks::value_trait] -pub trait ParallelChunkReference: AssetReference + ValueToString { - fn is_loaded_in_parallel(&self) -> BoolVc; -} - /// Specifies how a chunk interacts with other chunks when building a chunk /// group #[derive( @@ -182,59 +185,6 @@ pub trait ChunkableAssetReference: AssetReference + ValueToString { } } -/// A reference to a [Chunk]. Can be loaded in parallel, see [Chunk]. -#[turbo_tasks::value] -pub struct ChunkReference { - chunk: ChunkVc, - parallel: bool, -} - -#[turbo_tasks::value_impl] -impl ChunkReferenceVc { - #[turbo_tasks::function] - pub fn new(chunk: ChunkVc) -> Self { - Self::cell(ChunkReference { - chunk, - parallel: false, - }) - } - - #[turbo_tasks::function] - pub fn new_parallel(chunk: ChunkVc) -> Self { - Self::cell(ChunkReference { - chunk, - parallel: true, - }) - } -} - -#[turbo_tasks::value_impl] -impl AssetReference for ChunkReference { - #[turbo_tasks::function] - fn resolve_reference(&self) -> ResolveResultVc { - ResolveResult::asset(self.chunk.into()).into() - } -} - -#[turbo_tasks::value_impl] -impl ValueToString for ChunkReference { - #[turbo_tasks::function] - async fn to_string(&self) -> Result { - Ok(StringVc::cell(format!( - "chunk {}", - self.chunk.ident().to_string().await? - ))) - } -} - -#[turbo_tasks::value_impl] -impl ParallelChunkReference for ChunkReference { - #[turbo_tasks::function] - fn is_loaded_in_parallel(&self) -> BoolVc { - BoolVc::cell(self.parallel) - } -} - /// A reference to multiple chunks from a [ChunkGroup] #[turbo_tasks::value] pub struct ChunkGroupReference { @@ -419,12 +369,7 @@ where )); } ChunkingType::IsolatedParallel => { - let chunk = chunkable_asset.as_chunk( - context.chunking_context, - Value::new(AvailabilityInfo::Root { - current_availability_root: chunkable_asset.into(), - }), - ); + let chunk = chunkable_asset.as_root_chunk(context.chunking_context); graph_nodes.push(( Some((asset, chunking_type)), ChunkContentGraphNode::Chunk(chunk), diff --git a/crates/turbopack-core/src/chunk/optimize.rs b/crates/turbopack-core/src/chunk/optimize.rs index de3019ab95ea7..db0455815615f 100644 --- a/crates/turbopack-core/src/chunk/optimize.rs +++ b/crates/turbopack-core/src/chunk/optimize.rs @@ -3,310 +3,30 @@ //! Usually chunks are optimized by limiting their total count, restricting //! their size and eliminating duplicates between them. -use std::{cell::RefCell, mem::take, rc::Rc}; - use anyhow::Result; -use indexmap::{IndexMap, IndexSet}; -use turbo_tasks::TryJoinIterExt; -use turbo_tasks_fs::{FileSystemPathOptionVc, FileSystemPathVc}; - -use super::{ChunkVc, ChunksVc}; -use crate::{ - asset::{Asset, AssetVc}, - chunk::Chunk, -}; - -/// A functor to optimize a set of chunks. -#[turbo_tasks::value_trait] -pub trait ChunkOptimizer { - fn optimize(&self, chunks: ChunksVc) -> ChunksVc; -} - -/// Trait to mark a chunk as optimizable. -#[turbo_tasks::value_trait] -pub trait OptimizableChunk: Chunk + Asset { - fn get_optimizer(&self) -> ChunkOptimizerVc; -} - -/// Optimize all chunks that implement [OptimizableChunk]. -#[turbo_tasks::function] -pub async fn optimize(chunks: ChunksVc) -> Result { - let chunks = chunks.await?; - let mut by_optimizer = IndexMap::<_, Vec<_>>::new(); - for (chunk, optimizer) in chunks - .iter() - .map(|chunk| async move { - Ok(( - chunk, - if let Some(optimizable) = OptimizableChunkVc::resolve_from(chunk).await? { - Some(optimizable.get_optimizer().resolve().await?) - } else { - None - }, - )) - }) - .try_join() - .await? - { - by_optimizer.entry(optimizer).or_default().push(*chunk); - } - - let optimized_chunks = by_optimizer - .into_iter() - .map(|(optimizer, chunks)| async move { - // TODO keyed cell: this would benefit from keying the cell by optimizer - let chunks = ChunksVc::cell(chunks); - Ok(if let Some(optimizer) = optimizer { - optimizer.optimize(chunks).await? - } else { - chunks.await? - }) - }) - .try_join() - .await?; - let optimized_chunks = optimized_chunks - .iter() - .flat_map(|c| c.iter().copied()) - .collect(); - Ok(ChunksVc::cell(optimized_chunks)) -} - -#[derive(Default)] -pub struct ContainmentTree { - pub path: Option, - pub chunks: Option, - pub children: Vec, -} - -#[turbo_tasks::function] -fn orphan_chunk(chunk: ChunkVc) -> ChunksVc { - ChunksVc::cell(vec![chunk]) -} - -impl ContainmentTree { - async fn build( - chunks: impl Iterator, - ) -> Result { - async fn resolve( - chunks: impl Iterator, - ) -> Result, ChunkVc)>> { - chunks - .map(|(common_parent, chunk)| async move { - let common_parent = if let Some(common_parent) = *common_parent.await? { - Some(common_parent.resolve().await?) - } else { - None - }; - Ok((common_parent, chunk)) - }) - .try_join() - .await - } - - async fn expand_common_parents( - common_parents: &mut IndexSet, - ) -> Result<()> { - // This is mutated while iterating, so we need to loop with index - let mut i = 0; - while i < common_parents.len() { - let current = common_parents[i]; - let parent = current.parent().resolve().await?; - common_parents.insert(parent); - i += 1; - } - Ok(()) - } - - async fn compute_relationships( - common_parents: &IndexSet, - ) -> Result, FileSystemPathVc)>> { - common_parents - .iter() - .map(|&key| { - let common_parents = &common_parents; - async move { - let mut current = key; - loop { - let parent = current.parent().resolve().await?; - if parent == current { - return Ok((None, key)); - } - if common_parents.contains(&parent) { - // Can't insert here into the parent tree, since we want the order - // of children to be deterministic - return Ok((Some(parent), key)); - } - current = parent; - } - } - }) - .try_join() - .await - } - - // Temp structure which uses Rc and RefCell - struct Node { - path: FileSystemPathVc, - chunks: Vec, - children: Vec>>, - } - - fn create_node_tree( - common_parents: IndexSet, - ) -> IndexMap>> { - let mut trees = IndexMap::>>::new(); - for common_parent in common_parents { - trees.insert( - common_parent, - Rc::new(RefCell::new(Node { - path: common_parent, - chunks: Vec::new(), - children: Vec::new(), - })), - ); - } - trees - } - - fn add_chunks_to_tree( - trees: &mut IndexMap>>, - chunks: Vec<(Option, ChunkVc)>, - ) -> Vec { - let mut orphan_chunks = Vec::new(); - for (common_parent, chunk) in chunks { - if let Some(common_parent) = common_parent { - trees - .get_mut(&common_parent) - .unwrap() - .borrow_mut() - .chunks - .push(chunk); - } else { - orphan_chunks.push(chunk); - } - } - orphan_chunks - } - - fn treeify( - relationships: Vec<(Option, FileSystemPathVc)>, - trees: &mut IndexMap>>, - ) -> Vec>> { - relationships - .into_iter() - .flat_map(|(parent, key)| { - let tree = trees.get(&key).unwrap().clone(); - if let Some(parent) = parent { - trees.get(&parent).unwrap().borrow_mut().children.push(tree); - None - } else { - Some(tree) - } - }) - .collect::>() - } - - fn skip_unnessary_nodes(trees: &mut IndexMap>>) { - for tree in trees.values_mut() { - let mut tree = tree.borrow_mut(); - if tree.chunks.is_empty() && tree.children.len() == 1 { - let child = tree.children.pop().unwrap(); - let mut child = child.borrow_mut(); - tree.path = child.path; - tree.chunks.append(&mut child.chunks); - tree.children.append(&mut child.children); - } - } - } - - // Convert function to the real data structure - fn node_to_common_parent_tree(node: Rc>) -> ContainmentTree { - let mut node = node.borrow_mut(); - let children = take(&mut node.children) - .into_iter() - .map(node_to_common_parent_tree) - .collect(); - // TODO keyed cell: this would benefit from keying the cell by node.path - let chunks = Some(ChunksVc::cell(take(&mut node.chunks))); - ContainmentTree { - path: Some(node.path), - chunks, - children, - } - } - - fn convert_into_common_parent_tree( - roots: Vec>>, - orphan_chunks: Vec, - ) -> Vec { - roots - .into_iter() - .map(node_to_common_parent_tree) - .chain(orphan_chunks.into_iter().map(|chunk| ContainmentTree { - path: None, - chunks: Some(orphan_chunk(chunk)), - children: Vec::new(), - })) - .collect::>() - } - - // resolve all paths - let chunks = resolve(chunks).await?; - // compute all unique common_parents - let mut common_parents = chunks - .iter() - .filter_map(|&(path, _)| path) - .collect::>(); - // expand all common parents to include all their parents - expand_common_parents(&mut common_parents).await?; - // compute parent -> child relationships between common_parents - let relationships = compute_relationships(&common_parents).await?; - - // create the tree nodes - let mut trees = create_node_tree(common_parents); - - // add chunks to nodes - let orphan_chunks = add_chunks_to_tree(&mut trees, chunks); - - // nest each tree by relationship, compute the roots - let roots = treeify(relationships, &mut trees); - - // optimize tree by removing unnecessary nodes - skip_unnessary_nodes(&mut trees); - - // do conversion - let roots = convert_into_common_parent_tree(roots, orphan_chunks); - - // top level nesting - Ok(if roots.len() == 1 { - roots.into_iter().next().unwrap() - } else { - ContainmentTree { - path: None, - chunks: None, - children: roots, - } - }) - } -} - -pub async fn optimize_by_common_parent( - chunks: ChunksVc, - get_common_parent: impl Fn(ChunkVc) -> FileSystemPathOptionVc, - optimize: impl Fn(Option, Vec) -> ChunksVc, -) -> Result { +use turbo_tasks_fs::FileSystemPathOptionVc; + +use crate::chunk::containment_tree::ContainmentTree; + +pub async fn optimize_by_common_parent( + chunks: &[T], + get_common_parent: impl Fn(T) -> FileSystemPathOptionVc, + optimize: impl Fn(Option>, Vec) -> Acc, +) -> Result +where + T: Clone, +{ let tree = ContainmentTree::build( chunks - .await? .iter() - .map(|&chunk| (get_common_parent(chunk), chunk)), + .map(|chunk| (get_common_parent(chunk.clone()), chunk.clone())), ) .await?; - fn optimize_tree( - tree: ContainmentTree, - optimize: &impl Fn(Option, Vec) -> ChunksVc, - ) -> ChunksVc { + fn optimize_tree( + tree: ContainmentTree, + optimize: &impl Fn(Option>, Vec) -> Acc, + ) -> Acc { let children = tree .children .into_iter() diff --git a/crates/turbopack-core/src/introspect/asset.rs b/crates/turbopack-core/src/introspect/asset.rs index 5d63fdd28f41e..64a8b1f5d6b5a 100644 --- a/crates/turbopack-core/src/introspect/asset.rs +++ b/crates/turbopack-core/src/introspect/asset.rs @@ -6,10 +6,7 @@ use turbo_tasks_fs::FileContent; use super::{Introspectable, IntrospectableChildrenVc, IntrospectableVc}; use crate::{ asset::{Asset, AssetContent, AssetContentVc, AssetVc}, - chunk::{ - ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkingType, ParallelChunkReference, - ParallelChunkReferenceVc, - }, + chunk::{ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkingType}, reference::{AssetReference, AssetReferencesVc}, resolve::PrimaryResolveResult, }; @@ -128,10 +125,6 @@ pub async fn children_from_asset_references( Some(ChunkingType::PlacedOrParallel) => key = placed_or_parallel_reference_ty(), Some(ChunkingType::SeparateAsync) => key = async_reference_ty(), } - } else if let Some(parallel) = ParallelChunkReferenceVc::resolve_from(reference).await? { - if *parallel.is_loaded_in_parallel().await? { - key = parallel_reference_ty(); - } } for result in reference.resolve_reference().await?.primary.iter() { diff --git a/crates/turbopack-core/src/reference/mod.rs b/crates/turbopack-core/src/reference/mod.rs index 7c8bedbc91aab..e2819840ffefe 100644 --- a/crates/turbopack-core/src/reference/mod.rs +++ b/crates/turbopack-core/src/reference/mod.rs @@ -14,11 +14,10 @@ pub use source_map::SourceMapReferenceVc; /// A reference to one or multiple [Asset]s or other special things. /// There are a bunch of optional traits that can influence how these references -/// are handled. e. g. [ChunkableAssetReference] or [ParallelChunkReference] +/// are handled. e. g. [ChunkableAssetReference] /// /// [Asset]: crate::asset::Asset /// [ChunkableAssetReference]: crate::chunk::ChunkableAssetReference -/// [ParallelChunkReference]: crate::chunk::ParallelChunkReference #[turbo_tasks::value_trait] pub trait AssetReference: ValueToString { fn resolve_reference(&self) -> ResolveResultVc; diff --git a/crates/turbopack-css/src/chunk/mod.rs b/crates/turbopack-css/src/chunk/mod.rs index 6842839b244d4..05acbf490db82 100644 --- a/crates/turbopack-css/src/chunk/mod.rs +++ b/crates/turbopack-css/src/chunk/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod optimize; pub mod source_map; pub(crate) mod writer; @@ -11,11 +10,9 @@ use turbo_tasks_fs::{rope::Rope, File, FileSystemPathOptionVc}; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ - availability_info::AvailabilityInfo, - chunk_content, chunk_content_split, - optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkReferenceVc, - ChunkVc, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, FromChunkableAsset, + availability_info::AvailabilityInfo, chunk_content, chunk_content_split, Chunk, + ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkVc, + ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunksVc, FromChunkableAsset, ModuleId, ModuleIdVc, ModuleIdsVc, OutputChunk, OutputChunkRuntimeInfo, OutputChunkRuntimeInfoVc, OutputChunkVc, }, @@ -31,7 +28,7 @@ use turbopack_core::{ }; use writer::expand_imports; -use self::{optimize::CssChunkOptimizerVc, source_map::CssChunkSourceMapAssetReferenceVc}; +use self::source_map::CssChunkSourceMapAssetReferenceVc; use crate::{ embed::{CssEmbed, CssEmbeddable, CssEmbeddableVc}, parse::ParseResultSourceMapVc, @@ -41,11 +38,14 @@ use crate::{ #[turbo_tasks::value] pub struct CssChunk { - context: ChunkingContextVc, - main_entries: CssChunkPlaceablesVc, - availability_info: AvailabilityInfo, + pub context: ChunkingContextVc, + pub main_entries: CssChunkPlaceablesVc, + pub availability_info: AvailabilityInfo, } +#[turbo_tasks::value(transparent)] +pub struct CssChunks(Vec); + #[turbo_tasks::value_impl] impl CssChunkVc { #[turbo_tasks::function] @@ -281,13 +281,20 @@ impl Chunk for CssChunk { fn chunking_context(&self) -> ChunkingContextVc { self.context } -} -#[turbo_tasks::value_impl] -impl OptimizableChunk for CssChunk { #[turbo_tasks::function] - fn get_optimizer(&self) -> ChunkOptimizerVc { - CssChunkOptimizerVc::new(self.context).into() + async fn parallel_chunks(&self) -> Result { + let content = css_chunk_content( + self.context, + self.main_entries, + Value::new(self.availability_info), + ) + .await?; + let mut chunks = Vec::new(); + for chunk in content.chunks.iter() { + chunks.push(*chunk); + } + Ok(ChunksVc::cell(chunks)) } } @@ -367,9 +374,6 @@ impl Asset for CssChunk { } } } - for chunk in content.chunks.iter() { - references.push(ChunkReferenceVc::new_parallel(*chunk).into()); - } for entry in content.async_chunk_group_entries.iter() { references.push(ChunkGroupReferenceVc::new(this.context, *entry).into()); } diff --git a/crates/turbopack-dev/Cargo.toml b/crates/turbopack-dev/Cargo.toml index b4410cd828bc2..a55310dd71bda 100644 --- a/crates/turbopack-dev/Cargo.toml +++ b/crates/turbopack-dev/Cargo.toml @@ -26,6 +26,7 @@ turbo-tasks-fs = { workspace = true } turbo-tasks-hash = { workspace = true } turbopack = { workspace = true } turbopack-core = { workspace = true } +turbopack-css = { workspace = true } turbopack-ecmascript = { workspace = true } [build-dependencies] diff --git a/crates/turbopack-dev/src/chunking_context.rs b/crates/turbopack-dev/src/chunking_context.rs index 21b836dbb76aa..29c09c7a794da 100644 --- a/crates/turbopack-dev/src/chunking_context.rs +++ b/crates/turbopack-dev/src/chunking_context.rs @@ -12,25 +12,28 @@ use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64, DeterministicHash, Xxh3Hash use turbopack_core::{ asset::{Asset, AssetVc, AssetsVc}, chunk::{ - availability_info::AvailabilityInfo, optimize, ChunkVc, ChunkableAsset, ChunkableAssetVc, - ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, ParallelChunkReference, - ParallelChunkReferenceVc, + availability_info::AvailabilityInfo, Chunk, ChunkVc, ChunkableAsset, ChunkableAssetVc, + ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, }, environment::EnvironmentVc, ident::{AssetIdent, AssetIdentVc}, - reference::{AssetReference, AssetReferenceVc}, - resolve::{ModulePart, PrimaryResolveResult}, + resolve::ModulePart, }; +use turbopack_css::chunk::{CssChunkVc, CssChunksVc}; use turbopack_ecmascript::chunk::{ EcmascriptChunkItemVc, EcmascriptChunkVc, EcmascriptChunkingContext, - EcmascriptChunkingContextVc, + EcmascriptChunkingContextVc, EcmascriptChunksVc, }; -use crate::ecmascript::{ - chunk::EcmascriptDevChunkVc, - evaluate::chunk::EcmascriptDevEvaluateChunkVc, - list::asset::{EcmascriptDevChunkListSource, EcmascriptDevChunkListVc}, - manifest::{chunk_asset::DevManifestChunkAssetVc, loader_item::DevManifestLoaderItemVc}, +use crate::{ + css::optimize::optimize_css_chunks, + ecmascript::{ + chunk::EcmascriptDevChunkVc, + evaluate::chunk::EcmascriptDevEvaluateChunkVc, + list::asset::{EcmascriptDevChunkListSource, EcmascriptDevChunkListVc}, + manifest::{chunk_asset::DevManifestChunkAssetVc, loader_item::DevManifestLoaderItemVc}, + optimize::optimize_ecmascript_chunks, + }, }; pub struct DevChunkingContextBuilder { @@ -370,9 +373,11 @@ impl ChunkingContext for DevChunkingContext { #[turbo_tasks::function] async fn chunk_group(self_vc: DevChunkingContextVc, entry_chunk: ChunkVc) -> Result { - let chunks = get_optimized_parallel_chunks([entry_chunk]).await?; + let parallel_chunks = get_parallel_chunks([entry_chunk]).await?; + + let optimized_chunks = get_optimized_chunks(parallel_chunks).await?; - let mut assets: Vec = chunks + let mut assets: Vec = optimized_chunks .await? .iter() .map(|chunk| self_vc.generate_chunk(*chunk)) @@ -412,9 +417,11 @@ impl ChunkingContext for DevChunkingContext { entry_assets.insert(entry_chunk.resolve().await?); - let chunks = get_optimized_parallel_chunks(entry_assets).await?; + let parallel_chunks = get_parallel_chunks(entry_assets).await?; - let mut assets: Vec = chunks + let optimized_chunks = get_optimized_chunks(parallel_chunks).await?; + + let mut assets: Vec = optimized_chunks .await? .iter() .map(|chunk| self_vc.generate_chunk(*chunk)) @@ -447,60 +454,59 @@ impl EcmascriptChunkingContext for DevChunkingContext { } } -async fn get_optimized_parallel_chunks(entries: I) -> Result +async fn get_parallel_chunks(entries: I) -> Result> where I: IntoIterator, { - let chunks: Vec<_> = GraphTraversal::, _>>::visit( - entries, - get_chunk_children, + Ok( + GraphTraversal::, _>>::visit( + entries, + |chunk: ChunkVc| async move { + Ok(chunk + .parallel_chunks() + .await? + .iter() + .copied() + .collect::>() + .into_iter()) + }, + ) + .await + .completed()? + .into_inner() + .into_iter(), ) - .await - .completed()? - .into_inner() - .into_iter() - .collect(); +} - let chunks = ChunksVc::cell(chunks); - let chunks = optimize(chunks); +async fn get_optimized_chunks(chunks: I) -> Result +where + I: IntoIterator, +{ + let mut ecmascript_chunks = vec![]; + let mut css_chunks = vec![]; + let mut other_chunks = vec![]; + + for chunk in chunks.into_iter() { + if let Some(ecmascript_chunk) = EcmascriptChunkVc::resolve_from(&chunk).await? { + ecmascript_chunks.push(ecmascript_chunk); + } else if let Some(css_chunk) = CssChunkVc::resolve_from(&chunk).await? { + css_chunks.push(css_chunk); + } else { + other_chunks.push(chunk); + } + } - Ok(chunks) -} + let ecmascript_chunks = + optimize_ecmascript_chunks(EcmascriptChunksVc::cell(ecmascript_chunks)).await?; + let css_chunks = optimize_css_chunks(CssChunksVc::cell(css_chunks)).await?; -/// Computes the list of all chunk children of a given chunk. -async fn get_chunk_children(parent: ChunkVc) -> Result + Send> { - Ok(parent - .references() - .await? + let chunks = ecmascript_chunks .iter() .copied() - .map(reference_to_chunks) - .try_join() - .await? - .into_iter() - .flatten()) -} + .map(|chunk| chunk.as_chunk()) + .chain(css_chunks.iter().copied().map(|chunk| chunk.as_chunk())) + .chain(other_chunks.into_iter()) + .collect(); -/// Get all parallel chunks from a parallel chunk reference. -async fn reference_to_chunks(r: AssetReferenceVc) -> Result + Send> { - let mut result = Vec::new(); - if let Some(pc) = ParallelChunkReferenceVc::resolve_from(r).await? { - if *pc.is_loaded_in_parallel().await? { - result = r - .resolve_reference() - .await? - .primary - .iter() - .map(|r| async move { - Ok(if let PrimaryResolveResult::Asset(a) = r { - ChunkVc::resolve_from(a).await? - } else { - None - }) - }) - .try_join() - .await?; - } - } - Ok(result.into_iter().flatten()) + Ok(ChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/css/mod.rs b/crates/turbopack-dev/src/css/mod.rs new file mode 100644 index 0000000000000..2ca465e5e4b63 --- /dev/null +++ b/crates/turbopack-dev/src/css/mod.rs @@ -0,0 +1 @@ +pub(crate) mod optimize; diff --git a/crates/turbopack-css/src/chunk/optimize.rs b/crates/turbopack-dev/src/css/optimize.rs similarity index 52% rename from crates/turbopack-css/src/chunk/optimize.rs rename to crates/turbopack-dev/src/css/optimize.rs index 924c0d3d21a91..e71aa0d6373f1 100644 --- a/crates/turbopack-css/src/chunk/optimize.rs +++ b/crates/turbopack-dev/src/css/optimize.rs @@ -1,50 +1,23 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use indexmap::IndexSet; use turbo_tasks::{TryJoinIterExt, Value}; -use turbopack_core::chunk::{ - optimize::{ChunkOptimizer, ChunkOptimizerVc}, - ChunkVc, ChunkingContextVc, ChunksVc, -}; - -use super::{CssChunkPlaceablesVc, CssChunkVc}; - -#[turbo_tasks::value] -pub struct CssChunkOptimizer(ChunkingContextVc); - -#[turbo_tasks::value_impl] -impl CssChunkOptimizerVc { - #[turbo_tasks::function] - pub fn new(context: ChunkingContextVc) -> Self { - CssChunkOptimizer(context).cell() - } -} - -#[turbo_tasks::value_impl] -impl ChunkOptimizer for CssChunkOptimizer { - #[turbo_tasks::function] - async fn optimize(&self, chunks: ChunksVc) -> Result { - // The CSS optimizer works under the constraint that the order in which - // CSS chunks are loaded must be preserved, as CSS rules - // precedence is determined by the order in which they are - // loaded. This means that we may not merge chunks that are not - // adjacent to each other in a valid reverse topological order. - - // TODO(alexkirsz) It might be more interesting to only merge adjacent - // chunks when they are part of the same chunk subgraph. - // However, the optimizer currently does not have access to this - // information, as chunks are already fully flattened by the - // time they reach the optimizer. - - merge_adjacent_chunks(chunks).await - } -} - -async fn css(chunk: ChunkVc) -> Result { - if let Some(chunk) = CssChunkVc::resolve_from(chunk).await? { - Ok(chunk) - } else { - bail!("CssChunkOptimizer can only be used on CssChunks") - } +use turbopack_css::chunk::{CssChunkPlaceablesVc, CssChunkVc, CssChunksVc}; + +#[turbo_tasks::function] +pub async fn optimize_css_chunks(chunks: CssChunksVc) -> Result { + // The CSS optimizer works under the constraint that the order in which + // CSS chunks are loaded must be preserved, as CSS rules + // precedence is determined by the order in which they are + // loaded. This means that we may not merge chunks that are not + // adjacent to each other in a valid reverse topological order. + + // TODO(alexkirsz) It might be more interesting to only merge adjacent + // chunks when they are part of the same chunk subgraph. + // However, the optimizer currently does not have access to this + // information, as chunks are already fully flattened by the + // time they reach the optimizer. + + merge_adjacent_chunks(chunks).await } async fn merge_chunks( @@ -72,7 +45,7 @@ async fn merge_chunks( const MAX_CHUNK_COUNT: usize = 20; /// Groups adjacent chunks into at most `MAX_CHUNK_COUNT` groups. -fn aggregate_adjacent_chunks(chunks: &[ChunkVc]) -> Vec> { +fn aggregate_adjacent_chunks(chunks: &[CssChunkVc]) -> Vec> { // Each of the resulting merged chunks will have `chunks_per_merged_chunk` // chunks in them, except for the first `chunks_mod` chunks, which will have // one more chunk. @@ -103,7 +76,7 @@ fn aggregate_adjacent_chunks(chunks: &[ChunkVc]) -> Vec> { } /// Merges adjacent chunks into at most `MAX_CHUNK_COUNT` chunks. -async fn merge_adjacent_chunks(chunks_vc: ChunksVc) -> Result { +async fn merge_adjacent_chunks(chunks_vc: CssChunksVc) -> Result { let chunks = chunks_vc.await?; if chunks.len() <= MAX_CHUNK_COUNT { @@ -114,15 +87,9 @@ async fn merge_adjacent_chunks(chunks_vc: ChunksVc) -> Result { let chunks = chunks .into_iter() - .map(|chunks| async move { - let chunks = chunks.iter().copied().map(css).try_join().await?; - merge_chunks(*chunks.first().unwrap(), &chunks).await - }) + .map(|chunks| async move { merge_chunks(*chunks.first().unwrap(), &chunks).await }) .try_join() - .await? - .into_iter() - .map(|chunk| chunk.as_chunk()) - .collect(); + .await?; - Ok(ChunksVc::cell(chunks)) + Ok(CssChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/ecmascript/chunk.rs b/crates/turbopack-dev/src/ecmascript/chunk.rs index 11bd77975717b..aa14726146ba5 100644 --- a/crates/turbopack-dev/src/ecmascript/chunk.rs +++ b/crates/turbopack-dev/src/ecmascript/chunk.rs @@ -5,7 +5,7 @@ use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ ChunkingContext, OutputChunk, OutputChunkRuntimeInfo, OutputChunkRuntimeInfoVc, - OutputChunkVc, ParallelChunkReference, ParallelChunkReferenceVc, + OutputChunkVc, }, ident::AssetIdentVc, introspect::{Introspectable, IntrospectableChildrenVc, IntrospectableVc}, @@ -91,15 +91,7 @@ impl Asset for EcmascriptDevChunk { let chunk_references = this.chunk.references().await?; let mut references = Vec::with_capacity(chunk_references.len() + 1); - // In contrast to the inner chunk, the outer chunk should not have - // references of parallel chunk since these are already handled - // at the [ChunkGroup] level. for reference in &*chunk_references { - if let Some(parallel_ref) = ParallelChunkReferenceVc::resolve_from(*reference).await? { - if *parallel_ref.is_loaded_in_parallel().await? { - continue; - } - } references.push(*reference); } diff --git a/crates/turbopack-dev/src/ecmascript/mod.rs b/crates/turbopack-dev/src/ecmascript/mod.rs index 26a39fd28827b..6aa1c482f763a 100644 --- a/crates/turbopack-dev/src/ecmascript/mod.rs +++ b/crates/turbopack-dev/src/ecmascript/mod.rs @@ -7,5 +7,6 @@ pub(crate) mod list; pub(crate) mod manifest; pub(crate) mod merged; pub(crate) mod module_factory; +pub(crate) mod optimize; pub(crate) mod update; pub(crate) mod version; diff --git a/crates/turbopack-ecmascript/src/chunk/optimize.rs b/crates/turbopack-dev/src/ecmascript/optimize.rs similarity index 83% rename from crates/turbopack-ecmascript/src/chunk/optimize.rs rename to crates/turbopack-dev/src/ecmascript/optimize.rs index d3a2b9f56327e..b189d1dcde4fd 100644 --- a/crates/turbopack-ecmascript/src/chunk/optimize.rs +++ b/crates/turbopack-dev/src/ecmascript/optimize.rs @@ -2,50 +2,62 @@ use std::{cmp::Ordering, collections::HashSet}; -use anyhow::{bail, Result}; +use anyhow::Result; use indexmap::{IndexMap, IndexSet}; use turbo_tasks::{TryJoinIterExt, Value}; use turbo_tasks_fs::FileSystemPathOptionVc; -use turbopack_core::chunk::{ - optimize::{optimize_by_common_parent, ChunkOptimizer, ChunkOptimizerVc}, - ChunkVc, ChunksVc, +use turbopack_core::chunk::optimize::optimize_by_common_parent; +use turbopack_ecmascript::chunk::{ + EcmascriptChunkPlaceablesVc, EcmascriptChunkVc, EcmascriptChunkingContextVc, EcmascriptChunksVc, }; -use super::{EcmascriptChunkPlaceablesVc, EcmascriptChunkVc, EcmascriptChunkingContextVc}; - -#[turbo_tasks::value] -pub struct EcmascriptChunkOptimizer(EcmascriptChunkingContextVc); - -#[turbo_tasks::value_impl] -impl EcmascriptChunkOptimizerVc { - #[turbo_tasks::function] - pub fn new(context: EcmascriptChunkingContextVc) -> Self { - EcmascriptChunkOptimizer(context).cell() - } -} +#[turbo_tasks::function] +pub async fn optimize_ecmascript_chunks(chunks: EcmascriptChunksVc) -> Result { + // Ecmascript chunks in the same chunk group can have different chunking + // contexts (e.g. through Next.js' with-client-chunks transition). They must not + // be merged together, as this affects how module ids are computed within the + // chunk. + let chunks_by_chunking_context: IndexMap> = + chunks + .await? + .iter() + .map(|chunk| async move { + let chunking_context = chunk.await?.context.resolve().await?; + Ok((chunking_context, chunk)) + }) + .try_join() + .await? + .into_iter() + .fold(IndexMap::new(), |mut acc, (chunking_context, chunk)| { + acc.entry(chunking_context) + .or_insert_with(Vec::new) + .push(*chunk); + acc + }); -#[turbo_tasks::value_impl] -impl ChunkOptimizer for EcmascriptChunkOptimizer { - #[turbo_tasks::function] - async fn optimize(&self, chunks: ChunksVc) -> Result { - optimize_by_common_parent(chunks, get_common_parent, |local, children| { - optimize_ecmascript(local, children) + let optimized_chunks = chunks_by_chunking_context + .into_values() + .map(|chunks| async move { + Ok( + optimize_by_common_parent(&chunks, get_common_parent, |local, children| { + optimize_ecmascript(local.map(EcmascriptChunksVc::cell), children) + }) + .await? + .await?, + ) }) - .await - } -} + .try_join() + .await? + .into_iter() + .flat_map(|chunks| chunks.iter().copied().collect::>()) + .collect::>(); -async fn ecma(chunk: ChunkVc) -> Result { - if let Some(chunk) = EcmascriptChunkVc::resolve_from(chunk).await? { - Ok(chunk) - } else { - bail!("EcmascriptChunkOptimizer can only be used on EcmascriptChunks") - } + Ok(EcmascriptChunksVc::cell(optimized_chunks)) } #[turbo_tasks::function] -async fn get_common_parent(chunk: ChunkVc) -> Result { - Ok(ecma(chunk).await?.common_parent()) +async fn get_common_parent(chunk: EcmascriptChunkVc) -> Result { + Ok(chunk.common_parent()) } /// Merge a few chunks into a single chunk. @@ -91,7 +103,7 @@ const MAX_CHUNK_ITEMS_PER_CHUNK: usize = 3000; /// Merge chunks with high duplication between them. async fn merge_duplicated_and_contained( - chunks: &mut Vec<(EcmascriptChunkVc, Option)>, + chunks: &mut Vec<(EcmascriptChunkVc, Option)>, mut unoptimized_count: usize, ) -> Result<()> { struct Comparison { @@ -276,7 +288,7 @@ async fn merge_duplicated_and_contained( /// all chunks from a single source and if that's not enough it will merge /// chunks into equal sized groups. async fn merge_to_limit( - chunks: Vec<(EcmascriptChunkVc, Option)>, + chunks: Vec<(EcmascriptChunkVc, Option)>, target_count: usize, ) -> Result> { let mut remaining = chunks.len(); @@ -373,12 +385,15 @@ async fn merge_by_size( /// Chunk optimization for ecmascript chunks. #[turbo_tasks::function] -async fn optimize_ecmascript(local: Option, children: Vec) -> Result { - let mut chunks = Vec::<(EcmascriptChunkVc, Option)>::new(); +async fn optimize_ecmascript( + local: Option, + children: Vec, +) -> Result { + let mut chunks = Vec::<(EcmascriptChunkVc, Option)>::new(); // TODO optimize let mut unoptimized_count = 0; if let Some(local) = local { - let mut local = local.await?.iter().copied().map(ecma).try_join().await?; + let mut local = local.await?.iter().copied().collect::>(); // Merge all local chunks when they are too many if local.len() > LOCAL_CHUNK_MERGE_THRESHOLD { local = merge_by_size(local).await?; @@ -399,7 +414,7 @@ async fn optimize_ecmascript(local: Option, children: Vec) - let mut children = children_chunks .await? .iter() - .map(|child| async move { Ok((ecma(*child).await?, Some(children_chunks))) }) + .map(|child| async move { Ok((*child, Some(children_chunks))) }) .try_join() .await?; chunks.append(&mut children); @@ -422,14 +437,11 @@ async fn optimize_ecmascript(local: Option, children: Vec) - // When there are too many chunks, try hard to reduce the number of chunks to // limit the request count. - if chunks.len() > TOTAL_CHUNK_MERGE_THRESHOLD { - let chunks = merge_to_limit(chunks, TOTAL_CHUNK_MERGE_THRESHOLD).await?; - Ok(ChunksVc::cell( - chunks.into_iter().map(|c| c.as_chunk()).collect(), - )) + let chunks = if chunks.len() > TOTAL_CHUNK_MERGE_THRESHOLD { + merge_to_limit(chunks, TOTAL_CHUNK_MERGE_THRESHOLD).await? } else { - Ok(ChunksVc::cell( - chunks.into_iter().map(|(c, _)| c.as_chunk()).collect(), - )) - } + chunks.into_iter().map(|(c, _)| c).collect() + }; + + Ok(EcmascriptChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/lib.rs b/crates/turbopack-dev/src/lib.rs index cf7a556e9c0b3..d26b7c25b7614 100644 --- a/crates/turbopack-dev/src/lib.rs +++ b/crates/turbopack-dev/src/lib.rs @@ -1,7 +1,9 @@ #![feature(lint_reasons)] #![feature(iter_intersperse)] +#![feature(int_roundings)] pub(crate) mod chunking_context; +pub(crate) mod css; pub(crate) mod ecmascript; pub mod embed_js; pub mod react_refresh; diff --git a/crates/turbopack-ecmascript/src/chunk/mod.rs b/crates/turbopack-ecmascript/src/chunk/mod.rs index fc15313c40098..3a18e7c3289a1 100644 --- a/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod content; pub(crate) mod context; pub(crate) mod item; -pub(crate) mod optimize; pub(crate) mod placeable; use std::fmt::Write; @@ -16,10 +15,8 @@ use turbo_tasks_fs::FileSystemPathOptionVc; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ - availability_info::AvailabilityInfo, - optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkReferenceVc, ChunkVc, ChunkingContextVc, - ModuleIdsVc, + availability_info::AvailabilityInfo, Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkVc, + ChunkingContextVc, ChunksVc, ModuleIdsVc, }, ident::{AssetIdent, AssetIdentVc}, introspect::{ @@ -29,7 +26,7 @@ use turbopack_core::{ reference::AssetReferencesVc, }; -use self::{content::ecmascript_chunk_content, optimize::EcmascriptChunkOptimizerVc}; +use self::content::ecmascript_chunk_content; pub use self::{ content::{EcmascriptChunkContent, EcmascriptChunkContentVc}, context::{EcmascriptChunkingContext, EcmascriptChunkingContextVc}, @@ -46,12 +43,15 @@ use crate::utils::FormatIter; #[turbo_tasks::value] pub struct EcmascriptChunk { - context: EcmascriptChunkingContextVc, - main_entries: EcmascriptChunkPlaceablesVc, - omit_entries: Option, - availability_info: AvailabilityInfo, + pub context: EcmascriptChunkingContextVc, + pub main_entries: EcmascriptChunkPlaceablesVc, + pub omit_entries: Option, + pub availability_info: AvailabilityInfo, } +#[turbo_tasks::value(transparent)] +pub struct EcmascriptChunks(Vec); + #[turbo_tasks::value_impl] impl EcmascriptChunkVc { #[turbo_tasks::function] @@ -211,9 +211,9 @@ impl EcmascriptChunkVc { #[turbo_tasks::value] pub struct EcmascriptChunkComparison { - shared_chunk_items: usize, - left_chunk_items: usize, - right_chunk_items: usize, + pub shared_chunk_items: usize, + pub left_chunk_items: usize, + pub right_chunk_items: usize, } #[turbo_tasks::value_impl] @@ -222,13 +222,21 @@ impl Chunk for EcmascriptChunk { fn chunking_context(&self) -> ChunkingContextVc { self.context.into() } -} -#[turbo_tasks::value_impl] -impl OptimizableChunk for EcmascriptChunk { #[turbo_tasks::function] - fn get_optimizer(&self) -> ChunkOptimizerVc { - EcmascriptChunkOptimizerVc::new(self.context).into() + async fn parallel_chunks(&self) -> Result { + let content = ecmascript_chunk_content( + self.context, + self.main_entries, + self.omit_entries, + Value::new(self.availability_info), + ) + .await?; + let mut chunks = Vec::new(); + for chunk in content.chunks.iter() { + chunks.push(*chunk); + } + Ok(ChunksVc::cell(chunks)) } } @@ -283,7 +291,7 @@ impl EcmascriptChunkVc { } #[turbo_tasks::function] - async fn chunk_items_count(self) -> Result { + pub async fn chunk_items_count(self) -> Result { Ok(UsizeVc::cell(self.chunk_content().await?.chunk_items.len())) } } @@ -382,9 +390,6 @@ impl Asset for EcmascriptChunk { for r in content.external_asset_references.iter() { references.push(*r); } - for chunk in content.chunks.iter() { - references.push(ChunkReferenceVc::new_parallel(*chunk).into()); - } for entry in content.async_chunk_group_entries.iter() { references.push(ChunkGroupReferenceVc::new(this.context.into(), *entry).into()); } diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js similarity index 77% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js index 3c88707fff253..339aa3da7df4e 100644 --- a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js +++ b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js @@ -1,12 +1,12 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js", {}, ]); (globalThis.TURBOPACK_CHUNK_LISTS = globalThis.TURBOPACK_CHUNK_LISTS || []).push({ - "path": "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js", + "path": "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js", "chunks": [ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css", - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js" + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css" ], "source": "entry" }); \ No newline at end of file diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js similarity index 99% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js index b15ab60a6ef2d..f9c5fdd69bf81 100644 --- a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js +++ b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js @@ -1,7 +1,7 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js", {}, - {"otherChunks":[{"path":"output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.css (css)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]}],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]} + {"otherChunks":[{"path":"output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.css (css)"]}],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]} ]); (() => { if (!Array.isArray(globalThis.TURBOPACK)) { diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map similarity index 100% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js similarity index 97% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js index 3f685031df85f..436e5e4743cb7 100644 --- a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js +++ b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js @@ -1,7 +1,7 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js", + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js", {}, - {"otherChunks":[{"path":"output/8697f_foo_style.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.css (css)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/style.css (css)"]},{"path":"output/8697f_foo_style.module_b5a149.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.module.css (css, css module)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/style.module.css (css, css module)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]},{"path":"output/8697f_foo_style.module.css_7740ee._.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.module.css (css module)"]}],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]} + {"otherChunks":[{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]},{"path":"output/8697f_foo_style.module.css_7740ee._.js","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.module.css (css module)"]},{"path":"output/8697f_foo_style.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.css (css)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/style.css (css)"]},{"path":"output/8697f_foo_style.module_b5a149.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/node_modules/foo/style.module.css (css, css module)"]},{"path":"output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css","included":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/style.module.css (css, css module)"]}],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]} ]); (() => { if (!Array.isArray(globalThis.TURBOPACK)) { diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js.map b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js.map similarity index 100% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_a14e34.js.map rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_37a138.js.map diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js similarity index 83% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js index bcfc33be52d32..bf3446711661e 100644 --- a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js +++ b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js @@ -1,16 +1,16 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js", + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js", {}, ]); (globalThis.TURBOPACK_CHUNK_LISTS = globalThis.TURBOPACK_CHUNK_LISTS || []).push({ - "path": "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_79b395.js", + "path": "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_469b46.js", "chunks": [ + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js", + "output/8697f_foo_style.module.css_7740ee._.js", "output/8697f_foo_style.css", "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css", "output/8697f_foo_style.module_b5a149.css", - "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css", - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js", - "output/8697f_foo_style.module.css_7740ee._.js" + "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css" ], "source": "entry" }); \ No newline at end of file