From ba6c6d1fc64ab6658a2394d929aa7843da905aa0 Mon Sep 17 00:00:00 2001 From: Alex Kirszenberg Date: Wed, 12 Apr 2023 11:12:36 +0200 Subject: [PATCH 1/3] Replace {Parallel}ChunkReference with Chunk::parallel_chunks --- crates/turbopack-core/src/chunk/mod.rs | 78 ++++--------------- crates/turbopack-core/src/introspect/asset.rs | 9 +-- crates/turbopack-core/src/reference/mod.rs | 3 +- crates/turbopack-css/src/chunk/mod.rs | 22 ++++-- crates/turbopack-dev/src/chunking_context.rs | 56 +++---------- crates/turbopack-dev/src/ecmascript/chunk.rs | 10 +-- crates/turbopack-ecmascript/src/chunk/mod.rs | 21 ++++- 7 files changed, 63 insertions(+), 136 deletions(-) diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 494e3a644f1ca..b40615a246c49 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -97,13 +97,17 @@ pub trait ChunkableAsset: Asset { #[turbo_tasks::value(transparent)] pub struct Chunks(Vec); +#[turbo_tasks::value_impl] +impl ChunksVc { + /// Creates a new empty [ChunksVc]. + #[turbo_tasks::function] + pub fn empty() -> ChunksVc { + Self::cell(vec![]) + } +} + /// A chunk is one type of asset. /// It usually contains multiple chunk items. -/// There is an optional trait [ParallelChunkReference] that -/// [AssetReference]s from a [Chunk] can implement. -/// If they implement that and [ParallelChunkReference::is_loaded_in_parallel] -/// returns true, all referenced assets (if they are [Chunk]s) are placed in the -/// same chunk group. #[turbo_tasks::value_trait] pub trait Chunk: Asset { fn chunking_context(&self) -> ChunkingContextVc; @@ -114,12 +118,11 @@ pub trait Chunk: Asset { fn path(&self) -> FileSystemPathVc { self.ident().path() } -} - -/// see [Chunk] for explanation -#[turbo_tasks::value_trait] -pub trait ParallelChunkReference: AssetReference + ValueToString { - fn is_loaded_in_parallel(&self) -> BoolVc; + /// Returns a list of chunks that should be loaded in parallel to this + /// chunk. + fn parallel_chunks(&self) -> ChunksVc { + ChunksVc::empty() + } } /// Specifies how a chunk interacts with other chunks when building a chunk @@ -166,59 +169,6 @@ pub trait ChunkableAssetReference: AssetReference + ValueToString { } } -/// A reference to a [Chunk]. Can be loaded in parallel, see [Chunk]. -#[turbo_tasks::value] -pub struct ChunkReference { - chunk: ChunkVc, - parallel: bool, -} - -#[turbo_tasks::value_impl] -impl ChunkReferenceVc { - #[turbo_tasks::function] - pub fn new(chunk: ChunkVc) -> Self { - Self::cell(ChunkReference { - chunk, - parallel: false, - }) - } - - #[turbo_tasks::function] - pub fn new_parallel(chunk: ChunkVc) -> Self { - Self::cell(ChunkReference { - chunk, - parallel: true, - }) - } -} - -#[turbo_tasks::value_impl] -impl AssetReference for ChunkReference { - #[turbo_tasks::function] - fn resolve_reference(&self) -> ResolveResultVc { - ResolveResult::asset(self.chunk.into()).into() - } -} - -#[turbo_tasks::value_impl] -impl ValueToString for ChunkReference { - #[turbo_tasks::function] - async fn to_string(&self) -> Result { - Ok(StringVc::cell(format!( - "chunk {}", - self.chunk.ident().to_string().await? - ))) - } -} - -#[turbo_tasks::value_impl] -impl ParallelChunkReference for ChunkReference { - #[turbo_tasks::function] - fn is_loaded_in_parallel(&self) -> BoolVc { - BoolVc::cell(self.parallel) - } -} - /// A reference to multiple chunks from a [ChunkGroup] #[turbo_tasks::value] pub struct ChunkGroupReference { diff --git a/crates/turbopack-core/src/introspect/asset.rs b/crates/turbopack-core/src/introspect/asset.rs index 5d63fdd28f41e..64a8b1f5d6b5a 100644 --- a/crates/turbopack-core/src/introspect/asset.rs +++ b/crates/turbopack-core/src/introspect/asset.rs @@ -6,10 +6,7 @@ use turbo_tasks_fs::FileContent; use super::{Introspectable, IntrospectableChildrenVc, IntrospectableVc}; use crate::{ asset::{Asset, AssetContent, AssetContentVc, AssetVc}, - chunk::{ - ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkingType, ParallelChunkReference, - ParallelChunkReferenceVc, - }, + chunk::{ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkingType}, reference::{AssetReference, AssetReferencesVc}, resolve::PrimaryResolveResult, }; @@ -128,10 +125,6 @@ pub async fn children_from_asset_references( Some(ChunkingType::PlacedOrParallel) => key = placed_or_parallel_reference_ty(), Some(ChunkingType::SeparateAsync) => key = async_reference_ty(), } - } else if let Some(parallel) = ParallelChunkReferenceVc::resolve_from(reference).await? { - if *parallel.is_loaded_in_parallel().await? { - key = parallel_reference_ty(); - } } for result in reference.resolve_reference().await?.primary.iter() { diff --git a/crates/turbopack-core/src/reference/mod.rs b/crates/turbopack-core/src/reference/mod.rs index 7c8bedbc91aab..e2819840ffefe 100644 --- a/crates/turbopack-core/src/reference/mod.rs +++ b/crates/turbopack-core/src/reference/mod.rs @@ -14,11 +14,10 @@ pub use source_map::SourceMapReferenceVc; /// A reference to one or multiple [Asset]s or other special things. /// There are a bunch of optional traits that can influence how these references -/// are handled. e. g. [ChunkableAssetReference] or [ParallelChunkReference] +/// are handled. e. g. [ChunkableAssetReference] /// /// [Asset]: crate::asset::Asset /// [ChunkableAssetReference]: crate::chunk::ChunkableAssetReference -/// [ParallelChunkReference]: crate::chunk::ParallelChunkReference #[turbo_tasks::value_trait] pub trait AssetReference: ValueToString { fn resolve_reference(&self) -> ResolveResultVc; diff --git a/crates/turbopack-css/src/chunk/mod.rs b/crates/turbopack-css/src/chunk/mod.rs index 5fbd021a20c34..e518924f35da2 100644 --- a/crates/turbopack-css/src/chunk/mod.rs +++ b/crates/turbopack-css/src/chunk/mod.rs @@ -14,8 +14,8 @@ use turbopack_core::{ availability_info::AvailabilityInfo, chunk_content, chunk_content_split, optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkReferenceVc, - ChunkVc, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, FromChunkableAsset, + Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkVc, + ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunksVc, FromChunkableAsset, ModuleId, ModuleIdVc, }, code_builder::{CodeBuilder, CodeVc}, @@ -280,6 +280,21 @@ impl Chunk for CssChunk { fn chunking_context(&self) -> ChunkingContextVc { self.context } + + #[turbo_tasks::function] + async fn parallel_chunks(&self) -> Result { + let content = css_chunk_content( + self.context, + self.main_entries, + Value::new(self.availability_info), + ) + .await?; + let mut chunks = Vec::new(); + for chunk in content.chunks.iter() { + chunks.push(*chunk); + } + Ok(ChunksVc::cell(chunks)) + } } #[turbo_tasks::value_impl] @@ -348,9 +363,6 @@ impl Asset for CssChunk { } } } - for chunk in content.chunks.iter() { - references.push(ChunkReferenceVc::new_parallel(*chunk).into()); - } for entry in content.async_chunk_group_entries.iter() { references.push(ChunkGroupReferenceVc::new(this.context, *entry).into()); } diff --git a/crates/turbopack-dev/src/chunking_context.rs b/crates/turbopack-dev/src/chunking_context.rs index 21b836dbb76aa..1544780402f32 100644 --- a/crates/turbopack-dev/src/chunking_context.rs +++ b/crates/turbopack-dev/src/chunking_context.rs @@ -12,14 +12,12 @@ use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64, DeterministicHash, Xxh3Hash use turbopack_core::{ asset::{Asset, AssetVc, AssetsVc}, chunk::{ - availability_info::AvailabilityInfo, optimize, ChunkVc, ChunkableAsset, ChunkableAssetVc, - ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, ParallelChunkReference, - ParallelChunkReferenceVc, + availability_info::AvailabilityInfo, optimize, Chunk, ChunkVc, ChunkableAsset, + ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, }, environment::EnvironmentVc, ident::{AssetIdent, AssetIdentVc}, - reference::{AssetReference, AssetReferenceVc}, - resolve::{ModulePart, PrimaryResolveResult}, + resolve::ModulePart, }; use turbopack_ecmascript::chunk::{ EcmascriptChunkItemVc, EcmascriptChunkVc, EcmascriptChunkingContext, @@ -453,7 +451,15 @@ where { let chunks: Vec<_> = GraphTraversal::, _>>::visit( entries, - get_chunk_children, + |chunk: ChunkVc| async move { + Ok(chunk + .parallel_chunks() + .await? + .iter() + .copied() + .collect::>() + .into_iter()) + }, ) .await .completed()? @@ -466,41 +472,3 @@ where Ok(chunks) } - -/// Computes the list of all chunk children of a given chunk. -async fn get_chunk_children(parent: ChunkVc) -> Result + Send> { - Ok(parent - .references() - .await? - .iter() - .copied() - .map(reference_to_chunks) - .try_join() - .await? - .into_iter() - .flatten()) -} - -/// Get all parallel chunks from a parallel chunk reference. -async fn reference_to_chunks(r: AssetReferenceVc) -> Result + Send> { - let mut result = Vec::new(); - if let Some(pc) = ParallelChunkReferenceVc::resolve_from(r).await? { - if *pc.is_loaded_in_parallel().await? { - result = r - .resolve_reference() - .await? - .primary - .iter() - .map(|r| async move { - Ok(if let PrimaryResolveResult::Asset(a) = r { - ChunkVc::resolve_from(a).await? - } else { - None - }) - }) - .try_join() - .await?; - } - } - Ok(result.into_iter().flatten()) -} diff --git a/crates/turbopack-dev/src/ecmascript/chunk.rs b/crates/turbopack-dev/src/ecmascript/chunk.rs index 1512429439155..2c9fa4caecf4f 100644 --- a/crates/turbopack-dev/src/ecmascript/chunk.rs +++ b/crates/turbopack-dev/src/ecmascript/chunk.rs @@ -3,7 +3,7 @@ use indexmap::IndexSet; use turbo_tasks::{primitives::StringVc, ValueToString, ValueToStringVc}; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, - chunk::{ChunkingContext, ParallelChunkReference, ParallelChunkReferenceVc}, + chunk::ChunkingContext, ident::AssetIdentVc, introspect::{Introspectable, IntrospectableChildrenVc, IntrospectableVc}, reference::AssetReferencesVc, @@ -76,15 +76,7 @@ impl Asset for EcmascriptDevChunk { let chunk_references = this.chunk.references().await?; let mut references = Vec::with_capacity(chunk_references.len() + 1); - // In contrast to the inner chunk, the outer chunk should not have - // references of parallel chunk since these are already handled - // at the [ChunkGroup] level. for reference in &*chunk_references { - if let Some(parallel_ref) = ParallelChunkReferenceVc::resolve_from(*reference).await? { - if *parallel_ref.is_loaded_in_parallel().await? { - continue; - } - } references.push(*reference); } diff --git a/crates/turbopack-ecmascript/src/chunk/mod.rs b/crates/turbopack-ecmascript/src/chunk/mod.rs index 7b6ca24c49eaa..c5f240b9bf90f 100644 --- a/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -18,7 +18,7 @@ use turbopack_core::{ chunk::{ availability_info::AvailabilityInfo, optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkReferenceVc, ChunkVc, ChunkingContextVc, + Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkVc, ChunkingContextVc, ChunksVc, }, ident::{AssetIdent, AssetIdentVc}, introspect::{ @@ -209,6 +209,22 @@ impl Chunk for EcmascriptChunk { fn chunking_context(&self) -> ChunkingContextVc { self.context.into() } + + #[turbo_tasks::function] + async fn parallel_chunks(&self) -> Result { + let content = ecmascript_chunk_content( + self.context, + self.main_entries, + self.omit_entries, + Value::new(self.availability_info), + ) + .await?; + let mut chunks = Vec::new(); + for chunk in content.chunks.iter() { + chunks.push(*chunk); + } + Ok(ChunksVc::cell(chunks)) + } } #[turbo_tasks::value_impl] @@ -369,9 +385,6 @@ impl Asset for EcmascriptChunk { for r in content.external_asset_references.iter() { references.push(*r); } - for chunk in content.chunks.iter() { - references.push(ChunkReferenceVc::new_parallel(*chunk).into()); - } for entry in content.async_chunk_group_entries.iter() { references.push(ChunkGroupReferenceVc::new(this.context.into(), *entry).into()); } From 19fe5f9a9abb91390185b5bfbe3b9d2bd5447e08 Mon Sep 17 00:00:00 2001 From: Alex Kirszenberg Date: Wed, 12 Apr 2023 12:27:19 +0200 Subject: [PATCH 2/3] Move optimize functions into turbopack-dev --- Cargo.lock | 1 + .../src/chunk/containment_tree.rs | 220 ++++++++++++ crates/turbopack-core/src/chunk/mod.rs | 4 +- crates/turbopack-core/src/chunk/optimize.rs | 314 +----------------- crates/turbopack-css/src/chunk/mod.rs | 26 +- crates/turbopack-dev/Cargo.toml | 1 + crates/turbopack-dev/src/chunking_context.rs | 50 ++- crates/turbopack-dev/src/css/mod.rs | 1 + .../src/css}/optimize.rs | 79 ++--- crates/turbopack-dev/src/ecmascript/mod.rs | 1 + .../src/ecmascript}/optimize.rs | 78 ++--- crates/turbopack-dev/src/lib.rs | 2 + crates/turbopack-ecmascript/src/chunk/mod.rs | 35 +- 13 files changed, 357 insertions(+), 455 deletions(-) create mode 100644 crates/turbopack-core/src/chunk/containment_tree.rs create mode 100644 crates/turbopack-dev/src/css/mod.rs rename crates/{turbopack-css/src/chunk => turbopack-dev/src/css}/optimize.rs (52%) rename crates/{turbopack-ecmascript/src/chunk => turbopack-dev/src/ecmascript}/optimize.rs (87%) diff --git a/Cargo.lock b/Cargo.lock index 09a88f353b400..1736073c1deb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7799,6 +7799,7 @@ dependencies = [ "turbo-tasks-hash", "turbopack", "turbopack-core", + "turbopack-css", "turbopack-ecmascript", ] diff --git a/crates/turbopack-core/src/chunk/containment_tree.rs b/crates/turbopack-core/src/chunk/containment_tree.rs new file mode 100644 index 0000000000000..2960716365854 --- /dev/null +++ b/crates/turbopack-core/src/chunk/containment_tree.rs @@ -0,0 +1,220 @@ +use std::{cell::RefCell, mem::take, rc::Rc}; + +use anyhow::Result; +use indexmap::{IndexMap, IndexSet}; +use turbo_tasks::TryJoinIterExt; +use turbo_tasks_fs::{FileSystemPathOptionVc, FileSystemPathVc}; + +#[derive(Default)] +pub struct ContainmentTree { + pub path: Option, + pub chunks: Option>, + pub children: Vec>, +} + +impl ContainmentTree { + pub async fn build( + chunks: impl Iterator, + ) -> Result> { + async fn resolve( + chunks: impl Iterator, + ) -> Result, T)>> { + chunks + .map(|(common_parent, chunk)| async move { + let common_parent = if let Some(common_parent) = *common_parent.await? { + Some(common_parent.resolve().await?) + } else { + None + }; + Ok((common_parent, chunk)) + }) + .try_join() + .await + } + + async fn expand_common_parents( + common_parents: &mut IndexSet, + ) -> Result<()> { + // This is mutated while iterating, so we need to loop with index + let mut i = 0; + while i < common_parents.len() { + let current = common_parents[i]; + let parent = current.parent().resolve().await?; + common_parents.insert(parent); + i += 1; + } + Ok(()) + } + + async fn compute_relationships( + common_parents: &IndexSet, + ) -> Result, FileSystemPathVc)>> { + common_parents + .iter() + .map(|&key| { + let common_parents = &common_parents; + async move { + let mut current = key; + loop { + let parent = current.parent().resolve().await?; + if parent == current { + return Ok((None, key)); + } + if common_parents.contains(&parent) { + // Can't insert here into the parent tree, since we want the order + // of children to be deterministic + return Ok((Some(parent), key)); + } + current = parent; + } + } + }) + .try_join() + .await + } + + // Temp structure which uses Rc and RefCell + struct Node { + path: FileSystemPathVc, + chunks: Vec, + children: Vec>>>, + } + + fn create_node_tree( + common_parents: IndexSet, + ) -> IndexMap>>> { + let mut trees = IndexMap::>>>::new(); + for common_parent in common_parents { + trees.insert( + common_parent, + Rc::new(RefCell::new(Node { + path: common_parent, + chunks: Vec::new(), + children: Vec::new(), + })), + ); + } + trees + } + + fn add_chunks_to_tree( + trees: &mut IndexMap>>>, + chunks: Vec<(Option, T)>, + ) -> Vec { + let mut orphan_chunks = Vec::new(); + for (common_parent, chunk) in chunks { + if let Some(common_parent) = common_parent { + trees + .get_mut(&common_parent) + .unwrap() + .borrow_mut() + .chunks + .push(chunk); + } else { + orphan_chunks.push(chunk); + } + } + orphan_chunks + } + + fn treeify( + relationships: Vec<(Option, FileSystemPathVc)>, + trees: &mut IndexMap>>>, + ) -> Vec>>> { + relationships + .into_iter() + .flat_map(|(parent, key)| { + let tree = trees.get(&key).unwrap().clone(); + if let Some(parent) = parent { + trees.get(&parent).unwrap().borrow_mut().children.push(tree); + None + } else { + Some(tree) + } + }) + .collect::>() + } + + fn skip_unnessary_nodes(trees: &mut IndexMap>>>) { + for tree in trees.values_mut() { + let mut tree = tree.borrow_mut(); + if tree.chunks.is_empty() && tree.children.len() == 1 { + let child = tree.children.pop().unwrap(); + let mut child = child.borrow_mut(); + tree.path = child.path; + tree.chunks.append(&mut child.chunks); + tree.children.append(&mut child.children); + } + } + } + + // Convert function to the real data structure + fn node_to_common_parent_tree(node: Rc>>) -> ContainmentTree { + let mut node = node.borrow_mut(); + let children = take(&mut node.children) + .into_iter() + .map(node_to_common_parent_tree) + .collect(); + // TODO keyed cell: this would benefit from keying the cell by node.path + let chunks = Some(take(&mut node.chunks)); + ContainmentTree { + path: Some(node.path), + chunks, + children, + } + } + + fn convert_into_common_parent_tree( + roots: Vec>>>, + orphan_chunks: Vec, + ) -> Vec> { + roots + .into_iter() + .map(node_to_common_parent_tree) + .chain(orphan_chunks.into_iter().map(|chunk| ContainmentTree { + path: None, + chunks: Some(vec![chunk]), + children: Vec::new(), + })) + .collect::>() + } + + // resolve all paths + let chunks = resolve(chunks).await?; + // compute all unique common_parents + let mut common_parents = chunks + .iter() + .filter_map(|&(path, _)| path) + .collect::>(); + // expand all common parents to include all their parents + expand_common_parents(&mut common_parents).await?; + // compute parent -> child relationships between common_parents + let relationships = compute_relationships(&common_parents).await?; + + // create the tree nodes + let mut trees = create_node_tree(common_parents); + + // add chunks to nodes + let orphan_chunks = add_chunks_to_tree(&mut trees, chunks); + + // nest each tree by relationship, compute the roots + let roots = treeify(relationships, &mut trees); + + // optimize tree by removing unnecessary nodes + skip_unnessary_nodes(&mut trees); + + // do conversion + let roots = convert_into_common_parent_tree(roots, orphan_chunks); + + // top level nesting + Ok(if roots.len() == 1 { + roots.into_iter().next().unwrap() + } else { + ContainmentTree { + path: None, + chunks: None, + children: roots, + } + }) + } +} diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index b40615a246c49..7cd2825d8d955 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -1,6 +1,7 @@ pub mod availability_info; pub mod available_assets; pub(crate) mod chunking_context; +pub(crate) mod containment_tree; pub(crate) mod evaluate; pub mod optimize; @@ -16,7 +17,7 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::{ debug::ValueDebugFormat, graph::{GraphTraversal, GraphTraversalResult, ReverseTopological, Visit, VisitControlFlow}, - primitives::{BoolVc, StringVc}, + primitives::StringVc, trace::TraceRawVcs, TryJoinIterExt, Value, ValueToString, ValueToStringVc, }; @@ -27,7 +28,6 @@ use self::availability_info::AvailabilityInfo; pub use self::{ chunking_context::{ChunkingContext, ChunkingContextVc}, evaluate::{EvaluatableAsset, EvaluatableAssetVc, EvaluatableAssets, EvaluatableAssetsVc}, - optimize::optimize, }; use crate::{ asset::{Asset, AssetVc, AssetsVc}, diff --git a/crates/turbopack-core/src/chunk/optimize.rs b/crates/turbopack-core/src/chunk/optimize.rs index de3019ab95ea7..db0455815615f 100644 --- a/crates/turbopack-core/src/chunk/optimize.rs +++ b/crates/turbopack-core/src/chunk/optimize.rs @@ -3,310 +3,30 @@ //! Usually chunks are optimized by limiting their total count, restricting //! their size and eliminating duplicates between them. -use std::{cell::RefCell, mem::take, rc::Rc}; - use anyhow::Result; -use indexmap::{IndexMap, IndexSet}; -use turbo_tasks::TryJoinIterExt; -use turbo_tasks_fs::{FileSystemPathOptionVc, FileSystemPathVc}; - -use super::{ChunkVc, ChunksVc}; -use crate::{ - asset::{Asset, AssetVc}, - chunk::Chunk, -}; - -/// A functor to optimize a set of chunks. -#[turbo_tasks::value_trait] -pub trait ChunkOptimizer { - fn optimize(&self, chunks: ChunksVc) -> ChunksVc; -} - -/// Trait to mark a chunk as optimizable. -#[turbo_tasks::value_trait] -pub trait OptimizableChunk: Chunk + Asset { - fn get_optimizer(&self) -> ChunkOptimizerVc; -} - -/// Optimize all chunks that implement [OptimizableChunk]. -#[turbo_tasks::function] -pub async fn optimize(chunks: ChunksVc) -> Result { - let chunks = chunks.await?; - let mut by_optimizer = IndexMap::<_, Vec<_>>::new(); - for (chunk, optimizer) in chunks - .iter() - .map(|chunk| async move { - Ok(( - chunk, - if let Some(optimizable) = OptimizableChunkVc::resolve_from(chunk).await? { - Some(optimizable.get_optimizer().resolve().await?) - } else { - None - }, - )) - }) - .try_join() - .await? - { - by_optimizer.entry(optimizer).or_default().push(*chunk); - } - - let optimized_chunks = by_optimizer - .into_iter() - .map(|(optimizer, chunks)| async move { - // TODO keyed cell: this would benefit from keying the cell by optimizer - let chunks = ChunksVc::cell(chunks); - Ok(if let Some(optimizer) = optimizer { - optimizer.optimize(chunks).await? - } else { - chunks.await? - }) - }) - .try_join() - .await?; - let optimized_chunks = optimized_chunks - .iter() - .flat_map(|c| c.iter().copied()) - .collect(); - Ok(ChunksVc::cell(optimized_chunks)) -} - -#[derive(Default)] -pub struct ContainmentTree { - pub path: Option, - pub chunks: Option, - pub children: Vec, -} - -#[turbo_tasks::function] -fn orphan_chunk(chunk: ChunkVc) -> ChunksVc { - ChunksVc::cell(vec![chunk]) -} - -impl ContainmentTree { - async fn build( - chunks: impl Iterator, - ) -> Result { - async fn resolve( - chunks: impl Iterator, - ) -> Result, ChunkVc)>> { - chunks - .map(|(common_parent, chunk)| async move { - let common_parent = if let Some(common_parent) = *common_parent.await? { - Some(common_parent.resolve().await?) - } else { - None - }; - Ok((common_parent, chunk)) - }) - .try_join() - .await - } - - async fn expand_common_parents( - common_parents: &mut IndexSet, - ) -> Result<()> { - // This is mutated while iterating, so we need to loop with index - let mut i = 0; - while i < common_parents.len() { - let current = common_parents[i]; - let parent = current.parent().resolve().await?; - common_parents.insert(parent); - i += 1; - } - Ok(()) - } - - async fn compute_relationships( - common_parents: &IndexSet, - ) -> Result, FileSystemPathVc)>> { - common_parents - .iter() - .map(|&key| { - let common_parents = &common_parents; - async move { - let mut current = key; - loop { - let parent = current.parent().resolve().await?; - if parent == current { - return Ok((None, key)); - } - if common_parents.contains(&parent) { - // Can't insert here into the parent tree, since we want the order - // of children to be deterministic - return Ok((Some(parent), key)); - } - current = parent; - } - } - }) - .try_join() - .await - } - - // Temp structure which uses Rc and RefCell - struct Node { - path: FileSystemPathVc, - chunks: Vec, - children: Vec>>, - } - - fn create_node_tree( - common_parents: IndexSet, - ) -> IndexMap>> { - let mut trees = IndexMap::>>::new(); - for common_parent in common_parents { - trees.insert( - common_parent, - Rc::new(RefCell::new(Node { - path: common_parent, - chunks: Vec::new(), - children: Vec::new(), - })), - ); - } - trees - } - - fn add_chunks_to_tree( - trees: &mut IndexMap>>, - chunks: Vec<(Option, ChunkVc)>, - ) -> Vec { - let mut orphan_chunks = Vec::new(); - for (common_parent, chunk) in chunks { - if let Some(common_parent) = common_parent { - trees - .get_mut(&common_parent) - .unwrap() - .borrow_mut() - .chunks - .push(chunk); - } else { - orphan_chunks.push(chunk); - } - } - orphan_chunks - } - - fn treeify( - relationships: Vec<(Option, FileSystemPathVc)>, - trees: &mut IndexMap>>, - ) -> Vec>> { - relationships - .into_iter() - .flat_map(|(parent, key)| { - let tree = trees.get(&key).unwrap().clone(); - if let Some(parent) = parent { - trees.get(&parent).unwrap().borrow_mut().children.push(tree); - None - } else { - Some(tree) - } - }) - .collect::>() - } - - fn skip_unnessary_nodes(trees: &mut IndexMap>>) { - for tree in trees.values_mut() { - let mut tree = tree.borrow_mut(); - if tree.chunks.is_empty() && tree.children.len() == 1 { - let child = tree.children.pop().unwrap(); - let mut child = child.borrow_mut(); - tree.path = child.path; - tree.chunks.append(&mut child.chunks); - tree.children.append(&mut child.children); - } - } - } - - // Convert function to the real data structure - fn node_to_common_parent_tree(node: Rc>) -> ContainmentTree { - let mut node = node.borrow_mut(); - let children = take(&mut node.children) - .into_iter() - .map(node_to_common_parent_tree) - .collect(); - // TODO keyed cell: this would benefit from keying the cell by node.path - let chunks = Some(ChunksVc::cell(take(&mut node.chunks))); - ContainmentTree { - path: Some(node.path), - chunks, - children, - } - } - - fn convert_into_common_parent_tree( - roots: Vec>>, - orphan_chunks: Vec, - ) -> Vec { - roots - .into_iter() - .map(node_to_common_parent_tree) - .chain(orphan_chunks.into_iter().map(|chunk| ContainmentTree { - path: None, - chunks: Some(orphan_chunk(chunk)), - children: Vec::new(), - })) - .collect::>() - } - - // resolve all paths - let chunks = resolve(chunks).await?; - // compute all unique common_parents - let mut common_parents = chunks - .iter() - .filter_map(|&(path, _)| path) - .collect::>(); - // expand all common parents to include all their parents - expand_common_parents(&mut common_parents).await?; - // compute parent -> child relationships between common_parents - let relationships = compute_relationships(&common_parents).await?; - - // create the tree nodes - let mut trees = create_node_tree(common_parents); - - // add chunks to nodes - let orphan_chunks = add_chunks_to_tree(&mut trees, chunks); - - // nest each tree by relationship, compute the roots - let roots = treeify(relationships, &mut trees); - - // optimize tree by removing unnecessary nodes - skip_unnessary_nodes(&mut trees); - - // do conversion - let roots = convert_into_common_parent_tree(roots, orphan_chunks); - - // top level nesting - Ok(if roots.len() == 1 { - roots.into_iter().next().unwrap() - } else { - ContainmentTree { - path: None, - chunks: None, - children: roots, - } - }) - } -} - -pub async fn optimize_by_common_parent( - chunks: ChunksVc, - get_common_parent: impl Fn(ChunkVc) -> FileSystemPathOptionVc, - optimize: impl Fn(Option, Vec) -> ChunksVc, -) -> Result { +use turbo_tasks_fs::FileSystemPathOptionVc; + +use crate::chunk::containment_tree::ContainmentTree; + +pub async fn optimize_by_common_parent( + chunks: &[T], + get_common_parent: impl Fn(T) -> FileSystemPathOptionVc, + optimize: impl Fn(Option>, Vec) -> Acc, +) -> Result +where + T: Clone, +{ let tree = ContainmentTree::build( chunks - .await? .iter() - .map(|&chunk| (get_common_parent(chunk), chunk)), + .map(|chunk| (get_common_parent(chunk.clone()), chunk.clone())), ) .await?; - fn optimize_tree( - tree: ContainmentTree, - optimize: &impl Fn(Option, Vec) -> ChunksVc, - ) -> ChunksVc { + fn optimize_tree( + tree: ContainmentTree, + optimize: &impl Fn(Option>, Vec) -> Acc, + ) -> Acc { let children = tree .children .into_iter() diff --git a/crates/turbopack-css/src/chunk/mod.rs b/crates/turbopack-css/src/chunk/mod.rs index e518924f35da2..700c8dd0286eb 100644 --- a/crates/turbopack-css/src/chunk/mod.rs +++ b/crates/turbopack-css/src/chunk/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod optimize; pub mod source_map; pub(crate) mod writer; @@ -11,10 +10,8 @@ use turbo_tasks_fs::{rope::Rope, File, FileSystemPathOptionVc}; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ - availability_info::AvailabilityInfo, - chunk_content, chunk_content_split, - optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkVc, + availability_info::AvailabilityInfo, chunk_content, chunk_content_split, Chunk, + ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkVc, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunksVc, FromChunkableAsset, ModuleId, ModuleIdVc, }, @@ -30,7 +27,7 @@ use turbopack_core::{ }; use writer::expand_imports; -use self::{optimize::CssChunkOptimizerVc, source_map::CssChunkSourceMapAssetReferenceVc}; +use self::source_map::CssChunkSourceMapAssetReferenceVc; use crate::{ embed::{CssEmbed, CssEmbeddable, CssEmbeddableVc}, parse::ParseResultSourceMapVc, @@ -40,11 +37,14 @@ use crate::{ #[turbo_tasks::value] pub struct CssChunk { - context: ChunkingContextVc, - main_entries: CssChunkPlaceablesVc, - availability_info: AvailabilityInfo, + pub context: ChunkingContextVc, + pub main_entries: CssChunkPlaceablesVc, + pub availability_info: AvailabilityInfo, } +#[turbo_tasks::value(transparent)] +pub struct CssChunks(Vec); + #[turbo_tasks::value_impl] impl CssChunkVc { #[turbo_tasks::function] @@ -297,14 +297,6 @@ impl Chunk for CssChunk { } } -#[turbo_tasks::value_impl] -impl OptimizableChunk for CssChunk { - #[turbo_tasks::function] - fn get_optimizer(&self) -> ChunkOptimizerVc { - CssChunkOptimizerVc::new(self.context).into() - } -} - #[turbo_tasks::value_impl] impl Asset for CssChunk { #[turbo_tasks::function] diff --git a/crates/turbopack-dev/Cargo.toml b/crates/turbopack-dev/Cargo.toml index b4410cd828bc2..a55310dd71bda 100644 --- a/crates/turbopack-dev/Cargo.toml +++ b/crates/turbopack-dev/Cargo.toml @@ -26,6 +26,7 @@ turbo-tasks-fs = { workspace = true } turbo-tasks-hash = { workspace = true } turbopack = { workspace = true } turbopack-core = { workspace = true } +turbopack-css = { workspace = true } turbopack-ecmascript = { workspace = true } [build-dependencies] diff --git a/crates/turbopack-dev/src/chunking_context.rs b/crates/turbopack-dev/src/chunking_context.rs index 1544780402f32..a2884af19ddb7 100644 --- a/crates/turbopack-dev/src/chunking_context.rs +++ b/crates/turbopack-dev/src/chunking_context.rs @@ -12,23 +12,28 @@ use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64, DeterministicHash, Xxh3Hash use turbopack_core::{ asset::{Asset, AssetVc, AssetsVc}, chunk::{ - availability_info::AvailabilityInfo, optimize, Chunk, ChunkVc, ChunkableAsset, - ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, + availability_info::AvailabilityInfo, Chunk, ChunkVc, ChunkableAsset, ChunkableAssetVc, + ChunkingContext, ChunkingContextVc, ChunksVc, EvaluatableAssetsVc, }, environment::EnvironmentVc, ident::{AssetIdent, AssetIdentVc}, resolve::ModulePart, }; +use turbopack_css::chunk::{CssChunkVc, CssChunksVc}; use turbopack_ecmascript::chunk::{ EcmascriptChunkItemVc, EcmascriptChunkVc, EcmascriptChunkingContext, - EcmascriptChunkingContextVc, + EcmascriptChunkingContextVc, EcmascriptChunksVc, }; -use crate::ecmascript::{ - chunk::EcmascriptDevChunkVc, - evaluate::chunk::EcmascriptDevEvaluateChunkVc, - list::asset::{EcmascriptDevChunkListSource, EcmascriptDevChunkListVc}, - manifest::{chunk_asset::DevManifestChunkAssetVc, loader_item::DevManifestLoaderItemVc}, +use crate::{ + css::optimize::optimize_css_chunks, + ecmascript::{ + chunk::EcmascriptDevChunkVc, + evaluate::chunk::EcmascriptDevEvaluateChunkVc, + list::asset::{EcmascriptDevChunkListSource, EcmascriptDevChunkListVc}, + manifest::{chunk_asset::DevManifestChunkAssetVc, loader_item::DevManifestLoaderItemVc}, + optimize::optimize_ecmascript_chunks, + }, }; pub struct DevChunkingContextBuilder { @@ -467,8 +472,31 @@ where .into_iter() .collect(); - let chunks = ChunksVc::cell(chunks); - let chunks = optimize(chunks); + let mut ecmascript_chunks = vec![]; + let mut css_chunks = vec![]; + let mut other_chunks = vec![]; + + for chunk in chunks.iter() { + if let Some(ecmascript_chunk) = EcmascriptChunkVc::resolve_from(chunk).await? { + ecmascript_chunks.push(ecmascript_chunk); + } else if let Some(css_chunk) = CssChunkVc::resolve_from(chunk).await? { + css_chunks.push(css_chunk); + } else { + other_chunks.push(*chunk); + } + } + + let ecmascript_chunks = + optimize_ecmascript_chunks(EcmascriptChunksVc::cell(ecmascript_chunks)).await?; + let css_chunks = optimize_css_chunks(CssChunksVc::cell(css_chunks)).await?; + + let chunks = ecmascript_chunks + .iter() + .copied() + .map(|chunk| chunk.as_chunk()) + .chain(css_chunks.iter().copied().map(|chunk| chunk.as_chunk())) + .chain(other_chunks.into_iter()) + .collect(); - Ok(chunks) + Ok(ChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/css/mod.rs b/crates/turbopack-dev/src/css/mod.rs new file mode 100644 index 0000000000000..2ca465e5e4b63 --- /dev/null +++ b/crates/turbopack-dev/src/css/mod.rs @@ -0,0 +1 @@ +pub(crate) mod optimize; diff --git a/crates/turbopack-css/src/chunk/optimize.rs b/crates/turbopack-dev/src/css/optimize.rs similarity index 52% rename from crates/turbopack-css/src/chunk/optimize.rs rename to crates/turbopack-dev/src/css/optimize.rs index 924c0d3d21a91..e71aa0d6373f1 100644 --- a/crates/turbopack-css/src/chunk/optimize.rs +++ b/crates/turbopack-dev/src/css/optimize.rs @@ -1,50 +1,23 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use indexmap::IndexSet; use turbo_tasks::{TryJoinIterExt, Value}; -use turbopack_core::chunk::{ - optimize::{ChunkOptimizer, ChunkOptimizerVc}, - ChunkVc, ChunkingContextVc, ChunksVc, -}; - -use super::{CssChunkPlaceablesVc, CssChunkVc}; - -#[turbo_tasks::value] -pub struct CssChunkOptimizer(ChunkingContextVc); - -#[turbo_tasks::value_impl] -impl CssChunkOptimizerVc { - #[turbo_tasks::function] - pub fn new(context: ChunkingContextVc) -> Self { - CssChunkOptimizer(context).cell() - } -} - -#[turbo_tasks::value_impl] -impl ChunkOptimizer for CssChunkOptimizer { - #[turbo_tasks::function] - async fn optimize(&self, chunks: ChunksVc) -> Result { - // The CSS optimizer works under the constraint that the order in which - // CSS chunks are loaded must be preserved, as CSS rules - // precedence is determined by the order in which they are - // loaded. This means that we may not merge chunks that are not - // adjacent to each other in a valid reverse topological order. - - // TODO(alexkirsz) It might be more interesting to only merge adjacent - // chunks when they are part of the same chunk subgraph. - // However, the optimizer currently does not have access to this - // information, as chunks are already fully flattened by the - // time they reach the optimizer. - - merge_adjacent_chunks(chunks).await - } -} - -async fn css(chunk: ChunkVc) -> Result { - if let Some(chunk) = CssChunkVc::resolve_from(chunk).await? { - Ok(chunk) - } else { - bail!("CssChunkOptimizer can only be used on CssChunks") - } +use turbopack_css::chunk::{CssChunkPlaceablesVc, CssChunkVc, CssChunksVc}; + +#[turbo_tasks::function] +pub async fn optimize_css_chunks(chunks: CssChunksVc) -> Result { + // The CSS optimizer works under the constraint that the order in which + // CSS chunks are loaded must be preserved, as CSS rules + // precedence is determined by the order in which they are + // loaded. This means that we may not merge chunks that are not + // adjacent to each other in a valid reverse topological order. + + // TODO(alexkirsz) It might be more interesting to only merge adjacent + // chunks when they are part of the same chunk subgraph. + // However, the optimizer currently does not have access to this + // information, as chunks are already fully flattened by the + // time they reach the optimizer. + + merge_adjacent_chunks(chunks).await } async fn merge_chunks( @@ -72,7 +45,7 @@ async fn merge_chunks( const MAX_CHUNK_COUNT: usize = 20; /// Groups adjacent chunks into at most `MAX_CHUNK_COUNT` groups. -fn aggregate_adjacent_chunks(chunks: &[ChunkVc]) -> Vec> { +fn aggregate_adjacent_chunks(chunks: &[CssChunkVc]) -> Vec> { // Each of the resulting merged chunks will have `chunks_per_merged_chunk` // chunks in them, except for the first `chunks_mod` chunks, which will have // one more chunk. @@ -103,7 +76,7 @@ fn aggregate_adjacent_chunks(chunks: &[ChunkVc]) -> Vec> { } /// Merges adjacent chunks into at most `MAX_CHUNK_COUNT` chunks. -async fn merge_adjacent_chunks(chunks_vc: ChunksVc) -> Result { +async fn merge_adjacent_chunks(chunks_vc: CssChunksVc) -> Result { let chunks = chunks_vc.await?; if chunks.len() <= MAX_CHUNK_COUNT { @@ -114,15 +87,9 @@ async fn merge_adjacent_chunks(chunks_vc: ChunksVc) -> Result { let chunks = chunks .into_iter() - .map(|chunks| async move { - let chunks = chunks.iter().copied().map(css).try_join().await?; - merge_chunks(*chunks.first().unwrap(), &chunks).await - }) + .map(|chunks| async move { merge_chunks(*chunks.first().unwrap(), &chunks).await }) .try_join() - .await? - .into_iter() - .map(|chunk| chunk.as_chunk()) - .collect(); + .await?; - Ok(ChunksVc::cell(chunks)) + Ok(CssChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/ecmascript/mod.rs b/crates/turbopack-dev/src/ecmascript/mod.rs index e2ba5ec6ab78f..8879a347a49a1 100644 --- a/crates/turbopack-dev/src/ecmascript/mod.rs +++ b/crates/turbopack-dev/src/ecmascript/mod.rs @@ -6,5 +6,6 @@ pub(crate) mod list; pub(crate) mod manifest; pub(crate) mod merged; pub(crate) mod module_factory; +pub(crate) mod optimize; pub(crate) mod update; pub(crate) mod version; diff --git a/crates/turbopack-ecmascript/src/chunk/optimize.rs b/crates/turbopack-dev/src/ecmascript/optimize.rs similarity index 87% rename from crates/turbopack-ecmascript/src/chunk/optimize.rs rename to crates/turbopack-dev/src/ecmascript/optimize.rs index d3a2b9f56327e..d8a738e219a2e 100644 --- a/crates/turbopack-ecmascript/src/chunk/optimize.rs +++ b/crates/turbopack-dev/src/ecmascript/optimize.rs @@ -2,50 +2,26 @@ use std::{cmp::Ordering, collections::HashSet}; -use anyhow::{bail, Result}; +use anyhow::Result; use indexmap::{IndexMap, IndexSet}; use turbo_tasks::{TryJoinIterExt, Value}; use turbo_tasks_fs::FileSystemPathOptionVc; -use turbopack_core::chunk::{ - optimize::{optimize_by_common_parent, ChunkOptimizer, ChunkOptimizerVc}, - ChunkVc, ChunksVc, +use turbopack_core::chunk::optimize::optimize_by_common_parent; +use turbopack_ecmascript::chunk::{ + EcmascriptChunkPlaceablesVc, EcmascriptChunkVc, EcmascriptChunksVc, }; -use super::{EcmascriptChunkPlaceablesVc, EcmascriptChunkVc, EcmascriptChunkingContextVc}; - -#[turbo_tasks::value] -pub struct EcmascriptChunkOptimizer(EcmascriptChunkingContextVc); - -#[turbo_tasks::value_impl] -impl EcmascriptChunkOptimizerVc { - #[turbo_tasks::function] - pub fn new(context: EcmascriptChunkingContextVc) -> Self { - EcmascriptChunkOptimizer(context).cell() - } -} - -#[turbo_tasks::value_impl] -impl ChunkOptimizer for EcmascriptChunkOptimizer { - #[turbo_tasks::function] - async fn optimize(&self, chunks: ChunksVc) -> Result { - optimize_by_common_parent(chunks, get_common_parent, |local, children| { - optimize_ecmascript(local, children) - }) - .await - } -} - -async fn ecma(chunk: ChunkVc) -> Result { - if let Some(chunk) = EcmascriptChunkVc::resolve_from(chunk).await? { - Ok(chunk) - } else { - bail!("EcmascriptChunkOptimizer can only be used on EcmascriptChunks") - } +#[turbo_tasks::function] +pub async fn optimize_ecmascript_chunks(chunks: EcmascriptChunksVc) -> Result { + optimize_by_common_parent(&*chunks.await?, get_common_parent, |local, children| { + optimize_ecmascript(local.map(EcmascriptChunksVc::cell), children) + }) + .await } #[turbo_tasks::function] -async fn get_common_parent(chunk: ChunkVc) -> Result { - Ok(ecma(chunk).await?.common_parent()) +async fn get_common_parent(chunk: EcmascriptChunkVc) -> Result { + Ok(chunk.common_parent()) } /// Merge a few chunks into a single chunk. @@ -91,7 +67,7 @@ const MAX_CHUNK_ITEMS_PER_CHUNK: usize = 3000; /// Merge chunks with high duplication between them. async fn merge_duplicated_and_contained( - chunks: &mut Vec<(EcmascriptChunkVc, Option)>, + chunks: &mut Vec<(EcmascriptChunkVc, Option)>, mut unoptimized_count: usize, ) -> Result<()> { struct Comparison { @@ -276,7 +252,7 @@ async fn merge_duplicated_and_contained( /// all chunks from a single source and if that's not enough it will merge /// chunks into equal sized groups. async fn merge_to_limit( - chunks: Vec<(EcmascriptChunkVc, Option)>, + chunks: Vec<(EcmascriptChunkVc, Option)>, target_count: usize, ) -> Result> { let mut remaining = chunks.len(); @@ -373,12 +349,15 @@ async fn merge_by_size( /// Chunk optimization for ecmascript chunks. #[turbo_tasks::function] -async fn optimize_ecmascript(local: Option, children: Vec) -> Result { - let mut chunks = Vec::<(EcmascriptChunkVc, Option)>::new(); +async fn optimize_ecmascript( + local: Option, + children: Vec, +) -> Result { + let mut chunks = Vec::<(EcmascriptChunkVc, Option)>::new(); // TODO optimize let mut unoptimized_count = 0; if let Some(local) = local { - let mut local = local.await?.iter().copied().map(ecma).try_join().await?; + let mut local = local.await?.iter().copied().collect::>(); // Merge all local chunks when they are too many if local.len() > LOCAL_CHUNK_MERGE_THRESHOLD { local = merge_by_size(local).await?; @@ -399,7 +378,7 @@ async fn optimize_ecmascript(local: Option, children: Vec) - let mut children = children_chunks .await? .iter() - .map(|child| async move { Ok((ecma(*child).await?, Some(children_chunks))) }) + .map(|child| async move { Ok((*child, Some(children_chunks))) }) .try_join() .await?; chunks.append(&mut children); @@ -422,14 +401,11 @@ async fn optimize_ecmascript(local: Option, children: Vec) - // When there are too many chunks, try hard to reduce the number of chunks to // limit the request count. - if chunks.len() > TOTAL_CHUNK_MERGE_THRESHOLD { - let chunks = merge_to_limit(chunks, TOTAL_CHUNK_MERGE_THRESHOLD).await?; - Ok(ChunksVc::cell( - chunks.into_iter().map(|c| c.as_chunk()).collect(), - )) + let chunks = if chunks.len() > TOTAL_CHUNK_MERGE_THRESHOLD { + merge_to_limit(chunks, TOTAL_CHUNK_MERGE_THRESHOLD).await? } else { - Ok(ChunksVc::cell( - chunks.into_iter().map(|(c, _)| c.as_chunk()).collect(), - )) - } + chunks.into_iter().map(|(c, _)| c).collect() + }; + + Ok(EcmascriptChunksVc::cell(chunks)) } diff --git a/crates/turbopack-dev/src/lib.rs b/crates/turbopack-dev/src/lib.rs index b476fe4cc3197..93a674f2e0137 100644 --- a/crates/turbopack-dev/src/lib.rs +++ b/crates/turbopack-dev/src/lib.rs @@ -1,7 +1,9 @@ #![feature(lint_reasons)] #![feature(iter_intersperse)] +#![feature(int_roundings)] pub(crate) mod chunking_context; +pub(crate) mod css; pub(crate) mod ecmascript; pub mod embed_js; pub mod react_refresh; diff --git a/crates/turbopack-ecmascript/src/chunk/mod.rs b/crates/turbopack-ecmascript/src/chunk/mod.rs index c5f240b9bf90f..1a74e1cf61c38 100644 --- a/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod content; pub(crate) mod context; pub(crate) mod item; -pub(crate) mod optimize; pub(crate) mod placeable; use std::fmt::Write; @@ -16,9 +15,8 @@ use turbo_tasks_fs::FileSystemPathOptionVc; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ - availability_info::AvailabilityInfo, - optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkVc, ChunkingContextVc, ChunksVc, + availability_info::AvailabilityInfo, Chunk, ChunkGroupReferenceVc, ChunkItem, ChunkVc, + ChunkingContextVc, ChunksVc, }, ident::{AssetIdent, AssetIdentVc}, introspect::{ @@ -28,7 +26,7 @@ use turbopack_core::{ reference::AssetReferencesVc, }; -use self::{content::ecmascript_chunk_content, optimize::EcmascriptChunkOptimizerVc}; +use self::content::ecmascript_chunk_content; pub use self::{ content::{EcmascriptChunkContent, EcmascriptChunkContentVc}, context::{EcmascriptChunkingContext, EcmascriptChunkingContextVc}, @@ -45,12 +43,15 @@ use crate::utils::FormatIter; #[turbo_tasks::value] pub struct EcmascriptChunk { - context: EcmascriptChunkingContextVc, - main_entries: EcmascriptChunkPlaceablesVc, - omit_entries: Option, - availability_info: AvailabilityInfo, + pub context: EcmascriptChunkingContextVc, + pub main_entries: EcmascriptChunkPlaceablesVc, + pub omit_entries: Option, + pub availability_info: AvailabilityInfo, } +#[turbo_tasks::value(transparent)] +pub struct EcmascriptChunks(Vec); + #[turbo_tasks::value_impl] impl EcmascriptChunkVc { #[turbo_tasks::function] @@ -198,9 +199,9 @@ impl EcmascriptChunkVc { #[turbo_tasks::value] pub struct EcmascriptChunkComparison { - shared_chunk_items: usize, - left_chunk_items: usize, - right_chunk_items: usize, + pub shared_chunk_items: usize, + pub left_chunk_items: usize, + pub right_chunk_items: usize, } #[turbo_tasks::value_impl] @@ -227,14 +228,6 @@ impl Chunk for EcmascriptChunk { } } -#[turbo_tasks::value_impl] -impl OptimizableChunk for EcmascriptChunk { - #[turbo_tasks::function] - fn get_optimizer(&self) -> ChunkOptimizerVc { - EcmascriptChunkOptimizerVc::new(self.context).into() - } -} - #[turbo_tasks::value_impl] impl ValueToString for EcmascriptChunk { #[turbo_tasks::function] @@ -286,7 +279,7 @@ impl EcmascriptChunkVc { } #[turbo_tasks::function] - async fn chunk_items_count(self) -> Result { + pub async fn chunk_items_count(self) -> Result { Ok(UsizeVc::cell(self.chunk_content().await?.chunk_items.len())) } } From 92f5e5287756efb2c6bb8a486d0bef9edb33f78c Mon Sep 17 00:00:00 2001 From: Alex Kirszenberg Date: Wed, 12 Apr 2023 12:27:38 +0200 Subject: [PATCH 3/3] Update snapshots --- ...shot_css_absolute-uri-import_input_index_ec11e8.js} | 8 ++++---- ...shot_css_absolute-uri-import_input_index_fa9a30.js} | 4 ++-- ..._css_absolute-uri-import_input_index_fa9a30.js.map} | 0 ...tests_tests_snapshot_css_css_input_index_97aff3.js} | 4 ++-- ...s_tests_snapshot_css_css_input_index_97aff3.js.map} | 0 ...tests_tests_snapshot_css_css_input_index_f2447a.js} | 10 +++++----- 6 files changed, 13 insertions(+), 13 deletions(-) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js} (77%) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js} (99%) rename crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/{crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map => crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map} (100%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js => crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js} (99%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js.map => crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js.map} (100%) rename crates/turbopack-tests/tests/snapshot/css/css/output/{crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js => crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js} (83%) diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js similarity index 77% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js index 3c88707fff253..339aa3da7df4e 100644 --- a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js +++ b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js @@ -1,12 +1,12 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js", {}, ]); (globalThis.TURBOPACK_CHUNK_LISTS = globalThis.TURBOPACK_CHUNK_LISTS || []).push({ - "path": "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_2e2744.js", + "path": "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_ec11e8.js", "chunks": [ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css", - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js" + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css" ], "source": "entry" }); \ No newline at end of file diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js similarity index 99% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js index 8feeb92458ce4..edd7d00e93057 100644 --- a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js +++ b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js @@ -1,7 +1,7 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js", + "output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js", {}, - {"otherChunks":["output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css","output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js"],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]} + {"otherChunks":["output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_b53fce.js","output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index.css"],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/input/index.js (ecmascript)"]} ]); (() => { if (!Array.isArray(globalThis.TURBOPACK)) { diff --git a/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map b/crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map similarity index 100% rename from crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_082e10.js.map rename to crates/turbopack-tests/tests/snapshot/css/absolute-uri-import/output/crates_turbopack-tests_tests_snapshot_css_absolute-uri-import_input_index_fa9a30.js.map diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js similarity index 99% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js index 53b7de0bdf892..f49e9677fb920 100644 --- a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js +++ b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js @@ -1,7 +1,7 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js", + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js", {}, - {"otherChunks":["output/8697f_foo_style.css","output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css","output/8697f_foo_style.module_b5a149.css","output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css","output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js","output/8697f_foo_style.module.css_a724a8._.js"],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]} + {"otherChunks":["output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js","output/8697f_foo_style.module.css_a724a8._.js","output/8697f_foo_style.css","output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css","output/8697f_foo_style.module_b5a149.css","output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css"],"runtimeModuleIds":["[project]/crates/turbopack-tests/tests/snapshot/css/css/input/index.js (ecmascript)"]} ]); (() => { if (!Array.isArray(globalThis.TURBOPACK)) { diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js.map b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js.map similarity index 100% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_2081e1.js.map rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_97aff3.js.map diff --git a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js similarity index 83% rename from crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js rename to crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js index 4f2bb2d86fdab..4f9e2eb36a9b4 100644 --- a/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js +++ b/crates/turbopack-tests/tests/snapshot/css/css/output/crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js @@ -1,16 +1,16 @@ (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([ - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js", + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js", {}, ]); (globalThis.TURBOPACK_CHUNK_LISTS = globalThis.TURBOPACK_CHUNK_LISTS || []).push({ - "path": "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_c9488c.js", + "path": "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_f2447a.js", "chunks": [ + "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js", + "output/8697f_foo_style.module.css_a724a8._.js", "output/8697f_foo_style.css", "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.css", "output/8697f_foo_style.module_b5a149.css", - "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css", - "output/crates_turbopack-tests_tests_snapshot_css_css_input_index_b53fce.js", - "output/8697f_foo_style.module.css_a724a8._.js" + "output/crates_turbopack-tests_tests_snapshot_css_css_input_style.module_b5a149.css" ], "source": "entry" }); \ No newline at end of file