feat(turbopack-ecmascript): implement acyclic SCC graph for ESM imports

crates/turbo-tasks/src/debug/mod.rs

@@ -15,8 +15,8 @@ use internal::PassthroughDebug;
 ///
 /// We don't use `StringVc` directly because we don't want the `Debug`/`Display`
 /// representations to be escaped.
-#[derive(Clone)]
 #[turbo_tasks::value]
+#[derive(Clone)]
 pub struct ValueDebugString(String);
 
 impl Debug for ValueDebugString {

crates/turbo-tasks/src/join_iter_ext.rs

@@ -1,4 +1,8 @@
-use std::future::{Future, IntoFuture};
+use std::{
+    future::{Future, IntoFuture},
+    pin::Pin,
+    task::Poll,
+};
 
 use anyhow::Result;
 use futures::{
@@ -108,3 +112,68 @@ where
         }
     }
 }
+
+pin_project! {
+    /// Future for the [TryFlatJoinIterExt::try_flat_join] method.
+    pub struct TryFlatJoin<F>
+    where
+        F: Future,
+    {
+        #[pin]
+        inner: JoinAll<F>,
+    }
+}
+
+impl<F, I, U> Future for TryFlatJoin<F>
+where
+    F: Future<Output = Result<I>>,
+    I: IntoIterator<IntoIter = U, Item = U::Item>,
+    U: Iterator,
+{
+    type Output = Result<Vec<U::Item>>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
+        match self.project().inner.poll_unpin(cx) {
+            Poll::Ready(res) => {
+                let mut v = Vec::new();
+                for r in res {
+                    v.extend(r?);
+                }
+
+                Poll::Ready(Ok(v))
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+pub trait TryFlatJoinIterExt<F, I, U>: Iterator
+where
+    F: Future<Output = Result<I>>,
+    I: IntoIterator<IntoIter = U, Item = U::Item>,
+    U: Iterator,
+{
+    /// Returns a future that resolves to a vector of the outputs of the futures
+    /// in the iterator, or to an error if one of the futures fail.
+    ///
+    /// It also flattens the result.
+    ///
+    /// Unlike `Futures::future::try_join_all`, this returns the Error that
+    /// occurs first in the list of futures, not the first to fail in time.
+    fn try_flat_join(self) -> TryFlatJoin<F>;
+}
+
+impl<F, IF, It, I, U> TryFlatJoinIterExt<F, I, U> for It
+where
+    F: Future<Output = Result<I>>,
+    IF: IntoFuture<Output = Result<I>, IntoFuture = F>,
+    It: Iterator<Item = IF>,
+    I: IntoIterator<IntoIter = U, Item = U::Item>,
+    U: Iterator,
+{
+    fn try_flat_join(self) -> TryFlatJoin<F> {
+        TryFlatJoin {
+            inner: join_all(self.map(|f| f.into_future())),
+        }
+    }
+}

crates/turbo-tasks/src/lib.rs

@@ -79,7 +79,7 @@ pub use id::{
 pub use invalidation::{
     DynamicEqHash, InvalidationReason, InvalidationReasonKind, InvalidationReasonSet,
 };
-pub use join_iter_ext::{JoinIterExt, TryJoinIterExt};
+pub use join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt};
 pub use manager::{
     dynamic_call, emit, get_invalidator, mark_finished, mark_stateful, run_once,
     run_once_with_reason, spawn_blocking, spawn_thread, trait_call, turbo_tasks, Invalidator,

crates/turbopack-core/src/asset.rs

@@ -30,6 +30,15 @@ impl AssetsVc {
     }
 }
 
+#[turbo_tasks::value_impl]
+impl AssetsSetVc {
+    /// Creates an empty set of [Asset]s
+    #[turbo_tasks::function]
+    pub fn empty() -> Self {
+        AssetsSetVc::cell(IndexSet::new())
+    }
+}
+
 /// An asset. It also forms a graph when following [Asset::references].
 #[turbo_tasks::value_trait]
 pub trait Asset {

crates/turbopack-core/src/chunk/available_assets.rs

@@ -82,7 +82,7 @@ impl AvailableAssetsVc {
 }
 
 #[turbo_tasks::function]
-async fn chunkable_assets_set(root: AssetVc) -> Result<AssetsSetVc> {
+pub async fn chunkable_assets_set(root: AssetVc) -> Result<AssetsSetVc> {
     let assets = AdjacencyMap::new()
         .skip_duplicates()
         .visit(once(root), |&asset: &AssetVc| async move {

crates/turbopack-ecmascript/Cargo.toml

@@ -19,7 +19,7 @@ num-bigint = "0.4"
 num-traits = "0.2.15"
 once_cell = { workspace = true }
 parking_lot = { workspace = true }
-petgraph = "0.6.2"
+petgraph = { workspace = true }
 pin-project-lite = { workspace = true }
 regex = { workspace = true }
 rustc-hash = { workspace = true }

crates/turbopack-ecmascript/src/chunk/esm_scope.rs

@@ -0,0 +1,155 @@
+use std::collections::HashMap;
+
+use anyhow::{Context, Result};
+use petgraph::{algo::tarjan_scc, prelude::DiGraphMap};
+use turbo_tasks::{TryFlatJoinIterExt, Value};
+use turbopack_core::{
+    asset::{Asset, AssetsSetVc},
+    chunk::{availability_info::AvailabilityInfo, available_assets::chunkable_assets_set},
+};
+
+use crate::{
+    chunk::EcmascriptChunkPlaceableVc,
+    references::esm::{base::ReferencedAsset, EsmAssetReferenceVc},
+    EcmascriptModuleAssetVc, EcmascriptModuleAssetsVc,
+};
+
+/// A graph representing all ESM imports in a chunk group.
+#[turbo_tasks::value(serialization = "none", cell = "new", eq = "manual")]
+pub(crate) struct EsmScope {
+    scc_map: HashMap<EcmascriptChunkPlaceableVc, EsmScopeSccVc>,
+    #[turbo_tasks(trace_ignore, debug_ignore)]
+    scc_graph: DiGraphMap<EsmScopeSccVc, ()>,
+}
+
+#[turbo_tasks::value(transparent)]
+pub(crate) struct EsmScopeScc(Vec<EcmascriptChunkPlaceableVc>);
+
+#[turbo_tasks::value(transparent)]
+pub(crate) struct OptionEsmScopeScc(Option<EsmScopeSccVc>);
+
+#[turbo_tasks::value(transparent)]
+pub(crate) struct EsmScopeSccs(Vec<EsmScopeSccVc>);
+
+#[turbo_tasks::value_impl]
+impl EsmScopeVc {
+    #[turbo_tasks::function]
+    pub(crate) async fn new(availability_info: Value<AvailabilityInfo>) -> Result<Self> {
+        let assets = if let Some(root) = availability_info.current_availability_root() {
+            chunkable_assets_set(root)
+        } else {
+            AssetsSetVc::empty()
+        };
+
+        let esm_assets = get_ecmascript_module_assets(assets);
+        let import_references = collect_import_references(esm_assets).await?;
+
+        let mut graph = DiGraphMap::new();
+
+        for (parent, child) in &*import_references {
+            graph.add_edge(*parent, *child, ());
+        }
+
+        let sccs = tarjan_scc(&graph);
+
+        let mut scc_map = HashMap::new();
+        for scc in sccs {
+            let scc_vc = EsmScopeScc(scc.clone()).cell();
+
+            for placeable in scc {
+                scc_map.insert(placeable, scc_vc);
+            }
+        }
+
+        let mut scc_graph = DiGraphMap::new();
+        for (parent, child, _) in graph.all_edges() {
+            let parent_scc_vc = *scc_map
+                .get(&parent)
+                .context("unexpected missing SCC in map")?;
+            let child_scc_vc = *scc_map
+                .get(&child)
+                .context("unexpected missing SCC in map")?;
+
+            if parent_scc_vc != child_scc_vc {
+                scc_graph.add_edge(parent_scc_vc, child_scc_vc, ());
+            }
+        }
+
+        Ok(Self::cell(EsmScope { scc_map, scc_graph }))
+    }
+
+    #[turbo_tasks::function]
+    pub(crate) async fn get_scc(
+        self,
+        placeable: EcmascriptChunkPlaceableVc,
+    ) -> Result<OptionEsmScopeSccVc> {
+        let this = self.await?;
+
+        Ok(OptionEsmScopeSccVc::cell(
+            this.scc_map.get(&placeable).copied(),
+        ))
+    }
+
+    #[turbo_tasks::function]
+    pub(crate) async fn get_scc_children(self, scc: EsmScopeSccVc) -> Result<EsmScopeSccsVc> {
+        let this = self.await?;
+
+        let children = this.scc_graph.neighbors(scc).collect();
+
+        Ok(EsmScopeSccsVc::cell(children))
+    }
+}
+
+#[turbo_tasks::function]
+async fn get_ecmascript_module_assets(assets: AssetsSetVc) -> Result<EcmascriptModuleAssetsVc> {
+    let esm_assets = assets
+        .await?
+        .iter()
+        .copied()
+        .map(|r| async move { anyhow::Ok(EcmascriptModuleAssetVc::resolve_from(r).await?) })
+        .try_flat_join()
+        .await?;
+
+    Ok(EcmascriptModuleAssetsVc::cell(esm_assets))
+}
+
+#[turbo_tasks::value(transparent)]
+struct ImportReferences(Vec<(EcmascriptChunkPlaceableVc, EcmascriptChunkPlaceableVc)>);
+
+#[turbo_tasks::function]
+async fn collect_import_references(
+    esm_assets: EcmascriptModuleAssetsVc,
+) -> Result<ImportReferencesVc> {
+    let import_references = esm_assets
+        .await?
+        .iter()
+        .copied()
+        .map(|a| async move {
+            let placeable = a.as_ecmascript_chunk_placeable().resolve().await?;
+
+            a.references()
+                .await?
+                .iter()
+                .copied()
+                .map(|r| async move {
+                    let Some(r) = EsmAssetReferenceVc::resolve_from(r).await? else {
+                        return Ok(None);
+                    };
+
+                    let ReferencedAsset::Some(child_placeable) = &*r.get_referenced_asset().await?
+                    else {
+                        return Ok(None);
+                    };
+
+                    let child_placeable = child_placeable.resolve().await?;
+
+                    anyhow::Ok(Some((placeable, child_placeable)))
+                })
+                .try_flat_join()
+                .await
+        })
+        .try_flat_join()
+        .await?;
+
+    Ok(ImportReferencesVc::cell(import_references))
+}

crates/turbopack-ecmascript/src/chunk/mod.rs

@@ -1,6 +1,7 @@
 pub(crate) mod content;
 pub(crate) mod context;
 pub(crate) mod data;
+pub(crate) mod esm_scope;
 pub(crate) mod item;
 pub(crate) mod placeable;
 

crates/turbopack-ecmascript/src/lib.rs

@@ -214,6 +214,10 @@ pub struct EcmascriptModuleAsset {
 #[turbo_tasks::value(transparent)]
 pub struct OptionEcmascriptModuleAsset(Option<EcmascriptModuleAssetVc>);
 
+/// A list of [EcmascriptModuleAsset]s
+#[turbo_tasks::value(transparent)]
+pub struct EcmascriptModuleAssets(Vec<EcmascriptModuleAssetVc>);
+
 impl EcmascriptModuleAssetVc {
     pub fn builder(
         source: SourceVc,

crates/turbopack-ecmascript/src/references/esm/base.rs

@@ -117,7 +117,7 @@ impl EsmAssetReference {
 #[turbo_tasks::value_impl]
 impl EsmAssetReferenceVc {
     #[turbo_tasks::function]
-    pub(super) async fn get_referenced_asset(self) -> Result<ReferencedAssetVc> {
+    pub(crate) async fn get_referenced_asset(self) -> Result<ReferencedAssetVc> {
         let this = self.await?;
 
         Ok(ReferencedAssetVc::from_resolve_result(