feat(turbopack-ecmascript): Add implementation for webpackIgnore and …

…turbopackIgnore (revision of #69113) (#69768)

## History

@arlyon was working on this in #69113. He's out of office, and this is a
feature we want to get landed soon, so I was seeing what needed to
happen to get it to merge.

I encountered more issues, and I didn't feel entirely comfortable
pushing deep changes to @arlyon's branch, so this builds on top of his
(rebased) commit.

This is my attempted PR to fix these problems.

## What?

*Copied from #69113's description.*

Certain bundles may be designed specifically to be included at runtime.
To support this usecase, webpack implemented a comment directive
`webpackIgnore` that opted out that particular import from bundle time
optimisations. We need to support this, as some libraries take advantage
of this (mapbox).

Closes PACK-3046

## Changes from #69113

- Bail out much earlier in the analysis. The other PR appeared to bail
out too late, and would generate a `Promise.resolve()` call, instead of
preserving `import`/`require`. My understanding of webpack's feature is
that it should include the `import`/`require`.
- Skip "too dynamic" analysis for ignored imports.
- Switch to a snapshot test, so that we can verify that the output
actually contains the `import`/`require` expressions.
- Integrates my changes from #69755 to use an `ImportOverrides` struct
instead of a bool.

---------

Co-authored-by: Alexander Lyon <arlyon@me.com>
Co-authored-by: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com>

turbopack/crates/turbopack-ecmascript/src/analyzer/graph.rs

@@ -1242,7 +1242,13 @@ impl VisitAstPath for Analyzer<'_> {
                 if let Some(require_var_id) = extract_var_from_umd_factory(callee, &n.args) {
                     self.add_value(
                         require_var_id,
-                        JsValue::WellKnownFunction(WellKnownFunctionKind::Require),
+                        JsValue::WellKnownFunction(WellKnownFunctionKind::Require {
+                            ignore: self
+                                .eval_context
+                                .imports
+                                .get_overrides(n.callee.span())
+                                .ignore,
+                        }),
                     );
                 }
             }

turbopack/crates/turbopack-ecmascript/src/analyzer/imports.rs

@@ -6,7 +6,7 @@ use std::{
 use indexmap::{IndexMap, IndexSet};
 use once_cell::sync::Lazy;
 use swc_core::{
-    common::{comments::Comments, source_map::SmallPos, Span, Spanned},
+    common::{comments::Comments, source_map::SmallPos, BytePos, Span, Spanned},
     ecma::{
         ast::*,
         atoms::{js_word, JsWord},
@@ -140,16 +140,56 @@ pub(crate) struct ImportMap {
     /// True if the module is an ESM module due to top-level await.
     has_top_level_await: bool,
 
-    /// Locations of webpackIgnore or turbopackIgnore comments
-    /// This is a webpack feature that allows opting out of static
-    /// imports, which we should respect.
+    /// Locations of [webpack-style "magic comments"][magic] that override import behaviors.
+    ///
+    /// Most commonly, these are `/* webpackIgnore: true */` comments. See [ImportOverrides] for
+    /// full details.
+    ///
+    /// [magic]: https://webpack.js.org/api/module-methods/#magic-comments
+    overrides: HashMap<BytePos, ImportOverrides>,
+}
+
+/// Represents a collection of [webpack-style "magic comments"][magic] that override import
+/// behaviors.
+///
+/// [magic]: https://webpack.js.org/api/module-methods/#magic-comments
+#[derive(Debug)]
+pub(crate) struct ImportOverrides {
+    /// Should we ignore this import expression when bundling? If so, the import expression will be
+    /// left as-is in Turbopack's output.
+    ///
+    /// This is set by using either a `webpackIgnore` or `turbopackIgnore` comment.
     ///
     /// Example:
     /// ```js
     /// const a = import(/* webpackIgnore: true */ "a");
     /// const b = import(/* turbopackIgnore: true */ "b");
     /// ```
-    turbopack_ignores: HashMap<Span, bool>,
+    pub ignore: bool,
+}
+
+impl ImportOverrides {
+    pub const fn empty() -> Self {
+        ImportOverrides { ignore: false }
+    }
+
+    pub fn empty_ref() -> &'static Self {
+        // use `Self::empty` here as `Default::default` isn't const
+        static DEFAULT_VALUE: ImportOverrides = ImportOverrides::empty();
+        &DEFAULT_VALUE
+    }
+}
+
+impl Default for ImportOverrides {
+    fn default() -> Self {
+        ImportOverrides::empty()
+    }
+}
+
+impl Default for &ImportOverrides {
+    fn default() -> Self {
+        ImportOverrides::empty_ref()
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -194,6 +234,10 @@ impl ImportMap {
         None
     }
 
+    pub fn get_overrides(&self, span: Span) -> &ImportOverrides {
+        self.overrides.get(&span.lo).unwrap_or_default()
+    }
+
     // TODO this could return &str instead of String to avoid cloning
     pub fn get_binding(&self, id: &Id) -> Option<(usize, Option<RcStr>)> {
         if let Some((i, i_sym)) = self.imports.get(id) {
@@ -459,7 +503,7 @@ impl Visit for Analyzer<'_> {
             };
 
             // we are interested here in the last comment with a valid directive
-            let ignore_statement = n
+            let ignore_directive = n
                 .args
                 .first()
                 .map(|arg| arg.span_lo())
@@ -478,10 +522,15 @@ impl Visit for Analyzer<'_> {
                 })
                 .next();
 
-            if let Some((callee_span, ignore_statement)) = callee_span.zip(ignore_statement) {
-                self.data
-                    .turbopack_ignores
-                    .insert(*callee_span, ignore_statement);
+            // potentially support more webpack magic comments in the future:
+            // https://webpack.js.org/api/module-methods/#magic-comments
+            if let Some((callee_span, ignore_directive)) = callee_span.zip(ignore_directive) {
+                self.data.overrides.insert(
+                    callee_span.lo,
+                    ImportOverrides {
+                        ignore: ignore_directive,
+                    },
+                );
             };
         }
 

turbopack/crates/turbopack-ecmascript/src/analyzer/mod.rs

@@ -1627,11 +1627,11 @@ impl JsValue {
                         format!("path.resolve({cwd})"),
                         "The Node.js path.resolve method: https://nodejs.org/api/path.html#pathresolvepaths",
                     ),
-                    WellKnownFunctionKind::Import => (
+                    WellKnownFunctionKind::Import { .. } => (
                         "import".to_string(),
                         "The dynamic import() method from the ESM specification: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#dynamic_imports"
                     ),
-                    WellKnownFunctionKind::Require => ("require".to_string(), "The require method from CommonJS"),
+                    WellKnownFunctionKind::Require { .. } => ("require".to_string(), "The require method from CommonJS"),
                     WellKnownFunctionKind::RequireResolve => ("require.resolve".to_string(), "The require.resolve method from CommonJS"),
                     WellKnownFunctionKind::RequireContext => ("require.context".to_string(), "The require.context method from webpack"),
                     WellKnownFunctionKind::RequireContextRequire(..) => ("require.context(...)".to_string(), "The require.context(...) method from webpack: https://webpack.js.org/api/module-methods/#requirecontext"),
@@ -3626,8 +3626,14 @@ pub enum WellKnownFunctionKind {
     PathDirname,
     /// `0` is the current working directory.
     PathResolve(Box<JsValue>),
-    Import,
-    Require,
+    /// Import and Require can be ignored at compile time using the `turbopackIgnore` directive.
+    /// This is functionality that was introduced in webpack, so we also support `webpackIgnore`.
+    Import {
+        ignore: bool,
+    },
+    Require {
+        ignore: bool,
+    },
     RequireResolve,
     RequireContext,
     RequireContextRequire(Vc<RequireContextValue>),
@@ -3656,8 +3662,8 @@ pub enum WellKnownFunctionKind {
 impl WellKnownFunctionKind {
     pub fn as_define_name(&self) -> Option<&[&str]> {
         match self {
-            Self::Import => Some(&["import"]),
-            Self::Require => Some(&["require"]),
+            Self::Import { .. } => Some(&["import"]),
+            Self::Require { .. } => Some(&["require"]),
             Self::RequireResolve => Some(&["require", "resolve"]),
             Self::RequireContext => Some(&["require", "context"]),
             Self::Define => Some(&["define"]),
@@ -3704,7 +3710,7 @@ pub mod test_utils {
         let mut new_value = match v {
             JsValue::Call(
                 _,
-                box JsValue::WellKnownFunction(WellKnownFunctionKind::Import),
+                box JsValue::WellKnownFunction(WellKnownFunctionKind::Import { .. }),
                 ref args,
             ) => match &args[0] {
                 JsValue::Constant(v) => JsValue::Module(ModuleValue {
@@ -3740,8 +3746,12 @@ pub mod test_utils {
                 Err(err) => v.into_unknown(true, PrettyPrintError(&err).to_string()),
             },
             JsValue::FreeVar(ref var) => match &**var {
-                "import" => JsValue::WellKnownFunction(WellKnownFunctionKind::Import),
-                "require" => JsValue::WellKnownFunction(WellKnownFunctionKind::Require),
+                "import" => {
+                    JsValue::WellKnownFunction(WellKnownFunctionKind::Import { ignore: false })
+                }
+                "require" => {
+                    JsValue::WellKnownFunction(WellKnownFunctionKind::Require { ignore: false })
+                }
                 "define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define),
                 "__dirname" => "__dirname".into(),
                 "__filename" => "__filename".into(),
@@ -3772,7 +3782,7 @@ mod tests {
     use std::{mem::take, path::PathBuf, time::Instant};
 
     use swc_core::{
-        common::Mark,
+        common::{comments::SingleThreadedComments, Mark},
         ecma::{
             ast::EsVersion, parser::parse_file_as_program, transforms::base::resolver,
             visit::VisitMutWith,
@@ -3809,11 +3819,12 @@ mod tests {
             r.block_on(async move {
                 let fm = cm.load_file(&input).unwrap();
 
+                let comments = SingleThreadedComments::default();
                 let mut m = parse_file_as_program(
                     &fm,
                     Default::default(),
                     EsVersion::latest(),
-                    None,
+                    Some(&comments),
                     &mut vec![],
                 )
                 .map_err(|err| err.into_diagnostic(handler).emit())?;
@@ -3823,7 +3834,7 @@ mod tests {
                 m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
 
                 let eval_context =
-                    EvalContext::new(&m, unresolved_mark, top_level_mark, None, None);
+                    EvalContext::new(&m, unresolved_mark, top_level_mark, Some(&comments), None);
 
                 let mut var_graph = create_graph(&m, &eval_context);
 

turbopack/crates/turbopack-ecmascript/src/analyzer/well_known.rs

@@ -57,12 +57,12 @@ pub async fn well_known_function_call(
         WellKnownFunctionKind::PathJoin => path_join(args),
         WellKnownFunctionKind::PathDirname => path_dirname(args),
         WellKnownFunctionKind::PathResolve(cwd) => path_resolve(*cwd, args),
-        WellKnownFunctionKind::Import => JsValue::unknown(
+        WellKnownFunctionKind::Import { .. } => JsValue::unknown(
             JsValue::call(Box::new(JsValue::WellKnownFunction(kind)), args),
             true,
             "import() is not supported",
         ),
-        WellKnownFunctionKind::Require => require(args),
+        WellKnownFunctionKind::Require { ignore } => require(args, ignore),
         WellKnownFunctionKind::RequireContextRequire(value) => {
             require_context_require(value, args).await?
         }
@@ -322,7 +322,11 @@ pub fn path_dirname(mut args: Vec<JsValue>) -> JsValue {
     )
 }
 
-pub fn require(args: Vec<JsValue>) -> JsValue {
+/// Resolve the contents of a require call, throwing errors
+/// if we come across any unsupported syntax.
+///
+/// `ignore` is true if the require call is marked with `turbopackIgnore` or `webpackIgnore`.
+pub fn require(args: Vec<JsValue>, ignore: bool) -> JsValue {
     if args.len() == 1 {
         if let Some(s) = args[0].as_str() {
             JsValue::Module(ModuleValue {
@@ -332,7 +336,9 @@ pub fn require(args: Vec<JsValue>) -> JsValue {
         } else {
             JsValue::unknown(
                 JsValue::call(
-                    Box::new(JsValue::WellKnownFunction(WellKnownFunctionKind::Require)),
+                    Box::new(JsValue::WellKnownFunction(WellKnownFunctionKind::Require {
+                        ignore,
+                    })),
                     args,
                 ),
                 true,
@@ -342,7 +348,9 @@ pub fn require(args: Vec<JsValue>) -> JsValue {
     } else {
         JsValue::unknown(
             JsValue::call(
-                Box::new(JsValue::WellKnownFunction(WellKnownFunctionKind::Require)),
+                Box::new(JsValue::WellKnownFunction(WellKnownFunctionKind::Require {
+                    ignore,
+                })),
                 args,
             ),
             true,
@@ -520,13 +528,13 @@ pub fn path_to_file_url(args: Vec<JsValue>) -> JsValue {
 
 pub fn well_known_function_member(kind: WellKnownFunctionKind, prop: JsValue) -> (JsValue, bool) {
     let new_value = match (kind, prop.as_str()) {
-        (WellKnownFunctionKind::Require, Some("resolve")) => {
+        (WellKnownFunctionKind::Require { .. }, Some("resolve")) => {
             JsValue::WellKnownFunction(WellKnownFunctionKind::RequireResolve)
         }
-        (WellKnownFunctionKind::Require, Some("cache")) => {
+        (WellKnownFunctionKind::Require { .. }, Some("cache")) => {
             JsValue::WellKnownObject(WellKnownObjectKind::RequireCache)
         }
-        (WellKnownFunctionKind::Require, Some("context")) => {
+        (WellKnownFunctionKind::Require { .. }, Some("context")) => {
             JsValue::WellKnownFunction(WellKnownFunctionKind::RequireContext)
         }
         (WellKnownFunctionKind::RequireContextRequire(val), Some("resolve")) => {

turbopack/crates/turbopack-ecmascript/src/lib.rs

@@ -364,7 +364,7 @@ impl EcmascriptParsable for EcmascriptModuleAsset {
 impl EcmascriptAnalyzable for EcmascriptModuleAsset {
     #[turbo_tasks::function]
     fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> {
-        analyse_ecmascript_module(self, None, None)
+        analyse_ecmascript_module(self, None)
     }
 
     /// Generates module contents without an analysis pass. This is useful for
@@ -417,6 +417,7 @@ impl EcmascriptModuleAsset {
     pub fn new(
         source: Vc<Box<dyn Source>>,
         asset_context: Vc<Box<dyn AssetContext>>,
+
         ty: Value<EcmascriptModuleAssetType>,
         transforms: Vc<EcmascriptInputTransforms>,
         options: Vc<EcmascriptOptions>,
@@ -428,6 +429,7 @@ impl EcmascriptModuleAsset {
             ty: ty.into_value(),
             transforms,
             options,
+
             compile_time_info,
             inner_assets: None,
             last_successful_parse: Default::default(),
@@ -440,6 +442,7 @@ impl EcmascriptModuleAsset {
         asset_context: Vc<Box<dyn AssetContext>>,
         ty: Value<EcmascriptModuleAssetType>,
         transforms: Vc<EcmascriptInputTransforms>,
+
         options: Vc<EcmascriptOptions>,
         compile_time_info: Vc<CompileTimeInfo>,
         inner_assets: Vc<InnerAssets>,
@@ -461,6 +464,11 @@ impl EcmascriptModuleAsset {
         Ok(self.await?.source)
     }
 
+    #[turbo_tasks::function]
+    pub fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult> {
+        analyse_ecmascript_module(self, None)
+    }
+
     #[turbo_tasks::function]
     pub async fn options(self: Vc<Self>) -> Result<Vc<EcmascriptOptions>> {
         Ok(self.await?.options)

turbopack/crates/turbopack-ecmascript/src/parse.rs

@@ -206,7 +206,7 @@ async fn parse_internal(
             FileContent::Content(file) => match file.content().to_str() {
                 Ok(string) => {
                     let transforms = &*transforms.await?;
-                    match parse_content(
+                    match parse_file_content(
                         string.into_owned(),
                         fs_path_vc,
                         fs_path,
@@ -246,7 +246,7 @@ async fn parse_internal(
     })
 }
 
-async fn parse_content(
+async fn parse_file_content(
     string: String,
     fs_path_vc: Vc<FileSystemPath>,
     fs_path: &FileSystemPath,
@@ -433,7 +433,7 @@ async fn parse_content(
                 &parsed_program,
                 unresolved_mark,
                 top_level_mark,
-                None,
+                Some(&comments),
                 Some(source),
             );
 

turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs

@@ -94,6 +94,10 @@ pub struct EsmAssetReference {
     pub origin: Vc<Box<dyn ResolveOrigin>>,
     pub request: Vc<Request>,
     pub annotations: ImportAnnotations,
+    /// True if the import should be ignored
+    /// This can happen for example when the webpackIgnore or turbopackIgnore
+    /// directives are present
+    pub ignore: bool,
     pub issue_source: Option<Vc<IssueSource>>,
     pub export_name: Option<Vc<ModulePart>>,
     pub import_externals: bool,
@@ -123,12 +127,14 @@ impl EsmAssetReference {
         annotations: Value<ImportAnnotations>,
         export_name: Option<Vc<ModulePart>>,
         import_externals: bool,
+        ignore: bool,
     ) -> Vc<Self> {
         Self::cell(EsmAssetReference {
             origin,
             request,
             issue_source,
             annotations: annotations.into_value(),
+            ignore,
             export_name,
             import_externals,
         })
@@ -144,6 +150,9 @@ impl EsmAssetReference {
 impl ModuleReference for EsmAssetReference {
     #[turbo_tasks::function]
     async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
+        if self.ignore {
+            return Ok(ModuleResolveResult::ignored().cell());
+        }
         let ty = if matches!(self.annotations.module_type(), Some("json")) {
             EcmaScriptModulesReferenceSubType::ImportWithType(ImportWithType::Json)
         } else if let Some(part) = &self.export_name {