fix node.js externals (vercel/turborepo#6530)

### Description

* Extend resolving to allow reference type for plugins
* add extensions for MainField
* add an option `import_externals` to context to specify externals
behavior for ESM imports
  * false: require() is used
  * true: import() is used and the module becomes an async module
* adds `import_externals: false` support for dynamic import
* adds `import_externals: true` support for normal import

see also #58129

### Testing Instructions

<!--
  Give a quick description of steps to test your changes.
-->


Closes PACK-2011

crates/turbopack-core/src/resolve/mod.rs

@@ -1008,33 +1008,40 @@ pub async fn resolve_raw(
 #[turbo_tasks::function]
 pub async fn resolve(
     lookup_path: Vc<FileSystemPath>,
+    reference_type: Value<ReferenceType>,
     request: Vc<Request>,
     options: Vc<ResolveOptions>,
 ) -> Result<Vc<ResolveResult>> {
     let raw_result = resolve_internal_inline(lookup_path, request, options)
         .await?
         .resolve()
         .await?;
-    let result = handle_resolve_plugins(lookup_path, request, options, raw_result).await?;
+    let result =
+        handle_resolve_plugins(lookup_path, reference_type, request, options, raw_result).await?;
     Ok(result)
 }
 
 async fn handle_resolve_plugins(
     lookup_path: Vc<FileSystemPath>,
+    reference_type: Value<ReferenceType>,
     request: Vc<Request>,
     options: Vc<ResolveOptions>,
     result: Vc<ResolveResult>,
 ) -> Result<Vc<ResolveResult>> {
     async fn apply_plugins_to_path(
         path: Vc<FileSystemPath>,
         lookup_path: Vc<FileSystemPath>,
+        reference_type: Value<ReferenceType>,
         request: Vc<Request>,
         options: Vc<ResolveOptions>,
     ) -> Result<Option<Vc<ResolveResult>>> {
         for plugin in &options.await?.plugins {
             let after_resolve_condition = plugin.after_resolve_condition().resolve().await?;
             if *after_resolve_condition.matches(path).await? {
-                if let Some(result) = *plugin.after_resolve(path, lookup_path, request).await? {
+                if let Some(result) = *plugin
+                    .after_resolve(path, lookup_path, reference_type.clone(), request)
+                    .await?
+                {
                     return Ok(Some(result));
                 }
             }
@@ -1052,7 +1059,8 @@ async fn handle_resolve_plugins(
         if let &ResolveResultItem::Source(source) = primary {
             let path = source.ident().path().resolve().await?;
             if let Some(new_result) =
-                apply_plugins_to_path(path, lookup_path, request, options).await?
+                apply_plugins_to_path(path, lookup_path, reference_type.clone(), request, options)
+                    .await?
             {
                 let new_result = new_result.await?;
                 changed = true;
@@ -1333,12 +1341,21 @@ async fn resolve_into_folder(
                 return resolve_internal_inline(package_path, request.resolve().await?, options)
                     .await;
             }
-            ResolveIntoPackage::MainField(name) => {
+            ResolveIntoPackage::MainField {
+                field: name,
+                extensions,
+            } => {
                 if let Some(package_json) = &*read_package_json(package_json_path).await? {
                     if let Some(field_value) = package_json[name].as_str() {
                         let request =
                             Request::parse(Value::new(normalize_request(field_value).into()));
 
+                        let options = if let Some(extensions) = extensions {
+                            options.with_extensions(extensions.clone())
+                        } else {
+                            options
+                        };
+
                         let result = &*resolve_internal_inline(package_path, request, options)
                             .await?
                             .await?;
@@ -1542,7 +1559,7 @@ async fn resolve_into_package(
     if could_match_others {
         for resolve_into_package in options_value.into_package.iter() {
             match resolve_into_package {
-                ResolveIntoPackage::Default(_) | ResolveIntoPackage::MainField(_) => {
+                ResolveIntoPackage::Default(_) | ResolveIntoPackage::MainField { .. } => {
                     // doesn't affect packages with subpath
                     if path.is_match("/") {
                         results.push(resolve_into_folder(package_path, options, query));

crates/turbopack-core/src/resolve/node.rs

@@ -13,8 +13,9 @@ pub fn node_cjs_resolve_options(root: Vc<FileSystemPath>) -> Vc<ResolveOptions>
         ("require".to_string(), ConditionValue::Set),
     ]
     .into();
+    let extensions = vec![".js".to_string(), ".json".to_string(), ".node".to_string()];
     ResolveOptions {
-        extensions: vec![".js".to_string(), ".json".to_string(), ".node".to_string()],
+        extensions,
         modules: vec![ResolveModules::Nested(
             root,
             vec!["node_modules".to_string()],
@@ -24,7 +25,10 @@ pub fn node_cjs_resolve_options(root: Vc<FileSystemPath>) -> Vc<ResolveOptions>
                 conditions: conditions.clone(),
                 unspecified_conditions: ConditionValue::Unset,
             },
-            ResolveIntoPackage::MainField("main".to_string()),
+            ResolveIntoPackage::MainField {
+                field: "main".to_string(),
+                extensions: None,
+            },
             ResolveIntoPackage::Default("index".to_string()),
         ],
         in_package: vec![ResolveInPackage::ImportsField {
@@ -43,6 +47,7 @@ pub fn node_esm_resolve_options(root: Vc<FileSystemPath>) -> Vc<ResolveOptions>
         ("import".to_string(), ConditionValue::Set),
     ]
     .into();
+    let extensions = vec![".js".to_string(), ".json".to_string(), ".node".to_string()];
     ResolveOptions {
         extensions: vec![],
         modules: vec![ResolveModules::Nested(
@@ -54,10 +59,11 @@ pub fn node_esm_resolve_options(root: Vc<FileSystemPath>) -> Vc<ResolveOptions>
                 conditions: conditions.clone(),
                 unspecified_conditions: ConditionValue::Unset,
             },
-            ResolveIntoPackage::MainField("main".to_string()),
-            ResolveIntoPackage::Default("index.js".to_string()),
-            ResolveIntoPackage::Default("index.json".to_string()),
-            ResolveIntoPackage::Default("index.node".to_string()),
+            ResolveIntoPackage::MainField {
+                field: "main".to_string(),
+                extensions: Some(extensions),
+            },
+            ResolveIntoPackage::Default("index".to_string()),
         ],
         in_package: vec![ResolveInPackage::ImportsField {
             conditions,

crates/turbopack-core/src/resolve/options.rs

@@ -67,7 +67,10 @@ pub enum ResolveIntoPackage {
     /// [main]: https://nodejs.org/api/packages.html#main
     /// [module]: https://esbuild.github.io/api/#main-fields
     /// [browser]: https://esbuild.github.io/api/#main-fields
-    MainField(String),
+    MainField {
+        field: String,
+        extensions: Option<Vec<String>>,
+    },
     /// Default behavior of using the index.js file at the root of the package.
     Default(String),
 }
@@ -455,6 +458,14 @@ impl ResolveOptions {
         );
         Ok(resolve_options.into())
     }
+
+    /// Overrides the extensions used for resolving
+    #[turbo_tasks::function]
+    pub async fn with_extensions(self: Vc<Self>, extensions: Vec<String>) -> Result<Vc<Self>> {
+        let mut resolve_options = self.await?.clone_value();
+        resolve_options.extensions = extensions;
+        Ok(resolve_options.into())
+    }
 }
 
 #[turbo_tasks::value(shared)]

crates/turbopack-core/src/resolve/plugin.rs

@@ -1,8 +1,11 @@
 use anyhow::Result;
-use turbo_tasks::Vc;
+use turbo_tasks::{Value, Vc};
 use turbo_tasks_fs::{glob::Glob, FileSystemPath};
 
-use crate::resolve::{parse::Request, ResolveResultOption};
+use crate::{
+    reference_type::ReferenceType,
+    resolve::{parse::Request, ResolveResultOption},
+};
 
 /// A condition which determines if the hooks of a resolve plugin gets called.
 #[turbo_tasks::value]
@@ -48,6 +51,7 @@ pub trait ResolvePlugin {
         self: Vc<Self>,
         fs_path: Vc<FileSystemPath>,
         lookup_path: Vc<FileSystemPath>,
+        reference_type: Value<ReferenceType>,
         request: Vc<Request>,
     ) -> Vc<ResolveResultOption>;
 }

crates/turbopack-dev/src/react_refresh.rs

@@ -1,9 +1,10 @@
 use anyhow::Result;
-use turbo_tasks::Vc;
+use turbo_tasks::{Value, Vc};
 use turbo_tasks_fs::FileSystemPath;
 use turbopack::resolve_options_context::ResolveOptionsContext;
 use turbopack_core::{
     issue::{Issue, IssueExt, IssueSeverity, StyledString},
+    reference_type::{CommonJsReferenceSubType, ReferenceType},
     resolve::parse::Request,
 };
 use turbopack_ecmascript::resolve::apply_cjs_specific_options;
@@ -49,8 +50,13 @@ pub async fn assert_can_resolve_react_refresh(
     let resolve_options =
         apply_cjs_specific_options(turbopack::resolve_options(path, resolve_options_context));
     for request in [react_refresh_request_in_next(), react_refresh_request()] {
-        let result =
-            turbopack_core::resolve::resolve(path, request, resolve_options).first_source();
+        let result = turbopack_core::resolve::resolve(
+            path,
+            Value::new(ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined)),
+            request,
+            resolve_options,
+        )
+        .first_source();
 
         if result.await?.is_some() {
             return Ok(ResolveReactRefreshResult::Found(request).cell());

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/runtime-backend-nodejs.ts

@@ -13,7 +13,10 @@ interface RequireContextEntry {
   external: boolean;
 }
 
-type ExternalRequire = (id: ModuleId) => Exports | EsmNamespaceObject;
+type ExternalRequire = (
+  id: ModuleId,
+  esm?: boolean
+) => Exports | EsmNamespaceObject;
 type ExternalImport = (id: ModuleId) => Promise<Exports | EsmNamespaceObject>;
 
 interface TurbopackDevContext extends TurbopackDevBaseContext {

crates/turbopack-ecmascript-runtime/js/src/shared-node/node-utils.ts

@@ -13,8 +13,23 @@ function commonJsRequireContext(
     : commonJsRequire(sourceModule, entry.id());
 }
 
-function externalImport(id: ModuleId) {
-  return import(id);
+async function externalImport(id: ModuleId) {
+  let raw;
+  try {
+    raw = await import(id);
+  } catch (err) {
+    // TODO(alexkirsz) This can happen when a client-side module tries to load
+    // an external module we don't provide a shim for (e.g. querystring, url).
+    // For now, we fail semi-silently, but in the future this should be a
+    // compilation error.
+    throw new Error(`Failed to load external module ${id}: ${err}`);
+  }
+
+  if (raw && raw.__esModule && raw.default && "default" in raw.default) {
+    return interopEsm(raw.default, {}, true);
+  }
+
+  return raw;
 }
 
 function externalRequire(

crates/turbopack-ecmascript/src/lib.rs

@@ -98,6 +98,9 @@ pub struct EcmascriptOptions {
     /// This allows to construct url depends on the different building context,
     /// e.g. SSR, CSR, or Node.js.
     pub url_rewrite_behavior: Option<UrlRewriteBehavior>,
+    /// External imports should used `__turbopack_import__` instead of
+    /// `__turbopack_require__` and become async module references.
+    pub import_externals: bool,
 }
 
 #[turbo_tasks::value(serialization = "auto_for_input")]
@@ -175,7 +178,11 @@ impl EcmascriptModuleAssetBuilder {
             )
         };
         if let Some(part) = self.part {
-            Vc::upcast(EcmascriptModulePartAsset::new(base, part))
+            Vc::upcast(EcmascriptModulePartAsset::new(
+                base,
+                part,
+                self.options.import_externals,
+            ))
         } else {
             Vc::upcast(base)
         }

crates/turbopack-ecmascript/src/references/async_module.rs

@@ -6,7 +6,7 @@ use swc_core::{
     ecma::ast::{ArrayLit, ArrayPat, Expr, Ident, Pat, Program},
     quote,
 };
-use turbo_tasks::{trace::TraceRawVcs, TryFlatJoinIterExt, Vc};
+use turbo_tasks::{trace::TraceRawVcs, TryFlatJoinIterExt, TryJoinIterExt, Vc};
 use turbopack_core::chunk::{AsyncModuleInfo, ChunkableModule};
 
 use super::esm::base::ReferencedAsset;
@@ -46,6 +46,7 @@ pub struct AsyncModule {
     pub placeable: Vc<Box<dyn EcmascriptChunkPlaceable>>,
     pub references: IndexSet<Vc<EsmAssetReference>>,
     pub has_top_level_await: bool,
+    pub import_externals: bool,
 }
 
 /// Option<[AsyncModule]>.
@@ -80,22 +81,24 @@ struct AsyncModuleIdents(IndexSet<String>);
 impl AsyncModule {
     #[turbo_tasks::function]
     async fn get_async_idents(
-        self: Vc<Self>,
+        &self,
         chunking_context: Vc<Box<dyn EcmascriptChunkingContext>>,
         async_module_info: Vc<AsyncModuleInfo>,
     ) -> Result<Vc<AsyncModuleIdents>> {
-        let this = self.await?;
         let async_module_info = async_module_info.await?;
 
-        let reference_idents = this
+        let reference_idents = self
             .references
             .iter()
             .map(|r| async {
                 let referenced_asset = r.get_referenced_asset().await?;
                 Ok(match &*referenced_asset {
                     ReferencedAsset::OriginalReferenceTypeExternal(_) => {
-                        // TODO(WEB-1259): we need to detect if external modules are esm
-                        None
+                        if self.import_externals {
+                            referenced_asset.get_ident().await?
+                        } else {
+                            None
+                        }
                     }
                     ReferencedAsset::Some(placeable) => {
                         let chunk_item = placeable
@@ -121,8 +124,28 @@ impl AsyncModule {
     }
 
     #[turbo_tasks::function]
-    pub(crate) fn is_self_async(&self) -> Vc<bool> {
-        Vc::cell(self.has_top_level_await)
+    pub(crate) async fn is_self_async(&self) -> Result<Vc<bool>> {
+        if self.has_top_level_await {
+            return Ok(Vc::cell(true));
+        }
+
+        Ok(Vc::cell(
+            self.import_externals
+                && self
+                    .references
+                    .iter()
+                    .map(|r| async {
+                        let referenced_asset = r.get_referenced_asset().await?;
+                        Ok(matches!(
+                            &*referenced_asset,
+                            ReferencedAsset::OriginalReferenceTypeExternal(_)
+                        ))
+                    })
+                    .try_join()
+                    .await?
+                    .iter()
+                    .any(|&b| b),
+        ))
     }
 
     /// Returns