From 9d5f62eb4a0686cdf47de60440f6f7dda38808d5 Mon Sep 17 00:00:00 2001 From: OJ Kwon <1210596+kwonoj@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:42:28 -0800 Subject: [PATCH] refactor(analysis): rust based page-static-info, deprecate js parse interface in next-swc (#59300) ### Description This PR refactors existing `analysis/get-static-page-info`, moves over most of parse / ast visiting logic into next-swc's rust codebase. By having this, turbopack can reuse same logic to extract info for the analysis. Also as a side effect, this removes JS side parse which is known to be inefficient due to serialization / deserialization. The entrypoint `getPageStaticInfo` is still in the existing `get-page-static-info`, only for extracting / visiting logic is moved. There are some JS specific context to postprocess extracted information which would require additional effort to move into. Closes PACK-2088 --- Cargo.lock | 3 + Cargo.toml | 2 +- packages/next-swc/crates/napi/Cargo.toml | 1 + packages/next-swc/crates/napi/src/parse.rs | 283 +++++++++---- .../crates/next-custom-transforms/Cargo.toml | 1 + .../src/transforms/mod.rs | 1 + .../collect_exported_const_visitor.rs | 210 ++++++++++ .../collect_exports_visitor.rs | 183 +++++++++ .../src/transforms/page_static_info/mod.rs | 375 ++++++++++++++++++ packages/next-swc/crates/wasm/Cargo.toml | 4 +- packages/next-swc/crates/wasm/src/lib.rs | 170 +++++--- .../src/build/analysis/extract-const-value.ts | 249 ------------ .../build/analysis/get-page-static-info.ts | 274 +------------ .../next/src/build/analysis/parse-module.ts | 15 - packages/next/src/build/index.ts | 14 +- packages/next/src/build/swc/index.ts | 145 ++++++- .../middleware-errors/index.test.ts | 2 + .../index.test.ts | 10 +- 18 files changed, 1258 insertions(+), 684 deletions(-) create mode 100644 packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exported_const_visitor.rs create mode 100644 packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exports_visitor.rs create mode 100644 packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/mod.rs delete mode 100644 packages/next/src/build/analysis/extract-const-value.ts delete mode 100644 packages/next/src/build/analysis/parse-module.ts diff --git a/Cargo.lock b/Cargo.lock index 5c6b2911706b9..e7cedcccd13b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3450,6 +3450,7 @@ dependencies = [ "either", "fxhash", "hex", + "lazy_static", "once_cell", "pathdiff", "react_remove_properties", @@ -3481,6 +3482,7 @@ dependencies = [ "next-core", "next-custom-transforms", "once_cell", + "regex", "serde", "serde_json", "shadow-rs", @@ -8635,6 +8637,7 @@ dependencies = [ "once_cell", "parking_lot_core 0.8.0", "path-clean", + "regex", "serde", "serde-wasm-bindgen", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index daa7eed03fb7a..a8e9becdfe27e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ "packages/next-swc/crates/next-api", "packages/next-swc/crates/next-build", "packages/next-swc/crates/next-core", - "packages/next-swc/crates/next-custom-transforms", + "packages/next-swc/crates/next-custom-transforms" ] [workspace.lints.clippy] diff --git a/packages/next-swc/crates/napi/Cargo.toml b/packages/next-swc/crates/napi/Cargo.toml index d5c60f13fe582..d518551032fe1 100644 --- a/packages/next-swc/crates/napi/Cargo.toml +++ b/packages/next-swc/crates/napi/Cargo.toml @@ -76,6 +76,7 @@ next-build = { workspace = true } next-core = { workspace = true } turbo-tasks = { workspace = true } once_cell = { workspace = true } +regex = "1.5" serde = "1" serde_json = "1" shadow-rs = { workspace = true } diff --git a/packages/next-swc/crates/napi/src/parse.rs b/packages/next-swc/crates/napi/src/parse.rs index 0df7489849ff8..c231ffc49fa41 100644 --- a/packages/next-swc/crates/napi/src/parse.rs +++ b/packages/next-swc/crates/napi/src/parse.rs @@ -1,92 +1,233 @@ -use std::sync::Arc; +use std::collections::HashMap; use anyhow::Context as _; use napi::bindgen_prelude::*; -use turbopack_binding::swc::core::{ - base::{config::ParseOptions, try_with_handler}, - common::{ - comments::Comments, errors::ColorConfig, FileName, FilePathMapping, SourceMap, GLOBALS, - }, +use next_custom_transforms::transforms::page_static_info::{ + build_ast_from_source, collect_exports, collect_rsc_module_info, extract_expored_const_values, + Const, ExportInfo, ExportInfoWarning, RscModuleInfo, }; +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::Value; use crate::util::MapErr; -pub struct ParseTask { - pub filename: FileName, - pub src: String, - pub options: Buffer, +/// wrap read file to suppress errors conditionally. +/// [NOTE] currently next.js passes _every_ file in the paths regardless of if +/// it's an asset or an ecmascript, So skipping non-utf8 read errors. Probably +/// should skip based on file extension. +fn read_file_wrapped_err(path: &str, raise_err: bool) -> Result { + let buf = std::fs::read(path).map_err(|e| { + napi::Error::new( + Status::GenericFailure, + format!("Next.js ERROR: Failed to read file {}:\n{:#?}", path, e), + ) + }); + + match buf { + Ok(buf) => Ok(String::from_utf8(buf).ok().unwrap_or("".to_string())), + Err(e) if raise_err => Err(e), + _ => Ok("".to_string()), + } +} + +/// A regex pattern to determine if is_dynamic_metadata_route should continue to +/// parse the page or short circuit and return false. +static DYNAMIC_METADATA_ROUTE_SHORT_CURCUIT: Lazy = + Lazy::new(|| Regex::new("generateImageMetadata|generateSitemaps").unwrap()); + +/// A regex pattern to determine if get_page_static_info should continue to +/// parse the page or short circuit and return default. +static PAGE_STATIC_INFO_SHORT_CURCUIT: Lazy = Lazy::new(|| { + Regex::new( + r#"runtime|preferredRegion|getStaticProps|getServerSideProps|generateStaticParams|export const"#, + ) + .unwrap() +}); + +pub struct DetectMetadataRouteTask { + page_file_path: String, + file_content: Option, } #[napi] -impl Task for ParseTask { - type Output = String; - type JsValue = String; +impl Task for DetectMetadataRouteTask { + type Output = Option; + type JsValue = Object; fn compute(&mut self) -> napi::Result { - GLOBALS.set(&Default::default(), || { - let c = turbopack_binding::swc::core::base::Compiler::new(Arc::new(SourceMap::new( - FilePathMapping::empty(), - ))); - - let options: ParseOptions = serde_json::from_slice(self.options.as_ref())?; - let comments = c.comments().clone(); - let comments: Option<&dyn Comments> = if options.comments { - Some(&comments) - } else { - None - }; - let fm = - c.cm.new_source_file(self.filename.clone(), self.src.clone()); - let program = try_with_handler( - c.cm.clone(), - turbopack_binding::swc::core::base::HandlerOpts { - color: ColorConfig::Never, - skip_filename: false, - }, - |handler| { - c.parse_js( - fm, - handler, - options.target, - options.syntax, - options.is_module, - comments, - ) - }, - ) - .convert_err()?; - - let ast_json = serde_json::to_string(&program) - .context("failed to serialize Program") - .convert_err()?; + let file_content = if let Some(file_content) = &self.file_content { + file_content.clone() + } else { + read_file_wrapped_err(self.page_file_path.as_str(), true)? + }; + + if !DYNAMIC_METADATA_ROUTE_SHORT_CURCUIT.is_match(file_content.as_str()) { + return Ok(None); + } + + let (source_ast, _) = build_ast_from_source(&file_content, &self.page_file_path)?; + collect_exports(&source_ast).convert_err() + } + + fn resolve(&mut self, env: Env, exports_info: Self::Output) -> napi::Result { + let mut ret = env.create_object()?; + + let mut warnings = env.create_array(0)?; - Ok(ast_json) - }) + match exports_info { + Some(exports_info) => { + let is_dynamic_metadata_route = + !exports_info.generate_image_metadata.unwrap_or_default() + || !exports_info.generate_sitemaps.unwrap_or_default(); + ret.set_named_property( + "isDynamicMetadataRoute", + env.get_boolean(is_dynamic_metadata_route), + )?; + + for ExportInfoWarning { key, message } in exports_info.warnings { + let mut warning_obj = env.create_object()?; + warning_obj.set_named_property("key", env.create_string(&key)?)?; + warning_obj.set_named_property("message", env.create_string(&message)?)?; + warnings.insert(warning_obj)?; + } + ret.set_named_property("warnings", warnings)?; + } + None => { + ret.set_named_property("warnings", warnings)?; + ret.set_named_property("isDynamicMetadataRoute", env.get_boolean(false))?; + } + } + + Ok(ret) + } +} + +/// Detect if metadata routes is a dynamic route, which containing +/// generateImageMetadata or generateSitemaps as export +#[napi] +pub fn is_dynamic_metadata_route( + page_file_path: String, + file_content: Option, +) -> AsyncTask { + AsyncTask::new(DetectMetadataRouteTask { + page_file_path, + file_content, + }) +} + +#[napi(object, object_to_js = false)] +pub struct CollectPageStaticInfoOption { + pub page_file_path: String, + pub is_dev: Option, + pub page: Option, + pub page_type: String, //'pages' | 'app' | 'root' +} + +pub struct CollectPageStaticInfoTask { + option: CollectPageStaticInfoOption, + file_content: Option, +} + +#[napi] +impl Task for CollectPageStaticInfoTask { + type Output = Option<( + ExportInfo, + HashMap, + RscModuleInfo, + Vec, + )>; + type JsValue = Option; + + fn compute(&mut self) -> napi::Result { + let CollectPageStaticInfoOption { + page_file_path, + is_dev, + .. + } = &self.option; + let file_content = if let Some(file_content) = &self.file_content { + file_content.clone() + } else { + read_file_wrapped_err(page_file_path.as_str(), !is_dev.unwrap_or_default())? + }; + + if !PAGE_STATIC_INFO_SHORT_CURCUIT.is_match(file_content.as_str()) { + return Ok(None); + } + + let (source_ast, comments) = build_ast_from_source(&file_content, page_file_path)?; + let exports_info = collect_exports(&source_ast)?; + match exports_info { + None => Ok(None), + Some(exports_info) => { + let rsc_info = collect_rsc_module_info(&comments, true); + + let mut properties_to_extract = exports_info.extra_properties.clone(); + properties_to_extract.insert("config".to_string()); + + let mut exported_const_values = + extract_expored_const_values(&source_ast, properties_to_extract); + + let mut extracted_values = HashMap::new(); + let mut warnings = vec![]; + + for (key, value) in exported_const_values.drain() { + match value { + Some(Const::Value(v)) => { + extracted_values.insert(key.clone(), v); + } + Some(Const::Unsupported(msg)) => { + warnings.push(msg); + } + _ => {} + } + } + + Ok(Some((exports_info, extracted_values, rsc_info, warnings))) + } + } } fn resolve(&mut self, _env: Env, result: Self::Output) -> napi::Result { - Ok(result) + if let Some((exports_info, extracted_values, rsc_info, warnings)) = result { + // [TODO] this is stopgap; there are some non n-api serializable types in the + // nested result. However, this is still much smaller than passing whole ast. + // Should go away once all of logics in the getPageStaticInfo is internalized. + let ret = StaticPageInfo { + exports_info: Some(exports_info), + extracted_values, + rsc_info: Some(rsc_info), + warnings, + }; + + let ret = serde_json::to_string(&ret) + .context("failed to serialize static info result") + .convert_err()?; + + Ok(Some(ret)) + } else { + Ok(None) + } } } +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StaticPageInfo { + pub exports_info: Option, + pub extracted_values: HashMap, + pub rsc_info: Option, + pub warnings: Vec, +} + #[napi] -pub fn parse( - src: String, - options: Buffer, - filename: Option, - signal: Option, -) -> AsyncTask { - let filename = if let Some(value) = filename { - FileName::Real(value.into()) - } else { - FileName::Anon - }; - AsyncTask::with_optional_signal( - ParseTask { - filename, - src, - options, - }, - signal, - ) +pub fn get_page_static_info( + option: CollectPageStaticInfoOption, + file_content: Option, +) -> AsyncTask { + AsyncTask::new(CollectPageStaticInfoTask { + option, + file_content, + }) } diff --git a/packages/next-swc/crates/next-custom-transforms/Cargo.toml b/packages/next-swc/crates/next-custom-transforms/Cargo.toml index 4fc06476ab63e..bcee089770163 100644 --- a/packages/next-swc/crates/next-custom-transforms/Cargo.toml +++ b/packages/next-swc/crates/next-custom-transforms/Cargo.toml @@ -26,6 +26,7 @@ serde_json = { workspace = true, features = ["preserve_order"] } sha1 = "0.10.1" tracing = { version = "0.1.37" } anyhow = { workspace = true } +lazy_static = { workspace = true } turbopack-binding = { workspace = true, features = [ "__swc_core", diff --git a/packages/next-swc/crates/next-custom-transforms/src/transforms/mod.rs b/packages/next-swc/crates/next-custom-transforms/src/transforms/mod.rs index b35d569825461..f3c62e7c1d398 100644 --- a/packages/next-swc/crates/next-custom-transforms/src/transforms/mod.rs +++ b/packages/next-swc/crates/next-custom-transforms/src/transforms/mod.rs @@ -8,6 +8,7 @@ pub mod import_analyzer; pub mod next_ssg; pub mod optimize_server_react; pub mod page_config; +pub mod page_static_info; pub mod pure; pub mod react_server_components; pub mod server_actions; diff --git a/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exported_const_visitor.rs b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exported_const_visitor.rs new file mode 100644 index 0000000000000..999db1e8de656 --- /dev/null +++ b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exported_const_visitor.rs @@ -0,0 +1,210 @@ +use std::collections::{HashMap, HashSet}; + +use serde_json::{Map, Number, Value}; +use swc_core::{ + common::{pass::AstNodePath, Mark, SyntaxContext}, + ecma::{ + ast::{ + BindingIdent, Decl, ExportDecl, Expr, Lit, Pat, Prop, PropName, PropOrSpread, VarDecl, + VarDeclKind, VarDeclarator, + }, + utils::{ExprCtx, ExprExt}, + visit::{AstParentNodeRef, VisitAstPath, VisitWithPath}, + }, +}; + +/// The values extracted for the corresponding AST node. +/// refer extract_expored_const_values for the supported value types. +/// Undefined / null is treated as None. +pub enum Const { + Value(Value), + Unsupported(String), +} + +pub(crate) struct CollectExportedConstVisitor { + pub properties: HashMap>, + expr_ctx: ExprCtx, +} + +impl CollectExportedConstVisitor { + pub fn new(properties_to_extract: HashSet) -> Self { + Self { + properties: properties_to_extract + .into_iter() + .map(|p| (p, None)) + .collect(), + expr_ctx: ExprCtx { + unresolved_ctxt: SyntaxContext::empty().apply_mark(Mark::new()), + is_unresolved_ref_safe: false, + }, + } + } +} + +impl VisitAstPath for CollectExportedConstVisitor { + fn visit_export_decl<'ast: 'r, 'r>( + &mut self, + export_decl: &'ast ExportDecl, + ast_path: &mut AstNodePath>, + ) { + match &export_decl.decl { + Decl::Var(box VarDecl { kind, decls, .. }) if kind == &VarDeclKind::Const => { + for decl in decls { + if let VarDeclarator { + name: Pat::Ident(BindingIdent { id, .. }), + init: Some(init), + .. + } = decl + { + let id = id.sym.as_ref(); + if let Some(prop) = self.properties.get_mut(id) { + *prop = extract_value(&self.expr_ctx, init, id.to_string()); + }; + } + } + } + _ => {} + } + + export_decl.visit_children_with_path(self, ast_path); + } +} + +/// Coerece the actual value of the given ast node. +fn extract_value(ctx: &ExprCtx, init: &Expr, id: String) -> Option { + match init { + init if init.is_undefined(ctx) => Some(Const::Value(Value::Null)), + Expr::Ident(ident) => Some(Const::Unsupported(format!( + "Unknown identifier \"{}\" at \"{}\".", + ident.sym, id + ))), + Expr::Lit(lit) => match lit { + Lit::Num(num) => Some(Const::Value(Value::Number( + Number::from_f64(num.value).expect("Should able to convert f64 to Number"), + ))), + Lit::Null(_) => Some(Const::Value(Value::Null)), + Lit::Str(s) => Some(Const::Value(Value::String(s.value.to_string()))), + Lit::Bool(b) => Some(Const::Value(Value::Bool(b.value))), + Lit::Regex(r) => Some(Const::Value(Value::String(format!( + "/{}/{}", + r.exp, r.flags + )))), + _ => Some(Const::Unsupported("Unsupported Literal".to_string())), + }, + Expr::Array(arr) => { + let mut a = vec![]; + + for elem in &arr.elems { + match elem { + Some(elem) => { + if elem.spread.is_some() { + return Some(Const::Unsupported(format!( + "Unsupported spread operator in the Array Expression at \"{}\"", + id + ))); + } + + match extract_value(ctx, &elem.expr, id.clone()) { + Some(Const::Value(value)) => a.push(value), + _ => { + return Some(Const::Unsupported( + "Unsupported value in the Array Expression".to_string(), + )) + } + } + } + None => { + a.push(Value::Null); + } + } + } + + Some(Const::Value(Value::Array(a))) + } + Expr::Object(obj) => { + let mut o = Map::new(); + + for prop in &obj.props { + let kv = match prop { + PropOrSpread::Prop(box Prop::KeyValue(kv)) => match kv.key { + PropName::Ident(_) | PropName::Str(_) => kv, + _ => { + return Some(Const::Unsupported(format!( + "Unsupported key type in the Object Expression at \"{}\"", + id + ))) + } + }, + _ => { + return Some(Const::Unsupported(format!( + "Unsupported spread operator in the Object Expression at \"{}\"", + id + ))) + } + }; + + let key = match &kv.key { + PropName::Ident(i) => i.sym.as_ref(), + PropName::Str(s) => s.value.as_ref(), + _ => { + return Some(Const::Unsupported(format!( + "Unsupported key type \"{:#?}\" in the Object Expression", + kv.key + ))) + } + }; + let new_value = extract_value(ctx, &kv.value, format!("{}.{}", id, key)); + if let Some(Const::Unsupported(msg)) = new_value { + return Some(Const::Unsupported(msg)); + } + + if let Some(Const::Value(value)) = new_value { + o.insert(key.to_string(), value); + } + } + + Some(Const::Value(Value::Object(o))) + } + Expr::Tpl(tpl) => { + // [TODO] should we add support for `${'e'}d${'g'}'e'`? + if !tpl.exprs.is_empty() { + Some(Const::Unsupported(format!( + "Unsupported template literal with expressions at \"{}\".", + id + ))) + } else { + Some( + tpl.quasis + .first() + .map(|q| { + // When TemplateLiteral has 0 expressions, the length of quasis is + // always 1. Because when parsing + // TemplateLiteral, the parser yields the first quasi, + // then the first expression, then the next quasi, then the next + // expression, etc., until the last quasi. + // Thus if there is no expression, the parser ends at the frst and also + // last quasis + // + // A "cooked" interpretation where backslashes have special meaning, + // while a "raw" interpretation where + // backslashes do not have special meaning https://exploringjs.com/impatient-js/ch_template-literals.html#template-strings-cooked-vs-raw + let cooked = q.cooked.as_ref(); + let raw = q.raw.as_ref(); + + Const::Value(Value::String( + cooked.map(|c| c.to_string()).unwrap_or(raw.to_string()), + )) + }) + .unwrap_or(Const::Unsupported(format!( + "Unsupported node type at \"{}\"", + id + ))), + ) + } + } + _ => Some(Const::Unsupported(format!( + "Unsupported node type at \"{}\"", + id + ))), + } +} diff --git a/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exports_visitor.rs b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exports_visitor.rs new file mode 100644 index 0000000000000..444dc79829526 --- /dev/null +++ b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/collect_exports_visitor.rs @@ -0,0 +1,183 @@ +use std::collections::HashSet; + +use lazy_static::lazy_static; +use swc_core::ecma::{ + ast::{ + Decl, ExportDecl, ExportNamedSpecifier, ExportSpecifier, Expr, ExprOrSpread, ExprStmt, Lit, + ModuleExportName, ModuleItem, NamedExport, Pat, Stmt, Str, VarDeclarator, + }, + visit::{Visit, VisitWith}, +}; + +use super::{ExportInfo, ExportInfoWarning}; + +lazy_static! { + static ref EXPORTS_SET: HashSet<&'static str> = HashSet::from([ + "getStaticProps", + "getServerSideProps", + "generateImageMetadata", + "generateSitemaps", + "generateStaticParams", + ]); +} + +pub(crate) struct CollectExportsVisitor { + pub export_info: Option, +} + +impl CollectExportsVisitor { + pub fn new() -> Self { + Self { + export_info: Default::default(), + } + } +} + +impl Visit for CollectExportsVisitor { + fn visit_module_items(&mut self, stmts: &[swc_core::ecma::ast::ModuleItem]) { + for stmt in stmts { + if let ModuleItem::Stmt(Stmt::Expr(ExprStmt { + expr: box Expr::Lit(Lit::Str(Str { value, .. })), + .. + })) = stmt + { + if value == "use server" { + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.directives.insert("server".to_string()); + } + if value == "use client" { + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.directives.insert("client".to_string()); + } + } + + stmt.visit_children_with(self); + } + } + + fn visit_export_decl(&mut self, export_decl: &ExportDecl) { + match &export_decl.decl { + Decl::Var(box var_decl) => { + if let Some(VarDeclarator { + name: Pat::Ident(name), + .. + }) = var_decl.decls.first() + { + if EXPORTS_SET.contains(&name.sym.as_str()) { + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.ssg = name.sym == "getStaticProps"; + export_info.ssr = name.sym == "getServerSideProps"; + export_info.generate_image_metadata = + Some(name.sym == "generateImageMetadata"); + export_info.generate_sitemaps = Some(name.sym == "generateSitemaps"); + export_info.generate_static_params = name.sym == "generateStaticParams"; + } + } + + for decl in &var_decl.decls { + if let Pat::Ident(id) = &decl.name { + if id.sym == "runtime" { + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.runtime = decl.init.as_ref().and_then(|init| { + if let Expr::Lit(Lit::Str(Str { value, .. })) = &**init { + Some(value.to_string()) + } else { + None + } + }) + } else if id.sym == "preferredRegion" { + if let Some(init) = &decl.init { + if let Expr::Array(arr) = &**init { + for expr in arr.elems.iter().flatten() { + if let ExprOrSpread { + expr: box Expr::Lit(Lit::Str(Str { value, .. })), + .. + } = expr + { + let export_info = + self.export_info.get_or_insert(Default::default()); + export_info.preferred_region.push(value.to_string()); + } + } + } else if let Expr::Lit(Lit::Str(Str { value, .. })) = &**init { + let export_info = + self.export_info.get_or_insert(Default::default()); + export_info.preferred_region.push(value.to_string()); + } + } + } else { + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.extra_properties.insert(id.sym.to_string()); + } + } + } + } + Decl::Fn(fn_decl) => { + let id = &fn_decl.ident; + + let export_info = self.export_info.get_or_insert(Default::default()); + export_info.ssg = id.sym == "getStaticProps"; + export_info.ssr = id.sym == "getServerSideProps"; + export_info.generate_image_metadata = Some(id.sym == "generateImageMetadata"); + export_info.generate_sitemaps = Some(id.sym == "generateSitemaps"); + export_info.generate_static_params = id.sym == "generateStaticParams"; + } + _ => {} + } + + export_decl.visit_children_with(self); + } + + fn visit_named_export(&mut self, named_export: &NamedExport) { + for specifier in &named_export.specifiers { + if let ExportSpecifier::Named(ExportNamedSpecifier { + orig: ModuleExportName::Ident(value), + .. + }) = specifier + { + let export_info = self.export_info.get_or_insert(Default::default()); + + if !export_info.ssg && value.sym == "getStaticProps" { + export_info.ssg = true; + } + + if !export_info.ssr && value.sym == "getServerSideProps" { + export_info.ssr = true; + } + + if !export_info.generate_image_metadata.unwrap_or_default() + && value.sym == "generateImageMetadata" + { + export_info.generate_image_metadata = Some(true); + } + + if !export_info.generate_sitemaps.unwrap_or_default() + && value.sym == "generateSitemaps" + { + export_info.generate_sitemaps = Some(true); + } + + if !export_info.generate_static_params && value.sym == "generateStaticParams" { + export_info.generate_static_params = true; + } + + if export_info.runtime.is_none() && value.sym == "runtime" { + export_info.warnings.push(ExportInfoWarning::new( + value.sym.to_string(), + "it was not assigned to a string literal".to_string(), + )); + } + + if export_info.preferred_region.is_empty() && value.sym == "preferredRegion" { + export_info.warnings.push(ExportInfoWarning::new( + value.sym.to_string(), + "it was not assigned to a string literal or an array of string literals" + .to_string(), + )); + } + } + } + + named_export.visit_children_with(self); + } +} diff --git a/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/mod.rs b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/mod.rs new file mode 100644 index 0000000000000..031ac76e2b2d9 --- /dev/null +++ b/packages/next-swc/crates/next-custom-transforms/src/transforms/page_static_info/mod.rs @@ -0,0 +1,375 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::Arc, +}; + +use anyhow::Result; +pub use collect_exported_const_visitor::Const; +use collect_exports_visitor::CollectExportsVisitor; +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use swc_core::{ + base::{ + config::{IsModule, ParseOptions}, + try_with_handler, Compiler, HandlerOpts, SwcComments, + }, + common::{errors::ColorConfig, FilePathMapping, SourceMap, GLOBALS}, + ecma::{ + ast::Program, + parser::{EsConfig, Syntax, TsConfig}, + visit::{VisitWith, VisitWithPath}, + }, +}; + +pub mod collect_exported_const_visitor; +pub mod collect_exports_visitor; + +/// Parse given contents of the file as ecmascript via swc's parser. +/// [NOTE] this is being used outside of turbopack (next.js's analysis phase) +/// currently, so we can't use turbopack-ecmascript's parse. +pub fn build_ast_from_source(contents: &str, file_path: &str) -> Result<(Program, SwcComments)> { + GLOBALS.set(&Default::default(), || { + let c = Compiler::new(Arc::new(SourceMap::new(FilePathMapping::empty()))); + + let options = ParseOptions { + is_module: IsModule::Unknown, + syntax: if file_path.ends_with(".ts") || file_path.ends_with(".tsx") { + Syntax::Typescript(TsConfig { + tsx: true, + decorators: true, + ..Default::default() + }) + } else { + Syntax::Es(EsConfig { + jsx: true, + decorators: true, + ..Default::default() + }) + }, + ..Default::default() + }; + + let fm = c.cm.new_source_file( + swc_core::common::FileName::Real(PathBuf::from(file_path.to_string())), + contents.to_string(), + ); + + let comments = c.comments().clone(); + + try_with_handler( + c.cm.clone(), + HandlerOpts { + color: ColorConfig::Never, + skip_filename: false, + }, + |handler| { + c.parse_js( + fm, + handler, + options.target, + options.syntax, + options.is_module, + Some(&comments), + ) + }, + ) + .map(|p| (p, comments)) + }) +} + +#[derive(Debug, Default)] +pub struct MiddlewareConfig {} + +#[derive(Debug)] +pub enum Amp { + Boolean(bool), + Hybrid, +} + +#[derive(Debug, Default)] +pub struct PageStaticInfo { + // [TODO] next-core have NextRuntime type, but the order of dependency won't allow to import + // Since this value is being passed into JS context anyway, we can just use string for now. + pub runtime: Option, // 'nodejs' | 'experimental-edge' | 'edge' + pub preferred_region: Vec, + pub ssg: Option, + pub ssr: Option, + pub rsc: Option, // 'server' | 'client' + pub generate_static_params: Option, + pub middleware: Option, + pub amp: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportInfoWarning { + pub key: String, + pub message: String, +} + +impl ExportInfoWarning { + pub fn new(key: String, message: String) -> Self { + Self { key, message } + } +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExportInfo { + pub ssr: bool, + pub ssg: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub preferred_region: Vec, + pub generate_image_metadata: Option, + pub generate_sitemaps: Option, + pub generate_static_params: bool, + pub extra_properties: HashSet, + pub directives: HashSet, + /// extra properties to bubble up warning messages from visitor, + /// since this isn't a failure to abort the process. + pub warnings: Vec, +} + +/// Collects static page export information for the next.js from given source's +/// AST. This is being used for some places like detecting page +/// is a dynamic route or not, or building a PageStaticInfo object. +pub fn collect_exports(program: &Program) -> Result> { + let mut collect_export_visitor = CollectExportsVisitor::new(); + program.visit_with(&mut collect_export_visitor); + + Ok(collect_export_visitor.export_info) +} + +static CLIENT_MODULE_LABEL: Lazy = Lazy::new(|| { + Regex::new(" __next_internal_client_entry_do_not_use__ ([^ ]*) (cjs|auto) ").unwrap() +}); +static ACTION_MODULE_LABEL: Lazy = + Lazy::new(|| Regex::new(r#" __next_internal_action_entry_do_not_use__ (\{[^}]+\}) "#).unwrap()); + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RscModuleInfo { + #[serde(rename = "type")] + pub module_type: String, + pub actions: Option>, + pub is_client_ref: bool, + pub client_refs: Option>, + pub client_entry_type: Option, +} + +impl RscModuleInfo { + pub fn new(module_type: String) -> Self { + Self { + module_type, + actions: None, + is_client_ref: false, + client_refs: None, + client_entry_type: None, + } + } +} + +/// Parse comments from the given source code and collect the RSC module info. +/// This doesn't use visitor, only read comments to parse necessary information. +pub fn collect_rsc_module_info( + comments: &SwcComments, + is_react_server_layer: bool, +) -> RscModuleInfo { + let mut captured = None; + + for comment in comments.leading.iter() { + let parsed = comment.iter().find_map(|c| { + let actions_json = ACTION_MODULE_LABEL.captures(&c.text); + let client_info_match = CLIENT_MODULE_LABEL.captures(&c.text); + + if actions_json.is_none() && client_info_match.is_none() { + return None; + } + + let actions = if let Some(actions_json) = actions_json { + if let Ok(serde_json::Value::Object(map)) = + serde_json::from_str::(&actions_json[1]) + { + Some( + map.iter() + // values for the action json should be a string + .map(|(_, v)| v.as_str().unwrap_or_default().to_string()) + .collect::>(), + ) + } else { + None + } + } else { + None + }; + + let is_client_ref = client_info_match.is_some(); + let client_info = client_info_match.map(|client_info_match| { + ( + client_info_match[1] + .split(',') + .map(|s| s.to_string()) + .collect::>(), + client_info_match[2].to_string(), + ) + }); + + Some((actions, is_client_ref, client_info)) + }); + + if captured.is_none() { + captured = parsed; + break; + } + } + + match captured { + Some((actions, is_client_ref, client_info)) => { + if !is_react_server_layer { + let mut module_info = RscModuleInfo::new("client".to_string()); + module_info.actions = actions; + module_info.is_client_ref = is_client_ref; + module_info + } else { + let mut module_info = RscModuleInfo::new(if client_info.is_some() { + "client".to_string() + } else { + "server".to_string() + }); + module_info.actions = actions; + module_info.is_client_ref = is_client_ref; + if let Some((client_refs, client_entry_type)) = client_info { + module_info.client_refs = Some(client_refs); + module_info.client_entry_type = Some(client_entry_type); + } + + module_info + } + } + None => RscModuleInfo::new(if !is_react_server_layer { + "client".to_string() + } else { + "server".to_string() + }), + } +} + +/// Extracts the value of an exported const variable named `exportedName` +/// (e.g. "export const config = { runtime: 'edge' }") from swc's AST. +/// The value must be one of +/// - string +/// - boolean +/// - number +/// - null +/// - undefined +/// - array containing values listed in this list +/// - object containing values listed in this list +/// +/// Returns a map of the extracted values, or either contains corresponding +/// error. +pub fn extract_expored_const_values( + source_ast: &Program, + properties_to_extract: HashSet, +) -> HashMap> { + GLOBALS.set(&Default::default(), || { + let mut visitor = + collect_exported_const_visitor::CollectExportedConstVisitor::new(properties_to_extract); + + source_ast.visit_with_path(&mut visitor, &mut Default::default()); + + visitor.properties + }) +} + +#[cfg(test)] +mod tests { + use super::{build_ast_from_source, collect_rsc_module_info, RscModuleInfo}; + + #[test] + fn should_parse_server_info() { + let input = r#"export default function Page() { + return

app-edge-ssr

+ } + + export const runtime = 'edge' + export const maxDuration = 4 + "#; + + let (_, comments) = build_ast_from_source(input, "some-file.js") + .expect("Should able to parse test fixture input"); + + let module_info = collect_rsc_module_info(&comments, true); + let expected = RscModuleInfo { + module_type: "server".to_string(), + actions: None, + is_client_ref: false, + client_refs: None, + client_entry_type: None, + }; + + assert_eq!(module_info, expected); + } + + #[test] + fn should_parse_actions_json() { + let input = r#" + /* __next_internal_action_entry_do_not_use__ {"ab21efdafbe611287bc25c0462b1e0510d13e48b":"foo"} */ import { createActionProxy } from "private-next-rsc-action-proxy"; + import { encryptActionBoundArgs, decryptActionBoundArgs } from "private-next-rsc-action-encryption"; + export function foo() {} + import { ensureServerEntryExports } from "private-next-rsc-action-validate"; + ensureServerEntryExports([ + foo + ]); + createActionProxy("ab21efdafbe611287bc25c0462b1e0510d13e48b", foo); + "#; + + let (_, comments) = build_ast_from_source(input, "some-file.js") + .expect("Should able to parse test fixture input"); + + let module_info = collect_rsc_module_info(&comments, true); + let expected = RscModuleInfo { + module_type: "server".to_string(), + actions: Some(vec!["foo".to_string()]), + is_client_ref: false, + client_refs: None, + client_entry_type: None, + }; + + assert_eq!(module_info, expected); + } + + #[test] + fn should_parse_client_refs() { + let input = r#" + // This is a comment. + /* __next_internal_client_entry_do_not_use__ default,a,b,c,*,f auto */ const { createProxy } = require("private-next-rsc-mod-ref-proxy"); + module.exports = createProxy("/some-project/src/some-file.js"); + "#; + + let (_, comments) = build_ast_from_source(input, "some-file.js") + .expect("Should able to parse test fixture input"); + + let module_info = collect_rsc_module_info(&comments, true); + + let expected = RscModuleInfo { + module_type: "client".to_string(), + actions: None, + is_client_ref: true, + client_refs: Some(vec![ + "default".to_string(), + "a".to_string(), + "b".to_string(), + "c".to_string(), + "*".to_string(), + "f".to_string(), + ]), + client_entry_type: Some("auto".to_string()), + }; + + assert_eq!(module_info, expected); + } +} diff --git a/packages/next-swc/crates/wasm/Cargo.toml b/packages/next-swc/crates/wasm/Cargo.toml index 8b0bbb1ab0c5a..3db650000101f 100644 --- a/packages/next-swc/crates/wasm/Cargo.toml +++ b/packages/next-swc/crates/wasm/Cargo.toml @@ -35,8 +35,8 @@ turbopack-binding = { workspace = true, features = [ "__swc_core_binding_wasm", "__feature_mdx_rs", ] } -swc_core = { workspace = true, features = ["ecma_ast_serde", "common"] } - +swc_core = { workspace = true, features = ["ecma_ast_serde", "common", "ecma_visit_path"] } +regex = "1.5" # Workaround a bug [package.metadata.wasm-pack.profile.release] diff --git a/packages/next-swc/crates/wasm/src/lib.rs b/packages/next-swc/crates/wasm/src/lib.rs index 4c53eeb2c517a..a9bd190d67e4c 100644 --- a/packages/next-swc/crates/wasm/src/lib.rs +++ b/packages/next-swc/crates/wasm/src/lib.rs @@ -1,18 +1,23 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use anyhow::{Context, Error}; use js_sys::JsString; -use next_custom_transforms::chain_transforms::{custom_before_pass, TransformOptions}; +use next_custom_transforms::{ + chain_transforms::{custom_before_pass, TransformOptions}, + transforms::page_static_info::{ + build_ast_from_source, collect_exports, collect_rsc_module_info, + extract_expored_const_values, Const, ExportInfo, RscModuleInfo, + }, +}; +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{Deserialize, Serialize}; use swc_core::common::Mark; use turbopack_binding::swc::core::{ - base::{ - config::{JsMinifyOptions, ParseOptions}, - try_with_handler, Compiler, - }, + base::{config::JsMinifyOptions, try_with_handler, Compiler}, common::{ - comments::{Comments, SingleThreadedComments}, - errors::ColorConfig, - FileName, FilePathMapping, SourceMap, GLOBALS, + comments::SingleThreadedComments, errors::ColorConfig, FileName, FilePathMapping, + SourceMap, GLOBALS, }, ecma::transforms::base::pass::noop, }; @@ -21,6 +26,21 @@ use wasm_bindgen_futures::future_to_promise; pub mod mdx; +/// A regex pattern to determine if is_dynamic_metadata_route should continue to +/// parse the page or short circuit and return false. +static DYNAMIC_METADATA_ROUTE_SHORT_CURCUIT: Lazy = + Lazy::new(|| Regex::new("generateImageMetadata|generateSitemaps").unwrap()); + +/// A regex pattern to determine if get_page_static_info should continue to +/// parse the page or short circuit and return default. +static PAGE_STATIC_INFO_SHORT_CURCUIT: Lazy = Lazy::new(|| { + Regex::new( + "runtime|preferredRegion|getStaticProps|getServerSideProps|generateStaticParams|export \ + const", + ) + .unwrap() +}); + fn convert_err(err: Error) -> JsValue { format!("{:?}", err).into() } @@ -137,57 +157,97 @@ pub fn transform(s: JsValue, opts: JsValue) -> js_sys::Promise { future_to_promise(async { transform_sync(s, opts) }) } -#[wasm_bindgen(js_name = "parseSync")] -pub fn parse_sync(s: JsString, opts: JsValue) -> Result { - console_error_panic_hook::set_once(); - - let c = turbopack_binding::swc::core::base::Compiler::new(Arc::new(SourceMap::new( - FilePathMapping::empty(), - ))); - let opts: ParseOptions = serde_wasm_bindgen::from_value(opts)?; +/// Detect if metadata routes is a dynamic route, which containing +/// generateImageMetadata or generateSitemaps as export +/// Unlike native bindings, caller should provide the contents of the pages +/// sine our wasm bindings does not have access to the file system +#[wasm_bindgen(js_name = "isDynamicMetadataRoute")] +pub fn is_dynamic_metadata_route(page_file_path: String, page_contents: String) -> js_sys::Promise { + // Returning promise to conform existing interfaces + future_to_promise(async move { + if !DYNAMIC_METADATA_ROUTE_SHORT_CURCUIT.is_match(&page_contents) { + return Ok(JsValue::from(false)); + } - try_with_handler( - c.cm.clone(), - turbopack_binding::swc::core::base::HandlerOpts { - ..Default::default() - }, - |handler| { - c.run(|| { - GLOBALS.set(&Default::default(), || { - let fm = c.cm.new_source_file(FileName::Anon, s.into()); - - let cmts = c.comments().clone(); - let comments = if opts.comments { - Some(&cmts as &dyn Comments) - } else { - None - }; - - let program = c - .parse_js( - fm, - handler, - opts.target, - opts.syntax, - opts.is_module, - comments, + let (source_ast, _) = build_ast_from_source(&page_contents, &page_file_path) + .map_err(|e| JsValue::from_str(format!("{:?}", e).as_str()))?; + collect_exports(&source_ast) + .map(|exports_info| { + exports_info + .map(|exports_info| { + JsValue::from( + !exports_info.generate_image_metadata.unwrap_or_default() + || !exports_info.generate_sitemaps.unwrap_or_default(), ) - .context("failed to parse code")?; - - let s = serde_json::to_string(&program).unwrap(); - Ok(JsValue::from_str(&s)) - }) + }) + .unwrap_or_default() }) - }, - ) - .map_err(convert_err) + .map_err(|e| JsValue::from_str(format!("{:?}", e).as_str())) + }) } -#[wasm_bindgen(js_name = "parse")] -pub fn parse(s: JsString, opts: JsValue) -> js_sys::Promise { - // TODO: This'll be properly scheduled once wasm have standard backed thread - // support. - future_to_promise(async { parse_sync(s, opts) }) +#[derive(Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StaticPageInfo { + pub exports_info: Option, + pub extracted_values: HashMap, + pub rsc_info: Option, + pub warnings: Vec, +} + +#[wasm_bindgen(js_name = "getPageStaticInfo")] +pub fn get_page_static_info(page_file_path: String, page_contents: String) -> js_sys::Promise { + future_to_promise(async move { + if !PAGE_STATIC_INFO_SHORT_CURCUIT.is_match(&page_contents) { + return Ok(JsValue::null()); + } + + let (source_ast, comments) = build_ast_from_source(&page_contents, &page_file_path) + .map_err(|e| JsValue::from_str(format!("{:?}", e).as_str()))?; + let exports_info = collect_exports(&source_ast) + .map_err(|e| JsValue::from_str(format!("{:?}", e).as_str()))?; + + match exports_info { + None => Ok(JsValue::null()), + Some(exports_info) => { + let rsc_info = collect_rsc_module_info(&comments, true); + + let mut properties_to_extract = exports_info.extra_properties.clone(); + properties_to_extract.insert("config".to_string()); + + let mut exported_const_values = + extract_expored_const_values(&source_ast, properties_to_extract); + + let mut extracted_values = HashMap::new(); + let mut warnings = vec![]; + + for (key, value) in exported_const_values.drain() { + match value { + Some(Const::Value(v)) => { + extracted_values.insert(key.clone(), v); + } + Some(Const::Unsupported(msg)) => { + warnings.push(msg); + } + _ => {} + } + } + + let ret = StaticPageInfo { + exports_info: Some(exports_info), + extracted_values, + rsc_info: Some(rsc_info), + warnings, + }; + + let s = serde_json::to_string(&ret) + .map(|s| JsValue::from_str(&s)) + .unwrap_or(JsValue::null()); + + Ok(s) + } + } + }) } /// Get global sourcemap diff --git a/packages/next/src/build/analysis/extract-const-value.ts b/packages/next/src/build/analysis/extract-const-value.ts deleted file mode 100644 index 0ae1d5fc7993a..0000000000000 --- a/packages/next/src/build/analysis/extract-const-value.ts +++ /dev/null @@ -1,249 +0,0 @@ -import type { - ArrayExpression, - BooleanLiteral, - ExportDeclaration, - Identifier, - KeyValueProperty, - Module, - Node, - NullLiteral, - NumericLiteral, - ObjectExpression, - RegExpLiteral, - StringLiteral, - TemplateLiteral, - VariableDeclaration, -} from '@swc/core' - -export class NoSuchDeclarationError extends Error {} - -function isExportDeclaration(node: Node): node is ExportDeclaration { - return node.type === 'ExportDeclaration' -} - -function isVariableDeclaration(node: Node): node is VariableDeclaration { - return node.type === 'VariableDeclaration' -} - -function isIdentifier(node: Node): node is Identifier { - return node.type === 'Identifier' -} - -function isBooleanLiteral(node: Node): node is BooleanLiteral { - return node.type === 'BooleanLiteral' -} - -function isNullLiteral(node: Node): node is NullLiteral { - return node.type === 'NullLiteral' -} - -function isStringLiteral(node: Node): node is StringLiteral { - return node.type === 'StringLiteral' -} - -function isNumericLiteral(node: Node): node is NumericLiteral { - return node.type === 'NumericLiteral' -} - -function isArrayExpression(node: Node): node is ArrayExpression { - return node.type === 'ArrayExpression' -} - -function isObjectExpression(node: Node): node is ObjectExpression { - return node.type === 'ObjectExpression' -} - -function isKeyValueProperty(node: Node): node is KeyValueProperty { - return node.type === 'KeyValueProperty' -} - -function isRegExpLiteral(node: Node): node is RegExpLiteral { - return node.type === 'RegExpLiteral' -} - -function isTemplateLiteral(node: Node): node is TemplateLiteral { - return node.type === 'TemplateLiteral' -} - -export class UnsupportedValueError extends Error { - /** @example `config.runtime[0].value` */ - path?: string - - constructor(message: string, paths?: string[]) { - super(message) - - // Generating "path" that looks like "config.runtime[0].value" - let codePath: string | undefined - if (paths) { - codePath = '' - for (const path of paths) { - if (path[0] === '[') { - // "array" + "[0]" - codePath += path - } else { - if (codePath === '') { - codePath = path - } else { - // "object" + ".key" - codePath += `.${path}` - } - } - } - } - - this.path = codePath - } -} - -function extractValue(node: Node, path?: string[]): any { - if (isNullLiteral(node)) { - return null - } else if (isBooleanLiteral(node)) { - // e.g. true / false - return node.value - } else if (isStringLiteral(node)) { - // e.g. "abc" - return node.value - } else if (isNumericLiteral(node)) { - // e.g. 123 - return node.value - } else if (isRegExpLiteral(node)) { - // e.g. /abc/i - return new RegExp(node.pattern, node.flags) - } else if (isIdentifier(node)) { - switch (node.value) { - case 'undefined': - return undefined - default: - throw new UnsupportedValueError( - `Unknown identifier "${node.value}"`, - path - ) - } - } else if (isArrayExpression(node)) { - // e.g. [1, 2, 3] - const arr = [] - for (let i = 0, len = node.elements.length; i < len; i++) { - const elem = node.elements[i] - if (elem) { - if (elem.spread) { - // e.g. [ ...a ] - throw new UnsupportedValueError( - 'Unsupported spread operator in the Array Expression', - path - ) - } - - arr.push(extractValue(elem.expression, path && [...path, `[${i}]`])) - } else { - // e.g. [1, , 2] - // ^^ - arr.push(undefined) - } - } - return arr - } else if (isObjectExpression(node)) { - // e.g. { a: 1, b: 2 } - const obj: any = {} - for (const prop of node.properties) { - if (!isKeyValueProperty(prop)) { - // e.g. { ...a } - throw new UnsupportedValueError( - 'Unsupported spread operator in the Object Expression', - path - ) - } - - let key - if (isIdentifier(prop.key)) { - // e.g. { a: 1, b: 2 } - key = prop.key.value - } else if (isStringLiteral(prop.key)) { - // e.g. { "a": 1, "b": 2 } - key = prop.key.value - } else { - throw new UnsupportedValueError( - `Unsupported key type "${prop.key.type}" in the Object Expression`, - path - ) - } - - obj[key] = extractValue(prop.value, path && [...path, key]) - } - - return obj - } else if (isTemplateLiteral(node)) { - // e.g. `abc` - if (node.expressions.length !== 0) { - // TODO: should we add support for `${'e'}d${'g'}'e'`? - throw new UnsupportedValueError( - 'Unsupported template literal with expressions', - path - ) - } - - // When TemplateLiteral has 0 expressions, the length of quasis is always 1. - // Because when parsing TemplateLiteral, the parser yields the first quasi, - // then the first expression, then the next quasi, then the next expression, etc., - // until the last quasi. - // Thus if there is no expression, the parser ends at the frst and also last quasis - // - // A "cooked" interpretation where backslashes have special meaning, while a - // "raw" interpretation where backslashes do not have special meaning - // https://exploringjs.com/impatient-js/ch_template-literals.html#template-strings-cooked-vs-raw - const [{ cooked, raw }] = node.quasis - - return cooked ?? raw - } else { - throw new UnsupportedValueError( - `Unsupported node type "${node.type}"`, - path - ) - } -} - -/** - * Extracts the value of an exported const variable named `exportedName` - * (e.g. "export const config = { runtime: 'edge' }") from swc's AST. - * The value must be one of (or throws UnsupportedValueError): - * - string - * - boolean - * - number - * - null - * - undefined - * - array containing values listed in this list - * - object containing values listed in this list - * - * Throws NoSuchDeclarationError if the declaration is not found. - */ -export function extractExportedConstValue( - module: Module, - exportedName: string -): any { - for (const moduleItem of module.body) { - if (!isExportDeclaration(moduleItem)) { - continue - } - - const declaration = moduleItem.declaration - if (!isVariableDeclaration(declaration)) { - continue - } - - if (declaration.kind !== 'const') { - continue - } - - for (const decl of declaration.declarations) { - if ( - isIdentifier(decl.id) && - decl.id.value === exportedName && - decl.init - ) { - return extractValue(decl.init, [exportedName]) - } - } - } - - throw new NoSuchDeclarationError() -} diff --git a/packages/next/src/build/analysis/get-page-static-info.ts b/packages/next/src/build/analysis/get-page-static-info.ts index 57a008dbe9408..3551405bdf4e3 100644 --- a/packages/next/src/build/analysis/get-page-static-info.ts +++ b/packages/next/src/build/analysis/get-page-static-info.ts @@ -1,15 +1,9 @@ import type { NextConfig } from '../../server/config-shared' import type { Middleware, RouteHas } from '../../lib/load-custom-routes' -import { promises as fs } from 'fs' import LRUCache from 'next/dist/compiled/lru-cache' import picomatch from 'next/dist/compiled/picomatch' import type { ServerRuntime } from 'next/types' -import { - extractExportedConstValue, - UnsupportedValueError, -} from './extract-const-value' -import { parseModule } from './parse-module' import * as Log from '../output/log' import { SERVER_RUNTIME } from '../../lib/constants' import { checkCustomRoutes } from '../../lib/load-custom-routes' @@ -56,9 +50,6 @@ const CLIENT_MODULE_LABEL = const ACTION_MODULE_LABEL = /\/\* __next_internal_action_entry_do_not_use__ (\{[^}]+\}) \*\// -const CLIENT_DIRECTIVE = 'use client' -const SERVER_ACTION_DIRECTIVE = 'use server' - export type RSCModuleType = 'server' | 'client' export function getRSCModuleInformation( source: string, @@ -93,211 +84,6 @@ export function getRSCModuleInformation( } } -const warnedInvalidValueMap = { - runtime: new Map(), - preferredRegion: new Map(), -} as const -function warnInvalidValue( - pageFilePath: string, - key: keyof typeof warnedInvalidValueMap, - message: string -): void { - if (warnedInvalidValueMap[key].has(pageFilePath)) return - - Log.warn( - `Next.js can't recognize the exported \`${key}\` field in "${pageFilePath}" as ${message}.` + - '\n' + - 'The default runtime will be used instead.' - ) - - warnedInvalidValueMap[key].set(pageFilePath, true) -} -/** - * Receives a parsed AST from SWC and checks if it belongs to a module that - * requires a runtime to be specified. Those are: - * - Modules with `export function getStaticProps | getServerSideProps` - * - Modules with `export { getStaticProps | getServerSideProps } ` - * - Modules with `export const runtime = ...` - */ -function checkExports( - swcAST: any, - pageFilePath: string -): { - ssr: boolean - ssg: boolean - runtime?: string - preferredRegion?: string | string[] - generateImageMetadata?: boolean - generateSitemaps?: boolean - generateStaticParams: boolean - extraProperties?: Set - directives?: Set -} { - const exportsSet = new Set([ - 'getStaticProps', - 'getServerSideProps', - 'generateImageMetadata', - 'generateSitemaps', - 'generateStaticParams', - ]) - if (Array.isArray(swcAST?.body)) { - try { - let runtime: string | undefined - let preferredRegion: string | string[] | undefined - let ssr: boolean = false - let ssg: boolean = false - let generateImageMetadata: boolean = false - let generateSitemaps: boolean = false - let generateStaticParams = false - let extraProperties = new Set() - let directives = new Set() - let hasLeadingNonDirectiveNode = false - - for (const node of swcAST.body) { - // There should be no non-string literals nodes before directives - if ( - node.type === 'ExpressionStatement' && - node.expression.type === 'StringLiteral' - ) { - if (!hasLeadingNonDirectiveNode) { - const directive = node.expression.value - if (CLIENT_DIRECTIVE === directive) { - directives.add('client') - } - if (SERVER_ACTION_DIRECTIVE === directive) { - directives.add('server') - } - } - } else { - hasLeadingNonDirectiveNode = true - } - if ( - node.type === 'ExportDeclaration' && - node.declaration?.type === 'VariableDeclaration' - ) { - for (const declaration of node.declaration?.declarations) { - if (declaration.id.value === 'runtime') { - runtime = declaration.init.value - } else if (declaration.id.value === 'preferredRegion') { - if (declaration.init.type === 'ArrayExpression') { - const elements: string[] = [] - for (const element of declaration.init.elements) { - const { expression } = element - if (expression.type !== 'StringLiteral') { - continue - } - elements.push(expression.value) - } - preferredRegion = elements - } else { - preferredRegion = declaration.init.value - } - } else { - extraProperties.add(declaration.id.value) - } - } - } - - if ( - node.type === 'ExportDeclaration' && - node.declaration?.type === 'FunctionDeclaration' && - exportsSet.has(node.declaration.identifier?.value) - ) { - const id = node.declaration.identifier.value - ssg = id === 'getStaticProps' - ssr = id === 'getServerSideProps' - generateImageMetadata = id === 'generateImageMetadata' - generateSitemaps = id === 'generateSitemaps' - generateStaticParams = id === 'generateStaticParams' - } - - if ( - node.type === 'ExportDeclaration' && - node.declaration?.type === 'VariableDeclaration' - ) { - const id = node.declaration?.declarations[0]?.id.value - if (exportsSet.has(id)) { - ssg = id === 'getStaticProps' - ssr = id === 'getServerSideProps' - generateImageMetadata = id === 'generateImageMetadata' - generateSitemaps = id === 'generateSitemaps' - generateStaticParams = id === 'generateStaticParams' - } - } - - if (node.type === 'ExportNamedDeclaration') { - const values = node.specifiers.map( - (specifier: any) => - specifier.type === 'ExportSpecifier' && - specifier.orig?.type === 'Identifier' && - specifier.orig?.value - ) - - for (const value of values) { - if (!ssg && value === 'getStaticProps') ssg = true - if (!ssr && value === 'getServerSideProps') ssr = true - if (!generateImageMetadata && value === 'generateImageMetadata') - generateImageMetadata = true - if (!generateSitemaps && value === 'generateSitemaps') - generateSitemaps = true - if (!generateStaticParams && value === 'generateStaticParams') - generateStaticParams = true - if (!runtime && value === 'runtime') - warnInvalidValue( - pageFilePath, - 'runtime', - 'it was not assigned to a string literal' - ) - if (!preferredRegion && value === 'preferredRegion') - warnInvalidValue( - pageFilePath, - 'preferredRegion', - 'it was not assigned to a string literal or an array of string literals' - ) - } - } - } - - return { - ssr, - ssg, - runtime, - preferredRegion, - generateImageMetadata, - generateSitemaps, - generateStaticParams, - extraProperties, - directives, - } - } catch (err) {} - } - - return { - ssg: false, - ssr: false, - runtime: undefined, - preferredRegion: undefined, - generateImageMetadata: false, - generateSitemaps: false, - generateStaticParams: false, - extraProperties: undefined, - directives: undefined, - } -} - -async function tryToReadFile(filePath: string, shouldThrow: boolean) { - try { - return await fs.readFile(filePath, { - encoding: 'utf8', - }) - } catch (error: any) { - if (shouldThrow) { - error.message = `Next.js ERROR: Failed to read file ${filePath}:\n${error.message}` - throw error - } - } -} - export function getMiddlewareMatchers( matcherOrMatchers: unknown, nextConfig: NextConfig @@ -430,10 +216,11 @@ function warnAboutExperimentalEdge(apiRoute: string | null) { const warnedUnsupportedValueMap = new LRUCache({ max: 250 }) +// [TODO] next-swc does not returns path where unsupported value is found yet. function warnAboutUnsupportedValue( pageFilePath: string, page: string | undefined, - error: UnsupportedValueError + message: string ) { if (warnedUnsupportedValueMap.has(pageFilePath)) { return @@ -443,9 +230,8 @@ function warnAboutUnsupportedValue( `Next.js can't recognize the exported \`config\` field in ` + (page ? `route "${page}"` : `"${pageFilePath}"`) + ':\n' + - error.message + - (error.path ? ` at "${error.path}"` : '') + - '.\n' + + message + + '\n' + 'The default config will be used instead.\n' + 'Read More - https://nextjs.org/docs/messages/invalid-page-config' ) @@ -453,20 +239,6 @@ function warnAboutUnsupportedValue( warnedUnsupportedValueMap.set(pageFilePath, true) } -// Detect if metadata routes is a dynamic route, which containing -// generateImageMetadata or generateSitemaps as export -export async function isDynamicMetadataRoute( - pageFilePath: string -): Promise { - const fileContent = (await tryToReadFile(pageFilePath, true)) || '' - if (/generateImageMetadata|generateSitemaps/.test(fileContent)) { - const swcAST = await parseModule(pageFilePath, fileContent) - const exportsInfo = checkExports(swcAST, pageFilePath) - return !!(exportsInfo.generateImageMetadata || exportsInfo.generateSitemaps) - } - return false -} - /** * For a given pageFilePath and nextConfig, if the config supports it, this * function will read the file and return the runtime that should be used. @@ -483,13 +255,12 @@ export async function getPageStaticInfo(params: { }): Promise { const { isDev, pageFilePath, nextConfig, page, pageType } = params - const fileContent = (await tryToReadFile(pageFilePath, !isDev)) || '' - if ( - /(? { + warnAboutUnsupportedValue(pageFilePath, page, warning) + }) + // default / failsafe value for config - let config: any - try { - config = extractExportedConstValue(swcAST, 'config') - } catch (e) { - if (e instanceof UnsupportedValueError) { - warnAboutUnsupportedValue(pageFilePath, page, e) - } - // `export config` doesn't exist, or other unknown error throw by swc, silence them - } + let config = extractedValues.config const extraConfig: Record = {} if (extraProperties && pageType === PAGE_TYPES.APP) { for (const prop of extraProperties) { if (!AUTHORIZED_EXTRA_ROUTER_PROPS.includes(prop)) continue - try { - extraConfig[prop] = extractExportedConstValue(swcAST, prop) - } catch (e) { - if (e instanceof UnsupportedValueError) { - warnAboutUnsupportedValue(pageFilePath, page, e) - } - } + extraConfig[prop] = extractedValues[prop] } } else if (pageType === PAGE_TYPES.PAGES) { for (const key in config) { diff --git a/packages/next/src/build/analysis/parse-module.ts b/packages/next/src/build/analysis/parse-module.ts deleted file mode 100644 index 5ba1dd24a15c3..0000000000000 --- a/packages/next/src/build/analysis/parse-module.ts +++ /dev/null @@ -1,15 +0,0 @@ -import LRUCache from 'next/dist/compiled/lru-cache' -import { withPromiseCache } from '../../lib/with-promise-cache' -import { createHash } from 'crypto' -import { parse } from '../swc' - -/** - * Parses a module with SWC using an LRU cache where the parsed module will - * be indexed by a sha of its content holding up to 500 entries. - */ -export const parseModule = withPromiseCache( - new LRUCache({ max: 500 }), - async (filename: string, content: string) => - parse(content, { isModule: 'unknown', filename }).catch(() => null), - (_, content) => createHash('sha1').update(content).digest('hex') -) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 6e177b864ce59..04e21ac68a459 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -94,10 +94,7 @@ import { } from '../telemetry/events' import type { EventBuildFeatureUsage } from '../telemetry/events' import { Telemetry } from '../telemetry/storage' -import { - isDynamicMetadataRoute, - getPageStaticInfo, -} from './analysis/get-page-static-info' +import { getPageStaticInfo } from './analysis/get-page-static-info' import { createPagesMapping, getPageFilePath, sortByPageExts } from './entries' import { PAGE_TYPES } from '../lib/page-types' import { generateBuildId } from './generate-build-id' @@ -126,6 +123,7 @@ import { recursiveCopy } from '../lib/recursive-copy' import { recursiveReadDir } from '../lib/recursive-readdir' import { lockfilePatchPromise, + loadBindings, teardownTraceSubscriber, teardownHeapProfiler, } from './swc' @@ -763,7 +761,6 @@ export default async function build( const cacheDir = getCacheDir(distDir) const telemetry = new Telemetry({ distDir }) - setGlobal('telemetry', telemetry) const publicDir = path.join(dir, 'public') @@ -771,6 +768,8 @@ export default async function build( NextBuildContext.pagesDir = pagesDir NextBuildContext.appDir = appDir + const binding = await loadBindings(config?.experimental?.useWasmBinary) + const enabledDirectories: NextEnabledDirectories = { app: typeof appDir === 'string', pages: typeof pagesDir === 'string', @@ -962,7 +961,10 @@ export default async function build( rootDir, }) - const isDynamic = await isDynamicMetadataRoute(pageFilePath) + const isDynamic = await binding.analysis.isDynamicMetadataRoute( + pageFilePath + ) + if (!isDynamic) { delete mappedAppPages[pageKey] mappedAppPages[pageKey.replace('[[...__metadata_id__]]/', '')] = diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 936a2e098d716..083b5f6949808 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -2,9 +2,9 @@ import path from 'path' import { pathToFileURL } from 'url' import { platform, arch } from 'os' +import { promises as fs } from 'fs' import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples' import * as Log from '../output/log' -import { getParserOptions } from './options' import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure' import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile' import { downloadWasmSwc, downloadNativeNextSwc } from '../../lib/download-swc' @@ -15,6 +15,7 @@ import type { DefineEnvPluginOptions } from '../webpack/plugins/define-env-plugi import type { PageExtensions } from '../page-extensions-type' const nextVersion = process.env.__NEXT_VERSION as string +const isYarnPnP = !!process?.versions?.pnp const ArchName = arch() const PlatformName = platform() @@ -108,6 +109,19 @@ function checkVersionMismatch(pkgData: any) { } } +async function tryToReadFile(filePath: string, shouldThrow: boolean) { + try { + return await fs.readFile(filePath, { + encoding: 'utf8', + }) + } catch (error: any) { + if (shouldThrow) { + error.message = `Next.js ERROR: Failed to read file ${filePath}:\n${error.message}` + throw error + } + } +} + // These are the platforms we'll try to load wasm bindings first, // only try to load native bindings if loading wasm binding somehow fails. // Fallback to native binding is for migration period only, @@ -160,12 +174,13 @@ export interface Binding { turboEngineOptions?: TurboEngineOptions ) => Promise } + analysis: { + isDynamicMetadataRoute(pageFilePath: string): Promise + } minify: any minifySync: any transform: any transformSync: any - parse: any - parseSync: any getTargetTriple(): string | undefined initCustomTraceSubscriber?: any teardownTraceSubscriber?: any @@ -1132,6 +1147,26 @@ function bindingToApi(binding: any, _wasm: boolean) { return createProject } +const warnedInvalidValueMap = { + runtime: new Map(), + preferredRegion: new Map(), +} as const +function warnInvalidValue( + pageFilePath: string, + key: keyof typeof warnedInvalidValueMap, + message: string +): void { + if (warnedInvalidValueMap[key].has(pageFilePath)) return + + Log.warn( + `Next.js can't recognize the exported \`${key}\` field in "${pageFilePath}" as ${message}.` + + '\n' + + 'The default runtime will be used instead.' + ) + + warnedInvalidValueMap[key].set(pageFilePath, true) +} + async function loadWasm(importPath = '') { if (wasmBindings) { return wasmBindings @@ -1173,14 +1208,33 @@ async function loadWasm(importPath = '') { minifySync(src: string, options: any) { return bindings.minifySync(src.toString(), options) }, - parse(src: string, options: any) { - return bindings?.parse - ? bindings.parse(src.toString(), options) - : Promise.resolve(bindings.parseSync(src.toString(), options)) - }, - parseSync(src: string, options: any) { - const astStr = bindings.parseSync(src.toString(), options) - return astStr + analysis: { + isDynamicMetadataRoute: async (pageFilePath: string) => { + const fileContent = (await tryToReadFile(pageFilePath, true)) || '' + const { isDynamicMetadataRoute, warnings } = + await bindings.isDynamicMetadataRoute(pageFilePath, fileContent) + + warnings?.forEach( + ({ + key, + message, + }: { + key: keyof typeof warnedInvalidValueMap + message: string + }) => warnInvalidValue(pageFilePath, key, message) + ) + return isDynamicMetadataRoute + }, + getPageStaticInfo: async (params: Record) => { + const fileContent = + (await tryToReadFile(params.pageFilePath, !params.isDev)) || '' + + const raw = await bindings.getPageStaticInfo( + params.pageFilePath, + fileContent + ) + return coercePageStaticInfo(params.pageFilePath, raw) + }, }, getTargetTriple() { return undefined @@ -1345,8 +1399,40 @@ function loadNative(importPath?: string) { return bindings.minifySync(toBuffer(src), toBuffer(options ?? {})) }, - parse(src: string, options: any) { - return bindings.parse(src, toBuffer(options ?? {})) + analysis: { + isDynamicMetadataRoute: async (pageFilePath: string) => { + let fileContent: string | undefined = undefined + if (isYarnPnP) { + fileContent = (await tryToReadFile(pageFilePath, true)) || '' + } + + const { isDynamicMetadataRoute, warnings } = + await bindings.isDynamicMetadataRoute(pageFilePath, fileContent) + + // Instead of passing js callback into napi's context, bindings bubble up the warning messages + // and let next.js logger handles it. + warnings?.forEach( + ({ + key, + message, + }: { + key: keyof typeof warnedInvalidValueMap + message: string + }) => warnInvalidValue(pageFilePath, key, message) + ) + return isDynamicMetadataRoute + }, + + getPageStaticInfo: async (params: Record) => { + let fileContent: string | undefined = undefined + if (isYarnPnP) { + fileContent = + (await tryToReadFile(params.pageFilePath, !params.isDev)) || '' + } + + const raw = await bindings.getPageStaticInfo(params, fileContent) + return coercePageStaticInfo(params.pageFilePath, raw) + }, }, getTargetTriple: bindings.getTargetTriple, @@ -1430,6 +1516,31 @@ function toBuffer(t: any) { return Buffer.from(JSON.stringify(t)) } +function coercePageStaticInfo(pageFilePath: string, raw?: string) { + if (!raw) return raw + + const parsed = JSON.parse(raw) + + parsed?.exportsInfo?.warnings?.forEach( + ({ + key, + message, + }: { + key: keyof typeof warnedInvalidValueMap + message: string + }) => warnInvalidValue(pageFilePath, key, message) + ) + + return { + ...parsed, + exportsInfo: { + ...parsed.exportsInfo, + directives: new Set(parsed?.exportsInfo?.directives ?? []), + extraProperties: new Set(parsed?.exportsInfo?.extraProperties ?? []), + }, + } +} + export async function isWasm(): Promise { let bindings = await loadBindings() return bindings.isWasm @@ -1455,14 +1566,6 @@ export function minifySync(src: string, options: any): string { return bindings.minifySync(src, options) } -export async function parse(src: string, options: any): Promise { - let bindings = await loadBindings() - let parserOptions = getParserOptions(options) - return bindings - .parse(src, parserOptions) - .then((astStr: any) => JSON.parse(astStr)) -} - export function getBinaryMetadata() { let bindings try { diff --git a/test/development/middleware-errors/index.test.ts b/test/development/middleware-errors/index.test.ts index 89e39644fa3e1..589f9f6fa4efd 100644 --- a/test/development/middleware-errors/index.test.ts +++ b/test/development/middleware-errors/index.test.ts @@ -249,6 +249,8 @@ createNextDescribe( await next.fetch('/') await check(async () => { expect(next.cliOutput).toContain(`Expected '{', got '}'`) + // [NOTE] [Flaky] expect at least 2 occurrences of the error message, + // on CI sometimes have more message appended somehow expect( next.cliOutput.split(`Expected '{', got '}'`).length ).toBeGreaterThanOrEqual(2) diff --git a/test/production/exported-runtimes-value-validation/index.test.ts b/test/production/exported-runtimes-value-validation/index.test.ts index f99474d7b6070..94013f520f37a 100644 --- a/test/production/exported-runtimes-value-validation/index.test.ts +++ b/test/production/exported-runtimes-value-validation/index.test.ts @@ -46,9 +46,7 @@ describe('Exported runtimes value validation', () => { ) ) expect(result.stderr).toEqual( - expect.stringContaining( - 'Unsupported node type "BinaryExpression" at "config.runtime"' - ) + expect.stringContaining('Unsupported node type at "config.runtime"') ) // Spread Operator within Object Expression expect(result.stderr).toEqual( @@ -90,9 +88,7 @@ describe('Exported runtimes value validation', () => { ) ) expect(result.stderr).toEqual( - expect.stringContaining( - 'Unsupported node type "CallExpression" at "config.runtime"' - ) + expect.stringContaining('Unsupported node type at "config.runtime"') ) // Unknown Object Key expect(result.stderr).toEqual( @@ -102,7 +98,7 @@ describe('Exported runtimes value validation', () => { ) expect(result.stderr).toEqual( expect.stringContaining( - 'Unsupported key type "Computed" in the Object Expression at "config.runtime"' + 'Unsupported key type in the Object Expression at "config.runtime"' ) ) })