Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Append sitemap extension and optimize imafe metadata static generation #66477

Merged
merged 31 commits into from
Jun 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
7d5108a
Add new dynamic multi metadata route conventions
huozhi Jun 1, 2024
6f0f938
update tests
huozhi Jun 1, 2024
ed2f994
handle multi dynamic routes
huozhi Jun 2, 2024
2b40734
handle dynamic routes and update test
huozhi Jun 2, 2024
77bdda7
support turbopack
huozhi Jun 2, 2024
7a14479
handle numeric
huozhi Jun 2, 2024
c5d1c58
fix turbopack regex matcher
huozhi Jun 3, 2024
4ed3eac
fix test and remove logs
huozhi Jun 3, 2024
205c55c
update test
huozhi Jun 3, 2024
5a24007
fix cargo test
huozhi Jun 3, 2024
d3d6fd2
skip turbopack tests
huozhi Jun 3, 2024
c8605c2
polish test
huozhi Jun 3, 2024
8a9c3b8
fix test
huozhi Jun 3, 2024
db0a6b2
fix rust build
huozhi Jun 5, 2024
1b9c781
fix rust build
huozhi Jun 5, 2024
d0f5a70
handle missing extension cases for sitemap generation
huozhi Jun 5, 2024
5c0a51c
fix template
huozhi Jun 5, 2024
bf114da
fix rust build
huozhi Jun 6, 2024
467dc85
fix rust build
huozhi Jun 6, 2024
e0813f6
Analyze exports
huozhi Jun 6, 2024
bdf994d
parse exports
huozhi Jun 6, 2024
148e495
fix turbopack app paths manifest
huozhi Jun 6, 2024
115f4d2
handle decl export function
huozhi Jun 7, 2024
d6e1554
fix dynamic routes, tests should pass
huozhi Jun 7, 2024
c5933cd
handle static metadata sitemap route
huozhi Jun 7, 2024
6fe7370
Update docs
huozhi Jun 7, 2024
ea33238
fix dynamic routes manifest loading
huozhi Jun 7, 2024
fb28834
fix pages
huozhi Jun 7, 2024
2a0b48b
separate dev error tests
huozhi Jun 7, 2024
b93ca56
stablize new test
huozhi Jun 7, 2024
1ea9876
Merge remote-tracking branch 'origin/canary' into huozhi/metadata-dyn…
huozhi Jun 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ export default function sitemap() {

Output:

```xml filename="acme.com/sitemap"
```xml filename="acme.com/sitemap.xml"
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://acme.com</loc>
Expand Down Expand Up @@ -163,7 +163,7 @@ export default function sitemap(): MetadataRoute.Sitemap {

Output:

```xml filename="acme.com/sitemap"
```xml filename="acme.com/sitemap.xml"
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>https://acme.com</loc>
Expand Down Expand Up @@ -264,7 +264,7 @@ export default async function sitemap({ id }) {
}
```

Your generated sitemaps will be available at `/.../sitemap/[id]`. For example, `/product/sitemap/1`.
Your generated sitemaps will be available at `/.../sitemap/[id]`. For example, `/product/sitemap/1.xml`.

See the [`generateSitemaps` API reference](/docs/app/api-reference/functions/generate-sitemaps) for more information.

Expand Down
44 changes: 35 additions & 9 deletions packages/next-swc/crates/next-core/src/app_segment_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_binding::{
swc::core::{
common::{source_map::Pos, Span, Spanned, GLOBALS},
ecma::ast::{Expr, Ident, Program},
ecma::ast::{Decl, Expr, FnExpr, Ident, Program},
},
turbopack::{
core::{
Expand Down Expand Up @@ -73,6 +73,9 @@ pub struct NextSegmentConfig {
pub runtime: Option<NextRuntime>,
pub preferred_region: Option<Vec<RcStr>>,
pub experimental_ppr: Option<bool>,
/// Wether these metadata exports are defined in the source file.
pub generate_image_metadata: bool,
pub generate_sitemaps: bool,
}

#[turbo_tasks::value_impl]
Expand All @@ -95,6 +98,7 @@ impl NextSegmentConfig {
runtime,
preferred_region,
experimental_ppr,
..
} = self;
*dynamic = dynamic.or(parent.dynamic);
*dynamic_params = dynamic_params.or(parent.dynamic_params);
Expand Down Expand Up @@ -137,6 +141,7 @@ impl NextSegmentConfig {
runtime,
preferred_region,
experimental_ppr,
..
} = self;
merge_parallel(dynamic, &parallel_config.dynamic, "dynamic")?;
merge_parallel(
Expand Down Expand Up @@ -272,22 +277,35 @@ pub async fn parse_segment_config_from_source(
let mut config = NextSegmentConfig::default();

for item in &module_ast.body {
let Some(decl) = item
let Some(export_decl) = item
.as_module_decl()
.and_then(|mod_decl| mod_decl.as_export_decl())
.and_then(|export_decl| export_decl.decl.as_var())
else {
continue;
};

for decl in &decl.decls {
let Some(ident) = decl.name.as_ident().map(|ident| ident.deref()) else {
continue;
};
match &export_decl.decl {
Decl::Var(var_decl) => {
for decl in &var_decl.decls {
let Some(ident) = decl.name.as_ident().map(|ident| ident.deref()) else {
continue;
};

if let Some(init) = decl.init.as_ref() {
parse_config_value(source, &mut config, ident, init, eval_context);
if let Some(init) = decl.init.as_ref() {
parse_config_value(source, &mut config, ident, init, eval_context);
}
}
}
Decl::Fn(fn_decl) => {
let ident = &fn_decl.ident;
// create an empty expression of {}, we don't need init for function
let init = Expr::Fn(FnExpr {
ident: None,
function: fn_decl.function.clone(),
});
parse_config_value(source, &mut config, ident, &init, eval_context);
}
_ => {}
}
}
config
Expand Down Expand Up @@ -431,6 +449,14 @@ fn parse_config_value(

config.preferred_region = Some(preferred_region);
}
// Match exported generateImageMetadata function and generateSitemaps function, and pass
// them to config.
"generateImageMetadata" => {
config.generate_image_metadata = true;
}
"generateSitemaps" => {
config.generate_sitemaps = true;
}
"experimental_ppr" => {
let value = eval_context.eval(init);
let Some(val) = value.as_bool() else {
Expand Down
16 changes: 3 additions & 13 deletions packages/next-swc/crates/next-core/src/next_app/metadata/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,7 @@ pub fn normalize_metadata_route(mut page: AppPage) -> Result<AppPage> {
route += ".txt"
} else if route == "/manifest" {
route += ".webmanifest"
// Do not append the suffix for the sitemap route
} else if !route.ends_with("/sitemap") {
} else {
// Remove the file extension, e.g. /route-path/robots.txt -> /route-path
let pathname_prefix = split_directory(&route).0.unwrap_or_default();
suffix = get_metadata_route_suffix(pathname_prefix);
Expand All @@ -317,13 +316,8 @@ pub fn normalize_metadata_route(mut page: AppPage) -> Result<AppPage> {
// /<metadata-route>/route.ts. If it's a metadata file route, we need to
// append /[id]/route to the page.
if !route.ends_with("/route") {
let is_static_metadata_file = is_static_metadata_route_file(&page.to_string());
let (base_name, ext) = split_extension(&route);

let is_static_route = route.starts_with("/robots")
|| route.starts_with("/manifest")
|| is_static_metadata_file;

page.0.pop();

page.push(PageSegment::Static(
Expand All @@ -338,10 +332,6 @@ pub fn normalize_metadata_route(mut page: AppPage) -> Result<AppPage> {
.into(),
))?;

if !is_static_route {
page.push(PageSegment::OptionalCatchAll("__metadata_id__".into()))?;
}

page.push(PageSegment::PageType(PageType::Route))?;
}

Expand All @@ -358,11 +348,11 @@ mod test {
let cases = vec![
[
"/client/(meme)/more-route/twitter-image",
"/client/(meme)/more-route/twitter-image-769mad/[[...__metadata_id__]]/route",
"/client/(meme)/more-route/twitter-image-769mad/route",
],
[
"/client/(meme)/more-route/twitter-image2",
"/client/(meme)/more-route/twitter-image2-769mad/[[...__metadata_id__]]/route",
"/client/(meme)/more-route/twitter-image2-769mad/route",
],
["/robots.txt", "/robots.txt/route"],
["/manifest.webmanifest", "/manifest.webmanifest/route"],
Expand Down
102 changes: 65 additions & 37 deletions packages/next-swc/crates/next-core/src/next_app/metadata/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//!
//! See `next/src/build/webpack/loaders/next-metadata-route-loader`

use anyhow::{bail, Result};
use anyhow::{bail, Ok, Result};
use base64::{display::Base64Display, engine::general_purpose::STANDARD};
use indoc::{formatdoc, indoc};
use turbo_tasks::{ValueToString, Vc};
Expand All @@ -22,17 +22,19 @@ use super::get_content_type;
use crate::{
app_structure::MetadataItem,
mode::NextMode,
next_app::{app_entry::AppEntry, app_route_entry::get_app_route_entry, AppPage, PageSegment},
next_app::{
app_entry::AppEntry, app_route_entry::get_app_route_entry, AppPage, PageSegment, PageType,
},
next_config::NextConfig,
parse_segment_config_from_source,
};

/// Computes the route source for a Next.js metadata file.
#[turbo_tasks::function]
pub async fn get_app_metadata_route_source(
page: AppPage,
mode: NextMode,
metadata: MetadataItem,
is_multi_dynamic: bool,
) -> Result<Vc<Box<dyn Source>>> {
Ok(match metadata {
MetadataItem::Static { path } => static_route_source(mode, path),
Expand All @@ -43,7 +45,7 @@ pub async fn get_app_metadata_route_source(
if stem == "robots" || stem == "manifest" {
dynamic_text_route_source(path)
} else if stem == "sitemap" {
dynamic_site_map_route_source(mode, path, page)
dynamic_site_map_route_source(mode, path, is_multi_dynamic)
} else {
dynamic_image_route_source(path)
}
Expand All @@ -52,11 +54,11 @@ pub async fn get_app_metadata_route_source(
}

#[turbo_tasks::function]
pub fn get_app_metadata_route_entry(
pub async fn get_app_metadata_route_entry(
nodejs_context: Vc<ModuleAssetContext>,
edge_context: Vc<ModuleAssetContext>,
project_root: Vc<FileSystemPath>,
page: AppPage,
mut page: AppPage,
mode: NextMode,
metadata: MetadataItem,
next_config: Vc<NextConfig>,
Expand All @@ -69,11 +71,43 @@ pub fn get_app_metadata_route_entry(

let source = Vc::upcast(FileSource::new(original_path));
let segment_config = parse_segment_config_from_source(source);
let is_dynamic_metadata = matches!(metadata, MetadataItem::Dynamic { .. });
let is_multi_dynamic: bool = if Some(segment_config).is_some() {
// is_multi_dynamic is true when config.generateSitemaps or
// config.generateImageMetadata is defined in dynamic routes
let config = segment_config.await.unwrap();
config.generate_sitemaps || config.generate_image_metadata
} else {
false
};

// Map dynamic sitemap and image routes based on the exports.
// if there's generator export: add /[__metadata_id__] to the route;
// otherwise keep the original route.
// For sitemap, if the last segment is sitemap, appending .xml suffix.
if is_dynamic_metadata {
// remove the last /route segment of page
page.0.pop();

let _ = if is_multi_dynamic {
page.push(PageSegment::Dynamic("__metadata_id__".into()))
} else {
// if page last segment is sitemap, change to sitemap.xml
if page.last() == Some(&PageSegment::Static("sitemap".into())) {
page.0.pop();
page.push(PageSegment::Static("sitemap.xml".into()))
} else {
Ok(())
}
};
// Push /route back
let _ = page.push(PageSegment::PageType(PageType::Route));
};

get_app_route_entry(
nodejs_context,
edge_context,
get_app_metadata_route_source(page.clone(), mode, metadata),
get_app_metadata_route_source(mode, metadata, is_multi_dynamic),
page,
project_root,
Some(segment_config),
Expand Down Expand Up @@ -208,26 +242,24 @@ async fn dynamic_text_route_source(path: Vc<FileSystemPath>) -> Result<Vc<Box<dy
async fn dynamic_site_map_route_source(
mode: NextMode,
path: Vc<FileSystemPath>,
page: AppPage,
is_multi_dynamic: bool,
) -> Result<Vc<Box<dyn Source>>> {
let stem = path.file_stem().await?;
let stem = stem.as_deref().unwrap_or_default();
let ext = &*path.extension().await?;

let content_type = get_content_type(path).await?;

let mut static_generation_code = "";

if mode.is_production() && page.contains(&PageSegment::Dynamic("[__metadata_id__]".into())) {
if mode.is_production() && is_multi_dynamic {
static_generation_code = indoc! {
r#"
export async function generateStaticParams() {
const sitemaps = await generateSitemaps()
const params = []

for (const item of sitemaps) {
params.push({ __metadata_id__: item.id.toString() })
}
for (const item of sitemaps) {{
params.push({ __metadata_id__: item.id.toString() + '.xml' })
}}
return params
}
"#,
Expand All @@ -252,29 +284,25 @@ async fn dynamic_site_map_route_source(
}}

export async function GET(_, ctx) {{
const {{ __metadata_id__ = [], ...params }} = ctx.params || {{}}
const targetId = __metadata_id__[0]
let id = undefined
const sitemaps = generateSitemaps ? await generateSitemaps() : null
const {{ __metadata_id__: id, ...params }} = ctx.params || {{}}
const hasXmlExtension = id ? id.endsWith('.xml') : false
if (id && !hasXmlExtension) {{
return new NextResponse('Not Found', {{
status: 404,
}})
}}

if (sitemaps) {{
id = sitemaps.find((item) => {{
if (process.env.NODE_ENV !== 'production') {{
if (item?.id == null) {{
throw new Error('id property is required for every item returned from generateSitemaps')
}}
if (process.env.NODE_ENV !== 'production' && sitemapModule.generateSitemaps) {{
const sitemaps = await sitemapModule.generateSitemaps()
for (const item of sitemaps) {{
if (item?.id == null) {{
throw new Error('id property is required for every item returned from generateSitemaps')
}}
return item.id.toString() === targetId
}})?.id

if (id == null) {{
return new NextResponse('Not Found', {{
status: 404,
}})
}}
}}

const data = await handler({{ id }})

const targetId = id && hasXmlExtension ? id.slice(0, -4) : undefined
const data = await handler({{ id: targetId }})
const content = resolveRouteData(data, fileType)

return new NextResponse(content, {{
Expand Down Expand Up @@ -324,12 +352,12 @@ async fn dynamic_image_route_source(path: Vc<FileSystemPath>) -> Result<Vc<Box<d
}}

export async function GET(_, ctx) {{
const {{ __metadata_id__ = [], ...params }} = ctx.params || {{}}
const targetId = __metadata_id__[0]
const {{ __metadata_id__, ...params }} = ctx.params || {{}}
const targetId = __metadata_id__
let id = undefined
const imageMetadata = generateImageMetadata ? await generateImageMetadata({{ params }}) : null

if (imageMetadata) {{
if (generateImageMetadata) {{
const imageMetadata = await generateImageMetadata({{ params }})
id = imageMetadata.find((item) => {{
if (process.env.NODE_ENV !== 'production') {{
if (item?.id == null) {{
Expand Down
Loading
Loading