From 498249af5128e1a83fdd4832d3037025bbf68543 Mon Sep 17 00:00:00 2001 From: "Mr.Hope" Date: Fri, 3 Jun 2022 00:55:27 +0800 Subject: [PATCH] feat(sitemap): rebuild sitemap --- packages/sitemap/src/node/compact/convert.ts | 28 +++ packages/sitemap/src/node/compact/index.ts | 1 + packages/sitemap/src/node/compact/utils.ts | 43 +++++ packages/sitemap/src/node/generateSitemap.ts | 178 ++++++++++++++++++ packages/sitemap/src/node/index.ts | 28 +-- packages/sitemap/src/node/plugin.ts | 32 ++++ packages/sitemap/src/node/sitemap.ts | 179 ------------------- packages/sitemap/src/types/frontmatter.d.ts | 13 +- packages/sitemap/src/types/index.d.ts | 2 - packages/sitemap/src/types/options.d.ts | 16 +- 10 files changed, 305 insertions(+), 215 deletions(-) create mode 100644 packages/sitemap/src/node/compact/convert.ts create mode 100644 packages/sitemap/src/node/compact/index.ts create mode 100644 packages/sitemap/src/node/compact/utils.ts create mode 100644 packages/sitemap/src/node/generateSitemap.ts create mode 100644 packages/sitemap/src/node/plugin.ts delete mode 100644 packages/sitemap/src/node/sitemap.ts diff --git a/packages/sitemap/src/node/compact/convert.ts b/packages/sitemap/src/node/compact/convert.ts new file mode 100644 index 000000000..ddb9cd94a --- /dev/null +++ b/packages/sitemap/src/node/compact/convert.ts @@ -0,0 +1,28 @@ +import { deprecatedLogger } from "./utils"; +import type { SitemapOptions } from "../../types"; + +/** @deprecated */ +export const covertOptions = ( + options: SitemapOptions & Record +): void => { + deprecatedLogger({ + options, + deprecatedOption: "urls", + newOption: "extraUrls", + }); + deprecatedLogger({ + options, + deprecatedOption: "exclude", + newOption: "excludeUrls", + }); + deprecatedLogger({ + options, + deprecatedOption: "outFile", + newOption: "sitemapFilename", + }); + deprecatedLogger({ + options, + deprecatedOption: "dateFormatter", + newOption: "modifyTimeGetter", + }); +}; diff --git a/packages/sitemap/src/node/compact/index.ts b/packages/sitemap/src/node/compact/index.ts new file mode 100644 index 000000000..9d3baa9e7 --- /dev/null +++ b/packages/sitemap/src/node/compact/index.ts @@ -0,0 +1 @@ +export * from "./convert"; diff --git a/packages/sitemap/src/node/compact/utils.ts b/packages/sitemap/src/node/compact/utils.ts new file mode 100644 index 000000000..7ba506f3f --- /dev/null +++ b/packages/sitemap/src/node/compact/utils.ts @@ -0,0 +1,43 @@ +import { black, blue } from "chalk"; + +export interface DeprecatedLoggerOptions { + options: Record; + deprecatedOption: string; + newOption: string; + msg?: string; + scope?: string; +} + +export const deprecatedLogger = ({ + options, + deprecatedOption, + newOption, + msg = "", + scope = "", +}: DeprecatedLoggerOptions): void => { + if (deprecatedOption in options) { + console.warn( + blue("Sitemap:"), + black.bgYellow("warn"), + `"${deprecatedOption}" is deprecated${ + scope ? ` in ${scope}` : "" + }, please use "${newOption}" instead.${msg ? `\n${msg}` : ""}` + ); + + if (newOption.includes(".")) { + const keys = newOption.split("."); + let temp = options; + + keys.forEach((key, index) => { + if (index !== keys.length - 1) { + // ensure level exists + temp[key] = temp[key] || {}; + + temp = temp[key] as Record; + } else temp[key] = options[deprecatedOption]; + }); + } else options[newOption] = options[deprecatedOption]; + + delete options[deprecatedOption]; + } +}; diff --git a/packages/sitemap/src/node/generateSitemap.ts b/packages/sitemap/src/node/generateSitemap.ts new file mode 100644 index 000000000..e82130788 --- /dev/null +++ b/packages/sitemap/src/node/generateSitemap.ts @@ -0,0 +1,178 @@ +import { black, blue, cyan } from "chalk"; +import { createWriteStream, readFile, existsSync, writeFile } from "fs-extra"; +import { relative, resolve } from "path"; +import { SitemapStream } from "sitemap"; + +import type { Context, Page } from "@mr-hope/vuepress-types"; +import type { + SitemapFrontmatterOption, + SitemapImageOption, + SitemapLinkOption, + SitemapNewsOption, + SitemapOptions, + SitemapVideoOption, +} from "../types"; + +interface SitemapPageInfo { + lastmod?: string; + changefreq?: string; + priority?: number; + img?: SitemapImageOption[]; + video?: SitemapVideoOption[]; + links?: SitemapLinkOption[]; + news?: SitemapNewsOption[]; +} + +const stripLocalePrefix = ( + page: Page +): { + // path of root locale + defaultPath: string; + // Locale path prefix of the page + pathLocale: string; +} => ({ + defaultPath: page.path.replace(page._localePath, "/"), + pathLocale: page._localePath, +}); + +const generatePageMap = ( + options: SitemapOptions, + { base, pages, siteConfig }: Context +): Map => { + const { + changefreq, + excludeUrls = ["/404.html"], + modifyTimeGetter = (page: Page): string => + page.updateTimeStamp ? new Date(page.updateTimeStamp).toISOString() : "", + } = options; + + const { locales = {} } = siteConfig; + + const pageLocalesMap = pages.reduce( + (map, page) => { + const { defaultPath, pathLocale } = stripLocalePrefix(page); + const pathLocales = map.get(defaultPath) || []; + + pathLocales.push(pathLocale); + + return map.set(defaultPath, pathLocales); + }, + // a map with keys of defaultPath and string[] value with pathLocales + new Map() + ); + + const pagesMap = new Map(); + + pages.forEach((page) => { + const frontmatterOptions: SitemapFrontmatterOption = + (page.frontmatter["sitemap"] as SitemapFrontmatterOption) || {}; + + const metaRobots = (page.frontmatter.meta || []).find( + (meta) => meta.name === "robots" + ); + const excludePage = metaRobots + ? (metaRobots.content || "") + .split(/,/u) + .map((content) => content.trim()) + .includes("noindex") + : frontmatterOptions.exclude; + + if (excludePage || excludeUrls.includes(page.path)) return; + + const lastmodifyTime = modifyTimeGetter(page); + const { defaultPath } = stripLocalePrefix(page); + const relatedLocales = pageLocalesMap.get(defaultPath) || []; + + let links: SitemapLinkOption[] = []; + + if (relatedLocales.length > 1) { + links = relatedLocales.map((localePrefix) => ({ + lang: locales[localePrefix]?.lang || "en", + url: `${base}${defaultPath + .replace(/^\//u, localePrefix) + .replace(/\/$/, "")}`, + })); + } + + const sitemapInfo: SitemapPageInfo = { + ...(changefreq ? { changefreq } : {}), + links, + ...(lastmodifyTime ? { lastmod: lastmodifyTime } : {}), + ...frontmatterOptions, + }; + + pagesMap.set(page.path, sitemapInfo); + }); + + return pagesMap; +}; + +export const generateSiteMap = async ( + options: SitemapOptions, + context: Context +): Promise => { + const { extraUrls = [], xmlNameSpace: xmlns } = options; + const hostname = options.hostname.replace(/\/$/, ""); + const sitemapFilename = options.sitemapFilename + ? options.sitemapFilename.replace(/^\//, "") + : "sitemap.xml"; + + console.log( + blue("Sitemap:"), + black.bgYellow("wait"), + "Generating sitemap..." + ); + + const { base, cwd, outDir } = context; + const sitemap = new SitemapStream({ + hostname, + ...(xmlns ? { xmlns } : {}), + }); + const pagesMap = generatePageMap(options, context); + const sitemapXMLPath = resolve(outDir, sitemapFilename); + const writeStream = createWriteStream(sitemapXMLPath); + + sitemap.pipe(writeStream); + + pagesMap.forEach((page, path) => + sitemap.write({ + url: `${base}${path.replace(/^\//, "")}`, + ...page, + }) + ); + + extraUrls.forEach((item) => + sitemap.write({ url: `${base}${item.replace(/^\//, "")}` }) + ); + + await new Promise((resolve) => { + sitemap.end(() => { + resolve(); + + console.log( + blue("Sitemap:"), + black.bgGreen("Success"), + `Sitemap generated and saved to ${cyan(relative(cwd, sitemapXMLPath))}` + ); + }); + }); + + const robotTxtPath = resolve(outDir, "robots.txt"); + + if (existsSync(robotTxtPath)) { + const robotsTxt = await readFile(robotTxtPath, { encoding: "utf8" }); + + const newRobotsTxtContent = `${robotsTxt.replace( + /^Sitemap: .*$/u, + "" + )}\nSitemap: ${hostname}${base}${sitemapFilename}\n`; + + await writeFile(robotTxtPath, newRobotsTxtContent, { flag: "w" }); + + console.log( + blue("Sitemap:"), + black.bgGreen("Success"), + `Appended sitemap path to ${cyan("robots.txt")}` + ); + } +}; diff --git a/packages/sitemap/src/node/index.ts b/packages/sitemap/src/node/index.ts index f6b09dc9f..81be8ff77 100644 --- a/packages/sitemap/src/node/index.ts +++ b/packages/sitemap/src/node/index.ts @@ -1,29 +1,3 @@ -import { black, blue } from "chalk"; -import { genSiteMap } from "./sitemap"; - -import type { Plugin } from "@mr-hope/vuepress-types"; -import type { SitemapOptions } from "../types"; - -const sitemapPlugin: Plugin = (options, context) => { - if (!options.hostname) { - console.log( - blue("Sitemap"), - black.bgRed("Error"), - 'Not generating sitemap because required "hostname" option doesn’t exist' - ); - - return { name: "sitemap" }; - } - - return { - name: "sitemap", - - async generated(): Promise { - await genSiteMap(options, context); - }, - - plugins: [["@mr-hope/git", true]], - }; -}; +import { sitemapPlugin } from "./plugin"; export = sitemapPlugin; diff --git a/packages/sitemap/src/node/plugin.ts b/packages/sitemap/src/node/plugin.ts new file mode 100644 index 000000000..8996ac13b --- /dev/null +++ b/packages/sitemap/src/node/plugin.ts @@ -0,0 +1,32 @@ +import { black, blue } from "chalk"; +import { covertOptions } from "./compact"; +import { generateSiteMap } from "./generateSitemap"; + +import type { Plugin, PluginOptionAPI } from "@mr-hope/vuepress-types"; +import type { SitemapOptions } from "../types"; + +export const sitemapPlugin: Plugin = (options, context) => { + covertOptions(options as SitemapOptions & Record); + + const plugin: PluginOptionAPI = { + name: "@mr-hope/vuepress-plugin-sitemap", + }; + + if (!options.hostname) { + console.log( + blue("Sitemap"), + black.bgRed("Error"), + 'Not generating sitemap because required "hostname" option doesn’t exist' + ); + + return plugin; + } + + return { + ...plugin, + + generated: async (): Promise => generateSiteMap(options, context), + + plugins: [["@mr-hope/git", true]], + }; +}; diff --git a/packages/sitemap/src/node/sitemap.ts b/packages/sitemap/src/node/sitemap.ts deleted file mode 100644 index 95602a916..000000000 --- a/packages/sitemap/src/node/sitemap.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { black, blue, cyan } from "chalk"; -import { createWriteStream, readFile, existsSync, writeFile } from "fs-extra"; -import { relative, resolve } from "path"; -import { SitemapStream } from "sitemap"; - -import type { Context, PageComputed, SiteData } from "@mr-hope/vuepress-types"; -import type { - SitemapFrontmatterOption, - SitemapLinkOption, - SitemapImageOption, - SitemapVideoOption, - SitemapOptions, - SitemapNewsOption, -} from "../types"; - -interface SitemapPageInfo { - lastmod?: string; - changefreq?: string; - priority?: number; - img?: SitemapImageOption[]; - video?: SitemapVideoOption[]; - links?: SitemapLinkOption[]; - news?: SitemapNewsOption[]; -} - -const stripLocalePrefix = ( - path: string, - localePathPrefixes: string[] -): { - normalizedPath: string; - localePrefix: string; -} => { - const matchingPrefix = localePathPrefixes - .filter((prefix) => path.startsWith(prefix)) - .shift() as string; - - return { - normalizedPath: path.replace(matchingPrefix, "/"), - localePrefix: matchingPrefix, - }; -}; - -const generatePageMap = ( - siteData: SiteData, - base: string, - options: SitemapOptions -): Map => { - const { - changefreq = "daily", - exclude = ["/404.html"], - dateFormatter = (page: PageComputed): string => - page.updateTimeStamp ? new Date(page.updateTimeStamp).toISOString() : "", - } = options; - - const { pages, locales = {} } = siteData; - - // Sort the locale keys in reverse order so that longer locales, such as '/en/', match before the default '/' - const localeKeys = Object.keys(locales).sort().reverse(); - const localesByNormalizedPagePath = pages.reduce((map, page) => { - const { normalizedPath, localePrefix } = stripLocalePrefix( - page.path, - localeKeys - ); - const prefixesByPath = map.get(normalizedPath) || []; - - prefixesByPath.push(localePrefix); - - return map.set(normalizedPath, prefixesByPath); - }, new Map()); - - const pagesMap = new Map(); - - pages.forEach((page) => { - const frontmatterOptions: SitemapFrontmatterOption = - (page.frontmatter["sitemap"] as SitemapFrontmatterOption) || {}; - const metaRobots = (page.frontmatter.meta || []).find( - (meta) => meta.name === "robots" - ); - const excludePage = metaRobots - ? (metaRobots.content || "") - .split(/,/u) - .map((content) => content.trim()) - .includes("noindex") - : frontmatterOptions.exclude; - - if (excludePage) exclude.push(page.path); - - const lastmodifyTime = dateFormatter(page); - const { normalizedPath } = stripLocalePrefix(page.path, localeKeys); - const relatedLocales = - localesByNormalizedPagePath.get(normalizedPath) || []; - - let links: SitemapLinkOption[] = []; - - if (relatedLocales.length > 1) - links = relatedLocales.map((localePrefix) => ({ - lang: locales[localePrefix].lang || "en", - url: `${base}${normalizedPath.replace("/", localePrefix)}`, - })); - - const sitemapInfo: SitemapPageInfo = { - changefreq, - links, - ...(lastmodifyTime ? { lastmod: lastmodifyTime } : {}), - ...frontmatterOptions, - }; - - pagesMap.set(page.path, sitemapInfo); - }); - - return pagesMap; -}; - -export const genSiteMap = async ( - options: SitemapOptions, - context: Context -): Promise => { - console.log( - blue("Sitemap:"), - black.bgYellow("wait"), - "Generating sitemap..." - ); - - const siteData = context.getSiteData(); - - const { - hostname, - urls = [], - outFile = "sitemap.xml", - exclude = [], - xmlNameSpace: xmlns, - } = options; - const sitemap = new SitemapStream({ - hostname, - ...(xmlns ? { xmlns } : {}), - }); - const sitemapXMLPath = resolve(context.outDir, outFile); - const writeStream = createWriteStream(sitemapXMLPath); - - sitemap.pipe(writeStream); - - const base = siteData.base.replace(/\/$/u, ""); - const pagesMap = generatePageMap(siteData, base, options); - - pagesMap.forEach((page, url) => { - if (!exclude.includes(url)) - sitemap.write({ url: `${base}${url}`, ...page }); - }); - - urls.forEach((item) => sitemap.write(item)); - sitemap.end(); - console.log( - blue("Sitemap:"), - black.bgGreen("Success"), - `Sitemap generated and saved to ${cyan( - relative(context.cwd, sitemapXMLPath) - )}` - ); - - const robotTxtPath = resolve(context.outDir, "robots.txt"); - const robotsTxt = existsSync(robotTxtPath) - ? await readFile(robotTxtPath, { encoding: "utf8" }) - : ""; - - const newRobotsTxtContent = `${robotsTxt.replace( - /^Sitemap: .*$/u, - "" - )}\nSitemap: ${options.hostname.replace(/\/$/u, "")}${ - context.base - }${outFile}\n`; - - await writeFile(robotTxtPath, newRobotsTxtContent, { flag: "w" }); - - console.log( - blue("Sitemap:"), - black.bgGreen("Success"), - `Appended sitemap path to ${cyan("robots.txt")}` - ); -}; diff --git a/packages/sitemap/src/types/frontmatter.d.ts b/packages/sitemap/src/types/frontmatter.d.ts index 8351ced08..57ffcc9be 100644 --- a/packages/sitemap/src/types/frontmatter.d.ts +++ b/packages/sitemap/src/types/frontmatter.d.ts @@ -1,4 +1,8 @@ -import type { SitemapImageOption, SitemapVideoOption } from "./sitemap"; +import type { + SitemapImageOption, + SitemapNewsOption, + SitemapVideoOption, +} from "./sitemap"; export interface SitemapFrontmatterOption { /** @@ -42,4 +46,11 @@ export interface SitemapFrontmatterOption { * 视频配置 */ video?: SitemapVideoOption[]; + + /** + * News config + * + * 新闻配置 + */ + news?: SitemapNewsOption[]; } diff --git a/packages/sitemap/src/types/index.d.ts b/packages/sitemap/src/types/index.d.ts index 674d05903..1551d05e0 100644 --- a/packages/sitemap/src/types/index.d.ts +++ b/packages/sitemap/src/types/index.d.ts @@ -1,5 +1,3 @@ -/* eslint-disable @typescript-eslint/naming-convention */ - export * from "./frontmatter"; export * from "./options"; export * from "./sitemap"; diff --git a/packages/sitemap/src/types/options.d.ts b/packages/sitemap/src/types/options.d.ts index 34e4082ab..51246641c 100644 --- a/packages/sitemap/src/types/options.d.ts +++ b/packages/sitemap/src/types/options.d.ts @@ -1,10 +1,12 @@ import type { PageComputed } from "@mr-hope/vuepress-types"; +export type ModifyTimeGetter = (page: PageComputed) => string; + export interface SitemapOptions { /** * domain which to be deployed to * - * 网站域名 + * 部署的网站域名 */ hostname: string; @@ -13,23 +15,23 @@ export interface SitemapOptions { * * 需要额外包含的网址 */ - urls?: string[]; + extraUrls?: string[]; /** * Urls to be excluded * * 不被收录的页面 */ - exclude?: string[]; + excludeUrls?: string[]; /** - * Output file name, relative to dest folder + * Output filename, relative to dest folder * * 输出的文件名，相对于输出目录 * * @default 'sitemap.xml' */ - outFile?: string; + sitemapFilename?: string; /** * Page default update frequency @@ -49,8 +51,10 @@ export interface SitemapOptions { /** * Date format function + * + * 时间格式化器 */ - dateFormatter?: (page: PageComputed) => string; + modifyTimeGetter?: ModifyTimeGetter; /** * XML namespaces to turn on - all by default