From da004a2ae704d6213ece35c9d180178269f423a2 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 8 Dec 2023 20:36:11 +0100 Subject: [PATCH 01/15] implement parseFrontMatter --- packages/docusaurus-mdx-loader/src/loader.ts | 12 +- .../src/__tests__/feed.test.ts | 7 + .../src/__tests__/index.test.ts | 19 +- .../src/blogUtils.ts | 31 +++- .../src/docs.ts | 16 +- .../src/index.ts | 8 +- packages/docusaurus-types/src/config.d.ts | 22 +++ packages/docusaurus-types/src/index.d.ts | 2 + .../__snapshots__/markdownUtils.test.ts.snap | 46 +++-- .../src/__tests__/markdownUtils.test.ts | 174 +++++++++++------- packages/docusaurus-utils/src/index.ts | 4 +- .../docusaurus-utils/src/markdownUtils.ts | 35 +++- .../__snapshots__/config.test.ts.snap | 10 + .../__snapshots__/index.test.ts.snap | 1 + .../server/__tests__/configValidation.test.ts | 4 + .../docusaurus/src/server/configValidation.ts | 33 ++-- .../tests/visibility/force_unlisted.mdx | 10 + website/_dogfooding/dogfooding.config.ts | 9 + website/docusaurus.config.ts | 8 + 19 files changed, 329 insertions(+), 122 deletions(-) create mode 100644 website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index a475220cd5c6..bde02542beb3 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -8,7 +8,7 @@ import fs from 'fs-extra'; import logger from '@docusaurus/logger'; import { - parseFrontMatter, + DEFAULT_PARSE_FRONT_MATTER, escapePath, getFileLoaderUtils, getWebpackLoaderCompilerName, @@ -133,7 +133,7 @@ function extractContentTitleData(data: { export async function mdxLoader( this: LoaderContext, - fileString: string, + fileContent: string, ): Promise { const compilerName = getWebpackLoaderCompilerName(this); const callback = this.async(); @@ -143,11 +143,15 @@ export async function mdxLoader( ensureMarkdownConfig(reqOptions); - const {frontMatter} = parseFrontMatter(fileString); + const {frontMatter} = await reqOptions.markdownConfig.parseFrontMatter({ + filePath, + fileContent, + defaultParseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, + }); const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx); const preprocessedContent = preprocessor({ - fileContent: fileString, + fileContent, filePath, admonitions: reqOptions.admonitions, markdownConfig: reqOptions.markdownConfig, diff --git a/packages/docusaurus-plugin-content-blog/src/__tests__/feed.test.ts b/packages/docusaurus-plugin-content-blog/src/__tests__/feed.test.ts index f928296bf9df..022d4ce749a9 100644 --- a/packages/docusaurus-plugin-content-blog/src/__tests__/feed.test.ts +++ b/packages/docusaurus-plugin-content-blog/src/__tests__/feed.test.ts @@ -8,6 +8,7 @@ import {jest} from '@jest/globals'; import path from 'path'; import fs from 'fs-extra'; +import {DEFAULT_PARSE_FRONT_MATTER} from '@docusaurus/utils'; import {DEFAULT_OPTIONS} from '../options'; import {generateBlogPosts} from '../blogUtils'; import {createBlogFeedFiles} from '../feed'; @@ -31,6 +32,8 @@ const DefaultI18N: I18n = { }, }; +const markdown = {parseFrontMatter: DEFAULT_PARSE_FRONT_MATTER}; + function getBlogContentPaths(siteDir: string): BlogContentPaths { return { contentPath: path.resolve(siteDir, 'blog'), @@ -72,6 +75,7 @@ describe.each(['atom', 'rss', 'json'])('%s', (feedType) => { baseUrl: '/', url: 'https://docusaurus.io', favicon: 'image/favicon.ico', + markdown, }; const outDir = path.join(siteDir, 'build-snap'); @@ -110,6 +114,7 @@ describe.each(['atom', 'rss', 'json'])('%s', (feedType) => { baseUrl: '/myBaseUrl/', url: 'https://docusaurus.io', favicon: 'image/favicon.ico', + markdown, }; // Build is quite difficult to mock, so we built the blog beforehand and @@ -152,6 +157,7 @@ describe.each(['atom', 'rss', 'json'])('%s', (feedType) => { baseUrl: '/myBaseUrl/', url: 'https://docusaurus.io', favicon: 'image/favicon.ico', + markdown, }; // Build is quite difficult to mock, so we built the blog beforehand and @@ -204,6 +210,7 @@ describe.each(['atom', 'rss', 'json'])('%s', (feedType) => { baseUrl: '/myBaseUrl/', url: 'https://docusaurus.io', favicon: 'image/favicon.ico', + markdown, }; // Build is quite difficult to mock, so we built the blog beforehand and diff --git a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts index 8b392611e741..da0caf9b4541 100644 --- a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts +++ b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts @@ -8,7 +8,11 @@ import {jest} from '@jest/globals'; import path from 'path'; import {normalizePluginOptions} from '@docusaurus/utils-validation'; -import {posixPath, getFileCommitDate} from '@docusaurus/utils'; +import { + posixPath, + getFileCommitDate, + DEFAULT_PARSE_FRONT_MATTER, +} from '@docusaurus/utils'; import pluginContentBlog from '../index'; import {validateOptions} from '../options'; import type { @@ -16,6 +20,7 @@ import type { LoadContext, I18n, Validate, + MarkdownConfig, } from '@docusaurus/types'; import type { BlogPost, @@ -24,6 +29,17 @@ import type { EditUrlFunction, } from '@docusaurus/plugin-content-blog'; +const markdown: MarkdownConfig = { + format: 'mdx', + mermaid: true, + mdx1Compat: { + comments: true, + headingIds: true, + admonitions: true, + }, + parseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, +}; + function findByTitle( blogPosts: BlogPost[], title: string, @@ -81,6 +97,7 @@ const getPlugin = async ( title: 'Hello', baseUrl: '/', url: 'https://docusaurus.io', + markdown, } as DocusaurusConfig; return pluginContentBlog( { diff --git a/packages/docusaurus-plugin-content-blog/src/blogUtils.ts b/packages/docusaurus-plugin-content-blog/src/blogUtils.ts index 0a8d2a0e0b67..3bbb5301bf92 100644 --- a/packages/docusaurus-plugin-content-blog/src/blogUtils.ts +++ b/packages/docusaurus-plugin-content-blog/src/blogUtils.ts @@ -11,7 +11,7 @@ import _ from 'lodash'; import logger from '@docusaurus/logger'; import readingTime from 'reading-time'; import { - parseMarkdownString, + parseMarkdownFile, normalizeUrl, aliasedSitePath, getEditUrl, @@ -29,7 +29,7 @@ import { } from '@docusaurus/utils'; import {validateBlogPostFrontMatter} from './frontMatter'; import {type AuthorsMap, getAuthorsMap, getBlogPostAuthors} from './authors'; -import type {LoadContext} from '@docusaurus/types'; +import type {LoadContext, ParseFrontMatter} from '@docusaurus/types'; import type { PluginOptions, ReadingTimeFunction, @@ -180,10 +180,19 @@ function formatBlogPostDate( } } -async function parseBlogPostMarkdownFile(blogSourceAbsolute: string) { - const markdownString = await fs.readFile(blogSourceAbsolute, 'utf-8'); +async function parseBlogPostMarkdownFile({ + filePath, + parseFrontMatter, +}: { + filePath: string; + parseFrontMatter: ParseFrontMatter; +}) { + const fileContent = await fs.readFile(filePath, 'utf-8'); try { - const result = parseMarkdownString(markdownString, { + const result = await parseMarkdownFile({ + filePath, + fileContent, + parseFrontMatter, removeContentTitle: true, }); return { @@ -191,7 +200,7 @@ async function parseBlogPostMarkdownFile(blogSourceAbsolute: string) { frontMatter: validateBlogPostFrontMatter(result.frontMatter), }; } catch (err) { - logger.error`Error while parsing blog post file path=${blogSourceAbsolute}.`; + logger.error`Error while parsing blog post file path=${filePath}.`; throw err; } } @@ -207,7 +216,10 @@ async function processBlogSourceFile( authorsMap?: AuthorsMap, ): Promise { const { - siteConfig: {baseUrl}, + siteConfig: { + baseUrl, + markdown: {parseFrontMatter}, + }, siteDir, i18n, } = context; @@ -228,7 +240,10 @@ async function processBlogSourceFile( const blogSourceAbsolute = path.join(blogDirPath, blogSourceRelative); const {frontMatter, content, contentTitle, excerpt} = - await parseBlogPostMarkdownFile(blogSourceAbsolute); + await parseBlogPostMarkdownFile({ + filePath: blogSourceAbsolute, + parseFrontMatter, + }); const aliasedSource = aliasedSitePath(blogSourceAbsolute, siteDir); diff --git a/packages/docusaurus-plugin-content-docs/src/docs.ts b/packages/docusaurus-plugin-content-docs/src/docs.ts index 8ee73cf40628..2907ac7211f3 100644 --- a/packages/docusaurus-plugin-content-docs/src/docs.ts +++ b/packages/docusaurus-plugin-content-docs/src/docs.ts @@ -15,7 +15,7 @@ import { getFolderContainingFile, getContentPathList, normalizeUrl, - parseMarkdownString, + parseMarkdownFile, posixPath, Globby, normalizeFrontMatterTags, @@ -140,13 +140,23 @@ async function doProcessDocMetadata({ env: DocEnv; }): Promise { const {source, content, contentPath, filePath} = docFile; - const {siteDir, i18n} = context; + const { + siteDir, + i18n, + siteConfig: { + markdown: {parseFrontMatter}, + }, + } = context; const { frontMatter: unsafeFrontMatter, contentTitle, excerpt, - } = parseMarkdownString(content); + } = await parseMarkdownFile({ + filePath, + fileContent: content, + parseFrontMatter, + }); const frontMatter = validateDocFrontMatter(unsafeFrontMatter); const { diff --git a/packages/docusaurus-plugin-content-pages/src/index.ts b/packages/docusaurus-plugin-content-pages/src/index.ts index e62d07c6cbaf..a4707110f2b4 100644 --- a/packages/docusaurus-plugin-content-pages/src/index.ts +++ b/packages/docusaurus-plugin-content-pages/src/index.ts @@ -19,7 +19,7 @@ import { createAbsoluteFilePathMatcher, normalizeUrl, DEFAULT_PLUGIN_ID, - parseMarkdownString, + parseMarkdownFile, isUnlisted, isDraft, } from '@docusaurus/utils'; @@ -113,7 +113,11 @@ export default function pluginContentPages( frontMatter: unsafeFrontMatter, contentTitle, excerpt, - } = parseMarkdownString(content); + } = await parseMarkdownFile({ + filePath: source, + fileContent: content, + parseFrontMatter: siteConfig.markdown.parseFrontMatter, + }); const frontMatter = validatePageFrontMatter(unsafeFrontMatter); if (isDraft({frontMatter})) { diff --git a/packages/docusaurus-types/src/config.d.ts b/packages/docusaurus-types/src/config.d.ts index 3a7bb99ae743..0d872e4001c7 100644 --- a/packages/docusaurus-types/src/config.d.ts +++ b/packages/docusaurus-types/src/config.d.ts @@ -27,6 +27,20 @@ export type MDX1CompatOptions = { headingIds: boolean; }; +export type ParseFrontMatterParams = {filePath: string; fileContent: string}; +export type ParseFrontMatterResult = { + frontMatter: {[key: string]: unknown}; + content: string; +}; +export type DefaultParseFrontMatter = ( + params: ParseFrontMatterParams, +) => Promise; +export type ParseFrontMatter = ( + params: ParseFrontMatterParams & { + defaultParseFrontMatter: DefaultParseFrontMatter; + }, +) => Promise; + export type MarkdownConfig = { /** * The Markdown format to use by default. @@ -44,6 +58,14 @@ export type MarkdownConfig = { */ format: 'mdx' | 'md' | 'detect'; + /** + * A function callback that lets users parse the front matter themselves. + * Gives the opportunity to read it from a different source, or process it. + * + * @see https://github.com/facebook/docusaurus/issues/5568 + */ + parseFrontMatter: ParseFrontMatter; + /** * Allow mermaid language code blocks to be rendered into Mermaid diagrams: * diff --git a/packages/docusaurus-types/src/index.d.ts b/packages/docusaurus-types/src/index.d.ts index 53e83ce96345..257ec57811de 100644 --- a/packages/docusaurus-types/src/index.d.ts +++ b/packages/docusaurus-types/src/index.d.ts @@ -9,6 +9,8 @@ export { ReportingSeverity, ThemeConfig, MarkdownConfig, + DefaultParseFrontMatter, + ParseFrontMatter, DocusaurusConfig, Config, } from './config'; diff --git a/packages/docusaurus-utils/src/__tests__/__snapshots__/markdownUtils.test.ts.snap b/packages/docusaurus-utils/src/__tests__/__snapshots__/markdownUtils.test.ts.snap index 5a65f0bb828f..8fb7a03dfa2f 100644 --- a/packages/docusaurus-utils/src/__tests__/__snapshots__/markdownUtils.test.ts.snap +++ b/packages/docusaurus-utils/src/__tests__/__snapshots__/markdownUtils.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`parseMarkdownString deletes only first heading 1`] = ` +exports[`parseMarkdownFile deletes only first heading 1`] = ` { "content": "# Markdown Title @@ -15,7 +15,7 @@ test test test # test bar } `; -exports[`parseMarkdownString deletes only first heading 2 1`] = ` +exports[`parseMarkdownFile deletes only first heading 2 1`] = ` { "content": "# test @@ -30,7 +30,7 @@ test3", } `; -exports[`parseMarkdownString does not warn for duplicate title if markdown title is not at the top 1`] = ` +exports[`parseMarkdownFile does not warn for duplicate title if markdown title is not at the top 1`] = ` { "content": "foo @@ -43,7 +43,7 @@ exports[`parseMarkdownString does not warn for duplicate title if markdown title } `; -exports[`parseMarkdownString handles code blocks 1`] = ` +exports[`parseMarkdownFile handles code blocks 1`] = ` { "content": "\`\`\`js code @@ -56,7 +56,7 @@ Content", } `; -exports[`parseMarkdownString handles code blocks 2`] = ` +exports[`parseMarkdownFile handles code blocks 2`] = ` { "content": "\`\`\`\`js Foo @@ -73,7 +73,7 @@ Content", } `; -exports[`parseMarkdownString handles code blocks 3`] = ` +exports[`parseMarkdownFile handles code blocks 3`] = ` { "content": "\`\`\`\`js Foo @@ -88,7 +88,7 @@ Content", } `; -exports[`parseMarkdownString ignores markdown title if its not a first text 1`] = ` +exports[`parseMarkdownFile ignores markdown title if its not a first text 1`] = ` { "content": "foo # test", @@ -98,18 +98,32 @@ exports[`parseMarkdownString ignores markdown title if its not a first text 1`] } `; -exports[`parseMarkdownString parse markdown with front matter 1`] = ` +exports[`parseMarkdownFile parse markdown with custom front matter parser 1`] = ` { "content": "Some text", "contentTitle": undefined, "excerpt": "Some text", "frontMatter": { + "age": 84, + "extra": "value", + "great": true, "title": "Frontmatter title", }, } `; -exports[`parseMarkdownString parses first heading as contentTitle 1`] = ` +exports[`parseMarkdownFile parse markdown with front matter 1`] = ` +{ + "content": "Some text", + "contentTitle": undefined, + "excerpt": "Some text", + "frontMatter": { + "title": "Frontmatter title", + }, +} +`; + +exports[`parseMarkdownFile parses first heading as contentTitle 1`] = ` { "content": "# Markdown Title @@ -120,7 +134,7 @@ Some text", } `; -exports[`parseMarkdownString parses front-matter and ignore h2 1`] = ` +exports[`parseMarkdownFile parses front-matter and ignore h2 1`] = ` { "content": "## test", "contentTitle": undefined, @@ -131,7 +145,7 @@ exports[`parseMarkdownString parses front-matter and ignore h2 1`] = ` } `; -exports[`parseMarkdownString parses title only 1`] = ` +exports[`parseMarkdownFile parses title only 1`] = ` { "content": "# test", "contentTitle": "test", @@ -140,7 +154,7 @@ exports[`parseMarkdownString parses title only 1`] = ` } `; -exports[`parseMarkdownString parses title only alternate 1`] = ` +exports[`parseMarkdownFile parses title only alternate 1`] = ` { "content": "test ===", @@ -150,7 +164,7 @@ exports[`parseMarkdownString parses title only alternate 1`] = ` } `; -exports[`parseMarkdownString reads front matter only 1`] = ` +exports[`parseMarkdownFile reads front matter only 1`] = ` { "content": "", "contentTitle": undefined, @@ -161,7 +175,7 @@ exports[`parseMarkdownString reads front matter only 1`] = ` } `; -exports[`parseMarkdownString warns about duplicate titles (front matter + markdown alternate) 1`] = ` +exports[`parseMarkdownFile warns about duplicate titles (front matter + markdown alternate) 1`] = ` { "content": "Markdown Title alternate ================ @@ -175,7 +189,7 @@ Some text", } `; -exports[`parseMarkdownString warns about duplicate titles (front matter + markdown) 1`] = ` +exports[`parseMarkdownFile warns about duplicate titles (front matter + markdown) 1`] = ` { "content": "# Markdown Title @@ -188,7 +202,7 @@ Some text", } `; -exports[`parseMarkdownString warns about duplicate titles 1`] = ` +exports[`parseMarkdownFile warns about duplicate titles 1`] = ` { "content": "# test", "contentTitle": "test", diff --git a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts index 182c95b05ffe..0c14a07d9cc3 100644 --- a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts +++ b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts @@ -9,12 +9,13 @@ import dedent from 'dedent'; import { createExcerpt, parseMarkdownContentTitle, - parseMarkdownString, parseMarkdownHeadingId, writeMarkdownHeadingId, escapeMarkdownHeadingIds, unwrapMdxCodeBlocks, admonitionTitleToDirectiveLabel, + parseMarkdownFile, + DEFAULT_PARSE_FRONT_MATTER, } from '../markdownUtils'; describe('createExcerpt', () => { @@ -623,32 +624,73 @@ Lorem Ipsum }); }); -describe('parseMarkdownString', () => { - it('parse markdown with front matter', () => { - expect( - parseMarkdownString(dedent` +describe('parseMarkdownFile', () => { + async function test( + fileContent: string, + options?: Partial>[0], + ) { + return parseMarkdownFile({ + fileContent, + filePath: 'some-file-path.mdx', + parseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, + ...options, + }); + } + + it('parse markdown with front matter', async () => { + await expect( + test(dedent` --- title: Frontmatter title --- Some text `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('parses first heading as contentTitle', () => { - expect( - parseMarkdownString(dedent` + it('parse markdown with custom front matter parser', async () => { + await expect( + test( + dedent` + --- + title: Frontmatter title + age: 42 + --- + + Some text + `, + { + parseFrontMatter: async (params) => { + const result = await params.defaultParseFrontMatter(params); + return { + ...result, + frontMatter: { + ...result.frontMatter, + age: result.frontMatter.age * 2, + extra: 'value', + great: true, + }, + }; + }, + }, + ), + ).resolves.toMatchSnapshot(); + }); + + it('parses first heading as contentTitle', async () => { + await expect( + test(dedent` # Markdown Title Some text `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('warns about duplicate titles (front matter + markdown)', () => { - expect( - parseMarkdownString(dedent` + it('warns about duplicate titles (front matter + markdown)', async () => { + await expect( + test(dedent` --- title: Frontmatter title --- @@ -657,12 +699,12 @@ describe('parseMarkdownString', () => { Some text `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('warns about duplicate titles (front matter + markdown alternate)', () => { - expect( - parseMarkdownString(dedent` + it('warns about duplicate titles (front matter + markdown alternate)', async () => { + await expect( + test(dedent` --- title: Frontmatter title --- @@ -672,12 +714,12 @@ describe('parseMarkdownString', () => { Some text `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('does not warn for duplicate title if markdown title is not at the top', () => { - expect( - parseMarkdownString(dedent` + it('does not warn for duplicate title if markdown title is not at the top', async () => { + await expect( + test(dedent` --- title: Frontmatter title --- @@ -686,12 +728,12 @@ describe('parseMarkdownString', () => { # Markdown Title `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('deletes only first heading', () => { - expect( - parseMarkdownString(dedent` + it('deletes only first heading', async () => { + await expect( + test(dedent` # Markdown Title test test test # test bar @@ -700,12 +742,12 @@ describe('parseMarkdownString', () => { ### Markdown Title h3 `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('parses front-matter and ignore h2', () => { - expect( - parseMarkdownString( + it('parses front-matter and ignore h2', async () => { + await expect( + test( dedent` --- title: Frontmatter title @@ -713,55 +755,55 @@ describe('parseMarkdownString', () => { ## test `, ), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('reads front matter only', () => { - expect( - parseMarkdownString(dedent` + it('reads front matter only', async () => { + await expect( + test(dedent` --- title: test --- `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('parses title only', () => { - expect(parseMarkdownString('# test')).toMatchSnapshot(); + it('parses title only', async () => { + await expect(test('# test')).resolves.toMatchSnapshot(); }); - it('parses title only alternate', () => { - expect( - parseMarkdownString(dedent` + it('parses title only alternate', async () => { + await expect( + test(dedent` test === `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('warns about duplicate titles', () => { - expect( - parseMarkdownString(dedent` + it('warns about duplicate titles', async () => { + await expect( + test(dedent` --- title: Frontmatter title --- # test `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('ignores markdown title if its not a first text', () => { - expect( - parseMarkdownString(dedent` + it('ignores markdown title if its not a first text', async () => { + await expect( + test(dedent` foo # test `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('deletes only first heading 2', () => { - expect( - parseMarkdownString(dedent` + it('deletes only first heading 2', async () => { + await expect( + test(dedent` # test test test test test test test @@ -770,21 +812,21 @@ describe('parseMarkdownString', () => { ### test test3 `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('handles code blocks', () => { - expect( - parseMarkdownString(dedent` + it('handles code blocks', async () => { + await expect( + test(dedent` \`\`\`js code \`\`\` Content `), - ).toMatchSnapshot(); - expect( - parseMarkdownString(dedent` + ).resolves.toMatchSnapshot(); + await expect( + test(dedent` \`\`\`\`js Foo \`\`\`diff @@ -795,9 +837,9 @@ describe('parseMarkdownString', () => { Content `), - ).toMatchSnapshot(); - expect( - parseMarkdownString(dedent` + ).resolves.toMatchSnapshot(); + await expect( + test(dedent` \`\`\`\`js Foo \`\`\`diff @@ -806,17 +848,17 @@ describe('parseMarkdownString', () => { Content `), - ).toMatchSnapshot(); + ).resolves.toMatchSnapshot(); }); - it('throws for invalid front matter', () => { - expect(() => - parseMarkdownString(dedent` + it('throws for invalid front matter', async () => { + await expect( + test(dedent` --- foo: f: a --- `), - ).toThrowErrorMatchingInlineSnapshot(` + ).rejects.toThrowErrorMatchingInlineSnapshot(` "incomplete explicit mapping pair; a key node is missed; or followed by a non-tabulated empty line at line 2, column 7: foo: f: a ^" diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts index 5bb77a0c054f..5b374898bf62 100644 --- a/packages/docusaurus-utils/src/index.ts +++ b/packages/docusaurus-utils/src/index.ts @@ -70,9 +70,9 @@ export { unwrapMdxCodeBlocks, admonitionTitleToDirectiveLabel, createExcerpt, - parseFrontMatter, + DEFAULT_PARSE_FRONT_MATTER, parseMarkdownContentTitle, - parseMarkdownString, + parseMarkdownFile, writeMarkdownHeadingId, type WriteHeadingIDOptions, } from './markdownUtils'; diff --git a/packages/docusaurus-utils/src/markdownUtils.ts b/packages/docusaurus-utils/src/markdownUtils.ts index a2ca3db101e9..c79b3c100281 100644 --- a/packages/docusaurus-utils/src/markdownUtils.ts +++ b/packages/docusaurus-utils/src/markdownUtils.ts @@ -8,6 +8,10 @@ import logger from '@docusaurus/logger'; import matter from 'gray-matter'; import {createSlugger, type Slugger, type SluggerOptions} from './slugger'; +import type { + ParseFrontMatter, + DefaultParseFrontMatter, +} from '@docusaurus/types'; // Some utilities for parsing Markdown content. These things are only used on // server-side when we infer metadata like `title` and `description` from the @@ -214,18 +218,21 @@ export function createExcerpt(fileString: string): string | undefined { * --- * ``` */ -export function parseFrontMatter(markdownFileContent: string): { +function parseFileContentFrontMatter(fileContent: string): { /** Front matter as parsed by gray-matter. */ frontMatter: {[key: string]: unknown}; /** The remaining content, trimmed. */ content: string; } { - const {data, content} = matter(markdownFileContent); + const {data, content} = matter(fileContent); return { frontMatter: data, content: content.trim(), }; } +export const DEFAULT_PARSE_FRONT_MATTER: DefaultParseFrontMatter = async ( + params, +) => parseFileContentFrontMatter(params.fileContent); function toTextContentTitle(contentTitle: string): string { return contentTitle.replace(/`(?[^`]*)`/g, '$'); @@ -309,10 +316,16 @@ export function parseMarkdownContentTitle( * @throws Throws when `parseFrontMatter` throws, usually because of invalid * syntax. */ -export function parseMarkdownString( - markdownFileContent: string, - options?: ParseMarkdownContentTitleOptions, -): { +export async function parseMarkdownFile({ + filePath, + fileContent, + parseFrontMatter, + removeContentTitle, +}: { + filePath: string; + fileContent: string; + parseFrontMatter: ParseFrontMatter; +} & ParseMarkdownContentTitleOptions): Promise<{ /** @see {@link parseFrontMatter} */ frontMatter: {[key: string]: unknown}; /** @see {@link parseMarkdownContentTitle} */ @@ -324,14 +337,18 @@ export function parseMarkdownString( * the `removeContentTitle` option. */ content: string; -} { +}> { try { const {frontMatter, content: contentWithoutFrontMatter} = - parseFrontMatter(markdownFileContent); + await parseFrontMatter({ + filePath, + fileContent, + defaultParseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, + }); const {content, contentTitle} = parseMarkdownContentTitle( contentWithoutFrontMatter, - options, + {removeContentTitle}, ); const excerpt = createExcerpt(content); diff --git a/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap b/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap index 2ed2b796fd01..c5d21f81aefc 100644 --- a/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap +++ b/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap @@ -24,6 +24,7 @@ exports[`loadSiteConfig website with .cjs siteConfig 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -72,6 +73,7 @@ exports[`loadSiteConfig website with ts + js config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -120,6 +122,7 @@ exports[`loadSiteConfig website with valid JS CJS config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -168,6 +171,7 @@ exports[`loadSiteConfig website with valid JS ESM config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -216,6 +220,7 @@ exports[`loadSiteConfig website with valid TypeScript CJS config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -264,6 +269,7 @@ exports[`loadSiteConfig website with valid TypeScript ESM config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -312,6 +318,7 @@ exports[`loadSiteConfig website with valid async config 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -362,6 +369,7 @@ exports[`loadSiteConfig website with valid async config creator function 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -412,6 +420,7 @@ exports[`loadSiteConfig website with valid config creator function 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, @@ -465,6 +474,7 @@ exports[`loadSiteConfig website with valid siteConfig 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, diff --git a/packages/docusaurus/src/server/__tests__/__snapshots__/index.test.ts.snap b/packages/docusaurus/src/server/__tests__/__snapshots__/index.test.ts.snap index 45b94b869406..06caa4c997dc 100644 --- a/packages/docusaurus/src/server/__tests__/__snapshots__/index.test.ts.snap +++ b/packages/docusaurus/src/server/__tests__/__snapshots__/index.test.ts.snap @@ -98,6 +98,7 @@ exports[`load loads props for site with custom i18n path 1`] = ` "headingIds": true, }, "mermaid": false, + "parseFrontMatter": [Function], "preprocessor": undefined, }, "noIndex": false, diff --git a/packages/docusaurus/src/server/__tests__/configValidation.test.ts b/packages/docusaurus/src/server/__tests__/configValidation.test.ts index b3bc7b2611ef..925207b6366f 100644 --- a/packages/docusaurus/src/server/__tests__/configValidation.test.ts +++ b/packages/docusaurus/src/server/__tests__/configValidation.test.ts @@ -61,6 +61,8 @@ describe('normalizeConfig', () => { markdown: { format: 'md', mermaid: true, + parseFrontMatter: async (params) => + params.defaultParseFrontMatter(params), preprocessor: ({fileContent}) => fileContent, mdx1Compat: { comments: true, @@ -504,6 +506,8 @@ describe('markdown', () => { const markdown: DocusaurusConfig['markdown'] = { format: 'md', mermaid: true, + parseFrontMatter: async (params) => + params.defaultParseFrontMatter(params), preprocessor: ({fileContent}) => fileContent, mdx1Compat: { comments: false, diff --git a/packages/docusaurus/src/server/configValidation.ts b/packages/docusaurus/src/server/configValidation.ts index 3f9de2ce6807..193b8eb6a73d 100644 --- a/packages/docusaurus/src/server/configValidation.ts +++ b/packages/docusaurus/src/server/configValidation.ts @@ -6,6 +6,7 @@ */ import { + DEFAULT_PARSE_FRONT_MATTER, DEFAULT_STATIC_DIR_NAME, DEFAULT_I18N_DIR_NAME, addLeadingSlash, @@ -13,7 +14,11 @@ import { removeTrailingSlash, } from '@docusaurus/utils'; import {Joi, printWarning} from '@docusaurus/utils-validation'; -import type {DocusaurusConfig, I18nConfig} from '@docusaurus/types'; +import type { + DocusaurusConfig, + I18nConfig, + MarkdownConfig, +} from '@docusaurus/types'; const DEFAULT_I18N_LOCALE = 'en'; @@ -24,6 +29,18 @@ export const DEFAULT_I18N_CONFIG: I18nConfig = { localeConfigs: {}, }; +export const DEFAULT_MARKDOWN_CONFIG: MarkdownConfig = { + format: 'mdx', // TODO change this to "detect" in Docusaurus v4? + mermaid: false, + preprocessor: undefined, + parseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, + mdx1Compat: { + comments: true, + admonitions: true, + headingIds: true, + }, +}; + export const DEFAULT_CONFIG: Pick< DocusaurusConfig, | 'i18n' @@ -64,16 +81,7 @@ export const DEFAULT_CONFIG: Pick< tagline: '', baseUrlIssueBanner: true, staticDirectories: [DEFAULT_STATIC_DIR_NAME], - markdown: { - format: 'mdx', // TODO change this to "detect" in Docusaurus v4? - mermaid: false, - preprocessor: undefined, - mdx1Compat: { - comments: true, - admonitions: true, - headingIds: true, - }, - }, + markdown: DEFAULT_MARKDOWN_CONFIG, }; function createPluginSchema(theme: boolean) { @@ -280,6 +288,9 @@ export const ConfigSchema = Joi.object({ format: Joi.string() .equal('mdx', 'md', 'detect') .default(DEFAULT_CONFIG.markdown.format), + parseFrontMatter: Joi.function().default( + () => DEFAULT_CONFIG.markdown.parseFrontMatter, + ), mermaid: Joi.boolean().default(DEFAULT_CONFIG.markdown.mermaid), preprocessor: Joi.function() .arity(1) diff --git a/website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx b/website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx new file mode 100644 index 000000000000..08018984425a --- /dev/null +++ b/website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx @@ -0,0 +1,10 @@ +--- +unlisted: false +force_unlisted_parseFrontMatter_test: true +--- + +# force_unlisted_parseFrontMatter_test + +This doc is hidden despite `unlisted: false` + +We use `parseFrontMatter` to force it to true thanks to `force_unlisted_parseFrontMatter_test: true` diff --git a/website/_dogfooding/dogfooding.config.ts b/website/_dogfooding/dogfooding.config.ts index 10f70d71fb95..428e3af528b4 100644 --- a/website/_dogfooding/dogfooding.config.ts +++ b/website/_dogfooding/dogfooding.config.ts @@ -10,6 +10,15 @@ import type {Options as DocsOptions} from '@docusaurus/plugin-content-docs'; import type {Options as BlogOptions} from '@docusaurus/plugin-content-blog'; import type {Options as PageOptions} from '@docusaurus/plugin-content-pages'; +export function dogfoodingParseFrontMatter(frontMatter: { + [key: string]: unknown; +}): {[key: string]: unknown} { + if (frontMatter.force_unlisted_parseFrontMatter_test === true) { + return {...frontMatter, unlisted: true}; + } + return frontMatter; +} + export const dogfoodingThemeInstances: PluginConfig[] = [ function swizzleThemeTests(): Plugin { return { diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts index b3dca14db3ba..b2d348403de0 100644 --- a/website/docusaurus.config.ts +++ b/website/docusaurus.config.ts @@ -17,6 +17,7 @@ import { dogfoodingPluginInstances, dogfoodingThemeInstances, dogfoodingRedirects, + dogfoodingParseFrontMatter, } from './_dogfooding/dogfooding.config'; import ConfigLocalized from './docusaurus.config.localized.json'; @@ -176,6 +177,13 @@ export default async function createConfigAsync() { mdx1Compat: { // comments: false, }, + parseFrontMatter: async (params) => { + const result = await params.defaultParseFrontMatter(params); + return { + ...result, + frontMatter: dogfoodingParseFrontMatter(result.frontMatter), + }; + }, preprocessor: ({filePath, fileContent}) => { let result = fileContent; From ed101ae5b086a150199a9191cfbdd2677f0b41ec Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 8 Dec 2023 23:33:23 +0100 Subject: [PATCH 02/15] add link to ./force-unlisted.mdx --- .../tests/visibility/{force_unlisted.mdx => force-unlisted.mdx} | 0 website/_dogfooding/_docs tests/tests/visibility/index.mdx | 1 + 2 files changed, 1 insertion(+) rename website/_dogfooding/_docs tests/tests/visibility/{force_unlisted.mdx => force-unlisted.mdx} (100%) diff --git a/website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx b/website/_dogfooding/_docs tests/tests/visibility/force-unlisted.mdx similarity index 100% rename from website/_dogfooding/_docs tests/tests/visibility/force_unlisted.mdx rename to website/_dogfooding/_docs tests/tests/visibility/force-unlisted.mdx diff --git a/website/_dogfooding/_docs tests/tests/visibility/index.mdx b/website/_dogfooding/_docs tests/tests/visibility/index.mdx index 88a78b5d6b2c..71c3712f2d12 100644 --- a/website/_dogfooding/_docs tests/tests/visibility/index.mdx +++ b/website/_dogfooding/_docs tests/tests/visibility/index.mdx @@ -24,6 +24,7 @@ In production, unlisted items should remain accessible, but be hidden in the sid - [./some-unlisteds/unlisted1.md](./some-unlisteds/unlisted1.mdx) - [./some-unlisteds/unlisted2.md](./some-unlisteds/unlisted2.mdx) - [./some-unlisteds/unlisted-subcategory/unlisted3.md](./some-unlisteds/unlisted-subcategory/unlisted3.mdx) +- [./force-unlisted.mdx](./force-unlisted.mdx) --- From c67ceb854afd480fff3c3c3566a3fb24e371cfdb Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 8 Dec 2023 23:35:32 +0100 Subject: [PATCH 03/15] rename to dogfoodingTransformFrontMatter --- website/_dogfooding/dogfooding.config.ts | 2 +- website/docusaurus.config.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/website/_dogfooding/dogfooding.config.ts b/website/_dogfooding/dogfooding.config.ts index 428e3af528b4..3e57c13c36ad 100644 --- a/website/_dogfooding/dogfooding.config.ts +++ b/website/_dogfooding/dogfooding.config.ts @@ -10,7 +10,7 @@ import type {Options as DocsOptions} from '@docusaurus/plugin-content-docs'; import type {Options as BlogOptions} from '@docusaurus/plugin-content-blog'; import type {Options as PageOptions} from '@docusaurus/plugin-content-pages'; -export function dogfoodingParseFrontMatter(frontMatter: { +export function dogfoodingTransformFrontMatter(frontMatter: { [key: string]: unknown; }): {[key: string]: unknown} { if (frontMatter.force_unlisted_parseFrontMatter_test === true) { diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts index b2d348403de0..3f88352dd6e5 100644 --- a/website/docusaurus.config.ts +++ b/website/docusaurus.config.ts @@ -17,7 +17,7 @@ import { dogfoodingPluginInstances, dogfoodingThemeInstances, dogfoodingRedirects, - dogfoodingParseFrontMatter, + dogfoodingTransformFrontMatter, } from './_dogfooding/dogfooding.config'; import ConfigLocalized from './docusaurus.config.localized.json'; @@ -181,7 +181,7 @@ export default async function createConfigAsync() { const result = await params.defaultParseFrontMatter(params); return { ...result, - frontMatter: dogfoodingParseFrontMatter(result.frontMatter), + frontMatter: dogfoodingTransformFrontMatter(result.frontMatter), }; }, preprocessor: ({filePath, fileContent}) => { From 7a2ba280b62e768e9286f496bd9d67f6fd59c19b Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 15 Dec 2023 18:33:26 +0100 Subject: [PATCH 04/15] document parseFrontMatter --- website/docs/api/docusaurus.config.js.mdx | 17 ++++++++ .../markdown-features-intro.mdx | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/website/docs/api/docusaurus.config.js.mdx b/website/docs/api/docusaurus.config.js.mdx index e7357f4bf520..48cd331e407e 100644 --- a/website/docs/api/docusaurus.config.js.mdx +++ b/website/docs/api/docusaurus.config.js.mdx @@ -421,10 +421,20 @@ type MDX1CompatOptions = headingIds: boolean; }; +export type ParseFrontMatter = (params: { + filePath: string; + fileContent: string; + defaultParseFrontMatter: ParseFrontMatter; +}) => Promise<{ + frontMatter: {[key: string]: unknown}; + content: string; +}>; + type MarkdownConfig = { format: 'mdx' | 'md' | 'detect'; mermaid: boolean; preprocessor?: MarkdownPreprocessor; + parseFrontMatter?: ParseFrontMatter; mdx1Compat: MDX1CompatOptions; }; ``` @@ -439,6 +449,12 @@ export default { preprocessor: ({filePath, fileContent}) => { return fileContent.replaceAll('{{MY_VAR}}', 'MY_VALUE'); }, + parseFrontMatter: async (params) => { + const result = await params.defaultParseFrontMatter(params); + result.frontMatter.description = + result.frontMatter.description?.replaceAll('{{MY_VAR}}', 'MY_VALUE'); + return result; + }, mdx1Compat: { comments: true, admonitions: true, @@ -457,6 +473,7 @@ export default { | `format` | `'mdx' \| 'md' \| 'detect'` | `'mdx'` | The default parser format to use for Markdown content. Using 'detect' will select the appropriate format automatically based on file extensions: `.md` vs `.mdx`. | | `mermaid` | `boolean` | `false` | When `true`, allows Docusaurus to render Markdown code blocks with `mermaid` language as Mermaid diagrams. | | `preprocessor` | `MarkdownPreprocessor` | `undefined` | Gives you the ability to alter the Markdown content string before parsing. Use it as a last-resort escape hatch or workaround: it is almost always better to implement a Remark/Rehype plugin. | +| `parseFrontMatter` | `ParseFrontMatter` | `undefined` | Gives you the ability to provide your own front matter parser, or to enhance the default parser. Read our [front matter guide](../guides/markdown-features/markdown-features-intro.mdx#front-matter) for details. | | `mdx1Compat` | `MDX1CompatOptions` | `{comments: true, admonitions: true, headingIds: true}` | Compatibility options to make it easier to upgrade to Docusaurus v3+. | ```mdx-code-block diff --git a/website/docs/guides/markdown-features/markdown-features-intro.mdx b/website/docs/guides/markdown-features/markdown-features-intro.mdx index b3116e29983c..0eec49ff5526 100644 --- a/website/docs/guides/markdown-features/markdown-features-intro.mdx +++ b/website/docs/guides/markdown-features/markdown-features-intro.mdx @@ -120,6 +120,45 @@ The API documentation of each official plugin lists the supported attributes: ::: +:::tip enhance your front matter + +Use the [Markdown config `parseFrontMatter` function](../../api/docusaurus.config.js.mdx#markdown) to provide your own front matter parser, or to enhance the default parser. + +It is possible to reuse the default parser to wrap it with your own custom proprietary logic. This makes it possible to implement convenient front matter transformations, shortcuts, or to integrate with external systems using front matter that Docusaurus plugins do not support. + +```js title="docusaurus.config.js" +export default { + markdown: { + // highlight-start + parseFrontMatter: async (params) => { + // Reuse the default parser + const result = await params.defaultParseFrontMatter(params); + + // Process front matter description placeholders + result.frontMatter.description = + result.frontMatter.description?.replaceAll('{{MY_VAR}}', 'MY_VALUE'); + + // Create your own front matter shortcut + if (result.frontMatter.i_dont_want_docs_pagination) { + result.frontMatter.pagination_prev = null; + result.frontMatter.pagination_next = null; + } + + // Rename an unsupported front matter coming from another system + if (result.frontMatter.cms_seo_summary) { + result.frontMatter.description = result.frontMatter.cms_seo_summary; + delete result.frontMatter.cms_seo_summary; + } + + return result; + }, + // highlight-end + }, +}; +``` + +::: + ## Quotes {#quotes} Markdown quotes are beautifully styled: From 23521cd650a0156cb3da1dda13b494aef719d322 Mon Sep 17 00:00:00 2001 From: slorber Date: Fri, 15 Dec 2023 17:37:52 +0000 Subject: [PATCH 05/15] refactor: apply lint autofix --- project-words.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/project-words.txt b/project-words.txt index 5bb1628e9788..191bd81c37b3 100644 --- a/project-words.txt +++ b/project-words.txt @@ -78,6 +78,7 @@ dogfood Dogfooding dogfooding Dojocat +dont Dyte dyte Déja From 106307e603d23a8c0adeabb4ead42fc6002737a8 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 15 Dec 2023 19:35:52 +0100 Subject: [PATCH 06/15] fix parseFileContentFrontMatter, add structuredClone workaround --- .../src/__tests__/markdownUtils.test.ts | 33 +++++++++++++++++++ .../docusaurus-utils/src/markdownUtils.ts | 14 ++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts index 0c14a07d9cc3..83bb5e033a9b 100644 --- a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts +++ b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts @@ -16,6 +16,7 @@ import { admonitionTitleToDirectiveLabel, parseMarkdownFile, DEFAULT_PARSE_FRONT_MATTER, + parseFileContentFrontMatter, } from '../markdownUtils'; describe('createExcerpt', () => { @@ -624,6 +625,38 @@ Lorem Ipsum }); }); +describe('parseFileContentFrontMatter', () => { + function test(fileContent: string) { + return parseFileContentFrontMatter(fileContent); + } + + it('can parse front matter', () => { + const input = dedent` + --- + title: Frontmatter title + author: + age: 42 + --- + + Some text + `; + + const expectedResult: ReturnType = { + content: 'Some text', + frontMatter: {title: 'Frontmatter title', author: {age: 42}}, + }; + + const result = test(input); + expect(result).toEqual(expectedResult); + + // A regression test, ensure we don't return gray-matter cached objects + result.frontMatter.title = 'modified'; + // @ts-expect-error: ok + result.frontMatter.author.age = 53; + expect(test(input)).toEqual(expectedResult); + }); +}); + describe('parseMarkdownFile', () => { async function test( fileContent: string, diff --git a/packages/docusaurus-utils/src/markdownUtils.ts b/packages/docusaurus-utils/src/markdownUtils.ts index c79b3c100281..9f1349c9b9b2 100644 --- a/packages/docusaurus-utils/src/markdownUtils.ts +++ b/packages/docusaurus-utils/src/markdownUtils.ts @@ -218,15 +218,25 @@ export function createExcerpt(fileString: string): string | undefined { * --- * ``` */ -function parseFileContentFrontMatter(fileContent: string): { +export function parseFileContentFrontMatter(fileContent: string): { /** Front matter as parsed by gray-matter. */ frontMatter: {[key: string]: unknown}; /** The remaining content, trimmed. */ content: string; } { + // TODO replace gray-matter by a better lib + // gray-matter is unmaintained, not flexible, and the code doesn't look good const {data, content} = matter(fileContent); + + // gray-matter has an undocumented front matter caching behavior + // https://github.com/jonschlinkert/gray-matter/blob/ce67a86dba419381db0dd01cc84e2d30a1d1e6a5/index.js#L39 + // Unfortunately, this becomes a problem when we mutate returned front matter + // We want to make it possible as part of the parseFrontMatter API + // So we make it safe to mutate by always providing a deep copy + const frontMatter = structuredClone(data); + return { - frontMatter: data, + frontMatter, content: content.trim(), }; } From 1db3b771ac6319ef3b6178a7d3342f6800a632e2 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Fri, 15 Dec 2023 19:36:40 +0100 Subject: [PATCH 07/15] add docs parseFrontMatter test --- .../__fixtures__/simple-site/docusaurus.config.js | 12 ++++++++++++ .../src/__tests__/__snapshots__/index.test.ts.snap | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docusaurus.config.js b/packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docusaurus.config.js index ae48be19a450..bd7de0da9fcb 100644 --- a/packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docusaurus.config.js +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docusaurus.config.js @@ -11,4 +11,16 @@ module.exports = { url: 'https://your-docusaurus-site.example.com', baseUrl: '/', favicon: 'img/favicon.ico', + markdown: { + parseFrontMatter: async (params) => { + // Reuse the default parser + const result = await params.defaultParseFrontMatter(params); + if (result.frontMatter.last_update?.author) { + result.frontMatter.last_update.author = + result.frontMatter.last_update.author + + ' (processed by parseFrontMatter)'; + } + return result; + }, + }, }; diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap b/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap index bc57e764ce54..2a8b72873b07 100644 --- a/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap @@ -463,7 +463,7 @@ exports[`simple website content: data 1`] = ` "frontMatter": { "title": "Custom Last Update", "last_update": { - "author": "Custom Author", + "author": "Custom Author (processed by parseFrontMatter)", "date": "1/1/2000" } } @@ -686,7 +686,7 @@ exports[`simple website content: data 1`] = ` "frontMatter": { "title": "Last Update Author Only", "last_update": { - "author": "Custom Author" + "author": "Custom Author (processed by parseFrontMatter)" } } }", From b28c2c13e4f9581b8dd430bae4f95b50c844712c Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:07:46 +0100 Subject: [PATCH 08/15] fix docs tests --- .../src/__tests__/docs.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/docs.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/docs.test.ts index 026ac0da657a..7309620d737c 100644 --- a/packages/docusaurus-plugin-content-docs/src/__tests__/docs.test.ts +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/docs.test.ts @@ -567,14 +567,14 @@ describe('simple site', () => { description: 'Custom last update', frontMatter: { last_update: { - author: 'Custom Author', + author: 'Custom Author (processed by parseFrontMatter)', date: '1/1/2000', }, title: 'Custom Last Update', }, lastUpdatedAt: new Date('1/1/2000').getTime() / 1000, formattedLastUpdatedAt: 'Jan 1, 2000', - lastUpdatedBy: 'Custom Author', + lastUpdatedBy: 'Custom Author (processed by parseFrontMatter)', sidebarPosition: undefined, tags: [], unlisted: false, @@ -607,13 +607,13 @@ describe('simple site', () => { description: 'Only custom author, so it will still use the date from Git', frontMatter: { last_update: { - author: 'Custom Author', + author: 'Custom Author (processed by parseFrontMatter)', }, title: 'Last Update Author Only', }, lastUpdatedAt: 1539502055, formattedLastUpdatedAt: 'Oct 14, 2018', - lastUpdatedBy: 'Custom Author', + lastUpdatedBy: 'Custom Author (processed by parseFrontMatter)', sidebarPosition: undefined, tags: [], unlisted: false, @@ -685,7 +685,7 @@ describe('simple site', () => { description: 'Custom last update', frontMatter: { last_update: { - author: 'Custom Author', + author: 'Custom Author (processed by parseFrontMatter)', date: '1/1/2000', }, title: 'Custom Last Update', From 49a765597033175c45e4c53a20683c72cd42e370 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:21:11 +0100 Subject: [PATCH 09/15] add shitty Jest + structuredClone workaround for parseFileContentFrontMatter --- .../src/__tests__/markdownUtils.test.ts | 11 ++++++++--- packages/docusaurus-utils/src/markdownUtils.ts | 12 ++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts index 83bb5e033a9b..0e04dbf5c280 100644 --- a/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts +++ b/packages/docusaurus-utils/src/__tests__/markdownUtils.test.ts @@ -636,18 +636,23 @@ describe('parseFileContentFrontMatter', () => { title: Frontmatter title author: age: 42 + birth: 2000-07-23 --- Some text `; - const expectedResult: ReturnType = { + const expectedResult = { content: 'Some text', - frontMatter: {title: 'Frontmatter title', author: {age: 42}}, + frontMatter: { + title: 'Frontmatter title', + author: {age: 42, birth: new Date('2000-07-23')}, + }, }; - const result = test(input); + const result = test(input) as typeof expectedResult; expect(result).toEqual(expectedResult); + expect(result.frontMatter.author.birth).toBeInstanceOf(Date); // A regression test, ensure we don't return gray-matter cached objects result.frontMatter.title = 'modified'; diff --git a/packages/docusaurus-utils/src/markdownUtils.ts b/packages/docusaurus-utils/src/markdownUtils.ts index 9f1349c9b9b2..528858601d68 100644 --- a/packages/docusaurus-utils/src/markdownUtils.ts +++ b/packages/docusaurus-utils/src/markdownUtils.ts @@ -224,7 +224,7 @@ export function parseFileContentFrontMatter(fileContent: string): { /** The remaining content, trimmed. */ content: string; } { - // TODO replace gray-matter by a better lib + // TODO Docusaurus v4: replace gray-matter by a better lib // gray-matter is unmaintained, not flexible, and the code doesn't look good const {data, content} = matter(fileContent); @@ -233,13 +233,21 @@ export function parseFileContentFrontMatter(fileContent: string): { // Unfortunately, this becomes a problem when we mutate returned front matter // We want to make it possible as part of the parseFrontMatter API // So we make it safe to mutate by always providing a deep copy - const frontMatter = structuredClone(data); + const frontMatter = + // And of course structuredClone() doesn't work well with Date in Jest... + // See https://github.com/jestjs/jest/issues/2549 + // So we reparse for tests with a {} option object + // This undocumented empty option object disables gray-matter caching.. + process.env.JEST_WORKER_ID + ? matter(fileContent, {}).data + : structuredClone(data); return { frontMatter, content: content.trim(), }; } + export const DEFAULT_PARSE_FRONT_MATTER: DefaultParseFrontMatter = async ( params, ) => parseFileContentFrontMatter(params.fileContent); From 61e2ebd48229a0d83c5d5419bb72a0e5f3e6c153 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:29:24 +0100 Subject: [PATCH 10/15] add blog parseFrontMatter test --- .../src/__tests__/index.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts index da0caf9b4541..51b2f63f2fe7 100644 --- a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts +++ b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts @@ -8,11 +8,7 @@ import {jest} from '@jest/globals'; import path from 'path'; import {normalizePluginOptions} from '@docusaurus/utils-validation'; -import { - posixPath, - getFileCommitDate, - DEFAULT_PARSE_FRONT_MATTER, -} from '@docusaurus/utils'; +import {posixPath, getFileCommitDate} from '@docusaurus/utils'; import pluginContentBlog from '../index'; import {validateOptions} from '../options'; import type { @@ -37,7 +33,14 @@ const markdown: MarkdownConfig = { headingIds: true, admonitions: true, }, - parseFrontMatter: DEFAULT_PARSE_FRONT_MATTER, + parseFrontMatter: async (params) => { + // Reuse the default parser + const result = await params.defaultParseFrontMatter(params); + if (result.frontMatter.title === 'Complex Slug') { + result.frontMatter.custom_frontMatter = 'added by parseFrontMatter'; + } + return result; + }, }; function findByTitle( @@ -259,6 +262,7 @@ describe('blog plugin', () => { slug: '/hey/my super path/héllô', title: 'Complex Slug', tags: ['date', 'complex'], + custom_frontMatter: 'added by parseFrontMatter', }, tags: [ { From 6d9dbe76c31a7a3a4bff0c2f704f5de915752f02 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:33:59 +0100 Subject: [PATCH 11/15] add pages plugin parseFrontMatter tests --- .../__fixtures__/website/docusaurus.config.js | 7 +++++++ .../__tests__/__snapshots__/index.test.ts.snap | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/docusaurus.config.js b/packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/docusaurus.config.js index ae48be19a450..d048d2caf5a1 100644 --- a/packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/docusaurus.config.js +++ b/packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/docusaurus.config.js @@ -11,4 +11,11 @@ module.exports = { url: 'https://your-docusaurus-site.example.com', baseUrl: '/', favicon: 'img/favicon.ico', + markdown: { + parseFrontMatter: async (params) => { + const result = await params.defaultParseFrontMatter(params); + result.frontMatter.custom_frontMatter = 'added by parseFrontMatter'; + return result; + }, + }, }; diff --git a/packages/docusaurus-plugin-content-pages/src/__tests__/__snapshots__/index.test.ts.snap b/packages/docusaurus-plugin-content-pages/src/__tests__/__snapshots__/index.test.ts.snap index 56028498771e..fc5fa2196778 100644 --- a/packages/docusaurus-plugin-content-pages/src/__tests__/__snapshots__/index.test.ts.snap +++ b/packages/docusaurus-plugin-content-pages/src/__tests__/__snapshots__/index.test.ts.snap @@ -14,7 +14,9 @@ exports[`docusaurus-plugin-content-pages loads simple pages 1`] = ` }, { "description": "Markdown index page", - "frontMatter": {}, + "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", + }, "permalink": "/hello/", "source": "@site/src/pages/hello/index.md", "title": "Index", @@ -24,6 +26,7 @@ exports[`docusaurus-plugin-content-pages loads simple pages 1`] = ` { "description": "my MDX page", "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", "description": "my MDX page", "title": "MDX page", }, @@ -40,7 +43,9 @@ exports[`docusaurus-plugin-content-pages loads simple pages 1`] = ` }, { "description": "translated Markdown page", - "frontMatter": {}, + "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", + }, "permalink": "/hello/translatedMd", "source": "@site/src/pages/hello/translatedMd.md", "title": undefined, @@ -69,7 +74,9 @@ exports[`docusaurus-plugin-content-pages loads simple pages with french translat }, { "description": "Markdown index page", - "frontMatter": {}, + "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", + }, "permalink": "/fr/hello/", "source": "@site/src/pages/hello/index.md", "title": "Index", @@ -79,6 +86,7 @@ exports[`docusaurus-plugin-content-pages loads simple pages with french translat { "description": "my MDX page", "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", "description": "my MDX page", "title": "MDX page", }, @@ -95,7 +103,9 @@ exports[`docusaurus-plugin-content-pages loads simple pages with french translat }, { "description": "translated Markdown page (fr)", - "frontMatter": {}, + "frontMatter": { + "custom_frontMatter": "added by parseFrontMatter", + }, "permalink": "/fr/hello/translatedMd", "source": "@site/i18n/fr/docusaurus-plugin-content-pages/hello/translatedMd.md", "title": undefined, From f8e17d557950647967b0159c79668f69ccc16efc Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:36:29 +0100 Subject: [PATCH 12/15] typo --- .../docs/guides/markdown-features/markdown-features-intro.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/guides/markdown-features/markdown-features-intro.mdx b/website/docs/guides/markdown-features/markdown-features-intro.mdx index 0eec49ff5526..b6bcd756beff 100644 --- a/website/docs/guides/markdown-features/markdown-features-intro.mdx +++ b/website/docs/guides/markdown-features/markdown-features-intro.mdx @@ -139,7 +139,7 @@ export default { result.frontMatter.description?.replaceAll('{{MY_VAR}}', 'MY_VALUE'); // Create your own front matter shortcut - if (result.frontMatter.i_dont_want_docs_pagination) { + if (result.frontMatter.i_do_not_want_docs_pagination) { result.frontMatter.pagination_prev = null; result.frontMatter.pagination_next = null; } From 5627ca641f3eb47e29aee618df1aba835959c27a Mon Sep 17 00:00:00 2001 From: slorber Date: Sat, 16 Dec 2023 00:41:13 +0000 Subject: [PATCH 13/15] refactor: apply lint autofix --- project-words.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project-words.txt b/project-words.txt index 191bd81c37b3..6eea8321d082 100644 --- a/project-words.txt +++ b/project-words.txt @@ -78,7 +78,6 @@ dogfood Dogfooding dogfooding Dojocat -dont Dyte dyte Déja @@ -287,6 +286,7 @@ refactorings Rehype rehype renderable +reparse REPONAME Retrocompatibility retrocompatibility From d68816c4e2747e44e42518b7b677743687509da0 Mon Sep 17 00:00:00 2001 From: sebastienlorber Date: Sat, 16 Dec 2023 01:50:57 +0100 Subject: [PATCH 14/15] typo --- packages/docusaurus-utils/src/markdownUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docusaurus-utils/src/markdownUtils.ts b/packages/docusaurus-utils/src/markdownUtils.ts index 528858601d68..87aac88f09b1 100644 --- a/packages/docusaurus-utils/src/markdownUtils.ts +++ b/packages/docusaurus-utils/src/markdownUtils.ts @@ -236,7 +236,7 @@ export function parseFileContentFrontMatter(fileContent: string): { const frontMatter = // And of course structuredClone() doesn't work well with Date in Jest... // See https://github.com/jestjs/jest/issues/2549 - // So we reparse for tests with a {} option object + // So we parse again for tests with a {} option object // This undocumented empty option object disables gray-matter caching.. process.env.JEST_WORKER_ID ? matter(fileContent, {}).data From 516016a07a4f79d995770fc2ab9871ef6d3eb0ff Mon Sep 17 00:00:00 2001 From: slorber Date: Sat, 16 Dec 2023 00:55:40 +0000 Subject: [PATCH 15/15] refactor: apply lint autofix --- project-words.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/project-words.txt b/project-words.txt index 6eea8321d082..5bb1628e9788 100644 --- a/project-words.txt +++ b/project-words.txt @@ -286,7 +286,6 @@ refactorings Rehype rehype renderable -reparse REPONAME Retrocompatibility retrocompatibility