Skip to content

Commit

Permalink
Improve markdown rendering performance (#8532)
Browse files Browse the repository at this point in the history
  • Loading branch information
bluwy authored Sep 14, 2023
1 parent 61ac5c9 commit 7522bb4
Show file tree
Hide file tree
Showing 13 changed files with 208 additions and 124 deletions.
5 changes: 5 additions & 0 deletions .changeset/clever-parents-do.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'astro': patch
---

Improve markdown rendering performance by sharing processor instance
5 changes: 5 additions & 0 deletions .changeset/shaggy-actors-cheat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@astrojs/markdown-remark': minor
---

Export `createMarkdownProcessor` and deprecate `renderMarkdown` API
52 changes: 28 additions & 24 deletions packages/astro/src/vite-plugin-markdown/index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { renderMarkdown } from '@astrojs/markdown-remark';
import {
createMarkdownProcessor,
InvalidAstroDataError,
safelyGetAstroData,
} from '@astrojs/markdown-remark/dist/internal.js';
type MarkdownProcessor,
} from '@astrojs/markdown-remark';
import matter from 'gray-matter';
import fs from 'node:fs';
import path from 'node:path';
Expand Down Expand Up @@ -57,9 +57,14 @@ const astroErrorModulePath = normalizePath(
);

export default function markdown({ settings, logger }: AstroPluginOptions): Plugin {
let processor: MarkdownProcessor;

return {
enforce: 'pre',
name: 'astro:markdown',
async buildStart() {
processor = await createMarkdownProcessor(settings.config.markdown);
},
// Why not the "transform" hook instead of "load" + readFile?
// A: Vite transforms all "import.meta.env" references to their values before
// passing to the transform hook. This lets us get the truly raw value
Expand All @@ -70,33 +75,32 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
const rawFile = await fs.promises.readFile(fileId, 'utf-8');
const raw = safeMatter(rawFile, id);

const renderResult = await renderMarkdown(raw.content, {
...settings.config.markdown,
fileURL: new URL(`file://${fileId}`),
frontmatter: raw.data,
});
const renderResult = await processor
.render(raw.content, {
fileURL: new URL(`file://${fileId}`),
frontmatter: raw.data,
})
.catch((err) => {
// Improve error message for invalid astro data
if (err instanceof InvalidAstroDataError) {
throw new AstroError(AstroErrorData.InvalidFrontmatterInjectionError);
}
throw err;
});

let html = renderResult.code;
const { headings } = renderResult.metadata;
const { headings, imagePaths: rawImagePaths, frontmatter } = renderResult.metadata;

// Resolve all the extracted images from the content
let imagePaths: { raw: string; resolved: string }[] = [];
if (renderResult.vfile.data.imagePaths) {
for (let imagePath of renderResult.vfile.data.imagePaths.values()) {
imagePaths.push({
raw: imagePath,
resolved:
(await this.resolve(imagePath, id))?.id ?? path.join(path.dirname(id), imagePath),
});
}
}

const astroData = safelyGetAstroData(renderResult.vfile.data);
if (astroData instanceof InvalidAstroDataError) {
throw new AstroError(AstroErrorData.InvalidFrontmatterInjectionError);
const imagePaths: { raw: string; resolved: string }[] = [];
for (const imagePath of rawImagePaths.values()) {
imagePaths.push({
raw: imagePath,
resolved:
(await this.resolve(imagePath, id))?.id ?? path.join(path.dirname(id), imagePath),
});
}

const { frontmatter } = astroData;
const { layout } = frontmatter;

if (frontmatter.setup) {
Expand Down
7 changes: 7 additions & 0 deletions packages/markdown/remark/src/frontmatter-injection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ export function safelyGetAstroData(vfileData: Data): MarkdownAstroData | Invalid
return astro;
}

export function setAstroData(vfileData: Data, astroData: MarkdownAstroData) {
vfileData.astro = astroData;
}

/**
* @deprecated Use `setAstroData` instead
*/
export function toRemarkInitializeAstroData({
userFrontmatter,
}: {
Expand Down
157 changes: 102 additions & 55 deletions packages/markdown/remark/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
import type {
AstroMarkdownOptions,
MarkdownProcessor,
MarkdownRenderingOptions,
MarkdownRenderingResult,
MarkdownVFile,
} from './types.js';

import { toRemarkInitializeAstroData } from './frontmatter-injection.js';
import {
InvalidAstroDataError,
safelyGetAstroData,
setAstroData,
} from './frontmatter-injection.js';
import { loadPlugins } from './load-plugins.js';
import { rehypeHeadingIds } from './rehype-collect-headings.js';
import { remarkCollectImages } from './remark-collect-images.js';
Expand All @@ -15,13 +20,14 @@ import { remarkShiki } from './remark-shiki.js';
import rehypeRaw from 'rehype-raw';
import rehypeStringify from 'rehype-stringify';
import remarkGfm from 'remark-gfm';
import markdown from 'remark-parse';
import markdownToHtml from 'remark-rehype';
import remarkParse from 'remark-parse';
import remarkRehype from 'remark-rehype';
import remarkSmartypants from 'remark-smartypants';
import { unified } from 'unified';
import { VFile } from 'vfile';
import { rehypeImages } from './rehype-images.js';

export { InvalidAstroDataError } from './frontmatter-injection.js';
export { rehypeHeadingIds } from './rehype-collect-headings.js';
export { remarkCollectImages } from './remark-collect-images.js';
export { remarkPrism } from './remark-prism.js';
Expand All @@ -45,30 +51,29 @@ export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'draft
// Skip nonessential plugins during performance benchmark runs
const isPerformanceBenchmark = Boolean(process.env.ASTRO_PERFORMANCE_BENCHMARK);

/** Shared utility for rendering markdown */
export async function renderMarkdown(
content: string,
opts: MarkdownRenderingOptions
): Promise<MarkdownRenderingResult> {
let {
fileURL,
/**
* Create a markdown preprocessor to render multiple markdown files
*/
export async function createMarkdownProcessor(
opts?: AstroMarkdownOptions
): Promise<MarkdownProcessor> {
const {
syntaxHighlight = markdownConfigDefaults.syntaxHighlight,
shikiConfig = markdownConfigDefaults.shikiConfig,
remarkPlugins = markdownConfigDefaults.remarkPlugins,
rehypePlugins = markdownConfigDefaults.rehypePlugins,
remarkRehype = markdownConfigDefaults.remarkRehype,
remarkRehype: remarkRehypeOptions = markdownConfigDefaults.remarkRehype,
gfm = markdownConfigDefaults.gfm,
smartypants = markdownConfigDefaults.smartypants,
frontmatter: userFrontmatter = {},
} = opts;
const input = new VFile({ value: content, path: fileURL });
} = opts ?? {};

const loadedRemarkPlugins = await Promise.all(loadPlugins(remarkPlugins));
const loadedRehypePlugins = await Promise.all(loadPlugins(rehypePlugins));

let parser = unified()
.use(markdown)
.use(toRemarkInitializeAstroData({ userFrontmatter }))
.use([]);
const parser = unified().use(remarkParse);

if (!isPerformanceBenchmark && gfm) {
// gfm and smartypants
if (!isPerformanceBenchmark) {
if (gfm) {
parser.use(remarkGfm);
}
Expand All @@ -77,14 +82,13 @@ export async function renderMarkdown(
}
}

const loadedRemarkPlugins = await Promise.all(loadPlugins(remarkPlugins));
const loadedRehypePlugins = await Promise.all(loadPlugins(rehypePlugins));

loadedRemarkPlugins.forEach(([plugin, pluginOpts]) => {
parser.use([[plugin, pluginOpts]]);
});
// User remark plugins
for (const [plugin, pluginOpts] of loadedRemarkPlugins) {
parser.use(plugin, pluginOpts);
}

if (!isPerformanceBenchmark) {
// Syntax highlighting
if (syntaxHighlight === 'shiki') {
parser.use(remarkShiki, shikiConfig);
} else if (syntaxHighlight === 'prism') {
Expand All @@ -95,45 +99,88 @@ export async function renderMarkdown(
parser.use(remarkCollectImages);
}

parser.use([
[
markdownToHtml as any,
{
allowDangerousHtml: true,
passThrough: [],
...remarkRehype,
},
],
]);

loadedRehypePlugins.forEach(([plugin, pluginOpts]) => {
parser.use([[plugin, pluginOpts]]);
// Remark -> Rehype
parser.use(remarkRehype as any, {
allowDangerousHtml: true,
passThrough: [],
...remarkRehypeOptions,
});

// User rehype plugins
for (const [plugin, pluginOpts] of loadedRehypePlugins) {
parser.use(plugin, pluginOpts);
}

// Images / Assets support
parser.use(rehypeImages());

// Headings
if (!isPerformanceBenchmark) {
parser.use([rehypeHeadingIds]);
parser.use(rehypeHeadingIds);
}

parser.use([rehypeRaw]).use(rehypeStringify, { allowDangerousHtml: true });
// Stringify to HTML
parser.use(rehypeRaw).use(rehypeStringify, { allowDangerousHtml: true });

let vfile: MarkdownVFile;
try {
vfile = await parser.process(input);
} catch (err) {
// Ensure that the error message contains the input filename
// to make it easier for the user to fix the issue
err = prefixError(err, `Failed to parse Markdown file "${input.path}"`);
// eslint-disable-next-line no-console
console.error(err);
throw err;
}
return {
async render(content, renderOpts) {
const vfile = new VFile({ value: content, path: renderOpts?.fileURL });
setAstroData(vfile.data, { frontmatter: renderOpts?.frontmatter ?? {} });

const result: MarkdownVFile = await parser.process(vfile).catch((err) => {
// Ensure that the error message contains the input filename
// to make it easier for the user to fix the issue
err = prefixError(err, `Failed to parse Markdown file "${vfile.path}"`);
// eslint-disable-next-line no-console
console.error(err);
throw err;
});

const astroData = safelyGetAstroData(result.data);
if (astroData instanceof InvalidAstroDataError) {
throw astroData;
}

return {
code: String(result.value),
metadata: {
headings: result.data.__astroHeadings ?? [],
imagePaths: result.data.imagePaths ?? new Set(),
frontmatter: astroData.frontmatter ?? {},
},
// Compat for `renderMarkdown` only. Do not use!
__renderMarkdownCompat: {
result,
},
};
},
};
}

/**
* Shared utility for rendering markdown
*
* @deprecated Use `createMarkdownProcessor` instead for better performance
*/
export async function renderMarkdown(
content: string,
opts: MarkdownRenderingOptions
): Promise<MarkdownRenderingResult> {
const processor = await createMarkdownProcessor(opts);

const result = await processor.render(content, {
fileURL: opts.fileURL,
frontmatter: opts.frontmatter,
});

const headings = vfile?.data.__astroHeadings || [];
return {
metadata: { headings, source: content, html: String(vfile.value) },
code: String(vfile.value),
vfile,
code: result.code,
metadata: {
headings: result.metadata.headings,
source: content,
html: result.code,
},
vfile: (result as any).__renderMarkdownCompat.result,
};
}

Expand Down
1 change: 1 addition & 0 deletions packages/markdown/remark/src/internal.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export {
InvalidAstroDataError,
safelyGetAstroData,
setAstroData,
toRemarkInitializeAstroData,
} from './frontmatter-injection.js';
2 changes: 1 addition & 1 deletion packages/markdown/remark/src/load-plugins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ async function importPlugin(p: string | unified.Plugin): Promise<unified.Plugin>
} catch {}

// Try import from user project
const resolved = await importMetaResolve(p, cwdUrlStr);
const resolved = importMetaResolve(p, cwdUrlStr);
const importResult = await import(resolved);
return importResult.default;
}
Expand Down
22 changes: 21 additions & 1 deletion packages/markdown/remark/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,33 @@ export interface ImageMetadata {
type: string;
}

export interface MarkdownRenderingOptions extends AstroMarkdownOptions {
export interface MarkdownProcessor {
render: (
content: string,
opts?: MarkdownProcessorRenderOptions
) => Promise<MarkdownProcessorRenderResult>;
}

export interface MarkdownProcessorRenderOptions {
/** @internal */
fileURL?: URL;
/** Used for frontmatter injection plugins */
frontmatter?: Record<string, any>;
}

export interface MarkdownProcessorRenderResult {
code: string;
metadata: {
headings: MarkdownHeading[];
imagePaths: Set<string>;
frontmatter: Record<string, any>;
};
}

export interface MarkdownRenderingOptions
extends AstroMarkdownOptions,
MarkdownProcessorRenderOptions {}

export interface MarkdownHeading {
depth: number;
slug: string;
Expand Down
Loading

0 comments on commit 7522bb4

Please sign in to comment.