diff --git a/.vscode/settings.json b/.vscode/settings.json index 795f401e1..b1360f3d8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,5 +36,6 @@ "json", "jsonc", "yaml" - ] + ], + "references.preferredLocation": "peek" } diff --git a/README.md b/README.md index d8fbe68da..0810ae3a2 100644 --- a/README.md +++ b/README.md @@ -432,21 +432,23 @@ console.log(root) Since `shikiji` uses `hast` internally, you can use the `transforms` option to customize the generated HTML by manipulating the hast tree. You can pass custom functions to modify the tree for different types of nodes. For example: ```js +import { addClassToHast, codeToHtml } from 'shikiji' + const code = await codeToHtml('foo\bar', { lang: 'js', theme: 'vitesse-light', transformers: [ { code(node) { - node.properties.class = 'language-js' + addClassToHast(node, 'language-js') }, line(node, line) { node.properties['data-line'] = line if ([1, 3, 4].includes(line)) - node.properties.class += ' highlight' + addClassToHast(node, 'highlight') }, token(node, line, col) { - node.properties.class = `token:${line}:${col}` + node.properties['data-token'] = `token:${line}:${col}` }, }, ] diff --git a/eslint.config.js b/eslint.config.js index 13a76bb5f..6d32da619 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -4,11 +4,13 @@ export default antfu( { ignores: [ 'packages/shikiji/src/assets/*.ts', + '**/fixtures/**', ], }, { rules: { 'no-restricted-syntax': 'off', + 'ts/no-invalid-this': 'off', }, }, ) diff --git a/packages/markdown-it-shikiji/src/index.ts b/packages/markdown-it-shikiji/src/index.ts index 33cc0fb47..47dbf471b 100644 --- a/packages/markdown-it-shikiji/src/index.ts +++ b/packages/markdown-it-shikiji/src/index.ts @@ -1,5 +1,5 @@ import type MarkdownIt from 'markdown-it' -import { bundledLanguages, getHighlighter } from 'shikiji' +import { addClassToHast, bundledLanguages, getHighlighter } from 'shikiji' import type { BuiltinLanguage, BuiltinTheme, CodeOptionsThemes, CodeToHastOptions, Highlighter, LanguageInput } from 'shikiji' import { parseHighlightLines } from '../../shared/line-highlight' @@ -45,7 +45,7 @@ function setup(markdownit: MarkdownIt, highlighter: Highlighter, options: Markdo name: 'markdown-it-shikiji:line-class', line(node, line) { if (lines.includes(line)) - node.properties.class += ` ${className}` + addClassToHast(node, className) return node }, }) diff --git a/packages/rehype-shikiji/src/index.ts b/packages/rehype-shikiji/src/index.ts index 914e400fb..8873ad7f9 100644 --- a/packages/rehype-shikiji/src/index.ts +++ b/packages/rehype-shikiji/src/index.ts @@ -1,5 +1,5 @@ import type { BuiltinLanguage, BuiltinTheme, CodeOptionsThemes, CodeToHastOptions, LanguageInput } from 'shikiji' -import { bundledLanguages, getHighlighter } from 'shikiji' +import { addClassToHast, bundledLanguages, getHighlighter } from 'shikiji' import { toString } from 'hast-util-to-string' import { visit } from 'unist-util-visit' import type { Plugin } from 'unified' @@ -105,7 +105,7 @@ const rehypeShikiji: Plugin<[RehypeShikijiOptions], Root> = function (options = name: 'rehype-shikiji:line-class', line(node, line) { if (lines.includes(line)) - node.properties.class += ` ${className}` + addClassToHast(node, className) return node }, }) diff --git a/packages/shikiji-compat/package.json b/packages/shikiji-compat/package.json index 71df297d6..01a496e03 100644 --- a/packages/shikiji-compat/package.json +++ b/packages/shikiji-compat/package.json @@ -37,6 +37,7 @@ "prepublishOnly": "nr build" }, "dependencies": { - "shikiji": "workspace:*" + "shikiji": "workspace:*", + "shikiji-transformers": "workspace:*" } } diff --git a/packages/shikiji-compat/src/index.ts b/packages/shikiji-compat/src/index.ts index c6c477229..2b37a733f 100644 --- a/packages/shikiji-compat/src/index.ts +++ b/packages/shikiji-compat/src/index.ts @@ -2,6 +2,7 @@ import fs from 'node:fs' import fsp from 'node:fs/promises' import type { BuiltinLanguage, BuiltinTheme, CodeToThemedTokensOptions, MaybeGetter, StringLiteralUnion, ThemeInput, ThemeRegistration, ThemedToken } from 'shikiji' import { bundledLanguages, bundledThemes, getHighlighter as getShikiji, toShikiTheme } from 'shikiji' +import { transformerCompactLineOptions } from 'shikiji-transformers' import type { AnsiToHtmlOptions, CodeToHtmlOptions, CodeToHtmlOptionsExtra, HighlighterOptions } from './types' export const BUNDLED_LANGUAGES = bundledLanguages @@ -66,17 +67,7 @@ export async function getHighlighter(options: HighlighterOptions = {}) { if (options.lineOptions) { options.transformers ||= [] - options.transformers.push({ - name: 'shikiji-compat:line-class', - line(node, line) { - const lineOption = options.lineOptions?.find(o => o.line === line) - if (lineOption?.classes) { - node.properties ??= {} - node.properties.class = [node.properties.class, ...lineOption.classes].filter(Boolean).join(' ') - } - return node - }, - }) + options.transformers.push(transformerCompactLineOptions(options.lineOptions)) } return shikiji.codeToHtml(code, options as any) diff --git a/packages/shikiji-transformers/README.md b/packages/shikiji-transformers/README.md new file mode 100644 index 000000000..6950329aa --- /dev/null +++ b/packages/shikiji-transformers/README.md @@ -0,0 +1,98 @@ +# shikiji-transformers + +Collective of common transformers for [shikiji](https://github.com/antfu/shikiji), inspired by [shiki-processor](https://github.com/innocenzi/shiki-processor). + +## Install + +```bash +npm i -D shikiji-transformers +``` + +```ts +import { + codeToHtml, +} from 'shikiji' +import { + transformerNotationDiff, + // ... +} from 'shikiji-transformers' + +const html = codeToHtml(code, { + lang: 'ts', + transformers: [ + transformerNotationDiff(), + // ... + ], +}) +``` + +## Transformers + +### `transformerNotationDiff` + +Use `[!code ++]` and `[!code --]` to mark added and removed lines. + +For example, the following code + +```ts +export function foo() { + console.log('hewwo') // [!code --] + console.log('hello') // [!code ++] +} +``` + +will be transformed to + +```html + +
 
+  
+    
+    function(){
+      
+      console.log('hewwo') 
+    
+      
+      console.log('hello') 
+    
+    }
+    
+  
+
+``` + +With some CSS, you can make it look like this: + +image + +### `transformerNotationHighlight` + +Use `[!code highlight]` to highlight a line (adding `highlighted` class). + +### `transformerNotationFocus` + +Use `[!code focus]` to focus a line (adding `focused` class). + +### `transformerNotationErrorLevel` + +Use `[!code error]`, `[!code warning]`, to mark a line with an error level (adding `highlighted error`, `highlighted warning` class). + +### `transformerRenderWhitespace` + +Render whitespaces (tabs and spaces) as individual spans, with classes `tab` and `space`. + +With some CSS, you can make it look like this: + +image + +### `transformerCompactLineOptions` + +Support for `shiki`'s `lineOptions` that is removed in `shikiji`. + +### `transformerRemoveLineBreak` + +Remove line breaks between ``. Useful when you set `display: block` to `.line` in CSS. + +## License + +MIT diff --git a/packages/shikiji-transformers/build.config.ts b/packages/shikiji-transformers/build.config.ts new file mode 100644 index 000000000..2656e4688 --- /dev/null +++ b/packages/shikiji-transformers/build.config.ts @@ -0,0 +1,14 @@ +import { defineBuildConfig } from 'unbuild' + +export default defineBuildConfig({ + entries: [ + 'src/index.ts', + ], + declaration: true, + rollup: { + emitCJS: false, + }, + externals: [ + 'hast', + ], +}) diff --git a/packages/shikiji-transformers/package.json b/packages/shikiji-transformers/package.json new file mode 100644 index 000000000..4b6b0fb7e --- /dev/null +++ b/packages/shikiji-transformers/package.json @@ -0,0 +1,40 @@ +{ + "name": "shikiji-transformers", + "type": "module", + "version": "0.0.0", + "description": "Collective of common transformers transformers for Shikiji", + "author": "Anthony Fu ", + "license": "MIT", + "homepage": "https://github.com/antfu/shikiji#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/antfu/shikiji.git", + "directory": "packages/shikiji-transformers" + }, + "bugs": "https://github.com/antfu/shikiji/issues", + "keywords": [ + "shiki", + "shikiji-transformers" + ], + "sideEffects": false, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "dist" + ], + "scripts": { + "build": "unbuild", + "dev": "unbuild --stub", + "prepublishOnly": "nr build" + }, + "dependencies": { + "shikiji": "workspace:*" + } +} diff --git a/packages/shikiji-transformers/src/index.ts b/packages/shikiji-transformers/src/index.ts new file mode 100644 index 000000000..49c7103b8 --- /dev/null +++ b/packages/shikiji-transformers/src/index.ts @@ -0,0 +1,8 @@ +export * from './transformers/render-whitespace' +export * from './transformers/remove-line-breaks' +export * from './transformers/compact-line-options' +export * from './transformers/notation-focus' +export * from './transformers/notation-highlight' +export * from './transformers/notation-diff' +export * from './transformers/notation-error-level' +export * from './utils' diff --git a/packages/shikiji-transformers/src/transformers/compact-line-options.ts b/packages/shikiji-transformers/src/transformers/compact-line-options.ts new file mode 100644 index 000000000..3e19aabb5 --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/compact-line-options.ts @@ -0,0 +1,27 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' + +export interface TransformerCompactLineOption { + /** + * 1-based line number. + */ + line: number + classes?: string[] +} + +/** + * Transformer for `shiki`'s legacy `lineOptions` + */ +export function transformerCompactLineOptions( + lineOptions: TransformerCompactLineOption[] = [], +): ShikijiTransformer { + return { + name: 'shikiji-transformers:compact-line-options', + line(node, line) { + const lineOption = lineOptions.find(o => o.line === line) + if (lineOption?.classes) + addClassToHast(node, lineOption.classes) + return node + }, + } +} diff --git a/packages/shikiji-transformers/src/transformers/notation-diff.ts b/packages/shikiji-transformers/src/transformers/notation-diff.ts new file mode 100644 index 000000000..9d1716b88 --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/notation-diff.ts @@ -0,0 +1,45 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' +import { createCommentNotationTransformer } from '../utils' + +export interface TransformerNotationDiffOptions { + /** + * Class for added lines + */ + classAdded?: string + /** + * Class for removed lines + */ + classRemoved?: string + /** + * Class added to the root element when the current code has diff + */ + classRootActive?: string +} + +/** + * Use `[!code ++]` and `[!code --]` to mark added and removed lines. + */ +export function transformerNotationDiff( + options: TransformerNotationDiffOptions = {}, +): ShikijiTransformer { + const { + classAdded = 'diff add', + classRemoved = 'diff remove', + classRootActive = 'has-diff', + } = options + + return createCommentNotationTransformer( + 'shikiji-transformers:notation-diff', + /\[!code (\-\-|\+\+)\]/, + function ([_, match], line) { + const className = match === '--' + ? classRemoved + : classAdded + addClassToHast(line, className) + if (classRootActive) + addClassToHast(this.pre, classRootActive) + return true + }, + ) +} diff --git a/packages/shikiji-transformers/src/transformers/notation-error-level.ts b/packages/shikiji-transformers/src/transformers/notation-error-level.ts new file mode 100644 index 000000000..f76d272f9 --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/notation-error-level.ts @@ -0,0 +1,30 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' +import { createCommentNotationTransformer } from '../utils' + +export interface TransformerNotationErrorLevelOptions { + classMap?: Record +} + +/** + * Allow using `[!code error]` `[!code warning]` notation in code to mark highlighted lines. + */ +export function transformerNotationErrorLevel( + options: TransformerNotationErrorLevelOptions = {}, +): ShikijiTransformer { + const { + classMap = { + error: ['highlighted', 'error'], + warning: ['highlighted', 'warning'], + }, + } = options + + return createCommentNotationTransformer( + 'shikiji-transformers:notation-error-level', + new RegExp(`\\[!code (${Object.keys(classMap).join('|')})\\]`), + ([_, match], line) => { + addClassToHast(line, classMap[match]) + return true + }, + ) +} diff --git a/packages/shikiji-transformers/src/transformers/notation-focus.ts b/packages/shikiji-transformers/src/transformers/notation-focus.ts new file mode 100644 index 000000000..fcb4b09b3 --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/notation-focus.ts @@ -0,0 +1,37 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' +import { createCommentNotationTransformer } from '../utils' + +export interface TransformerNotationFocusOptions { + /** + * Class for focused lines + */ + classFocused?: string + /** + * Class added to the root element when the code has focused lines + */ + classRootActive?: string +} + +/** + * Allow using `[!code focus]` notation in code to mark focused lines. + */ +export function transformerNotationFocus( + options: TransformerNotationFocusOptions = {}, +): ShikijiTransformer { + const { + classFocused = 'focused', + classRootActive = 'has-focused', + } = options + + return createCommentNotationTransformer( + 'shikiji-transformers:notation-focus', + /\[!code focus\]/, + function (_, line) { + addClassToHast(line, classFocused) + if (classRootActive) + addClassToHast(this.pre, classRootActive) + return true + }, + ) +} diff --git a/packages/shikiji-transformers/src/transformers/notation-highlight.ts b/packages/shikiji-transformers/src/transformers/notation-highlight.ts new file mode 100644 index 000000000..948242202 --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/notation-highlight.ts @@ -0,0 +1,37 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' +import { createCommentNotationTransformer } from '../utils' + +export interface TransformerNotationHighlightOptions { + /** + * Class for highlighted lines + */ + classHighlight?: string + /** + * Class added to the root element when the code has highlighted lines + */ + classRootActive?: string +} + +/** + * Allow using `[!code highlight]` notation in code to mark highlighted lines. + */ +export function transformerNotationHighlight( + options: TransformerNotationHighlightOptions = {}, +): ShikijiTransformer { + const { + classHighlight = 'highlighted', + classRootActive = 'has-highlighted', + } = options + + return createCommentNotationTransformer( + 'shikiji-transformers:notation-highlight', + /\[!code (hl|highlight)\]/, + function (_, line) { + addClassToHast(line, classHighlight) + if (classRootActive) + addClassToHast(this.pre, classRootActive) + return true + }, + ) +} diff --git a/packages/shikiji-transformers/src/transformers/remove-line-breaks.ts b/packages/shikiji-transformers/src/transformers/remove-line-breaks.ts new file mode 100644 index 000000000..5a208e3fe --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/remove-line-breaks.ts @@ -0,0 +1,14 @@ +import type { ShikijiTransformer } from 'shikiji' + +/** + * Remove line breaks between lines. + * Useful when you override `display: block` to `.line` in CSS. + */ +export function transformerRemoveLineBreak(): ShikijiTransformer { + return { + name: 'shikiji-transformers:remove-line-break', + code(code) { + code.children = code.children.filter(line => !(line.type === 'text' && line.value === '\n')) + }, + } +} diff --git a/packages/shikiji-transformers/src/transformers/render-whitespace.ts b/packages/shikiji-transformers/src/transformers/render-whitespace.ts new file mode 100644 index 000000000..877552ccc --- /dev/null +++ b/packages/shikiji-transformers/src/transformers/render-whitespace.ts @@ -0,0 +1,79 @@ +import type { ShikijiTransformer } from 'shikiji' +import { addClassToHast } from 'shikiji' + +export interface TransformerRenderWhitespaceOptions { + /** + * Class for tab + * + * @default 'tab' + */ + classTab?: string + /** + * Class for space + * + * @default 'space' + */ + classSpace?: string + + /** + * Position of rendered whitespace + * @default all positions + */ + positions?: { + startOfLine?: boolean + endOfLine?: boolean + inline?: boolean + } +} + +/** + * Render whitespaces as separate tokens. + * Apply with CSS, it can be used to render tabs and spaces visually. + */ +export function transformerRenderWhitespace( + options: TransformerRenderWhitespaceOptions = {}, +): ShikijiTransformer { + const classMap: Record = { + ' ': options.classSpace ?? 'space', + '\t': options.classTab ?? 'tab', + } + + // TODO: support `positions` + + return { + name: 'shikiji-transformers:render-whitespace', + line(node) { + const first = node.children[0] + if (!first || first.type !== 'element') + return + const textNode = first.children[0] + if (!textNode || textNode.type !== 'text') + return + node.children = node.children.flatMap((child) => { + if (child.type !== 'element') + return child + const node = child.children[0] + if (node.type !== 'text' || !node.value) + return child + + // Split by whitespaces + const parts = node.value.split(/([ \t])/).filter(i => i.length) + if (parts.length <= 1) + return child + + return parts.map((part) => { + const clone = { + ...child, + properties: { ...child.properties }, + } + clone.children = [{ type: 'text', value: part }] + if (part in classMap) { + addClassToHast(clone, classMap[part]) + delete clone.properties.style + } + return clone + }) + }) + }, + } +} diff --git a/packages/shikiji-transformers/src/utils.ts b/packages/shikiji-transformers/src/utils.ts new file mode 100644 index 000000000..01ee39329 --- /dev/null +++ b/packages/shikiji-transformers/src/utils.ts @@ -0,0 +1,50 @@ +import type { Element } from 'hast' +import type { ShikijiTransformer, ShikijiTransformerContext } from 'shikiji' + +/** + * Check if a node is comment-like, + * e.g. ``, `/* comment ..`, `// comment` + */ +export function isCommentLike(node: Element, line: Element) { + if (node.children?.[0].type !== 'text') + return false + const text = node.children[0].value.trim() + if (text.startsWith('')) + return true + if (text.startsWith('/*') && text.endsWith('*/')) + return true + if (text.startsWith('//') && line.children.indexOf(node) === line.children.length - 1) + return true + return false +} + +export function createCommentNotationTransformer( + name: string, + regex: RegExp, + onMatch: (this: ShikijiTransformerContext, match: RegExpMatchArray, line: Element, commentNode: Element) => boolean, +): ShikijiTransformer { + return { + name, + line(line) { + let nodeToRemove: Element | undefined + for (const child of line.children) { + if (child.type !== 'element') + continue + if (!isCommentLike(child, line)) + continue + const text = child.children[0] + if (text.type !== 'text') + continue + const match = text.value.match(regex) + if (!match) + continue + if (onMatch.call(this, match, line, child)) { + nodeToRemove = child + break + } + } + if (nodeToRemove) + line.children.splice(line.children.indexOf(nodeToRemove), 1) + }, + } +} diff --git a/packages/shikiji-transformers/test/fixtures.test.ts b/packages/shikiji-transformers/test/fixtures.test.ts new file mode 100644 index 000000000..fdb7edcd5 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures.test.ts @@ -0,0 +1,116 @@ +/// + +import { describe, expect, it } from 'vitest' +import type { ShikijiTransformer } from 'shikiji' +import { codeToHtml } from 'shikiji' +import { + transformerNotationDiff, + transformerNotationErrorLevel, + transformerNotationFocus, + transformerNotationHighlight, + transformerRemoveLineBreak, + transformerRenderWhitespace, +} from '../src' + +function suite( + name: string, + files: Record, + transformers: ShikijiTransformer[], + replace?: (code: string) => string, +) { + describe(name, () => { + for (const path of Object.keys(files)) { + if (path.endsWith('.output.html')) + continue + + it(path, async () => { + const ext = path.split('.').pop()! + + let code = await codeToHtml(files[path], { + lang: ext, + theme: 'github-dark', + transformers, + }) + + if (replace) + code = replace(code) + + expect(code) + .toMatchFileSnapshot(`${path}.output.html`) + }) + } + }) +} + +suite( + 'diff', + import.meta.glob('./fixtures/diff/*.*', { as: 'raw', eager: true }), + [transformerNotationDiff(), transformerRemoveLineBreak()], + code => `${code} +`, +) + +suite( + 'focus', + import.meta.glob('./fixtures/focus/*.*', { as: 'raw', eager: true }), + [transformerNotationFocus(), transformerRemoveLineBreak()], + code => `${code} +`, +) + +suite( + 'highlight', + import.meta.glob('./fixtures/highlight/*.*', { as: 'raw', eager: true }), + [transformerNotationHighlight(), transformerRemoveLineBreak()], + code => `${code} +`, +) + +suite( + 'error-level', + import.meta.glob('./fixtures/error-level/*.*', { as: 'raw', eager: true }), + [transformerNotationErrorLevel(), transformerRemoveLineBreak()], + code => `${code} +`, +) + +suite( + 'whitespace', + import.meta.glob('./fixtures/whitespace/*.*', { as: 'raw', eager: true }), + [transformerRenderWhitespace()], + code => `${code} +`, +) diff --git a/packages/shikiji-transformers/test/fixtures/diff/a.js b/packages/shikiji-transformers/test/fixtures/diff/a.js new file mode 100644 index 000000000..d84dc5177 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/diff/a.js @@ -0,0 +1,4 @@ +export function foo() { + console.log('hewwo') // [!code --] + console.log('hello') // [!code ++] +} diff --git a/packages/shikiji-transformers/test/fixtures/diff/a.js.output.html b/packages/shikiji-transformers/test/fixtures/diff/a.js.output.html new file mode 100644 index 000000000..5c7745259 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/diff/a.js.output.html @@ -0,0 +1,12 @@ +
export function foo() {  console.log('hewwo')   console.log('hello') }
+ \ No newline at end of file diff --git a/packages/shikiji-transformers/test/fixtures/error-level/a.js b/packages/shikiji-transformers/test/fixtures/error-level/a.js new file mode 100644 index 000000000..6408a3543 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/error-level/a.js @@ -0,0 +1,4 @@ +export function foo() { + console.log('error') // [!code error] + console.log('warn') // [!code warning] +} diff --git a/packages/shikiji-transformers/test/fixtures/error-level/a.js.output.html b/packages/shikiji-transformers/test/fixtures/error-level/a.js.output.html new file mode 100644 index 000000000..3a4e280fc --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/error-level/a.js.output.html @@ -0,0 +1,8 @@ +
export function foo() {  console.log('error')   console.log('warn') }
+ \ No newline at end of file diff --git a/packages/shikiji-transformers/test/fixtures/focus/a.js b/packages/shikiji-transformers/test/fixtures/focus/a.js new file mode 100644 index 000000000..a0441c2ce --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/focus/a.js @@ -0,0 +1,6 @@ +export function foo() { + console.log('focus') // [!code focus] + + // should not be transformed: + console.log('[!code focus]') +} diff --git a/packages/shikiji-transformers/test/fixtures/focus/a.js.output.html b/packages/shikiji-transformers/test/fixtures/focus/a.js.output.html new file mode 100644 index 000000000..63e4dcfd1 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/focus/a.js.output.html @@ -0,0 +1,7 @@ +
export function foo() {  console.log('focus')   // should not be transformed:  console.log('[!code focus]')}
+ \ No newline at end of file diff --git a/packages/shikiji-transformers/test/fixtures/highlight/a.js b/packages/shikiji-transformers/test/fixtures/highlight/a.js new file mode 100644 index 000000000..26c3d0bb6 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/highlight/a.js @@ -0,0 +1,7 @@ +export function foo() { + console.log('highlight') // [!code highlight] + console.log('hl') // [!code hl] + + // should not be transformed: + console.log('[!code highlight]') +} diff --git a/packages/shikiji-transformers/test/fixtures/highlight/a.js.output.html b/packages/shikiji-transformers/test/fixtures/highlight/a.js.output.html new file mode 100644 index 000000000..a130211b9 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/highlight/a.js.output.html @@ -0,0 +1,7 @@ +
export function foo() {  console.log('highlight')   console.log('hl')   // should not be transformed:  console.log('[!code highlight]')}
+ \ No newline at end of file diff --git a/packages/shikiji-transformers/test/fixtures/whitespace/a.js b/packages/shikiji-transformers/test/fixtures/whitespace/a.js new file mode 100644 index 000000000..6a67ee38f --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/whitespace/a.js @@ -0,0 +1,4 @@ +function block( ) { + space() + table() +} diff --git a/packages/shikiji-transformers/test/fixtures/whitespace/a.js.output.html b/packages/shikiji-transformers/test/fixtures/whitespace/a.js.output.html new file mode 100644 index 000000000..83c968c98 --- /dev/null +++ b/packages/shikiji-transformers/test/fixtures/whitespace/a.js.output.html @@ -0,0 +1,13 @@ +
function block( ) {
+  space()
+		table() 
+}
+
+ \ No newline at end of file diff --git a/packages/shikiji/src/core/bundle-factory.ts b/packages/shikiji/src/core/bundle-factory.ts index c6b7a809b..ec34722d8 100644 --- a/packages/shikiji/src/core/bundle-factory.ts +++ b/packages/shikiji/src/core/bundle-factory.ts @@ -69,21 +69,21 @@ export function createSingletonShorthands(g let _shiki: ReturnType async function _getHighlighter(options: { - theme: MaybeArray - lang: MaybeArray - }) { + theme?: MaybeArray + lang?: MaybeArray + } = {}) { if (!_shiki) { _shiki = getHighlighter({ - themes: toArray(options.theme), - langs: toArray(options.lang), + themes: toArray(options.theme || []), + langs: toArray(options.lang || []), }) return _shiki } else { const s = await _shiki await Promise.all([ - s.loadTheme(...toArray(options.theme)), - s.loadLanguage(...toArray(options.lang)), + s.loadTheme(...toArray(options.theme || [])), + s.loadLanguage(...toArray(options.lang || [])), ]) return s } @@ -143,6 +143,7 @@ export function createSingletonShorthands(g } return { + getSingletonHighlighter: () => _getHighlighter(), codeToHtml, codeToHast, codeToThemedTokens, diff --git a/packages/shikiji/src/core/index.ts b/packages/shikiji/src/core/index.ts index 2e6936b0c..13f00f9ad 100644 --- a/packages/shikiji/src/core/index.ts +++ b/packages/shikiji/src/core/index.ts @@ -3,5 +3,6 @@ export { loadWasm } from '../oniguruma' export * from './context' export * from './highlighter' export * from './bundle-factory' +export * from './utils' export { toShikiTheme } from './normalize' diff --git a/packages/shikiji/src/core/renderer-hast.ts b/packages/shikiji/src/core/renderer-hast.ts index 53fac6e71..9d05d915d 100644 --- a/packages/shikiji/src/core/renderer-hast.ts +++ b/packages/shikiji/src/core/renderer-hast.ts @@ -1,5 +1,5 @@ import type { Element, Root, Text } from 'hast' -import type { CodeToHastOptions, HtmlRendererOptions, ShikiContext, ThemedToken } from '../types' +import type { CodeToHastOptions, HtmlRendererOptions, ShikiContext, ShikijiTransformerContext, ThemedToken } from '../types' import { codeToThemedTokens } from './tokenizer' import { FontStyle } from './stackElementMetadata' import { codeToTokensWithThemes } from './renderer-html-themes' @@ -151,6 +151,24 @@ export function tokensToHast( children: lines, } + const context: ShikijiTransformerContext = { + get tokens() { + return tokens + }, + get options() { + return options + }, + get root() { + return tree + }, + get pre() { + return preNode + }, + get code() { + return codeNode + }, + } + tokens.forEach((line, idx) => { if (idx) lines.push({ type: 'text', value: '\n' }) @@ -177,30 +195,30 @@ export function tokensToHast( tokenNode.properties.style = style for (const transformer of transformers) - tokenNode = transformer?.token?.(tokenNode, idx + 1, col, lineNode) || tokenNode + tokenNode = transformer?.token?.call(context, tokenNode, idx + 1, col, lineNode) || tokenNode lineNode.children.push(tokenNode) col += token.content.length } for (const transformer of transformers) - lineNode = transformer?.line?.(lineNode, idx + 1) || lineNode + lineNode = transformer?.line?.call(context, lineNode, idx + 1) || lineNode lines.push(lineNode) }) for (const transformer of transformers) - codeNode = transformer?.code?.(codeNode) || codeNode + codeNode = transformer?.code?.call(context, codeNode) || codeNode preNode.children.push(codeNode) for (const transformer of transformers) - preNode = transformer?.pre?.(preNode) || preNode + preNode = transformer?.pre?.call(context, preNode) || preNode tree.children.push(preNode) let result = tree for (const transformer of transformers) - result = transformer?.root?.(result) || result + result = transformer?.root?.call(context, result) || result return result } diff --git a/packages/shikiji/src/core/utils.ts b/packages/shikiji/src/core/utils.ts index 2e0ae8050..cd849d34c 100644 --- a/packages/shikiji/src/core/utils.ts +++ b/packages/shikiji/src/core/utils.ts @@ -1,3 +1,4 @@ +import type { Element } from 'hast' import type { MaybeArray } from '../types' export function isPlaintext(lang: string | null | undefined) { @@ -11,3 +12,18 @@ export function toArray(x: MaybeArray): T[] { export function isSpecialLang(lang: string) { return lang === 'ansi' || isPlaintext(lang) } + +export function addClassToHast(node: Element, className: string | string[]) { + node.properties ||= {} + node.properties.class ||= [] + if (typeof node.properties.class === 'string') + node.properties.class = node.properties.class.split(/\s+/g) + if (!Array.isArray(node.properties.class)) + node.properties.class = [] + + const targets = Array.isArray(className) ? className : className.split(/\s+/g) + for (const c of targets) { + if (c && !node.properties.class.includes(c)) + node.properties.class.push(c) + } +} diff --git a/packages/shikiji/src/index.ts b/packages/shikiji/src/index.ts index 4571e07bf..4e026969f 100644 --- a/packages/shikiji/src/index.ts +++ b/packages/shikiji/src/index.ts @@ -25,6 +25,7 @@ export const { codeToHast, codeToThemedTokens, codeToTokensWithThemes, + getSingletonHighlighter, } = /* @__PURE__ */ createSingletonShorthands< BuiltinLanguage, BuiltinTheme diff --git a/packages/shikiji/src/types.ts b/packages/shikiji/src/types.ts index 114459d98..e0a1a7f2e 100644 --- a/packages/shikiji/src/types.ts +++ b/packages/shikiji/src/types.ts @@ -271,6 +271,14 @@ export interface ThemeRegistration extends ThemeRegistrationRaw { colors?: Record } +export interface ShikijiTransformerContext { + readonly tokens: ThemedToken[][] + readonly options: CodeToHastOptions + readonly root: Root + readonly pre: Element + readonly code: Element +} + export interface ShikijiTransformer { /** * Name of the transformer @@ -278,20 +286,27 @@ export interface ShikijiTransformer { name?: string /** * Transform the entire generated HAST tree. Return a new Node will replace the original one. - * - * @param hast */ - root?: (hast: Root) => Root | void - pre?: (hast: Element) => Element | void - code?: (hast: Element) => Element | void + root?(this: ShikijiTransformerContext, hast: Root): Root | void + /** + * Transform the `
` element. Return a new Node will replace the original one.
+   */
+  pre?(this: ShikijiTransformerContext, hast: Element): Element | void
+  /**
+   * Transform the `` element. Return a new Node will replace the original one.
+   */
+  code?(this: ShikijiTransformerContext, hast: Element): Element | void
   /**
    * Transform each line element.
    *
    * @param hast
    * @param line 1-based line number
    */
-  line?: (hast: Element, line: number) => Element | void
-  token?: (hast: Element, line: number, col: number, lineElement: Element) => Element | void
+  line?(this: ShikijiTransformerContext, hast: Element, line: number): Element | void
+  /**
+   * Transform each token element.
+   */
+  token?(this: ShikijiTransformerContext, hast: Element, line: number, col: number, lineElement: Element): Element | void
 }
 
 export interface HtmlRendererOptionsCommon {
diff --git a/packages/shikiji/test/cf.ts b/packages/shikiji/test/cf.ts
index 9e68d9c12..5fb08c2e5 100644
--- a/packages/shikiji/test/cf.ts
+++ b/packages/shikiji/test/cf.ts
@@ -1,4 +1,5 @@
 import { getHighlighterCore, loadWasm } from 'shikiji/core'
+import type { LanguageRegistration } from 'shikiji'
 
 import nord from 'shikiji/themes/nord.mjs'
 import js from 'shikiji/langs/javascript.mjs'
@@ -12,7 +13,7 @@ export default {
   async fetch() {
     const highlighter = await getHighlighterCore({
       themes: [nord],
-      langs: [js],
+      langs: [js as LanguageRegistration[]],
     })
 
     return new Response(
diff --git a/packages/shikiji/test/hast.test.ts b/packages/shikiji/test/hast.test.ts
index ca23af9bb..14de3a794 100644
--- a/packages/shikiji/test/hast.test.ts
+++ b/packages/shikiji/test/hast.test.ts
@@ -92,52 +92,17 @@ it('render whitespace', async () => {
     '\t\ttab()',
   ].join('\n')
 
-  const classMap: Record = {
-    ' ': 'space',
-    '\t': 'tab',
-  }
-
   const code = await codeToHtml(snippet, {
     lang: 'js',
     theme: 'vitesse-light',
     transformers: [
-      {
-        line(node) {
-          const first = node.children[0]
-          if (!first || first.type !== 'element')
-            return
-          const textNode = first.children[0]
-          if (!textNode || textNode.type !== 'text')
-            return
-          node.children = node.children.flatMap((child) => {
-            if (child.type !== 'element')
-              return child
-            const node = child.children[0]
-            if (node.type !== 'text' || !node.value)
-              return child
-            const parts = node.value.split(/([ \t])/).filter(i => i.length)
-            if (parts.length <= 1)
-              return child
 
-            return parts.map((part) => {
-              const clone = {
-                ...child,
-                properties: { ...child.properties },
-              }
-              clone.children = [{ type: 'text', value: part }]
-              if (part in classMap)
-                clone.properties.class = [clone.properties.class, classMap[part]].filter(Boolean).join(' ')
-              return clone
-            })
-          })
-        },
-      },
     ],
   })
 
   expect(code)
     .toMatchInlineSnapshot(`
-      "
  space()
-      		tab()
"
+      "
  space()
+      		tab()
"
     `)
 })
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2b17c432c..ba8e13da0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -188,6 +188,15 @@ importers:
       shikiji:
         specifier: workspace:*
         version: link:../shikiji
+      shikiji-transformers:
+        specifier: workspace:*
+        version: link:../shikiji-transformers
+
+  packages/shikiji-transformers:
+    dependencies:
+      shikiji:
+        specifier: workspace:*
+        version: link:../shikiji
 
 packages:
 
diff --git a/tsconfig.json b/tsconfig.json
index 35bf652f5..9fc37e08d 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,12 +1,14 @@
 {
   "compilerOptions": {
-    "target": "es2020",
+    "target": "esnext",
     "lib": ["esnext", "DOM"],
     "rootDir": ".",
     "module": "esnext",
     "moduleResolution": "Bundler",
     "paths": {
-      "shikiji": ["./packages/shikiji/src/index.ts"]
+      "shikiji": ["./packages/shikiji/src/index.ts"],
+      "shikiji/core": ["./packages/shikiji/src/core/index.ts"],
+      "shikiji-transformers": ["./packages/shikiji-transformers/src/index.ts"]
     },
     "resolveJsonModule": true,
     "strict": true,
@@ -15,5 +17,12 @@
     "skipDefaultLibCheck": true,
     "skipLibCheck": true
   },
-  "exclude": ["node_modules", "dist"]
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "**/node_modules/**",
+    "**/dist/**",
+    "**/fixtures/**"
+  ]
 }
diff --git a/vitest.config.ts b/vitest.config.ts
index 9837d4eb2..3f8a9b1d9 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -1,7 +1,7 @@
 import { fileURLToPath } from 'node:url'
 import { defineConfig } from 'vitest/config'
 
-// @ts-expect-error no types
+// @ts-expect-error - no types
 import { wasmPlugin } from './packages/shikiji/rollup.config.mjs'
 
 export default defineConfig({
@@ -10,7 +10,9 @@ export default defineConfig({
   ],
   resolve: {
     alias: {
-      shikiji: fileURLToPath(new URL('./packages/shikiji/src/index.ts', import.meta.url)),
+      'shikiji': fileURLToPath(new URL('./packages/shikiji/src/index.ts', import.meta.url)),
+      'shikiji/core': fileURLToPath(new URL('./packages/shikiji/src/core/index.ts', import.meta.url)),
+      'shikiji-transformers': fileURLToPath(new URL('./packages/shikiji-transformers/src/index.ts', import.meta.url)),
     },
   },
   test: {
@@ -22,6 +24,7 @@ export default defineConfig({
       },
     },
     coverage: {
+      provider: 'v8',
       exclude: [
         '**/src/oniguruma/**',
         '**/src/assets/**',