feat: introduce transformers package (#32)

.vscode/settings.json

@@ -36,5 +36,6 @@
     "json",
     "jsonc",
     "yaml"
-  ]
+  ],
+  "references.preferredLocation": "peek"
 }

README.md

@@ -432,21 +432,23 @@ console.log(root)
 Since `shikiji` uses `hast` internally, you can use the `transforms` option to customize the generated HTML by manipulating the hast tree. You can pass custom functions to modify the tree for different types of nodes. For example:
 
 ```js
+import { addClassToHast, codeToHtml } from 'shikiji'
+
 const code = await codeToHtml('foo\bar', {
   lang: 'js',
   theme: 'vitesse-light',
   transformers: [
     {
       code(node) {
-        node.properties.class = 'language-js'
+        addClassToHast(node, 'language-js')
       },
       line(node, line) {
         node.properties['data-line'] = line
         if ([1, 3, 4].includes(line))
-          node.properties.class += ' highlight'
+          addClassToHast(node, 'highlight')
       },
       token(node, line, col) {
-        node.properties.class = `token:${line}:${col}`
+        node.properties['data-token'] = `token:${line}:${col}`
       },
     },
   ]

eslint.config.js

@@ -4,11 +4,13 @@ export default antfu(
   {
     ignores: [
       'packages/shikiji/src/assets/*.ts',
+      '**/fixtures/**',
     ],
   },
   {
     rules: {
       'no-restricted-syntax': 'off',
+      'ts/no-invalid-this': 'off',
     },
   },
 )

packages/markdown-it-shikiji/src/index.ts

@@ -1,5 +1,5 @@
 import type MarkdownIt from 'markdown-it'
-import { bundledLanguages, getHighlighter } from 'shikiji'
+import { addClassToHast, bundledLanguages, getHighlighter } from 'shikiji'
 import type { BuiltinLanguage, BuiltinTheme, CodeOptionsThemes, CodeToHastOptions, Highlighter, LanguageInput } from 'shikiji'
 import { parseHighlightLines } from '../../shared/line-highlight'
 
@@ -45,7 +45,7 @@ function setup(markdownit: MarkdownIt, highlighter: Highlighter, options: Markdo
           name: 'markdown-it-shikiji:line-class',
           line(node, line) {
             if (lines.includes(line))
-              node.properties.class += ` ${className}`
+              addClassToHast(node, className)
             return node
           },
         })

packages/rehype-shikiji/src/index.ts

@@ -1,5 +1,5 @@
 import type { BuiltinLanguage, BuiltinTheme, CodeOptionsThemes, CodeToHastOptions, LanguageInput } from 'shikiji'
-import { bundledLanguages, getHighlighter } from 'shikiji'
+import { addClassToHast, bundledLanguages, getHighlighter } from 'shikiji'
 import { toString } from 'hast-util-to-string'
 import { visit } from 'unist-util-visit'
 import type { Plugin } from 'unified'
@@ -105,7 +105,7 @@ const rehypeShikiji: Plugin<[RehypeShikijiOptions], Root> = function (options =
             name: 'rehype-shikiji:line-class',
             line(node, line) {
               if (lines.includes(line))
-                node.properties.class += ` ${className}`
+                addClassToHast(node, className)
               return node
             },
           })

packages/shikiji-compat/package.json

@@ -37,6 +37,7 @@
     "prepublishOnly": "nr build"
   },
   "dependencies": {
-    "shikiji": "workspace:*"
+    "shikiji": "workspace:*",
+    "shikiji-transformers": "workspace:*"
   }
 }

packages/shikiji-compat/src/index.ts

@@ -2,6 +2,7 @@ import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import type { BuiltinLanguage, BuiltinTheme, CodeToThemedTokensOptions, MaybeGetter, StringLiteralUnion, ThemeInput, ThemeRegistration, ThemedToken } from 'shikiji'
 import { bundledLanguages, bundledThemes, getHighlighter as getShikiji, toShikiTheme } from 'shikiji'
+import { transformerCompactLineOptions } from 'shikiji-transformers'
 import type { AnsiToHtmlOptions, CodeToHtmlOptions, CodeToHtmlOptionsExtra, HighlighterOptions } from './types'
 
 export const BUNDLED_LANGUAGES = bundledLanguages
@@ -66,17 +67,7 @@ export async function getHighlighter(options: HighlighterOptions = {}) {
 
     if (options.lineOptions) {
       options.transformers ||= []
-      options.transformers.push({
-        name: 'shikiji-compat:line-class',
-        line(node, line) {
-          const lineOption = options.lineOptions?.find(o => o.line === line)
-          if (lineOption?.classes) {
-            node.properties ??= {}
-            node.properties.class = [node.properties.class, ...lineOption.classes].filter(Boolean).join(' ')
-          }
-          return node
-        },
-      })
+      options.transformers.push(transformerCompactLineOptions(options.lineOptions))
     }
 
     return shikiji.codeToHtml(code, options as any)

packages/shikiji-transformers/README.md

@@ -0,0 +1,98 @@
+# shikiji-transformers
+
+Collective of common transformers for [shikiji](https://github.com/antfu/shikiji), inspired by [shiki-processor](https://github.com/innocenzi/shiki-processor).
+
+## Install
+
+```bash
+npm i -D shikiji-transformers
+```
+
+```ts
+import {
+  codeToHtml,
+} from 'shikiji'
+import {
+  transformerNotationDiff,
+  // ...
+} from 'shikiji-transformers'
+
+const html = codeToHtml(code, {
+  lang: 'ts',
+  transformers: [
+    transformerNotationDiff(),
+    // ...
+  ],
+})
+```
+
+## Transformers
+
+### `transformerNotationDiff`
+
+Use `[!code ++]` and `[!code --]` to mark added and removed lines.
+
+For example, the following code
+
+```ts
+export function foo() {
+  console.log('hewwo') // [!code --]
+  console.log('hello') // [!code ++]
+}
+```
+
+will be transformed to
+
+```html
+<!-- Output (stripped of `style` attributes for clarity) -->
+<pre class="shiki has-diff"> <!-- Notice `has-diff` -->
+  <code>
+    <span class="line"></span>
+    <span class="line"><span>function</span><span>()</span><span></span><span>{</span></span>
+    <span class="line diff remove">  <!-- Notice `diff` and `remove` -->
+      <span></span><span>console</span><span>.</span><span>log</span><span>(</span><span>&#39;</span><span>hewwo</span><span>&#39;</span><span>) </span>
+    </span>
+    <span class="line diff add">  <!-- Notice `diff` and `add` -->
+      <span></span><span>console</span><span>.</span><span>log</span><span>(</span><span>&#39;</span><span>hello</span><span>&#39;</span><span>) </span>
+    </span>
+    <span class="line"><span></span><span>}</span></span>
+    <span class="line"><span></span></span>
+  </code>
+</pre>
+```
+
+With some CSS, you can make it look like this:
+
+<img width="362" alt="image" src="https://github.com/antfu/shikiji/assets/11247099/c1b6d2b2-686c-4d36-87b1-f01877071cde">
+
+### `transformerNotationHighlight`
+
+Use `[!code highlight]` to highlight a line (adding `highlighted` class).
+
+### `transformerNotationFocus`
+
+Use `[!code focus]` to focus a line (adding `focused` class).
+
+### `transformerNotationErrorLevel`
+
+Use `[!code error]`, `[!code warning]`, to mark a line with an error level (adding `highlighted error`, `highlighted warning` class).
+
+### `transformerRenderWhitespace`
+
+Render whitespaces (tabs and spaces) as individual spans, with classes `tab` and `space`.
+
+With some CSS, you can make it look like this:
+
+<img width="293" alt="image" src="https://github.com/antfu/shikiji/assets/11247099/01b7c4ba-6d63-4e74-8fd7-68a9f901f3de">
+
+### `transformerCompactLineOptions`
+
+Support for `shiki`'s `lineOptions` that is removed in `shikiji`.
+
+### `transformerRemoveLineBreak`
+
+Remove line breaks between `<span class="line">`. Useful when you set `display: block` to `.line` in CSS.
+
+## License
+
+MIT

packages/shikiji-transformers/build.config.ts

@@ -0,0 +1,14 @@
+import { defineBuildConfig } from 'unbuild'
+
+export default defineBuildConfig({
+  entries: [
+    'src/index.ts',
+  ],
+  declaration: true,
+  rollup: {
+    emitCJS: false,
+  },
+  externals: [
+    'hast',
+  ],
+})

packages/shikiji-transformers/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "shikiji-transformers",
+  "type": "module",
+  "version": "0.0.0",
+  "description": "Collective of common transformers transformers for Shikiji",
+  "author": "Anthony Fu <anthonyfu117@hotmail.com>",
+  "license": "MIT",
+  "homepage": "https://github.com/antfu/shikiji#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/antfu/shikiji.git",
+    "directory": "packages/shikiji-transformers"
+  },
+  "bugs": "https://github.com/antfu/shikiji/issues",
+  "keywords": [
+    "shiki",
+    "shikiji-transformers"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "unbuild",
+    "dev": "unbuild --stub",
+    "prepublishOnly": "nr build"
+  },
+  "dependencies": {
+    "shikiji": "workspace:*"
+  }
+}

packages/shikiji-transformers/src/index.ts

@@ -0,0 +1,8 @@
+export * from './transformers/render-whitespace'
+export * from './transformers/remove-line-breaks'
+export * from './transformers/compact-line-options'
+export * from './transformers/notation-focus'
+export * from './transformers/notation-highlight'
+export * from './transformers/notation-diff'
+export * from './transformers/notation-error-level'
+export * from './utils'

packages/shikiji-transformers/src/transformers/compact-line-options.ts

@@ -0,0 +1,27 @@
+import type { ShikijiTransformer } from 'shikiji'
+import { addClassToHast } from 'shikiji'
+
+export interface TransformerCompactLineOption {
+  /**
+   * 1-based line number.
+   */
+  line: number
+  classes?: string[]
+}
+
+/**
+ * Transformer for `shiki`'s legacy `lineOptions`
+ */
+export function transformerCompactLineOptions(
+  lineOptions: TransformerCompactLineOption[] = [],
+): ShikijiTransformer {
+  return {
+    name: 'shikiji-transformers:compact-line-options',
+    line(node, line) {
+      const lineOption = lineOptions.find(o => o.line === line)
+      if (lineOption?.classes)
+        addClassToHast(node, lineOption.classes)
+      return node
+    },
+  }
+}

packages/shikiji-transformers/src/transformers/notation-diff.ts

@@ -0,0 +1,45 @@
+import type { ShikijiTransformer } from 'shikiji'
+import { addClassToHast } from 'shikiji'
+import { createCommentNotationTransformer } from '../utils'
+
+export interface TransformerNotationDiffOptions {
+  /**
+   * Class for added lines
+   */
+  classAdded?: string
+  /**
+   * Class for removed lines
+   */
+  classRemoved?: string
+  /**
+   * Class added to the root element when the current code has diff
+   */
+  classRootActive?: string
+}
+
+/**
+ * Use `[!code ++]` and `[!code --]` to mark added and removed lines.
+ */
+export function transformerNotationDiff(
+  options: TransformerNotationDiffOptions = {},
+): ShikijiTransformer {
+  const {
+    classAdded = 'diff add',
+    classRemoved = 'diff remove',
+    classRootActive = 'has-diff',
+  } = options
+
+  return createCommentNotationTransformer(
+    'shikiji-transformers:notation-diff',
+    /\[!code (\-\-|\+\+)\]/,
+    function ([_, match], line) {
+      const className = match === '--'
+        ? classRemoved
+        : classAdded
+      addClassToHast(line, className)
+      if (classRootActive)
+        addClassToHast(this.pre, classRootActive)
+      return true
+    },
+  )
+}

packages/shikiji-transformers/src/transformers/notation-error-level.ts

@@ -0,0 +1,30 @@
+import type { ShikijiTransformer } from 'shikiji'
+import { addClassToHast } from 'shikiji'
+import { createCommentNotationTransformer } from '../utils'
+
+export interface TransformerNotationErrorLevelOptions {
+  classMap?: Record<string, string | string[]>
+}
+
+/**
+ * Allow using `[!code error]` `[!code warning]` notation in code to mark highlighted lines.
+ */
+export function transformerNotationErrorLevel(
+  options: TransformerNotationErrorLevelOptions = {},
+): ShikijiTransformer {
+  const {
+    classMap = {
+      error: ['highlighted', 'error'],
+      warning: ['highlighted', 'warning'],
+    },
+  } = options
+
+  return createCommentNotationTransformer(
+    'shikiji-transformers:notation-error-level',
+    new RegExp(`\\[!code (${Object.keys(classMap).join('|')})\\]`),
+    ([_, match], line) => {
+      addClassToHast(line, classMap[match])
+      return true
+    },
+  )
+}