feat: introduce Decorations API (#574)

docs/.vitepress/config.ts

@@ -1,6 +1,9 @@
 import type { DefaultTheme } from 'vitepress'
 import { defineConfig } from 'vitepress'
 import { bundledThemes } from 'shiki'
+
+// @ts-expect-error missing types
+import { withMermaid } from 'vitepress-plugin-mermaid'
 import { transformerMetaWordHighlight, transformerNotationWordHighlight } from '../../packages/transformers/src'
 import { defaultHoverInfoProcessor, transformerTwoslash } from '../../packages/vitepress-twoslash/src/index'
 import { version } from '../../package.json'
@@ -11,6 +14,7 @@ const GUIDES: DefaultTheme.NavItemWithLink[] = [
   { text: 'Installation', link: '/guide/install' },
   { text: 'Bundles', link: '/guide/bundles' },
   { text: 'Dual Themes', link: '/guide/dual-themes' },
+  { text: 'Decorations', link: '/guide/decorations' },
   { text: 'Transformers', link: '/guide/transformers' },
   { text: 'Theme Colors Manipulation', link: '/guide/theme-colors' },
   { text: 'Migration', link: '/guide/migrate' },
@@ -48,7 +52,7 @@ const VERSIONS: (DefaultTheme.NavItemWithLink | DefaultTheme.NavItemChildren)[]
 ]
 
 // https://vitepress.dev/reference/site-config
-export default defineConfig({
+export default withMermaid(defineConfig({
   title: 'Shiki',
   description: 'A beautiful and powerful syntax highlighter',
   markdown: {
@@ -97,6 +101,18 @@ export default defineConfig({
           return code
         },
       },
+      {
+        name: 'shiki:inline-decorations',
+        preprocess(code, options) {
+          const reg = /^\/\/ @decorations:(.*?)\n/
+          code = code.replace(reg, (match, decorations) => {
+            options.decorations ||= []
+            options.decorations.push(...JSON.parse(decorations))
+            return ''
+          })
+          return code
+        },
+      },
       transformerTwoslash({
         processHoverInfo(info) {
           return defaultHoverInfoProcessor(info)
@@ -205,4 +221,4 @@ export default defineConfig({
     ['meta', { name: 'twitter:image', content: 'https://shiki.style/og.png' }],
     ['meta', { name: 'viewport', content: 'width=device-width, initial-scale=1.0, viewport-fit=cover' }],
   ],
-})
+}))

docs/api.md

@@ -0,0 +1,80 @@
+# API References
+
+## `codeToHast`
+
+You can also get the intermediate `hast` to do custom rendering without serializing them into HTML with `codeToHast`. You can also further integrate the AST with the [unified](https://github.com/unifiedjs) ecosystem.
+
+```ts twoslash
+import { getHighlighter } from 'shiki'
+
+const highlighter = await getHighlighter({
+  themes: ['nord', 'min-light'],
+  langs: ['javascript'],
+})
+// ---cut---
+const root = highlighter.codeToHast(
+  'const a = 1',
+  { lang: 'javascript', theme: 'nord' }
+)
+
+console.log(root)
+```
+
+<!-- eslint-skip -->
+
+```ts
+{
+  type: 'root',
+  children: [
+    {
+      type: 'element',
+      tagName: 'pre',
+      properties: {
+        class: 'shiki vitesse-light',
+        style: 'background-color:#ffffff;color:#393a34',
+        tabindex: '0'
+      },
+      children: [
+        {
+          type: 'element',
+          tagName: 'code',
+          properties: {},
+          children: [
+            {
+              type: 'element',
+              tagName: 'span',
+              properties: { class: 'line' },
+              children: [
+                {
+                  type: 'element',
+                  tagName: 'span',
+                  properties: { style: 'color:#AB5959' },
+                  children: [ { type: 'text', value: 'const' } ]
+                },
+                {
+                  type: 'element',
+                  tagName: 'span',
+                  properties: { style: 'color:#B07D48' },
+                  children: [ { type: 'text', value: ' a' } ]
+                },
+                {
+                  type: 'element',
+                  tagName: 'span',
+                  properties: { style: 'color:#999999' },
+                  children: [ { type: 'text', value: ' =' } ]
+                },
+                {
+                  type: 'element',
+                  tagName: 'span',
+                  properties: { style: 'color:#2F798A' },
+                  children: [ { type: 'text', value: ' 1' } ]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+```

docs/guide/decorations.md

@@ -0,0 +1,105 @@
+# Decorations
+
+We provide a decorations API allowing you to warp custom class and attributes to ranges of your code.
+
+```ts twoslash
+import { codeToHtml } from 'shiki'
+
+const code = `
+const x = 10
+console.log(x)
+`.trim()
+
+const html = await codeToHtml(code, {
+  theme: 'vitesse-light',
+  lang: 'ts',
+  decorations: [ // [!code hl:8]
+    {
+      // line and character are 0-indexed
+      start: { line: 1, character: 0 },
+      end: { line: 1, character: 11 },
+      properties: { class: 'highlighted-word' }
+    }
+  ]
+})
+```
+
+The result will be (styled with CSS in this example):
+
+```ts
+// @decorations:[{"start":{"line":1,"character":0},"end":{"line":1,"character":11},"properties":{"class":"highlighted-word"}}]
+const x = 10
+console.log(x)
+```
+
+The positions can also be 0-indexed offsets relative to the code:
+
+```ts twoslash
+import { codeToHtml } from 'shiki'
+
+const code = `
+const x = 10
+console.log(x)
+`.trim()
+// ---cut---
+const html = await codeToHtml(code, {
+  theme: 'vitesse-light',
+  lang: 'ts',
+  decorations: [ // [!code hl:7]
+    {
+      start: 21,
+      end: 24,
+      properties: { class: 'highlighted-word' }
+    }
+  ]
+})
+```
+
+It renders:
+
+```ts
+// @decorations:[{"start":21,"end":24,"properties":{"class":"highlighted-word"}}]
+const x = 10
+console.log(x)
+```
+
+## Use Decorations in Transformers
+
+For advanced use cases, you can use the [Transformers API](./transformers.md) to have full access to the tokens and the HAST tree.
+
+Meanwhile, if you want to append decorations within a transformer, you can do that with:
+
+```ts twoslash
+/* eslint-disable import/no-duplicates */
+import { DecorationItem } from 'shiki'
+function doSomethingWithCode(code: string): DecorationItem[] {
+  return []
+}
+const code: string = ''
+
+// ---cut---
+import { ShikiTransformer, codeToHtml } from 'shiki'
+
+const myTransformer: ShikiTransformer = {
+  name: 'my-transformer',
+  preprocess(code, options) {
+    // Generate the decorations somehow
+    const decorations = doSomethingWithCode(code)
+
+    // Make sure the decorations array exists
+    options.decorations ||= []
+    // Append the decorations
+    options.decorations.push(...decorations)
+  }
+}
+
+const html = await codeToHtml(code, {
+  theme: 'vitesse-light',
+  lang: 'ts',
+  transformers: [
+    myTransformer
+  ]
+})
+```
+
+Note that you can only provide decorations in or before the `preprocess` hook. In later hooks, changes to the decorations arrary will be ignored.

docs/guide/transformers.md

@@ -28,83 +28,28 @@ const code = await codeToHtml('foo\bar', {
 })
 ```
 
-We also provide some common transformers for you to use, see [`shiki-transforms`](/packages/transformers) for more details.
-
-## `codeToHast`
-
-You can also get the intermediate `hast` to do custom rendering without serializing them into HTML with `codeToHast`. You can also further integrate the AST with the [unified](https://github.com/unifiedjs) ecosystem.
-
-```ts twoslash
-import { getHighlighter } from 'shiki'
-
-const highlighter = await getHighlighter({
-  themes: ['nord', 'min-light'],
-  langs: ['javascript'],
-})
-// ---cut---
-const root = highlighter.codeToHast(
-  'const a = 1',
-  { lang: 'javascript', theme: 'nord' }
-)
-
-console.log(root)
+We also provide some common transformers for you to use, see [`@shikijs/transforms`](/packages/transformers) for more details.
+
+## Transformer Hooks
+
+```mermaid
+flowchart LR
+  preprocess --> tokens
+  tokens --> span
+  span --> span
+  span --> line
+  line --> span
+  line --> code
+  code --> pre
+  pre --> root
+  root --> postprocess
 ```
 
-<!-- eslint-skip -->
-
-```ts
-{
-  type: 'root',
-  children: [
-    {
-      type: 'element',
-      tagName: 'pre',
-      properties: {
-        class: 'shiki vitesse-light',
-        style: 'background-color:#ffffff;color:#393a34',
-        tabindex: '0'
-      },
-      children: [
-        {
-          type: 'element',
-          tagName: 'code',
-          properties: {},
-          children: [
-            {
-              type: 'element',
-              tagName: 'span',
-              properties: { class: 'line' },
-              children: [
-                {
-                  type: 'element',
-                  tagName: 'span',
-                  properties: { style: 'color:#AB5959' },
-                  children: [ { type: 'text', value: 'const' } ]
-                },
-                {
-                  type: 'element',
-                  tagName: 'span',
-                  properties: { style: 'color:#B07D48' },
-                  children: [ { type: 'text', value: ' a' } ]
-                },
-                {
-                  type: 'element',
-                  tagName: 'span',
-                  properties: { style: 'color:#999999' },
-                  children: [ { type: 'text', value: ' =' } ]
-                },
-                {
-                  type: 'element',
-                  tagName: 'span',
-                  properties: { style: 'color:#2F798A' },
-                  children: [ { type: 'text', value: ' 1' } ]
-                }
-              ]
-            }
-          ]
-        }
-      ]
-    }
-  ]
-}
-```
+- `preprocess` - Called before the code is tokenized. You can use this to modify the code before it is tokenized.
+- `tokens` - Called after the code is tokenized. You can use this to modify the tokens.
+- `span` - Called for each `<span>` tag, for each token.
+- `line` - Called for each line `<span>` tag.
+- `code` - Called for each `<code>` tag, wraps all the lines.
+- `pre` - Called for each `<pre>` tag, wraps the `<code>` tag.
+- `root` - The root of HAST tree. Usually with only one child `<pre>` tag.
+- `postprocess` - Called after the HTML is generated, get a chance to modify the final output. Will not been called in `codeToHast`.

package.json

@@ -61,6 +61,7 @@
     "unbuild": "^2.0.0",
     "vite": "^5.0.12",
     "vite-tsconfig-paths": "^4.3.1",
+    "vitepress-plugin-mermaid": "^2.0.16",
     "vitest": "^1.2.2",
     "vue-tsc": "^1.8.27",
     "wrangler": "^3.25.0"