feat(transformers): add legacy option for notation transformers

packages/transformers/src/shared/notation-transformer.ts

@@ -1,6 +1,6 @@
 import type { Element, Text } from 'hast'
 import type { ShikiTransformer, ShikiTransformerContext } from 'shiki'
-import { parseComments, type ParsedComments } from './parse-comments'
+import { legacyClearEndCommentPrefix, parseComments, type ParsedComments } from './parse-comments'
 
 export function createCommentNotationTransformer(
   name: string,
@@ -13,6 +13,7 @@ export function createCommentNotationTransformer(
     lines: Element[],
     index: number
   ) => boolean,
+  legacy = false,
 ): ShikiTransformer {
   return {
     name,
@@ -25,7 +26,7 @@ export function createCommentNotationTransformer(
         _shiki_notation?: ParsedComments
       }
 
-      data._shiki_notation ??= parseComments(lines, ['jsx', 'tsx'].includes(this.options.lang))
+      data._shiki_notation ??= parseComments(lines, ['jsx', 'tsx'].includes(this.options.lang), legacy)
       const parsed = data._shiki_notation
 
       for (const comment of parsed) {
@@ -34,17 +35,26 @@ export function createCommentNotationTransformer(
 
         const isLineCommentOnly = comment.line.children.length === (comment.isJsxStyle ? 3 : 1)
         let lineIdx = lines.indexOf(comment.line)
-        if (isLineCommentOnly)
+        if (isLineCommentOnly && !legacy)
           lineIdx++
 
+        let replaced = false
         comment.info[1] = comment.info[1].replace(regex, (...match) => {
           if (onMatch.call(this, match, comment.line, comment.token, lines, lineIdx)) {
+            replaced = true
             return ''
           }
 
           return match[0]
         })
 
+        if (!replaced)
+          continue
+
+        if (legacy) {
+          comment.info[1] = legacyClearEndCommentPrefix(comment.info[1])
+        }
+
         const isEmpty = comment.info[1].trim().length === 0
         // ignore comment node
         if (isEmpty)

packages/transformers/src/shared/parse-comments.ts

@@ -21,34 +21,42 @@ const matchers: [re: RegExp, endOfLine: boolean][] = [
   [/^(\*)(.+)$/, true],
 ]
 
-export function parseComments(lines: Element[], jsx: boolean): ParsedComments {
+/**
+ * @param lines line tokens
+ * @param jsx enable JSX parsing
+ * @param legacy support legacy behaviours, force to parse all tokens.
+ */
+export function parseComments(lines: Element[], jsx: boolean, legacy = false): ParsedComments {
   const out: ParsedComments = []
 
   for (const line of lines) {
     const elements = line.children
-    // one step further for JSX as it's inside `{}`
-    const start = jsx ? elements.length - 2 : elements.length - 1
+    let start = elements.length - 1
+    if (legacy)
+      start = 0
+    else if (jsx)
+      // one step further for JSX as comment is inside curly brackets
+      start = elements.length - 2
 
     for (let i = Math.max(start, 0); i < elements.length; i++) {
       const token = elements[i]
-
       if (token.type !== 'element')
         continue
+      const head = token.children.at(0)
+      if (head?.type !== 'text')
+        continue
 
       const isLast = i === elements.length - 1
-      const match = matchToken(token, isLast)
+      const match = matchToken(head.value, isLast)
       if (!match)
         continue
 
       if (jsx && !isLast && i !== 0) {
-        const left = elements[i - 1]
-        const right = elements[i + 1]
-
         out.push({
           info: match,
           line,
           token,
-          isJsxStyle: isValue(left, '{') && isValue(right, '}'),
+          isJsxStyle: isValue(elements[i - 1], '{') && isValue(elements[i + 1], '}'),
         })
       }
       else {
@@ -76,26 +84,22 @@ function isValue(element: ElementContent, value: string): boolean {
 }
 
 /**
- * @param token the token node (children of line)
+ * @param text text value of comment node
  * @param isLast whether the token is located at the end of line
  */
-function matchToken(token: Element, isLast: boolean): [prefix: string, content: string, suffix?: string] | undefined {
-  const text = token.children[0]
-  if (text.type !== 'text')
-    return
+function matchToken(text: string, isLast: boolean): [prefix: string, content: string, suffix?: string] | undefined {
+  // no leading and trailing spaces allowed for matchers
+  // we extract the spaces
+  let trimmed = text.trimStart()
+  const spaceFront = text.length - trimmed.length
+
+  trimmed = trimmed.trimEnd()
+  const spaceEnd = text.length - trimmed.length - spaceFront
 
   for (const [matcher, endOfLine] of matchers) {
     if (endOfLine && !isLast)
       continue
 
-    // no leading and trailing spaces allowed for matchers
-    // we extract the spaces
-    let trimmed = text.value.trimStart()
-    const spaceFront = text.value.length - trimmed.length
-
-    trimmed = trimmed.trimEnd()
-    const spaceEnd = text.value.length - trimmed.length - spaceFront
-
     const result = matcher.exec(trimmed)
     if (!result)
       continue
@@ -107,3 +111,17 @@ function matchToken(token: Element, isLast: boolean): [prefix: string, content:
     ]
   }
 }
+
+/**
+ * Remove empty comment prefixes at line end, e.g. `// `
+ */
+export function legacyClearEndCommentPrefix(text: string): string {
+  const regex = /(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(.*)$/
+  const result = regex.exec(text)
+
+  if (result && result[1].trim().length === 0) {
+    return text.slice(0, result.index)
+  }
+
+  return text
+}

packages/transformers/src/transformers/notation-diff.ts

@@ -14,6 +14,8 @@ export interface TransformerNotationDiffOptions {
    * Class added to the <pre> element when the current code has diff
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 /**
@@ -26,6 +28,7 @@ export function transformerNotationDiff(
     classLineAdd = 'diff add',
     classLineRemove = 'diff remove',
     classActivePre = 'has-diff',
+    legacy,
   } = options
 
   return transformerNotationMap(
@@ -34,6 +37,7 @@ export function transformerNotationDiff(
         '++': classLineAdd,
         '--': classLineRemove,
       },
+      legacy,
       classActivePre,
     },
     '@shikijs/transformers:notation-diff',

packages/transformers/src/transformers/notation-error-level.ts

@@ -7,6 +7,8 @@ export interface TransformerNotationErrorLevelOptions {
    * Class added to the <pre> element when the current code has diff
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 /**
@@ -21,12 +23,14 @@ export function transformerNotationErrorLevel(
       warning: ['highlighted', 'warning'],
     },
     classActivePre = 'has-highlighted',
+    legacy,
   } = options
 
   return transformerNotationMap(
     {
       classMap,
       classActivePre,
+      legacy,
     },
     '@shikijs/transformers:notation-error-level',
   )

packages/transformers/src/transformers/notation-focus.ts

@@ -10,6 +10,8 @@ export interface TransformerNotationFocusOptions {
    * Class added to the root element when the code has focused lines
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 /**
@@ -21,6 +23,7 @@ export function transformerNotationFocus(
   const {
     classActiveLine = 'focused',
     classActivePre = 'has-focused',
+    legacy,
   } = options
 
   return transformerNotationMap(
@@ -29,6 +32,7 @@ export function transformerNotationFocus(
         focus: classActiveLine,
       },
       classActivePre,
+      legacy,
     },
     '@shikijs/transformers:notation-focus',
   )

packages/transformers/src/transformers/notation-highlight-word.ts

@@ -11,6 +11,8 @@ export interface TransformerNotationWordHighlightOptions {
    * Class added to the root element when the code has highlighted words
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 export function transformerNotationWordHighlight(
@@ -19,6 +21,7 @@ export function transformerNotationWordHighlight(
   const {
     classActiveWord = 'highlighted-word',
     classActivePre = undefined,
+    legacy,
   } = options
 
   return createCommentNotationTransformer(
@@ -37,5 +40,6 @@ export function transformerNotationWordHighlight(
         this.addClassToHast(this.pre, classActivePre)
       return true
     },
+    legacy,
   )
 }

packages/transformers/src/transformers/notation-highlight.ts

@@ -10,6 +10,8 @@ export interface TransformerNotationHighlightOptions {
    * Class added to the root element when the code has highlighted lines
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 /**
@@ -21,6 +23,7 @@ export function transformerNotationHighlight(
   const {
     classActiveLine = 'highlighted',
     classActivePre = 'has-highlighted',
+    legacy,
   } = options
 
   return transformerNotationMap(
@@ -29,6 +32,7 @@ export function transformerNotationHighlight(
         highlight: classActiveLine,
         hl: classActiveLine,
       },
+      legacy,
       classActivePre,
     },
     '@shikijs/transformers:notation-highlight',

packages/transformers/src/transformers/notation-map.ts

@@ -7,6 +7,8 @@ export interface TransformerNotationMapOptions {
    * Class added to the <pre> element when the current code has diff
    */
   classActivePre?: string
+
+  legacy?: boolean
 }
 
 function escapeRegExp(str: string): string {
@@ -20,6 +22,7 @@ export function transformerNotationMap(
   const {
     classMap = {},
     classActivePre = undefined,
+    legacy,
   } = options
 
   return createCommentNotationTransformer(
@@ -36,5 +39,6 @@ export function transformerNotationMap(
         this.addClassToHast(this.pre, classActivePre)
       return true
     },
+    legacy,
   )
 }

packages/transformers/test/fixtures.test.ts

@@ -185,3 +185,27 @@ body { margin: 0; }
 .space::before { content: "\\B7"; position: absolute; opacity: 0.3; }
 </style>`,
 )
+
+suite(
+  'all',
+  import.meta.glob('./fixtures/legacy/*.*', { as: 'raw', eager: true }),
+  [
+    transformerNotationFocus({ legacy: true }),
+    transformerNotationHighlight({ legacy: true }),
+    transformerNotationErrorLevel({ legacy: true }),
+    transformerNotationWordHighlight({ legacy: true }),
+    transformerRemoveLineBreak(),
+  ],
+  code => `${code}
+<style>
+* { tab-size: 4; }
+body { margin: 0; }
+.line { display: block; width: 100%; height: 1.2em; }
+.shiki { padding: 1em; }
+.has-focused .focused { background-color: #8805; }
+.highlighted { background-color: #8885; }
+.highlighted-word { background-color: #8885; }
+.highlighted.warning { background-color: #9905; }
+.highlighted.error { background-color: #8005; }
+</style>`,
+)