feat: Use Oniguruma-To-ES in the JS engine (#828)

docs/guide/regex-engines.md

@@ -4,7 +4,7 @@ outline: deep
 
 # RegExp Engines
 
-TextMate grammars is based on regular expressions to match tokens. Usually, we use [Oniguruma](https://github.com/kkos/oniguruma) (a regular expression engine written in C) to parse the grammar. To make it work in JavaScript, we compile Oniguruma to WebAssembly to run in the browser or Node.js.
+TextMate grammars are based on regular expressions to match tokens. Usually, we use [Oniguruma](https://github.com/kkos/oniguruma) (a regular expression engine written in C) to parse the grammar. To make it work in JavaScript, we compile Oniguruma to WebAssembly to run in the browser or Node.js.
 
 Since v1.15, we expose the ability to for users to switch the RegExp engine and provide custom implementations.
 
@@ -20,7 +20,7 @@ const shiki = await createShiki({
 })
 ```
 
-Shiki come with two built-in engines:
+Shiki comes with two built-in engines:
 
 ## Oniguruma Engine
 
@@ -43,7 +43,7 @@ const shiki = await createShiki({
 This feature is experimental and may change without following semver.
 :::
 
-This experimental engine uses JavaScript's native RegExp. As TextMate grammars' regular expressions are in Oniguruma flavor that might contains syntaxes that are not supported by JavaScript's RegExp, we use [`oniguruma-to-js`](https://github.com/antfu/oniguruma-to-js) to lowering the syntaxes and try to make them compatible with JavaScript's RegExp.
+This engine uses JavaScript's native RegExp. As regular expressions used by TextMate grammars are written for Oniguruma, they might contain syntax that is not supported by JavaScript's RegExp, or expect different behavior for the same syntax. So we use [Oniguruma-To-ES](https://github.com/slevithan/oniguruma-to-es) to transpile Oniguruma patterns to native JavaScript RegExp.
 
 ```ts {2,4,9}
 import { createHighlighter } from 'shiki'
@@ -60,17 +60,17 @@ const shiki = await createHighlighter({
 const html = shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'nord' })
 ```
 
-Please check the [compatibility table](/references/engine-js-compat) to check the support status of the languages you are using.
+Please check the [compatibility table](/references/engine-js-compat) for the support status of the languages you are using.
 
-If mismatches are acceptable and you want it to get results whatever it can, you can enable the `forgiving` option to suppress any errors happened during the conversion:
+If mismatches are acceptable and you want best-effort results whenever possible, you can enable the `forgiving` option to suppress any errors that happened during the conversion:
 
 ```ts
 const jsEngine = createJavaScriptRegexEngine({ forgiving: true })
 // ...use the engine
 ```
 
 ::: info
-If you runs Shiki on Node.js (or at build time), we still recommend using the Oniguruma engine for the best result, as most of the time bundle size or WebAssembly support is not a concern.
+If you run Shiki on Node.js (or at build time) and bundle size or WebAssembly support is not a concern, we still recommend using the Oniguruma engine for the best result.
 
-The JavaScript engine is more suitable for running in the browser in some cases that you want to control the bundle size.
+The JavaScript engine is best when running in the browser and in cases when you want to control the bundle size.
 :::

packages/core/rollup.config.mjs

@@ -36,11 +36,9 @@ const external = [
   'hast',
   '@shikijs/vscode-textmate',
 
-  // Externalize them to make it easier to patch and experiments
+  // Externalize to make it easier to patch and experiment
   // Versions are pinned to avoid regressions
-  // Later we might consider to bundle them.
-  'oniguruma-to-js',
-  'regex',
+  'oniguruma-to-es',
 ]
 
 export default defineConfig([

packages/engine-javascript/README.md

@@ -1,6 +1,6 @@
 # @shikijs/engine-javascript
 
-Engine for Shiki using JavaScript's native RegExp (experimental).
+Engine for Shiki using JavaScript's native RegExp (experimental). Uses [Oniguruma-To-ES](https://github.com/slevithan/oniguruma-to-es) to transpile regex syntax and behavior.
 
 [Documentation](https://shiki.style/guide/regex-engines)
 

packages/engine-javascript/package.json

@@ -37,6 +37,6 @@
   "dependencies": {
     "@shikijs/types": "workspace:*",
     "@shikijs/vscode-textmate": "catalog:",
-    "oniguruma-to-js": "catalog:"
+    "oniguruma-to-es": "catalog:"
   }
 }

packages/engine-javascript/src/index.ts

@@ -4,8 +4,7 @@
   RegexEngineString,
 } from '@shikijs/types'
 import type { IOnigMatch } from '@shikijs/vscode-textmate'
-import { onigurumaToRegexp } from 'oniguruma-to-js'
-import { replacements } from './replacements'
+import { toRegExp } from 'oniguruma-to-es'
 
 export interface JavaScriptRegexEngineOptions {
   /**
@@ -16,7 +15,7 @@
   forgiving?: boolean
 
   /**
-   * Use JavaScript to simulate some unsupported regex features.
+   * Cleanup some grammar patterns before use.
    *
    * @default true
    */
@@ -30,7 +29,7 @@
   /**
    * Custom pattern to RegExp constructor.
    *
-   * By default `oniguruma-to-js` is used.
+   * By default `oniguruma-to-es` is used.
    */
   regexConstructor?: (pattern: string) => RegExp
 }
@@ -41,18 +40,19 @@
  * The default RegExp constructor for JavaScript regex engine.
  */
 export function defaultJavaScriptRegexConstructor(pattern: string): RegExp {
-  return onigurumaToRegexp(
+  return toRegExp(
     pattern,
     {
-      flags: 'dgm',
-      ignoreContiguousAnchors: true,
+      accuracy: 'loose',
+      global: true,
+      hasIndices: true,
+      tmGrammar: true,
     },
   )
 }
 
 export class JavaScriptScanner implements PatternScanner {
   regexps: (RegExp | null)[]
-  contiguousAnchorSimulation: boolean[]
 
   constructor(
     public patterns: string[],
@@ -65,8 +65,7 @@
       regexConstructor = defaultJavaScriptRegexConstructor,
     } = options
 
-    this.contiguousAnchorSimulation = Array.from({ length: patterns.length }, () => false)
-    this.regexps = patterns.map((p, idx) => {
+    this.regexps = patterns.map((p) => {
       /**
        * vscode-textmate replace anchors to \uFFFF, where we still not sure how to handle it correctly
        *
@@ -77,10 +76,6 @@
       if (simulation)
         p = p.replaceAll('(^|\\\uFFFF)', '(^|\\G)')
 
-      // Detect contiguous anchors for simulation
-      if (simulation && (p.startsWith('(^|\\G)') || p.startsWith('(\\G|^)')))
-        this.contiguousAnchorSimulation[idx] = true
-
       // Cache
       const cached = cache?.get(p)
       if (cached) {
@@ -92,13 +87,7 @@
         throw cached
       }
       try {
-        let pattern = p
-        if (simulation) {
-          for (const [from, to] of replacements) {
-            pattern = pattern.replaceAll(from, to)
-          }
-        }
-        const regex = regexConstructor(pattern)
+        const regex = regexConstructor(p)
         cache?.set(p, regex)
         return regex
       }
@@ -143,25 +132,18 @@
       if (!regexp)
         continue
       try {
-        let offset = 0
         regexp.lastIndex = startPosition
-        let match = regexp.exec(str)
+        const match = regexp.exec(str)
 
-        // If a regex starts with `(^|\\G)` or `(\\G|^)`, we simulate the behavior by cutting the string
-        if (!match && this.contiguousAnchorSimulation[i]) {
-          offset = startPosition
-          regexp.lastIndex = 0
-          match = regexp.exec(str.slice(startPosition))
-        }
         if (!match)
           continue
 
         // If the match is at the start position, return it immediately
         if (match.index === startPosition) {
-          return toResult(i, match, offset)
+          return toResult(i, match, 0)
         }
         // Otherwise, store it for later
-        pending.push([i, match, offset])
+        pending.push([i, match, 0])
       }
       catch (e) {
         if (this.options.forgiving)
@@ -187,9 +169,10 @@
 /**
  * Use the modern JavaScript RegExp engine to implement the OnigScanner.
  *
- * As Oniguruma regex is more powerful than JavaScript regex, some patterns may not be supported.
- * Errors will be thrown when parsing TextMate grammars with unsupported patterns.
- * Set `forgiving` to `true` to ignore these errors and skip the unsupported patterns.
+ * As Oniguruma supports some features that can't be emulated using native JavaScript regexes, some
+ * patterns are not supported. Errors will be thrown when parsing TextMate grammars with
+ * unsupported patterns, and when the grammar includes patterns that use invalid Oniguruma syntax.
+ * Set `forgiving` to `true` to ignore these errors and skip any unsupported or invalid patterns.
  *
  * @experimental
  */