From a606d4491f47a64947540657be479110bdb3c601 Mon Sep 17 00:00:00 2001
From: brokenbones <brokenbonesdd@gmail.com>
Date: Tue, 30 Apr 2024 17:27:18 +0800
Subject: [PATCH] fix(monaco): options for tokenize limit (#657)

Co-authored-by: Anthony Fu <anthonyfu117@hotmail.com>
Co-authored-by: Anthony Fu <github@antfu.me>
---
 packages/monaco/src/index.ts | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/packages/monaco/src/index.ts b/packages/monaco/src/index.ts
index b9b98c24e..40aa190de 100644
--- a/packages/monaco/src/index.ts
+++ b/packages/monaco/src/index.ts
@@ -5,6 +5,21 @@ import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate'
 
 export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {}
 
+export interface ShikiToMonacoOptions {
+  /**
+   * The maximum length of a line to tokenize.
+   *
+   * @default 20000
+   */
+  tokenizeMaxLineLength?: number
+  /**
+   * The time limit in milliseconds for tokenizing a line.
+   *
+   * @default 500
+   */
+  tokenizeTimeLimit?: number
+}
+
 export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): MonacoTheme {
   let rules = 'rules' in theme
     ? theme.rules as MonacoTheme['rules']
@@ -42,6 +57,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo
 export function shikiToMonaco(
   highlighter: ShikiInternal<any, any>,
   monaco: typeof monacoNs,
+  options: ShikiToMonacoOptions = {},
 ) {
   // Convert themes to Monaco themes and register them
   const themeMap = new Map<string, MonacoTheme>()
@@ -82,6 +98,13 @@ export function shikiToMonaco(
     return colorToScopeMap.get(color)
   }
 
+  // Do not attempt to tokenize if a line is too long
+  // default to 20000 (as in monaco-editor-core defaults)
+  const {
+    tokenizeMaxLineLength = 20000,
+    tokenizeTimeLimit = 500,
+  } = options
+
   const monacoLanguageIds = new Set(monaco.languages.getLanguages().map(l => l.id))
   for (const lang of highlighter.getLoadedLanguages()) {
     if (monacoLanguageIds.has(lang)) {
@@ -90,11 +113,6 @@ export function shikiToMonaco(
           return new TokenizerState(INITIAL)
         },
         tokenize(line, state: TokenizerState) {
-          // Do not attempt to tokenize if a line is too long
-          // default to 20000 (as in monaco-editor-core defaults)
-          const tokenizeMaxLineLength = 20000
-          const tokenizeTimeLimit = 500
-
           if (line.length >= tokenizeMaxLineLength) {
             return {
               endState: state,