From ac1167a9a646f2bb74f2b32e77c0f6eaeabd2773 Mon Sep 17 00:00:00 2001 From: hddhyq Date: Wed, 17 Apr 2024 17:01:06 +0800 Subject: [PATCH 1/5] fix(market): Update Monaco syntax highlighting configuration --- docs/packages/monaco.md | 8 ++++++-- packages/monaco/playground/src/main.ts | 8 ++++++-- packages/monaco/src/index.ts | 12 ++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/packages/monaco.md b/docs/packages/monaco.md index 2641cd275..252612d5b 100644 --- a/docs/packages/monaco.md +++ b/docs/packages/monaco.md @@ -37,8 +37,12 @@ monaco.languages.register({ id: 'vue' }) monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) -// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:2] -shikiToMonaco(highlighter, monaco) +// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:6] +// Custom token parsing configuration can be defined. +shikiToMonaco(highlighter, monaco, { + tokenizeMaxLineLength: 20000, + tokenizeTimeLimit: 500 +}) // Create the editor const editor = monaco.editor.create(document.getElementById('container'), { diff --git a/packages/monaco/playground/src/main.ts b/packages/monaco/playground/src/main.ts index 532f3dcb7..2afc9c236 100644 --- a/packages/monaco/playground/src/main.ts +++ b/packages/monaco/playground/src/main.ts @@ -19,8 +19,12 @@ monaco.languages.register({ id: 'vue' }) monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) -// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:2] -shikiToMonaco(highlighter, monaco) +// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:6] +// Custom token parsing configuration can be defined. +shikiToMonaco(highlighter, monaco, { + tokenizeMaxLineLength: 20000, + tokenizeTimeLimit: 500, +}) // Create the editor monaco.editor.create(document.getElementById('app')!, { diff --git a/packages/monaco/src/index.ts b/packages/monaco/src/index.ts index b9b98c24e..4bc0c8e89 100644 --- a/packages/monaco/src/index.ts +++ b/packages/monaco/src/index.ts @@ -5,6 +5,11 @@ import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate' export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {} +export interface ShikiToMonacoOptions { + tokenizeMaxLineLength?: number + tokenizeTimeLimit?: number +} + export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): MonacoTheme { let rules = 'rules' in theme ? theme.rules as MonacoTheme['rules'] @@ -42,6 +47,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo export function shikiToMonaco( highlighter: ShikiInternal, monaco: typeof monacoNs, + options?: ShikiToMonacoOptions, ) { // Convert themes to Monaco themes and register them const themeMap = new Map() @@ -92,8 +98,10 @@ export function shikiToMonaco( tokenize(line, state: TokenizerState) { // Do not attempt to tokenize if a line is too long // default to 20000 (as in monaco-editor-core defaults) - const tokenizeMaxLineLength = 20000 - const tokenizeTimeLimit = 500 + const { + tokenizeMaxLineLength = 20000, + tokenizeTimeLimit = 500, + } = options || {} if (line.length >= tokenizeMaxLineLength) { return { From 9a5ea59f0b44a249bb3a0978f7586d68bd193d51 Mon Sep 17 00:00:00 2001 From: hddhyq Date: Wed, 17 Apr 2024 22:13:20 +0800 Subject: [PATCH 2/5] docs: Update Monaco syntax highlighting configuration --- docs/packages/monaco.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/packages/monaco.md b/docs/packages/monaco.md index 252612d5b..2641cd275 100644 --- a/docs/packages/monaco.md +++ b/docs/packages/monaco.md @@ -37,12 +37,8 @@ monaco.languages.register({ id: 'vue' }) monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) -// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:6] -// Custom token parsing configuration can be defined. -shikiToMonaco(highlighter, monaco, { - tokenizeMaxLineLength: 20000, - tokenizeTimeLimit: 500 -}) +// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:2] +shikiToMonaco(highlighter, monaco) // Create the editor const editor = monaco.editor.create(document.getElementById('container'), { From 52e6284f8edde91ac55feca0e0e27e661a5676c7 Mon Sep 17 00:00:00 2001 From: hddhyq Date: Wed, 17 Apr 2024 22:25:26 +0800 Subject: [PATCH 3/5] chore: remove useless code --- packages/monaco/playground/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/monaco/playground/src/main.ts b/packages/monaco/playground/src/main.ts index 2afc9c236..2ccdecec0 100644 --- a/packages/monaco/playground/src/main.ts +++ b/packages/monaco/playground/src/main.ts @@ -19,7 +19,7 @@ monaco.languages.register({ id: 'vue' }) monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) -// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:6] +// Register the themes from Shiki, and provide syntax highlighting for Monaco. // Custom token parsing configuration can be defined. shikiToMonaco(highlighter, monaco, { tokenizeMaxLineLength: 20000, From 0d491bcc47cf2331f66988edf19996bea4430eec Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Tue, 30 Apr 2024 11:26:17 +0200 Subject: [PATCH 4/5] chore: update deps --- packages/monaco/playground/src/main.ts | 6 +----- packages/monaco/src/index.ts | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/monaco/playground/src/main.ts b/packages/monaco/playground/src/main.ts index 2ccdecec0..80b26d6e6 100644 --- a/packages/monaco/playground/src/main.ts +++ b/packages/monaco/playground/src/main.ts @@ -20,11 +20,7 @@ monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) // Register the themes from Shiki, and provide syntax highlighting for Monaco. -// Custom token parsing configuration can be defined. -shikiToMonaco(highlighter, monaco, { - tokenizeMaxLineLength: 20000, - tokenizeTimeLimit: 500, -}) +shikiToMonaco(highlighter, monaco) // Create the editor monaco.editor.create(document.getElementById('app')!, { diff --git a/packages/monaco/src/index.ts b/packages/monaco/src/index.ts index 4bc0c8e89..40aa190de 100644 --- a/packages/monaco/src/index.ts +++ b/packages/monaco/src/index.ts @@ -6,7 +6,17 @@ import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate' export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {} export interface ShikiToMonacoOptions { + /** + * The maximum length of a line to tokenize. + * + * @default 20000 + */ tokenizeMaxLineLength?: number + /** + * The time limit in milliseconds for tokenizing a line. + * + * @default 500 + */ tokenizeTimeLimit?: number } @@ -47,7 +57,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo export function shikiToMonaco( highlighter: ShikiInternal, monaco: typeof monacoNs, - options?: ShikiToMonacoOptions, + options: ShikiToMonacoOptions = {}, ) { // Convert themes to Monaco themes and register them const themeMap = new Map() @@ -88,6 +98,13 @@ export function shikiToMonaco( return colorToScopeMap.get(color) } + // Do not attempt to tokenize if a line is too long + // default to 20000 (as in monaco-editor-core defaults) + const { + tokenizeMaxLineLength = 20000, + tokenizeTimeLimit = 500, + } = options + const monacoLanguageIds = new Set(monaco.languages.getLanguages().map(l => l.id)) for (const lang of highlighter.getLoadedLanguages()) { if (monacoLanguageIds.has(lang)) { @@ -96,13 +113,6 @@ export function shikiToMonaco( return new TokenizerState(INITIAL) }, tokenize(line, state: TokenizerState) { - // Do not attempt to tokenize if a line is too long - // default to 20000 (as in monaco-editor-core defaults) - const { - tokenizeMaxLineLength = 20000, - tokenizeTimeLimit = 500, - } = options || {} - if (line.length >= tokenizeMaxLineLength) { return { endState: state, From 2b91aa9fb44626ce39dc03ac266bf86548ac9d20 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Tue, 30 Apr 2024 11:26:47 +0200 Subject: [PATCH 5/5] Discard changes to packages/monaco/playground/src/main.ts --- packages/monaco/playground/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/monaco/playground/src/main.ts b/packages/monaco/playground/src/main.ts index 80b26d6e6..532f3dcb7 100644 --- a/packages/monaco/playground/src/main.ts +++ b/packages/monaco/playground/src/main.ts @@ -19,7 +19,7 @@ monaco.languages.register({ id: 'vue' }) monaco.languages.register({ id: 'typescript' }) monaco.languages.register({ id: 'javascript' }) -// Register the themes from Shiki, and provide syntax highlighting for Monaco. +// Register the themes from Shiki, and provide syntax highlighting for Monaco. // [!code highlight:2] shikiToMonaco(highlighter, monaco) // Create the editor