refactor!: refactor internal context

packages/core/src/code-to-hast.ts

@@ -160,7 +160,7 @@ export function tokensToHast(
         tokenNode.properties.style = style
 
       for (const transformer of transformers)
-        tokenNode = (transformer?.span || transformer?.token)?.call(context, tokenNode, idx + 1, col, lineNode) || tokenNode
+        tokenNode = transformer?.span?.call(context, tokenNode, idx + 1, col, lineNode) || tokenNode
 
       lineNode.children.push(tokenNode)
       col += token.content.length

packages/core/src/code-to-tokens-base.ts

@@ -29,7 +29,7 @@ export function codeToTokensBase(
   if (lang === 'ansi')
     return tokenizeAnsiWithTheme(theme, code, options)
 
-  const _grammar = internal.getLangGrammar(lang)
+  const _grammar = internal.getLanguage(lang)
   return tokenizeWithTheme(code, _grammar, theme, colorMap, options)
 }
 

packages/core/src/highlighter.ts

@@ -4,9 +4,7 @@ import { codeToTokens } from './code-to-tokens'
 import { codeToTokensBase } from './code-to-tokens-base'
 import { codeToTokensWithThemes } from './code-to-tokens-themes'
 import { getShikiInternal } from './internal'
-import type { HighlighterCoreOptions, HighlighterGeneric } from './types'
-
-export type HighlighterCore = HighlighterGeneric<never, never>
+import type { HighlighterCore, HighlighterCoreOptions } from './types'
 
 /**
  * Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -23,17 +21,7 @@ export async function getHighlighterCore(options: HighlighterCoreOptions = {}):
     codeToTokens: (code, options) => codeToTokens(internal, code, options),
     codeToHast: (code, options) => codeToHast(internal, code, options),
     codeToHtml: (code, options) => codeToHtml(internal, code, options),
-
-    loadLanguage: internal.loadLanguage,
-    loadTheme: internal.loadTheme,
-
-    getTheme: internal.getTheme,
-    getLangGrammar: internal.getLangGrammar,
-    setTheme: internal.setTheme,
-
-    getLoadedThemes: internal.getLoadedThemes,
-    getLoadedLanguages: internal.getLoadedLanguages,
-
+    ...internal,
     getInternalContext: () => internal,
   }
 }

packages/core/src/internal.ts

@@ -61,7 +61,7 @@ export async function getShikiInternal(options: HighlighterCoreOptions = {}): Pr
 
   let _lastTheme: string | ThemeRegistrationAny
 
-  function getLangGrammar(name: string | LanguageRegistration) {
+  function getLanguage(name: string | LanguageRegistration) {
     const _lang = _registry.getGrammar(typeof name === 'string' ? name : name.name)
     if (!_lang)
       throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`)
@@ -112,22 +112,12 @@ export async function getShikiInternal(options: HighlighterCoreOptions = {}): Pr
     )
   }
 
-  function updateAlias(alias: Record<string, string>) {
-    Object.assign(_registry.alias, alias)
-  }
-
-  function getAlias() {
-    return _registry.alias
-  }
-
   return {
     setTheme,
     getTheme,
-    getLangGrammar,
+    getLanguage,
     getLoadedThemes,
     getLoadedLanguages,
-    getAlias,
-    updateAlias,
     loadLanguage,
     loadTheme,
   }

packages/core/src/types/highlighter.ts

@@ -5,31 +5,59 @@ import type { SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationR
 import type { CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, ThemedToken, ThemedTokenWithVariants, TokensResult } from './tokens'
 import type { CodeToHastOptions } from './options'
 
-export interface ShikiInternal {
-  setTheme: (name: string | ThemeRegistrationAny) => {
+/**
+ * Internal context of Shiki, core textmate logic
+ */
+export interface ShikiInternal<BundledLangKeys extends string = never, BundledThemeKeys extends string = never> {
+  /**
+   * Load a theme to the highlighter, so later it can be used synchronously.
+   */
+  loadTheme: (...themes: (ThemeInput | BundledThemeKeys | SpecialTheme)[]) => Promise<void>
+  /**
+   * Load a language to the highlighter, so later it can be used synchronously.
+   */
+  loadLanguage: (...langs: (LanguageInput | BundledLangKeys | SpecialLanguage)[]) => Promise<void>
+
+  /**
+   * Get the registered theme object
+   */
+  getTheme: (name: string | ThemeRegistrationAny) => ThemeRegistrationResolved
+  /**
+   * Get the registered language object
+   */
+  getLanguage: (name: string | LanguageRegistration) => Grammar
+
+  /**
+   * Set the current theme and get the resolved theme object and color map.
+   * @internal
+   */
+  setTheme: (themeName: string | ThemeRegistrationAny) => {
     theme: ThemeRegistrationResolved
     colorMap: string[]
   }
 
-  getTheme: (name: string | ThemeRegistrationAny) => ThemeRegistrationResolved
-  getLangGrammar: (name: string | LanguageRegistration) => Grammar
-
-  getLoadedThemes: () => string[]
+  /**
+   * Get the names of loaded languages
+   *
+   * Special-handled languages like `text`, `plain` and `ansi` are not included.
+   */
   getLoadedLanguages: () => string[]
-  loadLanguage: (...langs: (LanguageInput | SpecialLanguage)[]) => Promise<void>
-  loadTheme: (...themes: (ThemeInput | SpecialTheme)[]) => Promise<void>
-
-  getAlias: () => Record<string, string>
-  updateAlias: (alias: Record<string, string>) => void
+  /**
+   * Get the names of loaded themes
+   *
+   * Special-handled themes like `none` are not included.
+   */
+  getLoadedThemes: () => string[]
 }
-/**
- * Generic instance interface of Shiki
- */
 
-export interface HighlighterGeneric<BundledLangKeys extends string, BundledThemeKeys extends string> {
 /**
- * Get highlighted code in HTML string
+ * Generic instance interface of Shiki
  */
+export interface HighlighterGeneric<BundledLangKeys extends string, BundledThemeKeys extends string>
+  extends ShikiInternal<BundledLangKeys, BundledThemeKeys> {
+  /**
+   * Get highlighted code in HTML string
+   */
   codeToHtml: (
     code: string,
     options: CodeToHastOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>
@@ -69,47 +97,15 @@ export interface HighlighterGeneric<BundledLangKeys extends string, BundledTheme
     options: CodeToTokensWithThemesOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>
   ) => ThemedTokenWithVariants[][]
 
-  /**
-   * Load a theme to the highlighter, so later it can be used synchronously.
-   */
-  loadTheme: (...themes: (ThemeInput | BundledThemeKeys | SpecialTheme)[]) => Promise<void>
-  /**
-   * Load a language to the highlighter, so later it can be used synchronously.
-   */
-  loadLanguage: (...langs: (LanguageInput | BundledLangKeys | SpecialLanguage)[]) => Promise<void>
-
-  /**
-   * Get the registered theme object
-   */
-  getTheme: (name: string | ThemeRegistrationAny) => ThemeRegistrationResolved
-  /**
-   * Get the registered language object
-   */
-  getLangGrammar: (name: string | LanguageRegistration) => Grammar
-
-  /**
-   * Set the current theme and get the resolved theme object and color map.
-   * @internal
-   */
-  setTheme: ShikiInternal['setTheme']
-
-  /**
-   * Get the names of loaded languages
-   *
-   * Special-handled languages like `text`, `plain` and `ansi` are not included.
-   */
-  getLoadedLanguages: () => string[]
-  /**
-   * Get the names of loaded themes
-   *
-   * Special-handled themes like `none` are not included.
-   */
-  getLoadedThemes: () => string[]
-
   /**
    * Get internal context object
    * @internal
    * @deprecated
    */
   getInternalContext: () => ShikiInternal
 }
+
+/**
+ * The fine-grained core Shiki highlighter instance.
+ */
+export type HighlighterCore = HighlighterGeneric<never, never>

packages/monaco/src/index.ts

@@ -1,24 +1,15 @@
 import type * as monaco from 'monaco-editor-core'
-import type { HighlighterGeneric, ShikiInternal, ThemeRegistrationResolved } from '@shikijs/core'
+import type { ShikiInternal, ThemeRegistrationResolved } from '@shikijs/core'
 import type { StateStack } from '@shikijs/core/textmate'
 import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate'
 
 type Monaco = typeof monaco
-type AcceptableShiki = HighlighterGeneric<any, any> | ShikiInternal
 
 export interface MonacoInferface {
   editor: Monaco['editor']
   languages: Monaco['languages']
 }
 
-export interface HighlighterInterface {
-  getLoadedThemes: AcceptableShiki['getLoadedThemes']
-  getTheme: AcceptableShiki['getTheme']
-  getLoadedLanguages: AcceptableShiki['getLoadedLanguages']
-  getLangGrammar: AcceptableShiki['getLangGrammar']
-  setTheme: AcceptableShiki['setTheme']
-}
-
 export interface MonacoTheme extends monaco.editor.IStandaloneThemeData { }
 
 export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): MonacoTheme {
@@ -51,7 +42,7 @@ export function textmateThemeToMonacoTheme(theme: ThemeRegistrationResolved): Mo
 }
 
 export function shikiToMonaco(
-  highlighter: HighlighterInterface,
+  highlighter: ShikiInternal<any, any>,
   monaco: MonacoInferface,
 ) {
   // Convert themes to Monaco themes and register them
@@ -81,7 +72,7 @@ export function shikiToMonaco(
           return new TokenizerState(INITIAL, highlighter)
         },
         tokenize(line, state: TokenizerState) {
-          const grammar = state.highlighter.getLangGrammar(lang)
+          const grammar = state.highlighter.getLanguage(lang)
           const { colorMap } = state.highlighter.setTheme(currentTheme)
           const theme = themeMap.get(currentTheme)
           const result = grammar.tokenizeLine2(line, state.ruleStack)
@@ -126,7 +117,7 @@ export function shikiToMonaco(
 class TokenizerState implements monaco.languages.IState {
   constructor(
     private _ruleStack: StateStack,
-    public highlighter: HighlighterInterface,
+    public highlighter: ShikiInternal<any, any>,
   ) { }
 
   public get ruleStack(): StateStack {