refactor!: flat lang registration

package.json

@@ -25,6 +25,18 @@
       "types": "./dist/core.d.mts",
       "import": "./dist/core.mjs"
     },
+    "./wasm": {
+      "types": "./dist/wasm.d.mts",
+      "import": "./dist/wasm.mjs"
+    },
+    "./languages": {
+      "types": "./dist/languages.d.mts",
+      "import": "./dist/languages.mjs"
+    },
+    "./themes": {
+      "types": "./dist/themes.d.mts",
+      "import": "./dist/themes.mjs"
+    },
     "./dist/*": "./dist/*",
     "./*": "./dist/*"
   },

rollup.config.mjs

@@ -11,6 +11,15 @@ import fs from 'fs-extra'
 import fg from 'fast-glob'
 import { wasmPlugin } from './build/wasm.mjs'
 
+const entries = [
+  'src/index.ts',
+  'src/core.ts',
+  'src/types.ts',
+  'src/themes.ts',
+  'src/languages.ts',
+  'src/wasm.ts',
+]
+
 const plugins = [
   esbuild(),
   nodeResolve(),
@@ -25,11 +34,7 @@ const plugins = [
 
 export default defineConfig([
   {
-    input: [
-      'src/index.ts',
-      'src/core.ts',
-      'src/wasm.ts',
-    ],
+    input: entries,
     output: {
       dir: 'dist',
       format: 'esm',
@@ -49,12 +54,7 @@ export default defineConfig([
     ],
   },
   {
-    input: [
-      'src/core.ts',
-      'src/index.ts',
-      'src/wasm.ts',
-      'src/types.ts',
-    ],
+    input: entries,
     output: {
       dir: 'dist',
       format: 'esm',

scripts/prepare.ts

@@ -20,12 +20,15 @@ for (const file of files) {
   }
 
   await fs.writeJSON(`./src/vendor/languages/${lang.id}.json`, {
-    ...lang,
-    samplePath: undefined,
-    path: undefined,
-    displayName: undefined,
-    grammar: content,
-  })
+    ...content,
+    name: content.name || lang.id,
+    scopeName: content.scopeName || lang.scopeName,
+    displayName: lang.displayName,
+    aliases: lang.aliases,
+    embeddedLangs: lang.embeddedLangs,
+    balancedBracketSelectors: lang.balancedBracketSelectors,
+    unbalancedBracketSelectors: lang.unbalancedBracketSelectors,
+  }, { spaces: 2 })
 }
 
 const languages = Object.fromEntries(BUNDLED_LANGUAGES.map(i => [i.id, `__()=>import('./languages/${i.id}.json').then(r=>r.default as unknown as LanguageRegistration)__`]))
@@ -36,14 +39,14 @@ await fs.writeFile(
   'src/vendor/langs.ts',
   `import type { LanguageRegistration } from '../types'
 
-export const languages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
+export const bundledLanguages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
   'utf-8',
 )
 
 await fs.writeFile(
   'src/vendor/themes.ts',
   `import type { ThemeRegisterationRaw } from '../types'
 
-export const themes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
+export const bundledThemes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
   'utf-8',
 )

src/core.ts

@@ -1,4 +1,4 @@
-import type { IOptions } from 'vscode-oniguruma'
+import type { OnigurumaLoadOptions } from './oniguruma'
 import { createOnigScanner, createOnigString, loadWasm } from './oniguruma'
 import { Registry } from './registry'
 import type { CodeToHtmlOptions, LanguageInput, ThemeInput } from './types'
@@ -9,17 +9,17 @@ import { toShikiTheme } from './normalize'
 
 export * from './types'
 
-export interface CoreHighlighterOptions {
+export interface HighlighterCoreOptions {
   themes: ThemeInput[]
   langs: LanguageInput[]
-  loadWasm?: IOptions | (() => Promise<IOptions>)
+  loadWasm?: OnigurumaLoadOptions | (() => Promise<OnigurumaLoadOptions>)
 }
 
 export {
   loadWasm,
 }
 
-export async function getHighlighter(options: CoreHighlighterOptions) {
+export async function getHighlighterCore(options: HighlighterCoreOptions) {
   const [
     themes,
     langs,

src/index.ts

@@ -1,15 +1,15 @@
 import type { BuiltinLanguages, BuiltinThemes, LanguageInput, ThemeInput } from './types'
-import { themes } from './vendor/themes'
-import { languages } from './vendor/langs'
-import { getHighlighter as getCoreHighlighter } from './core'
-import { loadWasm } from './oniguruma'
+import { bundledThemes } from './vendor/themes'
+import { bundledLanguages } from './vendor/langs'
+import { getHighlighterCore } from './core'
 import { getWasmInlined } from './wasm'
 
-export * from './types'
+export * from './core'
 
 export {
-  loadWasm,
   getWasmInlined,
+  bundledLanguages,
+  bundledThemes,
 }
 
 export interface HighlighterOptions {
@@ -20,18 +20,18 @@ export interface HighlighterOptions {
 export async function getHighlighter(options: HighlighterOptions = {}) {
   const _themes = (options.themes ?? ['nord']).map((i) => {
     if (typeof i === 'string')
-      return themes[i]
+      return bundledThemes[i]
     return i
   }) as ThemeInput[]
 
-  const langs = (options.langs ?? Object.keys(languages) as BuiltinLanguages[])
+  const langs = (options.langs ?? Object.keys(bundledLanguages) as BuiltinLanguages[])
     .map((i) => {
       if (typeof i === 'string')
-        return languages[i]
+        return bundledLanguages[i]
       return i
     })
 
-  return getCoreHighlighter({
+  return getHighlighterCore({
     ...options,
     themes: _themes,
     langs,

src/languages.ts

@@ -0,0 +1 @@
+export * from './vendor/langs'

src/oniguruma/index.ts

@@ -361,7 +361,7 @@ interface IInstantiatorOptions extends ICommonOptions {
 interface IDataOptions extends ICommonOptions {
   data: ArrayBufferView | ArrayBuffer | Response
 }
-export type IOptions = IInstantiatorOptions | IDataOptions
+export type OnigurumaLoadOptions = IInstantiatorOptions | IDataOptions
 
 async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string) => void) | undefined): Promise<void> {
   onigBinding = await OnigasmModuleFactory({
@@ -379,25 +379,25 @@ async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string)
   })
 }
 
-function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IInstantiatorOptions {
+function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IInstantiatorOptions {
   return (typeof (<IInstantiatorOptions>dataOrOptions).instantiator === 'function')
 }
 
-function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IDataOptions {
+function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IDataOptions {
   return (typeof (<IDataOptions>dataOrOptions).data !== 'undefined')
 }
 
-function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is Response {
+function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is Response {
   return (typeof Response !== 'undefined' && dataOrOptions instanceof Response)
 }
 
 let initCalled = false
 let initPromise: Promise<void> | null = null
 
 export function loadWasm(loader: WebAssemblyInstantiator): Promise<void>
-export function loadWasm(options: IOptions): Promise<void>
+export function loadWasm(options: OnigurumaLoadOptions): Promise<void>
 export function loadWasm(data: ArrayBufferView | ArrayBuffer | Response): Promise<void>
-export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | IOptions): Promise<void> {
+export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): Promise<void> {
   if (initCalled) {
     // Already initialized
     return initPromise!

src/registry.ts

@@ -22,7 +22,7 @@ export class Registry extends TextMateRegistry {
       _themes.map(i => [i.name, i]),
     )
     this._langMap = Object.fromEntries(
-      _langs.map(i => [i.id, i]),
+      _langs.map(i => [i.name, i]),
     )
   }
 
@@ -57,7 +57,7 @@ export class Registry extends TextMateRegistry {
     }
 
     const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig)
-    this._resolvedGrammars[lang.id] = g!
+    this._resolvedGrammars[lang.name] = g!
     if (lang.aliases) {
       lang.aliases.forEach((la) => {
         this._resolvedGrammars[la] = g!
@@ -83,8 +83,8 @@ export class Registry extends TextMateRegistry {
   }
 
   private resolveEmbeddedLanguages(lang: LanguageRegistration) {
-    if (!this._langGraph.has(lang.id))
-      this._langGraph.set(lang.id, lang)
+    if (!this._langGraph.has(lang.name))
+      this._langGraph.set(lang.name, lang)
 
     if (lang.embeddedLangs) {
       for (const embeddedLang of lang.embeddedLangs)

src/renderer.ts

@@ -6,15 +6,12 @@ const defaultElements: ElementsOptions = {
   pre({ className, style, children }) {
     return `<pre class="${className}" style="${style}" tabindex="0">${children}</pre>`
   },
-
   code({ children }) {
     return `<code>${children}</code>`
   },
-
   line({ className, children }) {
     return `<span class="${className}">${children}</span>`
   },
-
   token({ style, children }) {
     return `<span style="${style}">${children}</span>`
   },

src/resolver.ts

@@ -3,8 +3,6 @@ import type { IOnigLib, RegistryOptions } from 'vscode-textmate'
 import type { LanguageRegistration } from './types'
 
 export class Resolver implements RegistryOptions {
-  public languagesPath: string = 'languages/'
-
   private readonly languageMap: { [langIdOrAlias: string]: LanguageRegistration } = {}
   private readonly scopeToLangMap: { [scope: string]: LanguageRegistration } = {}
 
@@ -31,19 +29,11 @@ export class Resolver implements RegistryOptions {
   }
 
   public async loadGrammar(scopeName: string): Promise<any> {
-    const lang = this.scopeToLangMap[scopeName]
-
-    if (!lang)
-      return null
-
-    if (lang.grammar)
-      return lang.grammar
-
-    return null
+    return this.scopeToLangMap[scopeName]
   }
 
   public addLanguage(l: LanguageRegistration) {
-    this.languageMap[l.id] = l
+    this.languageMap[l.name] = l
     if (l.aliases) {
       l.aliases.forEach((a) => {
         this.languageMap[a] = l

src/themes.ts

@@ -0,0 +1 @@
+export * from './vendor/themes'

src/types.ts

@@ -1,19 +1,19 @@
-import type { IGrammar, IRawTheme } from 'vscode-textmate'
-import type { languages } from './vendor/langs'
-import type { themes } from './vendor/themes'
+import type { IRawGrammar, IRawTheme } from 'vscode-textmate'
+import type { bundledLanguages } from './vendor/langs'
+import type { bundledThemes } from './vendor/themes'
 import type { IThemedToken } from './themedTokenizer'
 
-export type BuiltinLanguages = keyof typeof languages
-export type BuiltinThemes = keyof typeof themes
+export type BuiltinLanguages = keyof typeof bundledLanguages
+export type BuiltinThemes = keyof typeof bundledThemes
 
 export type Awaitable<T> = T | Promise<T>
 export type MaybeGetter<T> = T | (() => Awaitable<T>)
 
 export type ThemeInput = MaybeGetter<ThemeRegisteration | ThemeRegisterationRaw>
 export type LanguageInput = MaybeGetter<LanguageRegistration>
 
-export interface LanguageRegistration {
-  id: string
+export interface LanguageRegistration extends IRawGrammar {
+  name: string
   scopeName: string
   displayName?: string
   aliases?: string[]
@@ -25,7 +25,6 @@ export interface LanguageRegistration {
   embeddedLangs?: string[]
   balancedBracketSelectors?: string[]
   unbalancedBracketSelectors?: string[]
-  grammar?: IGrammar
 }
 
 export interface CodeToHtmlOptions {
@@ -125,3 +124,5 @@ interface TokenElementProps extends ElementProps {
   token: IThemedToken
   index: number
 }
+
+export {}

src/vendor/langs.ts

@@ -1,6 +1,6 @@
 import type { LanguageRegistration } from '../types'
 
-export const languages = {
+export const bundledLanguages = {
   'abap': () => import('./languages/abap.json').then(r => r.default as unknown as LanguageRegistration),
   'actionscript-3': () => import('./languages/actionscript-3.json').then(r => r.default as unknown as LanguageRegistration),
   'ada': () => import('./languages/ada.json').then(r => r.default as unknown as LanguageRegistration),

src/vendor/themes.ts

@@ -1,6 +1,6 @@
 import type { ThemeRegisterationRaw } from '../types'
 
-export const themes = {
+export const bundledThemes = {
   'css-variables': () => import('shiki/themes/css-variables.json').then(r => r.default as unknown as ThemeRegisterationRaw),
   'dark-plus': () => import('shiki/themes/dark-plus.json').then(r => r.default as unknown as ThemeRegisterationRaw),
   'dracula-soft': () => import('shiki/themes/dracula-soft.json').then(r => r.default as unknown as ThemeRegisterationRaw),

test/cf.ts

@@ -1,4 +1,4 @@
-import { getHighlighter, loadWasm } from '../src/core'
+import { getHighlighterCore, loadWasm } from '../src/core'
 
 import nord from '../dist/themes/nord.mjs'
 import js from '../dist/languages/javascript.mjs'
@@ -10,7 +10,7 @@ await loadWasm(obj => WebAssembly.instantiate(wasm, obj))
 
 export default {
   async fetch() {
-    const highlighter = await getHighlighter({
+    const highlighter = await getHighlighterCore({
       themes: [nord],
       langs: [js],
     })

test/core.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { getHighlighter } from '../src/core'
+import { getHighlighterCore } from '../src/core'
 
 import js from '../dist/languages/javascript.mjs'
 import nord from '../dist/themes/nord.mjs'
@@ -9,7 +9,7 @@ import onig from '../dist/onig.mjs'
 
 describe('should', () => {
   it('exported', async () => {
-    const shiki = await getHighlighter({
+    const shiki = await getHighlighterCore({
       themes: [nord],
       langs: [js],
       loadWasm: {