From d7f66fdf8f2da61aa4b541c48305b801ef51555b Mon Sep 17 00:00:00 2001
From: Anthony Fu <anthonyfu117@hotmail.com>
Date: Fri, 11 Aug 2023 12:21:27 +0200
Subject: [PATCH] refactor!: flat lang registration

---
 package.json           | 12 ++++++++++++
 rollup.config.mjs      | 22 +++++++++++-----------
 scripts/prepare.ts     | 19 +++++++++++--------
 src/core.ts            |  8 ++++----
 src/index.ts           | 20 ++++++++++----------
 src/languages.ts       |  1 +
 src/oniguruma/index.ts | 12 ++++++------
 src/registry.ts        |  8 ++++----
 src/renderer.ts        |  3 ---
 src/resolver.ts        | 14 ++------------
 src/themes.ts          |  1 +
 src/types.ts           | 17 +++++++++--------
 src/vendor/langs.ts    |  2 +-
 src/vendor/themes.ts   |  2 +-
 test/cf.ts             |  4 ++--
 test/core.test.ts      |  4 ++--
 16 files changed, 77 insertions(+), 72 deletions(-)
 create mode 100644 src/languages.ts
 create mode 100644 src/themes.ts

diff --git a/package.json b/package.json
index 4530689f2..bbc2f8211 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,18 @@
       "types": "./dist/core.d.mts",
       "import": "./dist/core.mjs"
     },
+    "./wasm": {
+      "types": "./dist/wasm.d.mts",
+      "import": "./dist/wasm.mjs"
+    },
+    "./languages": {
+      "types": "./dist/languages.d.mts",
+      "import": "./dist/languages.mjs"
+    },
+    "./themes": {
+      "types": "./dist/themes.d.mts",
+      "import": "./dist/themes.mjs"
+    },
     "./dist/*": "./dist/*",
     "./*": "./dist/*"
   },
diff --git a/rollup.config.mjs b/rollup.config.mjs
index 73d1f82e2..606f1f7e1 100644
--- a/rollup.config.mjs
+++ b/rollup.config.mjs
@@ -11,6 +11,15 @@ import fs from 'fs-extra'
 import fg from 'fast-glob'
 import { wasmPlugin } from './build/wasm.mjs'
 
+const entries = [
+  'src/index.ts',
+  'src/core.ts',
+  'src/types.ts',
+  'src/themes.ts',
+  'src/languages.ts',
+  'src/wasm.ts',
+]
+
 const plugins = [
   esbuild(),
   nodeResolve(),
@@ -25,11 +34,7 @@ const plugins = [
 
 export default defineConfig([
   {
-    input: [
-      'src/index.ts',
-      'src/core.ts',
-      'src/wasm.ts',
-    ],
+    input: entries,
     output: {
       dir: 'dist',
       format: 'esm',
@@ -49,12 +54,7 @@ export default defineConfig([
     ],
   },
   {
-    input: [
-      'src/core.ts',
-      'src/index.ts',
-      'src/wasm.ts',
-      'src/types.ts',
-    ],
+    input: entries,
     output: {
       dir: 'dist',
       format: 'esm',
diff --git a/scripts/prepare.ts b/scripts/prepare.ts
index 0c603421b..9b0dad6ec 100644
--- a/scripts/prepare.ts
+++ b/scripts/prepare.ts
@@ -20,12 +20,15 @@ for (const file of files) {
   }
 
   await fs.writeJSON(`./src/vendor/languages/${lang.id}.json`, {
-    ...lang,
-    samplePath: undefined,
-    path: undefined,
-    displayName: undefined,
-    grammar: content,
-  })
+    ...content,
+    name: content.name || lang.id,
+    scopeName: content.scopeName || lang.scopeName,
+    displayName: lang.displayName,
+    aliases: lang.aliases,
+    embeddedLangs: lang.embeddedLangs,
+    balancedBracketSelectors: lang.balancedBracketSelectors,
+    unbalancedBracketSelectors: lang.unbalancedBracketSelectors,
+  }, { spaces: 2 })
 }
 
 const languages = Object.fromEntries(BUNDLED_LANGUAGES.map(i => [i.id, `__()=>import('./languages/${i.id}.json').then(r=>r.default as unknown as LanguageRegistration)__`]))
@@ -36,7 +39,7 @@ await fs.writeFile(
   'src/vendor/langs.ts',
   `import type { LanguageRegistration } from '../types'
 
-export const languages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
+export const bundledLanguages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
   'utf-8',
 )
 
@@ -44,6 +47,6 @@ await fs.writeFile(
   'src/vendor/themes.ts',
   `import type { ThemeRegisterationRaw } from '../types'
 
-export const themes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
+export const bundledThemes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
   'utf-8',
 )
diff --git a/src/core.ts b/src/core.ts
index 17a6c631d..b4f2000e9 100644
--- a/src/core.ts
+++ b/src/core.ts
@@ -1,4 +1,4 @@
-import type { IOptions } from 'vscode-oniguruma'
+import type { OnigurumaLoadOptions } from './oniguruma'
 import { createOnigScanner, createOnigString, loadWasm } from './oniguruma'
 import { Registry } from './registry'
 import type { CodeToHtmlOptions, LanguageInput, ThemeInput } from './types'
@@ -9,17 +9,17 @@ import { toShikiTheme } from './normalize'
 
 export * from './types'
 
-export interface CoreHighlighterOptions {
+export interface HighlighterCoreOptions {
   themes: ThemeInput[]
   langs: LanguageInput[]
-  loadWasm?: IOptions | (() => Promise<IOptions>)
+  loadWasm?: OnigurumaLoadOptions | (() => Promise<OnigurumaLoadOptions>)
 }
 
 export {
   loadWasm,
 }
 
-export async function getHighlighter(options: CoreHighlighterOptions) {
+export async function getHighlighterCore(options: HighlighterCoreOptions) {
   const [
     themes,
     langs,
diff --git a/src/index.ts b/src/index.ts
index 0fa0c9a57..18d9620ef 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,15 +1,15 @@
 import type { BuiltinLanguages, BuiltinThemes, LanguageInput, ThemeInput } from './types'
-import { themes } from './vendor/themes'
-import { languages } from './vendor/langs'
-import { getHighlighter as getCoreHighlighter } from './core'
-import { loadWasm } from './oniguruma'
+import { bundledThemes } from './vendor/themes'
+import { bundledLanguages } from './vendor/langs'
+import { getHighlighterCore } from './core'
 import { getWasmInlined } from './wasm'
 
-export * from './types'
+export * from './core'
 
 export {
-  loadWasm,
   getWasmInlined,
+  bundledLanguages,
+  bundledThemes,
 }
 
 export interface HighlighterOptions {
@@ -20,18 +20,18 @@ export interface HighlighterOptions {
 export async function getHighlighter(options: HighlighterOptions = {}) {
   const _themes = (options.themes ?? ['nord']).map((i) => {
     if (typeof i === 'string')
-      return themes[i]
+      return bundledThemes[i]
     return i
   }) as ThemeInput[]
 
-  const langs = (options.langs ?? Object.keys(languages) as BuiltinLanguages[])
+  const langs = (options.langs ?? Object.keys(bundledLanguages) as BuiltinLanguages[])
     .map((i) => {
       if (typeof i === 'string')
-        return languages[i]
+        return bundledLanguages[i]
       return i
     })
 
-  return getCoreHighlighter({
+  return getHighlighterCore({
     ...options,
     themes: _themes,
     langs,
diff --git a/src/languages.ts b/src/languages.ts
new file mode 100644
index 000000000..0ca8f85b7
--- /dev/null
+++ b/src/languages.ts
@@ -0,0 +1 @@
+export * from './vendor/langs'
diff --git a/src/oniguruma/index.ts b/src/oniguruma/index.ts
index abb42df5e..02d75f437 100644
--- a/src/oniguruma/index.ts
+++ b/src/oniguruma/index.ts
@@ -361,7 +361,7 @@ interface IInstantiatorOptions extends ICommonOptions {
 interface IDataOptions extends ICommonOptions {
   data: ArrayBufferView | ArrayBuffer | Response
 }
-export type IOptions = IInstantiatorOptions | IDataOptions
+export type OnigurumaLoadOptions = IInstantiatorOptions | IDataOptions
 
 async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string) => void) | undefined): Promise<void> {
   onigBinding = await OnigasmModuleFactory({
@@ -379,15 +379,15 @@ async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string)
   })
 }
 
-function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IInstantiatorOptions {
+function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IInstantiatorOptions {
   return (typeof (<IInstantiatorOptions>dataOrOptions).instantiator === 'function')
 }
 
-function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IDataOptions {
+function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IDataOptions {
   return (typeof (<IDataOptions>dataOrOptions).data !== 'undefined')
 }
 
-function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is Response {
+function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is Response {
   return (typeof Response !== 'undefined' && dataOrOptions instanceof Response)
 }
 
@@ -395,9 +395,9 @@ let initCalled = false
 let initPromise: Promise<void> | null = null
 
 export function loadWasm(loader: WebAssemblyInstantiator): Promise<void>
-export function loadWasm(options: IOptions): Promise<void>
+export function loadWasm(options: OnigurumaLoadOptions): Promise<void>
 export function loadWasm(data: ArrayBufferView | ArrayBuffer | Response): Promise<void>
-export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | IOptions): Promise<void> {
+export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): Promise<void> {
   if (initCalled) {
     // Already initialized
     return initPromise!
diff --git a/src/registry.ts b/src/registry.ts
index e06a9face..f15d32fbe 100644
--- a/src/registry.ts
+++ b/src/registry.ts
@@ -22,7 +22,7 @@ export class Registry extends TextMateRegistry {
       _themes.map(i => [i.name, i]),
     )
     this._langMap = Object.fromEntries(
-      _langs.map(i => [i.id, i]),
+      _langs.map(i => [i.name, i]),
     )
   }
 
@@ -57,7 +57,7 @@ export class Registry extends TextMateRegistry {
     }
 
     const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig)
-    this._resolvedGrammars[lang.id] = g!
+    this._resolvedGrammars[lang.name] = g!
     if (lang.aliases) {
       lang.aliases.forEach((la) => {
         this._resolvedGrammars[la] = g!
@@ -83,8 +83,8 @@ export class Registry extends TextMateRegistry {
   }
 
   private resolveEmbeddedLanguages(lang: LanguageRegistration) {
-    if (!this._langGraph.has(lang.id))
-      this._langGraph.set(lang.id, lang)
+    if (!this._langGraph.has(lang.name))
+      this._langGraph.set(lang.name, lang)
 
     if (lang.embeddedLangs) {
       for (const embeddedLang of lang.embeddedLangs)
diff --git a/src/renderer.ts b/src/renderer.ts
index 0e479910b..9483fa2f3 100644
--- a/src/renderer.ts
+++ b/src/renderer.ts
@@ -6,15 +6,12 @@ const defaultElements: ElementsOptions = {
   pre({ className, style, children }) {
     return `<pre class="${className}" style="${style}" tabindex="0">${children}</pre>`
   },
-
   code({ children }) {
     return `<code>${children}</code>`
   },
-
   line({ className, children }) {
     return `<span class="${className}">${children}</span>`
   },
-
   token({ style, children }) {
     return `<span style="${style}">${children}</span>`
   },
diff --git a/src/resolver.ts b/src/resolver.ts
index 35ee136b5..b20bb066b 100644
--- a/src/resolver.ts
+++ b/src/resolver.ts
@@ -3,8 +3,6 @@ import type { IOnigLib, RegistryOptions } from 'vscode-textmate'
 import type { LanguageRegistration } from './types'
 
 export class Resolver implements RegistryOptions {
-  public languagesPath: string = 'languages/'
-
   private readonly languageMap: { [langIdOrAlias: string]: LanguageRegistration } = {}
   private readonly scopeToLangMap: { [scope: string]: LanguageRegistration } = {}
 
@@ -31,19 +29,11 @@ export class Resolver implements RegistryOptions {
   }
 
   public async loadGrammar(scopeName: string): Promise<any> {
-    const lang = this.scopeToLangMap[scopeName]
-
-    if (!lang)
-      return null
-
-    if (lang.grammar)
-      return lang.grammar
-
-    return null
+    return this.scopeToLangMap[scopeName]
   }
 
   public addLanguage(l: LanguageRegistration) {
-    this.languageMap[l.id] = l
+    this.languageMap[l.name] = l
     if (l.aliases) {
       l.aliases.forEach((a) => {
         this.languageMap[a] = l
diff --git a/src/themes.ts b/src/themes.ts
new file mode 100644
index 000000000..c997ae670
--- /dev/null
+++ b/src/themes.ts
@@ -0,0 +1 @@
+export * from './vendor/themes'
diff --git a/src/types.ts b/src/types.ts
index e517ce67c..4f2aa2ff3 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -1,10 +1,10 @@
-import type { IGrammar, IRawTheme } from 'vscode-textmate'
-import type { languages } from './vendor/langs'
-import type { themes } from './vendor/themes'
+import type { IRawGrammar, IRawTheme } from 'vscode-textmate'
+import type { bundledLanguages } from './vendor/langs'
+import type { bundledThemes } from './vendor/themes'
 import type { IThemedToken } from './themedTokenizer'
 
-export type BuiltinLanguages = keyof typeof languages
-export type BuiltinThemes = keyof typeof themes
+export type BuiltinLanguages = keyof typeof bundledLanguages
+export type BuiltinThemes = keyof typeof bundledThemes
 
 export type Awaitable<T> = T | Promise<T>
 export type MaybeGetter<T> = T | (() => Awaitable<T>)
@@ -12,8 +12,8 @@ export type MaybeGetter<T> = T | (() => Awaitable<T>)
 export type ThemeInput = MaybeGetter<ThemeRegisteration | ThemeRegisterationRaw>
 export type LanguageInput = MaybeGetter<LanguageRegistration>
 
-export interface LanguageRegistration {
-  id: string
+export interface LanguageRegistration extends IRawGrammar {
+  name: string
   scopeName: string
   displayName?: string
   aliases?: string[]
@@ -25,7 +25,6 @@ export interface LanguageRegistration {
   embeddedLangs?: string[]
   balancedBracketSelectors?: string[]
   unbalancedBracketSelectors?: string[]
-  grammar?: IGrammar
 }
 
 export interface CodeToHtmlOptions {
@@ -125,3 +124,5 @@ interface TokenElementProps extends ElementProps {
   token: IThemedToken
   index: number
 }
+
+export {}
diff --git a/src/vendor/langs.ts b/src/vendor/langs.ts
index 30cb97ae7..125e52ccb 100644
--- a/src/vendor/langs.ts
+++ b/src/vendor/langs.ts
@@ -1,6 +1,6 @@
 import type { LanguageRegistration } from '../types'
 
-export const languages = {
+export const bundledLanguages = {
   'abap': () => import('./languages/abap.json').then(r => r.default as unknown as LanguageRegistration),
   'actionscript-3': () => import('./languages/actionscript-3.json').then(r => r.default as unknown as LanguageRegistration),
   'ada': () => import('./languages/ada.json').then(r => r.default as unknown as LanguageRegistration),
diff --git a/src/vendor/themes.ts b/src/vendor/themes.ts
index 988545d50..ed8d026fb 100644
--- a/src/vendor/themes.ts
+++ b/src/vendor/themes.ts
@@ -1,6 +1,6 @@
 import type { ThemeRegisterationRaw } from '../types'
 
-export const themes = {
+export const bundledThemes = {
   'css-variables': () => import('shiki/themes/css-variables.json').then(r => r.default as unknown as ThemeRegisterationRaw),
   'dark-plus': () => import('shiki/themes/dark-plus.json').then(r => r.default as unknown as ThemeRegisterationRaw),
   'dracula-soft': () => import('shiki/themes/dracula-soft.json').then(r => r.default as unknown as ThemeRegisterationRaw),
diff --git a/test/cf.ts b/test/cf.ts
index 647f5f4d8..f5af9f5cb 100644
--- a/test/cf.ts
+++ b/test/cf.ts
@@ -1,4 +1,4 @@
-import { getHighlighter, loadWasm } from '../src/core'
+import { getHighlighterCore, loadWasm } from '../src/core'
 
 import nord from '../dist/themes/nord.mjs'
 import js from '../dist/languages/javascript.mjs'
@@ -10,7 +10,7 @@ await loadWasm(obj => WebAssembly.instantiate(wasm, obj))
 
 export default {
   async fetch() {
-    const highlighter = await getHighlighter({
+    const highlighter = await getHighlighterCore({
       themes: [nord],
       langs: [js],
     })
diff --git a/test/core.test.ts b/test/core.test.ts
index 4178e8877..1c4b2ac13 100644
--- a/test/core.test.ts
+++ b/test/core.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { getHighlighter } from '../src/core'
+import { getHighlighterCore } from '../src/core'
 
 import js from '../dist/languages/javascript.mjs'
 import nord from '../dist/themes/nord.mjs'
@@ -9,7 +9,7 @@ import onig from '../dist/onig.mjs'
 
 describe('should', () => {
   it('exported', async () => {
-    const shiki = await getHighlighter({
+    const shiki = await getHighlighterCore({
       themes: [nord],
       langs: [js],
       loadWasm: {