diff --git a/package.json b/package.json index a4896bd62..39179c02a 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "lint-staged": "^15.2.0", "markdown-it": "^14.0.0", "markdown-it-shikiji": "workspace:*", + "ofetch": "^1.3.3", "pnpm": "^8.12.1", "prettier": "^3.1.1", "rimraf": "^5.0.5", diff --git a/packages/shikiji-core/src/resolver.ts b/packages/shikiji-core/src/resolver.ts index 93c6cd6ce..cd707760c 100644 --- a/packages/shikiji-core/src/resolver.ts +++ b/packages/shikiji-core/src/resolver.ts @@ -2,14 +2,14 @@ import type { IOnigLib, RegistryOptions } from './textmate' import type { LanguageRegistration } from './types' export class Resolver implements RegistryOptions { - private readonly languageMap: { [langIdOrAlias: string]: LanguageRegistration } = {} - private readonly scopeToLangMap: { [scope: string]: LanguageRegistration } = {} + private readonly _langs = new Map() + private readonly _scopeToLang = new Map() + private readonly _injections = new Map() private readonly _onigLibPromise: Promise constructor(onigLibPromise: Promise, langs: LanguageRegistration[]) { this._onigLibPromise = onigLibPromise - langs.forEach(i => this.addLanguage(i)) } @@ -18,20 +18,37 @@ export class Resolver implements RegistryOptions { } public getLangRegistration(langIdOrAlias: string): LanguageRegistration { - return this.languageMap[langIdOrAlias] + return this._langs.get(langIdOrAlias)! } public async loadGrammar(scopeName: string): Promise { - return this.scopeToLangMap[scopeName] + return this._scopeToLang.get(scopeName)! } public addLanguage(l: LanguageRegistration) { - this.languageMap[l.name] = l + this._langs.set(l.name, l) if (l.aliases) { l.aliases.forEach((a) => { - this.languageMap[a] = l + this._langs.set(a, l) + }) + } + this._scopeToLang.set(l.scopeName, l) + if (l.injectTo) { + l.injectTo.forEach((i) => { + if (!this._injections.get(i)) + this._injections.set(i, []) + this._injections.get(i)!.push(l.scopeName) }) } - this.scopeToLangMap[l.scopeName] = l + } + + public getInjections(scopeName: string): string[] | undefined { + const scopeParts = scopeName.split('.') + let injections: string[] = [] + for (let i = 1; i <= scopeParts.length; i++) { + const subScopeName = scopeParts.slice(0, i).join('.') + injections = [...injections, ...(this._injections.get(subScopeName) || [])] + } + return injections } } diff --git a/packages/shikiji-core/src/types.ts b/packages/shikiji-core/src/types.ts index 2dacea620..ace32bcab 100644 --- a/packages/shikiji-core/src/types.ts +++ b/packages/shikiji-core/src/types.ts @@ -197,6 +197,14 @@ export interface LanguageRegistration extends RawGrammar { embeddedLangs?: string[] balancedBracketSelectors?: string[] unbalancedBracketSelectors?: string[] + + /** + * Inject this language to other scopes. + * Same as `injectTo` in VSCode's `contributes.grammars`. + * + * @see https://code.visualstudio.com/api/language-extensions/syntax-highlight-guide#injection-grammars + */ + injectTo?: string[] } export interface CodeToThemedTokensOptions { diff --git a/packages/shikiji/scripts/prepare.ts b/packages/shikiji/scripts/prepare.ts index 23fcfd1f5..dd102bd7c 100644 --- a/packages/shikiji/scripts/prepare.ts +++ b/packages/shikiji/scripts/prepare.ts @@ -1,194 +1,8 @@ import fs from 'fs-extra' -import { BUNDLED_LANGUAGES, BUNDLED_THEMES } from 'shiki' -import fg from 'fast-glob' - -const allLangFiles = await fg('*.json', { - cwd: './node_modules/shiki/languages', - absolute: true, - onlyFiles: true, -}) - -const comments = ` -/** - * Generated by scripts/prepare.ts - */ -`.trim() +import { prepareLangs } from './prepare/langs' +import { prepareTheme } from './prepare/themes' await fs.ensureDir('./src/assets/langs') await fs.emptyDir('./src/assets/langs') - -allLangFiles.sort() -for (const file of allLangFiles) { - const content = await fs.readJSON(file) - const lang = BUNDLED_LANGUAGES.find(i => i.id === content.name) - if (!lang) { - console.warn(`unknown ${content.name}`) - continue - } - - const json = { - ...content, - name: content.name || lang.id, - scopeName: content.scopeName || lang.scopeName, - displayName: lang.displayName, - aliases: lang.aliases, - embeddedLangs: lang.embeddedLangs, - balancedBracketSelectors: lang.balancedBracketSelectors, - unbalancedBracketSelectors: lang.unbalancedBracketSelectors, - } - - // F# and Markdown has circular dependency - if (lang.id === 'fsharp') - json.embeddedLangs = json.embeddedLangs.filter((i: string) => i !== 'markdown') - - const embedded = (json.embeddedLangs || []) as string[] - - await fs.writeFile(`./src/assets/langs/${lang.id}.ts`, `${comments} -import type { LanguageRegistration } from 'shikiji-core' - -${embedded.map(i => `import ${i.replace(/[^\w]/g, '_')} from './${i}'`).join('\n')} - -const lang = Object.freeze(${JSON.stringify(json)}) as unknown as LanguageRegistration - -export default [ -${[ - ...embedded.map(i => ` ...${i.replace(/[^\w]/g, '_')}`), - ' lang', -].join(',\n') || ''} -] -`, 'utf-8') -} - -async function writeLanguageBundleIndex(fileName: string, ids: string[]) { - const bundled = ids.map(id => BUNDLED_LANGUAGES.find(i => i.id === id)!) - - const info = bundled.map(i => ({ - id: i.id, - name: i.displayName, - aliases: i.aliases, - import: `__(() => import('./langs/${i.id}')) as DynamicLangReg__`, - }) as const) - .sort((a, b) => a.id.localeCompare(b.id)) - - const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => `'${i}'`).join(' | ') - - await fs.writeFile( - `src/assets/${fileName}.ts`, - `${comments} -import type { LanguageRegistration } from 'shikiji-core' - -type DynamicLangReg = () => Promise<{ default: LanguageRegistration[] }> - -export interface BundledLanguageInfo { - id: string - name: string - import: DynamicLangReg - aliases?: string[] -} - -export const bundledLanguagesInfo: BundledLanguageInfo[] = ${JSON.stringify(info, null, 2).replace(/"__|__"/g, '').replace(/"/g, '\'')} - -export const bundledLanguagesBase = Object.fromEntries(bundledLanguagesInfo.map(i => [i.id, i.import])) - -export const bundledLanguagesAlias = Object.fromEntries(bundledLanguagesInfo.flatMap(i => i.aliases?.map(a => [a, i.import]) || [])) - -export type BuiltinLanguage = ${type} - -export const bundledLanguages = { - ...bundledLanguagesBase, - ...bundledLanguagesAlias, -} as Record -`, - 'utf-8', - ) - - await fs.writeJSON( - `src/assets/${fileName}.json`, - BUNDLED_LANGUAGES.map(i => ({ - id: i.id, - name: i.displayName, - aliases: i.aliases, - })), - { spaces: 2 }, - ) -} - -await writeLanguageBundleIndex('langs', BUNDLED_LANGUAGES.map(i => i.id)) -// await writeLanguageBundleIndex('langs-common', BundleCommonLangs) - -const themes = BUNDLED_THEMES.sort() - .filter(i => i !== 'css-variables') - .map((id) => { - const theme = fs.readJSONSync(`./node_modules/shiki/themes/${id}.json`) - - return { - id, - name: guessThemeName(id, theme), - type: guessThemeType(id, theme), - import: `__(() => import('shiki/themes/${id}.json')) as unknown as DynamicThemeReg__`, - } - }) - -await fs.writeFile( - 'src/assets/themes.ts', - `${comments} -import type { ThemeRegistrationRaw } from 'shikiji-core' - -type DynamicThemeReg = () => Promise<{ default: ThemeRegistrationRaw }> - -export interface BundledThemeInfo { - id: string - name: string - type: 'light' | 'dark' - import: DynamicThemeReg -} - -export const bundledThemesInfo: BundledThemeInfo[] = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')} - -export type BuiltinTheme = ${themes.map(i => `'${i.id}'`).join(' | ')} - -export const bundledThemes = Object.fromEntries(bundledThemesInfo.map(i => [i.id, i.import])) as Record -`, - 'utf-8', -) - -await fs.writeJSON( - 'src/assets/themes.json', - BUNDLED_THEMES - .filter(i => i !== 'css-variables') - .map(i => ({ - id: i, - })), - { spaces: 2 }, -) - -function isLightColor(hex: string) { - const [r, g, b] = hex.slice(1).match(/.{2}/g)!.map(i => Number.parseInt(i, 16)) - return (r * 299 + g * 587 + b * 114) / 1000 > 128 -} - -function guessThemeType(id: string, theme: any) { - let color - if (id.includes('dark') || id.includes('dimmed') || id.includes('black')) - color = 'dark' - else if (id.includes('light') || id.includes('white') || id === 'slack-ochin') - color = 'light' - else if (theme.colors.background) - color = isLightColor(theme.colors.background) ? 'light' : 'dark' - else if (theme.colors['editor.background']) - color = isLightColor(theme.colors['editor.background']) ? 'light' : 'dark' - else if (theme.colors.foreground) - color = isLightColor(theme.colors.foreground) ? 'dark' : 'light' - else - color = 'light' - return color -} - -function guessThemeName(id: string, theme: any) { - if (theme.displayName) - return theme.displayName - let name: string = theme.name || id - name = name.split(/[_-]/).map(i => i[0].toUpperCase() + i.slice(1)).join(' ') - name = name.replace(/github/ig, 'GitHub') - return name -} +await prepareLangs() +await prepareTheme() diff --git a/packages/shikiji/scripts/prepare/constants.ts b/packages/shikiji/scripts/prepare/constants.ts new file mode 100644 index 000000000..923257455 --- /dev/null +++ b/packages/shikiji/scripts/prepare/constants.ts @@ -0,0 +1,5 @@ +export const COMMENT_HEAD = ` +/** + * Generated by scripts/prepare.ts + */ +`.trim() diff --git a/packages/shikiji/scripts/prepare/injections.ts b/packages/shikiji/scripts/prepare/injections.ts new file mode 100644 index 000000000..55332a07b --- /dev/null +++ b/packages/shikiji/scripts/prepare/injections.ts @@ -0,0 +1,61 @@ +// Download and prepare grammar injections + +import fs from 'fs-extra' +import { fetch } from 'ofetch' +import { COMMENT_HEAD } from './constants' + +interface Injection { + name: string + contents: any[] + /** + * Bundle into a language + */ + toLang?: string +} + +export async function prepareInjections() { + const injections = (await Promise.all([ + prepareVueInjections(), + ])).flat() + + for (const injection of injections) { + await fs.writeFile( + `src/assets/langs/${injection.name}.ts`, + `${COMMENT_HEAD} +import type { LanguageRegistration } from 'shikiji-core' + +export default ${JSON.stringify(injection.contents, null, 2)} as unknown as LanguageRegistration[] +`, + 'utf-8', + ) + } + + return injections +} + +export async function prepareVueInjections(): Promise { + const base = 'https://github.com/vuejs/language-tools/blob/master/extensions/vscode/' + const pkgJson = await fetchJson(`${base}package.json?raw=true`) + const grammars = pkgJson.contributes.grammars as any[] + const injections = await Promise.all(grammars + .filter(i => i.injectTo) + .map(async (i) => { + const content = await fetchJson(`${new URL(i.path, base).href}?raw=true`) + return { + ...content, + name: i.language, + injectTo: i.injectTo, + } + }), + ) + + return { + name: 'vue-injections', + toLang: 'vue', + contents: injections, + } +} + +export function fetchJson(url: string) { + return fetch(url).then(res => res.json()) +} diff --git a/packages/shikiji/scripts/prepare/langs.ts b/packages/shikiji/scripts/prepare/langs.ts new file mode 100644 index 000000000..b9ab232eb --- /dev/null +++ b/packages/shikiji/scripts/prepare/langs.ts @@ -0,0 +1,118 @@ +import fs from 'fs-extra' +import { BUNDLED_LANGUAGES } from 'shiki' +import fg from 'fast-glob' +import type { LanguageRegistration } from 'shikiji-core' +import { COMMENT_HEAD } from './constants' +import { prepareInjections } from './injections' + +export async function prepareLangs() { + const allLangFiles = await fg('*.json', { + cwd: './node_modules/shiki/languages', + absolute: true, + onlyFiles: true, + }) + + allLangFiles.sort() + + const injections = await prepareInjections() + + for (const file of allLangFiles) { + const content = await fs.readJSON(file) + const lang = BUNDLED_LANGUAGES.find(i => i.id === content.name) + if (!lang) { + console.warn(`unknown ${content.name}`) + continue + } + + const json: LanguageRegistration = { + ...content, + name: content.name || lang.id, + scopeName: content.scopeName || lang.scopeName, + displayName: lang.displayName, + aliases: lang.aliases, + embeddedLangs: lang.embeddedLangs, + balancedBracketSelectors: lang.balancedBracketSelectors, + unbalancedBracketSelectors: lang.unbalancedBracketSelectors, + } + + // F# and Markdown has circular dependency + if (lang.id === 'fsharp' && json.embeddedLangs) + json.embeddedLangs = json.embeddedLangs.filter((i: string) => i !== 'markdown') + + const deps: string[] = [ + ...(json.embeddedLangs || []), + ...injections.filter(i => i.toLang === lang.id).map(i => i.name), + ] + + await fs.writeFile(`./src/assets/langs/${lang.id}.ts`, `${COMMENT_HEAD} +import type { LanguageRegistration } from 'shikiji-core' + +${deps.map(i => `import ${i.replace(/[^\w]/g, '_')} from './${i}'`).join('\n')} + +const lang = Object.freeze(${JSON.stringify(json)}) as unknown as LanguageRegistration + +export default [ +${[ + ...deps.map(i => ` ...${i.replace(/[^\w]/g, '_')}`), + ' lang', +].join(',\n') || ''} +] +`, 'utf-8') + } + + async function writeLanguageBundleIndex(fileName: string, ids: string[]) { + const bundled = ids.map(id => BUNDLED_LANGUAGES.find(i => i.id === id)!) + + const info = bundled.map(i => ({ + id: i.id, + name: i.displayName, + aliases: i.aliases, + import: `__(() => import('./langs/${i.id}')) as DynamicLangReg__`, + }) as const) + .sort((a, b) => a.id.localeCompare(b.id)) + + const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => `'${i}'`).join(' | ') + + await fs.writeFile( + `src/assets/${fileName}.ts`, + `${COMMENT_HEAD} +import type { LanguageRegistration } from 'shikiji-core' + +type DynamicLangReg = () => Promise<{ default: LanguageRegistration[] }> + +export interface BundledLanguageInfo { + id: string + name: string + import: DynamicLangReg + aliases?: string[] +} + +export const bundledLanguagesInfo: BundledLanguageInfo[] = ${JSON.stringify(info, null, 2).replace(/"__|__"/g, '').replace(/"/g, '\'')} + +export const bundledLanguagesBase = Object.fromEntries(bundledLanguagesInfo.map(i => [i.id, i.import])) + +export const bundledLanguagesAlias = Object.fromEntries(bundledLanguagesInfo.flatMap(i => i.aliases?.map(a => [a, i.import]) || [])) + +export type BuiltinLanguage = ${type} + +export const bundledLanguages = { + ...bundledLanguagesBase, + ...bundledLanguagesAlias, +} as Record +`, + 'utf-8', + ) + + await fs.writeJSON( + `src/assets/${fileName}.json`, + BUNDLED_LANGUAGES.map(i => ({ + id: i.id, + name: i.displayName, + aliases: i.aliases, + })), + { spaces: 2 }, + ) + } + + await writeLanguageBundleIndex('langs', BUNDLED_LANGUAGES.map(i => i.id)) +} diff --git a/packages/shikiji/scripts/prepare/themes.ts b/packages/shikiji/scripts/prepare/themes.ts new file mode 100644 index 000000000..44420784b --- /dev/null +++ b/packages/shikiji/scripts/prepare/themes.ts @@ -0,0 +1,81 @@ +import fs from 'fs-extra' +import { BUNDLED_THEMES } from 'shiki' +import { COMMENT_HEAD } from './constants' + +export async function prepareTheme() { + const themes = BUNDLED_THEMES.sort() + .filter(i => i !== 'css-variables') + .map((id) => { + const theme = fs.readJSONSync(`./node_modules/shiki/themes/${id}.json`) + + return { + id, + name: guessThemeName(id, theme), + type: guessThemeType(id, theme), + import: `__(() => import('shiki/themes/${id}.json')) as unknown as DynamicThemeReg__`, + } + }) + await fs.writeFile( + 'src/assets/themes.ts', +`${COMMENT_HEAD} +import type { ThemeRegistrationRaw } from 'shikiji-core' + +type DynamicThemeReg = () => Promise<{ default: ThemeRegistrationRaw }> + +export interface BundledThemeInfo { + id: string + name: string + type: 'light' | 'dark' + import: DynamicThemeReg +} + +export const bundledThemesInfo: BundledThemeInfo[] = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')} + +export type BuiltinTheme = ${themes.map(i => `'${i.id}'`).join(' | ')} + +export const bundledThemes = Object.fromEntries(bundledThemesInfo.map(i => [i.id, i.import])) as Record +`, +'utf-8', + ) + await fs.writeJSON( + 'src/assets/themes.json', + BUNDLED_THEMES + .filter(i => i !== 'css-variables') + .map(i => ({ + id: i, + })), + { spaces: 2 }, + ) +} + +function isLightColor(hex: string) { + const [r, g, b] = hex.slice(1).match(/.{2}/g)!.map(i => Number.parseInt(i, 16)) + return (r * 299 + g * 587 + b * 114) / 1000 > 128 +} + +function guessThemeType(id: string, theme: any) { + let color + if (id.includes('dark') || id.includes('dimmed') || id.includes('black')) + color = 'dark' + else if (id.includes('light') || id.includes('white') || id === 'slack-ochin') + color = 'light' + else if (theme.colors.background) + color = isLightColor(theme.colors.background) ? 'light' : 'dark' + else if (theme.colors['editor.background']) + color = isLightColor(theme.colors['editor.background']) ? 'light' : 'dark' + else if (theme.colors.foreground) + color = isLightColor(theme.colors.foreground) ? 'dark' : 'light' + + else + color = 'light' + return color +} + +function guessThemeName(id: string, theme: any) { + if (theme.displayName) + return theme.displayName + let name: string = theme.name || id + name = name.split(/[_-]/).map(i => i[0].toUpperCase() + i.slice(1)).join(' ') + name = name.replace(/github/ig, 'GitHub') + return name +} diff --git a/packages/shikiji/test/index.test.ts b/packages/shikiji/test/index.test.ts index cbbe07835..f86d3b8ba 100644 --- a/packages/shikiji/test/index.test.ts +++ b/packages/shikiji/test/index.test.ts @@ -145,6 +145,10 @@ describe('should', () => { "typescript", "vb", "vue", + "vue-directives", + "vue-injection-markdown", + "vue-interpolations", + "vue-sfc-style-variable-injection", "xml", "xsl", "yaml", diff --git a/packages/shikiji/test/injections.test.ts b/packages/shikiji/test/injections.test.ts new file mode 100644 index 000000000..d16dbee67 --- /dev/null +++ b/packages/shikiji/test/injections.test.ts @@ -0,0 +1,22 @@ +import { codeToHtml } from 'shikiji' +import { expect, it } from 'vitest' + +// Basically we need to make sure that the syntax inside `v-if` and `{{}}` is highlighted correctly. +// This is done by a `vue-injections` patch that injects extra grammar into HTML. +it('vue-injections', async () => { + const code = ` + + + +` + + expect(`${await codeToHtml(code, { lang: 'vue', theme: 'vitesse-dark' })}`) + .toMatchFileSnapshot('./out/vue-injections.html') +}) diff --git a/packages/shikiji/test/out/vue-injections.html b/packages/shikiji/test/out/vue-injections.html new file mode 100644 index 000000000..d8b930889 --- /dev/null +++ b/packages/shikiji/test/out/vue-injections.html @@ -0,0 +1,12 @@ +

+<script setup lang="ts">
+import { ref } from 'vue'
+const count = ref(0)
+</script>
+
+<template>
+  <div>
+    <h1 v-if="count == 1 ? true : 'str'.toUpperCase()">{{ count * 2 }}</h1>
+  </div>
+</template>
+
\ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82667a5d8..cbd842f5e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,6 +99,9 @@ importers: markdown-it-shikiji: specifier: workspace:* version: link:packages/markdown-it-shikiji + ofetch: + specifier: ^1.3.3 + version: 1.3.3 pnpm: specifier: ^8.12.1 version: 8.12.1