feat: new shikiji-core package, improve bundling (#42)

.github/workflows/ci.yml

@@ -14,6 +14,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2
@@ -36,6 +38,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2
@@ -73,6 +77,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Install pnpm
         uses: pnpm/action-setup@v2

.github/workflows/release.yml

@@ -16,9 +16,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v2
-
       - name: Set node
         uses: actions/setup-node@v3
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/shikiji-core/vendor/vscode-textmate"]
+	path = packages/shikiji-core/vendor/vscode-textmate
+	url = https://github.com/microsoft/vscode-textmate

eslint.config.ts

@@ -13,6 +13,7 @@ export default antfu(
       '**/test/out/**',
       'docs/languages.md',
       'docs/themes.md',
+      '**/vendor/**',
     ],
   },
   {

package.json

@@ -14,6 +14,7 @@
     "@antfu/eslint-config": "^2.4.5",
     "@antfu/ni": "^0.21.12",
     "@antfu/utils": "^0.7.7",
+    "@rollup/plugin-alias": "^5.1.0",
     "@rollup/plugin-commonjs": "^25.0.7",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
@@ -42,6 +43,7 @@
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-dts": "^6.1.0",
     "rollup-plugin-esbuild": "^6.1.0",
+    "rollup-plugin-typescript2": "^0.36.0",
     "shiki": "^0.14.6",
     "shikiji": "workspace:*",
     "simple-git-hooks": "^2.9.0",
@@ -50,8 +52,6 @@
     "unbuild": "^2.0.0",
     "vite": "^5.0.9",
     "vitest": "^1.0.4",
-    "vscode-oniguruma": "^1.7.0",
-    "vscode-textmate": "^9.0.0",
     "wrangler": "^3.20.0"
   },
   "resolutions": {

packages/shikiji-core/README.md

@@ -0,0 +1,5 @@
+# shikiji-core
+
+The core functionality of [Shikiji](https://github.com/antfu/shikiji), without any grammar of themes bundled.
+
+It's the same as importing `shikiji/core`.

packages/shikiji-core/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "shikiji-core",
+  "type": "module",
+  "version": "0.9.3-alpha.0",
+  "description": "Core of Shikiji",
+  "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
+  "license": "MIT",
+  "homepage": "https://github.com/antfu/shikiji#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/antfu/shikiji.git",
+    "directory": "packages/shikiji-core"
+  },
+  "bugs": "https://github.com/antfu/shikiji/issues",
+  "keywords": [
+    "shiki"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./wasm": {
+      "types": "./dist/wasm.d.mts",
+      "default": "./dist/wasm.mjs"
+    },
+    "./textmate": {
+      "types": "./dist/textmate.d.mts",
+      "default": "./dist/textmate.mjs"
+    },
+    "./types": {
+      "types": "./dist/types.d.mts"
+    },
+    "./dist/*": "./dist/*",
+    "./*": "./dist/*"
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "typesVersions": {
+    "*": {
+      "wasm": [
+        "./dist/wasm.d.mts"
+      ],
+      "types": [
+        "./dist/types.d.mts"
+      ],
+      "textmate": [
+        "./dist/textmate.d.mts"
+      ],
+      "*": [
+        "./dist/*",
+        "./*"
+      ]
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "rimraf dist && rollup -c",
+    "dev": "rollup -cw",
+    "prepublishOnly": "nr build",
+    "test": "vitest"
+  },
+  "devDependencies": {
+    "hast-util-to-html": "^9.0.0",
+    "vscode-oniguruma": "^1.7.0"
+  }
+}

packages/shikiji-core/rollup.config.mjs

@@ -0,0 +1,94 @@
+// @ts-check
+import { defineConfig } from 'rollup'
+import { nodeResolve } from '@rollup/plugin-node-resolve'
+import commonjs from '@rollup/plugin-commonjs'
+import replace from '@rollup/plugin-replace'
+import dts from 'rollup-plugin-dts'
+import json from '@rollup/plugin-json'
+import fs from 'fs-extra'
+import ts from 'rollup-plugin-typescript2'
+
+const entries = [
+  'src/index.ts',
+  'src/types.ts',
+  'src/textmate.ts',
+  'src/wasm.ts',
+]
+
+const plugins = [
+  ts({
+    check: false,
+  }),
+  replace({
+    'DebugFlags.InDebugMode': 'false',
+    'preventAssignment': true,
+  }),
+  nodeResolve(),
+  commonjs(),
+  json({
+    namedExports: false,
+    preferConst: true,
+    compact: true,
+  }),
+  wasmPlugin(),
+]
+
+export default defineConfig([
+  {
+    input: entries,
+    output: {
+      dir: 'dist',
+      format: 'esm',
+      entryFileNames: '[name].mjs',
+      chunkFileNames: (f) => {
+        if (f.name === 'onig')
+          return 'onig.mjs'
+        return 'chunks-[name].mjs'
+      },
+    },
+    plugins: [
+      ...plugins,
+    ],
+  },
+  {
+    input: entries,
+    output: {
+      dir: 'dist',
+      format: 'esm',
+      chunkFileNames: '[name].d.mts',
+      entryFileNames: f => `${f.name.replace(/src[\\\/]/, '')}.d.mts`,
+    },
+    plugins: [
+      dts({
+        respectExternal: true,
+      }),
+      {
+        name: 'post',
+        async buildEnd() {
+          await fs.writeFile('dist/onig.d.mts', 'declare const binary: ArrayBuffer; export default binary;', 'utf-8')
+        },
+      },
+    ],
+    onwarn: (warning, warn) => {
+      if (!/Circular|an empty chunk/.test(warning.message))
+        warn(warning)
+    },
+    external: [],
+  },
+])
+
+/**
+ * @returns {import('rollup').Plugin} Plugin
+ */
+export function wasmPlugin() {
+  return {
+    name: 'wasm',
+    async load(id) {
+      if (!id.endsWith('.wasm'))
+        return
+      const binary = await fs.readFile(id)
+      const base64 = binary.toString('base64')
+      return `export default Uint8Array.from(atob(${JSON.stringify(base64)}), c => c.charCodeAt(0))`
+    },
+  }
+}

packages/shikiji/src/core/bundle-factory.ts → packages/shikiji-core/src/bundle-factory.ts

@@ -1,4 +1,4 @@
-import type { BundledHighlighterOptions, CodeToHastOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, RequireKeys, SpecialLanguage, ThemeInput, ThemeRegistration, ThemeRegistrationRaw } from '../types'
+import type { BundledHighlighterOptions, CodeToHastOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, RequireKeys, SpecialLanguage, ThemeInput, ThemeRegistration, ThemeRegistrationRaw } from './types'
 import { isSpecialLang, toArray } from './utils'
 import { getHighlighterCore } from './highlighter'
 

packages/shikiji/src/core/highlighter.ts → packages/shikiji-core/src/highlighter.ts

@@ -1,4 +1,4 @@
-import type { HighlighterCoreOptions, HighlighterGeneric } from '../types'
+import type { HighlighterCoreOptions, HighlighterGeneric } from './types'
 import { codeToHtml } from './renderer-html'
 import { codeToTokensWithThemes } from './renderer-html-themes'
 import { codeToThemedTokens } from './tokenizer'

packages/shikiji/src/core/index.ts → packages/shikiji-core/src/index.ts

@@ -1,8 +1,9 @@
-export { loadWasm } from '../oniguruma'
+export { loadWasm } from './oniguruma'
 
 export * from './highlighter'
 export * from './bundle-factory'
 export * from './utils'
+export * from './types'
 
 export { getShikiInternal, getShikiContext } from './internal'
 export { codeToThemedTokens } from './tokenizer'

packages/shikiji/src/core/internal.ts → packages/shikiji-core/src/internal.ts

@@ -1,5 +1,5 @@
-import type { HighlighterCoreOptions, LanguageInput, MaybeGetter, ShikiInternal, ThemeInput, ThemeRegistration } from '../types'
-import { createOnigScanner, createOnigString, loadWasm } from '../oniguruma'
+import type { HighlighterCoreOptions, LanguageInput, MaybeGetter, ShikiInternal, ThemeInput, ThemeRegistration } from './types'
+import { createOnigScanner, createOnigString, loadWasm } from './oniguruma'
 import { Registry } from './registry'
 import { Resolver } from './resolver'
 
@@ -30,14 +30,17 @@ export async function getShikiInternal(options: HighlighterCoreOptions = {}): Pr
         : undefined,
   ] as const)
 
-  const resolver = new Resolver(Promise.resolve({
-    createOnigScanner(patterns) {
-      return createOnigScanner(patterns)
-    },
-    createOnigString(s) {
-      return createOnigString(s)
-    },
-  }), 'vscode-oniguruma', langs)
+  const resolver = new Resolver(
+    Promise.resolve({
+      createOnigScanner(patterns) {
+        return createOnigScanner(patterns)
+      },
+      createOnigString(s) {
+        return createOnigString(s)
+      },
+    }),
+    langs,
+  )
 
   const _registry = new Registry(resolver, themes, langs)
   Object.assign(_registry.alias, options.langAlias)

packages/shikiji/src/core/normalize.ts → packages/shikiji-core/src/normalize.ts

@@ -1,5 +1,5 @@
-import type { IRawTheme } from 'vscode-textmate'
-import type { ThemeRegistration, ThemeRegistrationRaw } from '../types'
+import type { IRawTheme } from './textmate'
+import type { ThemeRegistration, ThemeRegistrationRaw } from './types'
 
 export function toShikiTheme(rawTheme: ThemeRegistrationRaw | ThemeRegistration): ThemeRegistration {
   const type = (<any>rawTheme).type || 'dark'

packages/shikiji/src/core/registry.ts → packages/shikiji-core/src/registry.ts

@@ -1,6 +1,6 @@
-import type { IGrammar, IGrammarConfiguration } from 'vscode-textmate'
-import { Registry as TextMateRegistry } from 'vscode-textmate'
-import type { LanguageRegistration, ThemeRegistration, ThemeRegistrationRaw } from '../types'
+import type { IGrammar, IGrammarConfiguration } from './textmate'
+import { Registry as TextMateRegistry } from './textmate'
+import type { LanguageRegistration, ThemeRegistration, ThemeRegistrationRaw } from './types'
 import type { Resolver } from './resolver'
 import { toShikiTheme } from './normalize'
 

packages/shikiji/src/core/renderer-hast.ts → packages/shikiji-core/src/renderer-hast.ts

@@ -8,9 +8,9 @@ import type {
   ThemedToken,
   ThemedTokenWithVariants,
   TokenStyles,
-} from '../types'
+} from './types'
 import { codeToThemedTokens } from './tokenizer'
-import { FontStyle } from './stackElementMetadata'
+import { FontStyle } from './types'
 import { codeToTokensWithThemes } from './renderer-html-themes'
 
 export function codeToHast(

packages/shikiji/src/core/renderer-html-themes.ts → packages/shikiji-core/src/renderer-html-themes.ts

@@ -3,7 +3,7 @@ import type {
   ShikiInternal,
   ThemedToken,
   ThemedTokenWithVariants,
-} from '../types'
+} from './types'
 import { codeToThemedTokens } from './tokenizer'
 
 /**

packages/shikiji/src/core/renderer-html.ts → packages/shikiji-core/src/renderer-html.ts

@@ -1,5 +1,5 @@
 import { toHtml as hastToHtml } from 'hast-util-to-html'
-import type { CodeToHastOptions, ShikiInternal, ShikijiTransformerContextCommon } from '../types'
+import type { CodeToHastOptions, ShikiInternal, ShikijiTransformerContextCommon } from './types'
 import { codeToHast } from './renderer-hast'
 
 /**

packages/shikiji/src/core/resolver.ts → packages/shikiji-core/src/resolver.ts

@@ -1,17 +1,14 @@
-import type { IOnigLib, RegistryOptions } from 'vscode-textmate'
-
-import type { LanguageRegistration } from '../types'
+import type { IOnigLib, RegistryOptions } from './textmate'
+import type { LanguageRegistration } from './types'
 
 export class Resolver implements RegistryOptions {
   private readonly languageMap: { [langIdOrAlias: string]: LanguageRegistration } = {}
   private readonly scopeToLangMap: { [scope: string]: LanguageRegistration } = {}
 
   private readonly _onigLibPromise: Promise<IOnigLib>
-  private readonly _onigLibName: string
 
-  constructor(onigLibPromise: Promise<IOnigLib>, onigLibName: string, langs: LanguageRegistration[]) {
+  constructor(onigLibPromise: Promise<IOnigLib>, langs: LanguageRegistration[]) {
     this._onigLibPromise = onigLibPromise
-    this._onigLibName = onigLibName
 
     langs.forEach(i => this.addLanguage(i))
   }
@@ -20,10 +17,6 @@ export class Resolver implements RegistryOptions {
     return this._onigLibPromise
   }
 
-  public getOnigLibName(): string {
-    return this._onigLibName
-  }
-
   public getLangRegistration(langIdOrAlias: string): LanguageRegistration {
     return this.languageMap[langIdOrAlias]
   }