feat(perf): skip bundling for assets (#795)

shikijs · Sep 29, 2024 · 0c5a51a · 0c5a51a
1 parent c8accc9
commit 0c5a51a
Show file tree

Hide file tree

Showing 27 changed files with 733 additions and 752 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,9 +11,11 @@ logs
 node_modules
 temp
 tmp
-packages/shiki/src/assets/langs
-packages/shiki/src/assets/themes
-packages/shiki/src/assets/*.json
+packages/shiki/src/langs
+packages/shiki/src/themes
+packages/shiki/src/*.json
+packages/shiki/src/langs
+packages/shiki/src/themes
 cache
 .eslintcache
 report-engine-js-compat.json

diff --git a/eslint.config.js b/eslint.config.js
@@ -22,12 +22,17 @@ export default antfu(
       },
     },
     ignores: [
-      'packages/shiki/src/assets/*.ts',
       '**/fixtures/**',
       '**/vendor/**',
       '**/test/out/**',
       'docs/languages.md',
       'docs/themes.md',
+      // Generated Files
+      'packages/shiki/src/langs/**',
+      'packages/shiki/src/themes/**',
+      'packages/shiki/src/langs-bundle-full.ts',
+      'packages/shiki/src/langs-bundle-web.ts',
+      'packages/shiki/src/themes.ts',
     ],
   },
   {

diff --git a/packages/engine-javascript/test/compare.test.ts b/packages/engine-javascript/test/compare.test.ts
@@ -46,51 +46,51 @@ export interface Cases {
 const cases: Cases[] = [
   {
     name: 'json-basic',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/json'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/json.mjs'),
     cases: [
       '{"foo":{"bar":1}}',
       '[undefined, null, true, false, 0, 1, 1.1, "foo", [], {}]',
     ],
   },
   {
     name: 'html-basic',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/html'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/html.mjs'),
     cases: [
       '<div class="foo">bar</div>',
       '<!DOCTYPE html><html><head><title>foo</title></head><body>bar</body></html>',
     ],
   },
   {
     name: 'ts-basic',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/typescript'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/typescript.mjs'),
     cases: [
       'const foo: string = "bar"',
     ],
   },
   {
     name: 'jsonc',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/jsonc'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/jsonc.mjs'),
     cases: [
       '// comment\n{"foo":"bar"}',
     ],
   },
   {
     name: 'vue',
-    theme: () => import('../../shiki/src/assets/themes/vitesse-dark'),
-    lang: () => import('../../shiki/src/assets/langs/vue'),
+    theme: () => import('../../shiki/src/themes/vitesse-dark.mjs'),
+    lang: () => import('../../shiki/src/langs/vue.mjs'),
     cases: [
       `<script setup>\nimport { ref } from 'vue'\n</script>`,
       `<template>\n<div>{{ foo }}</div>\n</template>`,
     ],
   },
   {
     name: 'toml',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/toml'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/toml.mjs'),
     cases: [
       [
         `# This is a TOML document`,
@@ -104,8 +104,8 @@ const cases: Cases[] = [
   },
   {
     name: 'sql',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/sql'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/sql.mjs'),
     cases: [
       'SELECT * FROM foo',
       [
@@ -119,8 +119,8 @@ const cases: Cases[] = [
   {
     skip: true,
     name: 'markdown',
-    theme: () => import('../../shiki/src/assets/themes/nord'),
-    lang: () => import('../../shiki/src/assets/langs/markdown'),
+    theme: () => import('../../shiki/src/themes/nord.mjs'),
+    lang: () => import('../../shiki/src/langs/markdown.mjs'),
     cases: [
       [
         '# Header',

diff --git a/packages/shiki/package.json b/packages/shiki/package.json
@@ -129,6 +129,7 @@
     "@types/hast": "catalog:"
   },
   "devDependencies": {
+    "rollup-plugin-copy": "^3.5.0",
     "tm-grammars": "catalog:",
     "tm-themes": "catalog:",
     "vscode-oniguruma": "catalog:"

diff --git a/packages/shiki/rollup.config.mjs b/packages/shiki/rollup.config.mjs
@@ -35,6 +35,7 @@ const external = [
   '@shikijs/engine-oniguruma',
   '@shikijs/vscode-textmate',
   'shiki/wasm',
+  /[/\\]src[/\\](langs|themes)[/\\]/g,
 ]
 
 const plugins = [
@@ -52,27 +53,20 @@ export default defineConfig([
   {
     input: [
       ...entries,
-      // add language files entries
-      ...fg.sync('src/assets/langs/*.js'),
     ],
     output: {
       dir: 'dist',
       format: 'esm',
-      entryFileNames: (f) => {
-        if (f.facadeModuleId?.match(/[\\/]langs[\\/]/))
-          return `langs/${f.name}.mjs`
-        return '[name].mjs'
-      },
-      chunkFileNames: (f) => {
-        if (f.moduleIds.some(i => i.match(/[\\/]langs[\\/]/)))
-          return `langs/${f.name}.mjs`
-        else if (f.moduleIds.some(i => i.match(/[\\/]themes[\\/]/)))
-          return 'themes/[name].mjs'
-        return 'chunks/[name].mjs'
-      },
+      entryFileNames: '[name].mjs',
     },
     plugins: [
       ...plugins,
+      copy({
+        targets: [
+          { src: './src/langs', dest: 'dist' },
+          { src: './src/themes', dest: 'dist' },
+        ],
+      }),
     ],
     external,
   },

diff --git a/packages/shiki/scripts/prepare.ts b/packages/shiki/scripts/prepare.ts
@@ -2,9 +2,9 @@ import fs from 'fs-extra'
 import { prepareLangs } from './prepare/langs'
 import { prepareTheme } from './prepare/themes'
 
-await fs.ensureDir('./src/assets/langs')
-await fs.emptyDir('./src/assets/langs')
-await fs.ensureDir('./src/assets/themes')
-await fs.emptyDir('./src/assets/themes')
+await fs.ensureDir('./src/langs')
+await fs.emptyDir('./src/langs')
+await fs.ensureDir('./src/themes')
+await fs.emptyDir('./src/themes')
 await prepareLangs()
 await prepareTheme()
diff --git a/packages/shiki/scripts/prepare/langs.ts b/packages/shiki/scripts/prepare/langs.ts
@@ -96,9 +96,8 @@ export async function prepareLangs() {
       console.log(json.name, json.embeddedLangs)
 
     await fs.writeFile(
-      `./src/assets/langs/${lang.name}.js`,
-      `${COMMENT_HEAD}
-${deps.map(i => `import ${i.replace(/\W/g, '_')} from './${i}'`).join('\n')}
+      `./src/langs/${lang.name}.mjs`,
+      `${deps.map(i => `import ${i.replace(/\W/g, '_')} from './${i}.mjs'`).join('\n')}
 
 const lang = Object.freeze(JSON.parse(${JSON.stringify(JSON.stringify(json))}))
 
@@ -108,18 +107,17 @@ ${[
     '  lang',
   ].join(',\n') || ''}
 ]
-`.replace(/\n{2,}/g, '\n\n'),
+`.replace(/\n{2,}/g, '\n\n').trimStart(),
       'utf-8',
     )
 
     for (const alias of json.aliases || []) {
       if (isInvalidFilename(alias))
         continue
       await fs.writeFile(
-        `./src/assets/langs/${alias}.js`,
-        `${COMMENT_HEAD}
-// ${alias} is an alias of ${lang.name}
-export { default } from './${lang.name}'
+        `./src/langs/${alias}.mjs`,
+        `/* Alias ${alias} for ${lang.name} */
+export { default } from './${lang.name}.mjs'
 `,
         'utf-8',
       )
@@ -129,9 +127,8 @@ export { default } from './${lang.name}'
       if (isInvalidFilename(name))
         continue
       await fs.writeFile(
-        `./src/assets/langs/${name}.d.ts`,
-        `${COMMENT_HEAD}
-import type { LanguageRegistration } from '@shikijs/core'
+        `./src/langs/${name}.d.mts`,
+        `import type { LanguageRegistration } from '@shikijs/core'
 const langs: LanguageRegistration []
 export default langs
 `,
@@ -169,14 +166,14 @@ export default langs
         id: i.name,
         name: i.displayName || i.name,
         aliases: i.aliases,
-        import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
+        import: `__(() => import('./langs/${i.name}.mjs')) as DynamicImportLanguageRegistration__`,
       }) as const)
       .sort((a, b) => a.id.localeCompare(b.id))
 
     const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => `  | '${i}'`).join('\n')
 
     await fs.writeFile(
-      `src/assets/${fileName}.ts`,
+      `src/${fileName}.ts`,
       `${COMMENT_HEAD}
 import type { DynamicImportLanguageRegistration, BundledLanguageInfo } from '@shikijs/core'
 

diff --git a/packages/shiki/scripts/prepare/themes.ts b/packages/shiki/scripts/prepare/themes.ts
@@ -8,18 +8,16 @@ export async function prepareTheme(): Promise<void> {
       const theme = await fs.readJSON(`./node_modules/tm-themes/themes/${t.name}.json`)
 
       await fs.writeFile(
-        `./src/assets/themes/${t.name}.js`,
-        `${COMMENT_HEAD}
+        `./src/themes/${t.name}.mjs`,
+        `/* Theme: ${theme.name} */
 export default Object.freeze(JSON.parse(${JSON.stringify(JSON.stringify(theme))}))
 `,
         'utf-8',
       )
 
       await fs.writeFile(
-        `./src/assets/themes/${t.name}.d.ts`,
-        `${COMMENT_HEAD}
-import type { ThemeRegistration } from '@shikijs/core'
-
+        `./src/themes/${t.name}.d.mts`,
+        `import type { ThemeRegistration } from '@shikijs/core'
 const theme: ThemeRegistration
 export default theme
 `,
@@ -30,11 +28,11 @@ export default theme
         id: t.name,
         displayName: theme.displayName,
         type: theme.type,
-        import: `__(() => import('./themes/${t.name}')) as unknown as DynamicImportThemeRegistration__`,
+        import: `__(() => import('./themes/${t.name}.mjs')) as unknown as DynamicImportThemeRegistration__`,
       }
     }))
   await fs.writeFile(
-    'src/assets/themes.ts',
+    'src/themes.ts',
     `${COMMENT_HEAD}
 import type { DynamicImportThemeRegistration, BundledThemeInfo } from '@shikijs/core'