feat!: merge Shikiji back into Shiki for v1.0

.gitattributes

@@ -1 +1 @@
-packages/shiki/languages/*.json binary
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -2,20 +2,36 @@
 
 Thanks for lending a hand 👋
 
-Here are a few ways to contribute:
+## Development
 
-- [Add a theme](../docs/themes.md#adding-theme)
-- [Add a language grammar](../docs/languages.md#adding-grammar)
-- Fix a bug
+### Clone
 
-## Coding conventions
+This repository contains a [git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to `vscode-textmate`. By default `git clone` does not clone submodules. To clone this repository and its submodules, use:
 
-- We use [prettier](https://prettier.io). Before you commit, all files will be formatted automatically.
-- We use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Please use a prefix. If your PR has multiple commits and some of them don't follow the Conventional Commits rule, we'll do a squash merge.
-  - If adding a language, use `feat(lang)`
-  - If adding a theme, use `feat(theme)`
+```bash
+git clone --recursive https://github.com/shikijs/shiki
+```
+
+Or if you have already cloned it, use:
+
+```bash
+git submodule update --init --recursive
+```
+
+Learn more at this [StackOverflow thread](https://stackoverflow.com/a/4438292).
+
+### Setup
 
-## Fix a bug
+- We use [pnpm](https://pnpm.js.org/) to manage dependencies. Install it with `npm i -g pnpm`.
+- Install dependencies with `pnpm i`.
+- Build all packages with `pnpm build`.
 
-- Reference the bug you are fixing in the PR
-- Add a test if possible
+### Testting
+
+- We use [Vitest](https://vitest.dev) to test the codebase. Run `pnpm test` to start the test runner.
+- We have quite some [Snapshot Tests](https://vitest.dev/guide/snapshot.html) to ensure the output is consistent. If you are making changes to the output, run `pnpm test -u` to update the snapshots.
+
+### Coding conventions
+
+- We use ESLint to lint and format the codebase. Before you commit, all files will be formatted automatically.
+- We use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Please use a prefix. If your PR has multiple commits and some of them don't follow the Conventional Commits rule, we'll do a squash merge.

.github/FUNDING.yml

@@ -1 +1 @@
-github: [octref]
+github: [antfu, octref]

.github/pull_request_template.md

@@ -13,4 +13,4 @@
 
 - [ ] I have read docs for [adding a language](https://github.com/shikijs/shiki/blob/main/docs/languages.md#adding-grammar).
 - [ ] I have searched around and this is the most up-to-date, actively maintained version of the language grammar.
-- [ ] I have added a sample file that includes a variety of language syntaxes and succinctly captures the idiosyncrasy of a language. See [docs](https://github.com/shikijs/shiki/blob/main/docs/languages.md#adding-grammar) for requirement.
+- [ ] I have added a sample file that includes a variety of language syntaxes and succinctly captures the idiosyncrasy of a language. See [docs](https://github.com/shikijs/shiki/blob/main/docs/languages.md#adding-grammar) for requirement.

.github/workflows/ci.yml

@@ -0,0 +1,106 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+
+      - name: Set node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Setup
+        run: npm i -g @antfu/ni
+
+      - name: Install
+        run: nci
+
+      - name: Lint
+        run: nr lint
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+
+      - name: Set node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Setup
+        run: npm i -g @antfu/ni
+
+      - name: Install
+        run: nci
+
+      - name: Build
+        run: nr build
+
+      - name: Typecheck
+        run: nr typecheck
+
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        node: [lts/*]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          - node: 16.x
+            os: ubuntu-latest
+          - node: 18.x
+            os: ubuntu-latest
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+
+      - name: Set node ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Setup
+        run: npm i -g @antfu/ni
+
+      - name: Install
+        run: nci
+
+      - name: Build
+        run: nr build
+
+      - name: Test
+        run: nr test --coverage
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Release
+
+permissions:
+  contents: write
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+
+      - run: npx changelogithub
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}