From 6861425a6498117106aba9940775b57e92d1444c Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Thu, 28 Dec 2023 15:55:17 +0100 Subject: [PATCH] feat!: support custom init --- README.md | 67 ++++++++++++++--- build.config.ts | 2 +- package.json | 9 ++- plugin.d.ts | 1 + src/_utils.ts | 5 -- src/index.ts | 6 ++ src/{plugin.ts => plugin/index.ts} | 93 +++-------------------- src/plugin/runtime.ts | 117 +++++++++++++++++++++++++++++ src/plugin/shared.ts | 30 ++++++++ test/fixture/_imports.mjs | 6 ++ test/fixture/dynamic-import.mjs | 9 +++ test/fixture/index.mjs | 1 - test/fixture/static-import.mjs | 8 ++ test/plugin.test.ts | 26 ++++--- vitest.config.ts | 12 +++ 15 files changed, 276 insertions(+), 116 deletions(-) create mode 100644 plugin.d.ts delete mode 100644 src/_utils.ts create mode 100644 src/index.ts rename src/{plugin.ts => plugin/index.ts} (63%) create mode 100644 src/plugin/runtime.ts create mode 100644 src/plugin/shared.ts create mode 100644 test/fixture/_imports.mjs create mode 100644 test/fixture/dynamic-import.mjs delete mode 100644 test/fixture/index.mjs create mode 100644 test/fixture/static-import.mjs create mode 100644 vitest.config.ts diff --git a/README.md b/README.md index 953e55f..516d799 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,20 @@ [![npm downloads][npm-downloads-src]][npm-downloads-href] [![Codecov][codecov-src]][codecov-href] -# 🇼 unwasm +# unwasm Universal [WebAssembly](https://webassembly.org/) tools for JavaScript. ## Goal -This project aims to make a common and future-proof solution for WebAssembly modules support suitable for various JavaScript runtimes, frameworks, and build Tools following [WebAssembly/ES Module Integration](https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration) proposal from WebAssembly Community Group as much as possible. +This project aims to make a common and future-proof solution for WebAssembly modules support suitable for various JavaScript runtimes, frameworks, and build Tools following [WebAssembly/ES Module Integration](https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration) proposal from WebAssembly Community Group as much as possible while also trying to keep compatibility with current ecosystem libraries. ## Roadmap The development will be split into multiple stages. > [!IMPORTANT] -> This Project is under development! Join the linked discussions to be involved! +> This Project is under development! See the linked discussions to be involved! - [ ] Universal builder plugins built with [unjs/unplugin](https://github.com/unjs/unplugin) ([unjs/unwasm#2](https://github.com/unjs/unwasm/issues/2)) - [x] Rollup @@ -25,29 +25,73 @@ The development will be split into multiple stages. - [ ] Integration with [Wasmer](https://github.com/wasmerio) ([unjs/unwasm#6](https://github.com/unjs/unwasm/issues/6)) - [ ] Convention and tools for library authors exporting wasm modules ([unjs/unwasm#7](https://github.com/unjs/unwasm/issues/7)) -## Install +## Bindings API -Install package from [npm](https://www.npmjs.com/package/unwasm): +When importing a `.wasm` module using unwasm, it will take steps to transform the binary and finally resolve to an ESM module that allows you to interact with the WASM module. The returned result is a [Proxy](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy) object. This proxy allows to use an elegant API while also having both backward and forward compatibility with WASM modules as ecosystem evolves. + +WebAssembly modules that don't require any imports, can be imported simply like you import any other esm module: + +**Using static import:** + +```js +import { func } from "lib/module.wasm"; +``` + +**Using dynamic import:** + +```js +const { func } = await import("lib/module.wasm").then((mod) => mod.default); +``` + +In case of your WebAssembly module requires imports object (which is likely!), the usage syntax would be slightly different as we need to initial module with imports object first: + +**Using static import with imports object:** + +```js +import { func, $init } from "lib/module.wasm"; + +await $init({ env: {} }); +``` + +**Using dynamic import with imports object:** + +```js +const { func } = await import("lib/module.wasm").then((mod) => mod.$init(env)); +``` + +> [!NOTE] > **When using static import syntax**, and before calling `$init`, the named exports will be wrapped into a function by proxy that waits for the module initialization and before that if called, will immediately try to call `$init()` and return a Promise that calls function after init. + +> [!NOTE] +> Named exports with `$` prefix are reserved for unwasm. In case your module uses them, you can access from `$exports` property. + +## Usage + +Unwasm needs to transform the `.wasm` imports to the compatible bindings. Currently only method is using a rollup plugin. In the future more usage methods will be introduced. + +### Install + +First, install the [`unwasm` npm package](https://www.npmjs.com/package/unwasm): ```sh # npm -npm install unwasm +npm install --dev unwasm # yarn -yarn add unwasm +yarn add -D unwasm # pnpm -pnpm install unwasm +pnpm i -D unwasm # bun -bun install unwasm +bun i -D unwasm ``` -## Using build plugin +### Builder Plugins ###### Rollup ```js +// rollup.config.js import unwasmPlugin from "unwasm/plugin"; export default { @@ -59,7 +103,7 @@ export default { }; ``` -### Options +### Plugin Options - `esmImport`: Direct import the wasm file instead of bundling, required in Cloudflare Workers (default is `false`) - `lazy`: Import `.wasm` files using a lazily evaluated promise for compatibility with runtimes without top-level await support (default is `false`) @@ -71,6 +115,7 @@ export default { - Enable [Corepack](https://github.com/nodejs/corepack) using `corepack enable` - Install dependencies using `pnpm install` - Run interactive tests using `pnpm dev` +- Optionally install [es6-string-html](https://marketplace.visualstudio.com/items?itemName=Tobermory.es6-string-html) extension to make it easier working with string templates. ## License diff --git a/build.config.ts b/build.config.ts index b7be3c1..64bfdf7 100644 --- a/build.config.ts +++ b/build.config.ts @@ -2,6 +2,6 @@ import { defineBuildConfig } from "unbuild"; export default defineBuildConfig({ declaration: true, - entries: ["src/plugin"], + entries: ["src/plugin/index"], externals: ["unwasm", "rollup"], }); diff --git a/package.json b/package.json index bdcb000..23b82ac 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,13 @@ "type": "module", "exports": { "./plugin": { - "types": "./dist/plugin.d.mts", - "import": "./dist/plugin.mjs" + "types": "./dist/plugin/index.d.mts", + "import": "./dist/plugin/index.mjs" } }, "files": [ - "dist" + "dist", + "*.d.ts" ], "scripts": { "build": "unbuild", @@ -48,4 +49,4 @@ "vitest": "^1.1.0" }, "packageManager": "pnpm@8.12.1" -} \ No newline at end of file +} diff --git a/plugin.d.ts b/plugin.d.ts new file mode 100644 index 0000000..e74ea70 --- /dev/null +++ b/plugin.d.ts @@ -0,0 +1 @@ +export * from "./dist/plugin"; diff --git a/src/_utils.ts b/src/_utils.ts deleted file mode 100644 index 6421283..0000000 --- a/src/_utils.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { createHash } from "node:crypto"; - -export function sha1(source: Buffer) { - return createHash("sha1").update(source).digest("hex").slice(0, 16); -} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..75fb319 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,6 @@ +/* eslint-disable unicorn/no-empty-file */ // <-- WTF! + +// No runtime utils yet! +// https://github.com/unjs/unwasm/issues/4 + +// See plugin/ for real stuff! diff --git a/src/plugin.ts b/src/plugin/index.ts similarity index 63% rename from src/plugin.ts rename to src/plugin/index.ts index 1c3762f..f2040c5 100644 --- a/src/plugin.ts +++ b/src/plugin/index.ts @@ -3,35 +3,17 @@ import { basename } from "pathe"; import MagicString from "magic-string"; import type { RenderedChunk, Plugin as RollupPlugin } from "rollup"; import { createUnplugin } from "unplugin"; -import { sha1 } from "./_utils"; - -const UNWASM_EXTERNAL_PREFIX = "\0unwasm:external:"; -const UMWASM_HELPERS_ID = "\0unwasm:helpers"; - -export interface UnwasmPluginOptions { - /** - * Direct import the wasm file instead of bundling, required in Cloudflare Workers - * - * @default false - */ - esmImport?: boolean; - - /** - * Import `.wasm` files using a lazily evaluated promise for compatibility with runtimes without top-level await support - * - * @default false - */ - lazy?: boolean; -} +import { + sha1, + UMWASM_HELPERS_ID, + UNWASM_EXTERNAL_PREFIX, + UnwasmPluginOptions, + WasmAsset, +} from "./shared"; +import { getPluginUtils, getWasmBinding } from "./runtime"; const unplugin = createUnplugin((opts) => { - type WasmAsset = { - name: string; - source: Buffer; - }; - const assets: Record = Object.create(null); - return { name: "unwasm", rollup: { @@ -78,9 +60,9 @@ const unplugin = createUnplugin((opts) => { } const source = await fs.readFile(id); const name = `wasm/${basename(id, ".wasm")}-${sha1(source)}.wasm`; - assets[id] = { name, source }; + assets[id] = { name, id, source }; // TODO: Can we parse wasm to extract exports and avoid syntheticNamedExports? - return `export default "WASM";`; // dummy + return `export default "UNWASM DUMMY EXPORT";`; }, transform(_code, id) { if (!id.endsWith(".wasm")) { @@ -91,46 +73,8 @@ const unplugin = createUnplugin((opts) => { return; } - const envCode: string = opts.esmImport - ? ` -async function _instantiate(imports) { - const _mod = await import("${UNWASM_EXTERNAL_PREFIX}${id}").then(r => r.default || r); - return WebAssembly.instantiate(_mod, imports) -} -` - : ` -import { base64ToUint8Array } from "${UMWASM_HELPERS_ID}"; - -function _instantiate(imports) { - const _mod = base64ToUint8Array("${asset.source.toString("base64")}") - return WebAssembly.instantiate(_mod, imports) -} - `; - - const code = `${envCode} -const _defaultImports = Object.create(null); - -// TODO: For testing only -Object.assign(_defaultImports, { env: { "seed": () => () => Date.now() * Math.random() } }) - -const instancePromises = new WeakMap(); -function instantiate(imports = _defaultImports) { - let p = instancePromises.get(imports); - if (!p) { - p = _instantiate(imports); - instancePromises.set(imports, p); - } - return p; -} - -const _instance = instantiate(); -const _exports = _instance.then(r => r?.instance?.exports || r?.exports || r); - -export default ${opts.lazy ? "" : "await "} _exports; - `; - return { - code, + code: getWasmBinding(asset, opts), map: { mappings: "" }, syntheticNamedExports: true, }; @@ -147,7 +91,6 @@ export default ${opts.lazy ? "" : "await "} _exports; ) || !code.includes(UNWASM_EXTERNAL_PREFIX) ) { - console.log(chunk); return; } const s = new MagicString(code); @@ -190,20 +133,6 @@ export default ${opts.lazy ? "" : "await "} _exports; }; }); -export function getPluginUtils() { - return ` -export function base64ToUint8Array(str) { - const data = atob(str); - const size = data.length; - const bytes = new Uint8Array(size); - for (let i = 0; i < size; i++) { - bytes[i] = data.charCodeAt(i); - } - return bytes; -} - `; -} - const rollup = unplugin.rollup as (opts: UnwasmPluginOptions) => RollupPlugin; export default { diff --git a/src/plugin/runtime.ts b/src/plugin/runtime.ts new file mode 100644 index 0000000..7a87a3e --- /dev/null +++ b/src/plugin/runtime.ts @@ -0,0 +1,117 @@ +import { + UMWASM_HELPERS_ID, + UNWASM_EXTERNAL_PREFIX, + WasmAsset, + UnwasmPluginOptions, +} from "./shared"; + +// https://marketplace.visualstudio.com/items?itemName=Tobermory.es6-string-html +const js = String.raw; + +export function getWasmBinding(asset: WasmAsset, opts: UnwasmPluginOptions) { + const envCode: string = opts.esmImport + ? js` +async function _instantiate(imports) { + const _mod = await import("${UNWASM_EXTERNAL_PREFIX}${asset.id}").then(r => r.default || r); + return WebAssembly.instantiate(_mod, imports) +} + ` + : js` +import { base64ToUint8Array } from "${UMWASM_HELPERS_ID}"; + +function _instantiate(imports) { + const _data = base64ToUint8Array("${asset.source.toString("base64")}") + return WebAssembly.instantiate(_data, imports) +} + `; + + return js` +import { createUnwasmModule } from "${UMWASM_HELPERS_ID}"; +${envCode} +const _mod = createUnwasmModule(_instantiate); + +export const $init = _mod.$init.bind(_mod); +export const exports = _mod; + +export default _mod; +`; +} + +export function getPluginUtils() { + return js` +export function debug(...args) { + console.log('[wasm]', ...args); +} + +function getExports(input) { + return input?.instance?.exports || input?.exports || input; +} + +export function base64ToUint8Array(str) { + const data = atob(str); + const size = data.length; + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) { + bytes[i] = data.charCodeAt(i); + } + return bytes; +} + +export function createUnwasmModule(_instantiator) { + const _exports = Object.create(null); + let _loaded; + let _promise; + + const $init = (imports) => { + if (_loaded) { + return Promise.resolve(_proxy); + } + if (_promise) { + return _promise; + } + return _promise = _instantiator(imports) + .then(r => { + Object.assign(_exports, getExports(r)); + _loaded = true; + _promise = undefined; + return _proxy; + }) + .catch(error => { + _promise = undefined; + console.error('[wasm]', error); + throw error; + }); + } + + const $exports = new Proxy(_exports, { + get(_, prop) { + if (_loaded) { + return _exports[prop]; + } + return (...args) => { + return _loaded + ? _exports[prop]?.(...args) + : $init().then(() => $exports[prop]?.(...args)); + }; + }, + }); + + const _instance = { + $init, + $exports, + }; + + const _proxy = new Proxy(_instance, { + get(_, prop) { + // Reserve all to avoid future breaking changes + if (prop.startsWith('$')) { + return _instance[prop]; + } + return $exports[prop]; + } + }); + + return _proxy; +} + `; +} diff --git a/src/plugin/shared.ts b/src/plugin/shared.ts new file mode 100644 index 0000000..d3ca71e --- /dev/null +++ b/src/plugin/shared.ts @@ -0,0 +1,30 @@ +import { createHash } from "node:crypto"; + +export interface UnwasmPluginOptions { + /** + * Direct import the wasm file instead of bundling, required in Cloudflare Workers + * + * @default false + */ + esmImport?: boolean; + + /** + * Import `.wasm` files using a lazily evaluated promise for compatibility with runtimes without top-level await support + * + * @default false + */ + lazy?: boolean; +} + +export type WasmAsset = { + id: string; + name: string; + source: Buffer; +}; + +export const UNWASM_EXTERNAL_PREFIX = "\0unwasm:external:"; +export const UMWASM_HELPERS_ID = "\0unwasm:helpers"; + +export function sha1(source: Buffer) { + return createHash("sha1").update(source).digest("hex").slice(0, 16); +} diff --git a/test/fixture/_imports.mjs b/test/fixture/_imports.mjs new file mode 100644 index 0000000..800c417 --- /dev/null +++ b/test/fixture/_imports.mjs @@ -0,0 +1,6 @@ +export const imports = { + env: { + // eslint-disable-next-line unicorn/consistent-function-scoping + seed: () => () => Math.random() * Date.now(), + }, +}; diff --git a/test/fixture/dynamic-import.mjs b/test/fixture/dynamic-import.mjs new file mode 100644 index 0000000..b74015c --- /dev/null +++ b/test/fixture/dynamic-import.mjs @@ -0,0 +1,9 @@ +import { imports } from "./_imports.mjs"; + +const { rand } = await import("@fixture/wasm/index.wasm").then((r) => + r.$init(imports), +); + +export function test() { + return rand(0, 1000) > 0 ? "OK" : "FALED"; +} diff --git a/test/fixture/index.mjs b/test/fixture/index.mjs deleted file mode 100644 index 4256f37..0000000 --- a/test/fixture/index.mjs +++ /dev/null @@ -1 +0,0 @@ -export { rand } from "@fixture/wasm/index.wasm"; diff --git a/test/fixture/static-import.mjs b/test/fixture/static-import.mjs new file mode 100644 index 0000000..9dd84c6 --- /dev/null +++ b/test/fixture/static-import.mjs @@ -0,0 +1,8 @@ +import { imports } from "./_imports.mjs"; +import { rand, $init } from "@fixture/wasm/index.wasm"; + +await $init(imports); + +export function test() { + return rand(0, 1000) > 0 ? "OK" : "FALED"; +} diff --git a/test/plugin.test.ts b/test/plugin.test.ts index cec1488..399ab68 100644 --- a/test/plugin.test.ts +++ b/test/plugin.test.ts @@ -9,39 +9,40 @@ import unwasm from "../src/plugin"; const r = (p: string) => fileURLToPath(new URL(p, import.meta.url)); -const entry = r("fixture/index.mjs"); - await rm(r(".tmp"), { recursive: true }).catch(() => {}); describe("plugin:rollup-inline", () => { it("works", async () => { const build = await rollup({ - input: entry, + input: r("fixture/static-import.mjs"), plugins: [rollupNodeResolve({}), unwasm.rollup({})], }); const { output } = await build.write({ format: "esm", - entryFileNames: "[name].mjs", + entryFileNames: "index.mjs", + chunkFileNames: "[name].mjs", dir: r(".tmp/rollup-inline"), }); const code = output[0].code; - const mod = await evalModule(code, { url: entry }); - expect(mod.rand(1, 1000)).toBeGreaterThan(0); + const mod = await evalModule(code, { url: r("fixture/static-import.mjs") }); + expect(mod.test()).toBe("OK"); }); }); describe("plugin:rollup-esm", () => { it("works", async () => { const build = await rollup({ - input: entry, + input: r("fixture/dynamic-import.mjs"), plugins: [rollupNodeResolve({}), unwasm.rollup({ esmImport: true })], }); const { output } = await build.write({ format: "esm", - entryFileNames: "[name].mjs", + entryFileNames: "index.mjs", + chunkFileNames: "[name].mjs", dir: r(".tmp/rollup-esm"), }); - const code = output[0].code; + + const code = (output[1] && "code" in output[1] && output[1].code) || ""; const esmImport = code.match(/["'](.+wasm)["']/)?.[1]; expect(esmImport).match(/\.\/wasm\/index-[\da-f]+\.wasm/); expect(existsSync(r(`.tmp/rollup-esm/${esmImport}`))).toBe(true); @@ -52,16 +53,17 @@ describe("plugin:rollup-esm", () => { modulesRules: [{ type: "CompiledWasm", include: ["**/*.wasm"] }], scriptPath: r(".tmp/rollup-esm/_mf.mjs"), script: ` - import * as _wasm from "./index.mjs"; + import { test } from "./index.mjs"; export default { async fetch(request, env, ctx) { - return new Response("" + _wasm.rand(1, 1000)); + return new Response(test()); } } `, }); const res = await mf.dispatchFetch("http://localhost"); - expect(Number.parseInt(await res.text())).toBeGreaterThan(0); + const resText = await res.text(); + expect(resText).toBe("OK"); await mf.dispose(); }); }); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..c5ebddd --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + forceRerunTriggers: [ + "**/package.json/**", + "**/vitest.config.*/**", + "**/vite.config.*/**", + "**/fixture/**", + ], + }, +});