From 3456c69744907cea8d00d0c9296ecaaf65da9537 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Wed, 6 Sep 2023 12:41:27 -0400 Subject: [PATCH] feat(emscripten): load and decode .wasm.zstd --- package-lock.json | 27 ++++++++++++++----- package.json | 3 ++- src/bindgen/typescript/typescript-bindings.js | 1 + src/build-emscripten.js | 7 +++++ .../internal/loadEmscriptenModuleWebWorker.ts | 13 +++++++-- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 19b2c5845..9d42380fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@babel/runtime": "^7.15.4", + "@thewtex/zstddec": "^0.1.1", "@types/emscripten": "^1.39.6", "axios": "^1.4.0", "commander": "^9.4.0", @@ -3186,6 +3187,11 @@ "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==", "dev": true }, + "node_modules/@thewtex/zstddec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@thewtex/zstddec/-/zstddec-0.1.1.tgz", + "integrity": "sha512-evgVH1iKPzR4s2tqZNNQvoZXLrzFocm58VEnje/8f63GO8OxY/XXJRJXG+zfnA8GcLTpxAZvpaAvfwL5OGuPGQ==" + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -5268,9 +5274,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001388", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001388.tgz", - "integrity": "sha512-znVbq4OUjqgLxMxoNX2ZeeLR0d7lcDiE5uJ4eUiWdml1J1EkxbnQq6opT9jb9SMfJxB0XA16/ziHwni4u1I3GQ==", + "version": "1.0.30001527", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001527.tgz", + "integrity": "sha512-YkJi7RwPgWtXVSgK4lG9AHH57nSzvvOp9MesgXmw4Q7n0C3H04L0foHqfxcmSAm5AcWb8dW9AYj2tR7/5GnddQ==", "dev": true, "funding": [ { @@ -5280,6 +5286,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, @@ -23013,6 +23023,11 @@ "integrity": "sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==", "dev": true }, + "@thewtex/zstddec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@thewtex/zstddec/-/zstddec-0.1.1.tgz", + "integrity": "sha512-evgVH1iKPzR4s2tqZNNQvoZXLrzFocm58VEnje/8f63GO8OxY/XXJRJXG+zfnA8GcLTpxAZvpaAvfwL5OGuPGQ==" + }, "@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -24592,9 +24607,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001388", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001388.tgz", - "integrity": "sha512-znVbq4OUjqgLxMxoNX2ZeeLR0d7lcDiE5uJ4eUiWdml1J1EkxbnQq6opT9jb9SMfJxB0XA16/ziHwni4u1I3GQ==", + "version": "1.0.30001527", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001527.tgz", + "integrity": "sha512-YkJi7RwPgWtXVSgK4lG9AHH57nSzvvOp9MesgXmw4Q7n0C3H04L0foHqfxcmSAm5AcWb8dW9AYj2tR7/5GnddQ==", "dev": true }, "cardinal": { diff --git a/package.json b/package.json index 187a8e24e..6aec2814d 100644 --- a/package.json +++ b/package.json @@ -167,7 +167,8 @@ "markdown-table": "^3.0.3", "mime-types": "^2.1.35", "wasm-feature-detect": "^1.5.1", - "webworker-promise": "^0.4.2" + "webworker-promise": "^0.4.2", + "@thewtex/zstddec": "^0.1.1" }, "bin": { "itk-wasm": "./src/itk-wasm-cli.js" diff --git a/src/bindgen/typescript/typescript-bindings.js b/src/bindgen/typescript/typescript-bindings.js index a0dee3057..b1ceb3ac4 100644 --- a/src/bindgen/typescript/typescript-bindings.js +++ b/src/bindgen/typescript/typescript-bindings.js @@ -87,6 +87,7 @@ function typescriptBindings (outputDir, buildDir, wasmBinaries, options, forNode if (err.code !== 'EEXIST') throw err } fs.copyFileSync(wasmBinaryRelativePath, path.join(distPipelinesDir, path.basename(wasmBinaryRelativePath))) + fs.copyFileSync(`${wasmBinaryRelativePath}.zstd`, path.join(distPipelinesDir, `${path.basename(wasmBinaryRelativePath)}.zstd`)) const prefix = wasmBinaryRelativePath.substring(0, wasmBinaryRelativePath.length-5) fs.copyFileSync(`${prefix}.js`, path.join(distPipelinesDir, `${path.basename(prefix)}.js`)) fs.copyFileSync(`${prefix}.umd.js`, path.join(distPipelinesDir, `${path.basename(prefix)}.umd.js`)) diff --git a/src/build-emscripten.js b/src/build-emscripten.js index cb77ac33f..bde454aaa 100644 --- a/src/build-emscripten.js +++ b/src/build-emscripten.js @@ -140,7 +140,9 @@ if (options.copyBuildArtifacts) { } let imageIOFiles = glob.sync(path.join(buildDir, 'image-io', '*.js')) imageIOFiles = imageIOFiles.concat(glob.sync(path.join(buildDir, 'image-io', '*.wasm'))) + imageIOFiles = imageIOFiles.concat(glob.sync(path.join(buildDir, 'image-io', '*.wasm.zstd'))) imageIOFiles = imageIOFiles.filter((fn) => !fn.endsWith('.umd.wasm')) + imageIOFiles = imageIOFiles.filter((fn) => !fn.endsWith('.umd.wasm.zstd')) const copyImageIOModules = function (imageIOFile, callback) { const io = path.basename(imageIOFile) const output = path.join('dist', 'image-io', io) @@ -154,7 +156,9 @@ if (options.copyBuildArtifacts) { } let meshIOFiles = glob.sync(path.join(buildDir, 'mesh-io', '*.js')) meshIOFiles = meshIOFiles.concat(glob.sync(path.join(buildDir, 'mesh-io', '*.wasm'))) + meshIOFiles = meshIOFiles.concat(glob.sync(path.join(buildDir, 'mesh-io', '*.wasm.zstd'))) meshIOFiles = meshIOFiles.filter((fn) => !fn.endsWith('.umd.wasm')) + meshIOFiles = meshIOFiles.filter((fn) => !fn.endsWith('.umd.wasm.zstd')) const copyMeshIOModules = function (meshIOFile, callback) { const io = path.basename(meshIOFile) const output = path.join('dist', 'mesh-io', io) @@ -168,7 +172,9 @@ if (options.copyBuildArtifacts) { } let dicomFiles = glob.sync(path.join(buildDir, 'dicom', '*.js')) dicomFiles = dicomFiles.concat(glob.sync(path.join(buildDir, 'dicom', '*.wasm'))) + dicomFiles = dicomFiles.concat(glob.sync(path.join(buildDir, 'dicom', '*.wasm.zstd'))) dicomFiles = dicomFiles.filter((fn) => !fn.endsWith('.umd.wasm')) + dicomFiles = dicomFiles.filter((fn) => !fn.endsWith('.umd.wasm.zstd')) const copyDICOMModules = function (dicomFile, callback) { const io = path.basename(dicomFile) const output = path.join('dist', 'dicom', 'public', 'pipelines', io) @@ -219,6 +225,7 @@ if (options.buildTestPipelines) { } let pipelineFiles = glob.sync(path.join(pipelinePath, 'emscripten-build', '*.js')) pipelineFiles = pipelineFiles.concat(glob.sync(path.join(pipelinePath, 'emscripten-build', '*.wasm'))) + pipelineFiles = pipelineFiles.concat(glob.sync(path.join(pipelinePath, 'emscripten-build', '*.wasm.zstd'))) pipelineFiles.forEach((file) => { const filename = path.basename(file) const output = path.join('dist', 'pipelines', filename) diff --git a/src/core/internal/loadEmscriptenModuleWebWorker.ts b/src/core/internal/loadEmscriptenModuleWebWorker.ts index c195c4c6d..15f922d3e 100644 --- a/src/core/internal/loadEmscriptenModuleWebWorker.ts +++ b/src/core/internal/loadEmscriptenModuleWebWorker.ts @@ -1,5 +1,9 @@ import axios from 'axios' +import { ZSTDDecoder } from '@thewtex/zstddec' +const decoder = new ZSTDDecoder() +let decoderInitialized = false + import ITKWasmEmscriptenModule from '../ITKWasmEmscriptenModule.js' import camelCase from './camelCase.js' @@ -26,8 +30,13 @@ async function loadEmscriptenModuleWebWorker(moduleRelativePathOrURL: string | U // adds worker dynamic import support: // https://bugzilla.mozilla.org/show_bug.cgi?id=1540913 const wasmBinaryPath = `${modulePrefix}.wasm` - const response = await axios.get(wasmBinaryPath, { responseType: 'arraybuffer' }) - const wasmBinary = response.data + const response = await axios.get(`${wasmBinaryPath}.zstd`, { responseType: 'arraybuffer' }) + if (!decoderInitialized) { + await decoder.init() + decoderInitialized = true + } + const decompressedArray = decoder.decode(new Uint8Array(response.data)) + const wasmBinary = decompressedArray.buffer const modulePath = `${modulePrefix}.umd.js` importScripts(modulePath) const moduleBaseName: string = camelCase(modulePrefix.replace(/.*\//, ''))