Optimize time-consuming tasks

package.json

@@ -4,7 +4,7 @@
   "description": "eip-4844 blobs upload sdk",
   "main": "dist/index.cjs.js",
   "module": "dist/index.esm.js",
-  "types": "types/index.d.ts",
+  "types": "index.d.ts",
   "exports": {
     ".": {
       "require": "./dist/index.cjs.js",
@@ -15,13 +15,15 @@
     }
   },
   "scripts": {
-    "build": "rollup -c"
+    "build": "rollup -c",
+    "prepublishOnly": "npm run build"
   },
   "dependencies": {
     "async-mutex": "^0.5.0",
     "dotenv": "^16.4.5",
     "ethers": "^6.13.1",
-    "kzg-wasm": "^0.4.0"
+    "kzg-wasm": "^0.4.0",
+    "workerpool": "^9.1.3"
   },
   "repository": {
     "type": "git",

rollup.config.mjs

@@ -17,17 +17,27 @@ export default [
             }
         ],
         plugins: [commonjs(), resolve()],
-        external: ["ethers", "kzg-wasm"]
+        external: ["ethers", "kzg-wasm", "workerpool"]
     },
     {
         input: 'src/node/file.js',
         output: {
             file: 'dist/file.cjs.js',
             format: 'cjs',
-            sourcemap: true
+            sourcemap: true,
         },
         plugins: [commonjs(), resolve()],
         external: ["ethers"]
     },
+    {
+        input: 'src/worker/worker.js',
+        output: {
+            file: 'dist/worker.cjs.js',
+            format: 'cjs',
+            sourcemap: true,
+        },
+        plugins: [commonjs(), resolve()],
+        external: ["kzg-wasm", "workerpool"]
+    },
 ];
 

src/flatdirectory.js

@@ -11,11 +11,14 @@ import {
 } from './param';
 import {
     BlobUploader, encodeBlobs,
-    getChainId, getFileChunk,
+    getChainId, getFileChunk, getHash,
     isBuffer, isFile,
     stringToHex
 } from "./utils";
 
+import workerpool from 'workerpool';
+const pool = workerpool.pool(__dirname + '/worker.cjs.js');
+
 const REMOVE_FAIL = -1;
 const REMOVE_NORMAL = 0;
 const REMOVE_SUCCESS = 1;
@@ -132,7 +135,7 @@ export class FlatDirectory {
         const provider = new ethers.JsonRpcProvider(this.#ethStorageRpc);
         const contract = new ethers.Contract(this.#contractAddr, FlatDirectoryAbi, provider);
         try {
-            const blobCount = await contract.countChunks(hexName);
+            const blobCount = await this.#countChunks(contract, hexName);
             for (let i = 0; i < blobCount; i++) {
                 const result = await contract.readChunk(hexName, i);
                 const chunk = ethers.getBytes(result[0]);
@@ -210,7 +213,7 @@ export class FlatDirectory {
         let gasLimit = 0;
         const [cost, oldChunkLength, maxFeePerBlobGas, gasFeeData] = await Promise.all([
             fileContract.upfrontPayment(),
-            fileContract.countChunks(hexName),
+            this.#countChunks(fileContract, hexName),
             this.#blobUploader.getBlobGasPrice(),
             this.#blobUploader.getGasPrice(),
         ]);
@@ -222,19 +225,19 @@ export class FlatDirectory {
             const blobArr = encodeBlobs(data);
             const chunkIdArr = [];
             const chunkSizeArr = [];
-            const blobHashArr = [];
             const blobHashRequestArr = [];
             for (let j = 0; j < blobArr.length; j++) {
                 chunkIdArr.push(i + j);
                 chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE);
 
-                blobHashArr.push(this.#blobUploader.getBlobHash(blobArr[j]));
                 blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j));
             }
 
+            let blobHashArr;
             // check change
             if (chunkIdArr[0] < oldChunkLength) {
-                const isChange = await this.#checkChange(fileContract, blobHashArr, blobHashRequestArr);
+                blobHashArr = await this.#getBlobHashes(blobArr);
+                const isChange = await this.#checkChange(blobHashArr, blobHashRequestArr);
                 if (!isChange) {
                     continue;
                 }
@@ -246,6 +249,7 @@ export class FlatDirectory {
             totalStorageCost += value;
             // gas cost
             if (gasLimit === 0) {
+                blobHashArr = blobHashArr ? blobHashArr : await this.#getBlobHashes(blobArr);
                 gasLimit = await fileContract.writeChunks.estimateGas(hexName, chunkIdArr, chunkSizeArr, {
                     value: value,
                     blobVersionedHashes: blobHashArr
@@ -317,7 +321,7 @@ export class FlatDirectory {
         }
 
         const [oldChunkLength, gasFeeData] = await Promise.all([
-            fileContract.countChunks(hexName),
+            this.#countChunks(fileContract, hexName),
             this.#blobUploader.getGasPrice(),
         ]);
 
@@ -398,7 +402,7 @@ export class FlatDirectory {
         // check old data
         const [cost, oldBlobLength] = await Promise.all([
             fileContract.upfrontPayment(),
-            fileContract.countChunks(hexName),
+            this.#countChunks(fileContract, hexName)
         ]);
         const clearState = await this.#clearOldFile(hexName, blobLength, oldBlobLength);
         if (clearState === REMOVE_FAIL) {
@@ -414,7 +418,6 @@ export class FlatDirectory {
             const blobArr = encodeBlobs(data);
             const chunkIdArr = [];
             const chunkSizeArr = [];
-            const blobHashArr = [];
             const blobHashRequestArr = [];
             for (let j = 0; j < blobArr.length; j++) {
                 chunkIdArr.push(i + j);
@@ -424,14 +427,15 @@ export class FlatDirectory {
                 } else {
                     chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE);
                 }
-                blobHashArr.push(this.#blobUploader.getBlobHash(blobArr[j]));
                 blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j));
             }
+            const blobCommitmentArr = await this.#getBlobCommitments(blobArr);
 
             // check change
             if (clearState === REMOVE_NORMAL) {
                 try {
-                    const isChange = await this.#checkChange(fileContract, blobHashArr, blobHashRequestArr);
+                    const blobHashArr = this.#getHashes(blobCommitmentArr);
+                    const isChange = await this.#checkChange(blobHashArr, blobHashRequestArr);
                     if (!isChange) {
                         callback.onProgress(chunkIdArr[chunkIdArr.length - 1], blobLength, false);
                         continue;
@@ -444,7 +448,7 @@ export class FlatDirectory {
 
             // upload
             try {
-                const status = await this.#uploadBlob(fileContract, key, hexName, blobArr, chunkIdArr, chunkSizeArr, cost, gasIncPct);
+                const status = await this.#uploadBlob(fileContract, key, hexName, blobArr, blobCommitmentArr, chunkIdArr, chunkSizeArr, cost, gasIncPct);
                 if (!status) {
                     callback.onFail(new Error("FlatDirectory: Sending transaction failed."));
                     break;
@@ -520,7 +524,7 @@ export class FlatDirectory {
         }
 
         // check old data
-        const oldChunkLength = await fileContract.countChunks(hexName);
+        const oldChunkLength = await this.#countChunks(fileContract, hexName);
         const clearState = await this.#clearOldFile(hexName, chunkLength, oldChunkLength);
         if (clearState === REMOVE_FAIL) {
             callback.onFail(new Error(`FlatDirectory: Failed to delete old data!`));
@@ -586,19 +590,17 @@ export class FlatDirectory {
         }
     }
 
-    async #checkChange(fileContract, blobHashArr, blobHashRequestArr) {
-        let hasChange = false;
+    async #checkChange(blobHashArr, blobHashRequestArr) {
         const dataHashArr = await Promise.all(blobHashRequestArr);
         for (let i = 0; i < blobHashArr.length; i++) {
             if (blobHashArr[i] !== dataHashArr[i]) {
-                hasChange = true;
-                break;
+                return true;
             }
         }
-        return hasChange;
+        return false;
     }
 
-    async #uploadBlob(fileContract, key, hexName, blobArr, chunkIdArr, chunkSizeArr, cost, gasIncPct) {
+    async #uploadBlob(fileContract, key, hexName, blobArr, blobCommitmentArr, chunkIdArr, chunkSizeArr, cost, gasIncPct) {
         // create tx
         const value = cost * BigInt(blobArr.length);
         const tx = await fileContract.writeChunks.populateTransaction(hexName, chunkIdArr, chunkSizeArr, {
@@ -615,8 +617,8 @@ export class FlatDirectory {
             tx.maxFeePerBlobGas = blobGas * BigInt(100 + gasIncPct) / BigInt(100);
         }
         // send
-        const txResponse = await this.#blobUploader.sendTxLock(tx, blobArr);
-        console.log(`FlatDirectory: The ${chunkIdArr} chunks hash is ${txResponse.hash}`, key);
+        const txResponse = await this.#blobUploader.sendTxLock(tx, blobArr, blobCommitmentArr);
+        console.log(`FlatDirectory: The ${chunkIdArr} chunks hash is ${txResponse.hash}`, "", key);
         const txReceipt = await txResponse.wait();
         return txReceipt && txReceipt.status;
     }
@@ -637,8 +639,31 @@ export class FlatDirectory {
 
         // send
         const txResponse = await this.#blobUploader.sendTxLock(tx);
-        console.log(`FlatDirectory: The ${chunkId} chunk hash is ${txResponse.hash}`, key);
+        console.log(`FlatDirectory: The ${chunkId} chunk hash is ${txResponse.hash}`, "", key);
         const txReceipt = await txResponse.wait();
         return txReceipt && txReceipt.status;
     }
+
+    async #countChunks(fileContract, hexName) {
+        const count = await fileContract.countChunks(hexName);
+        // Bigint to number
+        return Number(count);
+    }
+
+    async #getBlobCommitments(blobArr) {
+        const isNode = typeof process !== 'undefined' && !!process.versions && !!process.versions.node;
+        const promises = isNode
+            ? blobArr.map(blob => pool.exec('getCommitment', [blob]))
+            : blobArr.map(blob => this.#blobUploader.getCommitment(blob));
+        return await Promise.all(promises);
+    }
+
+    #getHashes(blobCommitmentArr) {
+        return blobCommitmentArr.map(comment => getHash(comment));
+    }
+
+    async #getBlobHashes(blobArr) {
+        const commitments = await this.#getBlobCommitments(blobArr);
+        return this.#getHashes(commitments);
+    }
 }

src/utils/uploader.js

@@ -1,18 +1,7 @@
 import {ethers} from "ethers";
 import {loadKZG} from 'kzg-wasm';
 import {Mutex} from 'async-mutex';
-
-function computeVersionedHash(commitment, blobCommitmentVersion) {
-    const computedVersionedHash = new Uint8Array(32);
-    computedVersionedHash.set([blobCommitmentVersion], 0);
-    const hash = ethers.getBytes(ethers.sha256(commitment));
-    computedVersionedHash.set(hash.subarray(1), 1);
-    return computedVersionedHash;
-}
-
-function commitmentsToVersionedHashes(commitment) {
-    return computeVersionedHash(commitment, 0x01);
-}
+import {getHash, commitmentsToVersionedHashes} from "./util";
 
 // blob gas price
 const MIN_BLOB_GASPRICE = 1n;
@@ -80,7 +69,7 @@ export class BlobUploader {
         return null;
     }
 
-    async sendTx(tx, blobs) {
+    async sendTx(tx, blobs = null, commitments = null) {
         if (!blobs) {
             return await this.#wallet.sendTransaction(tx);
         }
@@ -95,7 +84,7 @@ export class BlobUploader {
         const versionedHashes = [];
         for (let i = 0; i < blobs.length; i++) {
             const blob = blobs[i];
-            const commitment = kzg.blobToKzgCommitment(blob);
+            const commitment = (commitments && commitments.length > i) ? commitments[i] : kzg.blobToKzgCommitment(blob);
             const proof = kzg.computeBlobKzgProof(blob, commitment);
             ethersBlobs.push({
                 data: blob,
@@ -115,21 +104,21 @@ export class BlobUploader {
         return await this.#wallet.sendTransaction(tx);
     }
 
-    async sendTxLock(tx, blobs) {
+    async sendTxLock(tx, blobs = null, commitments = null) {
         const release = await this.#mutex.acquire();
         try {
-            return await this.sendTx(tx, blobs);
+            return await this.sendTx(tx, blobs, commitments);
         } finally {
             release();
         }
     }
 
+    getCommitment(blob) {
+        return this.#kzg.blobToKzgCommitment(blob);
+    }
+
     getBlobHash(blob) {
-        const kzg = this.#kzg;
-        const commit = kzg.blobToKzgCommitment(blob);
-        const localHash = commitmentsToVersionedHashes(commit);
-        const hash = new Uint8Array(32);
-        hash.set(localHash.subarray(0, 32 - 8));
-        return ethers.hexlify(hash);
+        const commit = this.getCommitment(blob);
+        return getHash(commit);
     }
 }

src/utils/util.js

@@ -25,3 +25,22 @@ export function isFile(content) {
         content.isNodeJs;
     return (content instanceof File) || isNodeFile;
 }
+
+function computeVersionedHash(commitment, blobCommitmentVersion) {
+    const computedVersionedHash = new Uint8Array(32);
+    computedVersionedHash.set([blobCommitmentVersion], 0);
+    const hash = ethers.getBytes(ethers.sha256(commitment));
+    computedVersionedHash.set(hash.subarray(1), 1);
+    return computedVersionedHash;
+}
+
+export function commitmentsToVersionedHashes(commitment) {
+    return computeVersionedHash(commitment, 0x01);
+}
+
+export function getHash(commit) {
+    const localHash = commitmentsToVersionedHashes(commit);
+    const hash = new Uint8Array(32);
+    hash.set(localHash.subarray(0, 32 - 8));
+    return ethers.hexlify(hash);
+}

src/worker/worker.js

@@ -0,0 +1,20 @@
+import workerpool from 'workerpool';
+import {loadKZG} from 'kzg-wasm';
+
+let kzgInstance = null;
+
+async function initializeKzg() {
+    if (!kzgInstance) {
+        kzgInstance = await loadKZG();
+    }
+    return kzgInstance;
+}
+
+async function getCommitment(blob) {
+    const kzg = await initializeKzg();
+    return kzg.blobToKzgCommitment(blob);
+}
+
+workerpool.worker({
+    getCommitment: getCommitment,
+});

yarn.lock

@@ -583,6 +583,11 @@ undici-types@~5.26.4:
   resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
+workerpool@^9.1.3:
+  version "9.1.3"
+  resolved "https://registry.npmjs.org/workerpool/-/workerpool-9.1.3.tgz#34b81f50f777a0e549c6dfaa0926575735e3f4b4"
+  integrity sha512-LhUrk4tbxJRDQmRrrFWA9EnboXI79fe0ZNTy3u8m+dqPN1EkVSIsQYAB8OF/fkyhG8Rtup+c/bzj/+bzbG8fqg==
+
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"