diff --git a/.changeset/wet-falcons-talk.md b/.changeset/wet-falcons-talk.md new file mode 100644 index 000000000000..7698788db0e9 --- /dev/null +++ b/.changeset/wet-falcons-talk.md @@ -0,0 +1,5 @@ +--- +'@eth-optimism/data-transport-layer': patch +--- + +improve slow blocking JSON parsing that occurs during l2 sync diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index c959e434e2c1..da0ca5267abf 100644 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -30,6 +30,7 @@ "@sentry/tracing": "^6.3.1", "@types/express": "^4.17.11", "bcfg": "^0.1.6", + "bfj": "^7.0.2", "browser-or-node": "^1.3.0", "cors": "^2.8.5", "dotenv": "^8.2.0", @@ -49,6 +50,7 @@ "@types/levelup": "^4.3.0", "@types/mocha": "^8.2.2", "@types/node-fetch": "^2.5.8", + "@types/workerpool": "^6.0.0", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "hardhat": "^2.2.1", diff --git a/packages/data-transport-layer/src/services/l2-ingestion/service.ts b/packages/data-transport-layer/src/services/l2-ingestion/service.ts index 120bb35ad734..6a32bc301a58 100644 --- a/packages/data-transport-layer/src/services/l2-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l2-ingestion/service.ts @@ -3,6 +3,8 @@ import { BaseService } from '@eth-optimism/common-ts' import { JsonRpcProvider } from '@ethersproject/providers' import { BigNumber } from 'ethers' import { LevelUp } from 'levelup' +import axios from 'axios'; +import bfj from 'bfj'; /* Imports: Internal */ import { TransportDB } from '../../db/transport-db' @@ -168,11 +170,27 @@ export class L2IngestionService extends BaseService { ) }) } else { - blocks = await this.state.l2RpcProvider.send('eth_getBlockRange', [ - toRpcHexString(startBlockNumber), - toRpcHexString(endBlockNumber), - true, - ]) + // This request returns a large response. Parsing it into JSON inside the ethers library is + // quite slow, and can block the event loop for upwards of multiple seconds. When this happens, + // incoming http requests will likely timeout and fail. + // Instead, we will parse the incoming http stream directly with the bfj package, which yields + // the event loop periodically so that we don't fail to serve requests. + const req = { + jsonrpc: '2.0', + method: 'eth_getBlockRange', + params: [ + toRpcHexString(startBlockNumber), + toRpcHexString(endBlockNumber), + true, + ], + id: '1', + }; + + const resp = await axios.post(this.state.l2RpcProvider.connection.url, req, {responseType: 'stream'}); + const respJson = await bfj.parse(resp.data, { + yieldRate: 4096 // this yields abit more often than the default of 16384 + }); + const blocks = respJson.data; } for (const block of blocks) { diff --git a/yarn.lock b/yarn.lock index 678b6dfabc8f..1be52e5f1a8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2601,6 +2601,13 @@ "@types/bn.js" "*" "@types/underscore" "*" +"@types/workerpool@^6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@types/workerpool/-/workerpool-6.0.0.tgz#068c31191f7df9b3d49ebe348b1eeb601e75e2d3" + integrity sha512-BjbKVHFBWblQ3vZ5yFq29kbM2TsaUaTOwYgVxqnNjMrT6CktVF8AvMxOJZgHGgNbAzP4z8DK+EshyZcYpdvAhQ== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "20.2.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9" @@ -3657,6 +3664,16 @@ better-path-resolve@1.0.0: dependencies: is-windows "^1.0.0" +bfj@^7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/bfj/-/bfj-7.0.2.tgz#1988ce76f3add9ac2913fd8ba47aad9e651bfbb2" + integrity sha512-+e/UqUzwmzJamNF50tBV6tZPTORow7gQ96iFow+8b562OdMpEK0BcJEq2OSPEDmAbSMBQ7PKZ87ubFkgxpYWgw== + dependencies: + bluebird "^3.5.5" + check-types "^11.1.1" + hoopy "^0.1.4" + tryer "^1.0.1" + bignumber.js@^9.0.0, bignumber.js@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" @@ -3711,7 +3728,7 @@ blakejs@^1.1.0: resolved "https://registry.yarnpkg.com/blakejs/-/blakejs-1.1.0.tgz#69df92ef953aa88ca51a32df6ab1c54a155fc7a5" integrity sha1-ad+S75U6qIylGjLfarHFShVfx6U= -bluebird@^3.5.0, bluebird@^3.5.2, bluebird@^3.5.3, bluebird@^3.7.2: +bluebird@^3.5.0, bluebird@^3.5.2, bluebird@^3.5.3, bluebird@^3.5.5, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -4190,6 +4207,11 @@ check-error@^1.0.2: resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" integrity sha1-V00xLt2Iu13YkS6Sht1sCu1KrII= +check-types@^11.1.1: + version "11.1.2" + resolved "https://registry.yarnpkg.com/check-types/-/check-types-11.1.2.tgz#86a7c12bf5539f6324eb0e70ca8896c0e38f3e2f" + integrity sha512-tzWzvgePgLORb9/3a0YenggReLKAIb2owL03H2Xdoe5pKcUyWRSEQ8xfCar8t2SIAuEDwtmx2da1YB52YuHQMQ== + checkpoint-store@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/checkpoint-store/-/checkpoint-store-1.1.0.tgz#04e4cb516b91433893581e6d4601a78e9552ea06" @@ -7304,6 +7326,11 @@ home-or-tmp@^2.0.0: os-homedir "^1.0.0" os-tmpdir "^1.0.1" +hoopy@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/hoopy/-/hoopy-0.1.4.tgz#609207d661100033a9a9402ad3dea677381c1b1d" + integrity sha512-HRcs+2mr52W0K+x8RzcLzuPPmVIKMSv97RGHy0Ea9y/mpcaK+xTrjICA04KAHi4GRzxliNqNJEFYWHghy3rSfQ== + hosted-git-info@^2.1.4, hosted-git-info@^2.6.0: version "2.8.8" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" @@ -12743,6 +12770,11 @@ trim-right@^1.0.1: resolved "https://registry.yarnpkg.com/true-case-path/-/true-case-path-2.2.1.tgz#c5bf04a5bbec3fd118be4084461b3a27c4d796bf" integrity sha512-0z3j8R7MCjy10kc/g+qg7Ln3alJTodw9aDuVWZa3uiWqfuBMKeAeP2ocWcxoyM3D73yz3Jt/Pu4qPr4wHSdB/Q== +tryer@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" + integrity sha512-c3zayb8/kWWpycWYg87P71E1S1ZL6b6IJxfb5fvsUgsf0S2MVGaDhDXXjDMpdCpfWXqptc+4mXwmiy1ypXqRAA== + ts-essentials@^1.0.0, ts-essentials@^1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/ts-essentials/-/ts-essentials-1.0.4.tgz#ce3b5dade5f5d97cf69889c11bf7d2da8555b15a"