Skip to content

Commit

Permalink
perf: Improve network request performance (#7636)
Browse files Browse the repository at this point in the history 
* perf: Improve fetch

* CHANGELOG

* lifo
  • Loading branch information
@mei23 @atsu1125
mei23 authored and atsu1125 committed Feb 19, 2024
1 parent a368204 commit b4e19fb
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 40 deletions.
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,8 @@
"bcryptjs": "2.4.3",
"blurhash": "1.1.3",
"bootstrap-icons-vue": "0.8.0",
"bull": "4.10.1",
"bull": "4.10.4",
"cacheable-lookup": "6.0.0",
"cafy": "15.2.1",
"cbor": "8.1.0",
"chalk": "4.1.2",
Expand Down Expand Up @@ -157,6 +158,7 @@
"gulp-terser": "2.1.0",
"gulp-typescript": "6.0.0-alpha.1",
"hard-source-webpack-plugin": "0.13.1",
"hpagent": "0.1.2",
"html-minifier": "4.0.0",
"http-proxy-agent": "5.0.0",
"http-signature": "1.3.6",
Expand All @@ -182,7 +184,6 @@
"koa-slow": "2.1.0",
"koa-views": "8.0.0",
"langmap": "0.0.16",
"lookup-dns-cache": "2.1.0",
"markdown-it": "12.3.2",
"markdown-it-anchor": "8.4.1",
"matter-js": "0.18.0",
Expand Down
9 changes: 0 additions & 9 deletions src/@types/lookup-dns-cache.d.ts
View file

This file was deleted.

33 changes: 27 additions & 6 deletions src/misc/fetch.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import * as http from 'http';
import * as https from 'https';
import * as cache from 'lookup-dns-cache';
import CacheableLookup from 'cacheable-lookup';
import fetch, { HeadersInit } from 'node-fetch';
import { HttpProxyAgent } from 'http-proxy-agent';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import config from '../config';
import { URL } from 'url';

Expand Down Expand Up @@ -49,6 +48,12 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout =
return await res.text();
}

const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});

/**
* Get http non-proxy agent
*/
Expand All @@ -65,20 +70,36 @@ const _https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
});
} as https.AgentOptions);

const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);

/**
* Get http proxy or non-proxy agent
*/
export const httpAgent = config.proxy
? new HttpProxyAgent(config.proxy)
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _http;

/**
* Get https proxy or non-proxy agent
*/
export const httpsAgent = config.proxy
? new HttpsProxyAgent(config.proxy)
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _https;

/**
Expand Down
74 changes: 51 additions & 23 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,11 @@
resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==

"@ioredis/commands@^1.1.1":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==

"@jridgewell/gen-mapping@^0.3.0":
version "0.3.2"
resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9"
Expand Down Expand Up @@ -1982,13 +1987,6 @@ async-settle@^1.0.0:
dependencies:
async-done "^1.2.2"

async@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/async/-/async-2.6.0.tgz#61a29abb6fcc026fea77e56d1c6ec53a795951f4"
integrity sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==
dependencies:
lodash "^4.14.0"

async@>=0.2.9:
version "3.2.3"
resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"
Expand Down Expand Up @@ -2296,7 +2294,7 @@ bufferutil@^4.0.1:
dependencies:
node-gyp-build "~3.7.0"

bull@*, bull@4.10.1:
bull@*:
version "4.10.1"
resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f"
integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g==
Expand All @@ -2311,6 +2309,20 @@ bull@*, bull@4.10.1:
semver "^7.3.2"
uuid "^8.3.0"

bull@4.10.4:
version "4.10.4"
resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.4.tgz#db39ee0c3bfbe3b76f1f35db800501de5bba4f84"
integrity sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==
dependencies:
cron-parser "^4.2.1"
debuglog "^1.0.0"
get-port "^5.1.1"
ioredis "^5.0.0"
lodash "^4.17.21"
msgpackr "^1.5.2"
semver "^7.3.2"
uuid "^8.3.0"

busboy@^1.0.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893"
Expand Down Expand Up @@ -2393,6 +2405,11 @@ cache-content-type@^1.0.0:
mime-types "^2.1.18"
ylru "^1.2.0"

cacheable-lookup@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-6.0.0.tgz#6fd7e364a0929ee50af00843aaf6e31b9b9f200e"
integrity sha512-5qeyMn8/BERrUPdIfcOLkdMrwltVbxIpgnYM61OLWOg3BuSSh9HrkUtTTRxYthQpBrocvYqD0tJ7vU0y6T7OWw==

cacheable-lookup@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3"
Expand Down Expand Up @@ -3475,6 +3492,11 @@ denque@^1.1.0, denque@^1.5.0:
resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==

denque@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==

depd@^1.1.2, depd@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
Expand Down Expand Up @@ -5056,6 +5078,11 @@ hosted-git-info@^2.1.4:
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==

hpagent@0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9"
integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==

hpagent@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.1.tgz#66f67f16e5c7a8b59a068e40c2658c2c749ad5e2"
Expand Down Expand Up @@ -5388,6 +5415,21 @@ ioredis@^4.28.5:
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"

ioredis@^5.0.0:
version "5.3.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7"
integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==
dependencies:
"@ioredis/commands" "^1.1.1"
cluster-key-slot "^1.1.0"
debug "^4.3.4"
denque "^2.1.0"
lodash.defaults "^4.2.0"
lodash.isarguments "^3.1.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
standard-as-callback "^2.1.0"

ip-address@^7.1.0:
version "7.1.0"
resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-7.1.0.tgz#4a9c699e75b51cbeb18b38de8ed216efa1a490c5"
Expand Down Expand Up @@ -6514,7 +6556,7 @@ lodash.uniq@^4.5.0:
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=

lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21:
lodash@^4.15.0, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
Expand All @@ -6527,15 +6569,6 @@ log-symbols@4.1.0:
chalk "^4.1.0"
is-unicode-supported "^0.1.0"

lookup-dns-cache@2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/lookup-dns-cache/-/lookup-dns-cache-2.1.0.tgz#6362340e269071e20b6f0bcf51da98873411e051"
integrity sha512-tLcJ7rkqWzZ77D7pN5R2ceWKZsIJ5/6HaLQdmhw3M9fBQQmqS4LZqvBcstKzQ6kuZet5LY4TWTiShx7QmO+Q8w==
dependencies:
async "2.6.0"
lodash "^4.17.10"
rr "0.1.0"

lower-case@^1.1.1:
version "1.1.4"
resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac"
Expand Down Expand Up @@ -9172,11 +9205,6 @@ rndstr@1.0.0:
rangestr "0.0.1"
seedrandom "2.4.2"

rr@0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/rr/-/rr-0.1.0.tgz#a18ec25ec94a67c35f210bb3a85d17914e79cd1e"
integrity sha1-oY7CXslKZ8NfIQuzqF0XkU55zR4=

run-parallel@^1.1.9:
version "1.2.0"
resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
Expand Down

0 comments on commit b4e19fb

Please sign in to comment.