From cb1afacd7bc71c1d4870503472d94ab5f1cd9be8 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Wed, 5 Jun 2024 19:35:54 +0545 Subject: [PATCH] feat(http): remove `dnsCache` option (#29449) --- lib/types/host-rules.ts | 1 - lib/util/http/dns.spec.ts | 50 -------------- lib/util/http/dns.ts | 111 ------------------------------- lib/util/http/host-rules.spec.ts | 17 ----- lib/util/http/host-rules.ts | 5 -- lib/workers/repository/index.ts | 3 - package.json | 2 - pnpm-lock.yaml | 6 -- 8 files changed, 195 deletions(-) delete mode 100644 lib/util/http/dns.spec.ts delete mode 100644 lib/util/http/dns.ts diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts index 23ac43d7f64cbd..00820c81a80f02 100644 --- a/lib/types/host-rules.ts +++ b/lib/types/host-rules.ts @@ -14,7 +14,6 @@ export interface HostRule { headers?: Record; maxRetryAfter?: number; - dnsCache?: boolean; keepAlive?: boolean; artifactAuth?: string[] | null; httpsCertificateAuthority?: string; diff --git a/lib/util/http/dns.spec.ts b/lib/util/http/dns.spec.ts deleted file mode 100644 index db574bc10e2f2a..00000000000000 --- a/lib/util/http/dns.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { logger } from '../../logger'; -import { clearDnsCache, dnsLookup, printDnsStats } from './dns'; - -describe('util/http/dns', () => { - describe('dnsLookup', () => { - it('works', async () => { - clearDnsCache(); - const ip = await new Promise((resolve) => - dnsLookup('api.github.com', 4, (_e, r, _f) => { - resolve(r); - }), - ); - expect(ip).toBeString(); - // uses cache - expect( - await new Promise((resolve) => - dnsLookup('api.github.com', (_e, r, _f) => { - resolve(r); - }), - ), - ).toBe(ip); - expect( - await new Promise((resolve) => - dnsLookup('api.github.com', {}, (_e, r, _f) => { - resolve(r); - }), - ), - ).toBe(ip); - }); - - it('throws', async () => { - clearDnsCache(); - const ip = new Promise((resolve, reject) => - dnsLookup('api.github.comcccccccc', 4, (_e, r, _f) => { - if (_e) { - reject(_e); - } else { - resolve(r); - } - }), - ); - await expect(ip).rejects.toThrow(); - }); - - it('prints stats', () => { - printDnsStats(); - expect(logger.debug).toHaveBeenCalled(); - }); - }); -}); diff --git a/lib/util/http/dns.ts b/lib/util/http/dns.ts deleted file mode 100644 index 808060f0d0de33..00000000000000 --- a/lib/util/http/dns.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { - LookupAllOptions, - LookupOneOptions, - lookup as _dnsLookup, -} from 'node:dns'; -import type { EntryObject, IPFamily, LookupOptions } from 'cacheable-lookup'; -import { LRUCache } from 'lru-cache'; -import { logger } from '../../logger'; - -const cache = new LRUCache({ max: 1000 }); - -function lookup( - ...[host, options, callback]: - | [ - hostname: string, - family: IPFamily, - callback: ( - error: NodeJS.ErrnoException, - address: string, - family: IPFamily, - ) => void, - ] - | [ - hostname: string, - callback: ( - error: NodeJS.ErrnoException, - address: string, - family: IPFamily, - ) => void, - ] - | [ - hostname: string, - options: LookupOptions & { all: true }, - callback: ( - error: NodeJS.ErrnoException, - result: ReadonlyArray, - ) => void, - ] - | [ - hostname: string, - options: LookupOptions, - callback: ( - error: NodeJS.ErrnoException, - address: string, - family: IPFamily, - ) => void, - ] -): void { - let opts: LookupOneOptions | LookupAllOptions; - // TODO: strict null incompatible types (#22198) - let cb: any; - - if (typeof options === 'function') { - opts = {}; - cb = options; - } else if (typeof options === 'number') { - opts = { family: options }; - cb = callback; - } else { - opts = options; - cb = callback; - } - - // istanbul ignore if: not used - if (opts.all) { - const key = `${host}_all`; - if (cache.has(key)) { - logger.trace({ host }, 'dns lookup cache hit'); - cb(null, cache.get(key)); - return; - } - - _dnsLookup(host, opts, (err, res) => { - if (err) { - logger.debug({ host, err }, 'dns lookup error'); - cb(err, null, null); - return; - } - logger.trace({ host, opts, res }, 'dns lookup'); - cache.set(key, res); - cb(null, res, null); - }); - } else { - if (cache.has(host)) { - logger.trace({ host }, 'dns lookup cache hit'); - cb(null, ...cache.get(host)); - return; - } - - _dnsLookup(host, opts, (err, ...res) => { - if (err) { - logger.debug({ host, err }, 'dns lookup error'); - cb(err); - return; - } - logger.trace({ host, opts, res }, 'dns lookup'); - cache.set(host, res); - cb(null, ...res); - }); - } -} - -export { lookup as dnsLookup }; - -export function printDnsStats(): void { - logger.debug({ hosts: Array.from(cache.keys()) }, 'dns cache'); -} - -export function clearDnsCache(): void { - cache.clear(); -} diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts index 2b452132838578..dcbaa06de70a4e 100644 --- a/lib/util/http/host-rules.spec.ts +++ b/lib/util/http/host-rules.spec.ts @@ -2,7 +2,6 @@ import { GlobalConfig } from '../../config/global'; import { bootstrap } from '../../proxy'; import type { HostRule } from '../../types'; import * as hostRules from '../host-rules'; -import { dnsLookup } from './dns'; import { applyHostRule, findMatchingRule } from './host-rules'; import type { GotOptions } from './types'; @@ -125,22 +124,6 @@ describe('util/http/host-rules', () => { }); }); - it('uses dnsCache', () => { - hostRules.add({ dnsCache: true }); - - const opts = { ...options, token: 'xxx' }; - const hostRule = findMatchingRule(url, opts); - expect(hostRule).toEqual({ - dnsCache: true, - token: 'token', - }); - expect(applyHostRule(url, opts, hostRule)).toMatchObject({ - hostType: 'github', - lookup: dnsLookup, - token: 'xxx', - }); - }); - it('uses http keep-alive', () => { hostRules.add({ keepAlive: true }); diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 09775fe9f412fe..2bdbbcebfc3d7f 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -12,7 +12,6 @@ import type { HostRule } from '../../types'; import * as hostRules from '../host-rules'; import { matchRegexOrGlobList } from '../string-match'; import { parseUrl } from '../url'; -import { dnsLookup } from './dns'; import { keepAliveAgents } from './keep-alive'; import type { GotOptions, InternalHttpOptions } from './types'; @@ -161,10 +160,6 @@ export function applyHostRule( options.timeout = hostRule.timeout; } - if (hostRule.dnsCache) { - options.lookup = dnsLookup; - } - if (hostRule.headers) { const allowedHeaders = GlobalConfig.get('allowedHeaders', []); const filteredHeaders: Record = {}; diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index f9d30bcf787aef..9b05fd8420af5b 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -16,7 +16,6 @@ import { removeDanglingContainers } from '../../util/exec/docker'; import { deleteLocalFile, privateCacheDir } from '../../util/fs'; import { isCloned } from '../../util/git'; import { detectSemanticCommits } from '../../util/git/semantic'; -import { clearDnsCache, printDnsStats } from '../../util/http/dns'; import * as queue from '../../util/http/queue'; import * as throttle from '../../util/http/throttle'; import { addSplit, getSplits, splitInit } from '../../util/split'; @@ -138,8 +137,6 @@ export async function renovateRepository( HttpStats.report(); HttpCacheStats.report(); LookupStats.report(); - printDnsStats(); - clearDnsCache(); const cloned = isCloned(); logger.info({ cloned, durationMs: splits.total }, 'Repository finished'); resetRepositoryLogLevelRemaps(); diff --git a/package.json b/package.json index c1dc55b68b1943..23d1df2e623dab 100644 --- a/package.json +++ b/package.json @@ -175,7 +175,6 @@ "azure-devops-node-api": "13.0.0", "bunyan": "1.8.15", "cacache": "18.0.3", - "cacheable-lookup": "5.0.4", "chalk": "4.1.2", "changelog-filename-regex": "2.0.1", "clean-git-ref": "2.0.1", @@ -213,7 +212,6 @@ "jsonata": "2.0.5", "jsonc-parser": "3.2.1", "klona": "2.0.6", - "lru-cache": "10.2.2", "luxon": "3.4.4", "markdown-it": "14.1.0", "markdown-table": "2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 025483e2d60c9b..575558ce74d128 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,9 +110,6 @@ importers: cacache: specifier: 18.0.3 version: 18.0.3 - cacheable-lookup: - specifier: 5.0.4 - version: 5.0.4 chalk: specifier: 4.1.2 version: 4.1.2 @@ -224,9 +221,6 @@ importers: klona: specifier: 2.0.6 version: 2.0.6 - lru-cache: - specifier: 10.2.2 - version: 10.2.2 luxon: specifier: 3.4.4 version: 3.4.4