diff --git a/bin/dev b/bin/dev index e79dad747..e042c9d02 100755 --- a/bin/dev +++ b/bin/dev @@ -2,9 +2,16 @@ const oclif = require('@oclif/core'); const path = require('path'); +const dotenv = require('dotenv'); +const fs = require('fs-extra'); +const envPath = path.resolve(process.cwd(), './.env') const project = path.join(__dirname, '..', 'tsconfig.json'); +if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }) +} + // In dev mode -> use ts-node and dev plugins process.env.NODE_ENV = 'development'; diff --git a/bin/run b/bin/run index 4e16c8385..1c0a97fc6 100755 --- a/bin/run +++ b/bin/run @@ -1,6 +1,15 @@ #!/usr/bin/env node const oclif = require('@oclif/core'); +const dotenv = require('dotenv'); +const { resolve } = require('path'); +const fs = require('fs-extra'); + +const envPath = resolve(process.cwd(), './.env') + +if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }) +} oclif .run() diff --git a/package.json b/package.json index 0e2bb8f82..4c89bc642 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,8 @@ "ali-oss": "^6.17.1", "bluebird": "^3.7.2", "bytes": "^3.1.2", + "cache-manager": "^5.2.3", + "cache-manager-ioredis-yet": "^1.2.2", "chalk": "^4.1.2", "change-case": "^4.1.2", "check-node-version": "^4.2.1", @@ -99,7 +101,6 @@ "lodash": "^4.17.21", "micromatch": "^4.0.5", "ms": "^2.1.3", - "node-cache": "^5.1.2", "node-dir": "^0.1.17", "nunjucks": "^3.2.4", "ora": "^5.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ddf43aab4..18825937b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,12 @@ dependencies: bytes: specifier: ^3.1.2 version: 3.1.2 + cache-manager: + specifier: ^5.2.3 + version: 5.2.3 + cache-manager-ioredis-yet: + specifier: ^1.2.2 + version: 1.2.2 chalk: specifier: ^4.1.2 version: 4.1.2 @@ -98,9 +104,6 @@ dependencies: ms: specifier: ^2.1.3 version: 2.1.3 - node-cache: - specifier: ^5.1.2 - version: 5.1.2 node-dir: specifier: ^0.1.17 version: 0.1.17 @@ -2965,6 +2968,23 @@ packages: engines: {node: '>=8'} dev: true + /cache-manager-ioredis-yet@1.2.2: + resolution: {integrity: sha512-o03N/tQxfFONZ1XLGgIxOFHuQQpjpRdnSAL1THG1YWZIVp1JMUfjU3ElSAjFN1LjbJXa55IpC8waG+VEoLUCUw==} + engines: {node: '>= 16.17.0'} + dependencies: + cache-manager: 5.2.3 + ioredis: 5.3.2 + transitivePeerDependencies: + - supports-color + dev: false + + /cache-manager@5.2.3: + resolution: {integrity: sha512-9OErI8fksFkxAMJ8Mco0aiZSdphyd90HcKiOMJQncSlU1yq/9lHHxrT8PDayxrmr9IIIZPOAEfXuGSD7g29uog==} + dependencies: + lodash.clonedeep: 4.5.0 + lru-cache: 9.1.2 + dev: false + /cacheable-lookup@2.0.1: resolution: {integrity: sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg==} engines: {node: '>=10'} @@ -3394,11 +3414,6 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} - /clone@2.1.2: - resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} - engines: {node: '>=0.8'} - dev: false - /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} @@ -6372,6 +6387,10 @@ packages: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} dev: true + /lodash.clonedeep@4.5.0: + resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} + dev: false + /lodash.defaults@4.2.0: resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} @@ -6984,13 +7003,6 @@ packages: - supports-color dev: true - /node-cache@5.1.2: - resolution: {integrity: sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==} - engines: {node: '>= 8.0.0'} - dependencies: - clone: 2.1.2 - dev: false - /node-dir@0.1.17: resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==} engines: {node: '>= 0.10.5'} diff --git a/src/__tests__/__snapshots__/index.test.ts.md b/src/__tests__/__snapshots__/index.test.ts.md index a1c274fb5..07d800a4a 100644 --- a/src/__tests__/__snapshots__/index.test.ts.md +++ b/src/__tests__/__snapshots__/index.test.ts.md @@ -9,6 +9,16 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { + cache: UnifiedCache { + del: AsyncFunction [], + get: AsyncFunction [], + keys: AsyncFunction [], + mdel: AsyncFunction [], + mget: AsyncFunction [], + mset: AsyncFunction [], + reset: AsyncFunction [], + set: AsyncFunction [], + }, categories: { CLASH: 'Clash', LOON: 'Loon', @@ -30,6 +40,226 @@ Generated by [AVA](https://avajs.dev). defineSurgioConfig: Function defineSurgioConfig {}, defineTrojanProvider: Function defineTrojanProvider {}, defineV2rayNSubscribeProvider: Function defineV2rayNSubscribeProvider {}, + httpClient: Function got { + CacheError: Function CacheError {}, + CancelError: Function CancelError {}, + HTTPError: Function HTTPError {}, + MaxRedirectsError: Function MaxRedirectsError {}, + ParseError: Function ParseError {}, + ReadError: Function ReadError {}, + RequestError: Function RequestError {}, + TimeoutError: Function TimeoutError {}, + UnsupportedProtocolError: Function UnsupportedProtocolError {}, + UploadError: Function UploadError {}, + defaults: { + _rawHandlers: [ + Function {}, + ], + handlers: [ + Function {}, + ], + mutableDefaults: false, + options: { + agent: { + http: Agent { + _events: { + free: [ + Function {}, + Function {}, + ], + newListener: Function maybeEnableKeylog {}, + }, + _eventsCount: 2, + _maxListeners: undefined, + closeSocketCount: 0, + closeSocketCountLastCheck: 0, + createSocketCount: 0, + createSocketCountLastCheck: 0, + createSocketErrorCount: 0, + createSocketErrorCountLastCheck: 0, + defaultPort: 80, + errorSocketCount: 0, + errorSocketCountLastCheck: 0, + freeSockets: {}, + keepAlive: true, + keepAliveMsecs: 1000, + maxFreeSockets: 256, + maxSockets: Infinity, + maxTotalSockets: Infinity, + options: { + freeSocketTimeout: 4000, + keepAlive: true, + noDelay: true, + path: null, + socketActiveTTL: 0, + timeout: 8000, + }, + protocol: 'http:', + requestCount: 0, + requestCountLastCheck: 0, + requests: {}, + scheduling: 'lifo', + sockets: {}, + timeoutSocketCount: 0, + timeoutSocketCountLastCheck: 0, + totalSocketCount: 0, + [Symbol(kCapture)]: false, + [Symbol(agentkeepalive#currentId)]: 0, + }, + https: HttpsAgent { + _events: { + free: [ + Function {}, + Function {}, + ], + newListener: Function maybeEnableKeylog {}, + }, + _eventsCount: 2, + _maxListeners: undefined, + _sessionCache: { + list: [], + map: {}, + }, + closeSocketCount: 0, + closeSocketCountLastCheck: 0, + createSocketCount: 0, + createSocketCountLastCheck: 0, + createSocketErrorCount: 0, + createSocketErrorCountLastCheck: 0, + defaultPort: 443, + errorSocketCount: 0, + errorSocketCountLastCheck: 0, + freeSockets: {}, + keepAlive: true, + keepAliveMsecs: 1000, + maxCachedSessions: 100, + maxFreeSockets: 256, + maxSockets: Infinity, + maxTotalSockets: Infinity, + options: { + freeSocketTimeout: 4000, + keepAlive: true, + noDelay: true, + path: null, + socketActiveTTL: 0, + timeout: 8000, + }, + protocol: 'https:', + requestCount: 0, + requestCountLastCheck: 0, + requests: {}, + scheduling: 'lifo', + sockets: {}, + timeoutSocketCount: 0, + timeoutSocketCountLastCheck: 0, + totalSocketCount: 0, + [Symbol(kCapture)]: false, + [Symbol(agentkeepalive#currentId)]: 0, + }, + }, + allowGetBody: false, + cache: undefined, + cacheOptions: {}, + decompress: true, + dnsCache: undefined, + followRedirect: true, + headers: { + 'user-agent': 'surgio/3.0.0-alpha.4', + }, + hooks: { + afterResponse: [], + beforeError: [], + beforeRedirect: [], + beforeRequest: [], + beforeRetry: [], + init: [], + }, + http2: false, + https: undefined, + ignoreInvalidCookies: false, + isStream: false, + maxRedirects: 10, + method: 'GET', + methodRewriting: true, + pagination: { + backoff: 0, + countLimit: Infinity, + filter: Function filter {}, + paginate: Function paginate {}, + requestLimit: 10000, + shouldContinue: Function shouldContinue {}, + stackAllItems: true, + transform: Function transform {}, + }, + parseJson: Function parseJson {}, + password: '', + prefixUrl: '', + resolveBodyOnly: false, + responseType: 'text', + retry: { + calculateDelay: Function calculateDelay {}, + errorCodes: [ + 'ETIMEDOUT', + 'ECONNRESET', + 'EADDRINUSE', + 'ECONNREFUSED', + 'EPIPE', + 'ENOTFOUND', + 'ENETUNREACH', + 'EAI_AGAIN', + ], + limit: 0, + maxRetryAfter: Infinity, + methods: [ + 'GET', + 'PUT', + 'HEAD', + 'DELETE', + 'OPTIONS', + 'TRACE', + ], + statusCodes: [ + 408, + 413, + 429, + 500, + 502, + 503, + 504, + 521, + 522, + 524, + ], + }, + stringifyJson: Function stringifyJson {}, + throwHttpErrors: true, + timeout: { + request: 5000, + }, + username: '', + }, + }, + delete: Function {}, + extend: Function {}, + get: Function {}, + head: Function {}, + mergeOptions: Function mergeOptions {}, + paginate: AsyncGeneratorFunction { + all: AsyncFunction {}, + each: [Circular], + }, + patch: Function {}, + post: Function {}, + put: Function {}, + stream: Function { + delete: Function {}, + get: Function {}, + head: Function {}, + patch: Function {}, + post: Function {}, + put: Function {}, + }, + }, utils: { SortFilterWithSortedFilters: Function SortFilterWithSortedFilters {}, SortFilterWithSortedKeywords: Function SortFilterWithSortedKeywords {}, diff --git a/src/__tests__/__snapshots__/index.test.ts.snap b/src/__tests__/__snapshots__/index.test.ts.snap index 6ebda458b..e044a763d 100644 Binary files a/src/__tests__/__snapshots__/index.test.ts.snap and b/src/__tests__/__snapshots__/index.test.ts.snap differ diff --git a/src/base-command.ts b/src/base-command.ts index c66372935..48ca6e9e7 100644 --- a/src/base-command.ts +++ b/src/base-command.ts @@ -3,8 +3,6 @@ import { Command, Flags, Interfaces, Config } from '@oclif/core' import { transports } from '@surgio/logger' import ora from 'ora' import { resolve } from 'path' -import fs from 'fs-extra' -import dotenv from 'dotenv' import { CommandConfig } from './types' import { loadConfig } from './config' @@ -50,13 +48,6 @@ abstract class BaseCommand extends Command { flags.project = resolve(process.cwd(), flags.project) } - const envPath = resolve(flags.project, './.env') - - // istanbul ignore next - if (fs.existsSync(envPath)) { - dotenv.config({ path: envPath }) - } - this.projectDir = flags.project this.surgioConfig = await loadConfig(this.projectDir) } diff --git a/src/index.ts b/src/index.ts index b797d6501..d60d4d2bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,8 @@ import { CATEGORIES } from './constant' export type { CommandConfigBeforeNormalize as SurgioConfig } from './types' export * from './configurables' +export { default as httpClient } from './utils/http-client' +export { unifiedCache as cache } from './utils/cache' export const utils = { ...filter, diff --git a/src/provider/BlackSSLProvider.ts b/src/provider/BlackSSLProvider.ts index 4702e2d88..617869558 100644 --- a/src/provider/BlackSSLProvider.ts +++ b/src/provider/BlackSSLProvider.ts @@ -10,7 +10,8 @@ import { SubscriptionUserinfo, } from '../types' import { SurgioError } from '../utils' -import { ConfigCache } from '../utils/cache' +import { unifiedCache } from '../utils/cache' +import { getProviderCacheMaxage } from '../utils/env-flag' import httpClient from '../utils/http-client' import Provider from './Provider' import { GetNodeListFunction, GetSubscriptionUserInfoFunction } from './types' @@ -88,9 +89,10 @@ export default class BlackSSLProvider extends Provider { assert(password, '未指定 BlackSSL password.') const key = `blackssl_${username}` + const cachedConfig = await unifiedCache.get(key) - const response = ConfigCache.has(key) - ? JSON.parse(ConfigCache.get(key) as string) + const response = cachedConfig + ? JSON.parse(cachedConfig) : await (async () => { const res = await httpClient.get( 'https://api.darkssl.com/v1/service/ssl_info', @@ -106,7 +108,7 @@ export default class BlackSSLProvider extends Provider { }, ) - ConfigCache.set(key, res.body) + await unifiedCache.set(key, res.body, getProviderCacheMaxage()) return JSON.parse(res.body) })() diff --git a/src/provider/Provider.ts b/src/provider/Provider.ts index c786cb3b2..5349f06a0 100644 --- a/src/provider/Provider.ts +++ b/src/provider/Provider.ts @@ -1,21 +1,16 @@ import { createLogger } from '@surgio/logger' import { CACHE_KEYS } from '../constant' -import { ProviderConfig, SupportProviderEnum } from '../types' import { - RedisCache, + ProviderConfig, SubsciptionCacheItem, - SubscriptionCache, -} from '../utils/cache' + SupportProviderEnum, +} from '../types' +import { unifiedCache } from '../utils/cache' import { getConfig } from '../config' import { getProviderCacheMaxage } from '../utils/env-flag' import httpClient, { getUserAgent } from '../utils/http-client' -import { - msToSeconds, - toMD5, - parseSubscriptionUserInfo, - SurgioError, -} from '../utils' +import { toMD5, parseSubscriptionUserInfo, SurgioError } from '../utils' import { ProviderValidator } from '../validators' import { GetNodeListFunction, GetSubscriptionUserInfoFunction } from './types' @@ -56,7 +51,6 @@ export default abstract class Provider { requestUserAgent?: string } = {}, ): Promise { - const cacheType = getConfig()?.cache?.type || 'default' const cacheKey = `${CACHE_KEYS.Provider}:${toMD5( getUserAgent(options.requestUserAgent) + url, )}` @@ -90,30 +84,19 @@ export default abstract class Provider { return subsciptionCacheItem } - if (cacheType === 'default') { - return SubscriptionCache.has(cacheKey) - ? (SubscriptionCache.get(cacheKey) as SubsciptionCacheItem) - : await (async () => { - const subsciptionCacheItem = await requestResource() - SubscriptionCache.set(cacheKey, subsciptionCacheItem) - return subsciptionCacheItem - })() - } else { - const redisCache = new RedisCache() - const cachedValue = await redisCache.getCache( - cacheKey, - ) - - return cachedValue - ? cachedValue - : await (async () => { - const subsciptionCacheItem = await requestResource() - await redisCache.setCache(cacheKey, subsciptionCacheItem, { - ttl: msToSeconds(getProviderCacheMaxage()), - }) - return subsciptionCacheItem - })() - } + const cachedValue = await unifiedCache.get(cacheKey) + + return cachedValue + ? cachedValue + : await (async () => { + const subsciptionCacheItem = await requestResource() + await unifiedCache.set( + cacheKey, + subsciptionCacheItem, + getProviderCacheMaxage(), + ) + return subsciptionCacheItem + })() } public determineRequestUserAgent( diff --git a/src/types.ts b/src/types.ts index 61d837ae3..0a12ebad7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -183,6 +183,11 @@ export interface SubscriptionUserinfo { readonly expire: number } +export interface SubsciptionCacheItem { + readonly body: string + subscriptionUserinfo?: SubscriptionUserinfo +} + export type NodeFilterType = z.infer export type SortedNodeFilterType = z.infer diff --git a/src/utils/__tests__/cache.test.ts b/src/utils/__tests__/cache.test.ts index 2d067d755..f43031957 100644 --- a/src/utils/__tests__/cache.test.ts +++ b/src/utils/__tests__/cache.test.ts @@ -2,13 +2,20 @@ import sinon from 'sinon' import test from 'ava' import MockRedis from 'ioredis-mock' -import { RedisCache } from '../cache' +import * as config from '../../config' import redis from '../../redis' +import { unifiedCache } from '../cache' const sandbox = sinon.createSandbox() -test.before(() => { +test.beforeEach(() => { + sandbox.restore() sandbox.stub(redis, 'getRedis').returns(new MockRedis()) + sandbox.stub(config, 'getConfig').returns({ + cache: { + type: 'redis', + }, + } as any) }) test.after(() => { @@ -16,14 +23,10 @@ test.after(() => { }) test('RedisCache should work', async (t) => { - const cache = new RedisCache('test') - - t.is(cache.getCacheKey('test'), 'test:test') - - await cache.setCache('key', 'value') - t.is(await cache.getCache('key'), 'value') - t.is(await cache.hasCache('key'), true) + await unifiedCache.set('key', 'value') + t.is(await unifiedCache.get('key'), 'value') + t.is(await unifiedCache.has('key'), true) - await cache.deleteCache('key') - t.is(await cache.hasCache('key'), false) + await unifiedCache.del('key') + t.is(await unifiedCache.has('key'), false) }) diff --git a/src/utils/cache.ts b/src/utils/cache.ts index e07c55af5..a59d40736 100644 --- a/src/utils/cache.ts +++ b/src/utils/cache.ts @@ -1,80 +1,108 @@ -import Redis from 'ioredis' -import NodeCache from 'node-cache' +import ms from 'ms' +import { caching, MemoryCache, MemoryStore } from 'cache-manager' +import { + redisInsStore, + RedisCache, + RedisStore, +} from 'cache-manager-ioredis-yet' +import { getConfig } from '../config' -import { SubscriptionUserinfo } from '../types' -import { getProviderCacheMaxage } from './env-flag' import redis from '../redis' -import { msToSeconds } from './time' -export interface SubsciptionCacheItem { - readonly body: string - subscriptionUserinfo?: SubscriptionUserinfo -} +type CacheType = MemoryCache | RedisCache +type StoreType = MemoryStore | RedisStore -export const ConfigCache = new NodeCache({ - stdTTL: msToSeconds(getProviderCacheMaxage()), - maxKeys: 300, - useClones: false, -}) +export class UnifiedCache { + #type: 'redis' | 'default' | undefined + #backend: Promise | undefined -export const SubscriptionCache = new NodeCache({ - stdTTL: msToSeconds(getProviderCacheMaxage()), - maxKeys: 300, - useClones: false, -}) + async prepare(): Promise { + if (!this.#type) { + this.#type = getConfig()?.cache?.type || 'default' + } -export class RedisCache { - private redisClient: Redis + if (this.#backend) { + return this.#backend + } else { + switch (this.#type) { + case 'redis': + this.#backend = caching(redisInsStore(redis.getRedis())) + break + default: + this.#backend = caching('memory', { + ttl: ms('1d'), + }) + } + return this.#backend + } + } - constructor(public namespace?: string) { - this.redisClient = redis.getRedis() + async getType() { + await this.prepare() + return this.#type as 'redis' | 'default' } - getCacheKey(key: string) { - return this.namespace ? `${this.namespace}:${key}` : key + async getBackend() { + return this.prepare() } - setCache = async ( - key: string, - value: unknown, - { ttl }: { ttl?: number } = {}, - ) => { - const storeValue: string = JSON.stringify(value) + get: CacheType['get'] = async (...args) => { + const cache = await this.prepare() + return cache.get(...args) + } - if (typeof ttl === 'undefined') { - await this.redisClient.set(this.getCacheKey(key), storeValue) - } else { - await this.redisClient.set(this.getCacheKey(key), storeValue, 'EX', ttl) - } + set: CacheType['set'] = async (...args) => { + const cache = await this.prepare() + return cache.set(...args) } - getCache = async (key: string): Promise => { - const value = await this.redisClient.get(this.getCacheKey(key)) + del: CacheType['del'] = async (...args) => { + const cache = await this.prepare() + return cache.del(...args) + } - if (!value) { - return undefined - } + reset: CacheType['reset'] = async (...args) => { + const cache = await this.prepare() + return cache.reset(...args) + } - return JSON.parse(value) as T + keys: StoreType['keys'] = async (...args) => { + const cache = await this.prepare() + return cache.store.keys(...args) } - hasCache = async (key: string): Promise => { - const value = await this.redisClient.exists(this.getCacheKey(key)) + mset: StoreType['mset'] = async (...args) => { + const cache = await this.prepare() + return cache.store.mset(...args) + } - return !!value + mget: StoreType['mget'] = async (...args) => { + const cache = await this.prepare() + return cache.store.mget(...args) } - deleteCache = async (key: string) => { - await this.redisClient.del(this.getCacheKey(key)) + mdel: StoreType['mdel'] = async (...args) => { + const cache = await this.prepare() + return cache.store.mdel(...args) + } + + async has(key: string): Promise { + await this.prepare() + + if (this.#type === 'redis') { + const keys = await this.keys() + return keys.includes(key) + } else { + const redisClient = redis.getRedis() + const value = await redisClient.exists(key) + return value === 1 + } } } +export const unifiedCache = new UnifiedCache() + // istanbul ignore next export const cleanCaches = async () => { - ConfigCache.flushAll() - SubscriptionCache.flushAll() - - if (redis.hasRedis()) { - await redis.cleanCache() - } + await unifiedCache.reset() } diff --git a/src/utils/dns.ts b/src/utils/dns.ts index 148ef611d..69e660ea4 100644 --- a/src/utils/dns.ts +++ b/src/utils/dns.ts @@ -1,12 +1,14 @@ import { promises as dns, RecordWithTtl } from 'dns' import { createLogger } from '@surgio/logger' import Bluebird from 'bluebird' -import NodeCache from 'node-cache' +import { caching } from 'cache-manager' +import ms from 'ms' import { getNetworkResolveTimeout } from './env-flag' -const DomainCache = new NodeCache({ - useClones: false, +const domainCache = caching('memory', { + ttl: ms('1d'), + max: 1000, }) const logger = createLogger({ service: 'surgio:utils:dns' }) @@ -14,8 +16,10 @@ export const resolveDomain = async ( domain: string, timeout: number = getNetworkResolveTimeout(), ): Promise> => { - if (DomainCache.has(domain)) { - return DomainCache.get(domain) as ReadonlyArray + const cached = await (await domainCache).get(domain) + + if (cached) { + return cached } logger.debug(`try to resolve domain ${domain}`) @@ -32,7 +36,7 @@ export const resolveDomain = async ( if (records.length) { const address = records.map((item) => item.address) - DomainCache.set(domain, address, records[0].ttl) // ttl is in seconds + await (await domainCache).set(domain, address, records[0].ttl) // ttl is in seconds return address } diff --git a/src/utils/remote-snippet.ts b/src/utils/remote-snippet.ts index ea2ef3c62..fae3322c0 100644 --- a/src/utils/remote-snippet.ts +++ b/src/utils/remote-snippet.ts @@ -1,15 +1,15 @@ import Bluebird from 'bluebird' import { logger } from '@surgio/logger' import detectNewline from 'detect-newline' +import ms from 'ms' import nunjucks from 'nunjucks' import * as babelParser from '@babel/parser' import { CACHE_KEYS } from '../constant' import { RemoteSnippet, RemoteSnippetConfig } from '../types' -import { ConfigCache } from './cache' +import { unifiedCache } from './cache' import { getNetworkConcurrency, getRemoteSnippetCacheMaxage } from './env-flag' import httpClient from './http-client' -import { getConfig } from '../config' import { toMD5 } from './index' import { createTmpFactory } from './tmp-helper' @@ -124,12 +124,12 @@ export const renderSurgioSnippet = (str: string, args: string[]): string => { return nunjucks.renderString(template, {}).trim() } -export const loadRemoteSnippetList = ( +export const loadRemoteSnippetList = async ( remoteSnippetList: ReadonlyArray, cacheSnippet = true, ): Promise> => { - const cacheType = getConfig()?.cache?.type || 'default' - const tmpFactory = createTmpFactory(CACHE_KEYS.RemoteSnippets, cacheType) + const cacheType = await unifiedCache.getType() + const useLocalFile = cacheSnippet && cacheType === 'default' function load(url: string): Promise { return httpClient @@ -146,51 +146,63 @@ export const loadRemoteSnippetList = ( return Bluebird.map( remoteSnippetList, - (item) => { + async (item) => { const fileMd5 = toMD5(item.url) const isSurgioSnippet = item.surgioSnippet - return (async () => { - if (cacheSnippet) { - const tmp = tmpFactory(fileMd5, getRemoteSnippetCacheMaxage()) - const tmpContent = await tmp.getContent() - let snippet: string - - if (tmpContent) { - snippet = tmpContent - } else { - snippet = await load(item.url) - await tmp.setContent(snippet) - } - - return { - main: (...args: string[]) => - isSurgioSnippet - ? renderSurgioSnippet(snippet, args) - : addProxyToSurgeRuleSet(snippet, args[0]), - name: item.name, - url: item.url, - text: snippet, // 原始内容 - } + if (useLocalFile) { + const tmpFactory = createTmpFactory( + CACHE_KEYS.RemoteSnippets, + 'default', + ) + const tmp = tmpFactory(fileMd5, getRemoteSnippetCacheMaxage()) + const tmpContent = await tmp.getContent() + let snippet: string + + if (tmpContent) { + snippet = tmpContent } else { - const snippet: string = ConfigCache.has(item.url) - ? (ConfigCache.get(item.url) as string) - : await load(item.url).then((res) => { - ConfigCache.set(item.url, res, getRemoteSnippetCacheMaxage()) - return res - }) + snippet = await load(item.url) + await tmp.setContent(snippet) + } - return { - main: (...args: string[]) => - isSurgioSnippet - ? renderSurgioSnippet(snippet, args) - : addProxyToSurgeRuleSet(snippet, args[0]), - name: item.name, - url: item.url, - text: snippet, // 原始内容 - } + return { + main: (...args: string[]) => + isSurgioSnippet + ? renderSurgioSnippet(snippet, args) + : addProxyToSurgeRuleSet(snippet, args[0]), + name: item.name, + url: item.url, + text: snippet, // 原始内容 + } + } else { + const cacheKey = `${CACHE_KEYS.RemoteSnippets}:${fileMd5}` + const cachedSnippet = await unifiedCache.get(cacheKey) + const snippet: string = cachedSnippet + ? cachedSnippet + : await load(item.url) + .then((res) => { + return Promise.all([ + unifiedCache.set( + cacheKey, + res, + cacheSnippet ? getRemoteSnippetCacheMaxage() : ms('1m'), + ), + res, + ]) + }) + .then(([, res]) => res) + + return { + main: (...args: string[]) => + isSurgioSnippet + ? renderSurgioSnippet(snippet, args) + : addProxyToSurgeRuleSet(snippet, args[0]), + name: item.name, + url: item.url, + text: snippet, // 原始内容 } - })() + } }, { concurrency: getNetworkConcurrency(), diff --git a/src/utils/tmp-helper.ts b/src/utils/tmp-helper.ts index e632133ac..3249f9e4a 100644 --- a/src/utils/tmp-helper.ts +++ b/src/utils/tmp-helper.ts @@ -6,7 +6,6 @@ import Redis from 'ioredis' import { TMP_FOLDER_NAME } from '../constant' import redis from '../redis' -import { msToSeconds } from './index' const logger = createLogger({ service: 'surgio:utils:tmp-helper' }) const tmpDir = path.join(os.tmpdir(), TMP_FOLDER_NAME) @@ -75,12 +74,7 @@ export class TmpRedis implements TmpHelper { public async setContent(content: string): Promise { if (this.maxAge) { - await this.redisClient.set( - this.cacheKey, - content, - 'EX', - msToSeconds(this.maxAge), - ) + await this.redisClient.set(this.cacheKey, content, 'PX', this.maxAge) } else { await this.redisClient.set(this.cacheKey, content) }