Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CON-332] Refactor ContentNodeInfoManager and consumers for redis caching #3819

Merged
merged 14 commits into from
Sep 8, 2022
Merged
50 changes: 30 additions & 20 deletions creator-node/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions creator-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
"@types/node": "^18.7.9",
"@types/proxyquire": "^1.3.28",
"@types/sinon": "^10.0.11",
"@types/sinon-chai": "^3.2.8",
"@typescript-eslint/eslint-plugin": "^5.5.0",
"@typescript-eslint/parser": "^5.5.0",
"chai": "4.3.6",
Expand Down
2 changes: 1 addition & 1 deletion creator-node/src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ const config = convict({
doc: 'interval (ms) to update the cNodeEndpoint->spId mapping',
format: 'nat',
env: 'fetchCNodeEndpointToSpIdMapIntervalMs',
default: 3_600_000 // 1hr
default: 600_000 // 10m
},
stateMonitoringQueueRateLimitInterval: {
doc: 'interval (ms) during which at most stateMonitoringQueueRateLimitJobsPerInterval monitor-state jobs will run',
Expand Down
141 changes: 141 additions & 0 deletions creator-node/src/services/ContentNodeInfoManager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/**
* Util functions for fetching+updating the redis cache of info about content nodes.
* Maintains a bidirectional mappings of SP ID <--> chain info (endpoint and delegateOwnerWallet).
* TODO: Rename this to SpInfoManager and cache Discovery Nodes
theoilie marked this conversation as resolved.
Show resolved Hide resolved
*/

import type Logger from 'bunyan'

import _ from 'lodash'

import initAudiusLibs from './initAudiusLibs'
import defaultRedisClient from '../redis'

const SP_ID_TO_CHAIN_INFO_MAP_KEY = 'contentNodeInfoManagerSpIdMap'

/**
* Updates redis cache of registered content nodes. Note that this queries ALL
* content nodes via ethContracts.getServiceProviderList.
*/
async function updateContentNodeChainInfo(
theoilie marked this conversation as resolved.
Show resolved Hide resolved
theoilie marked this conversation as resolved.
Show resolved Hide resolved
logger: Logger,
redisClient = defaultRedisClient,
ethContracts?: EthContracts
) {
try {
if (!ethContracts) ethContracts = await _initLibsAndGetEthContracts(logger)
const contentNodesFromLibs =
(await ethContracts.getServiceProviderList('content-node')) || []
const spIdToChainInfoFromChain = new Map(
contentNodesFromLibs.map((cn) => [
cn.spID,
_.pick(cn, ['endpoint', 'delegateOwnerWallet'])
])
)
if (spIdToChainInfoFromChain.size > 0) {
redisClient.set(
SP_ID_TO_CHAIN_INFO_MAP_KEY,
JSON.stringify(Array.from(spIdToChainInfoFromChain.entries()))
)
}
} catch (e: any) {
logger.error(
`Failed to fetch content nodes from chain and update mapping: ${e.message}`
)
}
}

async function getMapOfSpIdToChainInfo(
logger: Logger,
redisClient = defaultRedisClient
): Promise<Map<number, ContentNodeFromChain>> {
try {
const serializedMapFromRedis = await redisClient.get(
SP_ID_TO_CHAIN_INFO_MAP_KEY
)
if (_.isEmpty(serializedMapFromRedis)) return new Map()
return new Map<number, ContentNodeFromChain>(
JSON.parse(serializedMapFromRedis as string)
)
} catch (e: any) {
logger.error(
`ContentNodeInfoManager error: Failed to fetch and parse serialized mapping: ${e.message}: ${e.stack}`
)
return new Map()
}
}

async function getMapOfCNodeEndpointToSpId(
logger: Logger,
redisClient = defaultRedisClient
) {
const spIdToChainInfo: Map<number, ContentNodeFromChain> =
await getMapOfSpIdToChainInfo(logger, redisClient)
const cNodeEndpointToSpIdMap = new Map<string, number>()
spIdToChainInfo.forEach((chainInfo, spId) => {
cNodeEndpointToSpIdMap.set(chainInfo.endpoint, spId)
})
return cNodeEndpointToSpIdMap
}

async function getSpIdFromEndpoint(
endpoint: string,
logger: Logger,
redisClient = defaultRedisClient
): Promise<number | undefined> {
const cNodeEndpointToSpIdMap: Map<string, number> =
await getMapOfCNodeEndpointToSpId(logger, redisClient)
return cNodeEndpointToSpIdMap.get(endpoint)
}

async function getContentNodeInfoFromSpId(
spId: number,
logger: Logger,
redisClient = defaultRedisClient
): Promise<ContentNodeFromChain | undefined> {
const map = await getMapOfSpIdToChainInfo(logger, redisClient)
return map.get(spId)
}

async function _initLibsAndGetEthContracts(
SidSethi marked this conversation as resolved.
Show resolved Hide resolved
logger: Logger
): Promise<EthContracts> {
const audiusLibs = await initAudiusLibs({
enableEthContracts: true,
enableContracts: false,
enableDiscovery: false,
enableIdentity: false,
logger
})
return audiusLibs.ethContracts
}

export type EthContracts = {
getServiceProviderList: (spType: string) => Promise<LibsServiceProvider[]>
}
export type LibsServiceProvider = {
owner: any // Libs typed this as any, but the contract has it as address
delegateOwnerWallet: any // Libs typed this as any, but the contract has it as address
endpoint: any // Libs typed this as any, but the contract has it as string
spID: number
type: string
blockNumber: number
}
export type ContentNodeFromChain = {
endpoint: string
delegateOwnerWallet: string
}
export {
updateContentNodeChainInfo,
getMapOfSpIdToChainInfo,
getMapOfCNodeEndpointToSpId,
getSpIdFromEndpoint,
getContentNodeInfoFromSpId
}
module.exports = {
updateContentNodeChainInfo,
getMapOfSpIdToChainInfo,
getMapOfCNodeEndpointToSpId,
getSpIdFromEndpoint,
getContentNodeInfoFromSpId
}
Loading