From 7adda62b362194be1abf69c282eba6c0fcabc359 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Wed, 21 Aug 2024 16:46:06 +0200 Subject: [PATCH 1/9] Move route finding to a worker thread --- package.json | 5 +- src/router.ts | 161 +++++++++++++++---------------------------- src/router.worker.ts | 115 +++++++++++++++++++++++++++++++ src/utils.ts | 26 +++---- tsconfig.json | 8 ++- 5 files changed, 189 insertions(+), 126 deletions(-) create mode 100644 src/router.worker.ts diff --git a/package.json b/package.json index 15c28873..749cfc74 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,11 @@ "vue-eslint-parser": "^7.6.0" }, "dependencies": { + "@curvefi/ethcall": "6.0.7", "axios": "^0.21.1", "bignumber.js": "^9.0.1", - "@curvefi/ethcall": "6.0.7", "ethers": "^6.11.0", - "memoizee": "^0.4.15" + "memoizee": "^0.4.15", + "web-worker": "^1.3.0" } } diff --git a/src/router.ts b/src/router.ts index 9b84cf12..315a7c1b 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,52 +1,42 @@ import axios from "axios"; import memoize from "memoizee"; import BigNumber from "bignumber.js"; -import { ethers } from "ethers"; -import { curve } from "./curve.js"; -import { IDict, ISwapType, IRoute, IRouteStep, IRouteTvl, IRouteOutputAndCost } from "./interfaces"; +import {ethers} from "ethers"; +import {curve} from "./curve.js"; +import {IDict, IRoute, IRouteOutputAndCost, IRouteStep, ISwapType} from "./interfaces"; import { + _cutZeros, + _get_price_impact, + _get_small_x, _getCoinAddresses, _getCoinDecimals, _getUsdRate, + BN, + DIGas, ensureAllowance, ensureAllowanceEstimateGas, + ETH_ADDRESS, fromBN, + getGasPriceFromL1, + getTxCostsUsd, hasAllowance, - isEth, - toBN, - BN, + isEth, log, parseUnits, - _cutZeros, - ETH_ADDRESS, - _get_small_x, - _get_price_impact, - DIGas, smartNumber, - getTxCostsUsd, - getGasPriceFromL1, + toBN, } from "./utils.js"; -import { getPool } from "./pools/index.js"; -import { _getAmplificationCoefficientsFromApi } from "./pools/utils.js"; -import { L2Networks } from "./constants/L2Networks.js"; - +import {getPool} from "./pools"; +import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; +import {L2Networks} from "./constants/L2Networks.js"; +import Worker from 'web-worker'; +import {routerWorkerCode} from "./router.worker"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; const GRAPH_MAX_EDGES = 3; -const MAX_ROUTES_FOR_ONE_COIN = 5; const OLD_CHAINS = [1, 10, 56, 100, 137, 250, 1284, 2222, 8453, 42161, 42220, 43114, 1313161554]; // these chains have non-ng pools -const _removeDuplications = (routes: IRouteTvl[]) => { - return routes.filter((r, i, _routes) => { - const routesByPoolIds = _routes.map((r) => r.route.map((s) => s.poolId).toString()); - return routesByPoolIds.indexOf(r.route.map((s) => s.poolId).toString()) === i; - }) -} - -const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; -const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; - const _getTVL = memoize( async (poolId: string) => Number(await (getPool(poolId)).stats.totalLiquidity()), { @@ -54,14 +44,6 @@ const _getTVL = memoize( maxAge: 5 * 60 * 1000, // 5m }); -// 4 --> 6, 5 --> 7 not allowed -// 4 --> 7, 5 --> 6 allowed -const _handleSwapType = (swapType: ISwapType): string => { - if (swapType === 6) return "4"; - if (swapType === 7) return "5"; - return swapType.toString() -} - const SNX = { 10: { swap: "0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4".toLowerCase(), @@ -74,6 +56,12 @@ const SNX = { }, } +async function mapDict(entries: [string, T][], mapper: (val: string) => Promise): Promise> { + const result: Record = {}; + await Promise.all(entries.map(async ([key]) => result[key] = await mapper(key))); + return result; +} + const _buildRouteGraph = memoize(async (): Promise>> => { const routerGraph: IDict> = {} @@ -222,8 +210,11 @@ const _buildRouteGraph = memoize(async (): Promise>> = } } - const ALL_POOLS = Object.entries(curve.getPoolsData()).filter(([id, _]) => !["crveth", "y", "busd", "pax"].includes(id)); + let start = Date.now(); + const ALL_POOLS = Object.entries(curve.getPoolsData()).filter(([id]) => !["crveth", "y", "busd", "pax"].includes(id)); const amplificationCoefficientDict = await _getAmplificationCoefficientsFromApi(); + const poolTvlDict: Record = await mapDict(ALL_POOLS, _getTVL); + log(`Preparing ${ALL_POOLS.length} pools done`, `${Date.now() - start}ms`); start = Date.now(); for (const [poolId, poolData] of ALL_POOLS) { const wrappedCoinAddresses = poolData.wrapped_coin_addresses.map((a: string) => a.toLowerCase()); const underlyingCoinAddresses = poolData.underlying_coin_addresses.map((a: string) => a.toLowerCase()); @@ -250,7 +241,7 @@ const _buildRouteGraph = memoize(async (): Promise>> = const metaCoinAddresses = basePool ? basePool.underlying_coin_addresses.map((a: string) => a.toLowerCase()) : []; let swapAddress = poolData.is_fake ? poolData.deposit_address?.toLowerCase() as string : poolAddress; - const tvl = (await _getTVL(poolId)) * tvlMultiplier; + const tvl = poolTvlDict[poolId] * tvlMultiplier; // Skip empty pools if (curve.chainId === 1 && tvl < 1000) continue; if (curve.chainId !== 1 && tvl < 100) continue; @@ -381,7 +372,7 @@ const _buildRouteGraph = memoize(async (): Promise>> = } } } - + log(`Reading ${ALL_POOLS.length} pools done`, `${Date.now() - start}ms, routerGraph: #${Object.keys(routerGraph).length}`); return routerGraph }, { @@ -389,71 +380,34 @@ const _buildRouteGraph = memoize(async (): Promise>> = maxAge: 5 * 1000, // 5m }); -const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => { - return route.route.map((r) => r.inputCoinAddress).includes(coinAddress); -} - -const _isVisitedPool = (poolId: string, route: IRouteTvl): boolean => { - return route.route.map((r) => r.poolId).includes(poolId); -} - -// Breadth-first search -const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string): Promise => { - inputCoinAddress = inputCoinAddress.toLowerCase(); - outputCoinAddress = outputCoinAddress.toLowerCase(); - - const routes: IRouteTvl[] = [{ route: [], minTvl: Infinity, totalTvl: 0 }]; - let targetRoutes: IRouteTvl[] = []; - +const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string, timeout=30000): Promise => { + const blob = new Blob([routerWorkerCode], { type: 'application/javascript' }); + const worker = new Worker(URL.createObjectURL(blob), { type: 'module' }); const routerGraph = await _buildRouteGraph(); - const ALL_POOLS = curve.getPoolsData(); - while (routes.length > 0) { - // @ts-ignore - const route: IRouteTvl = routes.pop(); - const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; - - if (inCoin === outputCoinAddress) { - targetRoutes.push(route); - } else if (route.route.length < 5) { - for (const outCoin in routerGraph[inCoin]) { - if (_isVisitedCoin(outCoin, route)) continue; - - for (const step of routerGraph[inCoin][outCoin]) { - const poolData = ALL_POOLS[step.poolId]; - - if (!poolData?.is_lending && _isVisitedPool(step.poolId, route)) continue; - - // 4 --> 6, 5 --> 7 not allowed - // 4 --> 7, 5 --> 6 allowed - const routePoolIdsPlusSwapType = route.route.map((s) => s.poolId + "+" + _handleSwapType(s.swapParams[2])); - if (routePoolIdsPlusSwapType.includes(step.poolId + "+" + _handleSwapType(step.swapParams[2]))) continue; - - const poolCoins = poolData ? poolData.wrapped_coin_addresses.concat(poolData.underlying_coin_addresses) : []; - // Exclude such cases as: - // cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) - if (!poolData?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress) continue; - // Exclude such cases as: - // aave -> aave -> 3pool (aave -> aave instead) - if (poolData?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress && outCoin !== poolData.token_address) continue; - - routes.push({ - route: [...route.route, step], - minTvl: Math.min(step.tvl, route.minTvl), - totalTvl: route.totalTvl + step.tvl, - }); - } - } - } - } - - targetRoutes = _removeDuplications([ - ...targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ...targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ]); - - return targetRoutes.map((r) => r.route); -} + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Timeout')), timeout); + + worker.onerror = (e) => { + console.error('worker error', e); + clearTimeout(timer); + reject(e); + }; + worker.onmessage = (e) => { + const {type, routes} = e.data; + console.assert(type === 'findRoutes'); + clearTimeout(timer); + resolve(routes); + }; + worker.postMessage({ + type: 'findRoutes', + inputCoinAddress, + outputCoinAddress, + routerGraph, + allPools: curve.getPoolsData(), + }); + }).finally(() => worker.terminate()); +}; const _getRouteKey = (route: IRoute, inputCoinAddress: string, outputCoinAddress: string): string => { const sortedCoins = [inputCoinAddress, outputCoinAddress].sort(); @@ -462,7 +416,6 @@ const _getRouteKey = (route: IRoute, inputCoinAddress: string, outputCoinAddress key += `${routeStep.poolId}-->`; } key += sortedCoins[1]; - return key } @@ -737,7 +690,7 @@ export const swapRequired = async (inputCoin: string, outputCoin: string, outAmo const contract = curve.contracts[curve.constants.ALIASES.router].contract; const { _route, _swapParams, _pools, _basePools, _baseTokens, _secondBasePools, _secondBaseTokens } = _getExchangeArgs(route); - let _required = 0; + let _required; if ("get_dx(address[11],uint256[5][5],uint256,address[5],address[5],address[5],address[5],address[5])" in contract) { _required = await contract.get_dx(_route, _swapParams, _outAmount, _pools, _basePools, _baseTokens, _secondBasePools, _secondBaseTokens, curve.constantOptions); } else if (_pools) { diff --git a/src/router.worker.ts b/src/router.worker.ts new file mode 100644 index 00000000..7c45e34e --- /dev/null +++ b/src/router.worker.ts @@ -0,0 +1,115 @@ +// Breadth-first search +import type {IDict, IPoolData, IRoute, IRouteStep, IRouteTvl, ISwapType} from "./interfaces"; + +function routerWorker(): void { + // this is a workaround to avoid using [...] operator, as nextjs will try to use some stupid swc helpers + const concatArrays = (a: T[], b: T[]): T[] => a.map((x) => x).concat(b); + + function log(fnName: string, ...args: unknown[]): void { + if (process.env.NODE_ENV === 'development') { + console.log(`curve-js/router-worker@${new Date().toISOString()} -> ${fnName}:`, args) + } + } + + const MAX_ROUTES_FOR_ONE_COIN = 5; + const MAX_DEPTH = 5; + + const _removeDuplications = (routes: IRouteTvl[]) => + routes.filter( + (r, i, _routes) => _routes.map((r) => r.route.map((s) => s.poolId).toString()).indexOf(r.route.map((s) => s.poolId).toString()) === i + ) + + const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; + const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; + + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + const _handleSwapType = (swapType: ISwapType): string => { + if (swapType === 6) return "4"; + if (swapType === 7) return "5"; + return swapType.toString() + } + + const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => + route.route.find((r) => r.inputCoinAddress === coinAddress) !== undefined + + const _isVisitedPool = (poolId: string, route: IRouteTvl): boolean => + route.route.find((r) => r.poolId === poolId) !== undefined + + const _findRoutes = (inputCoinAddress: string, outputCoinAddress: string, routerGraph: IDict>, allPools: IDict): IRoute[] => { + inputCoinAddress = inputCoinAddress.toLowerCase(); + outputCoinAddress = outputCoinAddress.toLowerCase(); + + const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; + let targetRoutes: IRouteTvl[] = []; + + let count = 0; + let start = Date.now(); + + while (routes.length > 0) { + count++; + // @ts-ignore + const route: IRouteTvl = routes.pop(); + const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; + + if (inCoin === outputCoinAddress) { + targetRoutes.push(route); + } else if (route.route.length < MAX_DEPTH) { + const inCoinGraph = routerGraph[inCoin]; + for (const outCoin in inCoinGraph) { + if (_isVisitedCoin(outCoin, route)) continue; + + for (const step of inCoinGraph[outCoin]) { + const poolData = allPools[step.poolId]; + + if (!poolData?.is_lending && _isVisitedPool(step.poolId, route)) continue; + + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + const swapType = _handleSwapType(step.swapParams[2]); + if (route.route.find((s) => s.poolId === step.poolId && swapType === _handleSwapType(s.swapParams[2]))) + continue; + + if (poolData && outCoin !== outputCoinAddress && ( + poolData.wrapped_coin_addresses.includes(outputCoinAddress) || poolData.underlying_coin_addresses.includes(outputCoinAddress) + )) { + // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) + if (!poolData?.is_lending) continue; + // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) + if (outCoin !== poolData.token_address) continue; + } + + routes.push({ + route: concatArrays(route.route, [step]), + minTvl: Math.min(step.tvl, route.minTvl), + totalTvl: route.totalTvl + step.tvl, + }); + } + } + } + } + log(`Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); + start = Date.now(); + + targetRoutes = _removeDuplications( + concatArrays( + targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), + targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), + ) + ); + + const result = targetRoutes.map((r) => r.route); + log(`Deduplicated to ${result.length} routes`, `${Date.now() - start}ms`); + return result; + } + + addEventListener('message', (e) => { + const {type, routerGraph, outputCoinAddress, inputCoinAddress, allPools} = e.data; + console.assert(type === 'findRoutes'); + const routes = _findRoutes(inputCoinAddress, outputCoinAddress, routerGraph, allPools); + postMessage({ type, routes }); + }); +} + +// this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) +export const routerWorkerCode = `${routerWorker.toString()}; ${routerWorker.name}();`; diff --git a/src/utils.ts b/src/utils.ts index faacab0c..6d72c0c8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -21,7 +21,7 @@ import { import ERC20Abi from './constants/abis/ERC20.json' assert { type: 'json' }; import { L2Networks } from './constants/L2Networks.js'; import { volumeNetworks } from "./constants/volumeNetworks.js"; -import { getPool } from "./pools/index.js"; +import { getPool } from "./pools"; export const ETH_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; @@ -311,13 +311,6 @@ export const getPoolIdBySwapAddress = (swapAddress: string): string => { return poolIds[0][0]; } -const _getTokenAddressBySwapAddress = (swapAddress: string): string => { - const poolsData = curve.getPoolsData() - const res = Object.entries(poolsData).filter(([_, poolData]) => poolData.swap_address.toLowerCase() === swapAddress.toLowerCase()); - if (res.length === 0) return ""; - return res[0][1].token_address; -} - export const _getUsdPricesFromApi = async (): Promise> => { const network = curve.constants.NETWORK_NAME; const allTypesExtendedPoolData = await _getAllPoolsFromApi(network); @@ -378,19 +371,12 @@ export const _getUsdPricesFromApi = async (): Promise> => { } for(const address in priceDict) { - if(priceDict[address].length > 0) { - const maxTvlItem = priceDict[address].reduce((prev, current) => { - if (+current.tvl > +prev.tvl) { - return current; - } else { - return prev; - } - }); + if (priceDict[address].length) { + const maxTvlItem = priceDict[address].reduce((prev, current) => +current.tvl > +prev.tvl ? current : prev); priceDictByMaxTvl[address] = maxTvlItem.price } else { priceDictByMaxTvl[address] = 0 } - } return priceDictByMaxTvl @@ -824,4 +810,10 @@ export const memoizedMulticallContract = (): (address: string, abi: any) => Mult return result; } } +} + +export function log(fnName: string, ...args: unknown[]): void { + if (process.env.NODE_ENV === 'development') { + console.log(`curve-js@${new Date().toISOString()} -> ${fnName}:`, ...args) + } } \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 14ecd39b..1c081074 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,9 +4,11 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ - "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ - "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ - "lib": ["ES2020"], /* Specify library files to be included in the compilation. */ + "target": "es6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ + "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ + "lib": [ + "webworker" + ], /* Specify library files to be included in the compilation. */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ From 8a06511dec03b9e7e3c997ef24bffb0085325053 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 23 Aug 2024 14:34:51 +0200 Subject: [PATCH 2/9] Check routes before adding them to array, try to use sorted array --- src/interfaces.ts | 7 +- src/router.ts | 58 +++++++---- src/router.worker.ts | 240 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 235 insertions(+), 70 deletions(-) diff --git a/src/interfaces.ts b/src/interfaces.ts index 3cd2eb04..ee733716 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -128,12 +128,7 @@ export interface IPoolDataShort { address: string, } -export interface ISubgraphPoolData { - address: string, - volumeUSD: number, - latestDailyApy: number, - latestWeeklyApy: number, -} +export type IRoutePoolData = Pick; export interface IExtendedPoolDataFromApi { poolData: IPoolDataFromApi[], diff --git a/src/router.ts b/src/router.ts index 315a7c1b..30117da1 100644 --- a/src/router.ts +++ b/src/router.ts @@ -29,7 +29,7 @@ import {getPool} from "./pools"; import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; import {L2Networks} from "./constants/L2Networks.js"; import Worker from 'web-worker'; -import {routerWorkerCode} from "./router.worker"; +import {routerWorkerBlob, routerWorker, findRouteAlgos} from "./router.worker"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; @@ -56,9 +56,15 @@ const SNX = { }, } -async function mapDict(entries: [string, T][], mapper: (val: string) => Promise): Promise> { - const result: Record = {}; - await Promise.all(entries.map(async ([key]) => result[key] = await mapper(key))); +async function entriesToDictAsync(entries: [string, T][], mapper: (key: string, value: T) => Promise): Promise> { + const result: IDict = {}; + await Promise.all(entries.map(async ([key, value]) => result[key] = await mapper(key, value))); + return result; +} + +function mapDict(dict: IDict, mapper: (key: string, value: T) => U): IDict { + const result: IDict = {}; + Object.entries(dict).forEach(([key, value]) => result[key] = mapper(key, value)); return result; } @@ -213,7 +219,7 @@ const _buildRouteGraph = memoize(async (): Promise>> = let start = Date.now(); const ALL_POOLS = Object.entries(curve.getPoolsData()).filter(([id]) => !["crveth", "y", "busd", "pax"].includes(id)); const amplificationCoefficientDict = await _getAmplificationCoefficientsFromApi(); - const poolTvlDict: Record = await mapDict(ALL_POOLS, _getTVL); + const poolTvlDict: IDict = await entriesToDictAsync(ALL_POOLS, _getTVL); log(`Preparing ${ALL_POOLS.length} pools done`, `${Date.now() - start}ms`); start = Date.now(); for (const [poolId, poolData] of ALL_POOLS) { const wrappedCoinAddresses = poolData.wrapped_coin_addresses.map((a: string) => a.toLowerCase()); @@ -380,11 +386,30 @@ const _buildRouteGraph = memoize(async (): Promise>> = maxAge: 5 * 1000, // 5m }); -const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string, timeout=30000): Promise => { - const blob = new Blob([routerWorkerCode], { type: 'application/javascript' }); - const worker = new Worker(URL.createObjectURL(blob), { type: 'module' }); +const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string, timeout=30000, inWorker=false): Promise => { const routerGraph = await _buildRouteGraph(); - + const poolData = mapDict( + curve.getPoolsData(), + (_, { is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) => ({ is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) + ); + + if (!inWorker) { + routerWorker(); + const routerStr = JSON.stringify(routerGraph); + const poolsStr = JSON.stringify(poolData); + let firstResult = undefined; + for (const findRoute of findRouteAlgos) { + const found = findRoute(inputCoinAddress, outputCoinAddress, JSON.parse(routerStr), JSON.parse(poolsStr)); + if (firstResult === undefined) { + firstResult = found; + } else { + const areEqual = JSON.stringify(found) === JSON.stringify(firstResult); + console.log({ found, firstResult, areEqual }); + } + } + return firstResult!.map((r) => r.route); + } + const worker = new Worker(routerWorkerBlob, { type: 'module' }); return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('Timeout')), timeout); @@ -395,17 +420,12 @@ const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string, }; worker.onmessage = (e) => { const {type, routes} = e.data; - console.assert(type === 'findRoutes'); - clearTimeout(timer); - resolve(routes); + if (type === 'findRoutes') { + clearTimeout(timer); + resolve(routes); + } }; - worker.postMessage({ - type: 'findRoutes', - inputCoinAddress, - outputCoinAddress, - routerGraph, - allPools: curve.getPoolsData(), - }); + worker.postMessage({ type: 'findRoutes', inputCoinAddress, outputCoinAddress, routerGraph, poolData }); }).finally(() => worker.terminate()); }; diff --git a/src/router.worker.ts b/src/router.worker.ts index 7c45e34e..32f3148d 100644 --- a/src/router.worker.ts +++ b/src/router.worker.ts @@ -1,9 +1,15 @@ // Breadth-first search -import type {IDict, IPoolData, IRoute, IRouteStep, IRouteTvl, ISwapType} from "./interfaces"; +import {IDict, IRoutePoolData, IRouteStep, IRouteTvl, ISwapType} from "./interfaces"; -function routerWorker(): void { - // this is a workaround to avoid using [...] operator, as nextjs will try to use some stupid swc helpers - const concatArrays = (a: T[], b: T[]): T[] => a.map((x) => x).concat(b); +type FindRoute = (inputCoinAddress: string, outputCoinAddress: string, routerGraph: IDict>, poolData: IDict) => IRouteTvl[]; +export let findRouteAlgos: FindRoute[]; + +export function routerWorker(): void { + const addStep = (route: IRouteTvl, step: IRouteStep) => ({ + route: [...route.route, step], + minTvl: Math.min(step.tvl, route.minTvl), + totalTvl: route.totalTvl + step.tvl, + }); function log(fnName: string, ...args: unknown[]): void { if (process.env.NODE_ENV === 'development') { @@ -12,13 +18,14 @@ function routerWorker(): void { } const MAX_ROUTES_FOR_ONE_COIN = 5; - const MAX_DEPTH = 5; + const MAX_DEPTH = 4; const _removeDuplications = (routes: IRouteTvl[]) => routes.filter( (r, i, _routes) => _routes.map((r) => r.route.map((s) => s.poolId).toString()).indexOf(r.route.map((s) => s.poolId).toString()) === i ) - + const itemAt = (route: T[], index: number) => route.length > index ? route[index] : undefined; + const lastItem = (route: T[]) => itemAt(route, route.length - 1); const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; @@ -30,21 +37,42 @@ function routerWorker(): void { return swapType.toString() } + class SortedSizedArray { + readonly items: T[] = []; + constructor(private readonly compareFn: (a: T, b: T) => number, private readonly maxSize: number) {} + + push(item: T) { + if (this.items.length === this.maxSize) { + const last = this.items[this.items.length - 1]; + if (this.compareFn(item, last) >= 0) return; + this.items.pop(); + } + + const position = this.items.findIndex((existingItem) => this.compareFn(item, existingItem) < 0); + if (position === -1) { + this.items.push(item); + } else { + this.items.splice(position, 0, item); + } + } + } + const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => route.route.find((r) => r.inputCoinAddress === coinAddress) !== undefined - const _isVisitedPool = (poolId: string, route: IRouteTvl): boolean => - route.route.find((r) => r.poolId === poolId) !== undefined + const _findPool = (route: IRouteTvl, poolId: string) => route.route.find((r) => r.poolId === poolId); + + const _isVisitedPool = (poolId: string, route: IRouteTvl): boolean => _findPool(route, poolId) !== undefined + + const _findRoutes0: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { - const _findRoutes = (inputCoinAddress: string, outputCoinAddress: string, routerGraph: IDict>, allPools: IDict): IRoute[] => { inputCoinAddress = inputCoinAddress.toLowerCase(); outputCoinAddress = outputCoinAddress.toLowerCase(); - const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; + const routes: IRouteTvl[] = [{ route: [], minTvl: Infinity, totalTvl: 0 }]; let targetRoutes: IRouteTvl[] = []; - let count = 0; - let start = Date.now(); + const start = Date.now(); while (routes.length > 0) { count++; @@ -54,33 +82,31 @@ function routerWorker(): void { if (inCoin === outputCoinAddress) { targetRoutes.push(route); - } else if (route.route.length < MAX_DEPTH) { + } else if (route.route.length <= MAX_DEPTH) { const inCoinGraph = routerGraph[inCoin]; for (const outCoin in inCoinGraph) { if (_isVisitedCoin(outCoin, route)) continue; for (const step of inCoinGraph[outCoin]) { - const poolData = allPools[step.poolId]; + const pool = poolData[step.poolId]; - if (!poolData?.is_lending && _isVisitedPool(step.poolId, route)) continue; + if (!pool?.is_lending && _isVisitedPool(step.poolId, route)) continue; // 4 --> 6, 5 --> 7 not allowed // 4 --> 7, 5 --> 6 allowed - const swapType = _handleSwapType(step.swapParams[2]); - if (route.route.find((s) => s.poolId === step.poolId && swapType === _handleSwapType(s.swapParams[2]))) - continue; + const routePoolIdsPlusSwapType = route.route.map((s) => s.poolId + "+" + _handleSwapType(s.swapParams[2])); + if (routePoolIdsPlusSwapType.includes(step.poolId + "+" + _handleSwapType(step.swapParams[2]))) continue; - if (poolData && outCoin !== outputCoinAddress && ( - poolData.wrapped_coin_addresses.includes(outputCoinAddress) || poolData.underlying_coin_addresses.includes(outputCoinAddress) - )) { - // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) - if (!poolData?.is_lending) continue; - // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) - if (outCoin !== poolData.token_address) continue; - } + const poolCoins = pool ? pool.wrapped_coin_addresses.concat(pool.underlying_coin_addresses) : []; + // Exclude such cases as: + // cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) + if (!pool?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress) continue; + // Exclude such cases as: + // aave -> aave -> 3pool (aave -> aave instead) + if (pool?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress && outCoin !== pool.token_address) continue; routes.push({ - route: concatArrays(route.route, [step]), + route: [...route.route, step], minTvl: Math.min(step.tvl, route.minTvl), totalTvl: route.totalTvl + step.tvl, }); @@ -88,28 +114,152 @@ function routerWorker(): void { } } } - log(`Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); - start = Date.now(); - - targetRoutes = _removeDuplications( - concatArrays( - targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), - targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ) - ); - - const result = targetRoutes.map((r) => r.route); - log(`Deduplicated to ${result.length} routes`, `${Date.now() - start}ms`); - return result; + + targetRoutes = _removeDuplications([ + ...targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), + ...targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), + ]); + log(`[old algo] Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); + return targetRoutes; + } + + const _findRoutes1: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { + inputCoinAddress = inputCoinAddress.toLowerCase(); + outputCoinAddress = outputCoinAddress.toLowerCase(); + + const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; + const targetRoutesByTvl = new SortedSizedArray(_sortByTvl, MAX_ROUTES_FOR_ONE_COIN); + const targetRoutesByLength = new SortedSizedArray(_sortByLength, MAX_ROUTES_FOR_ONE_COIN); + + let count = 0; + const start = Date.now(); + + while (routes.length) { + count++; + const route = routes.pop() as IRouteTvl; + const inCoin = lastItem(route.route)?.outputCoinAddress ?? inputCoinAddress; + const inCoinGraph = routerGraph[inCoin]; + + for (const outCoin in inCoinGraph) { + if (_isVisitedCoin(outCoin, route)) continue; + + for (const step of inCoinGraph[outCoin]) { + const { + is_lending, + token_address, + underlying_coin_addresses = [], + wrapped_coin_addresses = [], + } = poolData[step.poolId] || {}; + + const currentPoolInRoute = route.route.find((r) => r.poolId === step.poolId); + if (currentPoolInRoute) { + if (!is_lending) continue; + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) { + continue; + } + } + + if (step.outputCoinAddress === outputCoinAddress) { + const updatedRoute = addStep(route, step); + targetRoutesByTvl.push(updatedRoute); + targetRoutesByLength.push(updatedRoute); + continue; + } + + if (wrapped_coin_addresses.includes(outputCoinAddress) || underlying_coin_addresses.includes(outputCoinAddress)) { + // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) + if (!is_lending) continue; + // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) + if (outCoin !== token_address) continue; + } + if (route.route.length < MAX_DEPTH) { + routes.push(addStep(route, step)); // try another step + } + } + } + } + log(`[new algo, sorted array] Searched ${count} routes resulting in ${targetRoutesByTvl.items.length + targetRoutesByLength.items.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); + + return _removeDuplications([...targetRoutesByTvl.items, ...targetRoutesByLength.items]); + } + + const _findRoutes2: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { + inputCoinAddress = inputCoinAddress.toLowerCase(); + outputCoinAddress = outputCoinAddress.toLowerCase(); + + const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; + const targetRoutes: IRouteTvl[] = []; + + let count = 0; + const start = Date.now(); + + while (routes.length) { + count++; + const route = routes.pop() as IRouteTvl; + const inCoin = lastItem(route.route)?.outputCoinAddress ?? inputCoinAddress; + const inCoinGraph = routerGraph[inCoin]; + + for (const outCoin in inCoinGraph) { + if (_isVisitedCoin(outCoin, route)) continue; + + for (const step of inCoinGraph[outCoin]) { + const { + is_lending, + token_address, + underlying_coin_addresses = [], + wrapped_coin_addresses = [], + } = poolData[step.poolId] || {}; + + const currentPoolInRoute = route.route.find((r) => r.poolId === step.poolId); + if (currentPoolInRoute) { + if (!is_lending) continue; + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) { + continue; + } + } + + if (step.outputCoinAddress === outputCoinAddress) { + const updatedRoute = addStep(route, step); + targetRoutes.push(updatedRoute); + continue; + } + + if (wrapped_coin_addresses.includes(outputCoinAddress) || underlying_coin_addresses.includes(outputCoinAddress)) { + // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) + if (!is_lending) continue; + // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) + if (outCoin !== token_address) continue; + } + if (route.route.length < MAX_DEPTH) { + routes.push(addStep(route, step)); // try another step + } + } + } + } + log(`[new algo, normal array] Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); + + return _removeDuplications([ + ...targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), + ...targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), + ]); } addEventListener('message', (e) => { - const {type, routerGraph, outputCoinAddress, inputCoinAddress, allPools} = e.data; - console.assert(type === 'findRoutes'); - const routes = _findRoutes(inputCoinAddress, outputCoinAddress, routerGraph, allPools); - postMessage({ type, routes }); + const {type, routerGraph, outputCoinAddress, inputCoinAddress, poolData} = e.data; + if (type === 'findRoutes') { + const routes = _findRoutes2(inputCoinAddress, outputCoinAddress, routerGraph, poolData); + postMessage({type, routes}); + } }); + + findRouteAlgos = [_findRoutes0, _findRoutes1, _findRoutes2]; } // this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) -export const routerWorkerCode = `${routerWorker.toString()}; ${routerWorker.name}();`; +const routerWorkerCode = `${routerWorker.toString()}; ${routerWorker.name}();`; +const blob = new Blob([routerWorkerCode], { type: 'application/javascript' }); +export const routerWorkerBlob = URL.createObjectURL(blob); From bad8491cb8a713080b31d488b8bcc65df84685c3 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 23 Aug 2024 23:15:26 +0200 Subject: [PATCH 3/9] Check routes before adding them to array, try to use sorted array --- src/router.worker.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/router.worker.ts b/src/router.worker.ts index 32f3148d..0921c2ca 100644 --- a/src/router.worker.ts +++ b/src/router.worker.ts @@ -42,12 +42,12 @@ export function routerWorker(): void { constructor(private readonly compareFn: (a: T, b: T) => number, private readonly maxSize: number) {} push(item: T) { + if (!this.fits(item)) { + return; + } if (this.items.length === this.maxSize) { - const last = this.items[this.items.length - 1]; - if (this.compareFn(item, last) >= 0) return; this.items.pop(); } - const position = this.items.findIndex((existingItem) => this.compareFn(item, existingItem) < 0); if (position === -1) { this.items.push(item); @@ -55,6 +55,11 @@ export function routerWorker(): void { this.items.splice(position, 0, item); } } + fits(item: T): boolean { + if (this.items.length < this.maxSize) return true; + const last = this.items[this.items.length - 1]; + return this.compareFn(item, last) < 0; + } } const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => @@ -162,9 +167,9 @@ export function routerWorker(): void { } if (step.outputCoinAddress === outputCoinAddress) { - const updatedRoute = addStep(route, step); - targetRoutesByTvl.push(updatedRoute); - targetRoutesByLength.push(updatedRoute); + const newRoute = addStep(route, step); + targetRoutesByTvl.push(newRoute); + targetRoutesByLength.push(newRoute); continue; } @@ -175,7 +180,10 @@ export function routerWorker(): void { if (outCoin !== token_address) continue; } if (route.route.length < MAX_DEPTH) { - routes.push(addStep(route, step)); // try another step + const newRoute = addStep(route, step); + if (targetRoutesByTvl.fits(newRoute) || targetRoutesByLength.fits(newRoute)) { + routes.push(newRoute); // try another step + } } } } From aace4339694c96bfc021d8335d2e171ae2413f19 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 23 Aug 2024 23:16:02 +0200 Subject: [PATCH 4/9] Code cleanup --- src/router.ts | 46 ++---------- src/router.worker.ts | 171 ++++++------------------------------------- src/utils.ts | 42 +++++++---- 3 files changed, 57 insertions(+), 202 deletions(-) diff --git a/src/router.ts b/src/router.ts index 30117da1..3ae14a81 100644 --- a/src/router.ts +++ b/src/router.ts @@ -20,16 +20,17 @@ import { getGasPriceFromL1, getTxCostsUsd, hasAllowance, - isEth, log, + isEth, + log, parseUnits, + runWorker, smartNumber, toBN, } from "./utils.js"; import {getPool} from "./pools"; import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; import {L2Networks} from "./constants/L2Networks.js"; -import Worker from 'web-worker'; -import {routerWorkerBlob, routerWorker, findRouteAlgos} from "./router.worker"; +import {routerWorkerBlob} from "./router.worker"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; @@ -386,47 +387,14 @@ const _buildRouteGraph = memoize(async (): Promise>> = maxAge: 5 * 1000, // 5m }); -const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string, timeout=30000, inWorker=false): Promise => { +const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string): Promise => { const routerGraph = await _buildRouteGraph(); + // extract only the fields we need for the worker const poolData = mapDict( curve.getPoolsData(), (_, { is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) => ({ is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) ); - - if (!inWorker) { - routerWorker(); - const routerStr = JSON.stringify(routerGraph); - const poolsStr = JSON.stringify(poolData); - let firstResult = undefined; - for (const findRoute of findRouteAlgos) { - const found = findRoute(inputCoinAddress, outputCoinAddress, JSON.parse(routerStr), JSON.parse(poolsStr)); - if (firstResult === undefined) { - firstResult = found; - } else { - const areEqual = JSON.stringify(found) === JSON.stringify(firstResult); - console.log({ found, firstResult, areEqual }); - } - } - return firstResult!.map((r) => r.route); - } - const worker = new Worker(routerWorkerBlob, { type: 'module' }); - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error('Timeout')), timeout); - - worker.onerror = (e) => { - console.error('worker error', e); - clearTimeout(timer); - reject(e); - }; - worker.onmessage = (e) => { - const {type, routes} = e.data; - if (type === 'findRoutes') { - clearTimeout(timer); - resolve(routes); - } - }; - worker.postMessage({ type: 'findRoutes', inputCoinAddress, outputCoinAddress, routerGraph, poolData }); - }).finally(() => worker.terminate()); + return runWorker(routerWorkerBlob, {type: 'findRoutes', inputCoinAddress, outputCoinAddress, routerGraph, poolData}); }; const _getRouteKey = (route: IRoute, inputCoinAddress: string, outputCoinAddress: string): string => { diff --git a/src/router.worker.ts b/src/router.worker.ts index 0921c2ca..97a7f41f 100644 --- a/src/router.worker.ts +++ b/src/router.worker.ts @@ -5,12 +5,6 @@ type FindRoute = (inputCoinAddress: string, outputCoinAddress: string, routerGra export let findRouteAlgos: FindRoute[]; export function routerWorker(): void { - const addStep = (route: IRouteTvl, step: IRouteStep) => ({ - route: [...route.route, step], - minTvl: Math.min(step.tvl, route.minTvl), - totalTvl: route.totalTvl + step.tvl, - }); - function log(fnName: string, ...args: unknown[]): void { if (process.env.NODE_ENV === 'development') { console.log(`curve-js/router-worker@${new Date().toISOString()} -> ${fnName}:`, args) @@ -20,12 +14,12 @@ export function routerWorker(): void { const MAX_ROUTES_FOR_ONE_COIN = 5; const MAX_DEPTH = 4; - const _removeDuplications = (routes: IRouteTvl[]) => - routes.filter( - (r, i, _routes) => _routes.map((r) => r.route.map((s) => s.poolId).toString()).indexOf(r.route.map((s) => s.poolId).toString()) === i - ) - const itemAt = (route: T[], index: number) => route.length > index ? route[index] : undefined; - const lastItem = (route: T[]) => itemAt(route, route.length - 1); + const _removeDuplications = (routesA: IRouteTvl[], routesB: IRouteTvl[]) => { + const routeToStr = (r: IRouteTvl) => r.route.map((s) => s.poolId).toString(); + const routeIdsA = new Set(routesA.map(routeToStr)); + return routesA.concat(routesB.filter((r) => !routeIdsA.has(routeToStr(r)))); + } + const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; @@ -37,14 +31,18 @@ export function routerWorker(): void { return swapType.toString() } + const _addStep = (route: IRouteTvl, step: IRouteStep) => ({ + route: route.route.concat(step), + minTvl: Math.min(step.tvl, route.minTvl), + totalTvl: route.totalTvl + step.tvl, + }); + class SortedSizedArray { readonly items: T[] = []; constructor(private readonly compareFn: (a: T, b: T) => number, private readonly maxSize: number) {} push(item: T) { - if (!this.fits(item)) { - return; - } + if (!this.fits(item)) return; if (this.items.length === this.maxSize) { this.items.pop(); } @@ -55,6 +53,7 @@ export function routerWorker(): void { this.items.splice(position, 0, item); } } + fits(item: T): boolean { if (this.items.length < this.maxSize) return true; const last = this.items[this.items.length - 1]; @@ -67,68 +66,7 @@ export function routerWorker(): void { const _findPool = (route: IRouteTvl, poolId: string) => route.route.find((r) => r.poolId === poolId); - const _isVisitedPool = (poolId: string, route: IRouteTvl): boolean => _findPool(route, poolId) !== undefined - - const _findRoutes0: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { - - inputCoinAddress = inputCoinAddress.toLowerCase(); - outputCoinAddress = outputCoinAddress.toLowerCase(); - - const routes: IRouteTvl[] = [{ route: [], minTvl: Infinity, totalTvl: 0 }]; - let targetRoutes: IRouteTvl[] = []; - let count = 0; - const start = Date.now(); - - while (routes.length > 0) { - count++; - // @ts-ignore - const route: IRouteTvl = routes.pop(); - const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; - - if (inCoin === outputCoinAddress) { - targetRoutes.push(route); - } else if (route.route.length <= MAX_DEPTH) { - const inCoinGraph = routerGraph[inCoin]; - for (const outCoin in inCoinGraph) { - if (_isVisitedCoin(outCoin, route)) continue; - - for (const step of inCoinGraph[outCoin]) { - const pool = poolData[step.poolId]; - - if (!pool?.is_lending && _isVisitedPool(step.poolId, route)) continue; - - // 4 --> 6, 5 --> 7 not allowed - // 4 --> 7, 5 --> 6 allowed - const routePoolIdsPlusSwapType = route.route.map((s) => s.poolId + "+" + _handleSwapType(s.swapParams[2])); - if (routePoolIdsPlusSwapType.includes(step.poolId + "+" + _handleSwapType(step.swapParams[2]))) continue; - - const poolCoins = pool ? pool.wrapped_coin_addresses.concat(pool.underlying_coin_addresses) : []; - // Exclude such cases as: - // cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) - if (!pool?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress) continue; - // Exclude such cases as: - // aave -> aave -> 3pool (aave -> aave instead) - if (pool?.is_lending && poolCoins.includes(outputCoinAddress) && outCoin !== outputCoinAddress && outCoin !== pool.token_address) continue; - - routes.push({ - route: [...route.route, step], - minTvl: Math.min(step.tvl, route.minTvl), - totalTvl: route.totalTvl + step.tvl, - }); - } - } - } - } - - targetRoutes = _removeDuplications([ - ...targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ...targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ]); - log(`[old algo] Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); - return targetRoutes; - } - - const _findRoutes1: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { + const _findRoutes: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { inputCoinAddress = inputCoinAddress.toLowerCase(); outputCoinAddress = outputCoinAddress.toLowerCase(); @@ -142,7 +80,7 @@ export function routerWorker(): void { while (routes.length) { count++; const route = routes.pop() as IRouteTvl; - const inCoin = lastItem(route.route)?.outputCoinAddress ?? inputCoinAddress; + const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; const inCoinGraph = routerGraph[inCoin]; for (const outCoin in inCoinGraph) { @@ -156,7 +94,7 @@ export function routerWorker(): void { wrapped_coin_addresses = [], } = poolData[step.poolId] || {}; - const currentPoolInRoute = route.route.find((r) => r.poolId === step.poolId); + const currentPoolInRoute = _findPool(route, step.poolId); if (currentPoolInRoute) { if (!is_lending) continue; // 4 --> 6, 5 --> 7 not allowed @@ -167,7 +105,7 @@ export function routerWorker(): void { } if (step.outputCoinAddress === outputCoinAddress) { - const newRoute = addStep(route, step); + const newRoute = _addStep(route, step); targetRoutesByTvl.push(newRoute); targetRoutesByLength.push(newRoute); continue; @@ -180,7 +118,7 @@ export function routerWorker(): void { if (outCoin !== token_address) continue; } if (route.route.length < MAX_DEPTH) { - const newRoute = addStep(route, step); + const newRoute = _addStep(route, step); if (targetRoutesByTvl.fits(newRoute) || targetRoutesByLength.fits(newRoute)) { routes.push(newRoute); // try another step } @@ -188,83 +126,16 @@ export function routerWorker(): void { } } } - log(`[new algo, sorted array] Searched ${count} routes resulting in ${targetRoutesByTvl.items.length + targetRoutesByLength.items.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); - - return _removeDuplications([...targetRoutesByTvl.items, ...targetRoutesByLength.items]); - } - - const _findRoutes2: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { - inputCoinAddress = inputCoinAddress.toLowerCase(); - outputCoinAddress = outputCoinAddress.toLowerCase(); - - const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; - const targetRoutes: IRouteTvl[] = []; - - let count = 0; - const start = Date.now(); - - while (routes.length) { - count++; - const route = routes.pop() as IRouteTvl; - const inCoin = lastItem(route.route)?.outputCoinAddress ?? inputCoinAddress; - const inCoinGraph = routerGraph[inCoin]; - - for (const outCoin in inCoinGraph) { - if (_isVisitedCoin(outCoin, route)) continue; - - for (const step of inCoinGraph[outCoin]) { - const { - is_lending, - token_address, - underlying_coin_addresses = [], - wrapped_coin_addresses = [], - } = poolData[step.poolId] || {}; - - const currentPoolInRoute = route.route.find((r) => r.poolId === step.poolId); - if (currentPoolInRoute) { - if (!is_lending) continue; - // 4 --> 6, 5 --> 7 not allowed - // 4 --> 7, 5 --> 6 allowed - if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) { - continue; - } - } - - if (step.outputCoinAddress === outputCoinAddress) { - const updatedRoute = addStep(route, step); - targetRoutes.push(updatedRoute); - continue; - } - - if (wrapped_coin_addresses.includes(outputCoinAddress) || underlying_coin_addresses.includes(outputCoinAddress)) { - // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) - if (!is_lending) continue; - // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) - if (outCoin !== token_address) continue; - } - if (route.route.length < MAX_DEPTH) { - routes.push(addStep(route, step)); // try another step - } - } - } - } - log(`[new algo, normal array] Searched ${count} routes resulting in ${targetRoutes.length} routes between ${inputCoinAddress} and ${outputCoinAddress}`, `${Date.now() - start}ms`); - - return _removeDuplications([ - ...targetRoutes.sort(_sortByTvl).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ...targetRoutes.sort(_sortByLength).slice(0, MAX_ROUTES_FOR_ONE_COIN), - ]); + return _removeDuplications(targetRoutesByTvl.items, targetRoutesByLength.items); } addEventListener('message', (e) => { const {type, routerGraph, outputCoinAddress, inputCoinAddress, poolData} = e.data; if (type === 'findRoutes') { - const routes = _findRoutes2(inputCoinAddress, outputCoinAddress, routerGraph, poolData); + const routes = _findRoutes(inputCoinAddress, outputCoinAddress, routerGraph, poolData); postMessage({type, routes}); } }); - - findRouteAlgos = [_findRoutes0, _findRoutes1, _findRoutes2]; } // this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) diff --git a/src/utils.ts b/src/utils.ts index 6d72c0c8..2178f9bf 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import {BrowserProvider, Contract, JsonRpcProvider, Signer} from 'ethers'; -import { Contract as MulticallContract } from "@curvefi/ethcall"; +import {Contract as MulticallContract} from "@curvefi/ethcall"; import BigNumber from 'bignumber.js'; import { IBasePoolShortItem, @@ -11,17 +11,13 @@ import { IVolumeAndAPYs, REFERENCE_ASSET, } from './interfaces'; -import { curve, NETWORK_CONSTANTS } from "./curve.js"; -import { - _getAllPoolsFromApi, - _getFactoryAPYs, - _getSubgraphData, - _getVolumes, -} from "./external-api.js"; -import ERC20Abi from './constants/abis/ERC20.json' assert { type: 'json' }; -import { L2Networks } from './constants/L2Networks.js'; -import { volumeNetworks } from "./constants/volumeNetworks.js"; -import { getPool } from "./pools"; +import {curve, NETWORK_CONSTANTS} from "./curve.js"; +import {_getAllPoolsFromApi, _getFactoryAPYs, _getSubgraphData, _getVolumes,} from "./external-api.js"; +import ERC20Abi from './constants/abis/ERC20.json' assert {type: 'json'}; +import {L2Networks} from './constants/L2Networks.js'; +import {volumeNetworks} from "./constants/volumeNetworks.js"; +import {getPool} from "./pools"; +import Worker from "web-worker"; export const ETH_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; @@ -816,4 +812,24 @@ export function log(fnName: string, ...args: unknown[]): void { if (process.env.NODE_ENV === 'development') { console.log(`curve-js@${new Date().toISOString()} -> ${fnName}:`, ...args) } -} \ No newline at end of file +} + +export function runWorker(blob: string, inputData: In, timeout = 30000): Promise { + const worker = new Worker(blob, {type: 'module'}); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Timeout')), timeout); + worker.onerror = (e) => { + console.error('worker error', e); + clearTimeout(timer); + reject(e); + }; + worker.onmessage = (e) => { + const {type, routes} = e.data; + if (type === inputData.type) { + clearTimeout(timer); + resolve(routes); + } + }; + worker.postMessage(inputData); + }).finally(() => worker.terminate()); +} From 7aff3bfe375c5110d4f099b0566dce92cca4b5d9 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 23 Aug 2024 23:18:08 +0200 Subject: [PATCH 5/9] Workify the graph creation too --- src/route-finder.worker.ts | 129 ++++++++++++++ src/route-graph.worker.ts | 348 +++++++++++++++++++++++++++++++++++++ src/router.ts | 347 ++---------------------------------- src/router.worker.ts | 144 --------------- src/utils.ts | 46 ++--- 5 files changed, 508 insertions(+), 506 deletions(-) create mode 100644 src/route-finder.worker.ts create mode 100644 src/route-graph.worker.ts delete mode 100644 src/router.worker.ts diff --git a/src/route-finder.worker.ts b/src/route-finder.worker.ts new file mode 100644 index 00000000..65d9502c --- /dev/null +++ b/src/route-finder.worker.ts @@ -0,0 +1,129 @@ +// important: only type imports, the worker needs to be standalone +import type {IDict, IRoutePoolData, IRouteStep, IRouteTvl, ISwapType} from "./interfaces"; + +export type IRouterWorkerInput = { + inputCoinAddress: string, + outputCoinAddress: string, + routerGraph: IDict>, + poolData: IDict +} + +export function routeFinderWorker(): void { + const MAX_ROUTES_FOR_ONE_COIN = 5; + const MAX_DEPTH = 4; + + const _removeDuplications = (routesA: IRouteTvl[], routesB: IRouteTvl[]) => { + const routeToStr = (r: IRouteTvl) => r.route.map((s) => s.poolId).toString(); + const routeIdsA = new Set(routesA.map(routeToStr)); + return routesA.concat(routesB.filter((r) => !routeIdsA.has(routeToStr(r)))); + } + + const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; + const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; + + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + const _handleSwapType = (swapType: ISwapType): string => { + if (swapType === 6) return "4"; + if (swapType === 7) return "5"; + return swapType.toString() + } + + /** Add step to route */ + const _addStep = (route: IRouteTvl, step: IRouteStep) => ({ + route: route.route.concat(step), + minTvl: Math.min(step.tvl, route.minTvl), + totalTvl: route.totalTvl + step.tvl, + }); + + /** Check if item fits in a sorted-sized array */ + function _fits(array: T[], item: T, compareFn: (a: T, b: T) => number, maxSize: number) { + if (array.length < maxSize) return true; + const last = array[array.length - 1]; + return compareFn(item, last) < 0; + } + + /** Add item to sorted-sized array */ + function _sortedPush(array: T[], item: T, compareFn: (a: T, b: T) => number, maxSize: number) { + if (!_fits(array, item, compareFn, maxSize)) return; + if (array.length === maxSize) { + array.pop(); + } + const position = array.findIndex((existingItem) => compareFn(item, existingItem) < 0); + if (position === -1) { + array.push(item); + } else { + array.splice(position, 0, item); + } + } + + const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => + route.route.find((r) => r.inputCoinAddress === coinAddress) !== undefined + + const _findPool = (route: IRouteTvl, poolId: string) => route.route.find((r) => r.poolId === poolId); + + const findRoutes = ({ inputCoinAddress, outputCoinAddress, routerGraph, poolData }: IRouterWorkerInput): IRouteStep[][] => { + inputCoinAddress = inputCoinAddress.toLowerCase(); + outputCoinAddress = outputCoinAddress.toLowerCase(); + + const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; + const targetRoutesByTvl: IRouteTvl[] = []; + const targetRoutesByLength: IRouteTvl[] = []; + + while (routes.length) { + const route = routes.pop() as IRouteTvl; + const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; + Object.entries(routerGraph[inCoin]).forEach((leaf) => { + const outCoin = leaf[0], steps = leaf[1]; + if (_isVisitedCoin(outCoin, route)) return; + + steps.forEach((step) => { + const pool = poolData[step.poolId]; + + const currentPoolInRoute = _findPool(route, step.poolId); + if (currentPoolInRoute) { + if (!pool?.is_lending) return; + // 4 --> 6, 5 --> 7 not allowed + // 4 --> 7, 5 --> 6 allowed + if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) { + return; + } + } + + if (step.outputCoinAddress === outputCoinAddress) { + const newRoute = _addStep(route, step); + _sortedPush(targetRoutesByTvl, newRoute, _sortByTvl, MAX_ROUTES_FOR_ONE_COIN); + _sortedPush(targetRoutesByLength, newRoute, _sortByLength, MAX_ROUTES_FOR_ONE_COIN); + return; + } + + if (pool?.wrapped_coin_addresses.includes(outputCoinAddress) || pool?.underlying_coin_addresses.includes(outputCoinAddress)) { + // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) + if (!pool?.is_lending) return; + // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) + if (outCoin !== pool?.token_address) return; + } + + if (route.route.length < MAX_DEPTH) { + const newRoute = _addStep(route, step); + if (_fits(targetRoutesByTvl, newRoute, _sortByTvl, MAX_ROUTES_FOR_ONE_COIN) || + _fits(targetRoutesByLength, newRoute, _sortByLength, MAX_ROUTES_FOR_ONE_COIN)) { + routes.push(newRoute); // try another step + } + } + }) + }) + } + return _removeDuplications(targetRoutesByTvl, targetRoutesByLength).map((r) => r.route); + } + + addEventListener('message', (e) => { + const { type } = e.data; + if (type === 'findRoutes') { + postMessage({ type, result: findRoutes(e.data) }); + } + }); +} + +// this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) +export const routeFinderWorkerCode = `${routeFinderWorker.toString()}; ${routeFinderWorker.name}();`; diff --git a/src/route-graph.worker.ts b/src/route-graph.worker.ts new file mode 100644 index 00000000..7b22175d --- /dev/null +++ b/src/route-graph.worker.ts @@ -0,0 +1,348 @@ +// important: only type imports, the worker needs to be standalone +import type {IChainId, IDict, IPoolData, IRouteStep, ISwapType} from "./interfaces"; +import type {curve} from "./curve"; + +export type IRouteGraphInput = { + constants: typeof curve['constants'], + chainId: IChainId, + allPools: [string, IPoolData][], + amplificationCoefficientDict: IDict, + poolTvlDict: IDict +}; + +function routeGraphWorker() { + const GRAPH_MAX_EDGES = 3; + const SNX = { + 10: { + swap: "0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4".toLowerCase(), + coins: [ // Optimism + "0x8c6f28f2f1a3c87f0f938b96d27520d9751ec8d9", // sUSD + "0xFBc4198702E81aE77c06D58f81b629BDf36f0a71", // sEUR + "0xe405de8f52ba7559f9df3c368500b6e6ae6cee49", // sETH + "0x298b9b95708152ff6968aafd889c6586e9169f1d", // sBTC + ].map((a) => a.toLowerCase()), + }, + } + + const createRouteGraph = ({constants, chainId, allPools, amplificationCoefficientDict, poolTvlDict}: IRouteGraphInput): IDict> => { + const routerGraph: IDict> = {} + // ETH <-> WETH (exclude Celo) + if (chainId !== 42220) { + routerGraph[constants.NATIVE_TOKEN.address] = {}; + routerGraph[constants.NATIVE_TOKEN.address][constants.NATIVE_TOKEN.wrappedAddress] = [{ + poolId: "WETH wrapper", + swapAddress: constants.NATIVE_TOKEN.wrappedAddress, + inputCoinAddress: constants.NATIVE_TOKEN.address, + outputCoinAddress: constants.NATIVE_TOKEN.wrappedAddress, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + + routerGraph[constants.NATIVE_TOKEN.wrappedAddress] = {}; + routerGraph[constants.NATIVE_TOKEN.wrappedAddress][constants.NATIVE_TOKEN.address] = [{ + poolId: "WETH wrapper", + swapAddress: constants.NATIVE_TOKEN.wrappedAddress, + inputCoinAddress: constants.NATIVE_TOKEN.wrappedAddress, + outputCoinAddress: constants.NATIVE_TOKEN.address, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + } + + // ETH -> stETH, ETH -> frxETH, ETH -> wBETH (Ethereum only) + if (chainId == 1) { + for (const outCoin of ["stETH", "frxETH", "wBETH"]) { + routerGraph[constants.NATIVE_TOKEN.address][constants.COINS[outCoin.toLowerCase()]] = [{ + poolId: outCoin + " minter", + swapAddress: outCoin === "frxETH" ? "0xbAFA44EFE7901E04E39Dad13167D089C559c1138".toLowerCase() : constants.COINS[outCoin.toLowerCase()], + inputCoinAddress: constants.NATIVE_TOKEN.address, + outputCoinAddress: constants.COINS[outCoin.toLowerCase()], + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }] + } + } + + // stETH <-> wstETH (Ethereum only) + if (chainId === 1) { + routerGraph[constants.COINS.steth] = {}; + routerGraph[constants.COINS.steth][constants.COINS.wsteth] = [{ + poolId: "wstETH wrapper", + swapAddress: constants.COINS.wsteth, + inputCoinAddress: constants.COINS.steth, + outputCoinAddress: constants.COINS.wsteth, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + + routerGraph[constants.COINS.wsteth] = {}; + routerGraph[constants.COINS.wsteth][constants.COINS.steth] = [{ + poolId: "wstETH wrapper", + swapAddress: constants.COINS.wsteth, + inputCoinAddress: constants.COINS.wsteth, + outputCoinAddress: constants.COINS.steth, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + } + + // frxETH <-> sfrxETH (Ethereum only) + if (chainId === 1) { + routerGraph[constants.COINS.frxeth] = {}; + routerGraph[constants.COINS.frxeth][constants.COINS.sfrxeth] = [{ + poolId: "sfrxETH wrapper", + swapAddress: constants.COINS.sfrxeth, + inputCoinAddress: constants.COINS.frxeth, + outputCoinAddress: constants.COINS.sfrxeth, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + + routerGraph[constants.COINS.sfrxeth] = {}; + routerGraph[constants.COINS.sfrxeth][constants.COINS.frxeth] = [{ + poolId: "sfrxETH wrapper", + swapAddress: constants.COINS.sfrxeth, + inputCoinAddress: constants.COINS.sfrxeth, + outputCoinAddress: constants.COINS.frxeth, + swapParams: [0, 0, 8, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + } + + // SNX swaps + if (chainId in SNX) { + // @ts-ignore + for (const inCoin of SNX[chainId].coins) { + // @ts-ignore + for (const outCoin of SNX[chainId].coins) { + if (inCoin === outCoin) continue; + + if (!routerGraph[inCoin]) routerGraph[inCoin] = {}; + routerGraph[inCoin][outCoin] = [{ + poolId: "SNX exchanger", + // @ts-ignore + swapAddress: SNX[chainId].swap, + inputCoinAddress: inCoin, + outputCoinAddress: outCoin, + swapParams: [0, 0, 9, 0, 0], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl: Infinity, + }]; + } + } + } + + let start = Date.now(); + console.log(`Preparing ${allPools.length} pools done`, `${Date.now() - start}ms`); + start = Date.now(); + for (const poolItem of allPools) { + const poolId = poolItem[0], poolData = poolItem[1]; + const wrappedCoinAddresses = poolData.wrapped_coin_addresses.map((a: string) => a.toLowerCase()); + const underlyingCoinAddresses = poolData.underlying_coin_addresses.map((a: string) => a.toLowerCase()); + const poolAddress = poolData.swap_address.toLowerCase(); + const tokenAddress = poolData.token_address.toLowerCase(); + const isAaveLikeLending = poolData.is_lending && wrappedCoinAddresses.length === 3 && !poolData.deposit_address; + // pool_type: 1 - stable, 2 - twocrypto, 3 - tricrypto, 4 - llamma + // 10 - stable-ng, 20 - twocrypto-ng, 30 - tricrypto-ng + let poolType = poolData.is_llamma ? 4 : poolData.is_crypto ? Math.min(poolData.wrapped_coins.length, 3) : 1; + if (poolData.is_ng) poolType *= 10; + const tvlMultiplier = poolData.is_crypto ? 1 : (amplificationCoefficientDict[poolData.swap_address] ?? 1); + const basePool = poolData.is_meta ? {...constants.POOLS_DATA, ...constants.FACTORY_POOLS_DATA}[poolData.base_pool as string] : null; + const basePoolAddress = basePool ? basePool.swap_address.toLowerCase() : constants.ZERO_ADDRESS; + let baseTokenAddress = basePool ? basePool.token_address.toLowerCase() : constants.ZERO_ADDRESS; + const secondBasePool = basePool && basePool.base_pool ? { + ...constants.POOLS_DATA, + ...constants.FACTORY_POOLS_DATA, + ...constants.CRVUSD_FACTORY_POOLS_DATA, + }[basePool.base_pool as string] : null; + const secondBasePoolAddress = secondBasePool ? secondBasePool.swap_address.toLowerCase() : constants.ZERO_ADDRESS; + // for double meta underlying (crv/tricrypto, wmatic/tricrypto) + if (basePool && secondBasePoolAddress !== constants.ZERO_ADDRESS) baseTokenAddress = basePool.deposit_address?.toLowerCase() as string; + const secondBaseTokenAddress = secondBasePool ? secondBasePool.token_address.toLowerCase() : constants.ZERO_ADDRESS; + const metaCoinAddresses = basePool ? basePool.underlying_coin_addresses.map((a: string) => a.toLowerCase()) : []; + let swapAddress = poolData.is_fake ? poolData.deposit_address?.toLowerCase() as string : poolAddress; + + const tvl = poolTvlDict[poolId] * tvlMultiplier; + // Skip empty pools + if (chainId === 1 && tvl < 1000) continue; + if (chainId !== 1 && tvl < 100) continue; + + const excludedUnderlyingSwaps = (poolId === 'ib' && chainId === 1) || + (poolId === 'geist' && chainId === 250) || + (poolId === 'saave' && chainId === 1); + + // Wrapped coin <-> LP "swaps" (actually add_liquidity/remove_liquidity_one_coin) + if (!poolData.is_fake && !poolData.is_llamma && wrappedCoinAddresses.length < 6) { + const coins = [tokenAddress].concat(wrappedCoinAddresses); + for (let k = 0; k < coins.length; k++) { + for (let l = 0; l < coins.length; l++) { + if (k > 0 && l > 0) continue; + if (k == 0 && l == 0) continue; + const i = Math.max(k - 1, 0); + const j = Math.max(l - 1, 0); + const swapType = k == 0 ? 6 : 4; + + if (!routerGraph[coins[k]]) routerGraph[coins[k]] = {}; + if (!routerGraph[coins[k]][coins[l]]) routerGraph[coins[k]][coins[l]] = []; + routerGraph[coins[k]][coins[l]].push({ + poolId, + swapAddress, + inputCoinAddress: coins[k], + outputCoinAddress: coins[l], + swapParams: [i, j, swapType, poolType, wrappedCoinAddresses.length], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl, + }); + } + } + } + + // Underlying coin <-> LP "swaps" (actually add_liquidity/remove_liquidity_one_coin) + if ((poolData.is_fake || isAaveLikeLending) && underlyingCoinAddresses.length < 6 && !excludedUnderlyingSwaps) { + const coins = [tokenAddress].concat(underlyingCoinAddresses); + for (let k = 0; k < coins.length; k++) { + for (let l = 0; l < coins.length; l++) { + if (k > 0 && l > 0) continue; + if (k == 0 && l == 0) continue; + const i = Math.max(k - 1, 0); + const j = Math.max(l - 1, 0); + let swapType: ISwapType = isAaveLikeLending ? 7 : 6; + if (k > 0) swapType = isAaveLikeLending ? 5 : 4; + + if (!routerGraph[coins[k]]) routerGraph[coins[k]] = {}; + if (!routerGraph[coins[k]][coins[l]]) routerGraph[coins[k]][coins[l]] = []; + routerGraph[coins[k]][coins[l]].push({ + poolId, + swapAddress, + inputCoinAddress: coins[k], + outputCoinAddress: coins[l], + swapParams: [i, j, swapType, poolType, underlyingCoinAddresses.length], + poolAddress: constants.ZERO_ADDRESS, + basePool: constants.ZERO_ADDRESS, + baseToken: constants.ZERO_ADDRESS, + secondBasePool: constants.ZERO_ADDRESS, + secondBaseToken: constants.ZERO_ADDRESS, + tvl, + }); + } + } + } + + // Wrapped swaps + if (!poolData.is_fake) { + for (let i = 0; i < wrappedCoinAddresses.length; i++) { + for (let j = 0; j < wrappedCoinAddresses.length; j++) { + if (i == j) continue; + if (!routerGraph[wrappedCoinAddresses[i]]) routerGraph[wrappedCoinAddresses[i]] = {}; + if (!routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]]) routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]] = []; + routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]] = routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]].concat({ + poolId, + swapAddress, + inputCoinAddress: wrappedCoinAddresses[i], + outputCoinAddress: wrappedCoinAddresses[j], + swapParams: [i, j, 1, poolType, wrappedCoinAddresses.length], + poolAddress, + basePool: basePoolAddress, + baseToken: baseTokenAddress, + secondBasePool: secondBasePoolAddress, + secondBaseToken: secondBaseTokenAddress, + tvl, + }).sort((a, b) => b.tvl - a.tvl).slice(0, GRAPH_MAX_EDGES); + } + } + } + + // Only for underlying swaps + swapAddress = (poolData.is_crypto && poolData.is_meta) || (basePool?.is_lending && poolData.is_factory) ? + poolData.deposit_address as string : poolData.swap_address; + + // Underlying swaps + if (!poolData.is_plain && !excludedUnderlyingSwaps) { + for (let i = 0; i < underlyingCoinAddresses.length; i++) { + for (let j = 0; j < underlyingCoinAddresses.length; j++) { + if (i === j) continue; + // Don't swap metacoins since they can be swapped directly in base pool + if (metaCoinAddresses.includes(underlyingCoinAddresses[i]) && metaCoinAddresses.includes(underlyingCoinAddresses[j])) continue; + // avWBTC is frozen by Aave on Avalanche, deposits are not working + if (chainId === 43114 && poolId === "atricrypto" && i === 3) continue; + + const hasEth = underlyingCoinAddresses.includes(constants.NATIVE_TOKEN.address); + const swapType = (poolData.is_crypto && poolData.is_meta && poolData.is_factory) || (basePool?.is_lending && poolData.is_factory) ? 3 + : hasEth && poolId !== 'avaxcrypto' ? 1 : 2; + + if (!routerGraph[underlyingCoinAddresses[i]]) routerGraph[underlyingCoinAddresses[i]] = {}; + if (!routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]]) routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]] = []; + routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]] = routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]].concat({ + poolId, + swapAddress, + inputCoinAddress: underlyingCoinAddresses[i], + outputCoinAddress: underlyingCoinAddresses[j], + swapParams: [i, j, swapType, poolType, underlyingCoinAddresses.length], + poolAddress, + basePool: basePoolAddress, + baseToken: baseTokenAddress, + secondBasePool: secondBasePoolAddress, + secondBaseToken: secondBaseTokenAddress, + tvl, + }).sort((a, b) => b.tvl - a.tvl).slice(0, GRAPH_MAX_EDGES); + } + } + } + } + console.log(`Reading ${allPools.length} pools done`, `${Date.now() - start}ms, routerGraph: #${Object.keys(routerGraph).length}`); + return routerGraph; + } + + addEventListener('message', (e) => { + const { type } = e.data; + if (type === 'createRouteGraph') { + postMessage({ type, result: createRouteGraph(e.data) }); + } + }); +} + +// this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) +export const routeGraphWorkerCode = `${routeGraphWorker.toString()}; ${routeGraphWorker.name}();`; diff --git a/src/router.ts b/src/router.ts index 3ae14a81..fb0f55a3 100644 --- a/src/router.ts +++ b/src/router.ts @@ -3,7 +3,7 @@ import memoize from "memoizee"; import BigNumber from "bignumber.js"; import {ethers} from "ethers"; import {curve} from "./curve.js"; -import {IDict, IRoute, IRouteOutputAndCost, IRouteStep, ISwapType} from "./interfaces"; +import {IDict, IRoute, IRouteOutputAndCost, IRouteStep} from "./interfaces"; import { _cutZeros, _get_price_impact, @@ -21,7 +21,6 @@ import { getTxCostsUsd, hasAllowance, isEth, - log, parseUnits, runWorker, smartNumber, @@ -30,11 +29,11 @@ import { import {getPool} from "./pools"; import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; import {L2Networks} from "./constants/L2Networks.js"; -import {routerWorkerBlob} from "./router.worker"; +import {IRouterWorkerInput, routeFinderWorkerCode} from "./route-finder.worker"; +import {IRouteGraphInput, routeGraphWorkerCode} from "./route-graph.worker"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; -const GRAPH_MAX_EDGES = 3; const OLD_CHAINS = [1, 10, 56, 100, 137, 250, 1284, 2222, 8453, 42161, 42220, 43114, 1313161554]; // these chains have non-ng pools @@ -45,18 +44,6 @@ const _getTVL = memoize( maxAge: 5 * 60 * 1000, // 5m }); -const SNX = { - 10: { - swap: "0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4".toLowerCase(), - coins: [ // Optimism - "0x8c6f28f2f1a3c87f0f938b96d27520d9751ec8d9", // sUSD - "0xFBc4198702E81aE77c06D58f81b629BDf36f0a71", // sEUR - "0xe405de8f52ba7559f9df3c368500b6e6ae6cee49", // sETH - "0x298b9b95708152ff6968aafd889c6586e9169f1d", // sBTC - ].map((a) => a.toLowerCase()), - }, -} - async function entriesToDictAsync(entries: [string, T][], mapper: (key: string, value: T) => Promise): Promise> { const result: IDict = {}; await Promise.all(entries.map(async ([key, value]) => result[key] = await mapper(key, value))); @@ -70,317 +57,13 @@ function mapDict(dict: IDict, mapper: (key: string, value: T) => U): ID } const _buildRouteGraph = memoize(async (): Promise>> => { - const routerGraph: IDict> = {} - - // ETH <-> WETH (exclude Celo) - if (curve.chainId !== 42220) { - routerGraph[curve.constants.NATIVE_TOKEN.address] = {}; - routerGraph[curve.constants.NATIVE_TOKEN.address][curve.constants.NATIVE_TOKEN.wrappedAddress] = [{ - poolId: "WETH wrapper", - swapAddress: curve.constants.NATIVE_TOKEN.wrappedAddress, - inputCoinAddress: curve.constants.NATIVE_TOKEN.address, - outputCoinAddress: curve.constants.NATIVE_TOKEN.wrappedAddress, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - - routerGraph[curve.constants.NATIVE_TOKEN.wrappedAddress] = {}; - routerGraph[curve.constants.NATIVE_TOKEN.wrappedAddress][curve.constants.NATIVE_TOKEN.address] = [{ - poolId: "WETH wrapper", - swapAddress: curve.constants.NATIVE_TOKEN.wrappedAddress, - inputCoinAddress: curve.constants.NATIVE_TOKEN.wrappedAddress, - outputCoinAddress: curve.constants.NATIVE_TOKEN.address, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - } - - // ETH -> stETH, ETH -> frxETH, ETH -> wBETH (Ethereum only) - if (curve.chainId == 1) { - for (const outCoin of ["stETH", "frxETH", "wBETH"]) { - routerGraph[curve.constants.NATIVE_TOKEN.address][curve.constants.COINS[outCoin.toLowerCase()]] = [{ - poolId: outCoin + " minter", - swapAddress: outCoin === "frxETH" ? "0xbAFA44EFE7901E04E39Dad13167D089C559c1138".toLowerCase() : curve.constants.COINS[outCoin.toLowerCase()], - inputCoinAddress: curve.constants.NATIVE_TOKEN.address, - outputCoinAddress: curve.constants.COINS[outCoin.toLowerCase()], - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }] - } - } - - // stETH <-> wstETH (Ethereum only) - if (curve.chainId === 1) { - routerGraph[curve.constants.COINS.steth] = {}; - routerGraph[curve.constants.COINS.steth][curve.constants.COINS.wsteth] = [{ - poolId: "wstETH wrapper", - swapAddress: curve.constants.COINS.wsteth, - inputCoinAddress: curve.constants.COINS.steth, - outputCoinAddress: curve.constants.COINS.wsteth, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - - routerGraph[curve.constants.COINS.wsteth] = {}; - routerGraph[curve.constants.COINS.wsteth][curve.constants.COINS.steth] = [{ - poolId: "wstETH wrapper", - swapAddress: curve.constants.COINS.wsteth, - inputCoinAddress: curve.constants.COINS.wsteth, - outputCoinAddress: curve.constants.COINS.steth, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - } - - // frxETH <-> sfrxETH (Ethereum only) - if (curve.chainId === 1) { - routerGraph[curve.constants.COINS.frxeth] = {}; - routerGraph[curve.constants.COINS.frxeth][curve.constants.COINS.sfrxeth] = [{ - poolId: "sfrxETH wrapper", - swapAddress: curve.constants.COINS.sfrxeth, - inputCoinAddress: curve.constants.COINS.frxeth, - outputCoinAddress: curve.constants.COINS.sfrxeth, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - - routerGraph[curve.constants.COINS.sfrxeth] = {}; - routerGraph[curve.constants.COINS.sfrxeth][curve.constants.COINS.frxeth] = [{ - poolId: "sfrxETH wrapper", - swapAddress: curve.constants.COINS.sfrxeth, - inputCoinAddress: curve.constants.COINS.sfrxeth, - outputCoinAddress: curve.constants.COINS.frxeth, - swapParams: [0, 0, 8, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - } - - // SNX swaps - if (curve.chainId in SNX) { - // @ts-ignore - for (const inCoin of SNX[curve.chainId].coins) { - // @ts-ignore - for (const outCoin of SNX[curve.chainId].coins) { - if (inCoin === outCoin) continue; - - if (!routerGraph[inCoin]) routerGraph[inCoin] = {}; - routerGraph[inCoin][outCoin] = [{ - poolId: "SNX exchanger", - // @ts-ignore - swapAddress: SNX[curve.chainId].swap, - inputCoinAddress: inCoin, - outputCoinAddress: outCoin, - swapParams: [0, 0, 9, 0, 0], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl: Infinity, - }]; - } - } - } - - let start = Date.now(); - const ALL_POOLS = Object.entries(curve.getPoolsData()).filter(([id]) => !["crveth", "y", "busd", "pax"].includes(id)); + const constants = curve.constants; + const chainId = curve.chainId; + const allPools = Object.entries(curve.getPoolsData()).filter(([id]) => !["crveth", "y", "busd", "pax"].includes(id)); const amplificationCoefficientDict = await _getAmplificationCoefficientsFromApi(); - const poolTvlDict: IDict = await entriesToDictAsync(ALL_POOLS, _getTVL); - log(`Preparing ${ALL_POOLS.length} pools done`, `${Date.now() - start}ms`); start = Date.now(); - for (const [poolId, poolData] of ALL_POOLS) { - const wrappedCoinAddresses = poolData.wrapped_coin_addresses.map((a: string) => a.toLowerCase()); - const underlyingCoinAddresses = poolData.underlying_coin_addresses.map((a: string) => a.toLowerCase()); - const poolAddress = poolData.swap_address.toLowerCase(); - const tokenAddress = poolData.token_address.toLowerCase(); - const isAaveLikeLending = poolData.is_lending && wrappedCoinAddresses.length === 3 && !poolData.deposit_address; - // pool_type: 1 - stable, 2 - twocrypto, 3 - tricrypto, 4 - llamma - // 10 - stable-ng, 20 - twocrypto-ng, 30 - tricrypto-ng - let poolType = poolData.is_llamma ? 4 : poolData.is_crypto ? Math.min(poolData.wrapped_coins.length, 3) : 1; - if (poolData.is_ng) poolType *= 10; - const tvlMultiplier = poolData.is_crypto ? 1 : (amplificationCoefficientDict[poolData.swap_address] ?? 1); - const basePool = poolData.is_meta ? { ...curve.constants.POOLS_DATA, ...curve.constants.FACTORY_POOLS_DATA }[poolData.base_pool as string] : null; - const basePoolAddress = basePool ? basePool.swap_address.toLowerCase() : curve.constants.ZERO_ADDRESS; - let baseTokenAddress = basePool ? basePool.token_address.toLowerCase() : curve.constants.ZERO_ADDRESS; - const secondBasePool = basePool && basePool.base_pool ? { - ...curve.constants.POOLS_DATA, - ...curve.constants.FACTORY_POOLS_DATA, - ...curve.constants.CRVUSD_FACTORY_POOLS_DATA, - }[basePool.base_pool as string] : null; - const secondBasePoolAddress = secondBasePool ? secondBasePool.swap_address.toLowerCase() : curve.constants.ZERO_ADDRESS; - // for double meta underlying (crv/tricrypto, wmatic/tricrypto) - if (basePool && secondBasePoolAddress !== curve.constants.ZERO_ADDRESS) baseTokenAddress = basePool.deposit_address?.toLowerCase() as string; - const secondBaseTokenAddress = secondBasePool ? secondBasePool.token_address.toLowerCase() : curve.constants.ZERO_ADDRESS; - const metaCoinAddresses = basePool ? basePool.underlying_coin_addresses.map((a: string) => a.toLowerCase()) : []; - let swapAddress = poolData.is_fake ? poolData.deposit_address?.toLowerCase() as string : poolAddress; - - const tvl = poolTvlDict[poolId] * tvlMultiplier; - // Skip empty pools - if (curve.chainId === 1 && tvl < 1000) continue; - if (curve.chainId !== 1 && tvl < 100) continue; - - const excludedUnderlyingSwaps = (poolId === 'ib' && curve.chainId === 1) || - (poolId === 'geist' && curve.chainId === 250) || - (poolId === 'saave' && curve.chainId === 1); - - // Wrapped coin <-> LP "swaps" (actually add_liquidity/remove_liquidity_one_coin) - if (!poolData.is_fake && !poolData.is_llamma && wrappedCoinAddresses.length < 6) { - const coins = [tokenAddress, ...wrappedCoinAddresses]; - for (let k = 0; k < coins.length; k++) { - for (let l = 0; l < coins.length; l++) { - if (k > 0 && l > 0) continue; - if (k == 0 && l == 0) continue; - const i = Math.max(k - 1, 0); - const j = Math.max(l - 1, 0); - const swapType = k == 0 ? 6 : 4; - - if (!routerGraph[coins[k]]) routerGraph[coins[k]] = {}; - if (!routerGraph[coins[k]][coins[l]]) routerGraph[coins[k]][coins[l]] = []; - routerGraph[coins[k]][coins[l]].push({ - poolId, - swapAddress, - inputCoinAddress: coins[k], - outputCoinAddress: coins[l], - swapParams: [i, j, swapType, poolType, wrappedCoinAddresses.length], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl, - }); - } - } - } - - // Underlying coin <-> LP "swaps" (actually add_liquidity/remove_liquidity_one_coin) - if ((poolData.is_fake || isAaveLikeLending) && underlyingCoinAddresses.length < 6 && !excludedUnderlyingSwaps) { - const coins = [tokenAddress, ...underlyingCoinAddresses]; - for (let k = 0; k < coins.length; k++) { - for (let l = 0; l < coins.length; l++) { - if (k > 0 && l > 0) continue; - if (k == 0 && l == 0) continue; - const i = Math.max(k - 1, 0); - const j = Math.max(l - 1, 0); - let swapType: ISwapType = isAaveLikeLending ? 7 : 6; - if (k > 0) swapType = isAaveLikeLending ? 5 : 4; - - if (!routerGraph[coins[k]]) routerGraph[coins[k]] = {}; - if (!routerGraph[coins[k]][coins[l]]) routerGraph[coins[k]][coins[l]] = []; - routerGraph[coins[k]][coins[l]].push({ - poolId, - swapAddress, - inputCoinAddress: coins[k], - outputCoinAddress: coins[l], - swapParams: [i, j, swapType, poolType, underlyingCoinAddresses.length], - poolAddress: curve.constants.ZERO_ADDRESS, - basePool: curve.constants.ZERO_ADDRESS, - baseToken: curve.constants.ZERO_ADDRESS, - secondBasePool: curve.constants.ZERO_ADDRESS, - secondBaseToken: curve.constants.ZERO_ADDRESS, - tvl, - }); - } - } - } - - // Wrapped swaps - if (!poolData.is_fake) { - for (let i = 0; i < wrappedCoinAddresses.length; i++) { - for (let j = 0; j < wrappedCoinAddresses.length; j++) { - if (i == j) continue; - if (!routerGraph[wrappedCoinAddresses[i]]) routerGraph[wrappedCoinAddresses[i]] = {}; - if (!routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]]) routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]] = []; - routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]] = routerGraph[wrappedCoinAddresses[i]][wrappedCoinAddresses[j]].concat({ - poolId, - swapAddress, - inputCoinAddress: wrappedCoinAddresses[i], - outputCoinAddress: wrappedCoinAddresses[j], - swapParams: [i, j, 1, poolType, wrappedCoinAddresses.length], - poolAddress, - basePool: basePoolAddress, - baseToken: baseTokenAddress, - secondBasePool: secondBasePoolAddress, - secondBaseToken: secondBaseTokenAddress, - tvl, - }).sort((a, b) => b.tvl - a.tvl).slice(0, GRAPH_MAX_EDGES); - } - } - } - - // Only for underlying swaps - swapAddress = (poolData.is_crypto && poolData.is_meta) || (basePool?.is_lending && poolData.is_factory) ? - poolData.deposit_address as string : poolData.swap_address; - - // Underlying swaps - if (!poolData.is_plain && !excludedUnderlyingSwaps) { - for (let i = 0; i < underlyingCoinAddresses.length; i++) { - for (let j = 0; j < underlyingCoinAddresses.length; j++) { - if (i === j) continue; - // Don't swap metacoins since they can be swapped directly in base pool - if (metaCoinAddresses.includes(underlyingCoinAddresses[i]) && metaCoinAddresses.includes(underlyingCoinAddresses[j])) continue; - // avWBTC is frozen by Aave on Avalanche, deposits are not working - if (curve.chainId === 43114 && poolId === "atricrypto" && i === 3) continue; - - const hasEth = underlyingCoinAddresses.includes(curve.constants.NATIVE_TOKEN.address); - const swapType = (poolData.is_crypto && poolData.is_meta && poolData.is_factory) || (basePool?.is_lending && poolData.is_factory) ? 3 - : hasEth && poolId !== 'avaxcrypto' ? 1 : 2; - - if (!routerGraph[underlyingCoinAddresses[i]]) routerGraph[underlyingCoinAddresses[i]] = {}; - if (!routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]]) routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]] = []; - routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]] = routerGraph[underlyingCoinAddresses[i]][underlyingCoinAddresses[j]].concat({ - poolId, - swapAddress, - inputCoinAddress: underlyingCoinAddresses[i], - outputCoinAddress: underlyingCoinAddresses[j], - swapParams: [i, j, swapType, poolType, underlyingCoinAddresses.length], - poolAddress, - basePool: basePoolAddress, - baseToken: baseTokenAddress, - secondBasePool: secondBasePoolAddress, - secondBaseToken: secondBaseTokenAddress, - tvl, - }).sort((a, b) => b.tvl - a.tvl).slice(0, GRAPH_MAX_EDGES); - } - } - } - } - log(`Reading ${ALL_POOLS.length} pools done`, `${Date.now() - start}ms, routerGraph: #${Object.keys(routerGraph).length}`); - return routerGraph + const poolTvlDict: IDict = await entriesToDictAsync(allPools, _getTVL); + const input: IRouteGraphInput = {constants, chainId, allPools, amplificationCoefficientDict, poolTvlDict}; + return runWorker(routeGraphWorkerCode, {type: 'createRouteGraph', ...input}); }, { promise: true, @@ -394,7 +77,8 @@ const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string): curve.getPoolsData(), (_, { is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) => ({ is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) ); - return runWorker(routerWorkerBlob, {type: 'findRoutes', inputCoinAddress, outputCoinAddress, routerGraph, poolData}); + const input: IRouterWorkerInput = {inputCoinAddress, outputCoinAddress, routerGraph, poolData}; + return runWorker(routeFinderWorkerCode, {type: 'findRoutes', ...input}); }; const _getRouteKey = (route: IRoute, inputCoinAddress: string, outputCoinAddress: string): string => { @@ -657,13 +341,10 @@ export const getArgs = (route: IRoute): { _baseTokens?: string[], _secondBasePools?: string[], _secondBaseTokens?: string[] -} => { - return _getExchangeArgs(route); -} +} => _getExchangeArgs(route) -export const swapExpected = async (inputCoin: string, outputCoin: string, amount: number | string): Promise => { - return (await getBestRouteAndOutput(inputCoin, outputCoin, amount))['output']; -} +export const swapExpected = async (inputCoin: string, outputCoin: string, amount: number | string): Promise => + (await getBestRouteAndOutput(inputCoin, outputCoin, amount))['output'] export const swapRequired = async (inputCoin: string, outputCoin: string, outAmount: number | string): Promise => { diff --git a/src/router.worker.ts b/src/router.worker.ts deleted file mode 100644 index 97a7f41f..00000000 --- a/src/router.worker.ts +++ /dev/null @@ -1,144 +0,0 @@ -// Breadth-first search -import {IDict, IRoutePoolData, IRouteStep, IRouteTvl, ISwapType} from "./interfaces"; - -type FindRoute = (inputCoinAddress: string, outputCoinAddress: string, routerGraph: IDict>, poolData: IDict) => IRouteTvl[]; -export let findRouteAlgos: FindRoute[]; - -export function routerWorker(): void { - function log(fnName: string, ...args: unknown[]): void { - if (process.env.NODE_ENV === 'development') { - console.log(`curve-js/router-worker@${new Date().toISOString()} -> ${fnName}:`, args) - } - } - - const MAX_ROUTES_FOR_ONE_COIN = 5; - const MAX_DEPTH = 4; - - const _removeDuplications = (routesA: IRouteTvl[], routesB: IRouteTvl[]) => { - const routeToStr = (r: IRouteTvl) => r.route.map((s) => s.poolId).toString(); - const routeIdsA = new Set(routesA.map(routeToStr)); - return routesA.concat(routesB.filter((r) => !routeIdsA.has(routeToStr(r)))); - } - - const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length; - const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl; - - // 4 --> 6, 5 --> 7 not allowed - // 4 --> 7, 5 --> 6 allowed - const _handleSwapType = (swapType: ISwapType): string => { - if (swapType === 6) return "4"; - if (swapType === 7) return "5"; - return swapType.toString() - } - - const _addStep = (route: IRouteTvl, step: IRouteStep) => ({ - route: route.route.concat(step), - minTvl: Math.min(step.tvl, route.minTvl), - totalTvl: route.totalTvl + step.tvl, - }); - - class SortedSizedArray { - readonly items: T[] = []; - constructor(private readonly compareFn: (a: T, b: T) => number, private readonly maxSize: number) {} - - push(item: T) { - if (!this.fits(item)) return; - if (this.items.length === this.maxSize) { - this.items.pop(); - } - const position = this.items.findIndex((existingItem) => this.compareFn(item, existingItem) < 0); - if (position === -1) { - this.items.push(item); - } else { - this.items.splice(position, 0, item); - } - } - - fits(item: T): boolean { - if (this.items.length < this.maxSize) return true; - const last = this.items[this.items.length - 1]; - return this.compareFn(item, last) < 0; - } - } - - const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean => - route.route.find((r) => r.inputCoinAddress === coinAddress) !== undefined - - const _findPool = (route: IRouteTvl, poolId: string) => route.route.find((r) => r.poolId === poolId); - - const _findRoutes: FindRoute = (inputCoinAddress, outputCoinAddress, routerGraph, poolData) => { - inputCoinAddress = inputCoinAddress.toLowerCase(); - outputCoinAddress = outputCoinAddress.toLowerCase(); - - const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}]; - const targetRoutesByTvl = new SortedSizedArray(_sortByTvl, MAX_ROUTES_FOR_ONE_COIN); - const targetRoutesByLength = new SortedSizedArray(_sortByLength, MAX_ROUTES_FOR_ONE_COIN); - - let count = 0; - const start = Date.now(); - - while (routes.length) { - count++; - const route = routes.pop() as IRouteTvl; - const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress; - const inCoinGraph = routerGraph[inCoin]; - - for (const outCoin in inCoinGraph) { - if (_isVisitedCoin(outCoin, route)) continue; - - for (const step of inCoinGraph[outCoin]) { - const { - is_lending, - token_address, - underlying_coin_addresses = [], - wrapped_coin_addresses = [], - } = poolData[step.poolId] || {}; - - const currentPoolInRoute = _findPool(route, step.poolId); - if (currentPoolInRoute) { - if (!is_lending) continue; - // 4 --> 6, 5 --> 7 not allowed - // 4 --> 7, 5 --> 6 allowed - if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) { - continue; - } - } - - if (step.outputCoinAddress === outputCoinAddress) { - const newRoute = _addStep(route, step); - targetRoutesByTvl.push(newRoute); - targetRoutesByLength.push(newRoute); - continue; - } - - if (wrapped_coin_addresses.includes(outputCoinAddress) || underlying_coin_addresses.includes(outputCoinAddress)) { - // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead) - if (!is_lending) continue; - // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead) - if (outCoin !== token_address) continue; - } - if (route.route.length < MAX_DEPTH) { - const newRoute = _addStep(route, step); - if (targetRoutesByTvl.fits(newRoute) || targetRoutesByLength.fits(newRoute)) { - routes.push(newRoute); // try another step - } - } - } - } - } - return _removeDuplications(targetRoutesByTvl.items, targetRoutesByLength.items); - } - - addEventListener('message', (e) => { - const {type, routerGraph, outputCoinAddress, inputCoinAddress, poolData} = e.data; - if (type === 'findRoutes') { - const routes = _findRoutes(inputCoinAddress, outputCoinAddress, routerGraph, poolData); - postMessage({type, routes}); - } - }); -} - -// this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code) -const routerWorkerCode = `${routerWorker.toString()}; ${routerWorker.name}();`; -const blob = new Blob([routerWorkerCode], { type: 'application/javascript' }); -export const routerWorkerBlob = URL.createObjectURL(blob); diff --git a/src/utils.ts b/src/utils.ts index 2178f9bf..4b98b8b2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -12,7 +12,7 @@ import { REFERENCE_ASSET, } from './interfaces'; import {curve, NETWORK_CONSTANTS} from "./curve.js"; -import {_getAllPoolsFromApi, _getFactoryAPYs, _getSubgraphData, _getVolumes,} from "./external-api.js"; +import {_getAllPoolsFromApi, _getFactoryAPYs, _getSubgraphData, _getVolumes} from "./external-api.js"; import ERC20Abi from './constants/abis/ERC20.json' assert {type: 'json'}; import {L2Networks} from './constants/L2Networks.js'; import {volumeNetworks} from "./constants/volumeNetworks.js"; @@ -302,7 +302,7 @@ export const ensureAllowance = async (coins: string[], amounts: (number | string export const getPoolIdBySwapAddress = (swapAddress: string): string => { const poolsData = curve.getPoolsData(); - const poolIds = Object.entries(poolsData).filter(([_, poolData]) => poolData.swap_address.toLowerCase() === swapAddress.toLowerCase()); + const poolIds = Object.entries(poolsData).filter(([, poolData]) => poolData.swap_address.toLowerCase() === swapAddress.toLowerCase()); if (poolIds.length === 0) return ""; return poolIds[0][0]; } @@ -507,7 +507,7 @@ export const getUsdRate = async (coin: string): Promise => { return await _getUsdRate(coinAddress); } -export const getBaseFeeByLastBlock = async () => { +export const getBaseFeeByLastBlock = async (): Promise => { const provider = curve.provider; try { @@ -597,17 +597,6 @@ const _getNetworkName = (network: INetworkName | IChainId = curve.chainId): INet } } -const _getChainId = (network: INetworkName | IChainId = curve.chainId): IChainId => { - if (typeof network === "number" && NETWORK_CONSTANTS[network]) { - return network; - } else if (typeof network === "string" && Object.values(NETWORK_CONSTANTS).map((n) => n.NAME).includes(network)) { - const idx = Object.values(NETWORK_CONSTANTS).map((n) => n.NAME).indexOf(network); - return Number(Object.keys(NETWORK_CONSTANTS)[idx]) as IChainId; - } else { - throw Error(`Wrong network name or id: ${network}`); - } -} - export const getTVL = async (network: INetworkName | IChainId = curve.chainId): Promise => { network = _getNetworkName(network); const allTypesExtendedPoolData = await _getAllPoolsFromApi(network); @@ -697,7 +686,7 @@ export const getCoinsData = async (...coins: string[] | string[][]): Promise<{na } const res: {name: string, symbol: string, decimals: number}[] = []; - coins.forEach((address: string, i: number) => { + coins.forEach(() => { res.push({ name: _response.shift() as string, symbol: _response.shift() as string, @@ -721,14 +710,8 @@ export const getCountArgsOfMethodByContract = (contract: Contract, methodName: s } } -export const isMethodExist = (contract: Contract, methodName: string): boolean => { - const func = contract.interface.fragments.find((item: any) => item.name === methodName); - if(func) { - return true; - } else { - return false; - } -} +export const isMethodExist = (contract: Contract, methodName: string): boolean => + contract.interface.fragments.find((item: any) => item.name === methodName) !== undefined export const getPoolName = (name: string): string => { const separatedName = name.split(": ") @@ -814,22 +797,27 @@ export function log(fnName: string, ...args: unknown[]): void { } } -export function runWorker(blob: string, inputData: In, timeout = 30000): Promise { - const worker = new Worker(blob, {type: 'module'}); +export function runWorker(code: string, inputData: In, timeout = 30000): Promise { + const blob = new Blob([code], { type: 'application/javascript' }); + const blobUrl = URL.createObjectURL(blob); + const worker = new Worker(blobUrl, {type: 'module'}); return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('Timeout')), timeout); worker.onerror = (e) => { - console.error('worker error', e); clearTimeout(timer); + console.error(code, inputData, e); reject(e); }; worker.onmessage = (e) => { - const {type, routes} = e.data; + const {type, result} = e.data; if (type === inputData.type) { clearTimeout(timer); - resolve(routes); + resolve(result); + console.log(code, inputData, result); } }; worker.postMessage(inputData); - }).finally(() => worker.terminate()); + }).finally(() => { + worker.terminate(); + }); } From 025bbaf8d0a1c272572c6e9653c77b9597928e55 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 26 Aug 2024 09:47:20 +0200 Subject: [PATCH 6/9] Remove log --- src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 4b98b8b2..70d9ecec 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -813,7 +813,7 @@ export function runWorker(code: string, inputD if (type === inputData.type) { clearTimeout(timer); resolve(result); - console.log(code, inputData, result); + // console.log(code, inputData, result, start - Date.now()); } }; worker.postMessage(inputData); From 6c268ac647af5c95537b59ba325b9a0c2d86b708 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 26 Aug 2024 10:30:58 +0200 Subject: [PATCH 7/9] Fix import issues Signed-off-by: Daniel Schiavini --- package.json | 4 ++-- src/curve.ts | 43 +++++++++++++++++++++++++++---------------- src/router.ts | 6 +++--- src/utils.ts | 30 +----------------------------- test/deploy.test.ts | 4 ++-- tsconfig.json | 6 ++---- 6 files changed, 37 insertions(+), 56 deletions(-) diff --git a/package.json b/package.json index 89b30802..da11ef85 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,11 @@ "vue-eslint-parser": "^7.6.0" }, "dependencies": { - "@curvefi/ethcall": "6.0.7", "axios": "^0.21.1", "bignumber.js": "^9.0.1", + "@curvefi/ethcall": "6.0.7", "ethers": "^6.11.0", "memoizee": "^0.4.15", - "web-worker": "^1.3.0" + "web-worker": "1.2.0" } } diff --git a/src/curve.ts b/src/curve.ts index c2fe3941..f011c592 100644 --- a/src/curve.ts +++ b/src/curve.ts @@ -5,6 +5,9 @@ import { BigNumberish, Numeric, AbstractProvider, + BrowserProvider, + JsonRpcProvider, + Signer, } from "ethers"; import { Provider as MulticallProvider, Contract as MulticallContract } from "@curvefi/ethcall"; import { getFactoryPoolData } from "./factory/factory.js"; @@ -103,26 +106,34 @@ import { COINS_FRAXTAL, cTokensFraxtal, yTokensFraxtal, ycTokensFraxtal, aToken import { COINS_XLAYER, cTokensXLayer, yTokensXLayer, ycTokensXLayer, aTokensXLayer } from "./constants/coins/xlayer.js"; import { COINS_MANTLE, cTokensMantle, yTokensMantle, ycTokensMantle, aTokensMantle } from "./constants/coins/mantle.js"; import { lowerCasePoolDataAddresses, extractDecimals, extractGauges } from "./constants/utils.js"; -import { _getAllGauges, _getHiddenPools } from "./external-api.js"; +import { _getHiddenPools } from "./external-api.js"; import { L2Networks } from "./constants/L2Networks.js"; import { getTwocryptoFactoryPoolData } from "./factory/factory-twocrypto.js"; -import {getGasInfoForL2, memoizedContract, memoizedMulticallContract} from "./utils.js"; - -const _killGauges = async (poolsData: IDict): Promise => { - const gaugeData = await _getAllGauges(); - const isKilled: IDict = {}; - const gaugeStatuses: IDict | null> = {}; - Object.values(gaugeData).forEach((d) => { - isKilled[d.gauge.toLowerCase()] = d.is_killed ?? false; - gaugeStatuses[d.gauge.toLowerCase()] = d.gaugeStatus ?? null; - }); - for (const poolId in poolsData) { - if (isKilled[poolsData[poolId].gauge_address]) { - poolsData[poolId].is_gauge_killed = true; +export const memoizedContract = (): (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer) => Contract => { + const cache: Record = {}; + return (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer): Contract => { + if (address in cache) { + return cache[address]; + } + else { + const result = new Contract(address, abi, provider) + cache[address] = result; + return result; + } + } +} + +export const memoizedMulticallContract = (): (address: string, abi: any) => MulticallContract => { + const cache: Record = {}; + return (address: string, abi: any): MulticallContract => { + if (address in cache) { + return cache[address]; } - if (gaugeStatuses[poolsData[poolId].gauge_address]) { - poolsData[poolId].gauge_status = gaugeStatuses[poolsData[poolId].gauge_address]; + else { + const result = new MulticallContract(address, abi) + cache[address] = result; + return result; } } } diff --git a/src/router.ts b/src/router.ts index fb0f55a3..32cb29dc 100644 --- a/src/router.ts +++ b/src/router.ts @@ -26,11 +26,11 @@ import { smartNumber, toBN, } from "./utils.js"; -import {getPool} from "./pools"; +import {getPool} from "./pools/index.js"; import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; import {L2Networks} from "./constants/L2Networks.js"; -import {IRouterWorkerInput, routeFinderWorkerCode} from "./route-finder.worker"; -import {IRouteGraphInput, routeGraphWorkerCode} from "./route-graph.worker"; +import {IRouterWorkerInput, routeFinderWorkerCode} from "./route-finder.worker.js"; +import {IRouteGraphInput, routeGraphWorkerCode} from "./route-graph.worker.js"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; diff --git a/src/utils.ts b/src/utils.ts index 542d1fba..bd94a2fc 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -16,7 +16,7 @@ import {_getAllPoolsFromApi, _getFactoryAPYs, _getSubgraphData, _getVolumes} fro import ERC20Abi from './constants/abis/ERC20.json' assert {type: 'json'}; import {L2Networks} from './constants/L2Networks.js'; import {volumeNetworks} from "./constants/volumeNetworks.js"; -import {getPool} from "./pools"; +import {getPool} from "./pools/index.js"; import Worker from "web-worker"; @@ -772,34 +772,6 @@ export const getBasePools = async (): Promise => { }) } -export const memoizedContract = (): (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer) => Contract => { - const cache: Record = {}; - return (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer): Contract => { - if (address in cache) { - return cache[address]; - } - else { - const result = new Contract(address, abi, provider) - cache[address] = result; - return result; - } - } -} - -export const memoizedMulticallContract = (): (address: string, abi: any) => MulticallContract => { - const cache: Record = {}; - return (address: string, abi: any): MulticallContract => { - if (address in cache) { - return cache[address]; - } - else { - const result = new MulticallContract(address, abi) - cache[address] = result; - return result; - } - } -} - export function log(fnName: string, ...args: unknown[]): void { if (process.env.NODE_ENV === 'development') { console.log(`curve-js@${new Date().toISOString()} -> ${fnName}:`, ...args) diff --git a/test/deploy.test.ts b/test/deploy.test.ts index 3ba8bb6a..a8b2dbec 100644 --- a/test/deploy.test.ts +++ b/test/deploy.test.ts @@ -730,7 +730,7 @@ describe('Factory deploy', function() { assert.equal(poolAddress.toLowerCase(), pool.address); assert.equal(gaugeAddress.toLowerCase(), pool.gauge.address); - const amounts = await pool.cryptoSeedAmounts(30); + const amounts = await pool.getSeedAmounts(30); await pool.depositAndStake(amounts); const underlyingBalances = await pool.stats.underlyingBalances(); const wrappedBalances = await pool.stats.wrappedBalances(); @@ -776,7 +776,7 @@ describe('Factory deploy', function() { assert.equal(poolAddress.toLowerCase(), pool.address); assert.equal(gaugeAddress.toLowerCase(), pool.gauge.address); - const amounts = await pool.cryptoSeedAmounts(30); + const amounts = await pool.getSeedAmounts(30); console.log(amounts); await pool.depositAndStake(amounts); const underlyingBalances = await pool.stats.underlyingBalances(); diff --git a/tsconfig.json b/tsconfig.json index 1c081074..41a01258 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,10 +5,8 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ "target": "es6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ - "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ - "lib": [ - "webworker" - ], /* Specify library files to be included in the compilation. */ + "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ + "lib": ["webworker"], /* Specify library files to be included in the compilation. */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ From 458d52491d601368fc2ad6256ed90aba2f632f7b Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Mon, 26 Aug 2024 14:03:29 +0200 Subject: [PATCH 8/9] Run synchronously in node, get rid of extra dependency Signed-off-by: Daniel Schiavini --- package.json | 3 +-- src/route-finder.worker.ts | 6 +++++- src/route-graph.worker.ts | 12 +++++++----- src/router.ts | 8 ++++---- src/utils.ts | 10 +++++++--- test/router.test.ts | 5 +++-- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index da11ef85..56a432b0 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "bignumber.js": "^9.0.1", "@curvefi/ethcall": "6.0.7", "ethers": "^6.11.0", - "memoizee": "^0.4.15", - "web-worker": "1.2.0" + "memoizee": "^0.4.15" } } diff --git a/src/route-finder.worker.ts b/src/route-finder.worker.ts index 65d9502c..5046ec88 100644 --- a/src/route-finder.worker.ts +++ b/src/route-finder.worker.ts @@ -8,7 +8,7 @@ export type IRouterWorkerInput = { poolData: IDict } -export function routeFinderWorker(): void { +export function routeFinderWorker() { const MAX_ROUTES_FOR_ONE_COIN = 5; const MAX_DEPTH = 4; @@ -117,6 +117,10 @@ export function routeFinderWorker(): void { return _removeDuplications(targetRoutesByTvl, targetRoutesByLength).map((r) => r.route); } + if (typeof addEventListener === 'undefined') { + return findRoutes; // for nodejs + } + addEventListener('message', (e) => { const { type } = e.data; if (type === 'findRoutes') { diff --git a/src/route-graph.worker.ts b/src/route-graph.worker.ts index 7b22175d..c576c7a1 100644 --- a/src/route-graph.worker.ts +++ b/src/route-graph.worker.ts @@ -10,7 +10,7 @@ export type IRouteGraphInput = { poolTvlDict: IDict }; -function routeGraphWorker() { +export function routeGraphWorker() { const GRAPH_MAX_EDGES = 3; const SNX = { 10: { @@ -171,9 +171,7 @@ function routeGraphWorker() { } } - let start = Date.now(); - console.log(`Preparing ${allPools.length} pools done`, `${Date.now() - start}ms`); - start = Date.now(); + const start = Date.now(); for (const poolItem of allPools) { const poolId = poolItem[0], poolData = poolItem[1]; const wrappedCoinAddresses = poolData.wrapped_coin_addresses.map((a: string) => a.toLowerCase()); @@ -332,10 +330,14 @@ function routeGraphWorker() { } } } - console.log(`Reading ${allPools.length} pools done`, `${Date.now() - start}ms, routerGraph: #${Object.keys(routerGraph).length}`); + console.log(`Read ${allPools.length} pools`, `${Date.now() - start}ms, routerGraph: ${Object.keys(routerGraph).length} items`); return routerGraph; } + if (typeof addEventListener === 'undefined') { + return createRouteGraph; // for nodejs + } + addEventListener('message', (e) => { const { type } = e.data; if (type === 'createRouteGraph') { diff --git a/src/router.ts b/src/router.ts index 32cb29dc..1ab41a5a 100644 --- a/src/router.ts +++ b/src/router.ts @@ -29,8 +29,8 @@ import { import {getPool} from "./pools/index.js"; import {_getAmplificationCoefficientsFromApi} from "./pools/utils.js"; import {L2Networks} from "./constants/L2Networks.js"; -import {IRouterWorkerInput, routeFinderWorkerCode} from "./route-finder.worker.js"; -import {IRouteGraphInput, routeGraphWorkerCode} from "./route-graph.worker.js"; +import {IRouterWorkerInput, routeFinderWorker, routeFinderWorkerCode} from "./route-finder.worker.js"; +import {IRouteGraphInput, routeGraphWorker, routeGraphWorkerCode} from "./route-graph.worker.js"; const MAX_STEPS = 5; const ROUTE_LENGTH = (MAX_STEPS * 2) + 1; @@ -63,7 +63,7 @@ const _buildRouteGraph = memoize(async (): Promise>> = const amplificationCoefficientDict = await _getAmplificationCoefficientsFromApi(); const poolTvlDict: IDict = await entriesToDictAsync(allPools, _getTVL); const input: IRouteGraphInput = {constants, chainId, allPools, amplificationCoefficientDict, poolTvlDict}; - return runWorker(routeGraphWorkerCode, {type: 'createRouteGraph', ...input}); + return runWorker(routeGraphWorkerCode, routeGraphWorker, {type: 'createRouteGraph', ...input}); }, { promise: true, @@ -78,7 +78,7 @@ const _findRoutes = async (inputCoinAddress: string, outputCoinAddress: string): (_, { is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) => ({ is_lending, wrapped_coin_addresses, underlying_coin_addresses, token_address }) ); const input: IRouterWorkerInput = {inputCoinAddress, outputCoinAddress, routerGraph, poolData}; - return runWorker(routeFinderWorkerCode, {type: 'findRoutes', ...input}); + return runWorker(routeFinderWorkerCode, routeFinderWorker, {type: 'findRoutes', ...input}); }; const _getRouteKey = (route: IRoute, inputCoinAddress: string, outputCoinAddress: string): string => { diff --git a/src/utils.ts b/src/utils.ts index bd94a2fc..64071c89 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,5 @@ import axios from 'axios'; -import {BrowserProvider, Contract, JsonRpcProvider, Signer} from 'ethers'; +import {Contract} from 'ethers'; import {Contract as MulticallContract} from "@curvefi/ethcall"; import BigNumber from 'bignumber.js'; import { @@ -17,7 +17,6 @@ import ERC20Abi from './constants/abis/ERC20.json' assert {type: 'json'}; import {L2Networks} from './constants/L2Networks.js'; import {volumeNetworks} from "./constants/volumeNetworks.js"; import {getPool} from "./pools/index.js"; -import Worker from "web-worker"; export const ETH_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; @@ -778,7 +777,12 @@ export function log(fnName: string, ...args: unknown[]): void { } } -export function runWorker(code: string, inputData: In, timeout = 30000): Promise { +export function runWorker(code: string, syncFn: () => ((val: In) => Out) | undefined, inputData: In, timeout = 30000): Promise { + if (typeof Worker === 'undefined') { + // in nodejs run worker in main thread + return Promise.resolve(syncFn()!(inputData)); + } + const blob = new Blob([code], { type: 'application/javascript' }); const blobUrl = URL.createObjectURL(blob); const worker = new Worker(blobUrl, {type: 'module'}); diff --git a/test/router.test.ts b/test/router.test.ts index 0f10d74e..14f873e7 100644 --- a/test/router.test.ts +++ b/test/router.test.ts @@ -87,8 +87,9 @@ describe('Router swap', async function () { try { await routerSwapTest(coin1, coin2); } catch (err: any) { - console.log(err.message); - assert.equal(err.message, "This pair can't be exchanged"); + if (err.message != "This pair can't be exchanged") { + throw err; + } } }); } From 5805aec0d7c15ddd7f2cbfc3c7d2532415224029 Mon Sep 17 00:00:00 2001 From: macket Date: Thu, 29 Aug 2024 16:34:34 +0400 Subject: [PATCH 9/9] build: v2.63.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 56a432b0..22b18f7a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@curvefi/api", - "version": "2.63.0", + "version": "2.63.1", "description": "JavaScript library for curve.fi", "main": "lib/index.js", "author": "Macket",