perf: router workers and optimizations

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@curvefi/api",
-  "version": "2.63.0",
+  "version": "2.63.1",
   "description": "JavaScript library for curve.fi",
   "main": "lib/index.js",
   "author": "Macket",

src/curve.ts

@@ -5,6 +5,9 @@ import {
     BigNumberish,
     Numeric,
     AbstractProvider,
+    BrowserProvider,
+    JsonRpcProvider,
+    Signer,
 } from "ethers";
 import { Provider as MulticallProvider, Contract as MulticallContract } from "@curvefi/ethcall";
 import { getFactoryPoolData } from "./factory/factory.js";
@@ -103,26 +106,34 @@ import { COINS_FRAXTAL, cTokensFraxtal,  yTokensFraxtal, ycTokensFraxtal, aToken
 import { COINS_XLAYER, cTokensXLayer,  yTokensXLayer, ycTokensXLayer, aTokensXLayer } from "./constants/coins/xlayer.js";
 import { COINS_MANTLE, cTokensMantle,  yTokensMantle, ycTokensMantle, aTokensMantle } from "./constants/coins/mantle.js";
 import { lowerCasePoolDataAddresses, extractDecimals, extractGauges } from "./constants/utils.js";
-import { _getAllGauges, _getHiddenPools } from "./external-api.js";
+import { _getHiddenPools } from "./external-api.js";
 import { L2Networks } from "./constants/L2Networks.js";
 import { getTwocryptoFactoryPoolData } from "./factory/factory-twocrypto.js";
-import {getGasInfoForL2, memoizedContract, memoizedMulticallContract} from "./utils.js";
-
-const _killGauges = async (poolsData: IDict<IPoolData>): Promise<void> => {
-    const gaugeData = await _getAllGauges();
-    const isKilled: IDict<boolean> = {};
-    const gaugeStatuses: IDict<Record<string, boolean> | null> = {};
-    Object.values(gaugeData).forEach((d) => {
-        isKilled[d.gauge.toLowerCase()] = d.is_killed ?? false;
-        gaugeStatuses[d.gauge.toLowerCase()] = d.gaugeStatus ?? null;
-    });
 
-    for (const poolId in poolsData) {
-        if (isKilled[poolsData[poolId].gauge_address]) {
-            poolsData[poolId].is_gauge_killed = true;
+export const memoizedContract = (): (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer) => Contract => {
+    const cache: Record<string, Contract> = {};
+    return (address: string, abi: any, provider: BrowserProvider | JsonRpcProvider | Signer): Contract => {
+        if (address in cache) {
+            return cache[address];
+        }
+        else {
+            const result = new Contract(address, abi, provider)
+            cache[address] = result;
+            return result;
+        }
+    }
+}
+
+export const memoizedMulticallContract = (): (address: string, abi: any) => MulticallContract => {
+    const cache: Record<string, MulticallContract> = {};
+    return (address: string, abi: any): MulticallContract => {
+        if (address in cache) {
+            return cache[address];
         }
-        if (gaugeStatuses[poolsData[poolId].gauge_address]) {
-            poolsData[poolId].gauge_status = gaugeStatuses[poolsData[poolId].gauge_address];
+        else {
+            const result = new MulticallContract(address, abi)
+            cache[address] = result;
+            return result;
         }
     }
 }

src/interfaces.ts

@@ -128,12 +128,7 @@ export interface IPoolDataShort {
     address: string,
 }
 
-export interface ISubgraphPoolData {
-    address: string,
-    volumeUSD: number,
-    latestDailyApy: number,
-    latestWeeklyApy: number,
-}
+export type IRoutePoolData = Pick<IPoolData, 'is_lending' | 'wrapped_coin_addresses' | 'underlying_coin_addresses' | 'token_address'>;
 
 export interface IExtendedPoolDataFromApi {
     poolData: IPoolDataFromApi[],

src/route-finder.worker.ts

@@ -0,0 +1,133 @@
+// important: only type imports, the worker needs to be standalone
+import type {IDict, IRoutePoolData, IRouteStep, IRouteTvl, ISwapType} from "./interfaces";
+
+export type IRouterWorkerInput = {
+    inputCoinAddress: string,
+    outputCoinAddress: string,
+    routerGraph: IDict<IDict<IRouteStep[]>>,
+    poolData: IDict<IRoutePoolData>
+}
+
+export function routeFinderWorker() {
+    const MAX_ROUTES_FOR_ONE_COIN = 5;
+    const MAX_DEPTH = 4;
+
+    const _removeDuplications = (routesA: IRouteTvl[], routesB: IRouteTvl[]) => {
+        const routeToStr = (r: IRouteTvl) => r.route.map((s) => s.poolId).toString();
+        const routeIdsA = new Set(routesA.map(routeToStr));
+        return routesA.concat(routesB.filter((r) => !routeIdsA.has(routeToStr(r))));
+    }
+
+    const _sortByTvl = (a: IRouteTvl, b: IRouteTvl) => b.minTvl - a.minTvl || b.totalTvl - a.totalTvl || a.route.length - b.route.length;
+    const _sortByLength = (a: IRouteTvl, b: IRouteTvl) => a.route.length - b.route.length || b.minTvl - a.minTvl || b.totalTvl - a.totalTvl;
+
+    // 4 --> 6, 5 --> 7 not allowed
+    // 4 --> 7, 5 --> 6 allowed
+    const _handleSwapType = (swapType: ISwapType): string => {
+        if (swapType === 6) return "4";
+        if (swapType === 7) return "5";
+        return swapType.toString()
+    }
+
+    /** Add step to route */
+    const _addStep = (route: IRouteTvl, step: IRouteStep) => ({
+        route: route.route.concat(step),
+        minTvl: Math.min(step.tvl, route.minTvl),
+        totalTvl: route.totalTvl + step.tvl,
+    });
+
+    /** Check if item fits in a sorted-sized array */
+    function _fits<T>(array: T[], item: T, compareFn: (a: T, b: T) => number, maxSize: number) {
+        if (array.length < maxSize) return true;
+        const last = array[array.length - 1];
+        return compareFn(item, last) < 0;
+    }
+
+    /** Add item to sorted-sized array */
+    function _sortedPush<T>(array: T[], item: T, compareFn: (a: T, b: T) => number, maxSize: number) {
+        if (!_fits(array, item, compareFn, maxSize)) return;
+        if (array.length === maxSize) {
+            array.pop();
+        }
+        const position = array.findIndex((existingItem) => compareFn(item, existingItem) < 0);
+        if (position === -1) {
+            array.push(item);
+        } else {
+            array.splice(position, 0, item);
+        }
+    }
+
+    const _isVisitedCoin = (coinAddress: string, route: IRouteTvl): boolean =>
+        route.route.find((r) => r.inputCoinAddress === coinAddress) !== undefined
+
+    const _findPool = (route: IRouteTvl, poolId: string) => route.route.find((r) => r.poolId === poolId);
+
+    const findRoutes = ({ inputCoinAddress, outputCoinAddress, routerGraph, poolData }: IRouterWorkerInput): IRouteStep[][] => {
+        inputCoinAddress = inputCoinAddress.toLowerCase();
+        outputCoinAddress = outputCoinAddress.toLowerCase();
+
+        const routes: IRouteTvl[] = [{route: [], minTvl: Infinity, totalTvl: 0}];
+        const targetRoutesByTvl: IRouteTvl[] = [];
+        const targetRoutesByLength: IRouteTvl[] = [];
+
+        while (routes.length) {
+            const route = routes.pop() as IRouteTvl;
+            const inCoin = route.route.length > 0 ? route.route[route.route.length - 1].outputCoinAddress : inputCoinAddress;
+            Object.entries(routerGraph[inCoin]).forEach((leaf) => {
+                const outCoin = leaf[0], steps = leaf[1];
+                if (_isVisitedCoin(outCoin, route)) return;
+
+                steps.forEach((step) => {
+                    const pool = poolData[step.poolId];
+
+                    const currentPoolInRoute = _findPool(route, step.poolId);
+                    if (currentPoolInRoute) {
+                        if (!pool?.is_lending) return;
+                        // 4 --> 6, 5 --> 7 not allowed
+                        // 4 --> 7, 5 --> 6 allowed
+                        if (_handleSwapType(step.swapParams[2]) === _handleSwapType(currentPoolInRoute.swapParams[2])) {
+                            return;
+                        }
+                    }
+
+                    if (step.outputCoinAddress === outputCoinAddress) {
+                        const newRoute = _addStep(route, step);
+                        _sortedPush(targetRoutesByTvl, newRoute, _sortByTvl, MAX_ROUTES_FOR_ONE_COIN);
+                        _sortedPush(targetRoutesByLength, newRoute, _sortByLength, MAX_ROUTES_FOR_ONE_COIN);
+                        return;
+                    }
+
+                    if (pool?.wrapped_coin_addresses.includes(outputCoinAddress) || pool?.underlying_coin_addresses.includes(outputCoinAddress)) {
+                        // Exclude such cases as: cvxeth -> tricrypto2 -> tusd -> susd (cvxeth -> tricrypto2 -> tusd instead)
+                        if (!pool?.is_lending) return;
+                        // Exclude such cases as: aave -> aave -> 3pool (aave -> aave instead)
+                        if (outCoin !== pool?.token_address) return;
+                    }
+
+                    if (route.route.length < MAX_DEPTH) {
+                        const newRoute = _addStep(route, step);
+                        if (_fits(targetRoutesByTvl, newRoute, _sortByTvl, MAX_ROUTES_FOR_ONE_COIN) ||
+                            _fits(targetRoutesByLength, newRoute, _sortByLength, MAX_ROUTES_FOR_ONE_COIN)) {
+                            routes.push(newRoute); // try another step
+                        }
+                    }
+                })
+            })
+        }
+        return _removeDuplications(targetRoutesByTvl, targetRoutesByLength).map((r) => r.route);
+    }
+
+    if (typeof addEventListener === 'undefined') {
+        return findRoutes; // for nodejs
+    }
+
+    addEventListener('message', (e) => {
+        const { type } = e.data;
+        if (type === 'findRoutes') {
+            postMessage({ type, result: findRoutes(e.data) });
+        }
+    });
+}
+
+// this is a workaround to avoid importing web-worker in the main bundle (nextjs will try to inject invalid hot-reloading code)
+export const routeFinderWorkerCode = `${routeFinderWorker.toString()}; ${routeFinderWorker.name}();`;