Merge pull request #39 from PowerLoom/feat/preloader_arch

Preloader architecture, memory, and other improvements
PowerLoom · Jul 28, 2023 · 56c3dd7 · 56c3dd7
2 parents 762fa59 + 4cecbbb
commit 56c3dd7
Show file tree

Hide file tree

Showing 121 changed files with 4,169 additions and 3,058 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,5 +8,5 @@ temp/*
 uniswapTokenData
 **/*projects.json
 **/*aggregator.json
-**/*settings.json
+config/*settings.json
 **/*.backup
diff --git a/README.md b/README.md
diff --git a/config/aggregator.example.json b/config/aggregator.example.json
@@ -6,7 +6,7 @@
         "projectId": "pairContract_trade_volume"
       },
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_24h",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.single_uniswap_trade_volume_24h",
         "class_name": "AggreagateTradeVolumeProcessor"
       }
     },
@@ -17,7 +17,7 @@
         "projectId": "aggregate_pairContract_24h_trade_volume"
       },
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.single_uniswap_trade_volume_7d",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.single_uniswap_trade_volume_7d",
         "class_name": "AggreagateTradeVolumeProcessor"
       }
     },
@@ -41,7 +41,7 @@
         "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
       ],
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_24h",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_pairs_24h",
         "class_name": "AggreagateTopPairsProcessor"
       }
     },
@@ -65,7 +65,7 @@
         "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
       ],
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_tokens",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_tokens",
         "class_name": "AggreagateTopTokensProcessor"
       }
     },
@@ -82,7 +82,7 @@
         "aggregate_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
       ],
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_7d",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_pairs_7d",
         "class_name": "AggreagateTopPairsProcessor"
       }
     },
@@ -106,7 +106,7 @@
         "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
       ],
       "processor": {
-        "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_stats",
+        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_stats",
         "class_name": "AggreagateStatsProcessor"
       }
     }

diff --git a/config/preloader.json b/config/preloader.json
@@ -0,0 +1,27 @@
+{
+    "preloaders": [
+        {
+          "task_type": "block_transactions",
+          "module": "snapshotter.utils.preloaders.tx_receipts.preloader",
+          "class_name": "TxPreloadWorker"
+        },
+        {
+          "task_type": "block_details",
+          "module": "snapshotter.utils.preloaders.block_details.preloader",
+          "class_name": "BlockDetailsPreloader"
+        },
+        {
+          "task_type": "eth_price",
+          "module": "snapshotter.utils.preloaders.eth_price.preloader",
+          "class_name": "EthPricePreloader"
+        }
+    ],
+    "delegate_tasks": [
+        {
+          "task_type": "txreceipt",
+          "module": "snapshotter.utils.preloaders.tx_receipts.delegated_worker.tx_receipts",
+          "class_name": "TxReceiptProcessor"
+        }
+    ],
+    "timeout": 60
+}
diff --git a/config/projects.example.json b/config/projects.example.json
@@ -1,6 +1,10 @@
 {
   "config": [{
       "project_type": "pairContract_pair_total_reserves",
+      "preload_tasks":[
+        "eth_price",
+        "block_details"
+      ],
       "projects":[
         "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
         "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
@@ -11,12 +15,16 @@
         "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
         ],
       "processor":{
-        "module": "pooler.modules.uniswapv2.pair_total_reserves",
+        "module": "snapshotter.modules.pooler.uniswapv2.pair_total_reserves",
         "class_name": "PairTotalReservesProcessor"
       }
     },
     {
       "project_type": "pairContract_trade_volume",
+      "preload_tasks":[
+        "eth_price",
+        "block_details"
+      ],
       "projects":[
         "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
         "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
@@ -27,7 +35,7 @@
         "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
         ],
         "processor":{
-          "module": "pooler.modules.uniswapv2.trade_volume",
+          "module": "snapshotter.modules.pooler.uniswapv2.trade_volume",
           "class_name": "TradeVolumeProcessor"
         }
     }

diff --git a/config/settings.example.json b/config/settings.example.json
@@ -47,6 +47,9 @@
       },
       "event_detector": {
         "exchange": "powerloom-backend-event-detector"
+      },
+      "delegated_worker": {
+        "exchange": "powerloom-backend-delegated-worker"
       }
     }
   },
@@ -74,14 +77,16 @@
   },
   "projects_config_path": "config/projects.json",
   "aggregator_config_path": "config/aggregator.json",
-  "pair_contract_abi": "pooler/static/abis/UniswapV2Pair.json",
+  "preloader_config_path": "config/preloader.json",
+  "pair_contract_abi": "snapshotter/static/abis/UniswapV2Pair.json",
   "protocol_state": {
-    "abi": "pooler/static/abis/ProtocolContract.json",
+    "abi": "snapshotter/static/abis/ProtocolContract.json",
     "address": "protocol-state-contract"
   },
   "callback_worker_config": {
-    "num_snapshot_workers": 8,
-    "num_aggregation_workers": 16
+    "num_delegate_workers": 8,
+    "num_snapshot_workers": 2,
+    "num_aggregation_workers": 4
   },
   "ipfs": {
     "url": "ipfs-writer-url",

diff --git a/init_processes.sh b/init_processes.sh
@@ -7,11 +7,11 @@ pm2 start pm2.config.js
 echo 'waiting for processes to start..';
 sleep 10
 
-poetry run python -m pooler.processhub_cmd start ProcessorDistributor
+poetry run python -m snapshotter.processhub_cmd start ProcessorDistributor
 sleep 3
 
-poetry run python -m pooler.processhub_cmd start SystemEventDetector
+poetry run python -m snapshotter.processhub_cmd start SystemEventDetector
 
-echo 'started all pooler scripts';
+echo 'started all snapshotter scripts';
 
 pm2 logs --lines 1000
diff --git a/pm2.config.js b/pm2.config.js
@@ -8,30 +8,32 @@ const MIN_UPTIME = 60000;
 module.exports = {
   apps : [
     {
-      name   : "pooler-process-hub-core",
-      script : `poetry run python -m pooler.launch_process_hub_core`,
+      name   : "process-hub-core",
+      script : `poetry run python -m snapshotter.launch_process_hub_core`,
       max_restarts: MAX_RESTART,
       min_uptime: MIN_UPTIME,
       env: {
         NODE_ENV: NODE_ENV,
       }
     },
     {
-      name   : "pooler-core-api",
-      script : `poetry run python -m pooler.gunicorn_core_launcher`,
+      name   : "core-api",
+      script : `poetry run python -m snapshotter.gunicorn_core_launcher`,
       max_restarts: MAX_RESTART,
       min_uptime: MIN_UPTIME,
       env: {
         NODE_ENV: NODE_ENV,
+        GUNICORN_WORKERS: 1,
       }
     },
     {
-      name   : "pooler-auth-api",
-      script : `poetry run python -m pooler.auth.gunicorn_auth_entry_launcher`,
+      name   : "auth-api",
+      script : `poetry run python -m snapshotter.auth.gunicorn_auth_entry_launcher`,
       max_restarts: MAX_RESTART,
       min_uptime: MIN_UPTIME,
       env: {
         NODE_ENV: NODE_ENV,
+        GUNICORN_WORKERS: 1,
       }
     }
   ]