PrimarySyncFromSecondary() improvements to error handling and logging (…

…#3703)
AudiusProject · Aug 17, 2022 · 9f9be2d · 9f9be2d
1 parent 6b38bac
commit 9f9be2d
Show file tree

Hide file tree

Showing 7 changed files with 314 additions and 194 deletions.
diff --git a/creator-node/src/middlewares.js b/creator-node/src/middlewares.js
@@ -508,32 +508,34 @@ async function getOwnEndpoint({ libs }) {
 }
 
 /**
- * Retrieves user replica set from discprov
+ * Retrieves user replica set endpoints from discprov
  *
  * Polls discprov conditionally as follows:
  *    - If blockNumber provided, polls discprov until it has indexed that blocknumber (for up to 200 seconds)
  *    - Else if ensurePrimary required, polls discprov until it has indexed myCnodeEndpoint (for up to 60 seconds)
  *      - Errors if retrieved primary does not match myCnodeEndpoint
  *    - If neither of above conditions are met, falls back to single discprov query without polling
  *
+ * @param {string} wallet - wallet used to query discprov for user data
  * @param {Object} serviceRegistry
  * @param {Object} logger
- * @param {string} wallet - wallet used to query discprov for user data
  * @param {number} blockNumber - blocknumber of eth TX preceding CN call
  * @param {string} myCnodeEndpoint - endpoint of this CN
  * @param {boolean} ensurePrimary - determines if function should error if this CN is not primary
  *
  * @returns {Array} - array of strings of replica set
  */
-async function getCreatorNodeEndpoints({
+async function getUserReplicaSetEndpointsFromDiscovery({
+  wallet,
   libs,
   logger,
-  wallet,
   blockNumber,
   ensurePrimary,
   myCnodeEndpoint
 }) {
-  logger.info(`Starting getCreatorNodeEndpoints for wallet ${wallet}`)
+  logger.info(
+    `Starting getUserReplicaSetEndpointsFromDiscovery for wallet ${wallet}`
+  )
   const start = Date.now()
 
   let user = null
@@ -551,7 +553,7 @@ async function getCreatorNodeEndpoints({
     let discprovBlockNumber = -1
     for (let retry = 1; retry <= MaxRetries; retry++) {
       logger.info(
-        `getCreatorNodeEndpoints retry #${retry}/${MaxRetries} || time from start: ${
+        `getUserReplicaSetEndpointsFromDiscovery retry #${retry}/${MaxRetries} || time from start: ${
           Date.now() - start2
         } discprovBlockNumber ${discprovBlockNumber} || blockNumber ${blockNumber}`
       )
@@ -584,7 +586,7 @@ async function getCreatorNodeEndpoints({
 
       await utils.timeout(RetryTimeout)
       logger.info(
-        `getCreatorNodeEndpoints AFTER TIMEOUT retry #${retry}/${MaxRetries} || time from start: ${
+        `getUserReplicaSetEndpointsFromDiscovery AFTER TIMEOUT retry #${retry}/${MaxRetries} || time from start: ${
           Date.now() - start2
         } discprovBlockNumber ${discprovBlockNumber} || blockNumber ${blockNumber}`
       )
@@ -609,7 +611,7 @@ async function getCreatorNodeEndpoints({
      * Errors if retrieved primary does not match myCnodeEndpoint
      */
     logger.info(
-      `getCreatorNodeEndpoints || no blockNumber passed, retrying until DN returns same endpoint`
+      `getUserReplicaSetEndpointsFromDiscovery || no blockNumber passed, retrying until DN returns same endpoint`
     )
 
     const start2 = Date.now()
@@ -621,7 +623,7 @@ async function getCreatorNodeEndpoints({
     let returnedPrimaryEndpoint = null
     for (let retry = 1; retry <= MaxRetries; retry++) {
       logger.info(
-        `getCreatorNodeEndpoints retry #${retry}/${MaxRetries} || time from start: ${
+        `getUserReplicaSetEndpointsFromDiscovery retry #${retry}/${MaxRetries} || time from start: ${
           Date.now() - start2
         } myCnodeEndpoint ${myCnodeEndpoint}`
       )
@@ -650,7 +652,7 @@ async function getCreatorNodeEndpoints({
 
       await utils.timeout(RetryTimeout)
       logger.info(
-        `getCreatorNodeEndpoints AFTER TIMEOUT retry #${retry}/${MaxRetries} || time from start: ${
+        `getUserReplicaSetEndpointsFromDiscovery AFTER TIMEOUT retry #${retry}/${MaxRetries} || time from start: ${
           Date.now() - start2
         } myCnodeEndpoint ${myCnodeEndpoint}`
       )
@@ -674,7 +676,7 @@ async function getCreatorNodeEndpoints({
      * If neither of above conditions are met, falls back to single discprov query without polling
      */
     logger.info(
-      `getCreatorNodeEndpoints || ensurePrimary === false, fetching user without retries`
+      `getUserReplicaSetEndpointsFromDiscovery || ensurePrimary === false, fetching user without retries`
     )
     user = await libs.User.getUsers(1, 0, null, wallet)
   }
@@ -692,7 +694,9 @@ async function getCreatorNodeEndpoints({
   const endpoint = user[0].creator_node_endpoint
   const userReplicaSet = endpoint ? endpoint.split(',') : []
 
-  logger.info(`getCreatorNodeEndpoints route time ${Date.now() - start}`)
+  logger.info(
+    `getUserReplicaSetEndpointsFromDiscovery route time ${Date.now() - start}`
+  )
   return userReplicaSet
 }
 
@@ -920,5 +924,5 @@ module.exports = {
   ensureValidSPMiddleware,
   issueAndWaitForSecondarySyncRequests,
   getOwnEndpoint,
-  getCreatorNodeEndpoints
+  getUserReplicaSetEndpointsFromDiscovery
 }
diff --git a/creator-node/src/services/stateMachineManager/ContentNodeInfoManager.js b/creator-node/src/services/stateMachineManager/ContentNodeInfoManager.js
@@ -4,6 +4,8 @@ const _ = require('lodash')
 /**
  * Queries to periodically keep the mapping of (Content Node endpoint -> SP ID)
  * up to date.
+ *
+ * @dev Since this class holds all state in memory, it is not concurrency-compatible
  */
 class ContentNodeInfoManager {
   constructor() {