diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index 6af2dc78e75..f48ab8cd0c2 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -131,6 +131,14 @@ "uri-js": "^4.2.2" } }, + "async-retry": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", + "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "requires": { + "retry": "0.12.0" + } + }, "bignumber.js": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", @@ -217,6 +225,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, + "retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + }, "web3": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/web3/-/web3-1.2.8.tgz", @@ -3156,11 +3169,11 @@ "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" }, "async-retry": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", - "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", "requires": { - "retry": "0.12.0" + "retry": "0.13.1" } }, "asynckit": { @@ -12386,9 +12399,9 @@ "dev": true }, "retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" }, "retry-as-promised": { "version": "2.3.2", diff --git a/creator-node/package.json b/creator-node/package.json index c137513090e..866e1d235f8 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -21,6 +21,7 @@ "dependencies": { "@audius/libs": "1.2.37", "JSONStream": "^1.3.5", + "async-retry": "^1.3.3", "axios": "^0.19.2", "base64-url": "^2.3.3", "bl": "^4.1.0", diff --git a/creator-node/src/snapbackSM/computeSyncModeForUserAndReplica.js b/creator-node/src/snapbackSM/computeSyncModeForUserAndReplica.js new file mode 100644 index 00000000000..4fce195f04c --- /dev/null +++ b/creator-node/src/snapbackSM/computeSyncModeForUserAndReplica.js @@ -0,0 +1,129 @@ +const DBManager = require('../dbManager.js') +const retry = require('async-retry') + +/** + * Sync mode for a (primary, secondary) pair for a user + */ +const SyncMode = Object.freeze({ + None: 'NONE', + SecondaryShouldSync: 'SECONDARY_SHOULD_SYNC', + PrimaryShouldSync: 'PRIMARY_SHOULD_SYNC' +}) + +const FetchFilesHashNumRetries = 3 + +/** + * Given user state info, determines required sync mode for user and replica. This fn is called for each (primary, secondary) pair + * @notice Is used when both replicas are running version >= 0.3.51 + * @param {Object} param + * @param {string} param.wallet user wallet + * @param {number} param.primaryClock clock value on user's primary + * @param {number} param.secondaryClock clock value on user's secondary + * @param {string} param.primaryFilesHash filesHash on user's primary + * @param {string} param.secondaryFilesHash filesHash on user's secondary + * @returns {SyncMode} syncMode one of None, SecondaryShouldSync, PrimaryShouldSync + */ +async function computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash, + logger +}) { + if ( + !Number.isInteger(primaryClock) || + !Number.isInteger(secondaryClock) || + // `null` is a valid filesHash value; `undefined` is not + primaryFilesHash === undefined || + secondaryFilesHash === undefined + ) { + throw new Error( + '[computeSyncModeForUserAndReplica] Error: Missing or invalid params' + ) + } + + if ( + primaryClock === secondaryClock && + primaryFilesHash === secondaryFilesHash + ) { + /** + * Nodes have identical data -> no sync needed + */ + return SyncMode.None + } else if ( + primaryClock === secondaryClock && + primaryFilesHash !== secondaryFilesHash + ) { + /** + * If clocks are same but filesHashes are not, this means secondary and primary states for user + * have diverged. To fix this issue, primary should sync content from secondary and + * subsequently force secondary to resync state from primary. + */ + return SyncMode.PrimaryShouldSync + } else if (primaryClock < secondaryClock) { + /** + * Secondary has more data than primary -> primary must sync from secondary + */ + return SyncMode.PrimaryShouldSync + } else if (primaryClock > secondaryClock && secondaryFilesHash === null) { + /** + * secondaryFilesHash will be null if secondary has no files for user -> secondary must sync from primary + */ + return SyncMode.SecondaryShouldSync + } else if (primaryClock > secondaryClock && secondaryFilesHash !== null) { + /** + * If primaryClock > secondaryClock, need to check that nodes have same content for each clock value. To do this, we compute filesHash from primary matching clock range from secondary. + */ + try { + // Throws error if failure after all retries + const primaryFilesHashForRange = await retry( + async () => + DBManager.fetchFilesHashFromDB({ + lookupKey: { lookupWallet: wallet }, + clockMin: 0, + clockMax: secondaryClock + 1 + }), + { retries: FetchFilesHashNumRetries } + ) + + if (primaryFilesHashForRange === secondaryFilesHash) { + return SyncMode.SecondaryShouldSync + } else { + return SyncMode.PrimaryShouldSync + } + } catch (e) { + const errorMsg = `[computeSyncModeForUserAndReplica] Error: failed DBManager.fetchFilesHashFromDB() - ${e.message}` + logger.error(errorMsg) + throw new Error(errorMsg) + } + } else { + return SyncMode.None + } +} + +/** + * Given user state info, determines required sync mode for user and replica. This fn is called for each (primary, secondary) pair + * @notice Is used when at least 1 replica is running version < 0.3.51 + * @param {Object} param + * @param {string} param.wallet user wallet + * @param {number} param.primaryClock clock value on user's primary + * @param {number} param.secondaryClock clock value on user's secondary + * @returns {SyncMode} syncMode one of None, SecondaryShouldSync, PrimaryShouldSync + */ +function computeSyncModeForUserAndReplicaLegacy({ + primaryClock, + secondaryClock +}) { + if (primaryClock > secondaryClock) { + return SyncMode.SecondaryShouldSync + } else { + return SyncMode.None + } +} + +module.exports = { + SyncMode, + computeSyncModeForUserAndReplica, + computeSyncModeForUserAndReplicaLegacy +} diff --git a/creator-node/src/snapbackSM/snapbackSM.js b/creator-node/src/snapbackSM/snapbackSM.js index d5a105d6562..d0b4a946f5d 100644 --- a/creator-node/src/snapbackSM/snapbackSM.js +++ b/creator-node/src/snapbackSM/snapbackSM.js @@ -11,6 +11,11 @@ const PeerSetManager = require('./peerSetManager') const CreatorNode = require('@audius/libs/src/services/creatorNode') const SecondarySyncHealthTracker = require('./secondarySyncHealthTracker') const { generateTimestampAndSignature } = require('../apiSigning') +const { + SyncMode, + computeSyncModeForUserAndReplica, + computeSyncModeForUserAndReplicaLegacy +} = require('./computeSyncModeForUserAndReplica.js') // Retry delay between requests during monitoring const SyncMonitoringRetryDelayMs = 15000 @@ -671,132 +676,137 @@ class SnapbackSM { } /** - * Given map(replica set node => userWallets[]), retrieves clock values for every (node, userWallet) pair + * Given map(replica set node => userWallets[]), retrieves user info for every (node, userWallet) pair * @param {Object} replicaSetNodesToUserWalletsMap map of * @param {Set} unhealthyPeers set of unhealthy peer endpoints * @param {number?} [maxUserClockFetchAttempts=10] max number of attempts to fetch clock values * - * @returns {Object} map of peer endpoints to (map of user wallet strings to clock value of replica set node for user) + * @returns {Object} map(replica => map(wallet => { clock, filesHash })) */ - async retrieveClockStatusesForUsersAcrossReplicaSet( - replicaSetNodesToUserWalletsMap, + async retrieveUserInfoFromReplicaSet( + replicasToWalletsMap, unhealthyPeers, maxUserClockFetchAttempts = 10 ) { - const replicaSetNodesToUserClockValuesMap = {} - - const replicaSetNodes = Object.keys(replicaSetNodesToUserWalletsMap) + const replicasToUserInfoMap = {} // TODO change to batched parallel requests + const replicas = Object.keys(replicasToWalletsMap) await Promise.all( - replicaSetNodes.map(async (replicaSetNode) => { - replicaSetNodesToUserClockValuesMap[replicaSetNode] = {} + replicas.map(async (replica) => { + replicasToUserInfoMap[replica] = {} - const replicaSetNodeUserWallets = - replicaSetNodesToUserWalletsMap[replicaSetNode] - const { timestamp, signature } = generateTimestampAndSignature( - { spID: this.spID }, - this.delegatePrivateKey - ) + const walletsOnReplica = replicasToWalletsMap[replica] const axiosReqParams = { - baseURL: replicaSetNode, - url: '/users/batch_clock_status', + baseURL: replica, + url: '/users/batch_clock_status?returnFilesHash=true', method: 'post', - data: { walletPublicKeys: replicaSetNodeUserWallets }, - timeout: BATCH_CLOCK_STATUS_REQUEST_TIMEOUT, - params: { - spID: this.spID, - timestamp, - signature - } + data: { walletPublicKeys: walletsOnReplica }, + timeout: BATCH_CLOCK_STATUS_REQUEST_TIMEOUT } - let userClockValuesResp = [] + // Generate and attach SP signature to bypass route rate limits + const { timestamp, signature } = generateTimestampAndSignature( + { spID: this.spID }, + this.delegatePrivateKey + ) + axiosReqParams.params = { spID: this.spID, timestamp, signature } + + // Make axios request with retries + // TODO replace with asyncRetry + let batchClockStatusResp = [] let userClockFetchAttempts = 0 let errorMsg while (userClockFetchAttempts++ < maxUserClockFetchAttempts) { try { - userClockValuesResp = (await axios(axiosReqParams)).data.data.users + batchClockStatusResp = (await axios(axiosReqParams)).data.data.users } catch (e) { errorMsg = e } } // If failed to get response after all attempts, add replica to `unhealthyPeers` list for reconfig - if (userClockValuesResp.length === 0) { + if (batchClockStatusResp.length === 0) { this.logError( - `[retrieveClockStatusesForUsersAcrossReplicaSet] Could not fetch clock values for wallets=${replicaSetNodeUserWallets} on replica node=${replicaSetNode} ${ + `[retrieveUserInfoFromReplicaSet] Could not fetch clock values for wallets=${walletsOnReplica} on replica node=${replica} ${ errorMsg ? ': ' + errorMsg.toString() : '' }` ) - unhealthyPeers.add(replicaSetNode) + unhealthyPeers.add(replica) } - userClockValuesResp.forEach((userClockValueResp) => { - const { walletPublicKey, clock } = userClockValueResp - try { - replicaSetNodesToUserClockValuesMap[replicaSetNode][ - walletPublicKey - ] = clock - } catch (e) { - // TODO: would this ever error actually? - this.log( - `Error updating replicaSetNodesToUserClockValuesMap for wallet ${walletPublicKey} to clock ${clock}` - ) - throw e - } + // Else, add response data to output aggregate map + batchClockStatusResp.forEach((clockStatusResp) => { + /** + * @notice `filesHash` will be undefined if node is running version < 0.3.51 + * @notice `filesHash` will be null if node is running version >= 0.3.51 but has no files for user + * - Note this can happen even if clock > 0 if user has AudiusUser or Track table records without any File table records + */ + const { walletPublicKey, clock, filesHash } = clockStatusResp + replicasToUserInfoMap[replica][walletPublicKey] = { clock, filesHash } }) }) ) - return replicaSetNodesToUserClockValuesMap + return replicasToUserInfoMap } /** - * Issues SyncRequests for every user from primary (this node) to secondary if needed - * Only issues requests if primary clock value is greater than secondary clock value + * Issues SyncRequests for every user from primary (this node) to secondary as needed * * @param {Object[]} userReplicaSets array of objects of schema { user_id, wallet, primary, secondary1, secondary2, endpoint } * `endpoint` field indicates secondary on which to issue SyncRequest - * @param {Object} replicaSetNodesToUserClockStatusesMap map(replica set node => map(userWallet => clockValue)) + * @param {Object} replicasToUserInfoMap map(replica => map(wallet => { clock, filesHash })) * @returns {Object} number of syncs required, enqueued, and errors if any */ - async issueSyncRequestsToSecondaries( - userReplicaSets, - replicaSetNodesToUserClockStatusesMap - ) { - // Retrieve clock values for all users on this node, which is their primary + async issueSyncRequestsToSecondaries(userReplicaSets, replicasToUserInfoMap) { let numSyncRequestsRequired = 0 let numSyncRequestsEnqueued = 0 const enqueueSyncRequestErrors = [] + // Only process users with this node as primary + userReplicaSets = userReplicaSets.filter( + (userReplicaSet) => userReplicaSet.primary === this.endpoint + ) + + // TODO change to sequential or batched parallel? await Promise.all( userReplicaSets.map(async (user) => { try { - const { - wallet, - primary, - secondary1, - secondary2, - endpoint: secondary - } = user - - // Short-circuit if primary is not self - this function is meant to be called from primary to secondaries only - if (primary !== this.endpoint) { - this.logError( - `issueSyncRequests || Can only be called by user's primary. User ${wallet} - replicaset [${primary}, ${secondary1}, ${secondary2}].` + const { wallet, primary, endpoint: secondary } = user + + const { clock: primaryClock, filesHash: primaryFilesHash } = + replicasToUserInfoMap[primary][wallet] + const { clock: secondaryClock, filesHash: secondaryFilesHash } = + replicasToUserInfoMap[secondary][wallet] + + let syncMode = SyncMode.None + + // filesHash value will be undefined if at least 1 replica is running version < 0.3.51 + if ( + primaryFilesHash === undefined || + secondaryFilesHash === undefined + ) { + this.logWarn( + `[issueSyncRequestsToSecondaries] Falling back to computeSyncModeForUserAndReplicaLegacy() [primaryFilesHash: ${primaryFilesHash}] secondaryFilesHash: ${secondaryFilesHash}` ) - return + syncMode = computeSyncModeForUserAndReplicaLegacy({ + primaryClock, + secondaryClock + }) + } else { + syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash, + logger + }) } - // Determine if secondary requires a sync by comparing clock values against primary (this node) - const userPrimaryClockVal = - replicaSetNodesToUserClockStatusesMap[primary][wallet] - const userSecondaryClockVal = - replicaSetNodesToUserClockStatusesMap[secondary][wallet] - - if (userPrimaryClockVal > userSecondaryClockVal) { + if (syncMode === SyncMode.SecondaryShouldSync) { numSyncRequestsRequired += 1 await this.enqueueSync({ @@ -807,10 +817,19 @@ class SnapbackSM { }) numSyncRequestsEnqueued += 1 + } else if (syncMode === SyncMode.PrimaryShouldSync) { + /** + * Log as placeholder + * TODO + * 1. await this.syncFromSecondary() + * 2. issue sync to secondary with forceResync = true + */ + this.log( + `[issueSyncRequestsToSecondaries] [PrimaryShouldSync = true] [SyncType = ${SyncType.Recurring}] wallet ${wallet} secondary ${secondary} Clocks: [${primaryClock},${secondaryClock}] Files hashes: [${primaryFilesHash},${secondaryFilesHash}]` + ) } - - // Swallow error without short-circuiting other processing } catch (e) { + // Swallow error without short-circuiting other processing enqueueSyncRequestErrors.push( `issueSyncRequestsToSecondaries() Error for user ${JSON.stringify( user @@ -935,26 +954,25 @@ class SnapbackSM { }) } - // Retrieve clock statuses for all users and their current replica sets - let replicaSetNodesToUserClockStatusesMap + // Retrieve user info for all users and their current replica sets + let replicasToUserInfoMap try { - replicaSetNodesToUserClockStatusesMap = - await this.retrieveClockStatusesForUsersAcrossReplicaSet( - replicaSetNodesToUserWalletsMap, - unhealthyPeers - ) + replicasToUserInfoMap = await this.retrieveUserInfoFromReplicaSet( + replicaSetNodesToUserWalletsMap, + unhealthyPeers + ) decisionTree.push({ - stage: 'retrieveClockStatusesForUsersAcrossReplicaSet() Success', + stage: 'retrieveUserInfoFromReplicaSet() Success', time: Date.now() }) } catch (e) { decisionTree.push({ - stage: 'retrieveClockStatusesForUsersAcrossReplicaSet() Error', + stage: 'retrieveUserInfoFromReplicaSet() Error', vals: e.message, time: Date.now() }) throw new Error( - 'processStateMachineOperation():retrieveClockStatusesForUsersAcrossReplicaSet() Error' + 'processStateMachineOperation():retrieveUserInfoFromReplicaSet() Error' ) } @@ -980,7 +998,7 @@ class SnapbackSM { try { const resp = await this.issueSyncRequestsToSecondaries( potentialSyncRequests, - replicaSetNodesToUserClockStatusesMap + replicasToUserInfoMap ) numSyncRequestsRequired = resp.numSyncRequestsRequired numSyncRequestsEnqueued = resp.numSyncRequestsEnqueued @@ -1778,4 +1796,9 @@ class SnapbackSM { } } -module.exports = { SnapbackSM, SyncType, RECONFIG_MODE_KEYS, RECONFIG_MODES } +module.exports = { + SnapbackSM, + SyncType, + RECONFIG_MODE_KEYS, + RECONFIG_MODES +} diff --git a/creator-node/src/snapbackSM/snapbackSM.test.js b/creator-node/src/snapbackSM/snapbackSM.test.js new file mode 100644 index 00000000000..3d0acfc4bfe --- /dev/null +++ b/creator-node/src/snapbackSM/snapbackSM.test.js @@ -0,0 +1,202 @@ +/** Unit tests for SnapbackSM module */ + +const assert = require('assert') +const proxyquire = require('proxyquire') + +const DBManager = require('../dbManager.js') +const { logger } = require('../logging.js') + +const { + SyncMode, + computeSyncModeForUserAndReplica +} = require('./computeSyncModeForUserAndReplica.js') +describe('Test computeSyncModeForUserAndReplica()', function () { + let primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash, + primaryFilesHashMock + + // Can be anything for test purposes + const wallet = 'wallet' + + it('Throws if missing or invalid params', async function () { + primaryClock = 10 + secondaryClock = 10 + primaryFilesHash = undefined + secondaryFilesHash = undefined + + try { + await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + } catch (e) { + assert.strictEqual( + e.message, + '[computeSyncModeForUserAndReplica] Error: Missing or invalid params' + ) + } + }) + + it('Returns SyncMode.None if clocks and filesHashes equal', async function () { + primaryClock = 10 + secondaryClock = primaryClock + primaryFilesHash = '0x123' + secondaryFilesHash = primaryFilesHash + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.None) + }) + + it('Returns SyncMode.PrimaryShouldSync if clocks equal and filesHashes unequal', async function () { + primaryClock = 10 + secondaryClock = primaryClock + primaryFilesHash = '0x123' + secondaryFilesHash = '0x456' + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.PrimaryShouldSync) + }) + + it('Returns SyncMode.PrimaryShouldSync if primaryClock < secondaryClock', async function () { + primaryClock = 5 + secondaryClock = 10 + primaryFilesHash = '0x123' + secondaryFilesHash = '0x456' + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.PrimaryShouldSync) + }) + + it('Returns SyncMode.SecondaryShouldSync if primaryClock > secondaryClock & secondaryFilesHash === null', async function () { + primaryClock = 10 + secondaryClock = 5 + primaryFilesHash = '0x123' + secondaryFilesHash = null + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.SecondaryShouldSync) + }) + + describe('primaryClock > secondaryClock', function () { + it('Returns SyncMode.SecondaryShouldSync if primaryFilesHashForRange = secondaryFilesHash', async function () { + primaryClock = 10 + secondaryClock = 5 + primaryFilesHash = '0x123' + secondaryFilesHash = '0x456' + + // Mock DBManager.fetchFilesHashFromDB() to return `secondaryFilesHash` for clock range + const DBManagerMock = DBManager + DBManagerMock.fetchFilesHashFromDB = async () => { + return secondaryFilesHash + } + proxyquire('./computeSyncModeForUserAndReplica.js', { + '../dbManager.js': DBManagerMock + }) + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.SecondaryShouldSync) + }) + + it('Returns SyncMode.PrimaryShouldSync if primaryFilesHashForRange != secondaryFilesHash', async function () { + primaryClock = 10 + secondaryClock = 5 + primaryFilesHash = '0x123' + secondaryFilesHash = '0x456' + primaryFilesHashMock = '0x789' + + // Mock DBManager.fetchFilesHashFromDB() to return different filesHash for clock range + const DBManagerMock = DBManager + DBManagerMock.fetchFilesHashFromDB = async () => { + return primaryFilesHashMock + } + proxyquire('./computeSyncModeForUserAndReplica.js', { + '../dbManager.js': DBManagerMock + }) + + const syncMode = await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash + }) + + assert.strictEqual(syncMode, SyncMode.PrimaryShouldSync) + }) + + it("Throws error primaryFilesHashForRange can't be retrieved", async function () { + // Increase mocha test timeout from default 2s to accommodate `async-retry` runtime + this.timeout(30000) // 30s + + primaryClock = 10 + secondaryClock = 5 + primaryFilesHash = '0x123' + secondaryFilesHash = '0x456' + + // Mock DBManager.fetchFilesHashFromDB() to throw error + const DBManagerMock = require('../dbManager.js') + DBManagerMock.fetchFilesHashFromDB = async () => { + throw new Error('Mock - Failed to fetch filesHash') + } + proxyquire('./computeSyncModeForUserAndReplica.js', { + '../dbManager.js': DBManagerMock + }) + + try { + await computeSyncModeForUserAndReplica({ + wallet, + primaryClock, + secondaryClock, + primaryFilesHash, + secondaryFilesHash, + logger + }) + } catch (e) { + assert.strictEqual( + e.message, + '[computeSyncModeForUserAndReplica] Error: failed DBManager.fetchFilesHashFromDB() - Mock - Failed to fetch filesHash' + ) + } + }) + }) +}) diff --git a/creator-node/test/snapbackSM.test.js b/creator-node/test/snapbackSM.test.js index f069a55fe33..4fd750e52f1 100644 --- a/creator-node/test/snapbackSM.test.js +++ b/creator-node/test/snapbackSM.test.js @@ -1,3 +1,5 @@ +/** Integration tests for SnapbackSM module */ + const nock = require('nock') const assert = require('assert')