From 5a25cacbc31acc8c9d6a02cabc00a7c8e90aa458 Mon Sep 17 00:00:00 2001 From: Sid Sethi <3323835+SidSethi@users.noreply.github.com> Date: Fri, 22 Jul 2022 18:28:11 +0000 Subject: [PATCH 1/5] Fix export bug --- .../replicaSet/exportComponentService.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/creator-node/src/components/replicaSet/exportComponentService.js b/creator-node/src/components/replicaSet/exportComponentService.js index 6ffe8f4c96d..a45f9704a80 100644 --- a/creator-node/src/components/replicaSet/exportComponentService.js +++ b/creator-node/src/components/replicaSet/exportComponentService.js @@ -1,3 +1,5 @@ +const _ = require('lodash') + const models = require('../../models') const { Transaction } = require('sequelize') @@ -105,13 +107,16 @@ const exportComponentService = async ({ } // Validate clock values or throw an error - const maxClockRecordId = Math.max( + const maxClockRecord = Math.max( ...clockRecords.map((record) => record.clock) ) - if (cnodeUser.clock !== maxClockRecordId) { - throw new Error( - `Cannot export - exported data is not consistent. Exported max clock val = ${cnodeUser.clock} and exported max ClockRecord val ${maxClockRecordId}` - ) + if (!_.isEmpty(clockRecords) && cnodeUser.clock !== maxClockRecord) { + const errorMsg = `Cannot export - exported data is not consistent. Exported max clock val = ${cnodeUser.clock} and exported max ClockRecord val ${maxClockRecord}` + if (forceExport) { + logger.error(errorMsg) + } else { + throw new Error(errorMsg) + } } cnodeUser.clockInfo = { From 1eb8886a81ef890fd153010d6d135c5845fa59a4 Mon Sep 17 00:00:00 2001 From: Sid Sethi <3323835+SidSethi@users.noreply.github.com> Date: Fri, 22 Jul 2022 18:50:39 +0000 Subject: [PATCH 2/5] whoops --- .../src/components/replicaSet/exportComponentService.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/creator-node/src/components/replicaSet/exportComponentService.js b/creator-node/src/components/replicaSet/exportComponentService.js index a45f9704a80..70022403bb4 100644 --- a/creator-node/src/components/replicaSet/exportComponentService.js +++ b/creator-node/src/components/replicaSet/exportComponentService.js @@ -112,11 +112,7 @@ const exportComponentService = async ({ ) if (!_.isEmpty(clockRecords) && cnodeUser.clock !== maxClockRecord) { const errorMsg = `Cannot export - exported data is not consistent. Exported max clock val = ${cnodeUser.clock} and exported max ClockRecord val ${maxClockRecord}` - if (forceExport) { - logger.error(errorMsg) - } else { - throw new Error(errorMsg) - } + throw new Error(errorMsg) } cnodeUser.clockInfo = { From 88cb36a11e8cdc6453a19edcd7ded3ed6c900ebc Mon Sep 17 00:00:00 2001 From: Sid Sethi <3323835+SidSethi@users.noreply.github.com> Date: Fri, 22 Jul 2022 21:32:42 +0000 Subject: [PATCH 3/5] test --- creator-node/test/sync.test.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/creator-node/test/sync.test.js b/creator-node/test/sync.test.js index c5985bef4ae..5a9ed60e85e 100644 --- a/creator-node/test/sync.test.js +++ b/creator-node/test/sync.test.js @@ -21,6 +21,7 @@ const { const { uploadTrack } = require('./lib/helpers') const BlacklistManager = require('../src/blacklistManager') const sessionManager = require('../src/sessionManager') +const exportComponentService = require('../src/components/replicaSet/exportComponentService') const redisClient = require('../src/redis') const { stringifiedDateFields } = require('./lib/utils') @@ -702,7 +703,7 @@ describe('test nodesync', async function () { }) }) - describe('Confirm export throws an error with inconsitent data', async function () { + describe('Confirm export throws an error with inconsistent data', async function () { beforeEach(setupDepsAndApp) beforeEach(createUserAndTrack) @@ -710,22 +711,24 @@ describe('test nodesync', async function () { it('Inconsistent clock values', async function () { // Mock findOne DB function for cnodeUsers and ClockRecords // Have them return inconsistent values + const clockRecordTableClock = 8 const clockRecordsFindAllStub = sandbox.stub().resolves([ { - clock: 8 + clock: clockRecordTableClock } ]) + const cnodeUserTableClock = 7 const cNodeUserFindAll = sandbox.stub().resolves([ { // Random UUID cnodeUserUUID: '48523a08-2a11-4200-8aac-ae74b8a39dd0', - clock: 7 + clock: cnodeUserTableClock } ]) const modelsMock = { ...require('../src/models'), - ClockRecords: { + ClockRecord: { findAll: clockRecordsFindAllStub }, CNodeUser: { @@ -747,13 +750,11 @@ describe('test nodesync', async function () { logger: console }) ).to.eventually.be.rejectedWith( - 'Cannot export - exported data is not consistent. Exported max clock val = 7 and exported max ClockRecord val -Infinity' + `Cannot export - exported data is not consistent. Exported max clock val = ${cnodeUserTableClock} and exported max ClockRecord val ${clockRecordTableClock}` ) - /** - * - * Verify - */ + expect(clockRecordsFindAllStub).to.have.been.calledOnce + expect(cNodeUserFindAll).to.have.been.calledOnce }) }) }) From f650db3f35107d09c679134d7b74ec992aa1fe99 Mon Sep 17 00:00:00 2001 From: Sid Sethi <3323835+SidSethi@users.noreply.github.com> Date: Fri, 22 Jul 2022 19:33:23 +0000 Subject: [PATCH 4/5] Force export flag --- .../src/components/replicaSet/exportComponentService.js | 6 +++++- creator-node/src/routes/nodeSync.js | 2 ++ creator-node/src/services/sync/primarySyncFromSecondary.js | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/creator-node/src/components/replicaSet/exportComponentService.js b/creator-node/src/components/replicaSet/exportComponentService.js index 70022403bb4..5221f3fe4d2 100644 --- a/creator-node/src/components/replicaSet/exportComponentService.js +++ b/creator-node/src/components/replicaSet/exportComponentService.js @@ -14,6 +14,7 @@ const exportComponentService = async ({ walletPublicKeys, requestedClockRangeMin, requestedClockRangeMax, + forceExport, logger }) => { const transaction = await models.sequelize.transaction({ @@ -112,7 +113,10 @@ const exportComponentService = async ({ ) if (!_.isEmpty(clockRecords) && cnodeUser.clock !== maxClockRecord) { const errorMsg = `Cannot export - exported data is not consistent. Exported max clock val = ${cnodeUser.clock} and exported max ClockRecord val ${maxClockRecord}` - throw new Error(errorMsg) + logger.error(errorMsg) + if (!forceExport) { + throw new Error(errorMsg) + } } cnodeUser.clockInfo = { diff --git a/creator-node/src/routes/nodeSync.js b/creator-node/src/routes/nodeSync.js index 8aa2ec870be..b69dea09c39 100644 --- a/creator-node/src/routes/nodeSync.js +++ b/creator-node/src/routes/nodeSync.js @@ -29,6 +29,7 @@ router.get( const walletPublicKeys = req.query.wallet_public_key // array const sourceEndpoint = req.query.source_endpoint // string + const forceExport = !!req.query.force_export // boolean const maxExportClockValueRange = config.get('maxExportClockValueRange') @@ -44,6 +45,7 @@ router.get( walletPublicKeys, requestedClockRangeMin, requestedClockRangeMax, + forceExport, logger: req.logger }) }, diff --git a/creator-node/src/services/sync/primarySyncFromSecondary.js b/creator-node/src/services/sync/primarySyncFromSecondary.js index 1de16b05280..9b72af45355 100644 --- a/creator-node/src/services/sync/primarySyncFromSecondary.js +++ b/creator-node/src/services/sync/primarySyncFromSecondary.js @@ -128,7 +128,8 @@ async function fetchExportFromSecondary({ const exportQueryParams = { wallet_public_key: [wallet], // export requires a wallet array clock_range_min: clockRangeMin, - source_endpoint: selfEndpoint + source_endpoint: selfEndpoint, + force_export: true } try { From b14bb758887042533a781d5b4fcf069652b43022 Mon Sep 17 00:00:00 2001 From: Sid Sethi <3323835+SidSethi@users.noreply.github.com> Date: Fri, 22 Jul 2022 22:00:18 +0000 Subject: [PATCH 5/5] default to false --- .../src/components/replicaSet/exportComponentService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creator-node/src/components/replicaSet/exportComponentService.js b/creator-node/src/components/replicaSet/exportComponentService.js index 5221f3fe4d2..fa2eeafdfab 100644 --- a/creator-node/src/components/replicaSet/exportComponentService.js +++ b/creator-node/src/components/replicaSet/exportComponentService.js @@ -14,7 +14,7 @@ const exportComponentService = async ({ walletPublicKeys, requestedClockRangeMin, requestedClockRangeMax, - forceExport, + forceExport = false, logger }) => { const transaction = await models.sequelize.transaction({