From 389114069ebab905de1ad02ec56a05e82a6b5c5e Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Wed, 27 Feb 2019 14:09:37 -0500 Subject: [PATCH] fix(node): detecting dupe transactions and coins with mismatched heights Improving verification scripts to detect duplicate transactions and coins with differing heights --- .../test/verification/db-repair.ts | 85 +++++++++++++------ .../test/verification/db-verify.ts | 25 +++++- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/packages/bitcore-node/test/verification/db-repair.ts b/packages/bitcore-node/test/verification/db-repair.ts index 73846a41b7d..21a5b68ae85 100755 --- a/packages/bitcore-node/test/verification/db-repair.ts +++ b/packages/bitcore-node/test/verification/db-repair.ts @@ -7,6 +7,7 @@ import { P2pWorker } from '../../src/services/p2p'; import { Config } from '../../src/services/config'; import { BlockStorage } from '../../src/models/block'; import { validateDataForBlock } from './db-verify'; +import { TransactionStorage } from '../../src/models/transaction'; (async () => { const { CHAIN, NETWORK, FILE, DRYRUN } = process.env; @@ -23,38 +24,70 @@ import { validateDataForBlock } from './db-verify'; const handleRepair = async data => { switch (data.type) { - case 'DUPE_COIN': - const coin = data.payload.coin; - const dupeCoins = await CoinStorage.collection - .find({ chain, network, mintTxid: coin.mintTxid, mintIndex: coin.mintIndex }) - .sort({ _id: -1 }) - .toArray(); + case 'DUPE_TRANSACTION': + { + const tx = data.payload.tx; + const dupeTxs = await TransactionStorage.collection + .find({ chain: tx.chain, network: tx.network, txid: tx.txid }) + .sort({ blockHeight: -1 }) + .toArray(); - if (dupeCoins.length < 2) { - console.log('No action required.', dupeCoins.length, 'coin'); - return; + if (dupeTxs.length < 2) { + console.log('No action required.', dupeTxs.length, 'transaction'); + return; + } + + let toKeep = dupeTxs[0]; + const wouldBeDeleted = dupeTxs.filter(c => c._id != toKeep._id); + + if (DRYRUN) { + console.log('WOULD DELETE'); + console.log(wouldBeDeleted); + } else { + console.log('Deleting', wouldBeDeleted.length, 'transactions'); + await TransactionStorage.collection.deleteMany({ + chain, + network, + _id: { $in: wouldBeDeleted.map(c => c._id) } + }); + } } + break; + case 'DUPE_COIN': + { + const coin = data.payload.coin; + const dupeCoins = await CoinStorage.collection + .find({ chain, network, mintTxid: coin.mintTxid, mintIndex: coin.mintIndex }) + .sort({ _id: -1 }) + .toArray(); - let toKeep = dupeCoins[0]; - const spentCoin = dupeCoins.find(c => c.spentHeight > toKeep.spentHeight); - toKeep = spentCoin || toKeep; - const wouldBeDeleted = dupeCoins.filter(c => c._id != toKeep._id); + if (dupeCoins.length < 2) { + console.log('No action required.', dupeCoins.length, 'coin'); + return; + } - if (DRYRUN) { - console.log('WOULD DELETE'); - console.log(wouldBeDeleted); - } else { - const { mintIndex, mintTxid } = toKeep; - console.log('Deleting', wouldBeDeleted.length, 'coins'); - await CoinStorage.collection.deleteMany({ - chain, - network, - mintTxid, - mintIndex, - _id: { $in: wouldBeDeleted.map(c => c._id) } - }); + let toKeep = dupeCoins[0]; + const spentCoin = dupeCoins.find(c => c.spentHeight > toKeep.spentHeight); + toKeep = spentCoin || toKeep; + const wouldBeDeleted = dupeCoins.filter(c => c._id != toKeep._id); + + if (DRYRUN) { + console.log('WOULD DELETE'); + console.log(wouldBeDeleted); + } else { + const { mintIndex, mintTxid } = toKeep; + console.log('Deleting', wouldBeDeleted.length, 'coins'); + await CoinStorage.collection.deleteMany({ + chain, + network, + mintTxid, + mintIndex, + _id: { $in: wouldBeDeleted.map(c => c._id) } + }); + } } break; + case 'COIN_HEIGHT_MISMATCH': case 'MISSING_BLOCK': case 'MISSING_TX': case 'MISSING_COIN_FOR_TXID': diff --git a/packages/bitcore-node/test/verification/db-verify.ts b/packages/bitcore-node/test/verification/db-verify.ts index 2498d9be414..5061aaef898 100755 --- a/packages/bitcore-node/test/verification/db-verify.ts +++ b/packages/bitcore-node/test/verification/db-verify.ts @@ -4,6 +4,7 @@ import { BlockStorage } from '../../src/models/block'; import { CoinStorage, ICoin } from '../../src/models/coin'; import { TransactionStorage, ITransaction } from '../../src/models/transaction'; import { Storage } from '../../src/services/storage'; +import * as _ from 'lodash'; const { CHAIN, NETWORK, HEIGHT } = process.env; const resumeHeight = Number(HEIGHT) || 1; @@ -33,10 +34,20 @@ export async function validateDataForBlock(blockNum: number, log = false) { console.log(JSON.stringify(error)); } } - seenTxs[tx.txid] = tx; + if (seenTxs[tx.txid]) { + success = false; + const error = { model: 'transaction', err: true, type: 'DUPE_TRANSACTION', payload: { tx, blockNum } }; + errors.push(error); + if (log) { + console.log(JSON.stringify(error)); + } + } else { + seenTxs[tx.txid] = tx; + } } const blockTxids = blockTxs.map(t => t.txid); + const coinsForTx = await CoinStorage.collection.find({ chain, network, mintTxid: { $in: blockTxids } }).toArray(); for (let coin of coinsForTx) { if (seenTxCoins[coin.mintTxid] && seenTxCoins[coin.mintTxid][coin.mintIndex]) { @@ -52,6 +63,16 @@ export async function validateDataForBlock(blockNum: number, log = false) { } } + const mintHeights = _.uniq(coinsForTx.map(c => c.mintHeight)); + if (mintHeights.length > 1) { + success = false; + const error = { model: 'coin', err: true, type: 'COIN_HEIGHT_MISMATCH', payload: { blockNum } }; + errors.push(error); + if (log) { + console.log(JSON.stringify(error)); + } + } + for (let txid of Object.keys(seenTxs)) { const coins = seenTxCoins[txid]; if (!coins) { @@ -152,7 +173,7 @@ if (require.main === module) { const tip = await BlockStorage.getLocalTip({ chain, network }); if (tip) { - for (let i = resumeHeight; i < tip.height; i++) { + for (let i = resumeHeight; i <= tip.height; i++) { const { success } = await validateDataForBlock(i, true); console.log({ block: i, success }); }