From 9cac60cf1919f0e79ee796f947bc9408f6cbfeca Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Thu, 12 Nov 2020 15:30:21 +0100 Subject: [PATCH] feat: integrate miner rewards into db and account balance calcuations --- docs/entities/balance/stx-balance.schema.json | 8 +- docs/index.d.ts | 4 + src/api/routes/address.ts | 4 + src/datastore/common.ts | 17 ++++ src/datastore/postgres-store.ts | 93 +++++++++++++++++-- src/event-stream/event-server.ts | 22 +++++ src/migrations/1605184662317_miner_rewards.ts | 53 +++++++++++ src/tests/api-tests.ts | 7 ++ src/tests/datastore-tests.ts | 25 ++++- src/tests/websocket-tests.ts | 5 + 10 files changed, 226 insertions(+), 12 deletions(-) create mode 100644 src/migrations/1605184662317_miner_rewards.ts diff --git a/docs/entities/balance/stx-balance.schema.json b/docs/entities/balance/stx-balance.schema.json index cfcae03f86..cda6f4f600 100644 --- a/docs/entities/balance/stx-balance.schema.json +++ b/docs/entities/balance/stx-balance.schema.json @@ -3,7 +3,7 @@ "description": "StxBalance", "type": "object", "additionalProperties": false, - "required": ["balance", "locked", "unlock_height", "total_sent", "total_received"], + "required": ["balance", "locked", "unlock_height", "total_sent", "total_received", "total_fees_sent", "total_miner_rewards_received"], "properties": { "balance": { "type": "string" @@ -19,6 +19,12 @@ }, "total_received": { "type": "string" + }, + "total_fees_sent": { + "type": "string" + }, + "total_miner_rewards_received": { + "type": "string" } } } diff --git a/docs/index.d.ts b/docs/index.d.ts index cc6351794e..eff0f862ed 100644 --- a/docs/index.d.ts +++ b/docs/index.d.ts @@ -27,6 +27,8 @@ export interface AddressBalanceResponse { unlock_height: number; total_sent: string; total_received: string; + total_fees_sent: string; + total_miner_rewards_received: string; }; fungible_tokens: { /** @@ -65,6 +67,8 @@ export interface AddressStxBalanceResponse { unlock_height: number; total_sent: string; total_received: string; + total_fees_sent: string; + total_miner_rewards_received: string; } /** diff --git a/src/api/routes/address.ts b/src/api/routes/address.ts index 1bfd79c145..81990bebf4 100644 --- a/src/api/routes/address.ts +++ b/src/api/routes/address.ts @@ -48,6 +48,8 @@ export function createAddressRouter(db: DataStore): RouterWithAsync { unlock_height: Number(stxBalanceResult.unlockHeight), total_sent: stxBalanceResult.totalSent.toString(), total_received: stxBalanceResult.totalReceived.toString(), + total_fees_sent: stxBalanceResult.totalFeesSent.toString(), + total_miner_rewards_received: stxBalanceResult.totalMinerRewardsReceived.toString(), }; res.json(result); }); @@ -88,6 +90,8 @@ export function createAddressRouter(db: DataStore): RouterWithAsync { unlock_height: Number(stxBalanceResult.unlockHeight), total_sent: stxBalanceResult.totalSent.toString(), total_received: stxBalanceResult.totalReceived.toString(), + total_fees_sent: stxBalanceResult.totalFeesSent.toString(), + total_miner_rewards_received: stxBalanceResult.totalMinerRewardsReceived.toString(), }, fungible_tokens: ftBalances, non_fungible_tokens: nftBalances, diff --git a/src/datastore/common.ts b/src/datastore/common.ts index d4418f837f..dc805001bd 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -28,6 +28,20 @@ export interface DbBlock { canonical: boolean; } +export interface DbMinerReward { + block_hash: string; + index_block_hash: string; + mature_block_height: number; + /** Set to `true` if entry corresponds to the canonical chain tip */ + canonical: boolean; + /** STX principal */ + recipient: string; + coinbase_amount: bigint; + tx_fees_anchored_shared: bigint; + tx_fees_anchored_exclusive: bigint; + tx_fees_streamed_confirmed: bigint; +} + export enum DbTxTypeId { TokenTransfer = 0x00, SmartContract = 0x01, @@ -222,6 +236,7 @@ export type DataStoreEventEmitter = StrictEventEmitter< export interface DataStoreUpdateData { block: DbBlock; + minerRewards: DbMinerReward[]; txs: { tx: DbTx; stxEvents: DbStxEvent[]; @@ -251,6 +266,8 @@ export interface DbStxBalance { unlockHeight: number; totalSent: bigint; totalReceived: bigint; + totalFeesSent: bigint; + totalMinerRewardsReceived: bigint; } export interface DataStore extends DataStoreEventEmitter { diff --git a/src/datastore/postgres-store.ts b/src/datastore/postgres-store.ts index e45717a0e4..619a67c582 100644 --- a/src/datastore/postgres-store.ts +++ b/src/datastore/postgres-store.ts @@ -40,6 +40,7 @@ import { DbStxBalance, DbStxLockEvent, DbFtBalance, + DbMinerReward, } from './common'; import { TransactionType } from '@blockstack/stacks-blockchain-api-types'; import { getTxTypeId } from '../api/controllers/db-controller'; @@ -265,6 +266,7 @@ interface FaucetRequestQueryResult { interface UpdatedEntities { markedCanonical: { blocks: number; + minerRewards: number; txs: number; stxLockEvents: number; stxEvents: number; @@ -275,6 +277,7 @@ interface UpdatedEntities { }; markedNonCanonical: { blocks: number; + minerRewards: number; txs: number; stxLockEvents: number; stxEvents: number; @@ -350,6 +353,9 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte } const blocksUpdated = await this.updateBlock(client, data.block); if (blocksUpdated !== 0) { + for (const minerRewards of data.minerRewards) { + await this.updateMinerReward(client, minerRewards); + } for (const entry of data.txs) { await this.updateTx(client, entry.tx); for (const stxLockEvent of entry.stxLockEvents) { @@ -482,6 +488,20 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte updatedEntities.markedNonCanonical.txs += txResult.rowCount; } + const minerRewardResults = await client.query( + ` + UPDATE miner_rewards + SET canonical = $2 + WHERE index_block_hash = $1 AND canonical != $2 + `, + [indexBlockHash, canonical] + ); + if (canonical) { + updatedEntities.markedCanonical.minerRewards += minerRewardResults.rowCount; + } else { + updatedEntities.markedNonCanonical.minerRewards += minerRewardResults.rowCount; + } + const stxLockResults = await client.query( ` UPDATE stx_lock_events @@ -663,6 +683,7 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte const updatedEntities: UpdatedEntities = { markedCanonical: { blocks: 0, + minerRewards: 0, txs: 0, stxLockEvents: 0, stxEvents: 0, @@ -673,6 +694,7 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte }, markedNonCanonical: { blocks: 0, + minerRewards: 0, txs: 0, stxLockEvents: 0, stxEvents: 0, @@ -730,6 +752,11 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte const updates = [ ['blocks', updatedEntities.markedCanonical.blocks, updatedEntities.markedNonCanonical.blocks], ['txs', updatedEntities.markedCanonical.txs, updatedEntities.markedNonCanonical.txs], + [ + 'miner-rewards', + updatedEntities.markedCanonical.minerRewards, + updatedEntities.markedNonCanonical.minerRewards, + ], [ 'stx-lock events', updatedEntities.markedCanonical.stxLockEvents, @@ -820,6 +847,30 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte } } + async updateMinerReward(client: ClientBase, minerReward: DbMinerReward): Promise { + const result = await client.query( + ` + INSERT INTO miner_rewards( + block_hash, index_block_hash, mature_block_height, canonical, recipient, coinbase_amount, tx_fees_anchored_shared, tx_fees_anchored_exclusive, tx_fees_streamed_confirmed + ) values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (index_block_hash) + DO NOTHING + `, + [ + hexToBuffer(minerReward.block_hash), + hexToBuffer(minerReward.index_block_hash), + minerReward.mature_block_height, + minerReward.canonical, + minerReward.recipient, + minerReward.coinbase_amount, + minerReward.tx_fees_anchored_shared, + minerReward.tx_fees_anchored_exclusive, + minerReward.tx_fees_streamed_confirmed, + ] + ); + return result.rowCount; + } + async updateBlock(client: ClientBase, block: DbBlock): Promise { const result = await client.query( ` @@ -1746,10 +1797,21 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte `stx_lock_events event query for ${stxAddress} should return zero or one rows but returned ${lockQuery.rowCount}` ); } - const totalFees = BigInt(feeQuery.rows[0].fee_sum ?? 0); - const totalSent = BigInt(result.rows[0].debit_total ?? 0); - const totalReceived = BigInt(result.rows[0].credit_total ?? 0); - const balance = totalReceived - totalSent - totalFees; + const minerRewardQuery = await client.query<{ amount: string }>( + ` + SELECT sum( + coinbase_amount + tx_fees_anchored_shared + tx_fees_anchored_exclusive + tx_fees_streamed_confirmed + ) amount + FROM miner_rewards + WHERE canonical = true AND recipient = $1 AND mature_block_height <= $2 + `, + [stxAddress, currentBlockHeight] + ); + const totalRewards = BigInt(minerRewardQuery.rows[0]?.amount ?? 0); + const totalFees = BigInt(feeQuery.rows[0]?.fee_sum ?? 0); + const totalSent = BigInt(result.rows[0]?.debit_total ?? 0); + const totalReceived = BigInt(result.rows[0]?.credit_total ?? 0); + const balance = totalReceived - totalSent - totalFees + totalRewards; const locked = BigInt(lockQuery.rows[0]?.locked_amount ?? 0); const unlockHeight = Number(lockQuery.rows[0]?.unlock_height ?? 0); return { @@ -1758,6 +1820,8 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte unlockHeight, totalSent, totalReceived, + totalFeesSent: totalFees, + totalMinerRewardsReceived: totalRewards, }; } catch (e) { await client.query('ROLLBACK'); @@ -1816,18 +1880,31 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte `stx_lock_events event query for ${stxAddress} should return zero or one rows but returned ${lockQuery.rowCount}` ); } + const minerRewardQuery = await client.query<{ amount: string }>( + ` + SELECT sum( + coinbase_amount + tx_fees_anchored_shared + tx_fees_anchored_exclusive + tx_fees_streamed_confirmed + ) amount + FROM miner_rewards + WHERE canonical = true AND recipient = $1 AND mature_block_height <= $2 + `, + [stxAddress, blockHeight] + ); + const totalRewards = BigInt(minerRewardQuery.rows[0]?.amount ?? 0); + const totalFees = BigInt(feeQuery.rows[0]?.fee_sum ?? 0); + const totalSent = BigInt(result.rows[0]?.debit_total ?? 0); + const totalReceived = BigInt(result.rows[0]?.credit_total ?? 0); + const balance = totalReceived - totalSent - totalFees + totalRewards; const locked = BigInt(lockQuery.rows[0]?.locked_amount ?? 0); const unlockHeight = Number(lockQuery.rows[0]?.unlock_height ?? 0); - const totalFees = BigInt(feeQuery.rows[0].fee_sum ?? 0); - const totalSent = BigInt(result.rows[0].debit_total ?? 0); - const totalReceived = BigInt(result.rows[0].credit_total ?? 0); - const balance = totalReceived - totalSent - totalFees; return { balance, locked, unlockHeight, totalSent, totalReceived, + totalFeesSent: totalFees, + totalMinerRewardsReceived: totalRewards, }; } catch (e) { await client.query('ROLLBACK'); diff --git a/src/event-stream/event-server.ts b/src/event-stream/event-server.ts index 36da8cb827..74ed848d55 100644 --- a/src/event-stream/event-server.ts +++ b/src/event-stream/event-server.ts @@ -26,6 +26,7 @@ import { DataStoreUpdateData, createDbMempoolTxFromCoreMsg, DbStxLockEvent, + DbMinerReward, } from '../datastore/common'; import { parseMessageTransactions, getTxSenderAddress, getTxSponsorAddress } from './reader'; import { TransactionPayloadTypeID, readTransaction } from '../p2p/tx'; @@ -86,8 +87,26 @@ async function handleClientMessage(msg: CoreNodeMessage, db: DataStore): Promise dbBlock ); + const dbMinerRewards: DbMinerReward[] = []; + for (const minerReward of msg.matured_miner_rewards) { + const dbMinerReward: DbMinerReward = { + canonical: true, + block_hash: minerReward.from_stacks_block_hash, + index_block_hash: minerReward.from_index_consensus_hash, + mature_block_height: parsedMsg.block_height, + recipient: minerReward.recipient, + coinbase_amount: BigInt(minerReward.coinbase_amount), + tx_fees_anchored_shared: BigInt(minerReward.tx_fees_anchored_shared), + tx_fees_anchored_exclusive: BigInt(minerReward.tx_fees_anchored_exclusive), + tx_fees_streamed_confirmed: BigInt(minerReward.tx_fees_streamed_confirmed), + }; + dbMinerRewards.push(dbMinerReward); + } + logger.verbose(`Received ${dbMinerRewards.length} matured miner rewards`); + const dbData: DataStoreUpdateData = { block: dbBlock, + minerRewards: dbMinerRewards, txs: new Array(parsedMsg.transactions.length), }; @@ -305,6 +324,9 @@ export async function startEventServer(opts: { app.postAsync('/new_block', async (req, res) => { try { const msg: CoreNodeMessage = req.body; + if (msg.matured_miner_rewards && msg.matured_miner_rewards.length > 0) { + console.log(msg.matured_miner_rewards); + } await messageHandler.handleBlockMessage(msg, db); res.status(200).json({ result: 'ok' }); } catch (error) { diff --git a/src/migrations/1605184662317_miner_rewards.ts b/src/migrations/1605184662317_miner_rewards.ts new file mode 100644 index 0000000000..5ea47fc03a --- /dev/null +++ b/src/migrations/1605184662317_miner_rewards.ts @@ -0,0 +1,53 @@ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; +// block_hash, index_block_hash, canonical, recipient, coinbase_amount, tx_fees_anchored_shared, tx_fees_anchored_exclusive, tx_fees_streamed_confirmed +export async function up(pgm: MigrationBuilder): Promise { + pgm.createTable('miner_rewards', { + id: { + type: 'serial', + primaryKey: true, + }, + block_hash: { + type: 'bytea', + notNull: true, + }, + index_block_hash: { + type: 'bytea', + notNull: true, + }, + mature_block_height: { + type: 'integer', + notNull: true, + }, + canonical: { + type: 'boolean', + notNull: true, + }, + recipient: { + type: 'string', + notNull: true, + }, + coinbase_amount: { + type: 'numeric', + notNull: true, + }, + tx_fees_anchored_shared: { + type: 'numeric', + notNull: true, + }, + tx_fees_anchored_exclusive: { + type: 'numeric', + notNull: true, + }, + tx_fees_streamed_confirmed: { + type: 'numeric', + notNull: true, + }, + }); + + pgm.createIndex('miner_rewards', 'block_hash'); + pgm.createIndex('miner_rewards', 'index_block_hash'); + pgm.createIndex('miner_rewards', 'mature_block_height'); + pgm.createIndex('miner_rewards', 'canonical'); + pgm.createIndex('miner_rewards', 'recipient'); + +} diff --git a/src/tests/api-tests.ts b/src/tests/api-tests.ts index 8c4e77b843..081679cdde 100644 --- a/src/tests/api-tests.ts +++ b/src/tests/api-tests.ts @@ -965,6 +965,8 @@ describe('api tests', () => { unlock_height: 0, total_sent: '1385', total_received: '100000', + total_fees_sent: '3702', + total_miner_rewards_received: '0', }, fungible_tokens: { bux: { balance: '99615', total_sent: '385', total_received: '100000' }, @@ -993,6 +995,8 @@ describe('api tests', () => { unlock_height: 0, total_sent: '15', total_received: '1350', + total_fees_sent: '1234', + total_miner_rewards_received: '0', }, fungible_tokens: { bux: { balance: '335', total_sent: '15', total_received: '350' }, @@ -1016,6 +1020,8 @@ describe('api tests', () => { unlock_height: 0, total_sent: '15', total_received: '1350', + total_fees_sent: '1234', + total_miner_rewards_received: '0', }; expect(JSON.parse(fetchAddrStxBalance1.text)).toEqual(expectedStxResp1); @@ -1269,6 +1275,7 @@ describe('api tests', () => { }; await db.update({ block: block1, + minerRewards: [], txs: [ { tx: tx1, diff --git a/src/tests/datastore-tests.ts b/src/tests/datastore-tests.ts index 8dc70de6b6..85e8a7c140 100644 --- a/src/tests/datastore-tests.ts +++ b/src/tests/datastore-tests.ts @@ -127,6 +127,8 @@ describe('postgres datastore', () => { unlockHeight: 0, totalReceived: 100000n, totalSent: 385n, + totalFeesSent: 1334n, + totalMinerRewardsReceived: 0n, }); expect(addrBResult).toEqual({ balance: 335n, @@ -134,6 +136,8 @@ describe('postgres datastore', () => { unlockHeight: 0, totalReceived: 350n, totalSent: 15n, + totalFeesSent: 0n, + totalMinerRewardsReceived: 0n, }); expect(addrCResult).toEqual({ balance: 50n, @@ -141,6 +145,8 @@ describe('postgres datastore', () => { unlockHeight: 0, totalReceived: 50n, totalSent: 0n, + totalFeesSent: 0n, + totalMinerRewardsReceived: 0n, }); expect(addrDResult).toEqual({ balance: 0n, @@ -148,6 +154,8 @@ describe('postgres datastore', () => { unlockHeight: 0, totalReceived: 0n, totalSent: 0n, + totalFeesSent: 0n, + totalMinerRewardsReceived: 0n, }); }); @@ -1624,6 +1632,7 @@ describe('postgres datastore', () => { }; await db.update({ block: block1, + minerRewards: [], txs: [ { tx: tx1, @@ -1818,12 +1827,14 @@ describe('postgres datastore', () => { for (const block of [block1, block2, block3]) { await db.update({ block: block, + minerRewards: [], txs: [], }); } await db.update({ block: block3B, + minerRewards: [], txs: [ { tx: tx1, @@ -1844,6 +1855,7 @@ describe('postgres datastore', () => { await db.update({ block: block4B, + minerRewards: [], txs: [], }); @@ -1864,6 +1876,7 @@ describe('postgres datastore', () => { for (const block of [block4, block5]) { await db.update({ block: block, + minerRewards: [], txs: [], }); } @@ -1885,6 +1898,7 @@ describe('postgres datastore', () => { // mine the same tx in the latest canonical block await db.update({ block: block6, + minerRewards: [], txs: [ { tx: tx1b, @@ -2038,6 +2052,7 @@ describe('postgres datastore', () => { expect(reorgResult).toEqual({ markedCanonical: { blocks: 4, + minerRewards: 0, txs: 2, stxLockEvents: 0, stxEvents: 0, @@ -2048,6 +2063,7 @@ describe('postgres datastore', () => { }, markedNonCanonical: { blocks: 1, + minerRewards: 0, txs: 0, stxLockEvents: 0, stxEvents: 0, @@ -2151,6 +2167,7 @@ describe('postgres datastore', () => { await db.update({ block: block1, + minerRewards: [], txs: [ { tx: tx1, @@ -2165,6 +2182,7 @@ describe('postgres datastore', () => { }); await db.update({ block: block2, + minerRewards: [], txs: [ { tx: tx2, @@ -2177,7 +2195,7 @@ describe('postgres datastore', () => { }, ], }); - await db.update({ block: block3, txs: [] }); + await db.update({ block: block3, minerRewards: [], txs: [] }); const block2b: DbBlock = { block_hash: '0x22bb', @@ -2221,6 +2239,7 @@ describe('postgres datastore', () => { }; await db.update({ block: block2b, + minerRewards: [], txs: [ { tx: tx3, @@ -2251,7 +2270,7 @@ describe('postgres datastore', () => { miner_txid: '0x4321', canonical: true, }; - await db.update({ block: block3b, txs: [] }); + await db.update({ block: block3b, minerRewards: [], txs: [] }); const blockQuery2 = await db.getBlock(block3b.block_hash); expect(blockQuery2.result?.canonical).toBe(false); const chainTip2 = await db.getChainTipHeight(client); @@ -2270,7 +2289,7 @@ describe('postgres datastore', () => { miner_txid: '0x4321', canonical: true, }; - await db.update({ block: block4b, txs: [] }); + await db.update({ block: block4b, minerRewards: [], txs: [] }); const blockQuery3 = await db.getBlock(block3b.block_hash); expect(blockQuery3.result?.canonical).toBe(true); const chainTip3 = await db.getChainTipHeight(client); diff --git a/src/tests/websocket-tests.ts b/src/tests/websocket-tests.ts index 0adcd227b4..9f657ba1a7 100644 --- a/src/tests/websocket-tests.ts +++ b/src/tests/websocket-tests.ts @@ -101,6 +101,7 @@ describe('websocket notifications', () => { const dbUpdate: DataStoreUpdateData = { block, + minerRewards: [], txs: [ { tx, @@ -233,6 +234,7 @@ describe('websocket notifications', () => { const dbUpdate: DataStoreUpdateData = { block, + minerRewards: [], txs: [ { tx, @@ -352,6 +354,7 @@ describe('websocket notifications', () => { const dbUpdate: DataStoreUpdateData = { block, + minerRewards: [], txs: [ { tx, @@ -465,6 +468,7 @@ describe('websocket notifications', () => { const dbUpdate: DataStoreUpdateData = { block, + minerRewards: [], txs: [ { tx, @@ -561,6 +565,7 @@ describe('websocket notifications', () => { const dbUpdate: DataStoreUpdateData = { block, + minerRewards: [], txs: [ { tx,