feat: integrate miner rewards into db and account balance calcuations

docs/entities/balance/stx-balance.schema.json

@@ -3,7 +3,7 @@
   "description": "StxBalance",
   "type": "object",
   "additionalProperties": false,
-  "required": ["balance", "locked", "unlock_height", "total_sent", "total_received"],
+  "required": ["balance", "locked", "unlock_height", "total_sent", "total_received", "total_fees_sent", "total_miner_rewards_received"],
   "properties": {
     "balance": {
       "type": "string"
@@ -19,6 +19,12 @@
     },
     "total_received": {
       "type": "string"
+    },
+    "total_fees_sent": {
+      "type": "string"
+    },
+    "total_miner_rewards_received": {
+      "type": "string"
     }
   }
 }

docs/index.d.ts

@@ -27,6 +27,8 @@ export interface AddressBalanceResponse {
     unlock_height: number;
     total_sent: string;
     total_received: string;
+    total_fees_sent: string;
+    total_miner_rewards_received: string;
   };
   fungible_tokens: {
     /**
@@ -65,6 +67,8 @@ export interface AddressStxBalanceResponse {
   unlock_height: number;
   total_sent: string;
   total_received: string;
+  total_fees_sent: string;
+  total_miner_rewards_received: string;
 }
 
 /**

src/api/routes/address.ts

@@ -48,6 +48,8 @@ export function createAddressRouter(db: DataStore): RouterWithAsync {
       unlock_height: Number(stxBalanceResult.unlockHeight),
       total_sent: stxBalanceResult.totalSent.toString(),
       total_received: stxBalanceResult.totalReceived.toString(),
+      total_fees_sent: stxBalanceResult.totalFeesSent.toString(),
+      total_miner_rewards_received: stxBalanceResult.totalMinerRewardsReceived.toString(),
     };
     res.json(result);
   });
@@ -88,6 +90,8 @@ export function createAddressRouter(db: DataStore): RouterWithAsync {
         unlock_height: Number(stxBalanceResult.unlockHeight),
         total_sent: stxBalanceResult.totalSent.toString(),
         total_received: stxBalanceResult.totalReceived.toString(),
+        total_fees_sent: stxBalanceResult.totalFeesSent.toString(),
+        total_miner_rewards_received: stxBalanceResult.totalMinerRewardsReceived.toString(),
       },
       fungible_tokens: ftBalances,
       non_fungible_tokens: nftBalances,

src/datastore/common.ts

@@ -28,6 +28,20 @@ export interface DbBlock {
   canonical: boolean;
 }
 
+export interface DbMinerReward {
+  block_hash: string;
+  index_block_hash: string;
+  mature_block_height: number;
+  /** Set to `true` if entry corresponds to the canonical chain tip */
+  canonical: boolean;
+  /** STX principal */
+  recipient: string;
+  coinbase_amount: bigint;
+  tx_fees_anchored_shared: bigint;
+  tx_fees_anchored_exclusive: bigint;
+  tx_fees_streamed_confirmed: bigint;
+}
+
 export enum DbTxTypeId {
   TokenTransfer = 0x00,
   SmartContract = 0x01,
@@ -222,6 +236,7 @@ export type DataStoreEventEmitter = StrictEventEmitter<
 
 export interface DataStoreUpdateData {
   block: DbBlock;
+  minerRewards: DbMinerReward[];
   txs: {
     tx: DbTx;
     stxEvents: DbStxEvent[];
@@ -251,6 +266,8 @@ export interface DbStxBalance {
   unlockHeight: number;
   totalSent: bigint;
   totalReceived: bigint;
+  totalFeesSent: bigint;
+  totalMinerRewardsReceived: bigint;
 }
 
 export interface DataStore extends DataStoreEventEmitter {

src/datastore/postgres-store.ts

@@ -40,6 +40,7 @@ import {
   DbStxBalance,
   DbStxLockEvent,
   DbFtBalance,
+  DbMinerReward,
 } from './common';
 import { TransactionType } from '@blockstack/stacks-blockchain-api-types';
 import { getTxTypeId } from '../api/controllers/db-controller';
@@ -265,6 +266,7 @@ interface FaucetRequestQueryResult {
 interface UpdatedEntities {
   markedCanonical: {
     blocks: number;
+    minerRewards: number;
     txs: number;
     stxLockEvents: number;
     stxEvents: number;
@@ -275,6 +277,7 @@ interface UpdatedEntities {
   };
   markedNonCanonical: {
     blocks: number;
+    minerRewards: number;
     txs: number;
     stxLockEvents: number;
     stxEvents: number;
@@ -350,6 +353,9 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
       }
       const blocksUpdated = await this.updateBlock(client, data.block);
       if (blocksUpdated !== 0) {
+        for (const minerRewards of data.minerRewards) {
+          await this.updateMinerReward(client, minerRewards);
+        }
         for (const entry of data.txs) {
           await this.updateTx(client, entry.tx);
           for (const stxLockEvent of entry.stxLockEvents) {
@@ -482,6 +488,20 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
       updatedEntities.markedNonCanonical.txs += txResult.rowCount;
     }
 
+    const minerRewardResults = await client.query(
+      `
+      UPDATE miner_rewards
+      SET canonical = $2
+      WHERE index_block_hash = $1 AND canonical != $2
+      `,
+      [indexBlockHash, canonical]
+    );
+    if (canonical) {
+      updatedEntities.markedCanonical.minerRewards += minerRewardResults.rowCount;
+    } else {
+      updatedEntities.markedNonCanonical.minerRewards += minerRewardResults.rowCount;
+    }
+
     const stxLockResults = await client.query(
       `
       UPDATE stx_lock_events
@@ -663,6 +683,7 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
     const updatedEntities: UpdatedEntities = {
       markedCanonical: {
         blocks: 0,
+        minerRewards: 0,
         txs: 0,
         stxLockEvents: 0,
         stxEvents: 0,
@@ -673,6 +694,7 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
       },
       markedNonCanonical: {
         blocks: 0,
+        minerRewards: 0,
         txs: 0,
         stxLockEvents: 0,
         stxEvents: 0,
@@ -730,6 +752,11 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
     const updates = [
       ['blocks', updatedEntities.markedCanonical.blocks, updatedEntities.markedNonCanonical.blocks],
       ['txs', updatedEntities.markedCanonical.txs, updatedEntities.markedNonCanonical.txs],
+      [
+        'miner-rewards',
+        updatedEntities.markedCanonical.minerRewards,
+        updatedEntities.markedNonCanonical.minerRewards,
+      ],
       [
         'stx-lock events',
         updatedEntities.markedCanonical.stxLockEvents,
@@ -820,6 +847,30 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
     }
   }
 
+  async updateMinerReward(client: ClientBase, minerReward: DbMinerReward): Promise<number> {
+    const result = await client.query(
+      `
+      INSERT INTO miner_rewards(
+        block_hash, index_block_hash, mature_block_height, canonical, recipient, coinbase_amount, tx_fees_anchored_shared, tx_fees_anchored_exclusive, tx_fees_streamed_confirmed
+      ) values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+      ON CONFLICT (index_block_hash)
+      DO NOTHING
+      `,
+      [
+        hexToBuffer(minerReward.block_hash),
+        hexToBuffer(minerReward.index_block_hash),
+        minerReward.mature_block_height,
+        minerReward.canonical,
+        minerReward.recipient,
+        minerReward.coinbase_amount,
+        minerReward.tx_fees_anchored_shared,
+        minerReward.tx_fees_anchored_exclusive,
+        minerReward.tx_fees_streamed_confirmed,
+      ]
+    );
+    return result.rowCount;
+  }
+
   async updateBlock(client: ClientBase, block: DbBlock): Promise<number> {
     const result = await client.query(
       `
@@ -1746,10 +1797,21 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
           `stx_lock_events event query for ${stxAddress} should return zero or one rows but returned ${lockQuery.rowCount}`
         );
       }
-      const totalFees = BigInt(feeQuery.rows[0].fee_sum ?? 0);
-      const totalSent = BigInt(result.rows[0].debit_total ?? 0);
-      const totalReceived = BigInt(result.rows[0].credit_total ?? 0);
-      const balance = totalReceived - totalSent - totalFees;
+      const minerRewardQuery = await client.query<{ amount: string }>(
+        `
+        SELECT sum(
+          coinbase_amount + tx_fees_anchored_shared + tx_fees_anchored_exclusive + tx_fees_streamed_confirmed
+        ) amount
+        FROM miner_rewards
+        WHERE canonical = true AND recipient = $1 AND mature_block_height <= $2
+        `,
+        [stxAddress, currentBlockHeight]
+      );
+      const totalRewards = BigInt(minerRewardQuery.rows[0]?.amount ?? 0);
+      const totalFees = BigInt(feeQuery.rows[0]?.fee_sum ?? 0);
+      const totalSent = BigInt(result.rows[0]?.debit_total ?? 0);
+      const totalReceived = BigInt(result.rows[0]?.credit_total ?? 0);
+      const balance = totalReceived - totalSent - totalFees + totalRewards;
       const locked = BigInt(lockQuery.rows[0]?.locked_amount ?? 0);
       const unlockHeight = Number(lockQuery.rows[0]?.unlock_height ?? 0);
       return {
@@ -1758,6 +1820,8 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
         unlockHeight,
         totalSent,
         totalReceived,
+        totalFeesSent: totalFees,
+        totalMinerRewardsReceived: totalRewards,
       };
     } catch (e) {
       await client.query('ROLLBACK');
@@ -1816,18 +1880,31 @@ export class PgDataStore extends (EventEmitter as { new (): DataStoreEventEmitte
           `stx_lock_events event query for ${stxAddress} should return zero or one rows but returned ${lockQuery.rowCount}`
         );
       }
+      const minerRewardQuery = await client.query<{ amount: string }>(
+        `
+        SELECT sum(
+          coinbase_amount + tx_fees_anchored_shared + tx_fees_anchored_exclusive + tx_fees_streamed_confirmed
+        ) amount
+        FROM miner_rewards
+        WHERE canonical = true AND recipient = $1 AND mature_block_height <= $2
+        `,
+        [stxAddress, blockHeight]
+      );
+      const totalRewards = BigInt(minerRewardQuery.rows[0]?.amount ?? 0);
+      const totalFees = BigInt(feeQuery.rows[0]?.fee_sum ?? 0);
+      const totalSent = BigInt(result.rows[0]?.debit_total ?? 0);
+      const totalReceived = BigInt(result.rows[0]?.credit_total ?? 0);
+      const balance = totalReceived - totalSent - totalFees + totalRewards;
       const locked = BigInt(lockQuery.rows[0]?.locked_amount ?? 0);
       const unlockHeight = Number(lockQuery.rows[0]?.unlock_height ?? 0);
-      const totalFees = BigInt(feeQuery.rows[0].fee_sum ?? 0);
-      const totalSent = BigInt(result.rows[0].debit_total ?? 0);
-      const totalReceived = BigInt(result.rows[0].credit_total ?? 0);
-      const balance = totalReceived - totalSent - totalFees;
       return {
         balance,
         locked,
         unlockHeight,
         totalSent,
         totalReceived,
+        totalFeesSent: totalFees,
+        totalMinerRewardsReceived: totalRewards,
       };
     } catch (e) {
       await client.query('ROLLBACK');

src/event-stream/event-server.ts

@@ -26,6 +26,7 @@ import {
   DataStoreUpdateData,
   createDbMempoolTxFromCoreMsg,
   DbStxLockEvent,
+  DbMinerReward,
 } from '../datastore/common';
 import { parseMessageTransactions, getTxSenderAddress, getTxSponsorAddress } from './reader';
 import { TransactionPayloadTypeID, readTransaction } from '../p2p/tx';
@@ -86,8 +87,26 @@ async function handleClientMessage(msg: CoreNodeMessage, db: DataStore): Promise
     dbBlock
   );
 
+  const dbMinerRewards: DbMinerReward[] = [];
+  for (const minerReward of msg.matured_miner_rewards) {
+    const dbMinerReward: DbMinerReward = {
+      canonical: true,
+      block_hash: minerReward.from_stacks_block_hash,
+      index_block_hash: minerReward.from_index_consensus_hash,
+      mature_block_height: parsedMsg.block_height,
+      recipient: minerReward.recipient,
+      coinbase_amount: BigInt(minerReward.coinbase_amount),
+      tx_fees_anchored_shared: BigInt(minerReward.tx_fees_anchored_shared),
+      tx_fees_anchored_exclusive: BigInt(minerReward.tx_fees_anchored_exclusive),
+      tx_fees_streamed_confirmed: BigInt(minerReward.tx_fees_streamed_confirmed),
+    };
+    dbMinerRewards.push(dbMinerReward);
+  }
+  logger.verbose(`Received ${dbMinerRewards.length} matured miner rewards`);
+
   const dbData: DataStoreUpdateData = {
     block: dbBlock,
+    minerRewards: dbMinerRewards,
     txs: new Array(parsedMsg.transactions.length),
   };
 
@@ -305,6 +324,9 @@ export async function startEventServer(opts: {
   app.postAsync('/new_block', async (req, res) => {
     try {
       const msg: CoreNodeMessage = req.body;
+      if (msg.matured_miner_rewards && msg.matured_miner_rewards.length > 0) {
+        console.log(msg.matured_miner_rewards);
+      }
       await messageHandler.handleBlockMessage(msg, db);
       res.status(200).json({ result: 'ok' });
     } catch (error) {

src/migrations/1605184662317_miner_rewards.ts

@@ -0,0 +1,53 @@
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+// block_hash, index_block_hash, canonical, recipient, coinbase_amount, tx_fees_anchored_shared, tx_fees_anchored_exclusive, tx_fees_streamed_confirmed
+export async function up(pgm: MigrationBuilder): Promise<void> {
+  pgm.createTable('miner_rewards', {
+    id: {
+      type: 'serial',
+      primaryKey: true,
+    },
+    block_hash: {
+      type: 'bytea',
+      notNull: true,
+    },
+    index_block_hash: {
+      type: 'bytea',
+      notNull: true,
+    },
+    mature_block_height: {
+      type: 'integer',
+      notNull: true,
+    },
+    canonical: {
+      type: 'boolean',
+      notNull: true,
+    },
+    recipient: {
+      type: 'string',
+      notNull: true,
+    },
+    coinbase_amount: {
+      type: 'numeric',
+      notNull: true,
+    },
+    tx_fees_anchored_shared: {
+      type: 'numeric',
+      notNull: true,
+    },
+    tx_fees_anchored_exclusive: {
+      type: 'numeric',
+      notNull: true,
+    },
+    tx_fees_streamed_confirmed: {
+      type: 'numeric',
+      notNull: true,
+    },
+  });
+
+  pgm.createIndex('miner_rewards', 'block_hash');
+  pgm.createIndex('miner_rewards', 'index_block_hash');
+  pgm.createIndex('miner_rewards', 'mature_block_height');
+  pgm.createIndex('miner_rewards', 'canonical');
+  pgm.createIndex('miner_rewards', 'recipient');
+
+}