feat: replace mmr + bitmap with smt (#5822)

Description
---
This replaces the UTXO MMR and the Roaring bitmap with a single SMT. 

Motivation and Context
---
See: tari-project/rfcs#105 as to why the SMT is
the better choice.

How Has This Been Tested?
---
Manual

applications/minotari_app_grpc/proto/block.proto

@@ -102,7 +102,7 @@ message HistoricalBlock {
     // confirmation.
     uint64 confirmations = 1;
     // The underlying block
-    Block block = 3;
+    Block block = 2;
 }
 
 

applications/minotari_app_grpc/src/conversions/block_header.rs

@@ -39,7 +39,7 @@ impl From<BlockHeader> for grpc::BlockHeader {
             timestamp: h.timestamp.as_u64(),
             input_mr: h.input_mr.to_vec(),
             output_mr: h.output_mr.to_vec(),
-            output_mmr_size: h.output_mmr_size,
+            output_mmr_size: h.output_smt_size,
             kernel_mr: h.kernel_mr.to_vec(),
             kernel_mmr_size: h.kernel_mmr_size,
             total_kernel_offset: h.total_kernel_offset.to_vec(),
@@ -75,7 +75,7 @@ impl TryFrom<grpc::BlockHeader> for BlockHeader {
             timestamp: EpochTime::from(header.timestamp),
             input_mr: FixedHash::try_from(header.input_mr).map_err(|err| err.to_string())?,
             output_mr: FixedHash::try_from(header.output_mr).map_err(|err| err.to_string())?,
-            output_mmr_size: header.output_mmr_size,
+            output_smt_size: header.output_mmr_size,
             kernel_mr: FixedHash::try_from(header.kernel_mr).map_err(|err| err.to_string())?,
             kernel_mmr_size: header.kernel_mmr_size,
             total_kernel_offset,

applications/minotari_app_grpc/src/conversions/historical_block.rs

@@ -32,11 +32,7 @@ impl TryFrom<HistoricalBlock> for grpc::HistoricalBlock {
     fn try_from(hb: HistoricalBlock) -> Result<Self, Self::Error> {
         Ok(Self {
             confirmations: hb.confirmations(),
-            block: Some(
-                hb.try_into_block()?
-                    .try_into()
-                    .map_err(ChainStorageError::ConversionError)?,
-            ),
+            block: Some(hb.into_block().try_into().map_err(ChainStorageError::ConversionError)?),
         })
     }
 }

applications/minotari_node/src/grpc/base_node_grpc_server.rs

@@ -437,7 +437,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
                         if is_reversed {
                             data.into_iter()
                                 .map(|chain_block| {
-                                    let (block, acc_data, confirmations, _) = chain_block.dissolve();
+                                    let (block, acc_data, confirmations) = chain_block.dissolve();
                                     match consensus_rules
                                         .calculate_coinbase_and_fees(block.header.height, block.body.kernels())
                                     {
@@ -459,7 +459,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
                         } else {
                             data.into_iter()
                                 .map(|chain_block| {
-                                    let (block, acc_data, confirmations, _) = chain_block.dissolve();
+                                    let (block, acc_data, confirmations) = chain_block.dissolve();
                                     match consensus_rules
                                         .calculate_coinbase_and_fees(block.header.height, block.body.kernels())
                                     {
@@ -1569,7 +1569,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
                 )
             })?;
 
-        let (block, acc_data, confirmations, _) = block.dissolve();
+        let (block, acc_data, confirmations) = block.dissolve();
         let total_block_reward = self
             .consensus_rules
             .calculate_coinbase_and_fees(block.header.height, block.body.kernels())

applications/minotari_node/src/recovery.rs

@@ -163,7 +163,7 @@ async fn do_recovery<D: BlockchainBackend + 'static>(
         let block = source_database
             .fetch_block(counter, true)
             .map_err(|e| anyhow!("Could not get block from recovery db: {}", e))?
-            .try_into_block()?;
+            .into_block();
         trace!(target: LOG_TARGET, "Adding block: {}", block);
         db.add_block(Arc::new(block))
             .await

base_layer/core/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2018"
 default = ["base_node"]
 transactions = []
 mempool_proto = []
-base_node = ["croaring", "tari_mmr", "transactions", "mempool_proto", "base_node_proto", "monero", "randomx-rs"]
+base_node = ["tari_mmr", "transactions", "mempool_proto", "base_node_proto", "monero", "randomx-rs"]
 base_node_proto = []
 benches = ["base_node", "criterion"]
 
@@ -45,7 +45,6 @@ bytes = "0.5"
 chacha20poly1305 = "0.10.1"
 chrono = { version = "0.4.19", default-features = false, features = ["serde"] }
 criterion = { version = "0.4.0", optional = true  }
-croaring = { version = "0.9", optional = true }
 decimal-rs = "0.1.42"
 derivative = "2.2.0"
 digest = "0.10"

base_layer/core/src/base_node/comms_interface/inbound_handlers.rs

@@ -47,7 +47,7 @@ use crate::{
         metrics,
     },
     blocks::{Block, BlockBuilder, BlockHeader, BlockHeaderValidationError, ChainBlock, NewBlock, NewBlockTemplate},
-    chain_storage::{async_db::AsyncBlockchainDb, BlockAddResult, BlockchainBackend, ChainStorageError, PrunedOutput},
+    chain_storage::{async_db::AsyncBlockchainDb, BlockAddResult, BlockchainBackend, ChainStorageError},
     consensus::{ConsensusConstants, ConsensusManager},
     mempool::Mempool,
     proof_of_work::{
@@ -163,14 +163,12 @@ where B: BlockchainBackend + 'static
             },
             NodeCommsRequest::FetchMatchingUtxos(utxo_hashes) => {
                 let mut res = Vec::with_capacity(utxo_hashes.len());
-                for (pruned_output, spent) in (self.blockchain_db.fetch_utxos(utxo_hashes).await?)
+                for (output, spent) in (self.blockchain_db.fetch_utxos(utxo_hashes).await?)
                     .into_iter()
                     .flatten()
                 {
-                    if let PrunedOutput::NotPruned { output } = pruned_output {
-                        if !spent {
-                            res.push(output);
-                        }
+                    if !spent {
+                        res.push(output);
                     }
                 }
                 Ok(NodeCommsResponse::TransactionOutputs(res))
@@ -367,7 +365,7 @@ where B: BlockchainBackend + 'static
                         },
                         Some,
                     ),
-                    Some(block) => Some(block.try_into_block()?),
+                    Some(block) => Some(block.into_block()),
                 };
 
                 Ok(NodeCommsResponse::Block(Box::new(maybe_block)))
@@ -417,12 +415,7 @@ where B: BlockchainBackend + 'static
             },
             NodeCommsRequest::FetchUnspentUtxosInBlock { block_hash } => {
                 let utxos = self.blockchain_db.fetch_outputs_in_block(block_hash).await?;
-                Ok(NodeCommsResponse::TransactionOutputs(
-                    utxos
-                        .into_iter()
-                        .filter_map(|utxo| utxo.into_unpruned_output())
-                        .collect(),
-                ))
+                Ok(NodeCommsResponse::TransactionOutputs(utxos))
             },
         }
     }
@@ -879,32 +872,22 @@ where B: BlockchainBackend + 'static
                         details: format!("Output {} to be spent does not exist in db", input.output_hash()),
                     })?;
 
-            match output_mined_info.output {
-                PrunedOutput::Pruned { .. } => {
-                    return Err(CommsInterfaceError::InvalidFullBlock {
-                        hash: block_hash,
-                        details: format!("Output {} to be spent is pruned", input.output_hash()),
-                    });
-                },
-                PrunedOutput::NotPruned { output } => {
-                    let rp_hash = match output.proof {
-                        Some(proof) => proof.hash(),
-                        None => FixedHash::zero(),
-                    };
-                    input.add_output_data(
-                        output.version,
-                        output.features,
-                        output.commitment,
-                        output.script,
-                        output.sender_offset_public_key,
-                        output.covenant,
-                        output.encrypted_data,
-                        output.metadata_signature,
-                        rp_hash,
-                        output.minimum_value_promise,
-                    );
-                },
-            }
+            let rp_hash = match output_mined_info.output.proof {
+                Some(proof) => proof.hash(),
+                None => FixedHash::zero(),
+            };
+            input.add_output_data(
+                output_mined_info.output.version,
+                output_mined_info.output.features,
+                output_mined_info.output.commitment,
+                output_mined_info.output.script,
+                output_mined_info.output.sender_offset_public_key,
+                output_mined_info.output.covenant,
+                output_mined_info.output.encrypted_data,
+                output_mined_info.output.metadata_signature,
+                rp_hash,
+                output_mined_info.output.minimum_value_promise,
+            );
         }
         debug!(
             target: LOG_TARGET,

base_layer/core/src/base_node/proto/rpc.proto

@@ -57,26 +57,12 @@ message SyncKernelsRequest {
 }
 
 message SyncUtxosRequest {
-  uint64 start = 1;
+  bytes start_header_hash = 1;
   bytes end_header_hash = 2;
-  bool include_pruned_utxos = 3;
-  bool include_deleted_bitmaps = 4;
 }
 message SyncUtxosResponse {
-  oneof utxo_or_deleted {
-    SyncUtxo utxo = 1;
-    bytes deleted_diff = 2;
-  }
-  uint64 mmr_index = 3;
-}
-
-message SyncUtxo {
-  oneof utxo {
-    // The unspent transaction output
-    tari.types.TransactionOutput output = 1;
-    // If the UTXO is deleted/pruned, the hashes are returned
-    PrunedOutput pruned_output = 2;
-  }
+  tari.types.TransactionOutput output = 1;
+  bytes mined_header = 2;
 }
 
 message PrunedOutput {

base_layer/core/src/base_node/proto/rpc.rs

@@ -24,7 +24,7 @@ use std::convert::{TryFrom, TryInto};
 
 use tari_utilities::ByteArray;
 
-use crate::{blocks::Block, chain_storage::PrunedOutput, mempool::FeePerGramStat, proto::base_node as proto};
+use crate::{blocks::Block, mempool::FeePerGramStat, proto::base_node as proto};
 
 impl TryFrom<Block> for proto::BlockBodyResponse {
     type Error = String;
@@ -37,22 +37,22 @@ impl TryFrom<Block> for proto::BlockBodyResponse {
     }
 }
 
-impl TryFrom<PrunedOutput> for proto::SyncUtxo {
-    type Error = String;
-
-    fn try_from(output: PrunedOutput) -> Result<Self, Self::Error> {
-        Ok(match output {
-            PrunedOutput::Pruned { output_hash } => proto::SyncUtxo {
-                utxo: Some(proto::sync_utxo::Utxo::PrunedOutput(proto::PrunedOutput {
-                    hash: output_hash.to_vec(),
-                })),
-            },
-            PrunedOutput::NotPruned { output } => proto::SyncUtxo {
-                utxo: Some(proto::sync_utxo::Utxo::Output(output.try_into()?)),
-            },
-        })
-    }
-}
+// impl TryFrom<PrunedOutput> for proto::SyncUtxo {
+//     type Error = String;
+//
+//     fn try_from(output: PrunedOutput) -> Result<Self, Self::Error> {
+//         Ok(match output {
+//             PrunedOutput::Pruned { output_hash } => proto::SyncUtxo {
+//                 utxo: Some(proto::sync_utxo::Utxo::PrunedOutput(proto::PrunedOutput {
+//                     hash: output_hash.to_vec(),
+//                 })),
+//             },
+//             PrunedOutput::NotPruned { output } => proto::SyncUtxo {
+//                 utxo: Some(proto::sync_utxo::Utxo::Output(output.try_into()?)),
+//             },
+//         })
+//     }
+// }
 
 impl From<Vec<FeePerGramStat>> for proto::GetMempoolFeePerGramStatsResponse {
     fn from(stats: Vec<FeePerGramStat>) -> Self {

base_layer/core/src/base_node/proto/wallet_rpc.proto

@@ -69,18 +69,21 @@ message FetchUtxosResponse {
 }
 
 message QueryDeletedRequest {
-  repeated uint64 mmr_positions = 1;
-  bytes chain_must_include_header = 2;
-  bool include_deleted_block_data = 3;
+  repeated bytes hashes = 1;
+  google.protobuf.BytesValue chain_must_include_header = 2;
 }
 
 message QueryDeletedResponse {
-  repeated uint64 deleted_positions = 1;
-  repeated uint64 not_deleted_positions = 2;
-  bytes best_block = 3;
-  uint64 height_of_longest_chain = 4;
-  repeated bytes blocks_deleted_in = 5;
-  repeated uint64 heights_deleted_at = 6;
+  repeated QueryDeletedData data = 1;
+  bytes best_block = 2;
+  uint64 height_of_longest_chain = 3;
+}
+
+message QueryDeletedData{
+  uint64 mined_height = 1;
+  bytes block_mined_in = 2;
+  uint64 height_deleted_at = 3;
+  bytes block_deleted_in = 4;
 }
 
 message UtxoQueryRequest {
@@ -95,11 +98,10 @@ message UtxoQueryResponses {
 
 message UtxoQueryResponse {
   tari.types.TransactionOutput output = 1;
-  uint64 mmr_position = 2;
-  uint64 mined_height = 3;
-  bytes mined_in_block = 4;
-  bytes output_hash = 5;
-  uint64 mined_timestamp = 6;
+  uint64 mined_height = 2;
+  bytes mined_in_block = 3;
+  bytes output_hash = 4;
+  uint64 mined_timestamp = 5;
 }
 
 message TipInfoResponse {

base_layer/core/src/base_node/proto/wallet_rpc.rs

@@ -29,7 +29,7 @@ use serde::{Deserialize, Serialize};
 use tari_common_types::types::{BlockHash, Signature};
 use tari_utilities::ByteArray;
 
-use crate::proto::{base_node as proto, types};
+use crate::proto::base_node as proto;
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct TxSubmissionResponse {
@@ -270,31 +270,3 @@ impl TryFrom<proto::TxQueryBatchResponse> for TxQueryBatchResponse {
         })
     }
 }
-
-impl proto::SyncUtxosResponse {
-    pub fn into_utxo(self) -> Option<proto::SyncUtxo> {
-        use proto::sync_utxos_response::UtxoOrDeleted::{DeletedDiff, Utxo};
-        match self.utxo_or_deleted? {
-            Utxo(utxo) => Some(utxo),
-            DeletedDiff(_) => None,
-        }
-    }
-
-    pub fn into_bitmap(self) -> Option<Vec<u8>> {
-        use proto::sync_utxos_response::UtxoOrDeleted::{DeletedDiff, Utxo};
-        match self.utxo_or_deleted? {
-            Utxo(_) => None,
-            DeletedDiff(bitmap) => Some(bitmap),
-        }
-    }
-}
-
-impl proto::sync_utxo::Utxo {
-    pub fn into_transaction_output(self) -> Option<types::TransactionOutput> {
-        use proto::sync_utxo::Utxo::{Output, PrunedOutput};
-        match self {
-            Output(output) => Some(output),
-            PrunedOutput(_) => None,
-        }
-    }
-}