feat(consensus)!: add balanced binary merkle tree

base_layer/core/src/blocks/genesis_block.rs

@@ -551,10 +551,7 @@ mod test {
         assert_eq!(kernel_mmr.get_merkle_root().unwrap(), block.header().kernel_mr,);
         assert_eq!(witness_mmr.get_merkle_root().unwrap(), block.header().witness_mr,);
         assert_eq!(output_mmr.get_merkle_root().unwrap(), block.header().output_mr,);
-        assert_eq!(
-            calculate_validator_node_mr(&vn_nodes).unwrap(),
-            block.header().validator_node_mr,
-        );
+        assert_eq!(calculate_validator_node_mr(&vn_nodes), block.header().validator_node_mr,);
 
         // Check that the faucet UTXOs balance (the faucet_value consensus constant is set correctly and faucet kernel
         // is correct)

base_layer/core/src/chain_storage/blockchain_database.rs

@@ -39,7 +39,7 @@ use tari_common_types::{
     types::{BlockHash, Commitment, FixedHash, HashOutput, PublicKey, Signature},
 };
 use tari_crypto::hash::blake2::Blake256;
-use tari_mmr::{error::MerkleMountainRangeError, pruned_hashset::PrunedHashSet};
+use tari_mmr::pruned_hashset::PrunedHashSet;
 use tari_utilities::{epoch_time::EpochTime, hex::Hex, ByteArray};
 
 use super::TemplateRegistrationEntry;
@@ -93,7 +93,7 @@ use crate::{
     PrunedInputMmr,
     PrunedKernelMmr,
     PrunedWitnessMmr,
-    ValidatorNodeMmr,
+    ValidatorNodeBMT,
 };
 
 const LOG_TARGET: &str = "c::cs::database";
@@ -1360,7 +1360,7 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
     let validator_node_mr = if block_height % epoch_len == 0 {
         // At epoch boundary, the MR is rebuilt from the current validator set
         let validator_nodes = db.fetch_active_validator_nodes(block_height)?;
-        FixedHash::try_from(calculate_validator_node_mr(&validator_nodes)?)?
+        FixedHash::try_from(calculate_validator_node_mr(&validator_nodes))?
     } else {
         // MR is unchanged except for epoch boundary
         let tip_header = fetch_header(db, block_height - 1)?;
@@ -1379,9 +1379,7 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
     Ok(mmr_roots)
 }
 
-pub fn calculate_validator_node_mr(
-    validator_nodes: &[(PublicKey, [u8; 32])],
-) -> Result<tari_mmr::Hash, MerkleMountainRangeError> {
+pub fn calculate_validator_node_mr(validator_nodes: &[(PublicKey, [u8; 32])]) -> tari_mmr::Hash {
     fn hash_node((pk, s): &(PublicKey, [u8; 32])) -> Vec<u8> {
         use digest::Digest;
         Blake256::new()
@@ -1391,12 +1389,8 @@ pub fn calculate_validator_node_mr(
             .to_vec()
     }
 
-    let mut vn_mmr = ValidatorNodeMmr::new(Vec::new());
-    for vn in validator_nodes {
-        vn_mmr.push(hash_node(vn))?;
-    }
-    let merkle_root = vn_mmr.get_merkle_root()?;
-    Ok(merkle_root)
+    let vn_bmt = ValidatorNodeBMT::create(validator_nodes.iter().map(hash_node).collect::<Vec<_>>());
+    vn_bmt.get_merkle_root()
 }
 
 pub fn fetch_header<T: BlockchainBackend>(db: &T, block_num: u64) -> Result<BlockHeader, ChainStorageError> {

base_layer/core/src/chain_storage/tests/blockchain_database.rs

@@ -583,6 +583,7 @@ mod validator_node_merkle_root {
     use super::*;
     use crate::{
         transactions::transaction_components::{OutputFeatures, ValidatorNodeSignature},
+        ValidatorNodeBMT,
         ValidatorNodeMmr,
     };
 
@@ -618,18 +619,13 @@ mod validator_node_merkle_root {
             .unwrap()
             .unwrap();
 
-        let mut vn_mmr = ValidatorNodeMmr::new(Vec::new());
-        vn_mmr
-            .push(
-                Blake256::new()
-                    .chain(public_key.as_bytes())
-                    .chain(shard_key.as_slice())
-                    .finalize()
-                    .to_vec(),
-            )
-            .unwrap();
+        let vn_bmt = ValidatorNodeBMT::create(vec![Blake256::new()
+            .chain(public_key.as_bytes())
+            .chain(shard_key.as_slice())
+            .finalize()
+            .to_vec()]);
 
         let tip = db.fetch_tip_header().unwrap();
-        assert_eq!(tip.header().validator_node_mr, vn_mmr.get_merkle_root().unwrap());
+        assert_eq!(tip.header().validator_node_mr, vn_bmt.get_merkle_root());
     }
 }

base_layer/core/src/lib.rs

@@ -72,7 +72,7 @@ pub use large_ints::{U256, U512};
 #[cfg(feature = "base_node")]
 mod domain_hashing {
     use tari_crypto::{hash::blake2::Blake256, hash_domain, hashing::DomainSeparatedHasher};
-    use tari_mmr::{pruned_hashset::PrunedHashSet, Hash, MerkleMountainRange, MutableMmr};
+    use tari_mmr::{pruned_hashset::PrunedHashSet, BalancedBinaryMerkleTree, Hash, MerkleMountainRange, MutableMmr};
 
     hash_domain!(
         KernelMmrHashDomain,
@@ -118,6 +118,7 @@ mod domain_hashing {
     );
     pub type ValidatorNodeMmrHasherBlake256 = DomainSeparatedHasher<Blake256, ValidatorNodeMmrHashDomain>;
     pub type ValidatorNodeMmr = MerkleMountainRange<ValidatorNodeMmrHasherBlake256, Vec<Hash>>;
+    pub type ValidatorNodeBMT = BalancedBinaryMerkleTree<ValidatorNodeMmrHasherBlake256>;
 }
 
 #[cfg(feature = "base_node")]

base_layer/core/tests/helpers/block_builders.rs

@@ -182,7 +182,7 @@ fn print_new_genesis_block(network: Network, extra: &str) {
     witness_mmr.push(utxo.witness_hash().to_vec()).unwrap();
     let mut output_mmr = MutableOutputMmr::new(Vec::new(), Bitmap::create()).unwrap();
     output_mmr.push(utxo.hash().to_vec()).unwrap();
-    let vn_mr = calculate_validator_node_mr(&[]).unwrap();
+    let vn_mr = calculate_validator_node_mr(&[]);
 
     header.kernel_mr = FixedHash::try_from(kernel_mmr.get_merkle_root().unwrap()).unwrap();
     header.kernel_mmr_size += 1;

base_layer/mmr/src/balanced_binary_merkle_proof.rs

@@ -0,0 +1,102 @@
+// Copyright 2019. The Tari Project
+//
+// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+// following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+// disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+// following disclaimer in the documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+// products derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+use std::marker::PhantomData;
+
+use digest::Digest;
+use tari_common::DomainDigest;
+
+use crate::{common::hash_together, BalancedBinaryMerkleTree, Hash};
+
+#[derive(Debug)]
+pub struct BalancedBinaryMerkleProof<D> {
+    pub path: Vec<Hash>,
+    pub node_index: usize,
+    _phantom: PhantomData<D>,
+}
+
+// Since this is balanced tree, the index `2k+1` is always left child and `2k` is right child
+
+impl<D> BalancedBinaryMerkleProof<D>
+where D: Digest + DomainDigest
+{
+    pub fn verify(&self, root: &Hash, leaf_hash: Hash) -> bool {
+        let mut computed_root = leaf_hash;
+        let mut node_index = self.node_index;
+        for sibling in &self.path {
+            if node_index & 1 == 1 {
+                computed_root = hash_together::<D>(&computed_root, sibling);
+            } else {
+                computed_root = hash_together::<D>(sibling, &computed_root);
+            }
+            node_index = (node_index - 1) >> 1;
+        }
+        &computed_root == root
+    }
+
+    pub fn generate_proof(tree: &BalancedBinaryMerkleTree<D>, leaf_index: usize) -> Self {
+        let mut node_index = tree.get_node_index(leaf_index);
+        let mut proof = Vec::new();
+        while node_index > 0 {
+            // Sibling
+            let parent = (node_index - 1) >> 1;
+            // The children are 2i+1 and 2i+2, so together are 4i+3, we substract one, we get the other.
+            let sibling = 4 * parent + 3 - node_index;
+            proof.push(tree.get_hash(sibling).clone());
+            // Traverse to parent
+            node_index = parent;
+        }
+        Self {
+            path: proof,
+            node_index: tree.get_node_index(leaf_index),
+            _phantom: PhantomData,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use tari_crypto::{hash::blake2::Blake256, hash_domain, hashing::DomainSeparatedHasher};
+
+    use crate::{BalancedBinaryMerkleProof, BalancedBinaryMerkleTree};
+    hash_domain!(TestDomain, "testing", 0);
+
+    #[test]
+    fn test_generate_and_verify_big_tree() {
+        for n in [1usize, 100, 1000, 10000] {
+            let leaves = (0..n)
+                .map(|i| [i.to_le_bytes().to_vec(), vec![0u8; 24]].concat())
+                .collect::<Vec<_>>();
+            let hash_0 = leaves[0].clone();
+            let hash_n_half = leaves[n / 2].clone();
+            let hash_last = leaves[n - 1].clone();
+            let bmt = BalancedBinaryMerkleTree::<DomainSeparatedHasher<Blake256, TestDomain>>::create(leaves);
+            let root = bmt.get_merkle_root();
+            let proof = BalancedBinaryMerkleProof::generate_proof(&bmt, 0);
+            assert!(proof.verify(&root, hash_0));
+            let proof = BalancedBinaryMerkleProof::generate_proof(&bmt, n / 2);
+            assert!(proof.verify(&root, hash_n_half));
+            let proof = BalancedBinaryMerkleProof::generate_proof(&bmt, n - 1);
+            assert!(proof.verify(&root, hash_last));
+        }
+    }
+}

base_layer/mmr/src/balanced_binary_merkle_tree.rs

@@ -0,0 +1,148 @@
+// Copyright 2019. The Tari Project
+//
+// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+// following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+// disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+// following disclaimer in the documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+// products derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+use std::marker::PhantomData;
+
+use digest::Digest;
+use tari_common::DomainDigest;
+use thiserror::Error;
+
+use crate::{common::hash_together, ArrayLike, Hash};
+
+#[derive(Clone, Debug, PartialEq, Eq, Error)]
+pub enum BalancedBinaryMerkleTreeError {
+    #[error("There is no leaf with the hash provided.")]
+    LeafNotFound,
+}
+
+// The hashes are perfectly balanced binary tree, so parent at index `i` (0-based) has children at positions `2*i+1` and
+// `2*i+1`.
+#[derive(Debug)]
+pub struct BalancedBinaryMerkleTree<D> {
+    hashes: Vec<Hash>,
+    _phantom: PhantomData<D>,
+}
+
+impl<D> BalancedBinaryMerkleTree<D>
+where D: Digest + DomainDigest
+{
+    // There is no push method for this tree. This tree is created at once and no modifications are allowed.
+    pub fn create(leaves: Vec<Hash>) -> Self {
+        let leaves_cnt = leaves.len();
+        if leaves_cnt == 0 {
+            return Self {
+                hashes: vec![],
+                _phantom: PhantomData,
+            };
+        }
+        // The size of the tree of `n` leaves is `2*n+1` where the leaves are at the end of the array.
+        let mut hashes = Vec::with_capacity(2 * leaves_cnt - 1);
+        hashes.extend(vec![vec![0; 32]; leaves_cnt - 1]);
+        hashes.extend(leaves);
+        // Now we compute the hashes from bottom to up of the tree.
+        for i in (0..leaves_cnt - 1).rev() {
+            hashes[i] = hash_together::<D>(&hashes[2 * i + 1], &hashes[2 * i + 2]);
+        }
+        Self {
+            hashes,
+            _phantom: PhantomData,
+        }
+    }
+
+    pub fn get_merkle_root(&self) -> Hash {
+        if self.hashes.is_empty() {
+            D::digest(b"").to_vec()
+        } else {
+            self.hashes[0].clone()
+        }
+    }
+
+    pub(crate) fn get_hash(&self, pos: usize) -> &Hash {
+        &self.hashes[pos]
+    }
+
+    pub fn get_leaf(&self, leaf_index: usize) -> &Hash {
+        self.get_hash(self.get_node_index(leaf_index))
+    }
+
+    pub(crate) fn get_node_index(&self, leaf_index: usize) -> usize {
+        leaf_index + (self.hashes.len() >> 1)
+    }
+
+    pub fn find_leaf_index_for_hash(&self, hash: &Hash) -> Result<usize, BalancedBinaryMerkleTreeError> {
+        let pos = self
+            .hashes
+            .position(hash)
+            .expect("Unexpected Balanced Binary Merkle Tree error")
+            .ok_or(BalancedBinaryMerkleTreeError::LeafNotFound)?;
+        if pos < (self.hashes.len() >> 1) {
+            // The hash provided was not for leaf, but for node.
+            Err(BalancedBinaryMerkleTreeError::LeafNotFound)
+        } else {
+            Ok(pos - (self.hashes.len() >> 1))
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use tari_crypto::{hash::blake2::Blake256, hash_domain, hashing::DomainSeparatedHasher};
+
+    use crate::{balanced_binary_merkle_tree::BalancedBinaryMerkleTreeError, BalancedBinaryMerkleTree};
+    hash_domain!(TestDomain, "testing", 0);
+
+    #[test]
+    fn test_empty_tree() {
+        let leaves = vec![];
+        let bmt = BalancedBinaryMerkleTree::<DomainSeparatedHasher<Blake256, TestDomain>>::create(leaves);
+        let root = bmt.get_merkle_root();
+        assert_eq!(root, vec![
+            72, 54, 179, 2, 214, 45, 9, 89, 161, 132, 177, 251, 229, 46, 124, 233, 32, 186, 46, 87, 127, 247, 19, 36,
+            225, 191, 167, 189, 58, 58, 39, 74
+        ]);
+    }
+
+    #[test]
+    fn test_single_node_tree() {
+        let leaves = vec![vec![0; 32]];
+        let bmt = BalancedBinaryMerkleTree::<DomainSeparatedHasher<Blake256, TestDomain>>::create(leaves);
+        let root = bmt.get_merkle_root();
+        assert_eq!(root, vec![0; 32]);
+    }
+
+    #[test]
+    fn test_find_leaf() {
+        let leaves = (0..100).map(|i| vec![i; 32]).collect::<Vec<_>>();
+        let bmt = BalancedBinaryMerkleTree::<DomainSeparatedHasher<Blake256, TestDomain>>::create(leaves);
+        assert_eq!(bmt.find_leaf_index_for_hash(&vec![42; 32]).unwrap(), 42);
+        // Non existing hash
+        assert_eq!(
+            bmt.find_leaf_index_for_hash(&vec![142; 32]),
+            Err(BalancedBinaryMerkleTreeError::LeafNotFound)
+        );
+        // This hash exists but it's not a leaf.
+        assert_eq!(
+            bmt.find_leaf_index_for_hash(&bmt.get_merkle_root()),
+            Err(BalancedBinaryMerkleTreeError::LeafNotFound)
+        );
+    }
+}

base_layer/mmr/src/lib.rs

@@ -136,6 +136,8 @@ pub type Hash = Vec<u8>;
 pub type HashSlice = [u8];
 
 mod backend;
+mod balanced_binary_merkle_proof;
+mod balanced_binary_merkle_tree;
 mod mem_backend_vec;
 mod merkle_mountain_range;
 mod merkle_proof;
@@ -150,6 +152,8 @@ pub mod pruned_hashset;
 // Commonly used exports
 /// A vector-based backend for [MerkleMountainRange]
 pub use backend::{ArrayLike, ArrayLikeExt};
+pub use balanced_binary_merkle_proof::BalancedBinaryMerkleProof;
+pub use balanced_binary_merkle_tree::BalancedBinaryMerkleTree;
 /// MemBackendVec is a shareable, memory only, vector that can be be used with MmrCache to store checkpoints.
 pub use mem_backend_vec::MemBackendVec;
 /// An immutable, append-only Merkle Mountain range (MMR) data structure