From d50d5e426d11972a3ca5f004cb2f77ec32812214 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 14:05:06 -0400 Subject: [PATCH 01/25] initial ideas --- Cargo.lock | 1 + substrate/primitives/runtime/Cargo.toml | 2 + substrate/primitives/runtime/src/lib.rs | 1 + .../primitives/runtime/src/proving_trie.rs | 111 ++++++++++++++++++ 4 files changed, 115 insertions(+) create mode 100644 substrate/primitives/runtime/src/proving_trie.rs diff --git a/Cargo.lock b/Cargo.lock index 413bd28abe03..5e61a544ee04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18979,6 +18979,7 @@ dependencies = [ "sp-state-machine", "sp-std 14.0.0", "sp-tracing 16.0.0", + "sp-trie", "sp-weights", "substrate-test-runtime-client", "zstd 0.12.4", diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index cacfc0597229..778bb7d23e32 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -31,6 +31,7 @@ sp-arithmetic = { path = "../arithmetic", default-features = false } sp-core = { path = "../core", default-features = false } sp-io = { path = "../io", default-features = false } sp-std = { path = "../std", default-features = false } +sp-trie = { path = "../trie", default-features = false } sp-weights = { path = "../weights", default-features = false } docify = { version = "0.2.7" } @@ -66,6 +67,7 @@ std = [ "sp-state-machine/std", "sp-std/std", "sp-tracing/std", + "sp-trie/std", "sp-weights/std", ] diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index 44bf3c969e54..455d6e7b9a50 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -85,6 +85,7 @@ pub mod generic; pub mod legacy; mod multiaddress; pub mod offchain; +pub mod proving_trie; pub mod runtime_logger; mod runtime_string; #[cfg(feature = "std")] diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs new file mode 100644 index 000000000000..cb9bd97d3eb5 --- /dev/null +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -0,0 +1,111 @@ +use sp_trie::{ + trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, + LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, +}; + +use crate::{traits::HashOutput, Decode, Encode, KeyTypeId}; + +/// A trie instance for checking and generating proofs. +pub struct ProvingTrie +where + Hashing: sp_core::Hasher, + Hash: HashOutput, + Item: Encode + Decode, +{ + db: MemoryDB, + root: Hash, + _phantom: core::marker::PhantomData, +} + +impl ProvingTrie +where + Hashing: sp_core::Hasher, + Hash: HashOutput, + Item: Encode + Decode, +{ + /// Access the underlying trie root. + pub fn root(&self) -> &Hash { + &self.root + } + + // Check a proof contained within the current memory-db. Returns `None` if the + // nodes within the current `MemoryDB` are insufficient to query the item. + fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { + let trie = TrieDBBuilder::new(&self.db, &self.root).build(); + let val_idx = (key_id, key_data) + .using_encoded(|s| trie.get(s)) + .ok()? + .and_then(|raw| u32::decode(&mut &*raw).ok())?; + + val_idx + .using_encoded(|s| trie.get(s)) + .ok()? + .and_then(|raw| Item::decode(&mut &*raw).ok()) + } + + /// Prove the full verification data for a given key and key ID. + pub fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { + let mut recorder = Recorder::>::new(); + { + let trie = + TrieDBBuilder::new(&self.db, &self.root).with_recorder(&mut recorder).build(); + let val_idx = (key_id, key_data).using_encoded(|s| { + trie.get(s).ok()?.and_then(|raw| u32::decode(&mut &*raw).ok()) + })?; + + val_idx.using_encoded(|s| { + trie.get(s).ok()?.and_then(|raw| Item::decode(&mut &*raw).ok()) + })?; + } + + Some(recorder.drain().into_iter().map(|r| r.data).collect()) + } + + fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { + use sp_trie::HashDBT; + + let mut memory_db = MemoryDB::default(); + for node in nodes { + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); + } + + ProvingTrie { db: memory_db, root, _phantom: Default::default() } + } + + // fn generate_for(items: I) -> Result + // where + // I: IntoIterator, + // { + // let mut db = MemoryDB::default(); + // let mut root = Default::default(); + + // { + // let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); + // for (i, (validator, full_id)) in validators.into_iter().enumerate() { + // let i = i as u32; + // let keys = match >::load_keys(&validator) { + // None => continue, + // Some(k) => k, + // }; + + // let full_id = (validator, full_id); + + // // map each key to the owner index. + // for key_id in T::Keys::key_ids() { + // let key = keys.get_raw(*key_id); + // let res = + // (key_id, key).using_encoded(|k| i.using_encoded(|v| trie.insert(k, v))); + + // let _ = res.map_err(|_| "failed to insert into trie")?; + // } + + // // map each owner index to the full identification. + // let _ = i + // .using_encoded(|k| full_id.using_encoded(|v| trie.insert(k, v))) + // .map_err(|_| "failed to insert into trie")?; + // } + // } + + // Ok(ProvingTrie { db, root }) + // } +} From 7f4422e090ca583bbb4ab20c9555e420d081046f Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 14:21:20 -0400 Subject: [PATCH 02/25] Update proving_trie.rs --- .../primitives/runtime/src/proving_trie.rs | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index cb9bd97d3eb5..b4307f9164e9 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -1,3 +1,22 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A generic implementation of a simple merkle trie used for making and verifying proofs. + use sp_trie::{ trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, @@ -28,9 +47,9 @@ where &self.root } - // Check a proof contained within the current memory-db. Returns `None` if the - // nodes within the current `MemoryDB` are insufficient to query the item. - fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { + /// Check a proof contained within the current memory-db. Returns `None` if the + /// nodes within the current `MemoryDB` are insufficient to query the item. + pub fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); let val_idx = (key_id, key_data) .using_encoded(|s| trie.get(s)) @@ -61,7 +80,8 @@ where Some(recorder.drain().into_iter().map(|r| r.data).collect()) } - fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { + /// Create a new instance of a `ProvingTrie` using a set of raw nodes. + pub fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { use sp_trie::HashDBT; let mut memory_db = MemoryDB::default(); @@ -72,40 +92,25 @@ where ProvingTrie { db: memory_db, root, _phantom: Default::default() } } - // fn generate_for(items: I) -> Result - // where - // I: IntoIterator, - // { - // let mut db = MemoryDB::default(); - // let mut root = Default::default(); - - // { - // let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); - // for (i, (validator, full_id)) in validators.into_iter().enumerate() { - // let i = i as u32; - // let keys = match >::load_keys(&validator) { - // None => continue, - // Some(k) => k, - // }; - - // let full_id = (validator, full_id); - - // // map each key to the owner index. - // for key_id in T::Keys::key_ids() { - // let key = keys.get_raw(*key_id); - // let res = - // (key_id, key).using_encoded(|k| i.using_encoded(|v| trie.insert(k, v))); - - // let _ = res.map_err(|_| "failed to insert into trie")?; - // } - - // // map each owner index to the full identification. - // let _ = i - // .using_encoded(|k| full_id.using_encoded(|v| trie.insert(k, v))) - // .map_err(|_| "failed to insert into trie")?; - // } - // } - - // Ok(ProvingTrie { db, root }) - // } + /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. + pub fn generate_for(items: I) -> Result + where + I: IntoIterator, + { + let mut db = MemoryDB::default(); + let mut root = Default::default(); + + { + let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); + for (i, item) in items.into_iter().enumerate() { + let i = i as u32; + + // insert each item into the trie + i.using_encoded(|k| item.using_encoded(|v| trie.insert(k, v))) + .map_err(|_| "failed to insert into trie")?; + } + } + + Ok(ProvingTrie { db, root, _phantom: Default::default() }) + } } From bdc0c8422ae02d42df9ab0cf4cbdf820925348e8 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 14:31:47 -0400 Subject: [PATCH 03/25] create trait --- .../primitives/runtime/src/proving_trie.rs | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index b4307f9164e9..bd24a25eb65c 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -15,17 +15,38 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! A generic implementation of a simple merkle trie used for making and verifying proofs. +//! Types for a simple merkle trie used for checking and generating proofs. use sp_trie::{ trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -use crate::{traits::HashOutput, Decode, Encode, KeyTypeId}; +use crate::{traits::HashOutput, Decode, DispatchError, Encode, KeyTypeId}; + +/// A trait for creating a merkle trie for checking and generating merkle proofs. +pub trait ProvingTrie +where + Hashing: sp_core::Hasher, + Hash: HashOutput, + Item: Encode + Decode, + Self: Sized, +{ + /// Access the underlying trie root. + fn root(&self) -> &Hash; + /// Check a proof contained within the current memory-db. Returns `None` if the + /// nodes within the current `MemoryDB` are insufficient to query the item. + fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option; + /// Prove the full verification data for a given key and key ID. + fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>>; + /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. + fn generate(items: I) -> Result + where + I: IntoIterator; +} /// A trie instance for checking and generating proofs. -pub struct ProvingTrie +pub struct BasicProvingTrie where Hashing: sp_core::Hasher, Hash: HashOutput, @@ -36,7 +57,7 @@ where _phantom: core::marker::PhantomData, } -impl ProvingTrie +impl BasicProvingTrie where Hashing: sp_core::Hasher, Hash: HashOutput, @@ -89,11 +110,11 @@ where HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); } - ProvingTrie { db: memory_db, root, _phantom: Default::default() } + Self { db: memory_db, root, _phantom: Default::default() } } /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - pub fn generate_for(items: I) -> Result + pub fn generate_for(items: I) -> Result where I: IntoIterator, { @@ -111,6 +132,6 @@ where } } - Ok(ProvingTrie { db, root, _phantom: Default::default() }) + Ok(Self { db, root, _phantom: Default::default() }) } } From da385ab4390ce36dee639014ddf0c6e0ead25f8a Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 14:38:05 -0400 Subject: [PATCH 04/25] use trait --- .../primitives/runtime/src/proving_trie.rs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index bd24a25eb65c..346a425d279b 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -57,20 +57,20 @@ where _phantom: core::marker::PhantomData, } -impl BasicProvingTrie +impl ProvingTrie for BasicProvingTrie where Hashing: sp_core::Hasher, Hash: HashOutput, Item: Encode + Decode, { /// Access the underlying trie root. - pub fn root(&self) -> &Hash { + fn root(&self) -> &Hash { &self.root } /// Check a proof contained within the current memory-db. Returns `None` if the /// nodes within the current `MemoryDB` are insufficient to query the item. - pub fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { + fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); let val_idx = (key_id, key_data) .using_encoded(|s| trie.get(s)) @@ -84,7 +84,7 @@ where } /// Prove the full verification data for a given key and key ID. - pub fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { + fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { let mut recorder = Recorder::>::new(); { let trie = @@ -101,20 +101,8 @@ where Some(recorder.drain().into_iter().map(|r| r.data).collect()) } - /// Create a new instance of a `ProvingTrie` using a set of raw nodes. - pub fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { - use sp_trie::HashDBT; - - let mut memory_db = MemoryDB::default(); - for node in nodes { - HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); - } - - Self { db: memory_db, root, _phantom: Default::default() } - } - /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - pub fn generate_for(items: I) -> Result + fn generate(items: I) -> Result where I: IntoIterator, { @@ -135,3 +123,22 @@ where Ok(Self { db, root, _phantom: Default::default() }) } } + +impl BasicProvingTrie +where + Hashing: sp_core::Hasher, + Hash: HashOutput, + Item: Encode + Decode, +{ + /// Create a new instance of a `ProvingTrie` using a set of raw nodes. + pub fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { + use sp_trie::HashDBT; + + let mut memory_db = MemoryDB::default(); + for node in nodes { + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); + } + + Self { db: memory_db, root, _phantom: Default::default() } + } +} From 6ab4bfbc6aa46a10cfefc5bdc5ebb856313d2728 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 15:09:45 -0400 Subject: [PATCH 05/25] clean up trait and basic trie further --- .../primitives/runtime/src/proving_trie.rs | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 346a425d279b..9974ca41100a 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -19,17 +19,14 @@ use sp_trie::{ trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, - LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, + LayoutV0, MemoryDB, Recorder, Trie, TrieMut, }; -use crate::{traits::HashOutput, Decode, DispatchError, Encode, KeyTypeId}; +use crate::{Decode, DispatchError, Encode, KeyTypeId}; /// A trait for creating a merkle trie for checking and generating merkle proofs. pub trait ProvingTrie where - Hashing: sp_core::Hasher, - Hash: HashOutput, - Item: Encode + Decode, Self: Sized, { /// Access the underlying trie root. @@ -40,7 +37,7 @@ where /// Prove the full verification data for a given key and key ID. fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>>; /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - fn generate(items: I) -> Result + fn generate_for(items: I) -> Result where I: IntoIterator; } @@ -49,8 +46,6 @@ where pub struct BasicProvingTrie where Hashing: sp_core::Hasher, - Hash: HashOutput, - Item: Encode + Decode, { db: MemoryDB, root: Hash, @@ -60,7 +55,7 @@ where impl ProvingTrie for BasicProvingTrie where Hashing: sp_core::Hasher, - Hash: HashOutput, + Hash: Default, Item: Encode + Decode, { /// Access the underlying trie root. @@ -102,7 +97,7 @@ where } /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - fn generate(items: I) -> Result + fn generate_for(items: I) -> Result where I: IntoIterator, { @@ -123,22 +118,3 @@ where Ok(Self { db, root, _phantom: Default::default() }) } } - -impl BasicProvingTrie -where - Hashing: sp_core::Hasher, - Hash: HashOutput, - Item: Encode + Decode, -{ - /// Create a new instance of a `ProvingTrie` using a set of raw nodes. - pub fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { - use sp_trie::HashDBT; - - let mut memory_db = MemoryDB::default(); - for node in nodes { - HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); - } - - Self { db: memory_db, root, _phantom: Default::default() } - } -} From 1cfb29f69dd02076c1ba051786a36096e354e6c1 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 15:09:58 -0400 Subject: [PATCH 06/25] use trait in session historical --- substrate/frame/session/src/historical/mod.rs | 47 ++++++++++--------- .../frame/session/src/historical/offchain.rs | 11 +++-- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/substrate/frame/session/src/historical/mod.rs b/substrate/frame/session/src/historical/mod.rs index b9cecea1a7f7..3c30a3126660 100644 --- a/substrate/frame/session/src/historical/mod.rs +++ b/substrate/frame/session/src/historical/mod.rs @@ -32,8 +32,9 @@ mod shared; use codec::{Decode, Encode}; use sp_runtime::{ + proving_trie::ProvingTrie, traits::{Convert, OpaqueKeys}, - KeyTypeId, + DispatchError, KeyTypeId, }; use sp_session::{MembershipProof, ValidatorCount}; use sp_staking::SessionIndex; @@ -176,7 +177,7 @@ impl> NoteHi if let Some(new_validators) = new_validators_and_id { let count = new_validators.len() as ValidatorCount; - match ProvingTrie::::generate_for(new_validators) { + match ValidatorProvingTrie::::generate_for(new_validators) { Ok(trie) => >::insert(new_index, &(trie.root, count)), Err(reason) => { print("Failed to generate historical ancestry-inclusion proof."); @@ -221,13 +222,15 @@ pub type IdentificationTuple = (::ValidatorId, ::FullIdentification); /// A trie instance for checking and generating proofs. -pub struct ProvingTrie { +pub struct ValidatorProvingTrie { db: MemoryDB, root: T::Hash, } -impl ProvingTrie { - fn generate_for(validators: I) -> Result +impl ProvingTrie> + for ValidatorProvingTrie +{ + fn generate_for(validators: I) -> Result where I: IntoIterator, { @@ -261,22 +264,11 @@ impl ProvingTrie { } } - Ok(ProvingTrie { db, root }) - } - - fn from_nodes(root: T::Hash, nodes: &[Vec]) -> Self { - use sp_trie::HashDBT; - - let mut memory_db = MemoryDB::default(); - for node in nodes { - HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); - } - - ProvingTrie { db: memory_db, root } + Ok(Self { db, root }) } /// Prove the full verification data for a given key and key ID. - pub fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { + fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { let mut recorder = Recorder::>::new(); { let trie = @@ -296,7 +288,7 @@ impl ProvingTrie { } /// Access the underlying trie root. - pub fn root(&self) -> &T::Hash { + fn root(&self) -> &T::Hash { &self.root } @@ -316,6 +308,19 @@ impl ProvingTrie { } } +impl ValidatorProvingTrie { + fn from_nodes(root: T::Hash, nodes: &[Vec]) -> Self { + use sp_trie::HashDBT; + + let mut memory_db = MemoryDB::default(); + for node in nodes { + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); + } + + Self { db: memory_db, root } + } +} + impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet { type Proof = MembershipProof; type IdentificationTuple = IdentificationTuple; @@ -332,7 +337,7 @@ impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet::generate_for(validators).ok()?; + let trie = ValidatorProvingTrie::::generate_for(validators).ok()?; let (id, data) = key; trie.prove(id, data.as_ref()).map(|trie_nodes| MembershipProof { @@ -364,7 +369,7 @@ impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet::from_nodes(root, &proof.trie_nodes); + let trie = ValidatorProvingTrie::::from_nodes(root, &proof.trie_nodes); trie.query(id, data.as_ref()) } } diff --git a/substrate/frame/session/src/historical/offchain.rs b/substrate/frame/session/src/historical/offchain.rs index 95f4d762949e..5a5a1114b9bf 100644 --- a/substrate/frame/session/src/historical/offchain.rs +++ b/substrate/frame/session/src/historical/offchain.rs @@ -20,17 +20,18 @@ //! Validator-set extracting an iterator from an off-chain worker stored list containing historical //! validator-sets. Based on the logic of historical slashing, but the validation is done off-chain. //! Use [`fn store_current_session_validator_set_to_offchain()`](super::onchain) to store the -//! required data to the offchain validator set. This is used in conjunction with [`ProvingTrie`] -//! and the off-chain indexing API. +//! required data to the offchain validator set. This is used in conjunction with +//! [`ValidatorProvingTrie`] and the off-chain indexing API. use sp_runtime::{ offchain::storage::{MutateStorageError, StorageRetrievalError, StorageValueRef}, + proving_trie::ProvingTrie, KeyTypeId, }; use sp_session::MembershipProof; use sp_std::prelude::*; -use super::{shared, Config, IdentificationTuple, ProvingTrie}; +use super::{shared, Config, IdentificationTuple, ValidatorProvingTrie}; use crate::{Pallet as SessionModule, SessionIndex}; /// A set of validators, which was used for a fixed session index. @@ -59,7 +60,7 @@ impl ValidatorSet { } /// Implement conversion into iterator for usage -/// with [ProvingTrie](super::ProvingTrie::generate_for). +/// with [ValidatorProvingTrie](super::ValidatorProvingTrie::generate_for). impl sp_std::iter::IntoIterator for ValidatorSet { type Item = (T::ValidatorId, T::FullIdentification); type IntoIter = sp_std::vec::IntoIter; @@ -79,7 +80,7 @@ pub fn prove_session_membership>( ) -> Option { let validators = ValidatorSet::::load_from_offchain_db(session_index)?; let count = validators.len() as u32; - let trie = ProvingTrie::::generate_for(validators.into_iter()).ok()?; + let trie = ValidatorProvingTrie::::generate_for(validators.into_iter()).ok()?; let (id, data) = session_key; trie.prove(id, data.as_ref()).map(|trie_nodes| MembershipProof { From 3080c7b3bd5ec3860da49eaa64bc9b45c261ed4d Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 17:44:42 -0400 Subject: [PATCH 07/25] fix api, add basic end to end test --- .../primitives/runtime/src/proving_trie.rs | 162 ++++++++++++------ 1 file changed, 110 insertions(+), 52 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 9974ca41100a..520d6379a0b2 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -17,104 +17,162 @@ //! Types for a simple merkle trie used for checking and generating proofs. +use crate::{Decode, DispatchError, Encode}; + +use sp_std::vec::Vec; use sp_trie::{ trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, - LayoutV0, MemoryDB, Recorder, Trie, TrieMut, + LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -use crate::{Decode, DispatchError, Encode, KeyTypeId}; - /// A trait for creating a merkle trie for checking and generating merkle proofs. -pub trait ProvingTrie +pub trait ProvingTrie where Self: Sized, { + /// Create a new instance of a `ProvingTrie` using an iterator of key/value pairs. + fn generate_for(items: I) -> Result + where + I: IntoIterator; /// Access the underlying trie root. fn root(&self) -> &Hash; /// Check a proof contained within the current memory-db. Returns `None` if the /// nodes within the current `MemoryDB` are insufficient to query the item. - fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option; - /// Prove the full verification data for a given key and key ID. - fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>>; - /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - fn generate_for(items: I) -> Result - where - I: IntoIterator; + fn query(&self, key: Key) -> Option; + /// Create the full verification data needed to prove a key and its value in the trie. Returns + /// `None` if the nodes nodes within the current `MemoryDB` are insufficient to create a proof. + fn create_proof(&self, key: Key) -> Option>>; + /// Create a new instance of `ProvingTrie` from raw nodes. Nodes can be generated using the + /// `create_proof` function. + fn from_nodes(root: Hash, nodes: &[Vec]) -> Self; } -/// A trie instance for checking and generating proofs. -pub struct BasicProvingTrie +/// A basic trie implementation for checking and generating proofs for a key / value pair. +pub struct BasicProvingTrie where Hashing: sp_core::Hasher, { db: MemoryDB, root: Hash, - _phantom: core::marker::PhantomData, + _phantom: core::marker::PhantomData<(Key, Value)>, } -impl ProvingTrie for BasicProvingTrie +impl ProvingTrie + for BasicProvingTrie where Hashing: sp_core::Hasher, - Hash: Default, - Item: Encode + Decode, + Hash: Default + Send + Sync, + Key: Encode, + Value: Encode + Decode, { - /// Access the underlying trie root. + fn generate_for(items: I) -> Result + where + I: IntoIterator, + { + let mut db = MemoryDB::default(); + let mut root = Default::default(); + + { + let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); + for (key, value) in items.into_iter() { + key.using_encoded(|k| value.using_encoded(|v| trie.insert(k, v))) + .map_err(|_| "failed to insert into trie")?; + } + } + + Ok(Self { db, root, _phantom: Default::default() }) + } + fn root(&self) -> &Hash { &self.root } - /// Check a proof contained within the current memory-db. Returns `None` if the - /// nodes within the current `MemoryDB` are insufficient to query the item. - fn query(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option { + fn query(&self, key: Key) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); - let val_idx = (key_id, key_data) - .using_encoded(|s| trie.get(s)) + key.using_encoded(|s| trie.get(s)) .ok()? - .and_then(|raw| u32::decode(&mut &*raw).ok())?; - - val_idx - .using_encoded(|s| trie.get(s)) - .ok()? - .and_then(|raw| Item::decode(&mut &*raw).ok()) + .and_then(|raw| Value::decode(&mut &*raw).ok()) } - /// Prove the full verification data for a given key and key ID. - fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { + fn create_proof(&self, key: Key) -> Option>> { let mut recorder = Recorder::>::new(); + { let trie = TrieDBBuilder::new(&self.db, &self.root).with_recorder(&mut recorder).build(); - let val_idx = (key_id, key_data).using_encoded(|s| { - trie.get(s).ok()?.and_then(|raw| u32::decode(&mut &*raw).ok()) - })?; - val_idx.using_encoded(|s| { - trie.get(s).ok()?.and_then(|raw| Item::decode(&mut &*raw).ok()) + key.using_encoded(|k| { + trie.get(k).ok()?.and_then(|raw| Value::decode(&mut &*raw).ok()) })?; } Some(recorder.drain().into_iter().map(|r| r.data).collect()) } - /// Create a new instance of a `ProvingTrie` using an iterator of items in the trie. - fn generate_for(items: I) -> Result - where - I: IntoIterator, - { - let mut db = MemoryDB::default(); - let mut root = Default::default(); + fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { + use sp_trie::HashDBT; - { - let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); - for (i, item) in items.into_iter().enumerate() { - let i = i as u32; + let mut memory_db = MemoryDB::default(); + for node in nodes { + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); + } - // insert each item into the trie - i.using_encoded(|k| item.using_encoded(|v| trie.insert(k, v))) - .map_err(|_| "failed to insert into trie")?; - } + Self { db: memory_db, root, _phantom: Default::default() } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::BlakeTwo256; + use sp_core::H256; + use sp_std::{collections::btree_map::BTreeMap, str::FromStr}; + + // A trie which simulates a trie of accounts (u32) and balances (u128). + type BalanceTrie = BasicProvingTrie; + + // The expected root hash for an empty trie. + fn empty_root() -> H256 { + H256::from_str("0x03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314") + .unwrap() + } + + #[test] + fn empty_trie_works() { + let empty_trie = BalanceTrie::generate_for(Vec::new()).unwrap(); + assert_eq!(*empty_trie.root(), empty_root()); + } + + #[test] + fn basic_end_to_end() { + // Create a map of users and their balances. + let mut map = BTreeMap::::new(); + for i in 0..10u32 { + map.insert(i, i.into()); } - Ok(Self { db, root, _phantom: Default::default() }) + // Put items into the trie. + let balance_trie = BalanceTrie::generate_for(map).unwrap(); + + // Root is changed. + let root = *balance_trie.root(); + assert!(root != empty_root()); + + // Assert valid key is queryable. + assert_eq!(balance_trie.query(6u32), Some(6u128)); + assert_eq!(balance_trie.query(9u32), Some(9u128)); + // Invalid key returns none. + assert_eq!(balance_trie.query(69u32), None); + + // Create a proof for a valid key. + let proof = balance_trie.prove(6u32).unwrap(); + // Can't create proof for invalid key. + assert_eq!(balance_trie.prove(69u32), None); + + // Create a new proving trie from the proof. + let new_balance_trie = BalanceTrie::from_nodes(root, &proof); + + // Assert valid key is queryable. + assert_eq!(new_balance_trie.query(6u32), Some(6u128)); } } From 39b0fca3553e4c49ce49548d39339a825cb1e2a8 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 17:46:31 -0400 Subject: [PATCH 08/25] fix test --- substrate/primitives/runtime/src/proving_trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 520d6379a0b2..3e72e89bc229 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -165,9 +165,9 @@ mod tests { assert_eq!(balance_trie.query(69u32), None); // Create a proof for a valid key. - let proof = balance_trie.prove(6u32).unwrap(); + let proof = balance_trie.create_proof(6u32).unwrap(); // Can't create proof for invalid key. - assert_eq!(balance_trie.prove(69u32), None); + assert_eq!(balance_trie.create_proof(69u32), None); // Create a new proving trie from the proof. let new_balance_trie = BalanceTrie::from_nodes(root, &proof); From 599f5761d552a1575e39a8ccf4abb279fa418959 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 28 Mar 2024 17:56:23 -0400 Subject: [PATCH 09/25] Revert "use trait in session historical" This reverts commit 1cfb29f69dd02076c1ba051786a36096e354e6c1. --- substrate/frame/session/src/historical/mod.rs | 47 +++++++++---------- .../frame/session/src/historical/offchain.rs | 11 ++--- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/substrate/frame/session/src/historical/mod.rs b/substrate/frame/session/src/historical/mod.rs index 3c30a3126660..b9cecea1a7f7 100644 --- a/substrate/frame/session/src/historical/mod.rs +++ b/substrate/frame/session/src/historical/mod.rs @@ -32,9 +32,8 @@ mod shared; use codec::{Decode, Encode}; use sp_runtime::{ - proving_trie::ProvingTrie, traits::{Convert, OpaqueKeys}, - DispatchError, KeyTypeId, + KeyTypeId, }; use sp_session::{MembershipProof, ValidatorCount}; use sp_staking::SessionIndex; @@ -177,7 +176,7 @@ impl> NoteHi if let Some(new_validators) = new_validators_and_id { let count = new_validators.len() as ValidatorCount; - match ValidatorProvingTrie::::generate_for(new_validators) { + match ProvingTrie::::generate_for(new_validators) { Ok(trie) => >::insert(new_index, &(trie.root, count)), Err(reason) => { print("Failed to generate historical ancestry-inclusion proof."); @@ -222,15 +221,13 @@ pub type IdentificationTuple = (::ValidatorId, ::FullIdentification); /// A trie instance for checking and generating proofs. -pub struct ValidatorProvingTrie { +pub struct ProvingTrie { db: MemoryDB, root: T::Hash, } -impl ProvingTrie> - for ValidatorProvingTrie -{ - fn generate_for(validators: I) -> Result +impl ProvingTrie { + fn generate_for(validators: I) -> Result where I: IntoIterator, { @@ -264,11 +261,22 @@ impl ProvingTrie> } } - Ok(Self { db, root }) + Ok(ProvingTrie { db, root }) + } + + fn from_nodes(root: T::Hash, nodes: &[Vec]) -> Self { + use sp_trie::HashDBT; + + let mut memory_db = MemoryDB::default(); + for node in nodes { + HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); + } + + ProvingTrie { db: memory_db, root } } /// Prove the full verification data for a given key and key ID. - fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { + pub fn prove(&self, key_id: KeyTypeId, key_data: &[u8]) -> Option>> { let mut recorder = Recorder::>::new(); { let trie = @@ -288,7 +296,7 @@ impl ProvingTrie> } /// Access the underlying trie root. - fn root(&self) -> &T::Hash { + pub fn root(&self) -> &T::Hash { &self.root } @@ -308,19 +316,6 @@ impl ProvingTrie> } } -impl ValidatorProvingTrie { - fn from_nodes(root: T::Hash, nodes: &[Vec]) -> Self { - use sp_trie::HashDBT; - - let mut memory_db = MemoryDB::default(); - for node in nodes { - HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); - } - - Self { db: memory_db, root } - } -} - impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet { type Proof = MembershipProof; type IdentificationTuple = IdentificationTuple; @@ -337,7 +332,7 @@ impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet::generate_for(validators).ok()?; + let trie = ProvingTrie::::generate_for(validators).ok()?; let (id, data) = key; trie.prove(id, data.as_ref()).map(|trie_nodes| MembershipProof { @@ -369,7 +364,7 @@ impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet::from_nodes(root, &proof.trie_nodes); + let trie = ProvingTrie::::from_nodes(root, &proof.trie_nodes); trie.query(id, data.as_ref()) } } diff --git a/substrate/frame/session/src/historical/offchain.rs b/substrate/frame/session/src/historical/offchain.rs index 5a5a1114b9bf..95f4d762949e 100644 --- a/substrate/frame/session/src/historical/offchain.rs +++ b/substrate/frame/session/src/historical/offchain.rs @@ -20,18 +20,17 @@ //! Validator-set extracting an iterator from an off-chain worker stored list containing historical //! validator-sets. Based on the logic of historical slashing, but the validation is done off-chain. //! Use [`fn store_current_session_validator_set_to_offchain()`](super::onchain) to store the -//! required data to the offchain validator set. This is used in conjunction with -//! [`ValidatorProvingTrie`] and the off-chain indexing API. +//! required data to the offchain validator set. This is used in conjunction with [`ProvingTrie`] +//! and the off-chain indexing API. use sp_runtime::{ offchain::storage::{MutateStorageError, StorageRetrievalError, StorageValueRef}, - proving_trie::ProvingTrie, KeyTypeId, }; use sp_session::MembershipProof; use sp_std::prelude::*; -use super::{shared, Config, IdentificationTuple, ValidatorProvingTrie}; +use super::{shared, Config, IdentificationTuple, ProvingTrie}; use crate::{Pallet as SessionModule, SessionIndex}; /// A set of validators, which was used for a fixed session index. @@ -60,7 +59,7 @@ impl ValidatorSet { } /// Implement conversion into iterator for usage -/// with [ValidatorProvingTrie](super::ValidatorProvingTrie::generate_for). +/// with [ProvingTrie](super::ProvingTrie::generate_for). impl sp_std::iter::IntoIterator for ValidatorSet { type Item = (T::ValidatorId, T::FullIdentification); type IntoIter = sp_std::vec::IntoIter; @@ -80,7 +79,7 @@ pub fn prove_session_membership>( ) -> Option { let validators = ValidatorSet::::load_from_offchain_db(session_index)?; let count = validators.len() as u32; - let trie = ValidatorProvingTrie::::generate_for(validators.into_iter()).ok()?; + let trie = ProvingTrie::::generate_for(validators.into_iter()).ok()?; let (id, data) = session_key; trie.prove(id, data.as_ref()).map(|trie_nodes| MembershipProof { From ead7951b3b701cded49789839157bee6619e897d Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 4 Apr 2024 13:28:36 -0400 Subject: [PATCH 10/25] Update substrate/primitives/runtime/src/proving_trie.rs Co-authored-by: Ankan <10196091+Ank4n@users.noreply.github.com> --- substrate/primitives/runtime/src/proving_trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 3e72e89bc229..ba126db28809 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -40,7 +40,7 @@ where /// nodes within the current `MemoryDB` are insufficient to query the item. fn query(&self, key: Key) -> Option; /// Create the full verification data needed to prove a key and its value in the trie. Returns - /// `None` if the nodes nodes within the current `MemoryDB` are insufficient to create a proof. + /// `None` if the nodes within the current `MemoryDB` are insufficient to create a proof. fn create_proof(&self, key: Key) -> Option>>; /// Create a new instance of `ProvingTrie` from raw nodes. Nodes can be generated using the /// `create_proof` function. From acfe734f8e149db8124e87cc09b181d6c3132093 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 14:28:47 -0400 Subject: [PATCH 11/25] fix some feedback --- substrate/primitives/runtime/Cargo.toml | 1 + .../primitives/runtime/src/proving_trie.rs | 44 +++++-------------- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index 2476d027f50c..800bf4bd0737 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -32,6 +32,7 @@ sp-arithmetic = { workspace = true } sp-core = { workspace = true } sp-io = { workspace = true } sp-std = { workspace = true } +sp-trie = { workspace = true } sp-weights = { workspace = true } docify = { workspace = true } tracing = { workspace = true, features = ["log"], default-features = false } diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index ba126db28809..0110cd2051e9 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -22,46 +22,24 @@ use crate::{Decode, DispatchError, Encode}; use sp_std::vec::Vec; use sp_trie::{ trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, - LayoutV0, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, + LayoutV1, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -/// A trait for creating a merkle trie for checking and generating merkle proofs. -pub trait ProvingTrie -where - Self: Sized, -{ - /// Create a new instance of a `ProvingTrie` using an iterator of key/value pairs. - fn generate_for(items: I) -> Result - where - I: IntoIterator; - /// Access the underlying trie root. - fn root(&self) -> &Hash; - /// Check a proof contained within the current memory-db. Returns `None` if the - /// nodes within the current `MemoryDB` are insufficient to query the item. - fn query(&self, key: Key) -> Option; - /// Create the full verification data needed to prove a key and its value in the trie. Returns - /// `None` if the nodes within the current `MemoryDB` are insufficient to create a proof. - fn create_proof(&self, key: Key) -> Option>>; - /// Create a new instance of `ProvingTrie` from raw nodes. Nodes can be generated using the - /// `create_proof` function. - fn from_nodes(root: Hash, nodes: &[Vec]) -> Self; -} +type HashOf = ::Out; /// A basic trie implementation for checking and generating proofs for a key / value pair. -pub struct BasicProvingTrie +pub struct BasicProvingTrie where - Hashing: sp_core::Hasher, + Hashing: sp_core::Hasher, { db: MemoryDB, - root: Hash, + root: HashOf, _phantom: core::marker::PhantomData<(Key, Value)>, } -impl ProvingTrie - for BasicProvingTrie +impl BasicProvingTrie where - Hashing: sp_core::Hasher, - Hash: Default + Send + Sync, + Hashing: sp_core::Hasher, Key: Encode, Value: Encode + Decode, { @@ -83,7 +61,7 @@ where Ok(Self { db, root, _phantom: Default::default() }) } - fn root(&self) -> &Hash { + fn root(&self) -> &HashOf { &self.root } @@ -95,7 +73,7 @@ where } fn create_proof(&self, key: Key) -> Option>> { - let mut recorder = Recorder::>::new(); + let mut recorder = Recorder::>::new(); { let trie = @@ -109,7 +87,7 @@ where Some(recorder.drain().into_iter().map(|r| r.data).collect()) } - fn from_nodes(root: Hash, nodes: &[Vec]) -> Self { + fn from_nodes(root: HashOf, nodes: &[Vec]) -> Self { use sp_trie::HashDBT; let mut memory_db = MemoryDB::default(); @@ -129,7 +107,7 @@ mod tests { use sp_std::{collections::btree_map::BTreeMap, str::FromStr}; // A trie which simulates a trie of accounts (u32) and balances (u128). - type BalanceTrie = BasicProvingTrie; + type BalanceTrie = BasicProvingTrie; // The expected root hash for an empty trie. fn empty_root() -> H256 { From 45f42877d91a56e672180725666de66607420ae3 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 14:30:55 -0400 Subject: [PATCH 12/25] update name --- substrate/primitives/runtime/src/proving_trie.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 0110cd2051e9..09447a49a091 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -72,7 +72,7 @@ where .and_then(|raw| Value::decode(&mut &*raw).ok()) } - fn create_proof(&self, key: Key) -> Option>> { + fn create_single_value_proof(&self, key: Key) -> Option>> { let mut recorder = Recorder::>::new(); { @@ -143,9 +143,9 @@ mod tests { assert_eq!(balance_trie.query(69u32), None); // Create a proof for a valid key. - let proof = balance_trie.create_proof(6u32).unwrap(); + let proof = balance_trie.create_single_value_proof(6u32).unwrap(); // Can't create proof for invalid key. - assert_eq!(balance_trie.create_proof(69u32), None); + assert_eq!(balance_trie.create_single_value_proof(69u32), None); // Create a new proving trie from the proof. let new_balance_trie = BalanceTrie::from_nodes(root, &proof); From efbe1c93e9031de5844fd240996f3b853486a0d4 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 14:58:26 -0400 Subject: [PATCH 13/25] docs and multi value proof --- .../primitives/runtime/src/proving_trie.rs | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 09447a49a091..4879b663b9cd 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -43,7 +43,8 @@ where Key: Encode, Value: Encode + Decode, { - fn generate_for(items: I) -> Result + /// Create a new instance of a `ProvingTrie` using an iterator of key/value pairs. + pub fn generate_for(items: I) -> Result where I: IntoIterator, { @@ -61,18 +62,46 @@ where Ok(Self { db, root, _phantom: Default::default() }) } - fn root(&self) -> &HashOf { + /// Access the underlying trie root. + pub fn root(&self) -> &HashOf { &self.root } - fn query(&self, key: Key) -> Option { + /// Check a proof contained within the current memory-db. Returns `None` if the + /// nodes within the current `MemoryDB` are insufficient to query the item. + pub fn query(&self, key: Key) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); key.using_encoded(|s| trie.get(s)) .ok()? .and_then(|raw| Value::decode(&mut &*raw).ok()) } - fn create_single_value_proof(&self, key: Key) -> Option>> { + /// Create the full verification data needed to prove all `keys` and their values in the trie. + /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a + /// proof. + pub fn create_proof(&self, keys: Vec) -> Option>> { + let mut recorder = Recorder::>::new(); + + { + let trie = + TrieDBBuilder::new(&self.db, &self.root).with_recorder(&mut recorder).build(); + + keys.iter() + .map(|key| { + key.using_encoded(|k| { + trie.get(k).ok()?.and_then(|raw| Value::decode(&mut &*raw).ok()) + }) + }) + .collect::>>()?; + } + + Some(recorder.drain().into_iter().map(|r| r.data).collect()) + } + + /// Create the full verification data needed to prove a single key and its value in the trie. + /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a + /// proof. + pub fn create_single_value_proof(&self, key: Key) -> Option>> { let mut recorder = Recorder::>::new(); { @@ -87,7 +116,9 @@ where Some(recorder.drain().into_iter().map(|r| r.data).collect()) } - fn from_nodes(root: HashOf, nodes: &[Vec]) -> Self { + /// Create a new instance of `ProvingTrie` from raw nodes. Nodes can be generated using the + /// `create_proof` function. + pub fn from_nodes(root: HashOf, nodes: &[Vec]) -> Self { use sp_trie::HashDBT; let mut memory_db = MemoryDB::default(); From cfa62ce6e8b34438d8544ebd17fd693dcc087432 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 15:09:47 -0400 Subject: [PATCH 14/25] improve test --- .../primitives/runtime/src/proving_trie.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 4879b663b9cd..5d11e3731955 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -67,7 +67,7 @@ where &self.root } - /// Check a proof contained within the current memory-db. Returns `None` if the + /// Check a proof contained within the current `MemoryDB`. Returns `None` if the /// nodes within the current `MemoryDB` are insufficient to query the item. pub fn query(&self, key: Key) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); @@ -153,10 +153,10 @@ mod tests { } #[test] - fn basic_end_to_end() { + fn basic_end_to_end_single_value() { // Create a map of users and their balances. let mut map = BTreeMap::::new(); - for i in 0..10u32 { + for i in 0..100u32 { map.insert(i, i.into()); } @@ -167,21 +167,28 @@ mod tests { let root = *balance_trie.root(); assert!(root != empty_root()); - // Assert valid key is queryable. + // Assert valid keys are queryable. assert_eq!(balance_trie.query(6u32), Some(6u128)); assert_eq!(balance_trie.query(9u32), Some(9u128)); + assert_eq!(balance_trie.query(69u32), Some(69u128)); // Invalid key returns none. - assert_eq!(balance_trie.query(69u32), None); + assert_eq!(balance_trie.query(6969u32), None); // Create a proof for a valid key. let proof = balance_trie.create_single_value_proof(6u32).unwrap(); // Can't create proof for invalid key. - assert_eq!(balance_trie.create_single_value_proof(69u32), None); + assert_eq!(balance_trie.create_single_value_proof(6969u32), None); // Create a new proving trie from the proof. let new_balance_trie = BalanceTrie::from_nodes(root, &proof); // Assert valid key is queryable. assert_eq!(new_balance_trie.query(6u32), Some(6u128)); + // A "neighbor" key is queryable, by happenstance. + assert_eq!(new_balance_trie.query(9u32), Some(9u128)); + // A "non-neighbor" key is not queryable. + assert_eq!(new_balance_trie.query(69u32), None); + // An invalid key is not queryable. + assert_eq!(new_balance_trie.query(6969u32), None); } } From 885fd942a1ea156eaaeb85771506ea33eb775d26 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 15:14:36 -0400 Subject: [PATCH 15/25] add multi-value query test --- .../primitives/runtime/src/proving_trie.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 5d11e3731955..3050394aabc8 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -191,4 +191,45 @@ mod tests { // An invalid key is not queryable. assert_eq!(new_balance_trie.query(6969u32), None); } + + #[test] + fn basic_end_to_end_multi_value() { + // Create a map of users and their balances. + let mut map = BTreeMap::::new(); + for i in 0..100u32 { + map.insert(i, i.into()); + } + + // Put items into the trie. + let balance_trie = BalanceTrie::generate_for(map).unwrap(); + + // Root is changed. + let root = *balance_trie.root(); + assert!(root != empty_root()); + + // Assert valid keys are queryable. + assert_eq!(balance_trie.query(6u32), Some(6u128)); + assert_eq!(balance_trie.query(9u32), Some(9u128)); + assert_eq!(balance_trie.query(69u32), Some(69u128)); + // Invalid key returns none. + assert_eq!(balance_trie.query(6969u32), None); + + // Create a proof for a valid key. + let proof = balance_trie.create_proof(vec![6u32, 69u32]).unwrap(); + // Can't create proof for invalid key. + assert_eq!(balance_trie.create_proof(vec![6u32, 69u32, 6969u32]), None); + + // Create a new proving trie from the proof. + let new_balance_trie = BalanceTrie::from_nodes(root, &proof); + + // Assert valid keys are queryable. + assert_eq!(new_balance_trie.query(6u32), Some(6u128)); + assert_eq!(new_balance_trie.query(69u32), Some(69u128)); + // A "neighbor" key is queryable, by happenstance. + assert_eq!(new_balance_trie.query(9u32), Some(9u128)); + // A "non-neighbor" key is not queryable. + assert_eq!(new_balance_trie.query(20u32), None); + // An invalid key is not queryable. + assert_eq!(new_balance_trie.query(6969u32), None); + } } From 5e3e51897a2b88e0b85197c70dcd8979079669ae Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 15 Aug 2024 15:19:32 -0400 Subject: [PATCH 16/25] Create pr_3881.prdoc --- prdoc/pr_3881.prdoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 prdoc/pr_3881.prdoc diff --git a/prdoc/pr_3881.prdoc b/prdoc/pr_3881.prdoc new file mode 100644 index 000000000000..57e11d91c885 --- /dev/null +++ b/prdoc/pr_3881.prdoc @@ -0,0 +1,15 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Introduce a Generic Proving Trie + +doc: + - audience: Runtime Dev + description: | + This PR introduces a Proving Trie object which can be used inside the runtime. This can allow + for things like airdrops where a single hash is stored on chain representing the whole airdrop + and individuals present a proof of their inclusion in the airdrop. + +crates: + - name: sp-runtime + bump: minor From e6c375906b51965d57563fcc29410ff1f3572c41 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Sat, 17 Aug 2024 05:34:45 +0200 Subject: [PATCH 17/25] use v1 --- substrate/primitives/runtime/src/proving_trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 3050394aabc8..1fe29e9b3882 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -21,7 +21,7 @@ use crate::{Decode, DispatchError, Encode}; use sp_std::vec::Vec; use sp_trie::{ - trie_types::{TrieDBBuilder, TrieDBMutBuilderV0}, + trie_types::{TrieDBBuilder, TrieDBMutBuilderV1}, LayoutV1, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; @@ -52,7 +52,7 @@ where let mut root = Default::default(); { - let mut trie = TrieDBMutBuilderV0::new(&mut db, &mut root).build(); + let mut trie = TrieDBMutBuilderV1::new(&mut db, &mut root).build(); for (key, value) in items.into_iter() { key.using_encoded(|k| value.using_encoded(|v| trie.insert(k, v))) .map_err(|_| "failed to insert into trie")?; From ffbd6563efe73de6f4032696ae5c38af3a67864a Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 12:27:48 -0400 Subject: [PATCH 18/25] use compact trie apis --- substrate/primitives/runtime/src/lib.rs | 15 + .../primitives/runtime/src/proving_trie.rs | 304 ++++++++++++------ 2 files changed, 219 insertions(+), 100 deletions(-) diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index e36c3ae70bba..ba1ea3769724 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -104,6 +104,8 @@ pub use crate::runtime_string::*; // Re-export Multiaddress pub use multiaddress::MultiAddress; +use proving_trie::TrieError; + /// Re-export these since they're only "kind of" generic. pub use generic::{Digest, DigestItem}; @@ -593,6 +595,8 @@ pub enum DispatchError { Unavailable, /// Root origin is not allowed. RootNotAllowed, + /// An error with tries. + Trie(TrieError), } /// Result of a `Dispatchable` which contains the `DispatchResult` and additional information about @@ -698,6 +702,12 @@ impl From for DispatchError { } } +impl From for DispatchError { + fn from(e: TrieError) -> DispatchError { + Self::Trie(e) + } +} + impl From<&'static str> for DispatchError { fn from(err: &'static str) -> DispatchError { Self::Other(err) @@ -722,6 +732,7 @@ impl From for &'static str { Corruption => "State corrupt", Unavailable => "Resource unavailable", RootNotAllowed => "Root not allowed", + Trie(e) => e.into(), } } } @@ -769,6 +780,10 @@ impl traits::Printable for DispatchError { Corruption => "State corrupt".print(), Unavailable => "Resource unavailable".print(), RootNotAllowed => "Root not allowed".print(), + Trie(e) => { + "Trie error: ".print(); + <&'static str>::from(*e).print(); + }, } } } diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 1fe29e9b3882..f3df7236b21f 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -15,18 +15,109 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Types for a simple merkle trie used for checking and generating proofs. +//! Types for a compact base-16 merkle trie used for checking and generating proofs within the +//! runtime. -use crate::{Decode, DispatchError, Encode}; +use crate::{Decode, DispatchError, Encode, MaxEncodedLen, TypeInfo}; use sp_std::vec::Vec; use sp_trie::{ - trie_types::{TrieDBBuilder, TrieDBMutBuilderV1}, - LayoutV1, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, + trie_types::{TrieDBBuilder, TrieDBMutBuilderV1, TrieError as SpTrieError}, + LayoutV1, MemoryDB, Trie, TrieMut, VerifyError, }; +#[cfg(feature = "serde")] +use crate::{Deserialize, Serialize}; + type HashOf = ::Out; +/// A runtime friendly error type for tries. +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, MaxEncodedLen)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub enum TrieError { + /* From TrieError */ + /// Attempted to create a trie with a state root not in the DB. + InvalidStateRoot, + /// Trie item not found in the database, + IncompleteDatabase, + /// A value was found in the trie with a nibble key that was not byte-aligned. + ValueAtIncompleteKey, + /// Corrupt Trie item. + DecoderError, + /// Hash is not value. + InvalidHash, + /* From VerifyError */ + /// The statement being verified contains multiple key-value pairs with the same key. + DuplicateKey, + /// The proof contains at least one extraneous node. + ExtraneousNode, + /// The proof contains at least one extraneous value which should have been omitted from the + /// proof. + ExtraneousValue, + /// The proof contains at least one extraneous hash reference the should have been omitted. + ExtraneousHashReference, + /// The proof contains an invalid child reference that exceeds the hash length. + InvalidChildReference, + /// The proof indicates that an expected value was not found in the trie. + ValueMismatch, + /// The proof is missing trie nodes required to verify. + IncompleteProof, + /// The root hash computed from the proof is incorrect. + RootMismatch, + /// One of the proof nodes could not be decoded. + DecodeError, +} + +impl From> for TrieError { + fn from(error: SpTrieError) -> Self { + match error { + SpTrieError::InvalidStateRoot(..) => Self::InvalidStateRoot, + SpTrieError::IncompleteDatabase(..) => Self::IncompleteDatabase, + SpTrieError::ValueAtIncompleteKey(..) => Self::ValueAtIncompleteKey, + SpTrieError::DecoderError(..) => Self::DecoderError, + SpTrieError::InvalidHash(..) => Self::InvalidHash, + } + } +} + +impl From> for TrieError { + fn from(error: VerifyError) -> Self { + match error { + VerifyError::DuplicateKey(..) => Self::DuplicateKey, + VerifyError::ExtraneousNode => Self::ExtraneousNode, + VerifyError::ExtraneousValue(..) => Self::ExtraneousValue, + VerifyError::ExtraneousHashReference(..) => Self::ExtraneousHashReference, + VerifyError::InvalidChildReference(..) => Self::InvalidChildReference, + VerifyError::ValueMismatch(..) => Self::ValueMismatch, + VerifyError::IncompleteProof => Self::IncompleteProof, + VerifyError::RootMismatch(..) => Self::RootMismatch, + VerifyError::DecodeError(..) => Self::DecodeError, + } + } +} + +impl From for &'static str { + fn from(e: TrieError) -> &'static str { + match e { + TrieError::InvalidStateRoot => "The state root is not in the database.", + TrieError::IncompleteDatabase => "A trie item was not found in the database.", + TrieError::ValueAtIncompleteKey => + "A value was found with a key that is not byte-aligned.", + TrieError::DecoderError => "A corrupt trie item was encountered.", + TrieError::InvalidHash => "The hash does not match the expected value.", + TrieError::DuplicateKey => "The proof contains duplicate keys.", + TrieError::ExtraneousNode => "The proof contains extraneous nodes.", + TrieError::ExtraneousValue => "The proof contains extraneous values.", + TrieError::ExtraneousHashReference => "The proof contains extraneous hash references.", + TrieError::InvalidChildReference => "The proof contains an invalid child reference.", + TrieError::ValueMismatch => "The proof indicates a value mismatch.", + TrieError::IncompleteProof => "The proof is incomplete.", + TrieError::RootMismatch => "The root hash computed from the proof is incorrect.", + TrieError::DecodeError => "One of the proof nodes could not be decoded.", + } + } +} + /// A basic trie implementation for checking and generating proofs for a key / value pair. pub struct BasicProvingTrie where @@ -79,55 +170,61 @@ where /// Create the full verification data needed to prove all `keys` and their values in the trie. /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. - pub fn create_proof(&self, keys: Vec) -> Option>> { - let mut recorder = Recorder::>::new(); - - { - let trie = - TrieDBBuilder::new(&self.db, &self.root).with_recorder(&mut recorder).build(); - - keys.iter() - .map(|key| { - key.using_encoded(|k| { - trie.get(k).ok()?.and_then(|raw| Value::decode(&mut &*raw).ok()) - }) - }) - .collect::>>()?; - } - - Some(recorder.drain().into_iter().map(|r| r.data).collect()) + pub fn create_proof(&self, keys: &[Key]) -> Result>, DispatchError> { + sp_trie::generate_trie_proof::, _, _, _>( + &self.db, + self.root, + &keys.into_iter().map(|k| k.encode()).collect::>>(), + ) + .map_err(|err| TrieError::from(*err).into()) } /// Create the full verification data needed to prove a single key and its value in the trie. /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. - pub fn create_single_value_proof(&self, key: Key) -> Option>> { - let mut recorder = Recorder::>::new(); - - { - let trie = - TrieDBBuilder::new(&self.db, &self.root).with_recorder(&mut recorder).build(); - - key.using_encoded(|k| { - trie.get(k).ok()?.and_then(|raw| Value::decode(&mut &*raw).ok()) - })?; - } - - Some(recorder.drain().into_iter().map(|r| r.data).collect()) + pub fn create_single_value_proof(&self, key: Key) -> Result>, DispatchError> { + self.create_proof(&[key]) } +} - /// Create a new instance of `ProvingTrie` from raw nodes. Nodes can be generated using the - /// `create_proof` function. - pub fn from_nodes(root: HashOf, nodes: &[Vec]) -> Self { - use sp_trie::HashDBT; +/// Verify the existence or non-existence of `key` and `value` in a trie proof. +pub fn verify_single_value_proof( + root: HashOf, + proof: &[Vec], + key: Key, + maybe_value: Option, +) -> Result<(), DispatchError> +where + Hashing: sp_core::Hasher, + Key: Encode, + Value: Encode, +{ + sp_trie::verify_trie_proof::, _, _, _>( + &root, + proof, + &[(key.encode(), maybe_value.map(|value| value.encode()))], + ) + .map_err(|err| TrieError::from(err).into()) +} - let mut memory_db = MemoryDB::default(); - for node in nodes { - HashDBT::insert(&mut memory_db, EMPTY_PREFIX, &node[..]); - } +/// Verify a proof which contains multiple keys and values. +pub fn verify_proof<'a, Hashing, Key, Value>( + root: HashOf, + proof: &[Vec], + items: &[(Key, Option)], +) -> Result<(), DispatchError> +where + Hashing: sp_core::Hasher, + Key: Encode, + Value: Encode, +{ + let items_encoded = items + .into_iter() + .map(|(key, maybe_value)| (key.encode(), maybe_value.as_ref().map(|value| value.encode()))) + .collect::, Option>)>>(); - Self { db: memory_db, root, _phantom: Default::default() } - } + sp_trie::verify_trie_proof::, _, _, _>(&root, proof, &items_encoded) + .map_err(|err| TrieError::from(err).into()) } #[cfg(test)] @@ -146,14 +243,7 @@ mod tests { .unwrap() } - #[test] - fn empty_trie_works() { - let empty_trie = BalanceTrie::generate_for(Vec::new()).unwrap(); - assert_eq!(*empty_trie.root(), empty_root()); - } - - #[test] - fn basic_end_to_end_single_value() { + fn create_balance_trie() -> BalanceTrie { // Create a map of users and their balances. let mut map = BTreeMap::::new(); for i in 0..100u32 { @@ -174,62 +264,76 @@ mod tests { // Invalid key returns none. assert_eq!(balance_trie.query(6969u32), None); - // Create a proof for a valid key. - let proof = balance_trie.create_single_value_proof(6u32).unwrap(); - // Can't create proof for invalid key. - assert_eq!(balance_trie.create_single_value_proof(6969u32), None); - - // Create a new proving trie from the proof. - let new_balance_trie = BalanceTrie::from_nodes(root, &proof); - - // Assert valid key is queryable. - assert_eq!(new_balance_trie.query(6u32), Some(6u128)); - // A "neighbor" key is queryable, by happenstance. - assert_eq!(new_balance_trie.query(9u32), Some(9u128)); - // A "non-neighbor" key is not queryable. - assert_eq!(new_balance_trie.query(69u32), None); - // An invalid key is not queryable. - assert_eq!(new_balance_trie.query(6969u32), None); + balance_trie } #[test] - fn basic_end_to_end_multi_value() { - // Create a map of users and their balances. - let mut map = BTreeMap::::new(); - for i in 0..100u32 { - map.insert(i, i.into()); - } - - // Put items into the trie. - let balance_trie = BalanceTrie::generate_for(map).unwrap(); + fn empty_trie_works() { + let empty_trie = BalanceTrie::generate_for(Vec::new()).unwrap(); + assert_eq!(*empty_trie.root(), empty_root()); + } - // Root is changed. + #[test] + fn basic_end_to_end_single_value() { + let balance_trie = create_balance_trie(); let root = *balance_trie.root(); - assert!(root != empty_root()); - - // Assert valid keys are queryable. - assert_eq!(balance_trie.query(6u32), Some(6u128)); - assert_eq!(balance_trie.query(9u32), Some(9u128)); - assert_eq!(balance_trie.query(69u32), Some(69u128)); - // Invalid key returns none. - assert_eq!(balance_trie.query(6969u32), None); // Create a proof for a valid key. - let proof = balance_trie.create_proof(vec![6u32, 69u32]).unwrap(); - // Can't create proof for invalid key. - assert_eq!(balance_trie.create_proof(vec![6u32, 69u32, 6969u32]), None); + let proof = balance_trie.create_single_value_proof(6u32).unwrap(); - // Create a new proving trie from the proof. - let new_balance_trie = BalanceTrie::from_nodes(root, &proof); + // Assert key is provable, all other keys are invalid. + for i in 0..200u32 { + if i == 6 { + assert_eq!( + verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i)) + ), + Ok(()) + ); + // Wrong value is invalid. + assert_eq!( + verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i + 1)) + ), + Err(TrieError::RootMismatch.into()) + ); + } else { + assert!(verify_single_value_proof::( + root, + &proof, + i, + Some(u128::from(i)) + ) + .is_err()); + assert!(verify_single_value_proof::( + root, + &proof, + i, + None:: + ) + .is_err()); + } + } + } - // Assert valid keys are queryable. - assert_eq!(new_balance_trie.query(6u32), Some(6u128)); - assert_eq!(new_balance_trie.query(69u32), Some(69u128)); - // A "neighbor" key is queryable, by happenstance. - assert_eq!(new_balance_trie.query(9u32), Some(9u128)); - // A "non-neighbor" key is not queryable. - assert_eq!(new_balance_trie.query(20u32), None); - // An invalid key is not queryable. - assert_eq!(new_balance_trie.query(6969u32), None); + #[test] + fn basic_end_to_end_multi_value() { + let balance_trie = create_balance_trie(); + let root = *balance_trie.root(); + + // Create a proof for a valid and invalid key. + let proof = balance_trie.create_proof(&[6u32, 69u32, 6969u32]).unwrap(); + let items = [(6u32, Some(6u128)), (69u32, Some(69u128)), (6969u32, None)]; + + assert_eq!(verify_proof::(root, &proof, &items), Ok(())); } + + #[test] + fn proof_fails_with_bad_data() {} } From 0217d0fc9e99dd58e2c9e389a52a5fbec2e5c7e0 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 12:43:00 -0400 Subject: [PATCH 19/25] Update proving_trie.rs --- substrate/primitives/runtime/src/proving_trie.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index f3df7236b21f..aaf42abb6c7f 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -16,9 +16,11 @@ // limitations under the License. //! Types for a compact base-16 merkle trie used for checking and generating proofs within the -//! runtime. +//! runtime. Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible with proofs using `LayoutV0`. use crate::{Decode, DispatchError, Encode, MaxEncodedLen, TypeInfo}; +#[cfg(feature = "serde")] +use crate::{Deserialize, Serialize}; use sp_std::vec::Vec; use sp_trie::{ @@ -26,9 +28,6 @@ use sp_trie::{ LayoutV1, MemoryDB, Trie, TrieMut, VerifyError, }; -#[cfg(feature = "serde")] -use crate::{Deserialize, Serialize}; - type HashOf = ::Out; /// A runtime friendly error type for tries. From f20038aba962a3bd26f514c3fc69626d5f0e22f5 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 13:37:45 -0400 Subject: [PATCH 20/25] more docs --- .../primitives/runtime/src/proving_trie.rs | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index aaf42abb6c7f..2a78de1e7a16 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -16,7 +16,13 @@ // limitations under the License. //! Types for a compact base-16 merkle trie used for checking and generating proofs within the -//! runtime. Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible with proofs using `LayoutV0`. +//! runtime. The `sp-trie` crate exposes all of these same functionality (and more), but this +//! library is designed to work more easily with runtime native types, which simply need to +//! implement `Encode`/`Decode`. It also exposes a runtime friendly `TrieError` type which can be +//! use inside of a FRAME Pallet. +//! +//! Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible with +//! proofs using `LayoutV0`. use crate::{Decode, DispatchError, Encode, MaxEncodedLen, TypeInfo}; #[cfg(feature = "serde")] @@ -117,7 +123,9 @@ impl From for &'static str { } } -/// A basic trie implementation for checking and generating proofs for a key / value pair. +/// A helper structure for building a basic base-16 merkle trie and creating compact proofs for that +/// trie. Proofs are created with latest substrate trie format (`LayoutV1`), and are not compatible +/// with proofs using `LayoutV0`. pub struct BasicProvingTrie where Hashing: sp_core::Hasher, @@ -157,7 +165,7 @@ where &self.root } - /// Check a proof contained within the current `MemoryDB`. Returns `None` if the + /// Query a value contained within the current trie. Returns `None` if the /// nodes within the current `MemoryDB` are insufficient to query the item. pub fn query(&self, key: Key) -> Option { let trie = TrieDBBuilder::new(&self.db, &self.root).build(); @@ -166,9 +174,13 @@ where .and_then(|raw| Value::decode(&mut &*raw).ok()) } - /// Create the full verification data needed to prove all `keys` and their values in the trie. + /// Create a compact merkle proof needed to prove all `keys` and their values are in the trie. /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. + /// + /// When verifying the proof created by this function, you must include all of the keys and + /// values of the proof, else the verifier will complain that extra nodes are provided in the + /// proof that are not needed. pub fn create_proof(&self, keys: &[Key]) -> Result>, DispatchError> { sp_trie::generate_trie_proof::, _, _, _>( &self.db, @@ -178,7 +190,7 @@ where .map_err(|err| TrieError::from(*err).into()) } - /// Create the full verification data needed to prove a single key and its value in the trie. + /// Create a compact merkle proof needed to prove a single key and its value are in the trie. /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. pub fn create_single_value_proof(&self, key: Key) -> Result>, DispatchError> { @@ -186,7 +198,9 @@ where } } -/// Verify the existence or non-existence of `key` and `value` in a trie proof. +/// Verify the existence or non-existence of `key` and `value` in a given trie root and proof. +/// +/// Proofs must be created with latest substrate trie format (`LayoutV1`). pub fn verify_single_value_proof( root: HashOf, proof: &[Vec], @@ -206,7 +220,9 @@ where .map_err(|err| TrieError::from(err).into()) } -/// Verify a proof which contains multiple keys and values. +/// Verify the existence or non-existence of multiple `items` in a given trie root and proof. +/// +/// Proofs must be created with latest substrate trie format (`LayoutV1`). pub fn verify_proof<'a, Hashing, Key, Value>( root: HashOf, proof: &[Vec], @@ -238,8 +254,7 @@ mod tests { // The expected root hash for an empty trie. fn empty_root() -> H256 { - H256::from_str("0x03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314") - .unwrap() + sp_trie::empty_trie_root::>() } fn create_balance_trie() -> BalanceTrie { From f934d8ddd95bc972f4e79e5ec6a05a45b8fa8601 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 14:54:20 -0400 Subject: [PATCH 21/25] bad data test --- .../primitives/runtime/src/proving_trie.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 2a78de1e7a16..5f9f53b91ca4 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -349,5 +349,20 @@ mod tests { } #[test] - fn proof_fails_with_bad_data() {} + fn proof_fails_with_bad_data() { + let balance_trie = create_balance_trie(); + let root = *balance_trie.root(); + + // Create a proof for a valid key. + let proof = balance_trie.create_single_value_proof(6u32).unwrap(); + + // Correct data verifies successfully + assert_eq!(verify_single_value_proof::(root, &proof, 6u32, Some(6u128)), Ok(())); + + // Fail to verify proof with wrong root + assert_eq!(verify_single_value_proof::(Default::default(), &proof, 6u32, Some(6u128)), Err(TrieError::RootMismatch.into())); + + // Fail to verify proof with wrong data + assert_eq!(verify_single_value_proof::(root, &[], 6u32, Some(6u128)), Err(TrieError::IncompleteProof.into())); + } } From 266a89b023fa0c3ba51f13a2bf3ffadb76c95023 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 15:30:33 -0400 Subject: [PATCH 22/25] additional warnings about layout --- .../primitives/runtime/src/proving_trie.rs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index 5f9f53b91ca4..d28ec318190d 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -178,6 +178,9 @@ where /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. /// + /// This function makes a proof with latest substrate trie format (`LayoutV1`), and is not + /// compatible with `LayoutV0`. + /// /// When verifying the proof created by this function, you must include all of the keys and /// values of the proof, else the verifier will complain that extra nodes are provided in the /// proof that are not needed. @@ -193,6 +196,9 @@ where /// Create a compact merkle proof needed to prove a single key and its value are in the trie. /// Returns `None` if the nodes within the current `MemoryDB` are insufficient to create a /// proof. + /// + /// This function makes a proof with latest substrate trie format (`LayoutV1`), and is not + /// compatible with `LayoutV0`. pub fn create_single_value_proof(&self, key: Key) -> Result>, DispatchError> { self.create_proof(&[key]) } @@ -357,12 +363,26 @@ mod tests { let proof = balance_trie.create_single_value_proof(6u32).unwrap(); // Correct data verifies successfully - assert_eq!(verify_single_value_proof::(root, &proof, 6u32, Some(6u128)), Ok(())); + assert_eq!( + verify_single_value_proof::(root, &proof, 6u32, Some(6u128)), + Ok(()) + ); // Fail to verify proof with wrong root - assert_eq!(verify_single_value_proof::(Default::default(), &proof, 6u32, Some(6u128)), Err(TrieError::RootMismatch.into())); + assert_eq!( + verify_single_value_proof::( + Default::default(), + &proof, + 6u32, + Some(6u128) + ), + Err(TrieError::RootMismatch.into()) + ); // Fail to verify proof with wrong data - assert_eq!(verify_single_value_proof::(root, &[], 6u32, Some(6u128)), Err(TrieError::IncompleteProof.into())); + assert_eq!( + verify_single_value_proof::(root, &[], 6u32, Some(6u128)), + Err(TrieError::IncompleteProof.into()) + ); } } From 3034aab836d26081e22f06ff558b5c3019b4900c Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 16:09:08 -0400 Subject: [PATCH 23/25] Update prdoc/pr_3881.prdoc Co-authored-by: Oliver Tale-Yazdi --- prdoc/pr_3881.prdoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prdoc/pr_3881.prdoc b/prdoc/pr_3881.prdoc index 57e11d91c885..4cf6425e73a5 100644 --- a/prdoc/pr_3881.prdoc +++ b/prdoc/pr_3881.prdoc @@ -12,4 +12,4 @@ doc: crates: - name: sp-runtime - bump: minor + bump: major From 1ceab64f59d8aec2ab9e6968caa8453a156d94a5 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 18:31:52 -0400 Subject: [PATCH 24/25] remove lifetime --- substrate/primitives/runtime/src/proving_trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index d28ec318190d..ddd50c323332 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -229,7 +229,7 @@ where /// Verify the existence or non-existence of multiple `items` in a given trie root and proof. /// /// Proofs must be created with latest substrate trie format (`LayoutV1`). -pub fn verify_proof<'a, Hashing, Key, Value>( +pub fn verify_proof( root: HashOf, proof: &[Vec], items: &[(Key, Option)], From c426a65094d1e14f978e3c5263037f7a7718f2c4 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 3 Sep 2024 18:43:11 -0400 Subject: [PATCH 25/25] remove unused --- substrate/primitives/runtime/src/proving_trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/primitives/runtime/src/proving_trie.rs b/substrate/primitives/runtime/src/proving_trie.rs index ddd50c323332..688bf81e0d7b 100644 --- a/substrate/primitives/runtime/src/proving_trie.rs +++ b/substrate/primitives/runtime/src/proving_trie.rs @@ -253,7 +253,7 @@ mod tests { use super::*; use crate::traits::BlakeTwo256; use sp_core::H256; - use sp_std::{collections::btree_map::BTreeMap, str::FromStr}; + use sp_std::collections::btree_map::BTreeMap; // A trie which simulates a trie of accounts (u32) and balances (u128). type BalanceTrie = BasicProvingTrie;