From ab0e651fd7f7fe69fe6c811fce056ee4e8c42e50 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 26 Oct 2022 12:36:30 +0200 Subject: [PATCH 1/4] remove size limit from trie codec --- primitives/trie/src/lib.rs | 1 - primitives/trie/src/node_codec.rs | 2 -- primitives/trie/src/node_header.rs | 5 +---- primitives/trie/src/trie_stream.rs | 3 +-- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index f40ed4d9b53a5..4d249284f732f 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -527,7 +527,6 @@ where /// Constants used into trie simplification codec. mod trie_constants { const FIRST_PREFIX: u8 = 0b_00 << 6; - pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 4b3e69adb7041..8172344e08f36 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -279,8 +279,6 @@ fn partial_from_iterator_encode>( nibble_count: usize, node_kind: NodeKind, ) -> Vec { - let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(4 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index c2c9510c5ac43..f3544be65b2e9 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -117,8 +117,6 @@ pub(crate) fn size_and_prefix_iterator( prefix: u8, prefix_mask: usize, ) -> impl Iterator { - let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let max_value = 255u8 >> prefix_mask; let l1 = sp_std::cmp::min((max_value as usize).saturating_sub(1), size); let (first_byte, mut rem) = if size == l1 { @@ -165,12 +163,11 @@ fn decode_size( return Ok(result) } result -= 1; - while result <= trie_constants::NIBBLE_SIZE_BOUND { + loop { let n = input.read_byte()? as usize; if n < 255 { return Ok(result + n + 1) } result += 255; } - Ok(trie_constants::NIBBLE_SIZE_BOUND) } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index ca798db47b552..435e6a986722e 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -54,8 +54,7 @@ fn branch_node_bit_mask(has_children: impl Iterator) -> (u8, u8) { /// Create a leaf/branch node, encoding a number of nibbles. fn fuse_nibbles_node(nibbles: &[u8], kind: NodeKind) -> impl Iterator + '_ { - let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibbles.len()); - + let size = nibbles.len(); let iter_start = match kind { NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), NodeKind::BranchNoValue => From 5ff99d0b9b9fb1bf2aacc688d327294c18aa474c Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 26 Oct 2022 15:52:44 +0200 Subject: [PATCH 2/4] test previous upper limit is not enforced anymore --- primitives/trie/src/lib.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 4d249284f732f..0f83906475d7a 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -985,4 +985,19 @@ mod tests { assert_eq!(first_storage_root, second_storage_root); } + + #[test] + fn big_key() { + let check = |keysize: usize| { + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + t.insert(&vec![0x01u8; keysize][..], &[0x01u8, 0x23]).unwrap(); + std::mem::drop(t); + let t = TrieDBBuilder::::new(&memdb, &root).build(); + assert_eq!(t.get(&vec![0x01u8; keysize][..]).unwrap(), Some(vec![0x01u8, 0x23])); + }; + check(u16::MAX as usize / 2); // old limit + check(u16::MAX as usize / 2 + 1); // value over old limit still works + } } From 3573895424ac027a21580d66e571959d1982f097 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Nov 2022 19:11:59 +0100 Subject: [PATCH 3/4] fmt --- primitives/trie/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index f5e87b52b5971..e4cf0d5b24613 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -999,7 +999,7 @@ mod tests { }; check(u16::MAX as usize / 2); // old limit check(u16::MAX as usize / 2 + 1); // value over old limit still works - } + } fn node_with_no_children_fail_decoding() { let branch = NodeCodec::::branch_node_nibbled( From d16f37dcdc4c7d7c479a5b6b858f1a62fc375b04 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Nov 2022 19:30:03 +0100 Subject: [PATCH 4/4] restore test --- primitives/trie/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index e4cf0d5b24613..d036db7b1fecd 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -1001,6 +1001,7 @@ mod tests { check(u16::MAX as usize / 2 + 1); // value over old limit still works } + #[test] fn node_with_no_children_fail_decoding() { let branch = NodeCodec::::branch_node_nibbled( b"some_partial".iter().copied(),