From 6f5fe51c5704074d5ef352adebdf0f7606ea99bf Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 15 Mar 2022 12:50:29 +0100 Subject: [PATCH 01/14] feat: c32 optimizations --- stacks-common/src/address/c32.rs | 54 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 6a331cb217..660800d73b 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -19,11 +19,18 @@ use super::Error; use sha2::Digest; use sha2::Sha256; -const C32_CHARACTERS: &str = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; +const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; -fn c32_encode(input_bytes: &[u8]) -> String { - let c32_chars: &[u8] = C32_CHARACTERS.as_bytes(); +// C32 chars as an array, indexed by their ASCII code for O(1) lookups +#[rustfmt::skip] +const C32_CHARACTERS_MAP: [u8; 91] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 10, 11, 12, 13, 14, 15, 16, + 17, 0, 18, 19, 0, 20, 21, 0, 22, 23, 24, 25, 26, 0, 27, 28, 29, 30, 31 +]; +fn c32_encode(input_bytes: &[u8]) -> String { let mut result = vec![]; let mut carry = 0; let mut carry_bits = 0; @@ -32,25 +39,25 @@ fn c32_encode(input_bytes: &[u8]) -> String { let low_bits_to_take = 5 - carry_bits; let low_bits = current_value & ((1 << low_bits_to_take) - 1); let c32_value = (low_bits << carry_bits) + carry; - result.push(c32_chars[c32_value as usize]); + result.push(C32_CHARACTERS[c32_value as usize]); carry_bits = (8 + carry_bits) - 5; carry = current_value >> (8 - carry_bits); if carry_bits >= 5 { let c32_value = carry & ((1 << 5) - 1); - result.push(c32_chars[c32_value as usize]); + result.push(C32_CHARACTERS[c32_value as usize]); carry_bits = carry_bits - 5; carry = carry >> 5; } } if carry_bits > 0 { - result.push(c32_chars[carry as usize]); + result.push(C32_CHARACTERS[carry as usize]); } // remove leading zeros from c32 encoding while let Some(v) = result.pop() { - if v != c32_chars[0] { + if v != C32_CHARACTERS[0] { result.push(v); break; } @@ -59,7 +66,7 @@ fn c32_encode(input_bytes: &[u8]) -> String { // add leading zeros from input. for current_value in input_bytes.iter() { if *current_value == 0 { - result.push(c32_chars[0]); + result.push(C32_CHARACTERS[0]); } else { break; } @@ -88,19 +95,16 @@ fn c32_decode(input_str: &str) -> Result, Error> { let mut carry: u16 = 0; let mut carry_bits = 0; // can be up to 5 - let iter_c32_digits_opts: Vec> = c32_normalize(input_str) - .chars() - .rev() - .map(|x| C32_CHARACTERS.find(x)) - .collect(); - - let iter_c32_digits: Vec = iter_c32_digits_opts + let normalized_str = c32_normalize(input_str); + let iter_c32_digits: Vec = normalized_str + .as_bytes() .iter() - .filter_map(|x| x.as_ref()) + .rev() + .filter_map(|x| C32_CHARACTERS_MAP.get(*x as usize)) .map(|ref_x| *ref_x) .collect(); - if iter_c32_digits.len() != iter_c32_digits_opts.len() { + if normalized_str.len() != iter_c32_digits.len() { // at least one char was None return Err(Error::InvalidCrockford32); } @@ -142,18 +146,8 @@ fn c32_decode(input_str: &str) -> Result, Error> { } fn double_sha256_checksum(data: &[u8]) -> Vec { - let mut sha2 = Sha256::new(); - let mut tmp = [0u8; 32]; - let mut tmp_2 = [0u8; 32]; - - sha2.update(data); - tmp.copy_from_slice(sha2.finalize().as_slice()); - - let mut sha2_2 = Sha256::new(); - sha2_2.update(&tmp); - tmp_2.copy_from_slice(sha2_2.finalize().as_slice()); - - tmp_2[0..4].to_vec() + let tmp = Sha256::digest(Sha256::digest(data)); + tmp[0..4].to_vec() } fn c32_check_encode(version: u8, data: &[u8]) -> Result { @@ -170,7 +164,7 @@ fn c32_check_encode(version: u8, data: &[u8]) -> Result { // working with ascii strings is awful. let mut c32_string = c32_encode(&encoding_data).into_bytes(); - let version_char = C32_CHARACTERS.as_bytes()[version as usize]; + let version_char = C32_CHARACTERS[version as usize]; c32_string.insert(0, version_char); Ok(String::from_utf8(c32_string).unwrap()) From 7ace1e9662b32399b4f703f3529a7314b2d94cdf Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 15 Mar 2022 15:45:08 +0100 Subject: [PATCH 02/14] chore: c32 optimizations, round 2 --- stacks-common/src/address/c32.rs | 50 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 660800d73b..01e28e17be 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -16,18 +16,41 @@ use super::Error; +use std::convert::TryFrom; use sha2::Digest; use sha2::Sha256; const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; -// C32 chars as an array, indexed by their ASCII code for O(1) lookups +/// C32 chars as an array, indexed by their ASCII code for O(1) lookups. +/// Table can be generated with: +/// ``` +/// let mut table: [isize; 128] = [-1; 128]; +/// let alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; +/// for (i, x) in alphabet.as_bytes().iter().enumerate() { +/// table[*x as usize] = i as isize; +/// } +/// let alphabet_lower = alphabet.to_lowercase(); +/// for (i, x) in alphabet_lower.as_bytes().iter().enumerate() { +/// table[*x as usize] = i as isize; +/// } +/// let specials = [('O', '0'), ('L', '1'), ('I', '1')]; +/// for pair in specials { +/// let i = alphabet.find(|a| a == pair.1).unwrap() as isize; +/// table[pair.0 as usize] = i; +/// table[pair.0.to_ascii_lowercase() as usize] = i; +/// } +/// ``` #[rustfmt::skip] -const C32_CHARACTERS_MAP: [u8; 91] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 10, 11, 12, 13, 14, 15, 16, - 17, 0, 18, 19, 0, 20, 21, 0, 22, 23, 24, 25, 26, 0, 27, 28, 29, 30, 31 +const C32_CHARACTERS_MAP: [i8; 128] = [ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, + 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, + 26, -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, 10, 11, 12, + 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, + -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1 ]; fn c32_encode(input_bytes: &[u8]) -> String { @@ -95,22 +118,21 @@ fn c32_decode(input_str: &str) -> Result, Error> { let mut carry: u16 = 0; let mut carry_bits = 0; // can be up to 5 - let normalized_str = c32_normalize(input_str); - let iter_c32_digits: Vec = normalized_str + let iter_c32_digits: Vec = input_str .as_bytes() .iter() .rev() .filter_map(|x| C32_CHARACTERS_MAP.get(*x as usize)) - .map(|ref_x| *ref_x) + .filter_map(|v| u8::try_from(*v).ok()) .collect(); - if normalized_str.len() != iter_c32_digits.len() { + if input_str.len() != iter_c32_digits.len() { // at least one char was None return Err(Error::InvalidCrockford32); } - for current_5bit in iter_c32_digits { - carry += (current_5bit as u16) << carry_bits; + for current_5bit in &iter_c32_digits { + carry += (*current_5bit as u16) << carry_bits; carry_bits += 5; if carry_bits >= 8 { @@ -133,8 +155,8 @@ fn c32_decode(input_str: &str) -> Result, Error> { } // add leading zeros from input. - for current_value in input_str.chars() { - if current_value == '0' { + for current_value in iter_c32_digits.iter().rev() { + if *current_value == 0 { result.push(0); } else { break; From 80ad9d2560a46971ff171c8afa0bf97c4d24d4d4 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 15 Mar 2022 16:07:28 +0100 Subject: [PATCH 03/14] chore: c32 optimizations, round 3 --- stacks-common/src/address/c32.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 01e28e17be..21c09d984a 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -202,8 +202,7 @@ fn c32_check_decode(check_data_unsanitized: &str) -> Result<(u8, Vec), Error return Err(Error::InvalidCrockford32); } - let check_data = c32_normalize(check_data_unsanitized); - let (version, data) = check_data.split_at(1); + let (version, data) = check_data_unsanitized.split_at(1); let data_sum_bytes = c32_decode(data)?; if data_sum_bytes.len() < 5 { From a1a681a59edcb4e8611b074589ed772b7c98cc52 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 15 Mar 2022 23:35:54 +0100 Subject: [PATCH 04/14] feat: reduce allocs in sha256 digests --- stacks-common/src/address/c32.rs | 36 ++++++++++++++------------------ stacks-common/src/util/hash.rs | 26 +++++------------------ 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 21c09d984a..a630c305ec 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -16,9 +16,9 @@ use super::Error; -use std::convert::TryFrom; use sha2::Digest; use sha2::Sha256; +use std::convert::TryFrom; const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; @@ -41,16 +41,13 @@ const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; /// table[pair.0.to_ascii_lowercase() as usize] = i; /// } /// ``` -#[rustfmt::skip] const C32_CHARACTERS_MAP: [i8; 128] = [ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, - 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, - 26, -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, 10, 11, 12, - 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, - -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, 16, 17, 1, + 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, 10, + 11, 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, + 31, -1, -1, -1, -1, -1, ]; fn c32_encode(input_bytes: &[u8]) -> String { @@ -99,16 +96,15 @@ fn c32_encode(input_bytes: &[u8]) -> String { String::from_utf8(result).unwrap() } -fn c32_normalize(input_str: &str) -> String { - let norm_str: String = input_str - .to_uppercase() - .replace("O", "0") - .replace("L", "1") - .replace("I", "1"); - norm_str +fn c32_decode(input_str: &str) -> Result, Error> { + // must be ASCII + if !input_str.is_ascii() { + return Err(Error::InvalidCrockford32); + } + c32_decode_ascii(input_str) } -fn c32_decode(input_str: &str) -> Result, Error> { +fn c32_decode_ascii(input_str: &str) -> Result, Error> { // must be ASCII if !input_str.is_ascii() { return Err(Error::InvalidCrockford32); @@ -204,14 +200,14 @@ fn c32_check_decode(check_data_unsanitized: &str) -> Result<(u8, Vec), Error let (version, data) = check_data_unsanitized.split_at(1); - let data_sum_bytes = c32_decode(data)?; + let data_sum_bytes = c32_decode_ascii(data)?; if data_sum_bytes.len() < 5 { return Err(Error::InvalidCrockford32); } let (data_bytes, expected_sum) = data_sum_bytes.split_at(data_sum_bytes.len() - 4); - let mut check_data = c32_decode(version)?; + let mut check_data = c32_decode_ascii(version)?; check_data.extend_from_slice(data_bytes); let computed_sum = double_sha256_checksum(&check_data); diff --git a/stacks-common/src/util/hash.rs b/stacks-common/src/util/hash.rs index 052edc31c3..2837292aab 100644 --- a/stacks-common/src/util/hash.rs +++ b/stacks-common/src/util/hash.rs @@ -15,6 +15,7 @@ // along with this program. If not, see . use std::char::from_digit; +use std::convert::TryInto; use std::fmt; use std::fmt::Write; use std::mem; @@ -307,21 +308,13 @@ impl MerkleHashFunc for Sha512Trunc256Sum { impl Keccak256Hash { pub fn from_data(data: &[u8]) -> Keccak256Hash { - let mut tmp = [0u8; 32]; - let mut digest = Keccak256::new(); - digest.update(data); - tmp.copy_from_slice(digest.finalize().as_slice()); - Keccak256Hash(tmp) + Keccak256Hash(Keccak256::digest(data).try_into().unwrap()) } } impl Sha256Sum { pub fn from_data(data: &[u8]) -> Sha256Sum { - let mut tmp = [0u8; 32]; - let mut sha2_1 = Sha256::new(); - sha2_1.update(data); - tmp.copy_from_slice(sha2_1.finalize().as_slice()); - Sha256Sum(tmp) + Sha256Sum(Sha256::digest(data).try_into().unwrap()) } pub fn zero() -> Sha256Sum { Sha256Sum([0u8; 32]) @@ -330,17 +323,8 @@ impl Sha256Sum { impl DoubleSha256 { pub fn from_data(data: &[u8]) -> DoubleSha256 { - let mut tmp = [0u8; 32]; - - let mut sha2 = Sha256::new(); - sha2.update(data); - tmp.copy_from_slice(sha2.finalize().as_slice()); - - let mut sha2_2 = Sha256::new(); - sha2_2.update(&tmp); - tmp.copy_from_slice(sha2_2.finalize().as_slice()); - - DoubleSha256(tmp) + let hashed = Sha256::digest(Sha256::digest(data)); + DoubleSha256(hashed.try_into().unwrap()) } /// Converts a hash to a little-endian Uint256 From 1a9ef3ca23e34d9ebc68377f8992a9c02cbba168 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 15 Mar 2022 23:48:03 +0100 Subject: [PATCH 05/14] chore: remove redundant ascii check scan --- stacks-common/src/address/c32.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index a630c305ec..cd061f82c4 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -105,11 +105,6 @@ fn c32_decode(input_str: &str) -> Result, Error> { } fn c32_decode_ascii(input_str: &str) -> Result, Error> { - // must be ASCII - if !input_str.is_ascii() { - return Err(Error::InvalidCrockford32); - } - let mut result = vec![]; let mut carry: u16 = 0; let mut carry_bits = 0; // can be up to 5 From 03107740595e5359f2381dfb97b5ee153125c44a Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 16 Mar 2022 00:03:57 +0100 Subject: [PATCH 06/14] chore: expand comment on C32_CHARACTERS_MAP table --- stacks-common/src/address/c32.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index cd061f82c4..29c8dd3142 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -23,6 +23,7 @@ use std::convert::TryFrom; const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; /// C32 chars as an array, indexed by their ASCII code for O(1) lookups. +/// Supports lookups by uppercase, lowercase, and special (i.e. `O, L, I`) chars. /// Table can be generated with: /// ``` /// let mut table: [isize; 128] = [-1; 128]; From 683e2c29b1854ca4f4175b991193cadc89830360 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 16 Mar 2022 18:27:50 +0100 Subject: [PATCH 07/14] test: preserve previous c32 code and use for regression testing --- stacks-common/src/address/c32.rs | 30 ++++ stacks-common/src/address/c32_old.rs | 233 +++++++++++++++++++++++++++ stacks-common/src/address/mod.rs | 1 + 3 files changed, 264 insertions(+) create mode 100644 stacks-common/src/address/c32_old.rs diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 29c8dd3142..ff22d2c01f 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -242,7 +242,37 @@ pub fn c32_address(version: u8, data: &[u8]) -> Result { #[cfg(test)] mod test { use super::*; + use rand::Rng; use util::hash::hex_bytes; + use super::super::c32_old::{c32_address as c32_address_old, c32_address_decode as c32_address_decode_old}; + + #[test] + fn old_c32_validation() { + for n in 0..5000 { + // random version + let random_version: u8 = rand::thread_rng().gen_range(0, 31); + + // random 20 bytes + let random_bytes = rand::thread_rng().gen::<[u8; 20]>(); + + let addr_new = c32_address(random_version, &random_bytes).unwrap(); + let addr_old = c32_address_old(random_version, &random_bytes).unwrap(); + + assert_eq!(&addr_new, &addr_old); + + let decoded_addrs = vec![ + c32_address_decode(&addr_new).unwrap(), + c32_address_decode(&addr_old).unwrap(), + c32_address_decode_old(&addr_new).unwrap(), + c32_address_decode_old(&addr_new).unwrap(), + ]; + + for decoded_addr in decoded_addrs { + assert_eq!(decoded_addr.0, random_version); + assert_eq!(decoded_addr.1, random_bytes); + } + } + } #[test] fn test_addresses() { diff --git a/stacks-common/src/address/c32_old.rs b/stacks-common/src/address/c32_old.rs new file mode 100644 index 0000000000..f6872b5132 --- /dev/null +++ b/stacks-common/src/address/c32_old.rs @@ -0,0 +1,233 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::Error; + +use sha2::Digest; +use sha2::Sha256; + +const C32_CHARACTERS: &str = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + +fn c32_encode(input_bytes: &[u8]) -> String { + let c32_chars: &[u8] = C32_CHARACTERS.as_bytes(); + + let mut result = vec![]; + let mut carry = 0; + let mut carry_bits = 0; + + for current_value in input_bytes.iter().rev() { + let low_bits_to_take = 5 - carry_bits; + let low_bits = current_value & ((1 << low_bits_to_take) - 1); + let c32_value = (low_bits << carry_bits) + carry; + result.push(c32_chars[c32_value as usize]); + carry_bits = (8 + carry_bits) - 5; + carry = current_value >> (8 - carry_bits); + + if carry_bits >= 5 { + let c32_value = carry & ((1 << 5) - 1); + result.push(c32_chars[c32_value as usize]); + carry_bits = carry_bits - 5; + carry = carry >> 5; + } + } + + if carry_bits > 0 { + result.push(c32_chars[carry as usize]); + } + + // remove leading zeros from c32 encoding + while let Some(v) = result.pop() { + if v != c32_chars[0] { + result.push(v); + break; + } + } + + // add leading zeros from input. + for current_value in input_bytes.iter() { + if *current_value == 0 { + result.push(c32_chars[0]); + } else { + break; + } + } + + let result: Vec = result.drain(..).rev().collect(); + String::from_utf8(result).unwrap() +} + +fn c32_normalize(input_str: &str) -> String { + let norm_str: String = input_str + .to_uppercase() + .replace("O", "0") + .replace("L", "1") + .replace("I", "1"); + norm_str +} + +fn c32_decode(input_str: &str) -> Result, Error> { + // must be ASCII + if !input_str.is_ascii() { + return Err(Error::InvalidCrockford32); + } + + let mut result = vec![]; + let mut carry: u16 = 0; + let mut carry_bits = 0; // can be up to 5 + + let iter_c32_digits_opts: Vec> = c32_normalize(input_str) + .chars() + .rev() + .map(|x| C32_CHARACTERS.find(x)) + .collect(); + + let iter_c32_digits: Vec = iter_c32_digits_opts + .iter() + .filter_map(|x| x.as_ref()) + .map(|ref_x| *ref_x) + .collect(); + + if iter_c32_digits.len() != iter_c32_digits_opts.len() { + // at least one char was None + return Err(Error::InvalidCrockford32); + } + + for current_5bit in iter_c32_digits { + carry += (current_5bit as u16) << carry_bits; + carry_bits += 5; + + if carry_bits >= 8 { + result.push((carry & ((1 << 8) - 1)) as u8); + carry_bits -= 8; + carry = carry >> 8; + } + } + + if carry_bits > 0 { + result.push(carry as u8); + } + + // remove leading zeros from Vec encoding + while let Some(v) = result.pop() { + if v != 0 { + result.push(v); + break; + } + } + + // add leading zeros from input. + for current_value in input_str.chars() { + if current_value == '0' { + result.push(0); + } else { + break; + } + } + + result.reverse(); + Ok(result) +} + +fn double_sha256_checksum(data: &[u8]) -> Vec { + let mut sha2 = Sha256::new(); + let mut tmp = [0u8; 32]; + let mut tmp_2 = [0u8; 32]; + + sha2.update(data); + tmp.copy_from_slice(sha2.finalize().as_slice()); + + let mut sha2_2 = Sha256::new(); + sha2_2.update(&tmp); + tmp_2.copy_from_slice(sha2_2.finalize().as_slice()); + + tmp_2[0..4].to_vec() +} + +fn c32_check_encode(version: u8, data: &[u8]) -> Result { + if version >= 32 { + return Err(Error::InvalidVersion(version)); + } + + let mut check_data = vec![version]; + check_data.extend_from_slice(data); + let checksum = double_sha256_checksum(&check_data); + + let mut encoding_data = data.to_vec(); + encoding_data.extend_from_slice(&checksum); + + // working with ascii strings is awful. + let mut c32_string = c32_encode(&encoding_data).into_bytes(); + let version_char = C32_CHARACTERS.as_bytes()[version as usize]; + c32_string.insert(0, version_char); + + Ok(String::from_utf8(c32_string).unwrap()) +} + +fn c32_check_decode(check_data_unsanitized: &str) -> Result<(u8, Vec), Error> { + // must be ASCII + if !check_data_unsanitized.is_ascii() { + return Err(Error::InvalidCrockford32); + } + + if check_data_unsanitized.len() < 2 { + return Err(Error::InvalidCrockford32); + } + + let check_data = c32_normalize(check_data_unsanitized); + let (version, data) = check_data.split_at(1); + + let data_sum_bytes = c32_decode(data)?; + if data_sum_bytes.len() < 5 { + return Err(Error::InvalidCrockford32); + } + + let (data_bytes, expected_sum) = data_sum_bytes.split_at(data_sum_bytes.len() - 4); + + let mut check_data = c32_decode(version)?; + check_data.extend_from_slice(data_bytes); + + let computed_sum = double_sha256_checksum(&check_data); + if computed_sum != expected_sum { + let computed_sum_u32 = (computed_sum[0] as u32) + | ((computed_sum[1] as u32) << 8) + | ((computed_sum[2] as u32) << 16) + | ((computed_sum[3] as u32) << 24); + + let expected_sum_u32 = (expected_sum[0] as u32) + | ((expected_sum[1] as u32) << 8) + | ((expected_sum[2] as u32) << 16) + | ((expected_sum[3] as u32) << 24); + + return Err(Error::BadChecksum(computed_sum_u32, expected_sum_u32)); + } + + let version = check_data[0]; + let data = data_bytes.to_vec(); + Ok((version, data)) +} + +pub fn c32_address_decode(c32_address_str: &str) -> Result<(u8, Vec), Error> { + if c32_address_str.len() <= 5 { + Err(Error::InvalidCrockford32) + } else { + c32_check_decode(&c32_address_str[1..]) + } +} + +pub fn c32_address(version: u8, data: &[u8]) -> Result { + let c32_string = c32_check_encode(version, data)?; + Ok(format!("S{}", c32_string)) +} \ No newline at end of file diff --git a/stacks-common/src/address/mod.rs b/stacks-common/src/address/mod.rs index 0f55c20d03..cc48826a4a 100644 --- a/stacks-common/src/address/mod.rs +++ b/stacks-common/src/address/mod.rs @@ -31,6 +31,7 @@ use std::convert::TryFrom; pub mod b58; pub mod c32; +pub mod c32_old; pub const C32_ADDRESS_VERSION_MAINNET_SINGLESIG: u8 = 22; // P pub const C32_ADDRESS_VERSION_MAINNET_MULTISIG: u8 = 20; // M From a280debf5ab2c149edd768efc45548e02f417894 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 16 Mar 2022 18:56:33 +0100 Subject: [PATCH 08/14] bench: c32 decode function benchmarks --- Cargo.toml | 4 ++++ benches/c32_bench.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 benches/c32_bench.rs diff --git a/Cargo.toml b/Cargo.toml index 7edf38afa9..e7fa9957bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,10 @@ harness = false name = "block_limits" harness = false +[[bench]] +name = "c32_bench" +harness = false + [dependencies] rand = "0.7.3" rand_chacha = "=0.2.2" diff --git a/benches/c32_bench.rs b/benches/c32_bench.rs new file mode 100644 index 0000000000..d44b26bc50 --- /dev/null +++ b/benches/c32_bench.rs @@ -0,0 +1,33 @@ +extern crate criterion; +extern crate rand; +extern crate blockstack_lib; + +use blockstack_lib::address::c32::{c32_address, c32_address_decode}; +use blockstack_lib::address::c32_old::{c32_address_decode as c32_address_decode_old}; +use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId}; +use rand::Rng; + +fn bench_c32_decoding(c: &mut Criterion) { + let mut group = c.benchmark_group("C32 Decoding"); + + let mut addrs: Vec = vec![]; + for _ in 0..5 { + // random version + let random_version: u8 = rand::thread_rng().gen_range(0, 31); + // random 20 bytes + let random_bytes = rand::thread_rng().gen::<[u8; 20]>(); + let addr = c32_address(random_version, &random_bytes).unwrap(); + addrs.push(addr); + } + + for addr in addrs.iter() { + group.bench_with_input(BenchmarkId::new("Legacy", addr), addr, + |b, i| b.iter(|| c32_address_decode_old(i))); + group.bench_with_input(BenchmarkId::new("Updated", addr), addr, + |b, i| b.iter(|| c32_address_decode(i))); + } + group.finish(); +} + +criterion_group!(benches, bench_c32_decoding); +criterion_main!(benches); From 5ccbee42c17d4bc23a20d4951432188049bc8912 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 17:40:48 +0200 Subject: [PATCH 09/14] chore: only compile c32_old in test builds --- stacks-common/src/address/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks-common/src/address/mod.rs b/stacks-common/src/address/mod.rs index 1357545462..fe4854a326 100644 --- a/stacks-common/src/address/mod.rs +++ b/stacks-common/src/address/mod.rs @@ -31,6 +31,7 @@ use std::convert::TryFrom; pub mod b58; pub mod c32; +#[cfg(test)] pub mod c32_old; pub const C32_ADDRESS_VERSION_MAINNET_SINGLESIG: u8 = 22; // P From 6355b7606662c81dbc59806da22fd82d9c52c448 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 17:43:20 +0200 Subject: [PATCH 10/14] chore: expand doc string for c32 table normalization rules --- stacks-common/src/address/c32.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 6dd791f1d7..16befa70cc 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -23,7 +23,13 @@ use std::convert::TryFrom; const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; /// C32 chars as an array, indexed by their ASCII code for O(1) lookups. -/// Supports lookups by uppercase, lowercase, and special (i.e. `O, L, I`) chars. +/// Supports lookups by uppercase and lowercase. +/// +/// The table also encodes the special characters `O, L, I`: +/// * `O` and `o` as `0` +/// * `L` and `l` as `1` +/// * `I` and `i` as `1` +/// /// Table can be generated with: /// ``` /// let mut table: [isize; 128] = [-1; 128]; From b4919704de7606b0d96a21fc419cc0e32be9af79 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 17:44:13 +0200 Subject: [PATCH 11/14] chore: docstring for c32_old module purpose --- stacks-common/src/address/c32_old.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stacks-common/src/address/c32_old.rs b/stacks-common/src/address/c32_old.rs index f6872b5132..3d448ee3ad 100644 --- a/stacks-common/src/address/c32_old.rs +++ b/stacks-common/src/address/c32_old.rs @@ -14,6 +14,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! This module (`c32_old`) is only here to test compatibility with the new `c32` +//! module. It will be removed in the next network upgrade. + use super::Error; use sha2::Digest; From 5fcdb07314bb1e4d90e8794128d8c6b41fef32b2 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 19:22:52 +0200 Subject: [PATCH 12/14] chore: use `Option` for c32 table rather than `i8` --- stacks-common/src/address/c32.rs | 161 +++++++++++++++++++++++++++---- 1 file changed, 142 insertions(+), 19 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 16befa70cc..84ae40a564 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -32,29 +32,151 @@ const C32_CHARACTERS: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; /// /// Table can be generated with: /// ``` -/// let mut table: [isize; 128] = [-1; 128]; +/// let mut table: [Option; 128] = [None; 128]; /// let alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; /// for (i, x) in alphabet.as_bytes().iter().enumerate() { -/// table[*x as usize] = i as isize; +/// table[*x as usize] = Some(i as u8); /// } /// let alphabet_lower = alphabet.to_lowercase(); /// for (i, x) in alphabet_lower.as_bytes().iter().enumerate() { -/// table[*x as usize] = i as isize; +/// table[*x as usize] = Some(i as u8); /// } /// let specials = [('O', '0'), ('L', '1'), ('I', '1')]; /// for pair in specials { /// let i = alphabet.find(|a| a == pair.1).unwrap() as isize; -/// table[pair.0 as usize] = i; -/// table[pair.0.to_ascii_lowercase() as usize] = i; +/// table[pair.0 as usize] = Some(i as u8); +/// table[pair.0.to_ascii_lowercase() as usize] = Some(i as u8); /// } /// ``` -const C32_CHARACTERS_MAP: [i8; 128] = [ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, 16, 17, 1, - 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, 10, - 11, 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, - 31, -1, -1, -1, -1, -1, +const C32_CHARACTERS_MAP: [Option; 128] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some(0), + Some(1), + Some(2), + Some(3), + Some(4), + Some(5), + Some(6), + Some(7), + Some(8), + Some(9), + None, + None, + None, + None, + None, + None, + None, + Some(10), + Some(11), + Some(12), + Some(13), + Some(14), + Some(15), + Some(16), + Some(17), + Some(1), + Some(18), + Some(19), + Some(1), + Some(20), + Some(21), + Some(0), + Some(22), + Some(23), + Some(24), + Some(25), + Some(26), + None, + Some(27), + Some(28), + Some(29), + Some(30), + Some(31), + None, + None, + None, + None, + None, + None, + Some(10), + Some(11), + Some(12), + Some(13), + Some(14), + Some(15), + Some(16), + Some(17), + Some(1), + Some(18), + Some(19), + Some(1), + Some(20), + Some(21), + Some(0), + Some(22), + Some(23), + Some(24), + Some(25), + Some(26), + None, + Some(27), + Some(28), + Some(29), + Some(30), + Some(31), + None, + None, + None, + None, + None, ]; fn c32_encode(input_bytes: &[u8]) -> String { @@ -116,13 +238,14 @@ fn c32_decode_ascii(input_str: &str) -> Result, Error> { let mut carry: u16 = 0; let mut carry_bits = 0; // can be up to 5 - let iter_c32_digits: Vec = input_str - .as_bytes() - .iter() - .rev() - .filter_map(|x| C32_CHARACTERS_MAP.get(*x as usize)) - .filter_map(|v| u8::try_from(*v).ok()) - .collect(); + let mut iter_c32_digits = Vec::::with_capacity(input_str.len()); + + for x in input_str.as_bytes().iter().rev() { + match C32_CHARACTERS_MAP[*x as usize] { + Some(x) => iter_c32_digits.push(x), + None => {} + } + } if input_str.len() != iter_c32_digits.len() { // at least one char was None From e2563fa58e8d2377407b8a8e6015405c00684d97 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 19:34:05 +0200 Subject: [PATCH 13/14] chore: cargo fmt --- benches/c32_bench.rs | 16 +++++++++------- stacks-common/src/address/c32.rs | 6 ++++-- stacks-common/src/address/c32_old.rs | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/benches/c32_bench.rs b/benches/c32_bench.rs index d44b26bc50..3fc4a93381 100644 --- a/benches/c32_bench.rs +++ b/benches/c32_bench.rs @@ -1,10 +1,10 @@ +extern crate blockstack_lib; extern crate criterion; extern crate rand; -extern crate blockstack_lib; use blockstack_lib::address::c32::{c32_address, c32_address_decode}; -use blockstack_lib::address::c32_old::{c32_address_decode as c32_address_decode_old}; -use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId}; +use blockstack_lib::address::c32_old::c32_address_decode as c32_address_decode_old; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use rand::Rng; fn bench_c32_decoding(c: &mut Criterion) { @@ -21,10 +21,12 @@ fn bench_c32_decoding(c: &mut Criterion) { } for addr in addrs.iter() { - group.bench_with_input(BenchmarkId::new("Legacy", addr), addr, - |b, i| b.iter(|| c32_address_decode_old(i))); - group.bench_with_input(BenchmarkId::new("Updated", addr), addr, - |b, i| b.iter(|| c32_address_decode(i))); + group.bench_with_input(BenchmarkId::new("Legacy", addr), addr, |b, i| { + b.iter(|| c32_address_decode_old(i)) + }); + group.bench_with_input(BenchmarkId::new("Updated", addr), addr, |b, i| { + b.iter(|| c32_address_decode(i)) + }); } group.finish(); } diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 84ae40a564..329cbbfb68 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -370,10 +370,12 @@ pub fn c32_address(version: u8, data: &[u8]) -> Result { #[cfg(test)] mod test { + use super::super::c32_old::{ + c32_address as c32_address_old, c32_address_decode as c32_address_decode_old, + }; use super::*; - use rand::Rng; use crate::util::hash::hex_bytes; - use super::super::c32_old::{c32_address as c32_address_old, c32_address_decode as c32_address_decode_old}; + use rand::Rng; #[test] fn old_c32_validation() { diff --git a/stacks-common/src/address/c32_old.rs b/stacks-common/src/address/c32_old.rs index 3d448ee3ad..d4ac6dbf77 100644 --- a/stacks-common/src/address/c32_old.rs +++ b/stacks-common/src/address/c32_old.rs @@ -233,4 +233,4 @@ pub fn c32_address_decode(c32_address_str: &str) -> Result<(u8, Vec), Error> pub fn c32_address(version: u8, data: &[u8]) -> Result { let c32_string = c32_check_encode(version, data)?; Ok(format!("S{}", c32_string)) -} \ No newline at end of file +} From 773026eaebd9581b3b96bf36456da65df5d5d7bb Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 19 Apr 2022 19:53:15 +0200 Subject: [PATCH 14/14] chore: preserve array indexer unwrap --- stacks-common/src/address/c32.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stacks-common/src/address/c32.rs b/stacks-common/src/address/c32.rs index 329cbbfb68..1978a661a1 100644 --- a/stacks-common/src/address/c32.rs +++ b/stacks-common/src/address/c32.rs @@ -241,9 +241,9 @@ fn c32_decode_ascii(input_str: &str) -> Result, Error> { let mut iter_c32_digits = Vec::::with_capacity(input_str.len()); for x in input_str.as_bytes().iter().rev() { - match C32_CHARACTERS_MAP[*x as usize] { - Some(x) => iter_c32_digits.push(x), - None => {} + match C32_CHARACTERS_MAP.get(*x as usize) { + Some(&Some(x)) => iter_c32_digits.push(x), + _ => {} } }