From 02b187f1ab851eb884e98f71700fed0e6e76a9de Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 12 Feb 2024 22:40:49 +0000 Subject: [PATCH] Drop `ahash` dependency in favor of core's `SipHasher` https://github.com/tkaitchuck/aHash/pull/196 bumped the MSRV of `ahash` in a patch release, which makes it rather difficult for us to have it as a dependency. Further, it seems that `ahash` hasn't been particularly robust in the past, notably https://github.com/tkaitchuck/aHash/issues/163 and https://github.com/tkaitchuck/aHash/issues/166. Luckily, `core` provides `SipHasher` even on no-std (sadly its SipHash-2-4 unlike the SipHash-1-3 used by the `DefaultHasher` in `std`). Thus, we drop the `ahash` dependency entirely here and simply wrap `SipHasher` for our `no-std` HashMaps. --- bench/Cargo.toml | 2 +- lightning/Cargo.toml | 14 +-- lightning/src/lib.rs | 103 +------------------ lightning/src/util/hash_tables.rs | 163 ++++++++++++++++++++++++++++++ lightning/src/util/mod.rs | 1 + 5 files changed, 168 insertions(+), 115 deletions(-) create mode 100644 lightning/src/util/hash_tables.rs diff --git a/bench/Cargo.toml b/bench/Cargo.toml index ff254e1d9cc..05354890c2a 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -9,7 +9,7 @@ name = "bench" harness = false [features] -hashbrown = ["lightning/hashbrown", "lightning/ahash"] +hashbrown = ["lightning/hashbrown"] [dependencies] lightning = { path = "../lightning", features = ["_test_utils", "criterion"] } diff --git a/lightning/Cargo.toml b/lightning/Cargo.toml index 8214193cc42..1f6c2bd2053 100644 --- a/lightning/Cargo.toml +++ b/lightning/Cargo.toml @@ -31,7 +31,7 @@ unsafe_revoked_tx_signing = [] # Override signing to not include randomness when generating signatures for test vectors. _test_vectors = [] -no-std = ["hashbrown", "ahash", "bitcoin/no-std", "core2/alloc", "libm"] +no-std = ["hashbrown", "possiblyrandom", "bitcoin/no-std", "core2/alloc", "libm"] std = ["bitcoin/std"] # Generates low-r bitcoin signatures, which saves 1 byte in 50% of the cases @@ -43,7 +43,7 @@ default = ["std", "grind_signatures"] bitcoin = { version = "0.30.2", default-features = false, features = ["secp-recovery"] } hashbrown = { version = "0.13", optional = true } -ahash = { version = "0.8", optional = true, default-features = false } +possiblyrandom = { path = "../possiblyrandom", optional = true, default-features = false } hex = { package = "hex-conservative", version = "0.1.1", default-features = false } regex = { version = "1.5.6", optional = true } backtrace = { version = "0.3", optional = true } @@ -51,16 +51,6 @@ backtrace = { version = "0.3", optional = true } core2 = { version = "0.3.0", optional = true, default-features = false } libm = { version = "0.2", optional = true, default-features = false } -# Because ahash no longer (kinda poorly) does it for us, (roughly) list out the targets that -# getrandom supports and turn on ahash's `runtime-rng` feature for them. -[target.'cfg(not(any(target_os = "unknown", target_os = "none")))'.dependencies] -ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] } - -# Not sure what target_os gets set to for sgx, so to be safe always enable runtime-rng for x86_64 -# platforms (assuming LDK isn't being used on embedded x86-64 running directly on metal). -[target.'cfg(target_arch = "x86_64")'.dependencies] -ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] } - [dev-dependencies] regex = "1.5.6" diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 9bc15832870..1adf3786b76 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -165,113 +165,12 @@ mod io_extras { } mod prelude { - #[cfg(feature = "hashbrown")] - extern crate hashbrown; - #[cfg(feature = "ahash")] - extern crate ahash; - pub use alloc::{vec, vec::Vec, string::String, collections::VecDeque, boxed::Box}; pub use alloc::borrow::ToOwned; pub use alloc::string::ToString; - // For no-std builds, we need to use hashbrown, however, by default, it doesn't randomize the - // hashing and is vulnerable to HashDoS attacks. Thus, when not fuzzing, we use its default - // ahash hashing algorithm but randomize, opting to not randomize when fuzzing to avoid false - // positive branch coverage. - - #[cfg(not(feature = "hashbrown"))] - mod std_hashtables { - pub(crate) use std::collections::{HashMap, HashSet, hash_map}; - - pub(crate) type OccupiedHashMapEntry<'a, K, V> = - std::collections::hash_map::OccupiedEntry<'a, K, V>; - pub(crate) type VacantHashMapEntry<'a, K, V> = - std::collections::hash_map::VacantEntry<'a, K, V>; - } - #[cfg(not(feature = "hashbrown"))] - pub(crate) use std_hashtables::*; - - #[cfg(feature = "hashbrown")] - pub(crate) use self::hashbrown::hash_map; - - #[cfg(all(feature = "hashbrown", fuzzing))] - mod nonrandomized_hashbrown { - pub(crate) use hashbrown::{HashMap, HashSet}; - - pub(crate) type OccupiedHashMapEntry<'a, K, V> = - hashbrown::hash_map::OccupiedEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; - pub(crate) type VacantHashMapEntry<'a, K, V> = - hashbrown::hash_map::VacantEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; - } - #[cfg(all(feature = "hashbrown", fuzzing))] - pub(crate) use nonrandomized_hashbrown::*; - - - #[cfg(all(feature = "hashbrown", not(fuzzing)))] - mod randomized_hashtables { - use super::*; - use ahash::RandomState; - - pub(crate) type HashMap = hashbrown::HashMap; - pub(crate) type HashSet = hashbrown::HashSet; - - pub(crate) type OccupiedHashMapEntry<'a, K, V> = - hashbrown::hash_map::OccupiedEntry<'a, K, V, RandomState>; - pub(crate) type VacantHashMapEntry<'a, K, V> = - hashbrown::hash_map::VacantEntry<'a, K, V, RandomState>; - - pub(crate) fn new_hash_map() -> HashMap { - HashMap::with_hasher(RandomState::new()) - } - pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { - HashMap::with_capacity_and_hasher(cap, RandomState::new()) - } - pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { - let iter = iter.into_iter(); - let min_size = iter.size_hint().0; - let mut res = HashMap::with_capacity_and_hasher(min_size, RandomState::new()); - res.extend(iter); - res - } - - pub(crate) fn new_hash_set() -> HashSet { - HashSet::with_hasher(RandomState::new()) - } - pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { - HashSet::with_capacity_and_hasher(cap, RandomState::new()) - } - pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { - let iter = iter.into_iter(); - let min_size = iter.size_hint().0; - let mut res = HashSet::with_capacity_and_hasher(min_size, RandomState::new()); - res.extend(iter); - res - } - } - - #[cfg(any(not(feature = "hashbrown"), fuzzing))] - mod randomized_hashtables { - use super::*; - - pub(crate) fn new_hash_map() -> HashMap { HashMap::new() } - pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { - HashMap::with_capacity(cap) - } - pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { - HashMap::from_iter(iter) - } - - pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } - pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { - HashSet::with_capacity(cap) - } - pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { - HashSet::from_iter(iter) - } - } - - pub(crate) use randomized_hashtables::*; + pub(crate) use crate::util::hash_tables::*; } #[cfg(all(not(ldk_bench), feature = "backtrace", feature = "std", test))] diff --git a/lightning/src/util/hash_tables.rs b/lightning/src/util/hash_tables.rs new file mode 100644 index 00000000000..6073bf2f3ee --- /dev/null +++ b/lightning/src/util/hash_tables.rs @@ -0,0 +1,163 @@ +//! Generally LDK uses `std`'s `HashMap`s, however when building for no-std, LDK uses `hashbrown`'s +//! `HashMap`s with the `std` `SipHasher` and uses `getrandom` to opportunistically randomize it, +//! if randomization is available. +//! +//! This module simply re-exports the `HashMap` used in LDK for public consumption. + +#[cfg(feature = "hashbrown")] +extern crate hashbrown; +#[cfg(feature = "possiblyrandom")] +extern crate possiblyrandom; + +// For no-std builds, we need to use hashbrown, however, by default, it doesn't randomize the +// hashing and is vulnerable to HashDoS attacks. Thus, we use the core SipHasher when not using +// std, but use `getrandom` to randomize it if its available. + +#[cfg(not(feature = "hashbrown"))] +mod std_hashtables { + pub use std::collections::HashMap; + pub use std::collections::hash_map::RandomState; + + pub(crate) use std::collections::{HashSet, hash_map}; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + std::collections::hash_map::OccupiedEntry<'a, K, V>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + std::collections::hash_map::VacantEntry<'a, K, V>; +} +#[cfg(not(feature = "hashbrown"))] +pub use std_hashtables::*; + +#[cfg(feature = "hashbrown")] +pub(crate) use self::hashbrown::hash_map; + +#[cfg(all(feature = "hashbrown", fuzzing))] +mod nonrandomized_hashbrown { + pub use hashbrown::HashMap; + pub use std::collections::hash_map::RandomState; + + pub(crate) use hashbrown::HashSet; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + hashbrown::hash_map::OccupiedEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + hashbrown::hash_map::VacantEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; +} +#[cfg(all(feature = "hashbrown", fuzzing))] +pub use nonrandomized_hashbrown::*; + +#[cfg(all(feature = "hashbrown", not(fuzzing)))] +mod randomized_hashtables { + #[cfg(feature = "std")] + mod hasher { + pub use std::collections::hash_map::RandomState; + } + #[cfg(not(feature = "std"))] + mod hasher { + #![allow(deprecated)] // hash::SipHasher was deprecated in favor of something only in std. + use core::hash::{BuildHasher, SipHasher}; + + #[derive(Clone, Copy)] + /// A simple implementation of [`BuildHasher`] that uses `getrandom` to opportunistically + /// randomize, if the platform supports it. + pub struct RandomState { + k0: u64, k1: u64, + } + + impl RandomState { + /// Constructs a new [`RandomState`] which may or may not be random, depending on the + /// target platform. + pub fn new() -> RandomState { + let (k0, k1); + #[cfg(feature = "possiblyrandom")] { + let mut keys = [0; 16]; + possiblyrandom::getpossiblyrandom(&mut keys); + + let mut k0_bytes = [0; 8]; + let mut k1_bytes = [0; 8]; + k0_bytes.copy_from_slice(&keys[..8]); + k1_bytes.copy_from_slice(&keys[8..]); + k0 = u64::from_le_bytes(k0_bytes); + k1 = u64::from_le_bytes(k1_bytes); + } + #[cfg(not(feature = "possiblyrandom"))] { + k0 = 0; + k1 = 0; + } + RandomState { k0, k1 } + } + } + + impl BuildHasher for RandomState { + type Hasher = SipHasher; + fn build_hasher(&self) -> SipHasher { + SipHasher::new_with_keys(self.k0, self.k1) + } + } + } + + pub use hasher::*; + use super::*; + + /// The HashMap type used in LDK. + pub type HashMap = hashbrown::HashMap; + pub(crate) type HashSet = hashbrown::HashSet; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + hashbrown::hash_map::OccupiedEntry<'a, K, V, RandomState>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + hashbrown::hash_map::VacantEntry<'a, K, V, RandomState>; + + pub(crate) fn new_hash_map() -> HashMap { + HashMap::with_hasher(RandomState::new()) + } + pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { + HashMap::with_capacity_and_hasher(cap, RandomState::new()) + } + pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + let mut res = HashMap::with_capacity_and_hasher(min_size, RandomState::new()); + res.extend(iter); + res + } + + pub(crate) fn new_hash_set() -> HashSet { + HashSet::with_hasher(RandomState::new()) + } + pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { + HashSet::with_capacity_and_hasher(cap, RandomState::new()) + } + pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + let mut res = HashSet::with_capacity_and_hasher(min_size, RandomState::new()); + res.extend(iter); + res + } +} +#[cfg(all(feature = "hashbrown", not(fuzzing)))] +pub use randomized_hashtables::*; + +#[cfg(any(not(feature = "hashbrown"), fuzzing))] +mod hashtable_constructors { + use super::*; + + pub(crate) fn new_hash_map() -> HashMap { HashMap::new() } + pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { + HashMap::with_capacity(cap) + } + pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { + HashMap::from_iter(iter) + } + + pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } + pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { + HashSet::with_capacity(cap) + } + pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { + HashSet::from_iter(iter) + } +} +#[cfg(any(not(feature = "hashbrown"), fuzzing))] +pub(crate) use hashtable_constructors::*; diff --git a/lightning/src/util/mod.rs b/lightning/src/util/mod.rs index 6ce00acab45..31bdf1ca53c 100644 --- a/lightning/src/util/mod.rs +++ b/lightning/src/util/mod.rs @@ -32,6 +32,7 @@ pub(crate) mod atomic_counter; pub(crate) mod byte_utils; pub(crate) mod transaction_utils; pub(crate) mod time; +pub mod hash_tables; pub mod indexed_map;