From 3d1cf88bfbfac1367f76ec3d9d3802b3099fcf5e Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 3 Nov 2021 13:45:45 +1000 Subject: [PATCH] Add CompactSize64 and CompactSizeMessage types But don't remove read_compactsize and write_compactsize yet. --- zebra-chain/src/serialization.rs | 12 +- zebra-chain/src/serialization/compact_size.rs | 350 ++++++++++++++++++ 2 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 zebra-chain/src/serialization/compact_size.rs diff --git a/zebra-chain/src/serialization.rs b/zebra-chain/src/serialization.rs index 99aedf99657..6a0df514341 100644 --- a/zebra-chain/src/serialization.rs +++ b/zebra-chain/src/serialization.rs @@ -6,6 +6,7 @@ //! `ReadZcashExt`, extension traits for `io::Read` and `io::Write` with utility functions //! for reading and writing data (e.g., the Bitcoin variable-integer format). +mod compact_size; mod constraint; mod date_time; mod error; @@ -14,13 +15,17 @@ mod write_zcash; mod zcash_deserialize; mod zcash_serialize; -pub(crate) mod serde_helpers; - pub mod sha256d; +pub(crate) mod serde_helpers; + #[cfg(any(test, feature = "proptest-impl"))] pub mod arbitrary; +#[cfg(test)] +pub mod tests; + +pub use compact_size::{CompactSize64, CompactSizeMessage}; pub use constraint::AtLeastOne; pub use date_time::{DateTime32, Duration32}; pub use error::SerializationError; @@ -34,6 +39,3 @@ pub use zcash_serialize::{ zcash_serialize_bytes, zcash_serialize_bytes_external_count, zcash_serialize_external_count, FakeWriter, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN, }; - -#[cfg(test)] -pub mod tests; diff --git a/zebra-chain/src/serialization/compact_size.rs b/zebra-chain/src/serialization/compact_size.rs new file mode 100644 index 00000000000..9f124905a8c --- /dev/null +++ b/zebra-chain/src/serialization/compact_size.rs @@ -0,0 +1,350 @@ +//! Types and serialization for the Bitcoin `CompactSize` format. +//! +//! Zebra decodes `CompactSize` fields into two different types, +//! depending on the consensus rules that apply to that type: +//! - [`CompactSizeMessage`] for sizes that must be less than the network message limit, and +//! - [`CompactSize64`] for flags, arbitrary counts, and sizes that span multiple blocks. + +use std::convert::{TryFrom, TryInto}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::serialization::{ + SerializationError, ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize, + MAX_PROTOCOL_MESSAGE_LEN, +}; + +/// A compactSize-encoded field that is limited to [`MAX_PROTOCOL_MESSAGE_LEN`]. +/// Used for sizes or counts of objects that are sent in network messages. +/// +/// # Security +/// +/// Deserialized sizes must be validated before being used to allocate memory. +/// +/// Preallocating vectors using untrusted `CompactSize`s allows memory +/// denial of service attacks. Valid sizes must be less than +/// `MAX_PROTOCOL_MESSAGE_LEN / min_serialized_item_bytes` (or a lower limit +/// specified by the Zcash consensus rules or Bitcoin network protocol). +/// +/// As a defence-in-depth for memory preallocation attacks, +/// Zebra rejects sizes greater than the protocol message length limit. +/// (These sizes should be impossible, because each array items takes at +/// least one byte.) +/// +/// # Serialization Examples +/// +/// ``` +/// use zebra_chain::serialization::{CompactSizeMessage, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN}; +/// use std::convert::TryFrom; +/// +/// let size = CompactSizeMessage::try_from(0x12).unwrap(); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\x12"); +/// +/// let size = CompactSizeMessage::try_from(0xfd).unwrap(); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfd\xfd\x00"); +/// +/// let size = CompactSizeMessage::try_from(0xaafd).unwrap(); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfd\xfd\xaa"); +/// +/// let max_size = u64::try_from(MAX_PROTOCOL_MESSAGE_LEN).unwrap(); +/// let size = CompactSizeMessage::try_from(max_size).unwrap(); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfe\x00\x00\x20\x00"); +/// ``` +/// +/// [`CompactSizeMessage`]s greater than the maximum network message length +/// can not be constructed: +/// ``` +/// # use zebra_chain::serialization::{CompactSizeMessage, MAX_PROTOCOL_MESSAGE_LEN}; +/// # use std::convert::TryFrom; +/// # let max_size = u64::try_from(MAX_PROTOCOL_MESSAGE_LEN).unwrap(); +/// let err = CompactSizeMessage::try_from(max_size + 1).unwrap_err(); +/// +/// let err = CompactSizeMessage::try_from(0xbbaafd).unwrap_err(); +/// +/// let err = CompactSizeMessage::try_from(0x22ccbbaafd).unwrap_err(); +/// ``` +/// +/// # Deserialization Examples +/// +/// ``` +/// use zebra_chain::serialization::{CompactSizeMessage, ZcashDeserializeInto, MAX_PROTOCOL_MESSAGE_LEN}; +/// use std::{convert::TryFrom, io::Cursor}; +/// +/// assert_eq!( +/// CompactSizeMessage::try_from(0x12).unwrap(), +/// Cursor::new(b"\x12").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSizeMessage::try_from(0xfd).unwrap(), +/// Cursor::new(b"\xfd\xfd\x00").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSizeMessage::try_from(0xaafd).unwrap(), +/// Cursor::new(b"\xfd\xfd\xaa").zcash_deserialize_into().unwrap(), +/// ); +/// +/// let max_size = u64::try_from(MAX_PROTOCOL_MESSAGE_LEN).unwrap(); +/// assert_eq!( +/// CompactSizeMessage::try_from(max_size).unwrap(), +/// Cursor::new(b"\xfe\x00\x00\x20\x00").zcash_deserialize_into().unwrap(), +/// ); +/// ``` +/// +/// [`CompactSizeMessage`]s greater than the maximum network message length are invalid. +/// They return a [`SerializationError::Parse`]: +/// ``` +/// # use zebra_chain::serialization::{CompactSizeMessage, ZcashDeserialize, MAX_PROTOCOL_MESSAGE_LEN}; +/// # use std::{convert::TryFrom, io::Cursor}; +/// let max_size_plus_one = Cursor::new(b"\xfe\x01\x00\x20\x00"); +/// let err = CompactSizeMessage::zcash_deserialize(max_size_plus_one).unwrap_err(); +/// +/// let bytes = Cursor::new(b"\xfe\xfd\xaa\xbb\x00"); +/// let err = CompactSizeMessage::zcash_deserialize(bytes).unwrap_err(); +/// +/// let bytes = Cursor::new(b"\xff\xfd\xaa\xbb\xcc\x22\x00\x00\x00"); +/// let err = CompactSizeMessage::zcash_deserialize(bytes).unwrap_err(); +/// ``` +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct CompactSizeMessage( + /// The number of items in a Zcash message. + /// + /// This field is private to enforce the [`MAX_PROTOCOL_MESSAGE_LEN`] limit. + u32, +); + +/// An arbitrary compactSize-encoded field. +/// Used for flags, arbitrary counts, and sizes that span multiple blocks. +/// +/// # Security +/// +/// Deserialized sizes must be validated before being used to allocate memory. +/// +/// Most allocation sizes should use [`CompactSizeMessage`], +/// because it is limited to [`MAX_PROTOCOL_MESSAGE_LEN`]. +/// +/// # Serialization Examples +/// +/// ``` +/// use zebra_chain::serialization::{CompactSize64, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN}; +/// use std::convert::TryFrom; +/// +/// let size = CompactSize64::from(0x12); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\x12"); +/// +/// let size = CompactSize64::from(0xfd); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfd\xfd\x00"); +/// +/// let size = CompactSize64::from(0xaafd); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfd\xfd\xaa"); +/// +/// let max_size = u64::try_from(MAX_PROTOCOL_MESSAGE_LEN).unwrap(); +/// let size = CompactSize64::from(max_size); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfe\x00\x00\x20\x00"); +/// +/// let size = CompactSize64::from(max_size + 1); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfe\x01\x00\x20\x00"); +/// +/// let size = CompactSize64::from(0xbbaafd); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xfe\xfd\xaa\xbb\x00"); +/// +/// let size = CompactSize64::from(0x22ccbbaafd); +/// let buf = size.zcash_serialize_to_vec().unwrap(); +/// assert_eq!(buf, b"\xff\xfd\xaa\xbb\xcc\x22\x00\x00\x00"); +/// ``` +/// +/// # Deserialization Examples +/// +/// ``` +/// use zebra_chain::serialization::{CompactSize64, ZcashDeserializeInto, MAX_PROTOCOL_MESSAGE_LEN}; +/// use std::{convert::TryFrom, io::Cursor}; +/// +/// assert_eq!( +/// CompactSize64::from(0x12), +/// Cursor::new(b"\x12").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSize64::from(0xfd), +/// Cursor::new(b"\xfd\xfd\x00").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSize64::from(0xaafd), +/// Cursor::new(b"\xfd\xfd\xaa").zcash_deserialize_into().unwrap(), +/// ); +/// +/// let max_size = u64::try_from(MAX_PROTOCOL_MESSAGE_LEN).unwrap(); +/// assert_eq!( +/// CompactSize64::from(max_size), +/// Cursor::new(b"\xfe\x00\x00\x20\x00").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSize64::from(max_size + 1), +/// Cursor::new(b"\xfe\x01\x00\x20\x00").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSize64::from(0xbbaafd), +/// Cursor::new(b"\xfe\xfd\xaa\xbb\x00").zcash_deserialize_into().unwrap(), +/// ); +/// +/// assert_eq!( +/// CompactSize64::from(0x22ccbbaafd), +/// Cursor::new(b"\xff\xfd\xaa\xbb\xcc\x22\x00\x00\x00").zcash_deserialize_into().unwrap(), +/// ); +///``` +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct CompactSize64( + /// A numeric value. + /// + /// This field is private for consistency with [`CompactSizeMessage`]. + u64, +); + +// We don't implement conversions between CompactSizeMessage and CompactSize64, +// because we want to distinguish fields with different numeric constraints. +// +// We don't implement From for u16 or u8, +// because we want all values to go through the same code paths. +// (And we don't want to accidentally truncate using `as`.) + +impl TryFrom for CompactSizeMessage { + type Error = SerializationError; + + #[inline] + fn try_from(size: u64) -> Result { + use SerializationError::Parse; + + let size: u32 = size.try_into()?; + + // # Security + // Defence-in-depth for memory DoS via preallocation. + if size + > MAX_PROTOCOL_MESSAGE_LEN + .try_into() + .expect("MAX_PROTOCOL_MESSAGE_LEN fits in u32") + { + // This could be invalid data from peers, so we return a parse error. + Err(Parse("compactsize larger than protocol message limit"))?; + } + + Ok(CompactSizeMessage(size)) + } +} + +impl From for u64 { + #[inline] + fn from(size: CompactSizeMessage) -> Self { + size.0.into() + } +} + +impl From for u64 { + #[inline] + fn from(size: CompactSize64) -> Self { + size.0 + } +} + +impl From for CompactSize64 { + #[inline] + fn from(size: u64) -> Self { + CompactSize64(size) + } +} + +impl ZcashSerialize for CompactSizeMessage { + /// Serialize a CompactSizeMessage into the Zcash consensus-critical format. + /// + /// # Panics + /// + /// If the value exceeds `MAX_PROTOCOL_MESSAGE_LEN`. + #[inline] + fn zcash_serialize(&self, writer: W) -> Result<(), std::io::Error> { + // # Security + // Defence-in-depth for memory DoS via preallocation. + if self.0 + > MAX_PROTOCOL_MESSAGE_LEN + .try_into() + .expect("usize fits in u64") + { + // This is validated data inside Zebra, so we panic. + panic!("compactsize larger than protocol message limit"); + } + + // Use the same serialization format as CompactSize64. + let size: u64 = self.0.into(); + CompactSize64(size).zcash_serialize(writer) + } +} + +impl ZcashDeserialize for CompactSizeMessage { + #[inline] + fn zcash_deserialize(reader: R) -> Result { + // Use the same deserialization format as CompactSize64. + let size: CompactSize64 = reader.zcash_deserialize_into()?; + + size.0.try_into() + } +} + +impl ZcashSerialize for CompactSize64 { + #[inline] + fn zcash_serialize(&self, mut writer: W) -> Result<(), std::io::Error> { + let n = self.0; + match n { + 0x0000_0000..=0x0000_00fc => writer.write_u8(n as u8), + 0x0000_00fd..=0x0000_ffff => { + writer.write_u8(0xfd)?; + writer.write_u16::(n as u16) + } + 0x0001_0000..=0xffff_ffff => { + writer.write_u8(0xfe)?; + writer.write_u32::(n as u32) + } + _ => { + writer.write_u8(0xff)?; + writer.write_u64::(n) + } + } + } +} + +impl ZcashDeserialize for CompactSize64 { + #[inline] + fn zcash_deserialize(mut reader: R) -> Result { + use SerializationError::Parse; + + let flag_byte = reader.read_u8()?; + let size = match flag_byte { + n @ 0x00..=0xfc => Ok(n as u64), + 0xfd => match reader.read_u16::()? { + n @ 0x0000_00fd..=0x0000_ffff => Ok(n as u64), + _ => Err(Parse("non-canonical compactsize")), + }, + 0xfe => match reader.read_u32::()? { + n @ 0x0001_0000..=0xffff_ffff => Ok(n as u64), + _ => Err(Parse("non-canonical compactsize")), + }, + 0xff => match reader.read_u64::()? { + n @ 0x1_0000_0000..=0xffff_ffff_ffff_ffff => Ok(n), + _ => Err(Parse("non-canonical compactsize")), + }, + }?; + + Ok(CompactSize64(size)) + } +}