From c309c66e40d5f8f428cf8f0ceb4e395c11f9aca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Zwoli=C5=84ski?= Date: Fri, 30 Aug 2024 10:09:02 +0200 Subject: [PATCH 01/12] wip --- Cargo.lock | 6 +- Cargo.toml | 4 +- rpc/Cargo.toml | 1 + rpc/src/share.rs | 9 + rpc/tests/share.rs | 28 +++ types/Cargo.toml | 1 + types/src/data_availability_header.rs | 293 ++++++++++++++++++++++++++ 7 files changed, 336 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77053abe..be30b864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -743,6 +743,7 @@ dependencies = [ "nmt-rs", "rand", "serde", + "serde_json", "thiserror", "tokio", "tracing", @@ -752,8 +753,6 @@ dependencies = [ [[package]] name = "celestia-tendermint" version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f93b5cbbd62b6cfde961889bf05d5fe19e70d8500c4465694306ed2695ac23" dependencies = [ "bytes", "celestia-tendermint-proto", @@ -782,8 +781,6 @@ dependencies = [ [[package]] name = "celestia-tendermint-proto" version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f7d49c1ececa30a4587c5fe8a4035b786b78a3253ed0f9636de591b3dc2b37" dependencies = [ "bytes", "flex-error", @@ -813,6 +810,7 @@ dependencies = [ "ed25519-consensus", "enum_dispatch", "getrandom", + "hex", "indoc", "leopard-codec", "libp2p-identity", diff --git a/Cargo.toml b/Cargo.toml index eb49771b..d5b91a27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ celestia-tendermint-proto = "0.32.1" # Uncomment to apply local changes #beetswap = { path = "../beetswap" } #blockstore = { path = "../blockstore" } -#celestia-tendermint = { path = "../celestia-tendermint-rs/tendermint" } -#celestia-tendermint-proto = { path = "../celestia-tendermint-rs/proto" } +celestia-tendermint = { path = "../celestia-tendermint-rs/tendermint" } +celestia-tendermint-proto = { path = "../celestia-tendermint-rs/proto" } #nmt-rs = { path = "../nmt-rs" } #libp2p = { path = "../../rust-libp2p/libp2p" } #libp2p-core = { path = "../../rust-libp2p/core" } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 3ccd6a33..b6effacd 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -25,6 +25,7 @@ jsonrpsee = { version = "0.24.2", features = ["client-core", "macros"] } serde = { version = "1.0.203", features = ["derive"] } thiserror = "1.0.61" tracing = "0.1.40" +serde_json = "*" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] http = "1.1.0" diff --git a/rpc/src/share.rs b/rpc/src/share.rs index c4ad9ef3..80aa2acd 100644 --- a/rpc/src/share.rs +++ b/rpc/src/share.rs @@ -8,6 +8,15 @@ pub trait Share { #[method(name = "share.GetEDS")] async fn share_get_eds(&self, root: &ExtendedHeader) -> Result; + /// GetRange gets a list of shares and their corresponding proof. + #[method(name = "share.GetRange")] + async fn share_get_range( + &self, + height: u64, + start: usize, + end: usize, + ) -> Result; + /// GetShare gets a Share by coordinates in EDS. #[method(name = "share.GetShare")] async fn share_get_share( diff --git a/rpc/tests/share.rs b/rpc/tests/share.rs index f53b35cc..0f0af48b 100644 --- a/rpc/tests/share.rs +++ b/rpc/tests/share.rs @@ -51,6 +51,34 @@ async fn get_shares_by_namespace() { assert_eq!(&reconstructed_data[..seq_len as usize], &data[..]); } +#[tokio::test] +async fn get_shares_range() { + let client = new_test_client(AuthLevel::Write).await.unwrap(); + let namespace = random_ns(); + let data = random_bytes(1024); + let blob = Blob::new(namespace, data.clone()).unwrap(); + let commitment = blob.commitment; + + let submitted_height = blob_submit(&client, &[blob]).await.unwrap(); + + let header = client.header_get_by_height(submitted_height).await.unwrap(); + let blob_on_chain = client + .blob_get(submitted_height, namespace, commitment) + .await + .unwrap(); + let index = blob_on_chain.index.unwrap(); + let shares = blob_on_chain.to_shares().unwrap().len(); + + let shares_range = client + .share_get_range(submitted_height, index as usize, index as usize + shares) + .await + .unwrap(); + + println!("{}", serde_json::to_string_pretty(&shares_range).unwrap()); + println!("{}", serde_json::to_string_pretty(&header).unwrap()); + panic!(); +} + #[tokio::test] async fn get_shares_by_namespace_wrong_ns() { let client = new_test_client(AuthLevel::Write).await.unwrap(); diff --git a/types/Cargo.toml b/types/Cargo.toml index f887e258..7ce02840 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -27,6 +27,7 @@ cid = { version = "0.11.1", default-features = false, features = ["std"] } const_format = "0.2.32" ed25519-consensus = { version = "2.1.0", optional = true } enum_dispatch = "0.3.13" +hex = "0.4.3" leopard-codec = "0.1.0" libp2p-identity = { version = "0.2.9", optional = true } multiaddr = { version = "0.18.1", optional = true } diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index 9db592d3..5bf96021 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -1,5 +1,7 @@ use celestia_proto::celestia::da::DataAvailabilityHeader as RawDataAvailabilityHeader; use celestia_tendermint::merkle::simple_hash_from_byte_vectors; +use celestia_tendermint_proto::v0_34::crypto::Proof as MerkleProof; +use celestia_tendermint_proto::v0_34::types::RowProof as RawRowProof; use celestia_tendermint_proto::Protobuf; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -244,6 +246,123 @@ impl ValidateBasic for DataAvailabilityHeader { } } +// TODO: it should follow our regular try from / into RawRowProof pattern +// utilizing proto/vendor/tendermint/types/types.proto. However we cannot +// easily generate it, as it's in .tendermint.types package which we override +// with celestia_tendermint_proto. so the correct solution here would be to +// update celestia_tendermint_proto proto definitions to a recent celestia-core +// version. Note, this wouldn't free us from merkle proof verification logic, +// they are not present in tendermint-rs +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct RowProof { + #[serde(with = "hash_vec_hexstring")] + row_roots: Vec, + proofs: Vec, + #[serde(default, with = "hash_base64string")] + root: Option, + start_row: usize, + end_row: usize, +} + +impl RowProof { + fn verify(&self, root: Hash) -> Result<()> { + assert_eq!(self.row_roots.len(), self.proofs.len()); + + for (row_root, proof) in self.row_roots.iter().zip(self.proofs.iter()) { + todo!() + // verify_merkle_proof(proof, &root, row_root); + } + + Ok(()) + } +} + +impl Protobuf for RowProof {} + +impl TryFrom for RowProof { + type Error = Error; + + fn try_from(value: RawRowProof) -> Result { + todo!() + } +} + +impl From for RawRowProof { + fn from(value: RowProof) -> Self { + todo!() + } +} + +fn verify_merkle_proof(proof: &MerkleProof, root: &Hash, leaf: &Hash) {} + +mod hash_base64string { + use base64::prelude::*; + use serde::{de, Deserialize, Deserializer, Serializer}; + + use super::Hash; + + pub fn serialize(value: &Option, serializer: S) -> Result + where + S: Serializer, + { + if let Some(hash) = value { + serializer.serialize_some(&BASE64_STANDARD.encode(hash)) + } else { + serializer.serialize_none() + } + } + + /// Deserialize base64string into `Hash` + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let Some(hash) = Option::<&str>::deserialize(deserializer)? else { + return Ok(None); + }; + BASE64_STANDARD + .decode(hash) + .map_err(de::Error::custom)? + .try_into() + .map(Some) + .map_err(de::Error::custom) + } +} + +mod hash_vec_hexstring { + use serde::{de, Deserialize, Deserializer, Serializer}; + + use super::{NamespacedHash, NamespacedHashExt}; + + /// Serialize from `Vec` into `Vec` + pub fn serialize(value: &[NamespacedHash], serializer: S) -> Result + where + S: Serializer, + { + let hashes: Vec<_> = value + .iter() + .map(|hash| hex::encode_upper(hash.to_array())) + .collect(); + serializer.serialize_some(&hashes) + } + + /// Deserialize vec_base64string into `NamespacedHash` + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Option::>::deserialize(deserializer)? + .unwrap_or_default() + .into_iter() + .map(|raw_hash| { + hex::decode(raw_hash) + .map_err(de::Error::custom) + .and_then(|hash| NamespacedHash::from_raw(&hash).map_err(de::Error::custom)) + }) + .collect() + } +} + #[cfg(test)] mod tests { use super::*; @@ -326,4 +445,178 @@ mod tests { dah.validate_basic().unwrap_err(); } + + #[test] + fn row_proof_verify_correct() { + let raw_row_proof = r#" + { + "end_row": 1, + "proofs": [ + { + "aunts": [ + "Ch+9PsBdsN5YUt8nvAmjdOAIcVdfmPAEUNmCA8KBe5A=", + "ojjC9H5JG/7OOrt5BzBXs/3w+n1LUI/0YR0d+RSfleU=", + "d6bMQbLTBfZGvqXOW9MPqRM+fTB2/wLJx6CkLc8glCI=" + ], + "index": 0, + "leaf_hash": "nOpM3A3d0JYOmNaaI5BFAeKPGwQ90TqmM/kx+sHr79s=", + "total": 8 + }, + { + "aunts": [ + "nOpM3A3d0JYOmNaaI5BFAeKPGwQ90TqmM/kx+sHr79s=", + "ojjC9H5JG/7OOrt5BzBXs/3w+n1LUI/0YR0d+RSfleU=", + "d6bMQbLTBfZGvqXOW9MPqRM+fTB2/wLJx6CkLc8glCI=" + ], + "index": 1, + "leaf_hash": "Ch+9PsBdsN5YUt8nvAmjdOAIcVdfmPAEUNmCA8KBe5A=", + "total": 8 + } + ], + "row_roots": [ + "000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000D8CBB533A24261C4C0A3D37F1CBFB6F4C5EA031472EBA390D482637933874AA0A2B9735E67629993852D", + "00000000000000000000000000000000000000D8CBB533A24261C4C0A300000000000000000000000000000000000000D8CBB533A24261C4C0A37E409334CCB1125C793EC040741137634C148F089ACB06BFFF4C1C4CA2CBBA8E" + ], + "start_row": 0 + } + "#; + let raw_dah = r#" + { + "row_roots": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo9N/HL+29MXqAxRy66OQ1IJjeTOHSqCiuXNeZ2KZk4Ut", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo35AkzTMsRJceT7AQHQRN2NMFI8ImssGv/9MHEyiy7qO", + "/////////////////////////////////////////////////////////////////////////////7mTwL+NxdxcYBd89/wRzW2k9vRkQehZiXsuqZXHy89X", + "/////////////////////////////////////////////////////////////////////////////2X/FT2ugeYdWmvnEisSgW+9Ih8paNvrji2NYPb8ujaK" + ], + "column_roots": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo/xEv//wkWzNtkcAZiZmSGU1Te6ERwUxTtTfHzoS4bv+", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo9FOCNvCjA42xYCwHrlo48iPEXLaKt+d+JdErCIrQIi6", + "/////////////////////////////////////////////////////////////////////////////y2UErq/83uv433HekCWokxqcY4g+nMQn3tZn2Tr6v74", + "/////////////////////////////////////////////////////////////////////////////z6fKmbJTvfLYFlNuDWHn87vJb6V7n44MlCkxv1dyfT2" + ] + } + "#; + + let row_proof: RowProof = serde_json::from_str(raw_row_proof).unwrap(); + let dah: DataAvailabilityHeader = serde_json::from_str(raw_dah).unwrap(); + + row_proof.verify(dah.hash()).unwrap(); + } } +// "Proof": { +// "data": [ +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMBAAAEAAK7jTduoBTIVHIsXZYBTeXT+ROAP0ErS1wBn3qRFHoNClY8r4gEOLhvoPDfYX5dN+qGDHdIFPG4F1aF+niSmbfQRSkw2QdjqKwDKhYUKvu10oUo5r/k0SyYJx5KImSJ0d2sBH/ajcpk +// +DWBD0tXJTmsfiATmM8BqaxRRa5biE1T9yV1WKndAyJUC00P8e2/MG+7P1t7a9tMjG+Oxgxx1EzxJ47FDiRwnYNz2/JBqzIC33fKoiWZSeL+NFLn0Dfx+Ev1GYaKpstd1x1tgJnkEceTFVC6r7qhqRbTFJjAjgYJAB4fbBd/+QUQkdbW0uCHLtmWhkeK9YB +// uY05L1v1c6wcXI9IhSlBLnFFdxSTonAaZYhOusiG6eNFn7FpTU0i0oHcksQL+MW3HhbOnIyyUE1Wyjsm6pFuHKBi4TwHTQOibOhvxehuxyrHkqk7QcEPK6/ioN08n2eqd1mlfXiG2wk8nDaZfdmIq3hCm2usrpmqxJHYoH/wbbMeB7AzhreueWRk38984H2 +// h1xX93ZpmUWJEJJGJ0St70Afb6RPjH9pX9vtbVXCvj65D+HPpxinReMBUj0rvGZ6IzNzoBhJYGszp5R4sdztGH8NLNWujwAFDThNRhWUX/r+APM1hdW9s=", +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAaRdGrcBVugXTHUqmX0/UvgFUVjY/T1lnVQuz0gKhCE1aN0WvNPJautwNmv/68eAucdCm6vPqzg4KEgL777G5navyLj/1TMhLJfgTf1YNayDJLN7R13d1QQ3Peagcg+N4Itv0ZmZ6p7/QMQfw +// aXh30yronynPhxV//932ODigrZVrbW+XBhPtgh+/DlbCrU8d65IGn3VGTDQNfZaajmogy4xNf9x089OgcOv8H1XEjj0X8iZQwW+K05wIE6STWGxXJSMywMM7A+FE5YDrnEHeIT9bsIeXvEAy2cwfrorAnQrbfPyZqSSHHQzGumhOYam1Cyz1oVUMAhJMOXR +// drsckPXQdy38cOw/2VNFCZnLDvJIdT9kL3fk+BX/tXUMdvmR0ATY1JiqjW1YPO8fpVaG+IrbUobxDUnrq5kSmK6Gi9WIF+NzCWpPD/bjV+6nwJXEUxzjb18wVitZJZsQpMVsB9t0KXzlvr9AsKob4AZZGAAqbI4cHKjW3PNMbpH6U1WTUPldy7NQvpWDcYs +// VbYQOEr9YiKGyUPIUdb+nPSyAd4aIpbce8fQhN9D9wE0SIDTm2hMorjJsemwdZSHifZbL4Yya7QR/Oa3x5K+82IZiuhm/y5HGEdlHB2Jg54wqJJrKvO2E=", +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAIt3G6MztjqXhUJ6Mbw/14mlqVbzIwJNU38ITmjBXACSyjgvQCMhVZYrvWQxCuEtPHboM1HrI1rgVjMC3B7vregAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +// ], +// "namespace_id": "AAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAow==", +// "namespace_version": 0, +// "share_proofs": [ +// { +// "end": 2, +// "nodes": [ +// "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCU0aUrR/wpx09HFWeoyuV1vuw5Ew3rhtCaf/Zd4chb9", +// "/////////////////////////////////////////////////////////////////////////////ypPU4ZqDz1t8YcunXI8ETuBth1gXLvPWIMd0JPoeJF3" +// ], +// "start": 1 +// }, +// { +// "end": 2, +// "nodes": [ +// "/////////////////////////////////////////////////////////////////////////////wdXw/2tc8hhuGLcsfU9pWo5BDIKSsNJFCytj++xtFgq" +// ] +// } +// ] +// }, +// "Shares": [ +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMBAAAEAAK7jTduoBTIVHIsXZYBTeXT+ROAP0ErS1wBn3qRFHoNClY8r4gEOLhvoPDfYX5dN+qGDHdIFPG4F1aF+niSmbfQRSkw2QdjqKwDKhYUKvu10oUo5r/k0SyYJx5KImSJ0d2sBH/ajcpk+D +// WBD0tXJTmsfiATmM8BqaxRRa5biE1T9yV1WKndAyJUC00P8e2/MG+7P1t7a9tMjG+Oxgxx1EzxJ47FDiRwnYNz2/JBqzIC33fKoiWZSeL+NFLn0Dfx+Ev1GYaKpstd1x1tgJnkEceTFVC6r7qhqRbTFJjAjgYJAB4fbBd/+QUQkdbW0uCHLtmWhkeK9YBuY +// 05L1v1c6wcXI9IhSlBLnFFdxSTonAaZYhOusiG6eNFn7FpTU0i0oHcksQL+MW3HhbOnIyyUE1Wyjsm6pFuHKBi4TwHTQOibOhvxehuxyrHkqk7QcEPK6/ioN08n2eqd1mlfXiG2wk8nDaZfdmIq3hCm2usrpmqxJHYoH/wbbMeB7AzhreueWRk38984H2h1 +// xX93ZpmUWJEJJGJ0St70Afb6RPjH9pX9vtbVXCvj65D+HPpxinReMBUj0rvGZ6IzNzoBhJYGszp5R4sdztGH8NLNWujwAFDThNRhWUX/r+APM1hdW9s=", +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAaRdGrcBVugXTHUqmX0/UvgFUVjY/T1lnVQuz0gKhCE1aN0WvNPJautwNmv/68eAucdCm6vPqzg4KEgL777G5navyLj/1TMhLJfgTf1YNayDJLN7R13d1QQ3Peagcg+N4Itv0ZmZ6p7/QMQfwaX +// h30yronynPhxV//932ODigrZVrbW+XBhPtgh+/DlbCrU8d65IGn3VGTDQNfZaajmogy4xNf9x089OgcOv8H1XEjj0X8iZQwW+K05wIE6STWGxXJSMywMM7A+FE5YDrnEHeIT9bsIeXvEAy2cwfrorAnQrbfPyZqSSHHQzGumhOYam1Cyz1oVUMAhJMOXRdr +// sckPXQdy38cOw/2VNFCZnLDvJIdT9kL3fk+BX/tXUMdvmR0ATY1JiqjW1YPO8fpVaG+IrbUobxDUnrq5kSmK6Gi9WIF+NzCWpPD/bjV+6nwJXEUxzjb18wVitZJZsQpMVsB9t0KXzlvr9AsKob4AZZGAAqbI4cHKjW3PNMbpH6U1WTUPldy7NQvpWDcYsVb +// YQOEr9YiKGyUPIUdb+nPSyAd4aIpbce8fQhN9D9wE0SIDTm2hMorjJsemwdZSHifZbL4Yya7QR/Oa3x5K+82IZiuhm/y5HGEdlHB2Jg54wqJJrKvO2E=", +// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAIt3G6MztjqXhUJ6Mbw/14mlqVbzIwJNU38ITmjBXACSyjgvQCMhVZYrvWQxCuEtPHboM1HrI1rgVjMC3B7vregAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +// ] +// } +// { +// "header": { +// "version": { +// "block": "11", +// "app": "1" +// }, +// "chain_id": "private", +// "height": "7621", +// "time": "2024-08-29T13:06:07.863632032Z", +// "last_block_id": { +// "hash": "A2EDE7E806EBF59F1E2914AC2D7801B76B3A789F033911AF14C5D62F5C3E7098", +// "parts": { +// "total": 1, +// "hash": "0C45BA8F78E4D8163C992AB47D7930583D1EF207D5C497FC59CC02281F625FEB" +// } +// }, +// "last_commit_hash": "526DB4A98D3661E05DF67B87239EC153515ACCD3F46BA268EFF4015295C6A2B3", +// "data_hash": "E414A1EEB8395CA0C353384A9EC77300698ABBF98CD1DEC688A2C5BC5525077C", +// "validators_hash": "73AE7622856703136CCEC52530EAEFB7BE02441ADC3395B469D82D13C1CD731F", +// "next_validators_hash": "73AE7622856703136CCEC52530EAEFB7BE02441ADC3395B469D82D13C1CD731F", +// "consensus_hash": "C0B6A634B72AE9687EA53B6D277A73ABA1386BA3CFC6D0F26963602F7F6FFCD6", +// "app_hash": "5A6329D9A806308D54C1A28EA0D89A5B2F4AC206BA2F3E5BF8C5E9B15F65CC2F", +// "last_results_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", +// "evidence_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", +// "proposer_address": "ECCF605776310B73EC57189AE89B1E587DBC2601" +// }, +// "commit": { +// "height": 7621, +// "round": 0, +// "block_id": { +// "hash": "A7A4A45F0C655DA558722A14A589BA580F8A06BEE12376772A4CCFA30A989877", +// "parts": { +// "total": 1, +// "hash": "617CFA4E6A5670FFCAF4D06FE72D16214FBAF3B7C96CA4FBD10B47A167B066D9" +// } +// }, +// "signatures": [ +// { +// "block_id_flag": 2, +// "validator_address": "ECCF605776310B73EC57189AE89B1E587DBC2601", +// "timestamp": "2024-08-29T13:06:08.875333629Z", +// "signature": "Beqnt4uw2reTFRguBXuQXMsEoysvUVlB4eyHBKJK6knigN478JUaKWfgXnuFyIA9XRdTpkiS/3OkqWimQXFfCQ==" +// } +// ] +// }, +// "validator_set": { +// "validators": [ +// { +// "address": "ECCF605776310B73EC57189AE89B1E587DBC2601", +// "pub_key": { +// "type": "tendermint/PubKeyEd25519", +// "value": "SIQRxnQENgPMQ8Vg1XBKtFEBLpqU4+hcxwrj6R3ZJp8=" +// }, +// "voting_power": "5000", +// "proposer_priority": "0" +// } +// ], +// "proposer": { +// "address": "ECCF605776310B73EC57189AE89B1E587DBC2601", +// "pub_key": { +// "type": "tendermint/PubKeyEd25519", +// "value": "SIQRxnQENgPMQ8Vg1XBKtFEBLpqU4+hcxwrj6R3ZJp8=" +// }, +// "voting_power": "5000", +// "proposer_priority": "0" +// } +// }, +// From f44b03ca95b07f5028ff8f6ad97cb441f6c80f52 Mon Sep 17 00:00:00 2001 From: zvolin Date: Fri, 30 Aug 2024 18:12:06 +0200 Subject: [PATCH 02/12] add merkle proofs --- types/src/data_availability_header.rs | 243 ++++---------------------- types/src/lib.rs | 2 + types/src/merkle_proof.rs | 190 ++++++++++++++++++++ 3 files changed, 230 insertions(+), 205 deletions(-) create mode 100644 types/src/merkle_proof.rs diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index 5bf96021..be356acd 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -1,6 +1,5 @@ use celestia_proto::celestia::da::DataAvailabilityHeader as RawDataAvailabilityHeader; use celestia_tendermint::merkle::simple_hash_from_byte_vectors; -use celestia_tendermint_proto::v0_34::crypto::Proof as MerkleProof; use celestia_tendermint_proto::v0_34::types::RowProof as RawRowProof; use celestia_tendermint_proto::Protobuf; use serde::{Deserialize, Serialize}; @@ -12,7 +11,10 @@ use crate::consts::data_availability_header::{ use crate::hash::Hash; use crate::nmt::{NamespacedHash, NamespacedHashExt}; use crate::rsmt2d::AxisType; -use crate::{bail_validation, Error, ExtendedDataSquare, Result, ValidateBasic, ValidationError}; +use crate::{ + bail_validation, bail_verification, Error, ExtendedDataSquare, MerkleProof, Result, + ValidateBasic, ValidationError, +}; /// Header with commitments of the data availability. /// @@ -246,31 +248,26 @@ impl ValidateBasic for DataAvailabilityHeader { } } -// TODO: it should follow our regular try from / into RawRowProof pattern -// utilizing proto/vendor/tendermint/types/types.proto. However we cannot -// easily generate it, as it's in .tendermint.types package which we override -// with celestia_tendermint_proto. so the correct solution here would be to -// update celestia_tendermint_proto proto definitions to a recent celestia-core -// version. Note, this wouldn't free us from merkle proof verification logic, -// they are not present in tendermint-rs #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct RowProof { - #[serde(with = "hash_vec_hexstring")] +#[serde(try_from = "RawRowProof", into = "RawRowProof")] +pub struct RowProof { row_roots: Vec, proofs: Vec, - #[serde(default, with = "hash_base64string")] - root: Option, start_row: usize, end_row: usize, } impl RowProof { - fn verify(&self, root: Hash) -> Result<()> { - assert_eq!(self.row_roots.len(), self.proofs.len()); + pub fn verify(&self, root: Hash) -> Result<()> { + if self.row_roots.len() != self.proofs.len() { + bail_verification!("invalid row proof: row_roots.len() != proofs.len()"); + } + let Hash::Sha256(root) = root else { + bail_verification!("empty hash"); + }; for (row_root, proof) in self.row_roots.iter().zip(self.proofs.iter()) { - todo!() - // verify_merkle_proof(proof, &root, row_root); + proof.verify(row_root.to_array(), root)?; } Ok(()) @@ -283,84 +280,37 @@ impl TryFrom for RowProof { type Error = Error; fn try_from(value: RawRowProof) -> Result { - todo!() + Ok(Self { + row_roots: value + .row_roots + .into_iter() + .map(|hash| NamespacedHash::from_raw(&hash)) + .collect::>()?, + proofs: value + .proofs + .into_iter() + .map(TryInto::try_into) + .collect::>()?, + start_row: value.start_row as usize, + end_row: value.end_row as usize, + }) } } impl From for RawRowProof { fn from(value: RowProof) -> Self { - todo!() - } -} - -fn verify_merkle_proof(proof: &MerkleProof, root: &Hash, leaf: &Hash) {} - -mod hash_base64string { - use base64::prelude::*; - use serde::{de, Deserialize, Deserializer, Serializer}; - - use super::Hash; - - pub fn serialize(value: &Option, serializer: S) -> Result - where - S: Serializer, - { - if let Some(hash) = value { - serializer.serialize_some(&BASE64_STANDARD.encode(hash)) - } else { - serializer.serialize_none() + Self { + row_roots: value + .row_roots + .into_iter() + .map(|hash| hash.to_vec()) + .collect(), + proofs: value.proofs.into_iter().map(Into::into).collect(), + start_row: value.start_row as u32, + end_row: value.end_row as u32, + root: vec![], } } - - /// Deserialize base64string into `Hash` - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let Some(hash) = Option::<&str>::deserialize(deserializer)? else { - return Ok(None); - }; - BASE64_STANDARD - .decode(hash) - .map_err(de::Error::custom)? - .try_into() - .map(Some) - .map_err(de::Error::custom) - } -} - -mod hash_vec_hexstring { - use serde::{de, Deserialize, Deserializer, Serializer}; - - use super::{NamespacedHash, NamespacedHashExt}; - - /// Serialize from `Vec` into `Vec` - pub fn serialize(value: &[NamespacedHash], serializer: S) -> Result - where - S: Serializer, - { - let hashes: Vec<_> = value - .iter() - .map(|hash| hex::encode_upper(hash.to_array())) - .collect(); - serializer.serialize_some(&hashes) - } - - /// Deserialize vec_base64string into `NamespacedHash` - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - Option::>::deserialize(deserializer)? - .unwrap_or_default() - .into_iter() - .map(|raw_hash| { - hex::decode(raw_hash) - .map_err(de::Error::custom) - .and_then(|hash| NamespacedHash::from_raw(&hash).map_err(de::Error::custom)) - }) - .collect() - } } #[cfg(test)] @@ -503,120 +453,3 @@ mod tests { row_proof.verify(dah.hash()).unwrap(); } } -// "Proof": { -// "data": [ -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMBAAAEAAK7jTduoBTIVHIsXZYBTeXT+ROAP0ErS1wBn3qRFHoNClY8r4gEOLhvoPDfYX5dN+qGDHdIFPG4F1aF+niSmbfQRSkw2QdjqKwDKhYUKvu10oUo5r/k0SyYJx5KImSJ0d2sBH/ajcpk -// +DWBD0tXJTmsfiATmM8BqaxRRa5biE1T9yV1WKndAyJUC00P8e2/MG+7P1t7a9tMjG+Oxgxx1EzxJ47FDiRwnYNz2/JBqzIC33fKoiWZSeL+NFLn0Dfx+Ev1GYaKpstd1x1tgJnkEceTFVC6r7qhqRbTFJjAjgYJAB4fbBd/+QUQkdbW0uCHLtmWhkeK9YB -// uY05L1v1c6wcXI9IhSlBLnFFdxSTonAaZYhOusiG6eNFn7FpTU0i0oHcksQL+MW3HhbOnIyyUE1Wyjsm6pFuHKBi4TwHTQOibOhvxehuxyrHkqk7QcEPK6/ioN08n2eqd1mlfXiG2wk8nDaZfdmIq3hCm2usrpmqxJHYoH/wbbMeB7AzhreueWRk38984H2 -// h1xX93ZpmUWJEJJGJ0St70Afb6RPjH9pX9vtbVXCvj65D+HPpxinReMBUj0rvGZ6IzNzoBhJYGszp5R4sdztGH8NLNWujwAFDThNRhWUX/r+APM1hdW9s=", -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAaRdGrcBVugXTHUqmX0/UvgFUVjY/T1lnVQuz0gKhCE1aN0WvNPJautwNmv/68eAucdCm6vPqzg4KEgL777G5navyLj/1TMhLJfgTf1YNayDJLN7R13d1QQ3Peagcg+N4Itv0ZmZ6p7/QMQfw -// aXh30yronynPhxV//932ODigrZVrbW+XBhPtgh+/DlbCrU8d65IGn3VGTDQNfZaajmogy4xNf9x089OgcOv8H1XEjj0X8iZQwW+K05wIE6STWGxXJSMywMM7A+FE5YDrnEHeIT9bsIeXvEAy2cwfrorAnQrbfPyZqSSHHQzGumhOYam1Cyz1oVUMAhJMOXR -// drsckPXQdy38cOw/2VNFCZnLDvJIdT9kL3fk+BX/tXUMdvmR0ATY1JiqjW1YPO8fpVaG+IrbUobxDUnrq5kSmK6Gi9WIF+NzCWpPD/bjV+6nwJXEUxzjb18wVitZJZsQpMVsB9t0KXzlvr9AsKob4AZZGAAqbI4cHKjW3PNMbpH6U1WTUPldy7NQvpWDcYs -// VbYQOEr9YiKGyUPIUdb+nPSyAd4aIpbce8fQhN9D9wE0SIDTm2hMorjJsemwdZSHifZbL4Yya7QR/Oa3x5K+82IZiuhm/y5HGEdlHB2Jg54wqJJrKvO2E=", -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAIt3G6MztjqXhUJ6Mbw/14mlqVbzIwJNU38ITmjBXACSyjgvQCMhVZYrvWQxCuEtPHboM1HrI1rgVjMC3B7vregAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -// ], -// "namespace_id": "AAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAow==", -// "namespace_version": 0, -// "share_proofs": [ -// { -// "end": 2, -// "nodes": [ -// "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCU0aUrR/wpx09HFWeoyuV1vuw5Ew3rhtCaf/Zd4chb9", -// "/////////////////////////////////////////////////////////////////////////////ypPU4ZqDz1t8YcunXI8ETuBth1gXLvPWIMd0JPoeJF3" -// ], -// "start": 1 -// }, -// { -// "end": 2, -// "nodes": [ -// "/////////////////////////////////////////////////////////////////////////////wdXw/2tc8hhuGLcsfU9pWo5BDIKSsNJFCytj++xtFgq" -// ] -// } -// ] -// }, -// "Shares": [ -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMBAAAEAAK7jTduoBTIVHIsXZYBTeXT+ROAP0ErS1wBn3qRFHoNClY8r4gEOLhvoPDfYX5dN+qGDHdIFPG4F1aF+niSmbfQRSkw2QdjqKwDKhYUKvu10oUo5r/k0SyYJx5KImSJ0d2sBH/ajcpk+D -// WBD0tXJTmsfiATmM8BqaxRRa5biE1T9yV1WKndAyJUC00P8e2/MG+7P1t7a9tMjG+Oxgxx1EzxJ47FDiRwnYNz2/JBqzIC33fKoiWZSeL+NFLn0Dfx+Ev1GYaKpstd1x1tgJnkEceTFVC6r7qhqRbTFJjAjgYJAB4fbBd/+QUQkdbW0uCHLtmWhkeK9YBuY -// 05L1v1c6wcXI9IhSlBLnFFdxSTonAaZYhOusiG6eNFn7FpTU0i0oHcksQL+MW3HhbOnIyyUE1Wyjsm6pFuHKBi4TwHTQOibOhvxehuxyrHkqk7QcEPK6/ioN08n2eqd1mlfXiG2wk8nDaZfdmIq3hCm2usrpmqxJHYoH/wbbMeB7AzhreueWRk38984H2h1 -// xX93ZpmUWJEJJGJ0St70Afb6RPjH9pX9vtbVXCvj65D+HPpxinReMBUj0rvGZ6IzNzoBhJYGszp5R4sdztGH8NLNWujwAFDThNRhWUX/r+APM1hdW9s=", -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAaRdGrcBVugXTHUqmX0/UvgFUVjY/T1lnVQuz0gKhCE1aN0WvNPJautwNmv/68eAucdCm6vPqzg4KEgL777G5navyLj/1TMhLJfgTf1YNayDJLN7R13d1QQ3Peagcg+N4Itv0ZmZ6p7/QMQfwaX -// h30yronynPhxV//932ODigrZVrbW+XBhPtgh+/DlbCrU8d65IGn3VGTDQNfZaajmogy4xNf9x089OgcOv8H1XEjj0X8iZQwW+K05wIE6STWGxXJSMywMM7A+FE5YDrnEHeIT9bsIeXvEAy2cwfrorAnQrbfPyZqSSHHQzGumhOYam1Cyz1oVUMAhJMOXRdr -// sckPXQdy38cOw/2VNFCZnLDvJIdT9kL3fk+BX/tXUMdvmR0ATY1JiqjW1YPO8fpVaG+IrbUobxDUnrq5kSmK6Gi9WIF+NzCWpPD/bjV+6nwJXEUxzjb18wVitZJZsQpMVsB9t0KXzlvr9AsKob4AZZGAAqbI4cHKjW3PNMbpH6U1WTUPldy7NQvpWDcYsVb -// YQOEr9YiKGyUPIUdb+nPSyAd4aIpbce8fQhN9D9wE0SIDTm2hMorjJsemwdZSHifZbL4Yya7QR/Oa3x5K+82IZiuhm/y5HGEdlHB2Jg54wqJJrKvO2E=", -// "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAIt3G6MztjqXhUJ6Mbw/14mlqVbzIwJNU38ITmjBXACSyjgvQCMhVZYrvWQxCuEtPHboM1HrI1rgVjMC3B7vregAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -// ] -// } -// { -// "header": { -// "version": { -// "block": "11", -// "app": "1" -// }, -// "chain_id": "private", -// "height": "7621", -// "time": "2024-08-29T13:06:07.863632032Z", -// "last_block_id": { -// "hash": "A2EDE7E806EBF59F1E2914AC2D7801B76B3A789F033911AF14C5D62F5C3E7098", -// "parts": { -// "total": 1, -// "hash": "0C45BA8F78E4D8163C992AB47D7930583D1EF207D5C497FC59CC02281F625FEB" -// } -// }, -// "last_commit_hash": "526DB4A98D3661E05DF67B87239EC153515ACCD3F46BA268EFF4015295C6A2B3", -// "data_hash": "E414A1EEB8395CA0C353384A9EC77300698ABBF98CD1DEC688A2C5BC5525077C", -// "validators_hash": "73AE7622856703136CCEC52530EAEFB7BE02441ADC3395B469D82D13C1CD731F", -// "next_validators_hash": "73AE7622856703136CCEC52530EAEFB7BE02441ADC3395B469D82D13C1CD731F", -// "consensus_hash": "C0B6A634B72AE9687EA53B6D277A73ABA1386BA3CFC6D0F26963602F7F6FFCD6", -// "app_hash": "5A6329D9A806308D54C1A28EA0D89A5B2F4AC206BA2F3E5BF8C5E9B15F65CC2F", -// "last_results_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", -// "evidence_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", -// "proposer_address": "ECCF605776310B73EC57189AE89B1E587DBC2601" -// }, -// "commit": { -// "height": 7621, -// "round": 0, -// "block_id": { -// "hash": "A7A4A45F0C655DA558722A14A589BA580F8A06BEE12376772A4CCFA30A989877", -// "parts": { -// "total": 1, -// "hash": "617CFA4E6A5670FFCAF4D06FE72D16214FBAF3B7C96CA4FBD10B47A167B066D9" -// } -// }, -// "signatures": [ -// { -// "block_id_flag": 2, -// "validator_address": "ECCF605776310B73EC57189AE89B1E587DBC2601", -// "timestamp": "2024-08-29T13:06:08.875333629Z", -// "signature": "Beqnt4uw2reTFRguBXuQXMsEoysvUVlB4eyHBKJK6knigN478JUaKWfgXnuFyIA9XRdTpkiS/3OkqWimQXFfCQ==" -// } -// ] -// }, -// "validator_set": { -// "validators": [ -// { -// "address": "ECCF605776310B73EC57189AE89B1E587DBC2601", -// "pub_key": { -// "type": "tendermint/PubKeyEd25519", -// "value": "SIQRxnQENgPMQ8Vg1XBKtFEBLpqU4+hcxwrj6R3ZJp8=" -// }, -// "voting_power": "5000", -// "proposer_priority": "0" -// } -// ], -// "proposer": { -// "address": "ECCF605776310B73EC57189AE89B1E587DBC2601", -// "pub_key": { -// "type": "tendermint/PubKeyEd25519", -// "value": "SIQRxnQENgPMQ8Vg1XBKtFEBLpqU4+hcxwrj6R3ZJp8=" -// }, -// "voting_power": "5000", -// "proposer_priority": "0" -// } -// }, -// diff --git a/types/src/lib.rs b/types/src/lib.rs index db66e3a8..3c1169ae 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -10,6 +10,7 @@ mod error; mod extended_header; pub mod fraud_proof; pub mod hash; +mod merkle_proof; pub mod namespaced_data; pub mod nmt; #[cfg(feature = "p2p")] @@ -35,6 +36,7 @@ pub use crate::data_availability_header::*; pub use crate::error::*; pub use crate::extended_header::*; pub use crate::fraud_proof::FraudProof; +pub use crate::merkle_proof::MerkleProof; pub use crate::rsmt2d::{AxisType, ExtendedDataSquare}; pub use crate::share::*; pub use crate::sync::*; diff --git a/types/src/merkle_proof.rs b/types/src/merkle_proof.rs new file mode 100644 index 00000000..cafda9ab --- /dev/null +++ b/types/src/merkle_proof.rs @@ -0,0 +1,190 @@ +use celestia_tendermint::crypto::default::Sha256; +use celestia_tendermint::merkle::{Hash, MerkleHash}; +use celestia_tendermint_proto::{v0_34::crypto::Proof as RawMerkleProof, Protobuf}; +use serde::{Deserialize, Serialize}; + +use crate::{ + bail_validation, bail_verification, validation_error, verification_error, Error, Result, +}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(try_from = "RawMerkleProof", into = "RawMerkleProof")] +pub struct MerkleProof { + index: usize, + total: usize, + leaf_hash: Hash, + aunts: Vec, +} + +impl MerkleProof { + pub fn new(leaf_to_prove: usize, leaves: &[impl AsRef<[u8]>]) -> Result<(Self, Hash)> { + let total = leaves.len().next_power_of_two(); + if leaf_to_prove >= leaves.len() { + // todo, error + bail_validation!("leaf index out of bounds"); + } + + let mut hasher = Sha256::default(); + let tree_height = total.ilog2() as usize; + + // create lowest level of the tree, padding with empty hashes + let mut tree_level: Vec<_> = leaves + .iter() + .map(|leaf| hasher.leaf_hash(leaf.as_ref())) + .collect(); + tree_level.resize(total, hasher.empty_hash()); + + // save the leaf we're about to prove + let proven_leaf = tree_level[leaf_to_prove]; + let mut aunts = Vec::with_capacity(tree_height); + let mut current_leaf = leaf_to_prove; + + for _ in 0..tree_height { + // the sibling node will be used in proof to reconstruct root + let sibling = current_leaf ^ 1; + aunts.push(tree_level[sibling]); + + // construct higher tree level + tree_level = tree_level + .chunks(2) + .map(|pair| hasher.inner_hash(pair[0], pair[1])) + .collect(); + current_leaf /= 2; + } + + // last tree level is just root + debug_assert_eq!(tree_level.len(), 1); + let root = tree_level[0]; + let proof = Self { + index: leaf_to_prove, + total, + leaf_hash: proven_leaf, + aunts, + }; + + Ok((proof, root)) + } + + pub fn verify(&self, leaf: impl AsRef<[u8]>, root: Hash) -> Result<()> { + let mut hasher = Sha256::default(); + let leaf = hasher.leaf_hash(leaf.as_ref()); + + if leaf != self.leaf_hash { + return Err(verification_error!("proof created for a different leaf").into()); + } + + let computed_root = subtree_root_from_aunts(self.index, self.total, leaf, &self.aunts)?; + + if computed_root != root { + return Err(Error::RootMismatch); + } + + Ok(()) + } +} + +// aunts are effectively a merkle proof for the leaf, so inner hashes needed +// to recompute root hash of a merkle tree +fn subtree_root_from_aunts(index: usize, total: usize, leaf: Hash, aunts: &[Hash]) -> Result { + debug_assert_ne!(total, 0); + + let root = if total == 1 { + // we reached the leaf + if !aunts.is_empty() { + bail_verification!("extra aunts in proof"); + } + leaf + } else { + let mut hasher = Sha256::default(); + let subtrees_split = total.next_power_of_two() / 2; + + // take next subtree root's sibling + let (sibling, aunts) = aunts + .split_last() + .ok_or_else(|| verification_error!("aunts missing in proof"))?; + + if index < subtrees_split { + // and recurse into left subtree + let left_hash = subtree_root_from_aunts(index, subtrees_split, leaf, aunts)?; + hasher.inner_hash(left_hash, *sibling) + } else { + // and recurse into right subtree + let right_hash = subtree_root_from_aunts( + index - subtrees_split, + total - subtrees_split, + leaf, + aunts, + )?; + hasher.inner_hash(*sibling, right_hash) + } + }; + + Ok(root) +} + +impl Protobuf for MerkleProof {} + +impl TryFrom for MerkleProof { + type Error = Error; + + fn try_from(value: RawMerkleProof) -> Result { + if value.index < 0 { + bail_validation!("negative index"); + } + if value.total <= 0 { + bail_validation!("total <= 0"); + } + Ok(Self { + index: value.index as usize, + total: value.total as usize, + leaf_hash: value + .leaf_hash + .try_into() + .map_err(|_| validation_error!("invalid hash size"))?, + aunts: value + .aunts + .into_iter() + .map(TryInto::try_into) + .collect::>() + .map_err(|_| validation_error!("invalid hash size"))?, + }) + } +} + +impl From for RawMerkleProof { + fn from(value: MerkleProof) -> Self { + Self { + index: value.index as i64, + total: value.total as i64, + leaf_hash: value.leaf_hash.into(), + aunts: value.aunts.into_iter().map(Into::into).collect(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::random_bytes; + + use super::MerkleProof; + + #[test] + fn create_and_verify() { + for _ in 0..5 { + let leaf_size = (rand::random::() % 1024) + 1; + let leaves_amount = (rand::random::() % 128) + 1; + let data = random_bytes(leaves_amount * leaf_size); + + let leaves: Vec<_> = data.chunks(leaf_size).collect(); + let leaf_to_prove = rand::random::() % leaves_amount; + + let (proof, root) = MerkleProof::new(leaf_to_prove, &leaves).unwrap(); + + proof.verify(leaves[leaf_to_prove], root).unwrap(); + proof.verify(random_bytes(leaf_size), root).unwrap_err(); + proof + .verify(leaves[leaf_to_prove], rand::random()) + .unwrap_err(); + } + } +} From 02f2f0b407f64cf81550e040fd85ac1c4a87b514 Mon Sep 17 00:00:00 2001 From: zvolin Date: Tue, 3 Sep 2024 15:48:48 +0200 Subject: [PATCH 03/12] add share proofs --- Cargo.lock | 2 + Cargo.toml | 4 +- rpc/src/lib.rs | 2 +- rpc/src/share.rs | 14 +- rpc/tests/share.rs | 34 ++++- types/src/data_availability_header.rs | 142 +++++++++++++++++- types/src/nmt/namespace_proof.rs | 38 +++++ types/src/share.rs | 8 +- types/src/share/proof.rs | 200 ++++++++++++++++++++++++++ 9 files changed, 423 insertions(+), 21 deletions(-) create mode 100644 types/src/share/proof.rs diff --git a/Cargo.lock b/Cargo.lock index be30b864..fc0b3491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -753,6 +753,7 @@ dependencies = [ [[package]] name = "celestia-tendermint" version = "0.32.1" +source = "git+https://github.com/zvolin/celestia-tendermint-rs?branch=fix/proof-serialization#c62d61749af98f0adaf9a271b174fda48e5c2461" dependencies = [ "bytes", "celestia-tendermint-proto", @@ -781,6 +782,7 @@ dependencies = [ [[package]] name = "celestia-tendermint-proto" version = "0.32.1" +source = "git+https://github.com/zvolin/celestia-tendermint-rs?branch=fix/proof-serialization#c62d61749af98f0adaf9a271b174fda48e5c2461" dependencies = [ "bytes", "flex-error", diff --git a/Cargo.toml b/Cargo.toml index d5b91a27..7d625256 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ celestia-tendermint-proto = "0.32.1" # Uncomment to apply local changes #beetswap = { path = "../beetswap" } #blockstore = { path = "../blockstore" } -celestia-tendermint = { path = "../celestia-tendermint-rs/tendermint" } -celestia-tendermint-proto = { path = "../celestia-tendermint-rs/proto" } +celestia-tendermint = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } +celestia-tendermint-proto = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } #nmt-rs = { path = "../nmt-rs" } #libp2p = { path = "../../rust-libp2p/libp2p" } #libp2p-core = { path = "../../rust-libp2p/core" } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index c0a1a6cd..bf02329a 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -7,7 +7,7 @@ mod error; mod header; #[cfg(feature = "p2p")] mod p2p; -mod share; +pub mod share; mod state; pub use crate::blob::BlobClient; diff --git a/rpc/src/share.rs b/rpc/src/share.rs index 80aa2acd..9b287b76 100644 --- a/rpc/src/share.rs +++ b/rpc/src/share.rs @@ -1,6 +1,16 @@ use celestia_types::nmt::Namespace; -use celestia_types::{ExtendedDataSquare, ExtendedHeader, NamespacedShares, Share}; +use celestia_types::{ + ExtendedDataSquare, ExtendedHeader, NamespacedShares, RawShare, Share, ShareProof, +}; use jsonrpsee::proc_macros::rpc; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct GetRangeResponse { + pub shares: Vec, + pub proof: ShareProof, +} #[rpc(client)] pub trait Share { @@ -15,7 +25,7 @@ pub trait Share { height: u64, start: usize, end: usize, - ) -> Result; + ) -> Result; /// GetShare gets a Share by coordinates in EDS. #[method(name = "share.GetShare")] diff --git a/rpc/tests/share.rs b/rpc/tests/share.rs index 0f0af48b..b6dcfdb1 100644 --- a/rpc/tests/share.rs +++ b/rpc/tests/share.rs @@ -6,7 +6,7 @@ use celestia_types::consts::appconsts::{ SHARE_INFO_BYTES, }; use celestia_types::nmt::{Namespace, NamespacedSha2Hasher}; -use celestia_types::Blob; +use celestia_types::{Blob, Share}; pub mod utils; @@ -67,16 +67,38 @@ async fn get_shares_range() { .await .unwrap(); let index = blob_on_chain.index.unwrap(); - let shares = blob_on_chain.to_shares().unwrap().len(); + let shares = blob_on_chain.to_shares().unwrap(); let shares_range = client - .share_get_range(submitted_height, index as usize, index as usize + shares) + .share_get_range( + submitted_height, + index as usize, + index as usize + shares.len(), + ) .await .unwrap(); - println!("{}", serde_json::to_string_pretty(&shares_range).unwrap()); - println!("{}", serde_json::to_string_pretty(&header).unwrap()); - panic!(); + shares_range.proof.verify(header.dah.hash()).unwrap(); + + for (share, received) in shares.into_iter().zip(shares_range.shares.into_iter()) { + assert_eq!(share, Share::try_from(received).unwrap()); + } +} + +#[tokio::test] +async fn get_shares_range_unexisting() { + let client = new_test_client(AuthLevel::Write).await.unwrap(); + let header = client.header_network_head().await.unwrap(); + let shares_in_block = header.dah.square_width().pow(2); + + client + .share_get_range( + header.height().value(), + shares_in_block as usize - 2, + shares_in_block as usize + 2, + ) + .await + .unwrap_err(); } #[tokio::test] diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index be356acd..9be1ec1d 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -12,8 +12,8 @@ use crate::hash::Hash; use crate::nmt::{NamespacedHash, NamespacedHashExt}; use crate::rsmt2d::AxisType; use crate::{ - bail_validation, bail_verification, Error, ExtendedDataSquare, MerkleProof, Result, - ValidateBasic, ValidationError, + bail_validation, bail_verification, validation_error, Error, ExtendedDataSquare, MerkleProof, + Result, ValidateBasic, ValidationError, }; /// Header with commitments of the data availability. @@ -182,6 +182,33 @@ impl DataAvailabilityHeader { // On validated DAH this never happens .expect("len is bigger than u16::MAX") } + + pub fn row_proof(&self, start_row: u16, end_row: u16) -> Result { + let all_roots: Vec<_> = self + .row_roots + .iter() + .chain(self.column_roots.iter()) + .map(|root| root.to_array()) + .collect(); + + let rows = 1 + end_row + .checked_sub(start_row) + .ok_or_else(|| validation_error!("todo"))? as usize; + let mut proofs = Vec::with_capacity(rows); + let mut row_roots = Vec::with_capacity(rows); + + for idx in start_row..=end_row { + proofs.push(MerkleProof::new(idx as usize, &all_roots)?.0); + row_roots.push(self.row_root(idx).expect("todo")); + } + + Ok(RowProof { + proofs, + row_roots, + start_row, + end_row, + }) + } } impl Protobuf for DataAvailabilityHeader {} @@ -253,15 +280,37 @@ impl ValidateBasic for DataAvailabilityHeader { pub struct RowProof { row_roots: Vec, proofs: Vec, - start_row: usize, - end_row: usize, + start_row: u16, + end_row: u16, } impl RowProof { + pub fn row_roots(&self) -> &[NamespacedHash] { + &self.row_roots + } + pub fn verify(&self, root: Hash) -> Result<()> { if self.row_roots.len() != self.proofs.len() { bail_verification!("invalid row proof: row_roots.len() != proofs.len()"); } + + if self.end_row < self.start_row { + bail_verification!( + "start_row ({}) > end_row ({})", + self.start_row, + self.end_row + ); + } + + let length = self.end_row - self.start_row + 1; + if length as usize != self.proofs.len() { + bail_verification!( + "length based on start_row and end_row ({}) != length of proofs ({})", + length, + self.proofs.len() + ); + } + let Hash::Sha256(root) = root else { bail_verification!("empty hash"); }; @@ -291,8 +340,14 @@ impl TryFrom for RowProof { .into_iter() .map(TryInto::try_into) .collect::>()?, - start_row: value.start_row as usize, - end_row: value.end_row as usize, + start_row: value + .start_row + .try_into() + .map_err(|_| validation_error!("start_row ({}) exceeds u16", value.start_row))?, + end_row: value + .end_row + .try_into() + .map_err(|_| validation_error!("end_row ({}) exceeds u16", value.end_row))?, }) } } @@ -315,6 +370,8 @@ impl From for RawRowProof { #[cfg(test)] mod tests { + use crate::nmt::Namespace; + use super::*; #[cfg(target_arch = "wasm32")] @@ -397,7 +454,7 @@ mod tests { } #[test] - fn row_proof_verify_correct() { + fn row_proof_serde() { let raw_row_proof = r#" { "end_row": 1, @@ -452,4 +509,75 @@ mod tests { row_proof.verify(dah.hash()).unwrap(); } + + #[test] + fn row_proof_verify_correct() { + for square_width in [2, 4, 8, 16] { + let dah = random_dah(square_width); + let dah_root = dah.hash(); + + for start_row in 0..dah.square_width() - 1 { + for end_row in start_row..dah.square_width() { + let proof = dah.row_proof(start_row, end_row).unwrap(); + + proof.verify(dah_root).unwrap() + } + } + } + } + + #[test] + fn row_proof_verify_malformed() { + let dah = random_dah(16); + let dah_root = dah.hash(); + + let valid_proof = dah.row_proof(0, 1).unwrap(); + + // start_row > end_row + let mut proof = valid_proof.clone(); + proof.end_row = 0; + proof.start_row = 1; + proof.verify(dah_root).unwrap_err(); + dah.row_proof(1, 0).unwrap_err(); + + // length incorrect based on start and end + let mut proof = valid_proof.clone(); + proof.end_row = 2; + proof.verify(dah_root).unwrap_err(); + + // incorrect amount of proofs + let mut proof = valid_proof.clone(); + proof.proofs.push(proof.proofs[0].clone()); + proof.verify(dah_root).unwrap_err(); + + // incorrect amount of roots + let mut proof = valid_proof.clone(); + proof.row_roots.pop(); + proof.verify(dah_root).unwrap_err(); + + // wrong proof order + let mut proof = valid_proof.clone(); + proof.row_roots = proof.row_roots.into_iter().rev().collect(); + proof.verify(dah_root).unwrap_err(); + } + + fn random_dah(square_width: u16) -> DataAvailabilityHeader { + let namespaces: Vec<_> = (0..square_width) + .map(|n| Namespace::new_v0(&[n as u8]).unwrap()) + .collect(); + let (row_roots, col_roots): (Vec<_>, Vec<_>) = namespaces + .iter() + .map(|&ns| { + let row = NamespacedHash::new(*ns, *ns, rand::random()); + let col = NamespacedHash::new( + **namespaces.first().unwrap(), + **namespaces.last().unwrap(), + rand::random(), + ); + (row, col) + }) + .unzip(); + + DataAvailabilityHeader::new(row_roots, col_roots).unwrap() + } } diff --git a/types/src/nmt/namespace_proof.rs b/types/src/nmt/namespace_proof.rs index 3e162ec5..2c78c60b 100644 --- a/types/src/nmt/namespace_proof.rs +++ b/types/src/nmt/namespace_proof.rs @@ -1,6 +1,7 @@ use std::ops::{Deref, DerefMut}; use celestia_proto::proof::pb::Proof as RawProof; +use celestia_tendermint_proto::v0_34::types::NmtProof as RawTendermintProof; use celestia_tendermint_proto::Protobuf; use nmt_rs::simple_merkle::proof::Proof as NmtProof; use serde::{Deserialize, Serialize}; @@ -162,3 +163,40 @@ impl From for RawProof { } } } + +impl TryFrom for NamespaceProof { + type Error = Error; + + fn try_from(value: RawTendermintProof) -> Result { + let siblings = value + .nodes + .iter() + .map(|bytes| NamespacedHash::from_raw(bytes)) + .collect::>>()?; + + let mut proof = NmtNamespaceProof::PresenceProof { + proof: NmtProof { + siblings, + range: value.start as u32..value.end as u32, + }, + ignore_max_ns: true, + }; + + if !value.leaf_hash.is_empty() { + proof.convert_to_absence_proof(NamespacedHash::from_raw(&value.leaf_hash)?); + } + + Ok(NamespaceProof(proof)) + } +} + +impl From for RawTendermintProof { + fn from(value: NamespaceProof) -> Self { + RawTendermintProof { + start: value.start_idx() as i32, + end: value.end_idx() as i32, + nodes: value.siblings().iter().map(|hash| hash.to_vec()).collect(), + leaf_hash: value.leaf().map(|hash| hash.to_vec()).unwrap_or_default(), + } + } +} diff --git a/types/src/share.rs b/types/src/share.rs index 5ed7d72c..8733323d 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -15,8 +15,10 @@ use crate::nmt::{ use crate::{Error, Result}; mod info_byte; +mod proof; pub use info_byte::InfoByte; +pub use proof::ShareProof; const SHARE_SEQUENCE_LENGTH_OFFSET: usize = NS_SIZE + appconsts::SHARE_INFO_BYTES; @@ -184,11 +186,11 @@ impl From for RawNamespacedShares { } } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(transparent)] -struct RawShare { +pub struct RawShare { #[serde(with = "celestia_tendermint_proto::serializers::bytes::base64string")] - data: Vec, + pub data: Vec, } impl TryFrom for Share { diff --git a/types/src/share/proof.rs b/types/src/share/proof.rs new file mode 100644 index 00000000..21a07421 --- /dev/null +++ b/types/src/share/proof.rs @@ -0,0 +1,200 @@ +use celestia_tendermint::Hash; +use celestia_tendermint_proto::v0_34::types::ShareProof as RawShareProof; +use celestia_tendermint_proto::Protobuf; +use serde::{Deserialize, Serialize}; + +use crate::consts::appconsts::SHARE_SIZE; +use crate::nmt::NamespaceProof; +use crate::{bail_verification, validation_error, RowProof}; +use crate::{nmt::Namespace, Error, Result}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(try_from = "RawShareProof", into = "RawShareProof")] +pub struct ShareProof { + data: Vec<[u8; SHARE_SIZE]>, + namespace_id: Namespace, + share_proofs: Vec, + row_proof: RowProof, +} + +impl ShareProof { + pub fn verify(&self, root: Hash) -> Result<()> { + let row_roots = self.row_proof.row_roots(); + + if self.share_proofs.len() != row_roots.len() { + bail_verification!( + "share proofs length ({}) != row roots length ({})", + self.share_proofs.len(), + row_roots.len() + ); + } + + let mut shares_needed = 0; + for proof in &self.share_proofs { + if proof.is_of_absence() { + bail_verification!("only presence proofs allowed"); + } + if proof.start_idx() >= proof.end_idx() { + bail_verification!("proof without data"); + } + + shares_needed += proof.end_idx() - proof.start_idx(); + } + + if shares_needed as usize != self.data.len() { + bail_verification!( + "shares needed ({}) != proof's data length ({})", + shares_needed, + self.data.len() + ); + } + + self.row_proof.verify(root)?; + + let mut data = self.data.as_slice(); + + for (proof, row) in self.share_proofs.iter().zip(row_roots) { + let amount = proof.end_idx() - proof.start_idx(); + let leaves = &data[..amount as usize]; + proof + .verify_range(row, leaves, *self.namespace_id) + .map_err(Error::RangeProofError)?; + data = &data[amount as usize..]; + } + + Ok(()) + } +} + +impl Protobuf for ShareProof {} + +impl TryFrom for ShareProof { + type Error = Error; + + fn try_from(value: RawShareProof) -> Result { + println!("{:?}", value.namespace_id); + Ok(Self { + data: value + .data + .into_iter() + .map(TryInto::try_into) + .collect::>() + .map_err(|_| validation_error!("todo"))?, + namespace_id: Namespace::new( + value + .namespace_version + .try_into() + .map_err(|_| validation_error!("namespace version must be single byte"))?, + &value.namespace_id, + )?, + share_proofs: value + .share_proofs + .into_iter() + .map(TryInto::try_into) + .collect::>()?, + row_proof: value + .row_proof + .ok_or_else(|| validation_error!("todo"))? + .try_into()?, + }) + } +} + +impl From for RawShareProof { + fn from(value: ShareProof) -> Self { + Self { + data: value.data.into_iter().map(Into::into).collect(), + namespace_id: value.namespace_id.id().to_vec(), + namespace_version: value.namespace_id.version() as u32, + share_proofs: value.share_proofs.into_iter().map(Into::into).collect(), + row_proof: Some(value.row_proof.into()), + } + } +} + +#[cfg(test)] +mod tests { + use crate::DataAvailabilityHeader; + + use super::ShareProof; + + #[test] + fn share_proof_serde() { + let raw_share_proof = r#"{ + "data": [ + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMBAAAEAAK7jTduoBTIVHIsXZYBTeXT+ROAP0ErS1wBn3qRFHoNClY8r4gEOLhvoPDfYX5dN+qGDHdIFPG4F1aF+niSmbfQRSkw2QdjqKwDKhYUKvu10oUo5r/k0SyYJx5KImSJ0d2sBH/ajcpk+DWBD0tXJTmsfiATmM8BqaxRRa5biE1T9yV1WKndAyJUC00P8e2/MG+7P1t7a9tMjG+Oxgxx1EzxJ47FDiRwnYNz2/JBqzIC33fKoiWZSeL+NFLn0Dfx+Ev1GYaKpstd1x1tgJnkEceTFVC6r7qhqRbTFJjAjgYJAB4fbBd/+QUQkdbW0uCHLtmWhkeK9YBuY05L1v1c6wcXI9IhSlBLnFFdxSTonAaZYhOusiG6eNFn7FpTU0i0oHcksQL+MW3HhbOnIyyUE1Wyjsm6pFuHKBi4TwHTQOibOhvxehuxyrHkqk7QcEPK6/ioN08n2eqd1mlfXiG2wk8nDaZfdmIq3hCm2usrpmqxJHYoH/wbbMeB7AzhreueWRk38984H2h1xX93ZpmUWJEJJGJ0St70Afb6RPjH9pX9vtbVXCvj65D+HPpxinReMBUj0rvGZ6IzNzoBhJYGszp5R4sdztGH8NLNWujwAFDThNRhWUX/r+APM1hdW9s=", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAaRdGrcBVugXTHUqmX0/UvgFUVjY/T1lnVQuz0gKhCE1aN0WvNPJautwNmv/68eAucdCm6vPqzg4KEgL777G5navyLj/1TMhLJfgTf1YNayDJLN7R13d1QQ3Peagcg+N4Itv0ZmZ6p7/QMQfwaXh30yronynPhxV//932ODigrZVrbW+XBhPtgh+/DlbCrU8d65IGn3VGTDQNfZaajmogy4xNf9x089OgcOv8H1XEjj0X8iZQwW+K05wIE6STWGxXJSMywMM7A+FE5YDrnEHeIT9bsIeXvEAy2cwfrorAnQrbfPyZqSSHHQzGumhOYam1Cyz1oVUMAhJMOXRdrsckPXQdy38cOw/2VNFCZnLDvJIdT9kL3fk+BX/tXUMdvmR0ATY1JiqjW1YPO8fpVaG+IrbUobxDUnrq5kSmK6Gi9WIF+NzCWpPD/bjV+6nwJXEUxzjb18wVitZJZsQpMVsB9t0KXzlvr9AsKob4AZZGAAqbI4cHKjW3PNMbpH6U1WTUPldy7NQvpWDcYsVbYQOEr9YiKGyUPIUdb+nPSyAd4aIpbce8fQhN9D9wE0SIDTm2hMorjJsemwdZSHifZbL4Yya7QR/Oa3x5K+82IZiuhm/y5HGEdlHB2Jg54wqJJrKvO2E=", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAIt3G6MztjqXhUJ6Mbw/14mlqVbzIwJNU38ITmjBXACSyjgvQCMhVZYrvWQxCuEtPHboM1HrI1rgVjMC3B7vregAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + ], + "namespace_id": "AAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAow==", + "namespace_version": 0, + "share_proofs": [ + { + "end": 2, + "nodes": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCU0aUrR/wpx09HFWeoyuV1vuw5Ew3rhtCaf/Zd4chb9", + "/////////////////////////////////////////////////////////////////////////////ypPU4ZqDz1t8YcunXI8ETuBth1gXLvPWIMd0JPoeJF3" + ], + "start": 1 + }, + { + "end": 2, + "nodes": [ + "/////////////////////////////////////////////////////////////////////////////wdXw/2tc8hhuGLcsfU9pWo5BDIKSsNJFCytj++xtFgq" + ] + } + ], + "row_proof": { + "end_row": 1, + "proofs": [ + { + "aunts": [ + "Ch+9PsBdsN5YUt8nvAmjdOAIcVdfmPAEUNmCA8KBe5A=", + "ojjC9H5JG/7OOrt5BzBXs/3w+n1LUI/0YR0d+RSfleU=", + "d6bMQbLTBfZGvqXOW9MPqRM+fTB2/wLJx6CkLc8glCI=" + ], + "index": 0, + "leaf_hash": "nOpM3A3d0JYOmNaaI5BFAeKPGwQ90TqmM/kx+sHr79s=", + "total": 8 + }, + { + "aunts": [ + "nOpM3A3d0JYOmNaaI5BFAeKPGwQ90TqmM/kx+sHr79s=", + "ojjC9H5JG/7OOrt5BzBXs/3w+n1LUI/0YR0d+RSfleU=", + "d6bMQbLTBfZGvqXOW9MPqRM+fTB2/wLJx6CkLc8glCI=" + ], + "index": 1, + "leaf_hash": "Ch+9PsBdsN5YUt8nvAmjdOAIcVdfmPAEUNmCA8KBe5A=", + "total": 8 + } + ], + "row_roots": [ + "000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000D8CBB533A24261C4C0A3D37F1CBFB6F4C5EA031472EBA390D482637933874AA0A2B9735E67629993852D", + "00000000000000000000000000000000000000D8CBB533A24261C4C0A300000000000000000000000000000000000000D8CBB533A24261C4C0A37E409334CCB1125C793EC040741137634C148F089ACB06BFFF4C1C4CA2CBBA8E" + ], + "start_row": 0 + } + }"#; + let raw_dah = r#" + { + "row_roots": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo9N/HL+29MXqAxRy66OQ1IJjeTOHSqCiuXNeZ2KZk4Ut", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo35AkzTMsRJceT7AQHQRN2NMFI8ImssGv/9MHEyiy7qO", + "/////////////////////////////////////////////////////////////////////////////7mTwL+NxdxcYBd89/wRzW2k9vRkQehZiXsuqZXHy89X", + "/////////////////////////////////////////////////////////////////////////////2X/FT2ugeYdWmvnEisSgW+9Ih8paNvrji2NYPb8ujaK" + ], + "column_roots": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo/xEv//wkWzNtkcAZiZmSGU1Te6ERwUxTtTfHzoS4bv+", + "AAAAAAAAAAAAAAAAAAAAAAAAANjLtTOiQmHEwKMAAAAAAAAAAAAAAAAAAAAAAAAA2Mu1M6JCYcTAo9FOCNvCjA42xYCwHrlo48iPEXLaKt+d+JdErCIrQIi6", + "/////////////////////////////////////////////////////////////////////////////y2UErq/83uv433HekCWokxqcY4g+nMQn3tZn2Tr6v74", + "/////////////////////////////////////////////////////////////////////////////z6fKmbJTvfLYFlNuDWHn87vJb6V7n44MlCkxv1dyfT2" + ] + } + "#; + + let proof: ShareProof = serde_json::from_str(raw_share_proof).unwrap(); + let dah: DataAvailabilityHeader = serde_json::from_str(raw_dah).unwrap(); + + proof.verify(dah.hash()).unwrap() + } +} From 5b8030ac5479f04a6b606020142ccf586f712447 Mon Sep 17 00:00:00 2001 From: zvolin Date: Tue, 3 Sep 2024 16:15:06 +0200 Subject: [PATCH 04/12] dry new nmt proof conversion --- Cargo.lock | 1 - types/Cargo.toml | 1 - types/src/nmt/namespace_proof.rs | 33 ++++++++++++-------------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc0b3491..5338bc5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,7 +812,6 @@ dependencies = [ "ed25519-consensus", "enum_dispatch", "getrandom", - "hex", "indoc", "leopard-codec", "libp2p-identity", diff --git a/types/Cargo.toml b/types/Cargo.toml index 7ce02840..f887e258 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -27,7 +27,6 @@ cid = { version = "0.11.1", default-features = false, features = ["std"] } const_format = "0.2.32" ed25519-consensus = { version = "2.1.0", optional = true } enum_dispatch = "0.3.13" -hex = "0.4.3" leopard-codec = "0.1.0" libp2p-identity = { version = "0.2.9", optional = true } multiaddr = { version = "0.18.1", optional = true } diff --git a/types/src/nmt/namespace_proof.rs b/types/src/nmt/namespace_proof.rs index 2c78c60b..79cbe572 100644 --- a/types/src/nmt/namespace_proof.rs +++ b/types/src/nmt/namespace_proof.rs @@ -168,35 +168,26 @@ impl TryFrom for NamespaceProof { type Error = Error; fn try_from(value: RawTendermintProof) -> Result { - let siblings = value - .nodes - .iter() - .map(|bytes| NamespacedHash::from_raw(bytes)) - .collect::>>()?; - - let mut proof = NmtNamespaceProof::PresenceProof { - proof: NmtProof { - siblings, - range: value.start as u32..value.end as u32, - }, - ignore_max_ns: true, + let raw_proof = RawProof { + start: value.start as i64, + end: value.end as i64, + nodes: value.nodes, + leaf_hash: value.leaf_hash, + is_max_namespace_ignored: true, }; - if !value.leaf_hash.is_empty() { - proof.convert_to_absence_proof(NamespacedHash::from_raw(&value.leaf_hash)?); - } - - Ok(NamespaceProof(proof)) + raw_proof.try_into() } } impl From for RawTendermintProof { fn from(value: NamespaceProof) -> Self { + let raw_proof = RawProof::from(value); RawTendermintProof { - start: value.start_idx() as i32, - end: value.end_idx() as i32, - nodes: value.siblings().iter().map(|hash| hash.to_vec()).collect(), - leaf_hash: value.leaf().map(|hash| hash.to_vec()).unwrap_or_default(), + start: raw_proof.start as i32, + end: raw_proof.end as i32, + nodes: raw_proof.nodes, + leaf_hash: raw_proof.leaf_hash, } } } From 07c4cfe2707a10dc8fde16dcfa28a888d0f7e4ef Mon Sep 17 00:00:00 2001 From: zvolin Date: Tue, 3 Sep 2024 16:18:15 +0200 Subject: [PATCH 05/12] remove leftover dependency --- Cargo.lock | 1 - rpc/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5338bc5f..9920125d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -743,7 +743,6 @@ dependencies = [ "nmt-rs", "rand", "serde", - "serde_json", "thiserror", "tokio", "tracing", diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index b6effacd..3ccd6a33 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -25,7 +25,6 @@ jsonrpsee = { version = "0.24.2", features = ["client-core", "macros"] } serde = { version = "1.0.203", features = ["derive"] } thiserror = "1.0.61" tracing = "0.1.40" -serde_json = "*" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] http = "1.1.0" From 188ae95d71adaed6a0febe01158f837d72c7bc20 Mon Sep 17 00:00:00 2001 From: zvolin Date: Mon, 9 Sep 2024 14:00:24 +0200 Subject: [PATCH 06/12] tendermint merkle consistency + docs --- rpc/tests/share.rs | 7 +- types/src/data_availability_header.rs | 58 ++++++++--- types/src/error.rs | 4 + types/src/merkle_proof.rs | 139 +++++++++++++++++++------- types/src/share/proof.rs | 23 +++++ 5 files changed, 180 insertions(+), 51 deletions(-) diff --git a/rpc/tests/share.rs b/rpc/tests/share.rs index b6dcfdb1..4ffe85cb 100644 --- a/rpc/tests/share.rs +++ b/rpc/tests/share.rs @@ -80,8 +80,13 @@ async fn get_shares_range() { shares_range.proof.verify(header.dah.hash()).unwrap(); - for (share, received) in shares.into_iter().zip(shares_range.shares.into_iter()) { + for ((share, received), proven) in shares + .into_iter() + .zip(shares_range.shares.into_iter()) + .zip(shares_range.proof.shares().iter()) + { assert_eq!(share, Share::try_from(received).unwrap()); + assert_eq!(share.as_ref(), proven.as_ref()); } } diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index 9be1ec1d..dcff55f7 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -1,3 +1,5 @@ +use std::ops::RangeInclusive; + use celestia_proto::celestia::da::DataAvailabilityHeader as RawDataAvailabilityHeader; use celestia_tendermint::merkle::simple_hash_from_byte_vectors; use celestia_tendermint_proto::v0_34::types::RowProof as RawRowProof; @@ -183,7 +185,8 @@ impl DataAvailabilityHeader { .expect("len is bigger than u16::MAX") } - pub fn row_proof(&self, start_row: u16, end_row: u16) -> Result { + /// Get the [`RowProof`] for given rows. + pub fn row_proof(&self, rows: RangeInclusive) -> Result { let all_roots: Vec<_> = self .row_roots .iter() @@ -191,15 +194,17 @@ impl DataAvailabilityHeader { .map(|root| root.to_array()) .collect(); - let rows = 1 + end_row - .checked_sub(start_row) - .ok_or_else(|| validation_error!("todo"))? as usize; - let mut proofs = Vec::with_capacity(rows); - let mut row_roots = Vec::with_capacity(rows); + let start_row = *rows.start(); + let end_row = *rows.end(); + let mut proofs = Vec::with_capacity(rows.len()); + let mut row_roots = Vec::with_capacity(rows.len()); - for idx in start_row..=end_row { + for idx in rows { proofs.push(MerkleProof::new(idx as usize, &all_roots)?.0); - row_roots.push(self.row_root(idx).expect("todo")); + let row = self + .row_root(idx) + .ok_or_else(|| Error::IndexOutOfRange(idx as usize, self.row_roots.len()))?; + row_roots.push(row); } Ok(RowProof { @@ -275,6 +280,7 @@ impl ValidateBasic for DataAvailabilityHeader { } } +/// A proof of inclusion of a range of row roots in a [`DataAvailabilityHeader`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(try_from = "RawRowProof", into = "RawRowProof")] pub struct RowProof { @@ -285,10 +291,36 @@ pub struct RowProof { } impl RowProof { + /// Get the list of row roots this proof proves. pub fn row_roots(&self) -> &[NamespacedHash] { &self.row_roots } + /// Verify the proof against the hash of [`DataAvailabilityHeader`], proving + /// the inclusion of rows. + /// + /// # Errors + /// + /// This function will return an error if: + /// - the proof is malformed, meaning some inconsistency between start/end row, + /// row roots or merkle proofs amounts + /// - the verification of any inner merkle proof fails + /// + /// # Example + /// + /// ``` + /// # use celestia_types::ExtendedHeader; + /// # fn get_extended_header() -> ExtendedHeader { + /// # let s = include_str!("../test_data/chain1/extended_header_block_1.json"); + /// # serde_json::from_str(s).unwrap() + /// # } + /// let eh = get_extended_header(); + /// let dah = eh.dah; + /// + /// let proof = dah.row_proof(0..=1).unwrap(); + /// + /// assert!(proof.verify(dah.hash()).is_ok()); + /// ``` pub fn verify(&self, root: Hash) -> Result<()> { if self.row_roots.len() != self.proofs.len() { bail_verification!("invalid row proof: row_roots.len() != proofs.len()"); @@ -518,7 +550,7 @@ mod tests { for start_row in 0..dah.square_width() - 1 { for end_row in start_row..dah.square_width() { - let proof = dah.row_proof(start_row, end_row).unwrap(); + let proof = dah.row_proof(start_row..=end_row).unwrap(); proof.verify(dah_root).unwrap() } @@ -531,14 +563,12 @@ mod tests { let dah = random_dah(16); let dah_root = dah.hash(); - let valid_proof = dah.row_proof(0, 1).unwrap(); + let valid_proof = dah.row_proof(0..=1).unwrap(); // start_row > end_row - let mut proof = valid_proof.clone(); - proof.end_row = 0; - proof.start_row = 1; + #[allow(clippy::reversed_empty_ranges)] + let proof = dah.row_proof(1..=0).unwrap(); proof.verify(dah_root).unwrap_err(); - dah.row_proof(1, 0).unwrap_err(); // length incorrect based on start and end let mut proof = valid_proof.clone(); diff --git a/types/src/error.rs b/types/src/error.rs index 741c3e1b..7cbdf5d9 100644 --- a/types/src/error.rs +++ b/types/src/error.rs @@ -164,6 +164,10 @@ pub enum Error { #[error("Unsupported fraud proof type: {0}")] UnsupportedFraudProofType(String), + /// Data square index out of range. + #[error("Index ({0}) out of range ({1})")] + IndexOutOfRange(usize, usize), + /// Data square index out of range. #[error("Data square index out of range. row: {0}, column: {1}")] EdsIndexOutOfRange(u16, u16), diff --git a/types/src/merkle_proof.rs b/types/src/merkle_proof.rs index cafda9ab..5955249c 100644 --- a/types/src/merkle_proof.rs +++ b/types/src/merkle_proof.rs @@ -17,54 +17,51 @@ pub struct MerkleProof { } impl MerkleProof { + /// Create a merkle proof of inclusion of a given leaf in a list. + /// + /// Returns a proof together with the root hash of a merkle tree built from given leaves. + /// + /// # Errors + /// + /// This function will return an error if `leaf_to_prove` index is out of `leaves` bounds. + /// + /// # Example + /// + /// ``` + /// use celestia_types::MerkleProof; + /// + /// let (proof, root) = MerkleProof::new(0, &[b"a", b"b", b"c"]).unwrap(); + /// + /// assert!(proof.verify(b"a", root).is_ok()); + /// ``` pub fn new(leaf_to_prove: usize, leaves: &[impl AsRef<[u8]>]) -> Result<(Self, Hash)> { - let total = leaves.len().next_power_of_two(); if leaf_to_prove >= leaves.len() { - // todo, error - bail_validation!("leaf index out of bounds"); + return Err(Error::IndexOutOfRange(leaf_to_prove, leaves.len())); } - let mut hasher = Sha256::default(); - let tree_height = total.ilog2() as usize; - - // create lowest level of the tree, padding with empty hashes - let mut tree_level: Vec<_> = leaves - .iter() - .map(|leaf| hasher.leaf_hash(leaf.as_ref())) - .collect(); - tree_level.resize(total, hasher.empty_hash()); - - // save the leaf we're about to prove - let proven_leaf = tree_level[leaf_to_prove]; + let tree_height = leaves.len().next_power_of_two().ilog2() as usize; let mut aunts = Vec::with_capacity(tree_height); - let mut current_leaf = leaf_to_prove; - - for _ in 0..tree_height { - // the sibling node will be used in proof to reconstruct root - let sibling = current_leaf ^ 1; - aunts.push(tree_level[sibling]); - - // construct higher tree level - tree_level = tree_level - .chunks(2) - .map(|pair| hasher.inner_hash(pair[0], pair[1])) - .collect(); - current_leaf /= 2; - } - // last tree level is just root - debug_assert_eq!(tree_level.len(), 1); - let root = tree_level[0]; + let root = hash_leaves_collecting_aunts(0, leaf_to_prove, leaves, &mut aunts); let proof = Self { index: leaf_to_prove, - total, - leaf_hash: proven_leaf, + total: leaves.len(), + leaf_hash: Sha256::default().leaf_hash(leaves[leaf_to_prove].as_ref()), aunts, }; Ok((proof, root)) } + /// Verify that given leaf is included under the root hash. + /// + /// # Errors + /// + /// This function will return an error if: + /// - provided leaf is different than the one for which proof was created + /// - proof is malformed, meaning some inconsistency between leaf index, leaves count, + /// or amount of inner nodes + /// - the recomputed root hash differs from expected one pub fn verify(&self, leaf: impl AsRef<[u8]>, root: Hash) -> Result<()> { let mut hasher = Sha256::default(); let leaf = hasher.leaf_hash(leaf.as_ref()); @@ -83,6 +80,55 @@ impl MerkleProof { } } +// creates a merkle tree out of the given leaves and returns it's root hash. +// inner nodes needed to prove leaf with given index are collected in `aunts`. +fn hash_leaves_collecting_aunts( + // for recursion, index of the first leaf in current subtree wrt the whole tree + first_leaf_index: usize, + leaf_to_prove: usize, + leaves: &[impl AsRef<[u8]>], + aunts: &mut Vec, +) -> Hash { + let mut hasher = Sha256::default(); + let total = leaves.len(); + + match total { + // can only happen if there are 0 leaves without recursing + 0 => hasher.empty_hash(), + // we reached a leaf + 1 => hasher.leaf_hash(leaves[0].as_ref()), + // split leaves into subtrees and hash them recursively + _ => { + let subtrees_split = total.next_power_of_two() / 2; + let left = hash_leaves_collecting_aunts( + first_leaf_index, + leaf_to_prove, + &leaves[..subtrees_split], + aunts, + ); + let right = hash_leaves_collecting_aunts( + first_leaf_index + subtrees_split, + leaf_to_prove, + &leaves[subtrees_split..], + aunts, + ); + + // if current subtree has the leaf to prove + if (first_leaf_index..first_leaf_index + total).contains(&leaf_to_prove) { + if leaf_to_prove < first_leaf_index + subtrees_split { + // leaf was in left subtree, so it's aunt is right hash + aunts.push(right) + } else { + // leaf was in right subtree, so it's aunt is left hash + aunts.push(left) + } + } + + hasher.inner_hash(left, right) + } + } +} + // aunts are effectively a merkle proof for the leaf, so inner hashes needed // to recompute root hash of a merkle tree fn subtree_root_from_aunts(index: usize, total: usize, leaf: Hash, aunts: &[Hash]) -> Result { @@ -164,15 +210,18 @@ impl From for RawMerkleProof { #[cfg(test)] mod tests { + use celestia_tendermint::crypto::default::Sha256; + use celestia_tendermint::merkle::simple_hash_from_byte_vectors; + use crate::test_utils::random_bytes; use super::MerkleProof; #[test] fn create_and_verify() { - for _ in 0..5 { + for _ in 0..100 { let leaf_size = (rand::random::() % 1024) + 1; - let leaves_amount = (rand::random::() % 128) + 1; + let leaves_amount = 8; let data = random_bytes(leaves_amount * leaf_size); let leaves: Vec<_> = data.chunks(leaf_size).collect(); @@ -187,4 +236,22 @@ mod tests { .unwrap_err(); } } + + #[test] + fn tendermint_compatibility() { + for _ in 0..100 { + let leaves_amount = (rand::random::() % 500) + 1; + let leaves: Vec<_> = (0..leaves_amount) + .map(|_| { + let leaf_size = (rand::random::() % 1024) + 1; + random_bytes(leaf_size) + }) + .collect(); + + let (_, root) = MerkleProof::new(0, &leaves).unwrap(); + let tendermint_root = simple_hash_from_byte_vectors::(&leaves); + + assert_eq!(root, tendermint_root); + } + } } diff --git a/types/src/share/proof.rs b/types/src/share/proof.rs index 21a07421..405e9ec5 100644 --- a/types/src/share/proof.rs +++ b/types/src/share/proof.rs @@ -8,6 +8,13 @@ use crate::nmt::NamespaceProof; use crate::{bail_verification, validation_error, RowProof}; use crate::{nmt::Namespace, Error, Result}; +/// A proof of inclusion of a continouous range of shares of some namespace +/// in a [`DataAvailabilityHeader`]. +/// +/// The proof will proof the inclusion of shares in row roots they span +/// and the inclusion of those row roots in the dah. +/// +/// [`DataAvailabilityHeader`]: crate::DataAvailabilityHeader #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(try_from = "RawShareProof", into = "RawShareProof")] pub struct ShareProof { @@ -18,6 +25,22 @@ pub struct ShareProof { } impl ShareProof { + /// Get the shares proven by this proof. + pub fn shares(&self) -> &[[u8; SHARE_SIZE]] { + &self.data + } + + /// Verify the proof against the hash of [`DataAvailabilityHeader`], proving + /// the inclusion of shares. + /// + /// # Errors + /// + /// This function will return an error if: + /// - the proof is malformed, meaning some inconsistency between + /// shares, nmt proofs or row proofs amounts + /// - the verification of any inner row proof fails + /// + /// [`DataAvailabilityHeader`]: crate::DataAvailabilityHeader pub fn verify(&self, root: Hash) -> Result<()> { let row_roots = self.row_proof.row_roots(); From cdc9309808a446778a3720de10b5f60251d68c9f Mon Sep 17 00:00:00 2001 From: zvolin Date: Mon, 9 Sep 2024 14:13:40 +0200 Subject: [PATCH 07/12] missing docs --- types/src/merkle_proof.rs | 1 + types/src/share.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/types/src/merkle_proof.rs b/types/src/merkle_proof.rs index 5955249c..6e7c3684 100644 --- a/types/src/merkle_proof.rs +++ b/types/src/merkle_proof.rs @@ -7,6 +7,7 @@ use crate::{ bail_validation, bail_verification, validation_error, verification_error, Error, Result, }; +/// A proof of inclusion of some leaf in a merkle tree. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(try_from = "RawMerkleProof", into = "RawMerkleProof")] pub struct MerkleProof { diff --git a/types/src/share.rs b/types/src/share.rs index 8733323d..d09790d0 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -186,9 +186,13 @@ impl From for RawNamespacedShares { } } +/// Raw share with no namespace or length checks. +/// +/// To be used as intermediary type in various RPC's. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(transparent)] pub struct RawShare { + /// Data held by the share. #[serde(with = "celestia_tendermint_proto::serializers::bytes::base64string")] pub data: Vec, } From 95907bdd3b51c2652d8743d56810155e785af1d8 Mon Sep 17 00:00:00 2001 From: zvolin Date: Mon, 9 Sep 2024 14:19:16 +0200 Subject: [PATCH 08/12] more missing docs --- rpc/src/share.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpc/src/share.rs b/rpc/src/share.rs index 9b287b76..2199eb46 100644 --- a/rpc/src/share.rs +++ b/rpc/src/share.rs @@ -1,3 +1,5 @@ +//! celestia-node rpc types and methods related to shares +//! use celestia_types::nmt::Namespace; use celestia_types::{ ExtendedDataSquare, ExtendedHeader, NamespacedShares, RawShare, Share, ShareProof, @@ -5,10 +7,13 @@ use celestia_types::{ use jsonrpsee::proc_macros::rpc; use serde::{Deserialize, Serialize}; +/// Response type for [`ShareClient::share_get_range`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct GetRangeResponse { + /// Shares contained in given range. pub shares: Vec, + /// Proof of inclusion of the shares. pub proof: ShareProof, } From 3f10a6ef46b67fc4a48f0e0f6bc6291b564aa4bc Mon Sep 17 00:00:00 2001 From: zvolin Date: Thu, 19 Sep 2024 11:33:07 +0200 Subject: [PATCH 09/12] fix leftover todos and println --- types/src/share/proof.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/types/src/share/proof.rs b/types/src/share/proof.rs index 405e9ec5..a11a6bc9 100644 --- a/types/src/share/proof.rs +++ b/types/src/share/proof.rs @@ -95,14 +95,13 @@ impl TryFrom for ShareProof { type Error = Error; fn try_from(value: RawShareProof) -> Result { - println!("{:?}", value.namespace_id); Ok(Self { data: value .data .into_iter() .map(TryInto::try_into) .collect::>() - .map_err(|_| validation_error!("todo"))?, + .map_err(|_| validation_error!("invalid share size"))?, namespace_id: Namespace::new( value .namespace_version @@ -117,7 +116,7 @@ impl TryFrom for ShareProof { .collect::>()?, row_proof: value .row_proof - .ok_or_else(|| validation_error!("todo"))? + .ok_or_else(|| validation_error!("row proof missing"))? .try_into()?, }) } From 46c2824259fffb693164510bb117be028548317e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Zwoli=C5=84ski?= Date: Thu, 19 Sep 2024 11:34:08 +0200 Subject: [PATCH 10/12] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikołaj Florkiewicz Signed-off-by: Maciej Zwoliński --- rpc/tests/share.rs | 2 +- types/src/data_availability_header.rs | 3 +-- types/src/merkle_proof.rs | 6 +++--- types/src/share/proof.rs | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/rpc/tests/share.rs b/rpc/tests/share.rs index 4ffe85cb..577ad0a5 100644 --- a/rpc/tests/share.rs +++ b/rpc/tests/share.rs @@ -91,7 +91,7 @@ async fn get_shares_range() { } #[tokio::test] -async fn get_shares_range_unexisting() { +async fn get_shares_range_not_existing() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let header = client.header_network_head().await.unwrap(); let shares_in_block = header.dah.square_width().pow(2); diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index dcff55f7..bf09508e 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -302,8 +302,7 @@ impl RowProof { /// # Errors /// /// This function will return an error if: - /// - the proof is malformed, meaning some inconsistency between start/end row, - /// row roots or merkle proofs amounts + /// - the proof is malformed. Number of proofs, row roots and the span between starting and ending row need to match. /// - the verification of any inner merkle proof fails /// /// # Example diff --git a/types/src/merkle_proof.rs b/types/src/merkle_proof.rs index 6e7c3684..c3288124 100644 --- a/types/src/merkle_proof.rs +++ b/types/src/merkle_proof.rs @@ -81,7 +81,7 @@ impl MerkleProof { } } -// creates a merkle tree out of the given leaves and returns it's root hash. +// creates a merkle tree out of the given leaves and returns its root hash. // inner nodes needed to prove leaf with given index are collected in `aunts`. fn hash_leaves_collecting_aunts( // for recursion, index of the first leaf in current subtree wrt the whole tree @@ -117,10 +117,10 @@ fn hash_leaves_collecting_aunts( // if current subtree has the leaf to prove if (first_leaf_index..first_leaf_index + total).contains(&leaf_to_prove) { if leaf_to_prove < first_leaf_index + subtrees_split { - // leaf was in left subtree, so it's aunt is right hash + // leaf was in left subtree, so its aunt is right hash aunts.push(right) } else { - // leaf was in right subtree, so it's aunt is left hash + // leaf was in right subtree, so its aunt is left hash aunts.push(left) } } diff --git a/types/src/share/proof.rs b/types/src/share/proof.rs index a11a6bc9..92d38818 100644 --- a/types/src/share/proof.rs +++ b/types/src/share/proof.rs @@ -36,8 +36,7 @@ impl ShareProof { /// # Errors /// /// This function will return an error if: - /// - the proof is malformed, meaning some inconsistency between - /// shares, nmt proofs or row proofs amounts + /// - the proof is malformed. Number of shares, nmt proofs and row proofs needs to match. /// - the verification of any inner row proof fails /// /// [`DataAvailabilityHeader`]: crate::DataAvailabilityHeader From eef229d5ef0fcba081f04504411e19368f646deb Mon Sep 17 00:00:00 2001 From: zvolin Date: Thu, 19 Sep 2024 12:45:54 +0200 Subject: [PATCH 11/12] update tendermint --- Cargo.lock | 10 ++++++---- Cargo.toml | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9920125d..1813e005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -751,8 +751,9 @@ dependencies = [ [[package]] name = "celestia-tendermint" -version = "0.32.1" -source = "git+https://github.com/zvolin/celestia-tendermint-rs?branch=fix/proof-serialization#c62d61749af98f0adaf9a271b174fda48e5c2461" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8c92a01145f79a0f3ac7c44a43a9b5ee58e8a4c716b56d98833a3848db1afd" dependencies = [ "bytes", "celestia-tendermint-proto", @@ -780,8 +781,9 @@ dependencies = [ [[package]] name = "celestia-tendermint-proto" -version = "0.32.1" -source = "git+https://github.com/zvolin/celestia-tendermint-rs?branch=fix/proof-serialization#c62d61749af98f0adaf9a271b174fda48e5c2461" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a95746c5221a74d7b913a415fdbb9e7c90e1b4d818dbbff59bddc034cfce2ec" dependencies = [ "bytes", "flex-error", diff --git a/Cargo.toml b/Cargo.toml index 7d625256..86ea2f78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,15 +11,15 @@ celestia-rpc = { version = "0.4.0", path = "rpc", default-features = false } celestia-types = { version = "0.4.0", path = "types", default-features = false } libp2p = "0.54.0" nmt-rs = "0.2.1" -celestia-tendermint = { version = "0.32.1", default-features = false } -celestia-tendermint-proto = "0.32.1" +celestia-tendermint = { version = "0.32.2", default-features = false } +celestia-tendermint-proto = "0.32.2" [patch.crates-io] # Uncomment to apply local changes #beetswap = { path = "../beetswap" } #blockstore = { path = "../blockstore" } -celestia-tendermint = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } -celestia-tendermint-proto = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } +#celestia-tendermint = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } +#celestia-tendermint-proto = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } #nmt-rs = { path = "../nmt-rs" } #libp2p = { path = "../../rust-libp2p/libp2p" } #libp2p-core = { path = "../../rust-libp2p/core" } From 326fb31222f84fee0b87a74e6ca4c20d73ffb55a Mon Sep 17 00:00:00 2001 From: zvolin Date: Thu, 19 Sep 2024 12:48:58 +0200 Subject: [PATCH 12/12] restore commented out patches to tendermint --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 86ea2f78..78946b2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ celestia-tendermint-proto = "0.32.2" # Uncomment to apply local changes #beetswap = { path = "../beetswap" } #blockstore = { path = "../blockstore" } -#celestia-tendermint = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } -#celestia-tendermint-proto = { git = "https://github.com/zvolin/celestia-tendermint-rs", branch = "fix/proof-serialization" } +#celestia-tendermint = { path = "../celestia-tendermint-rs/tendermint" } +#celestia-tendermint-proto = { path = "../celestia-tendermint-rs/proto" } #nmt-rs = { path = "../nmt-rs" } #libp2p = { path = "../../rust-libp2p/libp2p" } #libp2p-core = { path = "../../rust-libp2p/core" }