Upgrade to libp2p 0.51.3 (paritytech#13587)

* client/network: upgrade to libp2p 0.51.0 * make discovery.rs compile * make peer_info.rs compile * changes to notifications and request-response proto * make service.rs compile * towards making request_responses.rs compile * make request_responses.rs compile * make request_responses.rs compile * fix notifications/behaviour.rs tests * fix warnings * remove old code * allow deprecated code (temporary) * upgrade to libp2p 0.51.1 * add TODO for behaviour tests * return empty vec if peer_id is absent paritytech#13587 (comment) fyi: I don't really know what the old behaviour was. * update comment to reflect new defaults Closes paritytech#13338 * Revert "update comment to reflect new defaults" This reverts commit 7a981ab. * remove config.rs (from wrong merge) * upgrade to libp2p 0.51.2 * fix formatting * use handle_pending_outbound_connection in networt_state RPC * update deps * use re-exports when we use other libp2p packages * Apply suggestions from code review Co-authored-by: Dmitry Markin <dmitry@markin.tech> * format code * handle potential errors in network_state RPC * only update libp2p crate * update libp2p-core * fix docs * use libp2p-identity instead of libp2p where it's possible. libp2p-identity is much smaller, hence makes sense to use it instead of larger libp2p crate. * Update client/network/src/discovery.rs Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> * update Cargo.lock * add comment for per_connection_event_buffer_size current value is somewhat arbitrary and needs to be tweaked depending on memory usage and network worker sleep stats. * fix link format * update Cargo.lock * upgrade to libp2p 0.51.3 * deprecate mplex * Revert "deprecate mplex" This reverts commit 9e25820. * Revert "upgrade to libp2p 0.51.3" This reverts commit 6544dd4. * use new libp2p version in `statement` crate * pin version temporarily * libp2p 0.51.3 * deprecate mplex * deprecate legacy noise handshake * fix build error * update libp2p-identity * enable libp2p-identity:ed25519 feature in sc-consensus * enable ed25519 for peerset as well --------- Co-authored-by: Dmitry Markin <dmitry@markin.tech> Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> Co-authored-by: parity-processbot <>
nathanwhit · Jul 19, 2023 · 35d3e6f · 35d3e6f
1 parent 7fe74b8
commit 35d3e6f
Show file tree

Hide file tree

Showing 48 changed files with 1,002 additions and 924 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml
@@ -21,7 +21,8 @@ codec = { package = "parity-scale-codec", version = "3.2.2", default-features =
 futures = "0.3.21"
 futures-timer = "3.0.1"
 ip_network = "0.4.1"
-libp2p = { version = "0.50.0", features = ["kad"] }
+libp2p = { version = "0.51.3", features = ["kad", "ed25519"] }
+multihash = { version = "0.17.0", default-features = false, features = ["std", "sha2"] }
 log = "0.4.17"
 prost = "0.11"
 rand = "0.8.5"

diff --git a/client/authority-discovery/src/error.rs b/client/authority-discovery/src/error.rs
@@ -56,7 +56,7 @@ pub enum Error {
 	ParsingMultiaddress(#[from] libp2p::core::multiaddr::Error),
 
 	#[error("Failed to parse a libp2p key.")]
-	ParsingLibp2pIdentity(#[from] libp2p::identity::error::DecodingError),
+	ParsingLibp2pIdentity(#[from] libp2p::identity::DecodingError),
 
 	#[error("Failed to sign: {0}.")]
 	CannotSign(String),

diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs
@@ -25,8 +25,9 @@ use crate::{
 };
 
 use futures::{channel::mpsc::channel, executor::LocalPool, task::LocalSpawn};
-use libp2p::core::{
-	multiaddr::{Multiaddr, Protocol},
+use libp2p::{
+	core::multiaddr::{Multiaddr, Protocol},
+	identity::ed25519,
 	PeerId,
 };
 use std::{collections::HashSet, sync::Arc};
@@ -86,18 +87,16 @@ fn get_addresses_and_authority_id() {
 fn cryptos_are_compatible() {
 	use sp_core::crypto::Pair;
 
-	let libp2p_secret = libp2p::identity::Keypair::generate_ed25519();
-	let libp2p_public = libp2p_secret.public();
+	let libp2p_keypair = ed25519::Keypair::generate();
+	let libp2p_public = libp2p_keypair.public();
 
-	let sp_core_secret = {
-		let libp2p::identity::Keypair::Ed25519(libp2p_ed_secret) = libp2p_secret.clone();
-		sp_core::ed25519::Pair::from_seed_slice(&libp2p_ed_secret.secret().as_ref()).unwrap()
-	};
+	let sp_core_secret =
+		{ sp_core::ed25519::Pair::from_seed_slice(&libp2p_keypair.secret().as_ref()).unwrap() };
 	let sp_core_public = sp_core_secret.public();
 
 	let message = b"we are more powerful than not to be better";
 
-	let libp2p_signature = libp2p_secret.sign(message).unwrap();
+	let libp2p_signature = libp2p_keypair.sign(message);
 	let sp_core_signature = sp_core_secret.sign(message); // no error expected...
 
 	assert!(sp_core::ed25519::Pair::verify(

diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs
@@ -34,11 +34,9 @@ use futures::{channel::mpsc, future, stream::Fuse, FutureExt, Stream, StreamExt}
 use addr_cache::AddrCache;
 use codec::{Decode, Encode};
 use ip_network::IpNetwork;
-use libp2p::{
-	core::multiaddr,
-	multihash::{Multihash, MultihashDigest},
-	Multiaddr, PeerId,
-};
+use libp2p::{core::multiaddr, identity::PublicKey, multihash::Multihash, Multiaddr, PeerId};
+use multihash::{Code, MultihashDigest};
+
 use log::{debug, error, log_enabled};
 use prometheus_endpoint::{register, Counter, CounterVec, Gauge, Opts, U64};
 use prost::Message;
@@ -551,10 +549,8 @@ where
 				// properly signed by the owner of the PeerId
 
 				if let Some(peer_signature) = peer_signature {
-					let public_key = libp2p::identity::PublicKey::from_protobuf_encoding(
-						&peer_signature.public_key,
-					)
-					.map_err(Error::ParsingLibp2pIdentity)?;
+					let public_key = PublicKey::try_decode_protobuf(&peer_signature.public_key)
+						.map_err(Error::ParsingLibp2pIdentity)?;
 					let signature = Signature { public_key, bytes: peer_signature.signature };
 
 					if !signature.verify(record, &remote_peer_id) {
@@ -625,7 +621,7 @@ pub trait NetworkProvider: NetworkDHTProvider + NetworkStateInfo + NetworkSigner
 impl<T> NetworkProvider for T where T: NetworkDHTProvider + NetworkStateInfo + NetworkSigner {}
 
 fn hash_authority_id(id: &[u8]) -> KademliaKey {
-	KademliaKey::new(&libp2p::multihash::Code::Sha2_256.digest(id).digest())
+	KademliaKey::new(&Code::Sha2_256.digest(id).digest())
 }
 
 // Makes sure all values are the same and returns it
@@ -662,7 +658,7 @@ fn sign_record_with_peer_id(
 	let signature = network
 		.sign_with_local_identity(serialized_record)
 		.map_err(|e| Error::CannotSign(format!("{} (network packet)", e)))?;
-	let public_key = signature.public_key.to_protobuf_encoding();
+	let public_key = signature.public_key.encode_protobuf();
 	let signature = signature.bytes;
 	Ok(schema::PeerSignature { signature, public_key })
 }

diff --git a/client/authority-discovery/src/worker/schema/tests.rs b/client/authority-discovery/src/worker/schema/tests.rs
@@ -21,7 +21,7 @@ mod schema_v1 {
 }
 
 use super::*;
-use libp2p::{multiaddr::Multiaddr, PeerId};
+use libp2p::{identity::Keypair, multiaddr::Multiaddr, PeerId};
 use prost::Message;
 
 #[test]
@@ -55,7 +55,7 @@ fn v2_decodes_v1() {
 
 #[test]
 fn v1_decodes_v2() {
-	let peer_secret = libp2p::identity::Keypair::generate_ed25519();
+	let peer_secret = Keypair::generate_ed25519();
 	let peer_public = peer_secret.public();
 	let peer_id = peer_public.to_peer_id();
 	let multiaddress: Multiaddr =
@@ -67,7 +67,7 @@ fn v1_decodes_v2() {
 	let record_v2 = AuthorityRecord { addresses: vec_addresses.clone() };
 	let mut vec_record_v2 = vec![];
 	record_v2.encode(&mut vec_record_v2).unwrap();
-	let vec_peer_public = peer_public.to_protobuf_encoding();
+	let vec_peer_public = peer_public.encode_protobuf();
 	let peer_signature_v2 =
 		PeerSignature { public_key: vec_peer_public, signature: vec_peer_signature };
 	let signed_record_v2 = SignedAuthorityRecord {

diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs
@@ -31,7 +31,7 @@ use futures::{
 };
 use libp2p::{
 	core::multiaddr,
-	identity::{error::SigningError, Keypair},
+	identity::{Keypair, SigningError},
 	kad::record::Key as KademliaKey,
 	PeerId,
 };

diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml
@@ -18,7 +18,7 @@ chrono = "0.4.10"
 clap = { version = "4.2.5", features = ["derive", "string"] }
 fdlimit = "0.2.1"
 futures = "0.3.21"
-libp2p = "0.50.0"
+libp2p-identity = { version = "0.1.2", features = ["peerid", "ed25519"]}
 log = "0.4.17"
 names = { version = "0.13.0", default-features = false }
 parity-scale-codec = "3.2.2"

diff --git a/client/cli/src/commands/generate_node_key.rs b/client/cli/src/commands/generate_node_key.rs
@@ -19,7 +19,7 @@
 
 use crate::Error;
 use clap::Parser;
-use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey};
+use libp2p_identity::{ed25519, Keypair};
 use std::{
 	fs,
 	io::{self, Write},
@@ -48,7 +48,7 @@ pub struct GenerateNodeKeyCmd {
 impl GenerateNodeKeyCmd {
 	/// Run the command
 	pub fn run(&self) -> Result<(), Error> {
-		let keypair = libp2p_ed25519::Keypair::generate();
+		let keypair = ed25519::Keypair::generate();
 
 		let secret = keypair.secret();
 
@@ -63,7 +63,7 @@ impl GenerateNodeKeyCmd {
 			None => io::stdout().lock().write_all(&file_data)?,
 		}
 
-		eprintln!("{}", PublicKey::Ed25519(keypair.public()).to_peer_id());
+		eprintln!("{}", Keypair::from(keypair).public().to_peer_id());
 
 		Ok(())
 	}

diff --git a/client/cli/src/commands/inspect_node_key.rs b/client/cli/src/commands/inspect_node_key.rs
@@ -19,7 +19,7 @@
 
 use crate::Error;
 use clap::Parser;
-use libp2p::identity::{ed25519, PublicKey};
+use libp2p_identity::Keypair;
 use std::{
 	fs,
 	io::{self, Read},
@@ -68,12 +68,10 @@ impl InspectNodeKeyCmd {
 				.map_err(|_| "failed to decode secret as hex")?;
 		}
 
-		let secret =
-			ed25519::SecretKey::from_bytes(&mut file_data).map_err(|_| "Bad node key file")?;
+		let keypair =
+			Keypair::ed25519_from_bytes(&mut file_data).map_err(|_| "Bad node key file")?;
 
-		let keypair = ed25519::Keypair::from(secret);
-
-		println!("{}", PublicKey::Ed25519(keypair.public()).to_peer_id());
+		println!("{}", keypair.public().to_peer_id());
 
 		Ok(())
 	}

diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs
@@ -101,7 +101,7 @@ fn invalid_node_key(e: impl std::fmt::Display) -> error::Error {
 /// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`.
 fn parse_ed25519_secret(hex: &str) -> error::Result<sc_network::config::Ed25519Secret> {
 	H256::from_str(hex).map_err(invalid_node_key).and_then(|bytes| {
-		ed25519::SecretKey::from_bytes(bytes)
+		ed25519::SecretKey::try_from_bytes(bytes)
 			.map(sc_network::config::Secret::Input)
 			.map_err(invalid_node_key)
 	})
@@ -111,7 +111,7 @@ fn parse_ed25519_secret(hex: &str) -> error::Result<sc_network::config::Ed25519S
 mod tests {
 	use super::*;
 	use clap::ValueEnum;
-	use sc_network::config::identity::{ed25519, Keypair};
+	use libp2p_identity::ed25519;
 	use std::fs;
 
 	#[test]
@@ -154,9 +154,12 @@ mod tests {
 				.into_keypair()
 				.expect("Creates node key pair");
 
-			match node_key {
-				Keypair::Ed25519(ref pair) if pair.secret().as_ref() == key.as_ref() => {},
-				_ => panic!("Invalid key"),
+			if let Ok(pair) = node_key.try_into_ed25519() {
+				if pair.secret().as_ref() != key.as_ref() {
+					panic!("Invalid key")
+				}
+			} else {
+				panic!("Invalid key")
 			}
 		}
 

diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml
@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 async-trait = "0.1.57"
 futures = { version = "0.3.21", features = ["thread-pool"] }
 futures-timer = "3.0.1"
-libp2p = "0.50.0"
+libp2p-identity = { version = "0.1.2", features = ["peerid", "ed25519"] }
 log = "0.4.17"
 mockall = "0.11.3"
 parking_lot = "0.12.1"

diff --git a/client/consensus/common/src/import_queue.rs b/client/consensus/common/src/import_queue.rs
@@ -66,7 +66,7 @@ pub type BoxJustificationImport<B> =
 	Box<dyn JustificationImport<B, Error = ConsensusError> + Send + Sync>;
 
 /// Maps to the RuntimeOrigin used by the network.
-pub type RuntimeOrigin = libp2p::PeerId;
+pub type RuntimeOrigin = libp2p_identity::PeerId;
 
 /// Block data used by the queue.
 #[derive(Debug, PartialEq, Eq, Clone)]

diff --git a/client/consensus/common/src/import_queue/basic_queue.rs b/client/consensus/common/src/import_queue/basic_queue.rs
@@ -635,7 +635,7 @@ mod tests {
 			let hash = Hash::random();
 			finality_sender
 				.unbounded_send(worker_messages::ImportJustification(
-					libp2p::PeerId::random(),
+					libp2p_identity::PeerId::random(),
 					hash,
 					1,
 					(*b"TEST", Vec::new()),

diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml
@@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 ahash = "0.8.2"
 futures = "0.3.21"
 futures-timer = "3.0.1"
-libp2p = "0.50.0"
+libp2p = "0.51.3"
 log = "0.4.17"
 lru = "0.8.1"
 tracing = "0.1.29"

diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml
@@ -25,7 +25,7 @@ fnv = "1.0.6"
 futures = "0.3.21"
 futures-timer = "3.0.2"
 ip_network = "0.4.1"
-libp2p = { version = "0.50.0", features = ["dns", "identify", "kad", "macros", "mdns", "mplex", "noise", "ping", "tcp",  "tokio", "yamux", "websocket"] }
+libp2p = { version = "0.51.3", features = ["dns", "identify", "kad", "macros", "mdns", "noise", "ping", "tcp",  "tokio", "yamux", "websocket", "request-response"] }
 linked_hash_set = "0.1.3"
 log = "0.4.17"
 lru = "0.8.1"

diff --git a/client/network/README.md b/client/network/README.md
@@ -66,8 +66,6 @@ negotiated and applied. The exact handshake protocol is experimental and is subj
 
 The following multiplexing protocols are supported:
 
-- [Mplex](https://github.com/libp2p/specs/tree/master/mplex). Support for mplex will likely
-be deprecated in the future.
 - [Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md).
 
 ## Substreams

diff --git a/client/network/bitswap/Cargo.toml b/client/network/bitswap/Cargo.toml
@@ -18,7 +18,7 @@ prost-build = "0.11"
 [dependencies]
 cid = "0.8.6"
 futures = "0.3.21"
-libp2p = "0.50.0"
+libp2p-identity = { version = "0.1.2", features = ["peerid"] }
 log = "0.4.17"
 prost = "0.11"
 thiserror = "1.0"

diff --git a/client/network/bitswap/src/lib.rs b/client/network/bitswap/src/lib.rs
@@ -22,7 +22,7 @@
 
 use cid::{self, Version};
 use futures::{channel::mpsc, StreamExt};
-use libp2p::core::PeerId;
+use libp2p_identity::PeerId;
 use log::{debug, error, trace};
 use prost::Message;
 use sc_client_api::BlockBackend;

diff --git a/client/network/common/Cargo.toml b/client/network/common/Cargo.toml
@@ -25,7 +25,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", features = [
 ] }
 futures = "0.3.21"
 futures-timer = "3.0.2"
-libp2p = { version = "0.50.0", features = ["request-response", "kad"] }
+libp2p-identity = { version = "0.1.2", features = ["peerid"] }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../../utils/prometheus" }
 smallvec = "1.8.0"
 sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" }

diff --git a/client/network/common/src/sync.rs b/client/network/common/src/sync.rs
@@ -25,7 +25,7 @@ pub mod warp;
 use crate::role::Roles;
 use futures::Stream;
 
-use libp2p::PeerId;
+use libp2p_identity::PeerId;
 
 use message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse};
 use sc_consensus::{import_queue::RuntimeOrigin, IncomingBlock};

diff --git a/client/network/light/Cargo.toml b/client/network/light/Cargo.toml
@@ -21,7 +21,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", features = [
     "derive",
 ] }
 futures = "0.3.21"
-libp2p = "0.50.0"
+libp2p-identity = { version = "0.1.2", features = ["peerid"] }
 log = "0.4.16"
 prost = "0.11"
 sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }

diff --git a/client/network/light/src/light_client_requests/handler.rs b/client/network/light/src/light_client_requests/handler.rs
@@ -25,7 +25,7 @@
 use crate::schema;
 use codec::{self, Decode, Encode};
 use futures::{channel::mpsc, prelude::*};
-use libp2p::PeerId;
+use libp2p_identity::PeerId;
 use log::{debug, trace};
 use prost::Message;
 use sc_client_api::{BlockBackend, ProofProvider};

diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs
@@ -28,10 +28,8 @@ use crate::{
 use bytes::Bytes;
 use futures::channel::oneshot;
 use libp2p::{
-	core::{Multiaddr, PeerId, PublicKey},
-	identify::Info as IdentifyInfo,
-	kad::record,
-	swarm::NetworkBehaviour,
+	core::Multiaddr, identify::Info as IdentifyInfo, identity::PublicKey, kad::RecordKey,
+	swarm::NetworkBehaviour, PeerId,
 };
 
 use sc_network_common::role::{ObservedRole, Roles};
@@ -256,13 +254,13 @@ impl<B: BlockT> Behaviour<B> {
 
 	/// Start querying a record from the DHT. Will later produce either a `ValueFound` or a
 	/// `ValueNotFound` event.
-	pub fn get_value(&mut self, key: record::Key) {
+	pub fn get_value(&mut self, key: RecordKey) {
 		self.discovery.get_value(key);
 	}
 
 	/// Starts putting a record into DHT. Will later produce either a `ValuePut` or a
 	/// `ValuePutFailed` event.
-	pub fn put_value(&mut self, key: record::Key, value: Vec<u8>) {
+	pub fn put_value(&mut self, key: RecordKey, value: Vec<u8>) {
 		self.discovery.put_value(key, value);
 	}
 }