frame/utils: introduce substrate-rpc-client crate for RPC utils

Cargo.toml

@@ -228,6 +228,7 @@ members = [
 	"utils/frame/rpc/system",
 	"utils/frame/generate-bags",
 	"utils/frame/generate-bags/node-runtime",
+	"utils/frame/rpc-utils",
 	"utils/prometheus",
 	"utils/wasm-builder",
 ]

frame/bags-list/remote-tests/src/migration.rs

@@ -24,11 +24,14 @@ use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Test voter bags migration. `currency_unit` is the number of planks per the the runtimes `UNITS`
 /// (i.e. number of decimal places per DOT, KSM etc)
-pub async fn execute<Runtime: RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<Runtime: RuntimeT, Block>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,
-) {
+) where
+	Block: BlockT,
+	Block::Header: DeserializeOwned,
+{
 	let mut ext = Builder::<Block>::new()
 		.mode(Mode::Online(OnlineConfig {
 			transport: ws_url.to_string().into(),

frame/bags-list/remote-tests/src/snapshot.rs

@@ -22,11 +22,14 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute create a snapshot from pallet-staking.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<Runtime: crate::RuntimeT, Block>(
 	voter_limit: Option<usize>,
 	currency_unit: u64,
 	ws_url: String,
-) {
+) where
+	Block: BlockT,
+	Block::Header: DeserializeOwned,
+{
 	use frame_support::storage::generator::StorageMap;
 
 	let mut ext = Builder::<Block>::new()

frame/bags-list/remote-tests/src/try_state.rs

@@ -25,11 +25,14 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute the sanity check of the bags-list.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<Runtime: crate::RuntimeT, Block>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,
-) {
+) where
+	Block: BlockT,
+	Block::Header: DeserializeOwned,
+{
 	let mut ext = Builder::<Block>::new()
 		.mode(Mode::Online(OnlineConfig {
 			transport: ws_url.to_string().into(),

utils/frame/remote-externalities/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
-description = "An externalities provided environemnt that can load itself from remote nodes or cache files"
+description = "An externalities provided environment that can load itself from remote nodes or cached files"
 readme = "README.md"
 
 [package.metadata.docs.rs]
@@ -15,7 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0" }
 env_logger = "0.9"
-jsonrpsee = { version = "0.15.1", features = ["ws-client", "macros"] }
 log = "0.4.17"
 serde = "1.0.136"
 serde_json = "1.0"
@@ -24,6 +23,7 @@ sp-core = { version = "6.0.0", path = "../../../primitives/core" }
 sp-io = { version = "6.0.0", path = "../../../primitives/io" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
 sp-version = { version = "5.0.0", path = "../../../primitives/version" }
+rpc-utils = { path = "../rpc-utils" }
 
 [dev-dependencies]
 tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread"] }

utils/frame/remote-externalities/src/lib.rs

@@ -22,14 +22,8 @@
 
 use codec::{Decode, Encode};
 
-use jsonrpsee::{
-	core::{client::ClientT, Error as RpcError},
-	proc_macros::rpc,
-	rpc_params,
-	ws_client::{WsClient, WsClientBuilder},
-};
-
 use log::*;
+use rpc_utils::{rpc_params, ChainApi, ClientT, StateApi, WsClient};
 use serde::de::DeserializeOwned;
 use sp_core::{
 	hashing::twox_128,
@@ -47,8 +41,6 @@ use std::{
 	sync::Arc,
 };
 
-pub mod rpc_api;
-
 type KeyValue = (StorageKey, StorageData);
 type TopKeyValues = Vec<KeyValue>;
 type ChildKeyValues = Vec<(ChildInfo, Vec<KeyValue>)>;
@@ -58,40 +50,6 @@ const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io:443";
 const BATCH_SIZE: usize = 1000;
 const PAGE: u32 = 1000;
 
-#[rpc(client)]
-pub trait RpcApi<Hash> {
-	#[method(name = "childstate_getKeys")]
-	fn child_get_keys(
-		&self,
-		child_key: PrefixedStorageKey,
-		prefix: StorageKey,
-		hash: Option<Hash>,
-	) -> Result<Vec<StorageKey>, RpcError>;
-
-	#[method(name = "childstate_getStorage")]
-	fn child_get_storage(
-		&self,
-		child_key: PrefixedStorageKey,
-		prefix: StorageKey,
-		hash: Option<Hash>,
-	) -> Result<StorageData, RpcError>;
-
-	#[method(name = "state_getStorage")]
-	fn get_storage(&self, prefix: StorageKey, hash: Option<Hash>) -> Result<StorageData, RpcError>;
-
-	#[method(name = "state_getKeysPaged")]
-	fn get_keys_paged(
-		&self,
-		prefix: Option<StorageKey>,
-		count: u32,
-		start_key: Option<StorageKey>,
-		hash: Option<Hash>,
-	) -> Result<Vec<StorageKey>, RpcError>;
-
-	#[method(name = "chain_getFinalizedHead")]
-	fn finalized_head(&self) -> Result<Hash, RpcError>;
-}
-
 /// The execution mode.
 #[derive(Clone)]
 pub enum Mode<B: BlockT> {
@@ -140,14 +98,10 @@ impl Transport {
 		if let Self::Uri(uri) = self {
 			log::debug!(target: LOG_TARGET, "initializing remote client to {:?}", uri);
 
-			let ws_client = WsClientBuilder::default()
-				.max_request_body_size(u32::MAX)
-				.build(&uri)
-				.await
-				.map_err(|e| {
-					log::error!(target: LOG_TARGET, "error: {:?}", e);
-					"failed to build ws client"
-				})?;
+			let ws_client = rpc_utils::ws_client(uri).await.map_err(|e| {
+				log::error!(target: LOG_TARGET, "error: {:?}", e);
+				"failed to build ws client"
+			})?;
 
 			*self = Self::RemoteClient(Arc::new(ws_client))
 		}
@@ -258,7 +212,7 @@ pub struct Builder<B: BlockT> {
 
 // NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for
 // that.
-impl<B: BlockT + DeserializeOwned> Default for Builder<B> {
+impl<B: BlockT> Default for Builder<B> {
 	fn default() -> Self {
 		Self {
 			mode: Default::default(),
@@ -272,7 +226,7 @@ impl<B: BlockT + DeserializeOwned> Default for Builder<B> {
 }
 
 // Mode methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B> {
 	fn as_online(&self) -> &OnlineConfig<B> {
 		match &self.mode {
 			Mode::Online(config) => config,
@@ -291,26 +245,38 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // RPC methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B>
+where
+	B::Hash: DeserializeOwned,
+	B::Header: DeserializeOwned,
+{
 	async fn rpc_get_storage(
 		&self,
 		key: StorageKey,
 		maybe_at: Option<B::Hash>,
 	) -> Result<StorageData, &'static str> {
 		trace!(target: LOG_TARGET, "rpc: get_storage");
-		self.as_online().rpc_client().get_storage(key, maybe_at).await.map_err(|e| {
-			error!(target: LOG_TARGET, "Error = {:?}", e);
-			"rpc get_storage failed."
-		})
+		match self.as_online().rpc_client().storage(key, maybe_at).await {
+			Ok(Some(res)) => Ok(res),
+			Ok(None) => Err("get_storage not found"),
+			Err(e) => {
+				error!(target: LOG_TARGET, "Error = {:?}", e);
+				Err("rpc get_storage failed.")
+			},
+		}
 	}
 
 	/// Get the latest finalized head.
 	async fn rpc_get_head(&self) -> Result<B::Hash, &'static str> {
 		trace!(target: LOG_TARGET, "rpc: finalized_head");
-		self.as_online().rpc_client().finalized_head().await.map_err(|e| {
-			error!(target: LOG_TARGET, "Error = {:?}", e);
-			"rpc finalized_head failed."
-		})
+
+		// sadly this pretty much unreadable...
+		ChainApi::<(), _, B::Header, ()>::finalized_head(self.as_online().rpc_client())
+			.await
+			.map_err(|e| {
+				error!(target: LOG_TARGET, "Error = {:?}", e);
+				"rpc finalized_head failed."
+			})
 	}
 
 	/// Get all the keys at `prefix` at `hash` using the paged, safe RPC methods.
@@ -325,7 +291,7 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 			let page = self
 				.as_online()
 				.rpc_client()
-				.get_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at))
+				.storage_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at))
 				.await
 				.map_err(|e| {
 					error!(target: LOG_TARGET, "Error = {:?}", e);
@@ -471,19 +437,17 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 		child_prefix: StorageKey,
 		at: B::Hash,
 	) -> Result<Vec<StorageKey>, &'static str> {
-		let child_keys = self
-			.as_online()
-			.rpc_client()
-			.child_get_keys(
-				PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()),
-				child_prefix,
-				Some(at),
-			)
-			.await
-			.map_err(|e| {
-				error!(target: LOG_TARGET, "Error = {:?}", e);
-				"rpc child_get_keys failed."
-			})?;
+		let child_keys = rpc_utils::ChildStateApi::storage_keys(
+			self.as_online().rpc_client(),
+			PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()),
+			child_prefix,
+			Some(at),
+		)
+		.await
+		.map_err(|e| {
+			error!(target: LOG_TARGET, "Error = {:?}", e);
+			"rpc child_get_keys failed."
+		})?;
 
 		debug!(
 			target: LOG_TARGET,
@@ -497,7 +461,11 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // Internal methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B>
+where
+	B::Hash: DeserializeOwned,
+	B::Header: DeserializeOwned,
+{
 	/// Save the given data to the top keys snapshot.
 	fn save_top_snapshot(&self, data: &[KeyValue], path: &PathBuf) -> Result<(), &'static str> {
 		let mut path = path.clone();
@@ -726,12 +694,13 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 
 		let child_kv = match self.mode.clone() {
 			Mode::Online(_) => self.load_child_remote_and_maybe_save(&top_kv).await?,
-			Mode::OfflineOrElseOnline(offline_config, _) =>
+			Mode::OfflineOrElseOnline(offline_config, _) => {
 				if let Ok(kv) = self.load_child_snapshot(&offline_config.state_snapshot.path) {
 					kv
 				} else {
 					self.load_child_remote_and_maybe_save(&top_kv).await?
-				},
+				}
+			},
 			Mode::Offline(ref config) => self
 				.load_child_snapshot(&config.state_snapshot.path)
 				.map_err(|why| {
@@ -749,7 +718,7 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // Public methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B> {
 	/// Create a new builder.
 	pub fn new() -> Self {
 		Default::default()
@@ -824,7 +793,13 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 		}
 		self
 	}
+}
 
+// Public methods
+impl<B: BlockT> Builder<B>
+where
+	B::Header: DeserializeOwned,
+{
 	/// Build the test externalities.
 	pub async fn build(self) -> Result<TestExternalities, &'static str> {
 		let state_version = self.state_version;