From f5deceae4f316c24792271d2007fd58c23a72b4e Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Mon, 12 Dec 2022 11:32:55 +0100 Subject: [PATCH] rpc server with HTTP/WS on the same socket (#12663) * jsonrpsee v0.16 add backwards compatibility run old http server on http only * cargo fmt * update jsonrpsee 0.16.1 * less verbose cors log * fix nit in log: WS -> HTTP * revert needless changes in Cargo.lock * remove unused features in tower * fix nits; add client-core feature * jsonrpsee v0.16.2 --- Cargo.lock | 165 ++++++++++-------- bin/node-template/node/Cargo.toml | 2 +- bin/node/cli/Cargo.toml | 2 +- bin/node/rpc/Cargo.toml | 2 +- client/beefy/rpc/Cargo.toml | 2 +- client/beefy/rpc/src/lib.rs | 2 +- client/consensus/babe/rpc/Cargo.toml | 2 +- client/consensus/manual-seal/Cargo.toml | 2 +- client/finality-grandpa/rpc/Cargo.toml | 2 +- client/finality-grandpa/rpc/src/lib.rs | 2 +- client/merkle-mountain-range/rpc/Cargo.toml | 2 +- client/rpc-api/Cargo.toml | 2 +- client/rpc-servers/Cargo.toml | 5 +- client/rpc-servers/src/lib.rs | 142 ++++++++------- client/rpc-servers/src/middleware.rs | 130 +++++--------- client/rpc-spec-v2/Cargo.toml | 2 +- client/rpc-spec-v2/src/chain_spec/tests.rs | 2 +- client/rpc/Cargo.toml | 2 +- client/rpc/src/author/tests.rs | 2 +- client/rpc/src/chain/tests.rs | 2 +- client/rpc/src/state/mod.rs | 3 +- client/rpc/src/state/tests.rs | 2 +- client/rpc/src/system/tests.rs | 2 +- client/service/Cargo.toml | 2 +- client/service/src/lib.rs | 50 ++---- client/sync-state-rpc/Cargo.toml | 2 +- frame/transaction-payment/rpc/Cargo.toml | 2 +- utils/frame/remote-externalities/src/lib.rs | 120 ++++++++----- utils/frame/rpc/client/Cargo.toml | 2 +- utils/frame/rpc/client/src/lib.rs | 5 +- .../rpc/state-trie-migration-rpc/Cargo.toml | 2 +- utils/frame/rpc/support/Cargo.toml | 4 +- utils/frame/rpc/system/Cargo.toml | 2 +- 33 files changed, 355 insertions(+), 317 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 110f6fcdc9a19..2f2b4aa7cf35d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,7 +288,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", ] [[package]] @@ -2293,7 +2293,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "waker-fn", ] @@ -2350,7 +2350,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "pin-utils", "slab", ] @@ -2667,15 +2667,21 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "httparse" version = "1.8.0" @@ -2710,7 +2716,7 @@ dependencies = [ "httparse", "httpdate", "itoa 1.0.4", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "socket2", "tokio", "tower-service", @@ -2907,24 +2913,23 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd0d559d5e679b1ab2f869b486a11182923863b1b3ee8b421763cdd707b783a" +checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e" dependencies = [ "jsonrpsee-core", - "jsonrpsee-http-server", "jsonrpsee-proc-macros", + "jsonrpsee-server", "jsonrpsee-types", "jsonrpsee-ws-client", - "jsonrpsee-ws-server", "tracing", ] [[package]] name = "jsonrpsee-client-transport" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8752740ecd374bcbf8b69f3e80b0327942df76f793f8d4e60d3355650c31fb74" +checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" dependencies = [ "futures-util", "http", @@ -2943,9 +2948,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3dc3e9cf2ba50b7b1d7d76a667619f82846caa39e8e8daa8a4962d74acaddca" +checksum = "a4e70b4439a751a5de7dd5ed55eacff78ebf4ffe0fc009cb1ebb11417f5b536b" dependencies = [ "anyhow", "arrayvec 0.7.2", @@ -2956,10 +2961,8 @@ dependencies = [ "futures-timer", "futures-util", "globset", - "http", "hyper", "jsonrpsee-types", - "lazy_static", "parking_lot 0.12.1", "rand 0.8.5", "rustc-hash", @@ -2969,45 +2972,48 @@ dependencies = [ "thiserror", "tokio", "tracing", - "tracing-futures", - "unicase", ] [[package]] -name = "jsonrpsee-http-server" -version = "0.15.1" +name = "jsonrpsee-proc-macros" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa6da1e4199c10d7b1d0a6e5e8bd8e55f351163b6f4b3cbb044672a69bd4c1c" +dependencies = [ + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jsonrpsee-server" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03802f0373a38c2420c70b5144742d800b509e2937edc4afb116434f07120117" +checksum = "1fb69dad85df79527c019659a992498d03f8495390496da2f07e6c24c2b356fc" dependencies = [ "futures-channel", "futures-util", + "http", "hyper", "jsonrpsee-core", "jsonrpsee-types", "serde", "serde_json", + "soketto", "tokio", + "tokio-stream", + "tokio-util", + "tower", "tracing", - "tracing-futures", -] - -[[package]] -name = "jsonrpsee-proc-macros" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd67957d4280217247588ac86614ead007b301ca2fa9f19c19f880a536f029e3" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", ] [[package]] name = "jsonrpsee-types" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e290bba767401b646812f608c099b922d8142603c9e73a50fb192d3ac86f4a0d" +checksum = "5bd522fe1ce3702fd94812965d7bb7a3364b1c9aba743944c5a00529aae80f8c" dependencies = [ "anyhow", "beef", @@ -3019,9 +3025,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee5feddd5188e62ac08fcf0e56478138e581509d4730f3f7be9b57dd402a4ff" +checksum = "0b83daeecfc6517cfe210df24e570fb06213533dfb990318fae781f4c7119dd9" dependencies = [ "http", "jsonrpsee-client-transport", @@ -3029,26 +3035,6 @@ dependencies = [ "jsonrpsee-types", ] -[[package]] -name = "jsonrpsee-ws-server" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d488ba74fb369e5ab68926feb75a483458b88e768d44319f37e4ecad283c7325" -dependencies = [ - "futures-channel", - "futures-util", - "http", - "jsonrpsee-core", - "jsonrpsee-types", - "serde_json", - "soketto", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "tracing-futures", -] - [[package]] name = "k256" version = "0.11.5" @@ -6334,9 +6320,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -8110,11 +8096,14 @@ name = "sc-rpc-server" version = "4.0.0-dev" dependencies = [ "futures", + "http", "jsonrpsee", "log", "serde_json", "substrate-prometheus-endpoint", "tokio", + "tower", + "tower-http", ] [[package]] @@ -8822,6 +8811,7 @@ dependencies = [ "bytes", "flate2", "futures", + "http", "httparse", "log", "rand 0.8.5", @@ -10297,7 +10287,7 @@ dependencies = [ "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "signal-hook-registry", "socket2", "tokio-macros", @@ -10333,7 +10323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "tokio", ] @@ -10360,7 +10350,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "pin-project-lite 0.2.6", + "pin-project-lite 0.2.9", "tokio", "tracing", ] @@ -10374,6 +10364,41 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite 0.2.9", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.1" @@ -10387,7 +10412,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if", - "pin-project-lite 0.2.6", + "log", + "pin-project-lite 0.2.9", "tracing-attributes", "tracing-core", ] @@ -10657,15 +10683,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.4" diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index d609edc88401d..2ea841093d0e2 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -43,7 +43,7 @@ frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } # These dependencies are used for the node template's RPCs -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } sc-rpc = { version = "4.0.0-dev", path = "../../../client/rpc" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sc-rpc-api = { version = "0.10.0-dev", path = "../../../client/rpc-api" } diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 2ca37e7febe16..4ee4bcd033921 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -39,7 +39,7 @@ array-bytes = "4.1" clap = { version = "4.0.9", features = ["derive"], optional = true } codec = { package = "parity-scale-codec", version = "3.0.0" } serde = { version = "1.0.136", features = ["derive"] } -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } futures = "0.3.21" log = "0.4.17" rand = "0.8" diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index 9d2810413613f..f34922a287dfe 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -13,7 +13,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } node-primitives = { version = "2.0.0", path = "../primitives" } pallet-transaction-payment-rpc = { version = "4.0.0-dev", path = "../../../frame/transaction-payment/rpc/" } mmr-rpc = { version = "4.0.0-dev", path = "../../../client/merkle-mountain-range/rpc/" } diff --git a/client/beefy/rpc/Cargo.toml b/client/beefy/rpc/Cargo.toml index d27225824539a..f5b5770153477 100644 --- a/client/beefy/rpc/Cargo.toml +++ b/client/beefy/rpc/Cargo.toml @@ -11,7 +11,7 @@ homepage = "https://substrate.io" [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } futures = "0.3.21" -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } log = "0.4" parking_lot = "0.12.1" serde = { version = "1.0.136", features = ["derive"] } diff --git a/client/beefy/rpc/src/lib.rs b/client/beefy/rpc/src/lib.rs index d29ed433c38db..59a133b86214e 100644 --- a/client/beefy/rpc/src/lib.rs +++ b/client/beefy/rpc/src/lib.rs @@ -172,7 +172,7 @@ mod tests { }; use beefy_primitives::{known_payloads, Payload, SignedCommitment}; use codec::{Decode, Encode}; - use jsonrpsee::{types::EmptyParams, RpcModule}; + use jsonrpsee::{types::EmptyServerParams as EmptyParams, RpcModule}; use sp_runtime::traits::{BlakeTwo256, Hash}; use substrate_test_runtime_client::runtime::Block; diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index d0a65a3fc3193..4f5aaf85494b9 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -13,7 +13,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } futures = "0.3.21" serde = { version = "1.0.136", features = ["derive"] } thiserror = "1.0" diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index cf151424c2ee5..fb89445a97002 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -13,7 +13,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } assert_matches = "1.3.0" async-trait = "0.1.57" codec = { package = "parity-scale-codec", version = "3.0.0" } diff --git a/client/finality-grandpa/rpc/Cargo.toml b/client/finality-grandpa/rpc/Cargo.toml index 7be77c122bab2..252c5e3871a64 100644 --- a/client/finality-grandpa/rpc/Cargo.toml +++ b/client/finality-grandpa/rpc/Cargo.toml @@ -12,7 +12,7 @@ homepage = "https://substrate.io" [dependencies] finality-grandpa = { version = "0.16.0", features = ["derive-codec"] } futures = "0.3.16" -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } log = "0.4.8" parity-scale-codec = { version = "3.0.0", features = ["derive"] } serde = { version = "1.0.105", features = ["derive"] } diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs index 85df72de77b54..dfdad666ba8f3 100644 --- a/client/finality-grandpa/rpc/src/lib.rs +++ b/client/finality-grandpa/rpc/src/lib.rs @@ -138,7 +138,7 @@ mod tests { use std::{collections::HashSet, convert::TryInto, sync::Arc}; use jsonrpsee::{ - types::{EmptyParams, SubscriptionId}, + types::{EmptyServerParams as EmptyParams, SubscriptionId}, RpcModule, }; use parity_scale_codec::{Decode, Encode}; diff --git a/client/merkle-mountain-range/rpc/Cargo.toml b/client/merkle-mountain-range/rpc/Cargo.toml index ca14544000bdb..dcc5e49c52051 100644 --- a/client/merkle-mountain-range/rpc/Cargo.toml +++ b/client/merkle-mountain-range/rpc/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0" } -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } serde = { version = "1.0.136", features = ["derive"] } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index cb82a3b26706b..c46488db2d8e1 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -28,4 +28,4 @@ sp-rpc = { version = "6.0.0", path = "../../primitives/rpc" } sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" } sp-tracing = { version = "6.0.0", path = "../../primitives/tracing" } sp-version = { version = "5.0.0", path = "../../primitives/version" } -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["server", "client-core", "macros"] } diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index a3e64c367afb6..b494749ffd26a 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -14,8 +14,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.21" -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } log = "0.4.17" serde_json = "1.0.85" tokio = { version = "1.22.0", features = ["parking_lot"] } prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } +tower-http = { version = "0.3.4", features = ["cors"] } +tower = "0.4.13" +http = "0.2.8" diff --git a/client/rpc-servers/src/lib.rs b/client/rpc-servers/src/lib.rs index 7eb825e169bfa..1fa2ba81d8672 100644 --- a/client/rpc-servers/src/lib.rs +++ b/client/rpc-servers/src/lib.rs @@ -21,17 +21,21 @@ #![warn(missing_docs)] use jsonrpsee::{ - http_server::{AccessControlBuilder, HttpServerBuilder, HttpServerHandle}, - ws_server::{WsServerBuilder, WsServerHandle}, + server::{ + middleware::proxy_get_request::ProxyGetRequestLayer, AllowHosts, ServerBuilder, + ServerHandle, + }, RpcModule, }; use std::{error::Error as StdError, net::SocketAddr}; -pub use crate::middleware::{RpcMetrics, RpcMiddleware}; +pub use crate::middleware::RpcMetrics; +use http::header::HeaderValue; pub use jsonrpsee::core::{ id_providers::{RandomIntegerIdProvider, RandomStringIdProvider}, traits::IdProvider, }; +use tower_http::cors::{AllowOrigin, CorsLayer}; const MEGABYTE: usize = 1024 * 1024; @@ -46,12 +50,11 @@ const WS_MAX_SUBS_PER_CONN: usize = 1024; pub mod middleware; -/// Type alias for http server -pub type HttpServer = HttpServerHandle; -/// Type alias for ws server -pub type WsServer = WsServerHandle; +/// Type alias JSON-RPC server +pub type Server = ServerHandle; -/// WebSocket specific settings on the server. +/// Server config. +#[derive(Debug, Clone)] pub struct WsConfig { /// Maximum connections. pub max_connections: Option, @@ -67,8 +70,8 @@ impl WsConfig { // Deconstructs the config to get the finalized inner values. // // `Payload size` or `max subs per connection` bigger than u32::MAX will be truncated. - fn deconstruct(self) -> (u32, u32, u64, u32) { - let max_conns = self.max_connections.unwrap_or(WS_MAX_CONNECTIONS) as u64; + fn deconstruct(self) -> (u32, u32, u32, u32) { + let max_conns = self.max_connections.unwrap_or(WS_MAX_CONNECTIONS) as u32; let max_payload_in_mb = payload_size_or_default(self.max_payload_in_mb) as u32; let max_payload_out_mb = payload_size_or_default(self.max_payload_out_mb) as u32; let max_subs_per_conn = self.max_subs_per_conn.unwrap_or(WS_MAX_SUBS_PER_CONN) as u32; @@ -86,31 +89,27 @@ pub async fn start_http( metrics: Option, rpc_api: RpcModule, rt: tokio::runtime::Handle, -) -> Result> { - let max_payload_in = payload_size_or_default(max_payload_in_mb); - let max_payload_out = payload_size_or_default(max_payload_out_mb); +) -> Result> { + let max_payload_in = payload_size_or_default(max_payload_in_mb) as u32; + let max_payload_out = payload_size_or_default(max_payload_out_mb) as u32; + let host_filter = hosts_filter(cors.is_some(), &addrs); - let mut acl = AccessControlBuilder::new(); + let middleware = tower::ServiceBuilder::new() + // Proxy `GET /health` requests to internal `system_health` method. + .layer(ProxyGetRequestLayer::new("/health", "system_health")?) + .layer(try_into_cors(cors)?); - if let Some(cors) = cors { - // Whitelist listening address. - // NOTE: set_allowed_hosts will whitelist both ports but only one will used. - acl = acl.set_allowed_hosts(format_allowed_hosts(&addrs[..]))?; - acl = acl.set_allowed_origins(cors)?; - }; - - let builder = HttpServerBuilder::new() - .max_request_body_size(max_payload_in as u32) - .max_response_body_size(max_payload_out as u32) - .set_access_control(acl.build()) - .health_api("/health", "system_health")? - .custom_tokio_runtime(rt); + let builder = ServerBuilder::new() + .max_request_body_size(max_payload_in) + .max_response_body_size(max_payload_out) + .set_host_filtering(host_filter) + .set_middleware(middleware) + .custom_tokio_runtime(rt) + .http_only(); let rpc_api = build_rpc_api(rpc_api); let (handle, addr) = if let Some(metrics) = metrics { - let middleware = RpcMiddleware::new(metrics, "http".into()); - let builder = builder.set_middleware(middleware); - let server = builder.build(&addrs[..]).await?; + let server = builder.set_logger(metrics).build(&addrs[..]).await?; let addr = server.local_addr(); (server.start(rpc_api)?, addr) } else { @@ -120,16 +119,16 @@ pub async fn start_http( }; log::info!( - "Running JSON-RPC HTTP server: addr={}, allowed origins={:?}", + "Running JSON-RPC HTTP server: addr={}, allowed origins={}", addr.map_or_else(|_| "unknown".to_string(), |a| a.to_string()), - cors + format_cors(cors) ); Ok(handle) } -/// Start WS server listening on given address. -pub async fn start_ws( +/// Start a JSON-RPC server listening on given address that supports both HTTP and WS. +pub async fn start( addrs: [SocketAddr; 2], cors: Option<&Vec>, ws_config: WsConfig, @@ -137,27 +136,26 @@ pub async fn start_ws( rpc_api: RpcModule, rt: tokio::runtime::Handle, id_provider: Option>, -) -> Result> { +) -> Result> { let (max_payload_in, max_payload_out, max_connections, max_subs_per_conn) = ws_config.deconstruct(); - let mut acl = AccessControlBuilder::new(); + let host_filter = hosts_filter(cors.is_some(), &addrs); - if let Some(cors) = cors { - // Whitelist listening address. - // NOTE: set_allowed_hosts will whitelist both ports but only one will used. - acl = acl.set_allowed_hosts(format_allowed_hosts(&addrs[..]))?; - acl = acl.set_allowed_origins(cors)?; - }; + let middleware = tower::ServiceBuilder::new() + // Proxy `GET /health` requests to internal `system_health` method. + .layer(ProxyGetRequestLayer::new("/health", "system_health")?) + .layer(try_into_cors(cors)?); - let mut builder = WsServerBuilder::new() + let mut builder = ServerBuilder::new() .max_request_body_size(max_payload_in) .max_response_body_size(max_payload_out) .max_connections(max_connections) .max_subscriptions_per_connection(max_subs_per_conn) .ping_interval(std::time::Duration::from_secs(30)) - .custom_tokio_runtime(rt) - .set_access_control(acl.build()); + .set_host_filtering(host_filter) + .set_middleware(middleware) + .custom_tokio_runtime(rt); if let Some(provider) = id_provider { builder = builder.set_id_provider(provider); @@ -167,9 +165,7 @@ pub async fn start_ws( let rpc_api = build_rpc_api(rpc_api); let (handle, addr) = if let Some(metrics) = metrics { - let middleware = RpcMiddleware::new(metrics, "ws".into()); - let builder = builder.set_middleware(middleware); - let server = builder.build(&addrs[..]).await?; + let server = builder.set_logger(metrics).build(&addrs[..]).await?; let addr = server.local_addr(); (server.start(rpc_api)?, addr) } else { @@ -179,23 +175,14 @@ pub async fn start_ws( }; log::info!( - "Running JSON-RPC WS server: addr={}, allowed origins={:?}", + "Running JSON-RPC WS server: addr={}, allowed origins={}", addr.map_or_else(|_| "unknown".to_string(), |a| a.to_string()), - cors + format_cors(cors) ); Ok(handle) } -fn format_allowed_hosts(addrs: &[SocketAddr]) -> Vec { - let mut hosts = Vec::with_capacity(addrs.len() * 2); - for addr in addrs { - hosts.push(format!("localhost:{}", addr.port())); - hosts.push(format!("127.0.0.1:{}", addr.port())); - } - hosts -} - fn build_rpc_api(mut rpc_api: RpcModule) -> RpcModule { let mut available_methods = rpc_api.method_names().collect::>(); available_methods.sort(); @@ -214,3 +201,40 @@ fn build_rpc_api(mut rpc_api: RpcModule) -> RpcModu fn payload_size_or_default(size_mb: Option) -> usize { size_mb.map_or(RPC_MAX_PAYLOAD_DEFAULT, |mb| mb.saturating_mul(MEGABYTE)) } + +fn hosts_filter(enabled: bool, addrs: &[SocketAddr]) -> AllowHosts { + if enabled { + // NOTE The listening addresses are whitelisted by default. + let mut hosts = Vec::with_capacity(addrs.len() * 2); + for addr in addrs { + hosts.push(format!("localhost:{}", addr.port()).into()); + hosts.push(format!("127.0.0.1:{}", addr.port()).into()); + } + AllowHosts::Only(hosts) + } else { + AllowHosts::Any + } +} + +fn try_into_cors( + maybe_cors: Option<&Vec>, +) -> Result> { + if let Some(cors) = maybe_cors { + let mut list = Vec::new(); + for origin in cors { + list.push(HeaderValue::from_str(origin)?); + } + Ok(CorsLayer::new().allow_origin(AllowOrigin::list(list))) + } else { + // allow all cors + Ok(CorsLayer::permissive()) + } +} + +fn format_cors(maybe_cors: Option<&Vec>) -> String { + if let Some(cors) = maybe_cors { + format!("{:?}", cors) + } else { + format!("{:?}", ["*"]) + } +} diff --git a/client/rpc-servers/src/middleware.rs b/client/rpc-servers/src/middleware.rs index 9e0d422b2350e..1c25ac1dfd1b3 100644 --- a/client/rpc-servers/src/middleware.rs +++ b/client/rpc-servers/src/middleware.rs @@ -16,9 +16,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! RPC middlware to collect prometheus metrics on RPC calls. +//! RPC middleware to collect prometheus metrics on RPC calls. -use jsonrpsee::core::middleware::{Headers, HttpMiddleware, MethodKind, Params, WsMiddleware}; +use jsonrpsee::server::logger::{HttpRequest, Logger, MethodKind, Params, TransportProtocol}; use prometheus_endpoint::{ register, Counter, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, U64, @@ -54,9 +54,9 @@ pub struct RpcMetrics { calls_started: CounterVec, /// Number of calls completed. calls_finished: CounterVec, - /// Number of Websocket sessions opened (Websocket only). + /// Number of Websocket sessions opened. ws_sessions_opened: Option>, - /// Number of Websocket sessions closed (Websocket only). + /// Number of Websocket sessions closed. ws_sessions_closed: Option>, } @@ -139,62 +139,61 @@ impl RpcMetrics { } } -#[derive(Clone)] -/// Middleware for RPC calls -pub struct RpcMiddleware { - metrics: RpcMetrics, - transport_label: &'static str, -} +impl Logger for RpcMetrics { + type Instant = std::time::Instant; -impl RpcMiddleware { - /// Create a new [`RpcMiddleware`] with the provided [`RpcMetrics`]. - pub fn new(metrics: RpcMetrics, transport_label: &'static str) -> Self { - Self { metrics, transport_label } + fn on_connect( + &self, + _remote_addr: SocketAddr, + _request: &HttpRequest, + transport: TransportProtocol, + ) { + if let TransportProtocol::WebSocket = transport { + self.ws_sessions_opened.as_ref().map(|counter| counter.inc()); + } } - /// Called when a new JSON-RPC request comes to the server. - fn on_request(&self) -> std::time::Instant { + fn on_request(&self, transport: TransportProtocol) -> Self::Instant { + let transport_label = transport_label_str(transport); let now = std::time::Instant::now(); - self.metrics.requests_started.with_label_values(&[self.transport_label]).inc(); + self.requests_started.with_label_values(&[transport_label]).inc(); now } - /// Called on each JSON-RPC method call, batch requests will trigger `on_call` multiple times. - fn on_call(&self, name: &str, params: Params, kind: MethodKind) { + fn on_call(&self, name: &str, params: Params, kind: MethodKind, transport: TransportProtocol) { + let transport_label = transport_label_str(transport); log::trace!( target: "rpc_metrics", "[{}] on_call name={} params={:?} kind={}", - self.transport_label, + transport_label, name, params, kind, ); - self.metrics - .calls_started - .with_label_values(&[self.transport_label, name]) - .inc(); + self.calls_started.with_label_values(&[transport_label, name]).inc(); } - /// Called on each JSON-RPC method completion, batch requests will trigger `on_result` multiple - /// times. - fn on_result(&self, name: &str, success: bool, started_at: std::time::Instant) { + fn on_result( + &self, + name: &str, + success: bool, + started_at: Self::Instant, + transport: TransportProtocol, + ) { + let transport_label = transport_label_str(transport); let micros = started_at.elapsed().as_micros(); log::debug!( target: "rpc_metrics", "[{}] {} call took {} μs", - self.transport_label, + transport_label, name, micros, ); - self.metrics - .calls_time - .with_label_values(&[self.transport_label, name]) - .observe(micros as _); + self.calls_time.with_label_values(&[transport_label, name]).observe(micros as _); - self.metrics - .calls_finished + self.calls_finished .with_label_values(&[ - self.transport_label, + transport_label, name, // the label "is_error", so `success` should be regarded as false // and vice-versa to be registrered correctly. @@ -203,58 +202,23 @@ impl RpcMiddleware { .inc(); } - /// Called once the JSON-RPC request is finished and response is sent to the output buffer. - fn on_response(&self, result: &str, started_at: std::time::Instant) { - log::trace!(target: "rpc_metrics", "[{}] on_response started_at={:?}", self.transport_label, started_at); - log::trace!(target: "rpc_metrics::extra", "[{}] result={:?}", self.transport_label, result); - self.metrics.requests_finished.with_label_values(&[self.transport_label]).inc(); - } -} - -impl WsMiddleware for RpcMiddleware { - type Instant = std::time::Instant; - - fn on_connect(&self, _remote_addr: SocketAddr, _headers: &Headers) { - self.metrics.ws_sessions_opened.as_ref().map(|counter| counter.inc()); - } - - fn on_request(&self) -> Self::Instant { - self.on_request() - } - - fn on_call(&self, name: &str, params: Params, kind: MethodKind) { - self.on_call(name, params, kind) + fn on_response(&self, result: &str, started_at: Self::Instant, transport: TransportProtocol) { + let transport_label = transport_label_str(transport); + log::trace!(target: "rpc_metrics", "[{}] on_response started_at={:?}", transport_label, started_at); + log::trace!(target: "rpc_metrics::extra", "[{}] result={:?}", transport_label, result); + self.requests_finished.with_label_values(&[transport_label]).inc(); } - fn on_result(&self, name: &str, success: bool, started_at: Self::Instant) { - self.on_result(name, success, started_at) - } - - fn on_response(&self, _result: &str, started_at: Self::Instant) { - self.on_response(_result, started_at) - } - - fn on_disconnect(&self, _remote_addr: SocketAddr) { - self.metrics.ws_sessions_closed.as_ref().map(|counter| counter.inc()); + fn on_disconnect(&self, _remote_addr: SocketAddr, transport: TransportProtocol) { + if let TransportProtocol::WebSocket = transport { + self.ws_sessions_closed.as_ref().map(|counter| counter.inc()); + } } } -impl HttpMiddleware for RpcMiddleware { - type Instant = std::time::Instant; - - fn on_request(&self, _remote_addr: SocketAddr, _headers: &Headers) -> Self::Instant { - self.on_request() - } - - fn on_call(&self, name: &str, params: Params, kind: MethodKind) { - self.on_call(name, params, kind) - } - - fn on_result(&self, name: &str, success: bool, started_at: Self::Instant) { - self.on_result(name, success, started_at) - } - - fn on_response(&self, _result: &str, started_at: Self::Instant) { - self.on_response(_result, started_at) +fn transport_label_str(t: TransportProtocol) -> &'static str { + match t { + TransportProtocol::Http => "http", + TransportProtocol::WebSocket => "ws", } } diff --git a/client/rpc-spec-v2/Cargo.toml b/client/rpc-spec-v2/Cargo.toml index a0ae3038378ff..930aeb4bd8956 100644 --- a/client/rpc-spec-v2/Cargo.toml +++ b/client/rpc-spec-v2/Cargo.toml @@ -13,7 +13,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } # Internal chain structures for "chain_spec". sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } # Pool for submitting extrinsics required by "transaction" diff --git a/client/rpc-spec-v2/src/chain_spec/tests.rs b/client/rpc-spec-v2/src/chain_spec/tests.rs index 6c078b2974e98..6f662ba422bc4 100644 --- a/client/rpc-spec-v2/src/chain_spec/tests.rs +++ b/client/rpc-spec-v2/src/chain_spec/tests.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use super::*; -use jsonrpsee::{types::EmptyParams, RpcModule}; +use jsonrpsee::{types::EmptyServerParams as EmptyParams, RpcModule}; use sc_chain_spec::Properties; const CHAIN_NAME: &'static str = "TEST_CHAIN_NAME"; diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index d690e2c7b4cf1..a241807cc242b 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.0.0" } futures = "0.3.21" hash-db = { version = "0.15.2", default-features = false } -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } lazy_static = { version = "1.4.0", optional = true } log = "0.4.17" parking_lot = "0.12.1" diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index f969812e5b14c..573d01630de32 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -23,7 +23,7 @@ use assert_matches::assert_matches; use codec::Encode; use jsonrpsee::{ core::Error as RpcError, - types::{error::CallError, EmptyParams}, + types::{error::CallError, EmptyServerParams as EmptyParams}, RpcModule, }; use sc_transaction_pool::{BasicPool, FullChainApi}; diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index 1e6dbd5aca148..224d021f9409e 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -19,7 +19,7 @@ use super::*; use crate::testing::{test_executor, timeout_secs}; use assert_matches::assert_matches; -use jsonrpsee::types::EmptyParams; +use jsonrpsee::types::EmptyServerParams as EmptyParams; use sc_block_builder::BlockBuilderProvider; use sp_consensus::BlockOrigin; use sp_rpc::list::ListOrValue; diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 7213e4360ae2b..fd802e5a80391 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -28,9 +28,8 @@ use std::sync::Arc; use crate::SubscriptionTaskExecutor; use jsonrpsee::{ - core::{Error as JsonRpseeError, RpcResult}, + core::{server::rpc_module::SubscriptionSink, Error as JsonRpseeError, RpcResult}, types::SubscriptionResult, - ws_server::SubscriptionSink, }; use sc_rpc_api::{state::ReadProof, DenyUnsafe}; diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 53dd8ebf50499..3ef59e5ca9a7c 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -23,7 +23,7 @@ use assert_matches::assert_matches; use futures::executor; use jsonrpsee::{ core::Error as RpcError, - types::{error::CallError as RpcCallError, EmptyParams, ErrorObject}, + types::{error::CallError as RpcCallError, EmptyServerParams as EmptyParams, ErrorObject}, }; use sc_block_builder::BlockBuilderProvider; use sc_rpc_api::DenyUnsafe; diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index 2f91648008ff7..00ab9c46861e2 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -21,7 +21,7 @@ use assert_matches::assert_matches; use futures::prelude::*; use jsonrpsee::{ core::Error as RpcError, - types::{error::CallError, EmptyParams}, + types::{error::CallError, EmptyServerParams as EmptyParams}, RpcModule, }; use sc_network::{self, config::Role, PeerId}; diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 87949ef12d888..4d1d267d45c97 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -22,7 +22,7 @@ test-helpers = [] runtime-benchmarks = ["sc-client-db/runtime-benchmarks"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["server"] } thiserror = "1.0.30" futures = "0.3.21" rand = "0.7.3" diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 091b4bbe9fe5f..f0e3f72510c28 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -43,7 +43,6 @@ use log::{debug, error, warn}; use sc_client_api::{blockchain::HeaderBackend, BlockBackend, BlockchainEvents, ProofProvider}; use sc_network::PeerId; use sc_network_common::{config::MultiaddrWithPeerId, service::NetworkBlock}; -use sc_rpc_server::WsConfig; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; use sp_consensus::SyncOracle; @@ -294,20 +293,9 @@ async fn build_network_future< // Wrapper for HTTP and WS servers that makes sure they are properly shut down. mod waiting { - pub struct HttpServer(pub Option); + pub struct Server(pub Option); - impl Drop for HttpServer { - fn drop(&mut self) { - if let Some(server) = self.0.take() { - // This doesn't not wait for the server to be stopped but fires the signal. - let _ = server.stop(); - } - } - } - - pub struct WsServer(pub Option); - - impl Drop for WsServer { + impl Drop for Server { fn drop(&mut self) { if let Some(server) = self.0.take() { // This doesn't not wait for the server to be stopped but fires the signal. @@ -326,9 +314,6 @@ fn start_rpc_servers( where R: Fn(sc_rpc::DenyUnsafe) -> Result, Error>, { - let (max_request_size, ws_max_response_size, http_max_response_size) = - legacy_cli_parsing(config); - fn deny_unsafe(addr: SocketAddr, methods: &RpcMethods) -> sc_rpc::DenyUnsafe { let is_exposed_addr = !addr.ip().is_loopback(); match (is_exposed_addr, methods) { @@ -337,6 +322,9 @@ where } } + let (max_request_size, ws_max_response_size, http_max_response_size) = + legacy_cli_parsing(config); + let random_port = |mut addr: SocketAddr| { addr.set_port(0); addr @@ -346,6 +334,7 @@ where .rpc_ws .unwrap_or_else(|| "127.0.0.1:9944".parse().expect("valid sockaddr; qed")); let ws_addr2 = random_port(ws_addr); + let http_addr = config .rpc_http .unwrap_or_else(|| "127.0.0.1:9933".parse().expect("valid sockaddr; qed")); @@ -353,29 +342,29 @@ where let metrics = sc_rpc_server::RpcMetrics::new(config.prometheus_registry())?; + let server_config = sc_rpc_server::WsConfig { + max_connections: config.rpc_ws_max_connections, + max_payload_in_mb: max_request_size, + max_payload_out_mb: ws_max_response_size, + max_subs_per_conn: config.rpc_max_subs_per_conn, + }; + let http_fut = sc_rpc_server::start_http( [http_addr, http_addr2], config.rpc_cors.as_ref(), max_request_size, http_max_response_size, metrics.clone(), - gen_rpc_module(deny_unsafe(ws_addr, &config.rpc_methods))?, + gen_rpc_module(deny_unsafe(http_addr, &config.rpc_methods))?, config.tokio_handle.clone(), ); - let ws_config = WsConfig { - max_connections: config.rpc_ws_max_connections, - max_payload_in_mb: max_request_size, - max_payload_out_mb: ws_max_response_size, - max_subs_per_conn: config.rpc_max_subs_per_conn, - }; - - let ws_fut = sc_rpc_server::start_ws( + let ws_fut = sc_rpc_server::start( [ws_addr, ws_addr2], config.rpc_cors.as_ref(), - ws_config, - metrics, - gen_rpc_module(deny_unsafe(http_addr, &config.rpc_methods))?, + server_config.clone(), + metrics.clone(), + gen_rpc_module(deny_unsafe(ws_addr, &config.rpc_methods))?, config.tokio_handle.clone(), rpc_id_provider, ); @@ -383,8 +372,7 @@ where match tokio::task::block_in_place(|| { config.tokio_handle.block_on(futures::future::try_join(http_fut, ws_fut)) }) { - Ok((http, ws)) => - Ok(Box::new((waiting::HttpServer(Some(http)), waiting::WsServer(Some(ws))))), + Ok((http, ws)) => Ok(Box::new((waiting::Server(Some(http)), waiting::Server(Some(ws))))), Err(e) => Err(Error::Application(e)), } } diff --git a/client/sync-state-rpc/Cargo.toml b/client/sync-state-rpc/Cargo.toml index 9730eb56e9bd6..a72b4106ba873 100644 --- a/client/sync-state-rpc/Cargo.toml +++ b/client/sync-state-rpc/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0" } -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.85" thiserror = "1.0.30" diff --git a/frame/transaction-payment/rpc/Cargo.toml b/frame/transaction-payment/rpc/Cargo.toml index 06dcaca937381..b77143201ffd4 100644 --- a/frame/transaction-payment/rpc/Cargo.toml +++ b/frame/transaction-payment/rpc/Cargo.toml @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0" } -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } pallet-transaction-payment-rpc-runtime-api = { version = "4.0.0-dev", path = "./runtime-api" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 86cfc767bf3b5..4f95331c03bc8 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -39,7 +39,9 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use substrate_rpc_client::{rpc_params, ws_client, ChainApi, ClientT, StateApi, WsClient}; +use substrate_rpc_client::{ + rpc_params, ws_client, BatchRequestBuilder, ChainApi, ClientT, StateApi, WsClient, +}; type KeyValue = (StorageKey, StorageData); type TopKeyValues = Vec; @@ -334,33 +336,50 @@ where log::debug!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); let mut key_values: Vec = vec![]; + let mut batch_success = true; + let client = self.as_online().rpc_client(); for chunk_keys in keys.chunks(BATCH_SIZE) { - let batch = chunk_keys - .iter() - .cloned() - .map(|key| ("state_getStorage", rpc_params![key, at])) - .collect::>(); - - let values = client.batch_request::>(batch).await.map_err(|e| { - log::error!( - target: LOG_TARGET, - "failed to execute batch: {:?}. Error: {:?}", - chunk_keys.iter().map(HexDisplay::from).collect::>(), - e - ); - "batch failed." - })?; + let mut batch = BatchRequestBuilder::new(); - assert_eq!(chunk_keys.len(), values.len()); + for key in chunk_keys.iter() { + batch + .insert("state_getStorage", rpc_params![key, at]) + .map_err(|_| "Invalid batch params")?; + } + + let batch_response = + client.batch_request::>(batch).await.map_err(|e| { + log::error!( + target: LOG_TARGET, + "failed to execute batch: {:?}. Error: {:?}", + chunk_keys.iter().map(HexDisplay::from).collect::>(), + e + ); + "batch failed." + })?; + + assert_eq!(chunk_keys.len(), batch_response.len()); + + for (key, maybe_value) in chunk_keys.into_iter().zip(batch_response) { + match maybe_value { + Ok(Some(v)) => { + key_values.push((key.clone(), v)); + }, + Ok(None) => { + log::warn!( + target: LOG_TARGET, + "key {:?} had none corresponding value.", + &key + ); + key_values.push((key.clone(), StorageData(vec![]))); + }, + Err(e) => { + log::error!(target: LOG_TARGET, "key {:?} failed: {:?}", &key, e); + batch_success = false; + }, + }; - for (idx, key) in chunk_keys.iter().enumerate() { - let maybe_value = values[idx].clone(); - let value = maybe_value.unwrap_or_else(|| { - log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key); - StorageData(vec![]) - }); - key_values.push((key.clone(), value)); if key_values.len() % (10 * BATCH_SIZE) == 0 { let ratio: f64 = key_values.len() as f64 / keys_count as f64; log::debug!( @@ -374,7 +393,11 @@ where } } - Ok(key_values) + if batch_success { + Ok(key_values) + } else { + Err("batch failed.") + } } /// Get the values corresponding to `child_keys` at the given `prefixed_top_key`. @@ -385,12 +408,14 @@ where at: B::Hash, ) -> Result, &'static str> { let mut child_kv_inner = vec![]; + let mut batch_success = true; + for batch_child_key in child_keys.chunks(BATCH_SIZE) { - let batch_request = batch_child_key - .iter() - .cloned() - .map(|key| { - ( + let mut batch_request = BatchRequestBuilder::new(); + + for key in batch_child_key { + batch_request + .insert( "childstate_getStorage", rpc_params![ PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()), @@ -398,8 +423,8 @@ where at ], ) - }) - .collect::>(); + .map_err(|_| "Invalid batch params")?; + } let batch_response = self .as_online() @@ -418,17 +443,32 @@ where assert_eq!(batch_child_key.len(), batch_response.len()); - for (idx, key) in batch_child_key.iter().enumerate() { - let maybe_value = batch_response[idx].clone(); - let value = maybe_value.unwrap_or_else(|| { - log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key); - StorageData(vec![]) - }); - child_kv_inner.push((key.clone(), value)); + for (key, maybe_value) in batch_child_key.iter().zip(batch_response) { + match maybe_value { + Ok(Some(v)) => { + child_kv_inner.push((key.clone(), v)); + }, + Ok(None) => { + log::warn!( + target: LOG_TARGET, + "key {:?} had none corresponding value.", + &key + ); + child_kv_inner.push((key.clone(), StorageData(vec![]))); + }, + Err(e) => { + log::error!(target: LOG_TARGET, "key {:?} failed: {:?}", &key, e); + batch_success = false; + }, + }; } } - Ok(child_kv_inner) + if batch_success { + Ok(child_kv_inner) + } else { + Err("batch failed.") + } } pub(crate) async fn rpc_child_get_keys( diff --git a/utils/frame/rpc/client/Cargo.toml b/utils/frame/rpc/client/Cargo.toml index bbe8879818092..ee9982971cee3 100644 --- a/utils/frame/rpc/client/Cargo.toml +++ b/utils/frame/rpc/client/Cargo.toml @@ -12,7 +12,7 @@ description = "Shared JSON-RPC client" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee = { version = "0.15.1", features = ["ws-client"] } +jsonrpsee = { version = "0.16.2", features = ["ws-client"] } sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } async-trait = "0.1.57" serde = "1" diff --git a/utils/frame/rpc/client/src/lib.rs b/utils/frame/rpc/client/src/lib.rs index 254cc193c0e67..a211fc6c6983e 100644 --- a/utils/frame/rpc/client/src/lib.rs +++ b/utils/frame/rpc/client/src/lib.rs @@ -43,7 +43,10 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::collections::VecDeque; pub use jsonrpsee::{ - core::client::{ClientT, Subscription, SubscriptionClientT}, + core::{ + client::{ClientT, Subscription, SubscriptionClientT}, + params::BatchRequestBuilder, + }, rpc_params, ws_client::{WsClient, WsClientBuilder}, }; diff --git a/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml index 4886563a99440..3a1b4b8b6cbf8 100644 --- a/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml +++ b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -25,7 +25,7 @@ sp-state-machine = { path = "../../../../primitives/state-machine" } sp-trie = { path = "../../../../primitives/trie" } trie-db = "0.24.0" -jsonrpsee = { version = "0.15.1", features = ["server", "macros"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } # Substrate Dependencies sc-client-api = { version = "4.0.0-dev", path = "../../../../client/api" } diff --git a/utils/frame/rpc/support/Cargo.toml b/utils/frame/rpc/support/Cargo.toml index 119acbd937c8a..d098877e7302c 100644 --- a/utils/frame/rpc/support/Cargo.toml +++ b/utils/frame/rpc/support/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0" } futures = "0.3.21" -jsonrpsee = { version = "0.15.1", features = ["jsonrpsee-types"] } +jsonrpsee = { version = "0.16.2", features = ["jsonrpsee-types"] } serde = "1" frame-support = { version = "4.0.0-dev", path = "../../../../frame/support" } sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } @@ -25,7 +25,7 @@ sp-storage = { version = "7.0.0", path = "../../../../primitives/storage" } [dev-dependencies] scale-info = "2.1.1" -jsonrpsee = { version = "0.15.1", features = ["ws-client", "jsonrpsee-types"] } +jsonrpsee = { version = "0.16.2", features = ["ws-client", "jsonrpsee-types"] } tokio = "1.22.0" sp-core = { version = "7.0.0", path = "../../../../primitives/core" } sp-runtime = { version = "7.0.0", path = "../../../../primitives/runtime" } diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml index 56b8a79f8c080..92cf6882a10f1 100644 --- a/utils/frame/rpc/system/Cargo.toml +++ b/utils/frame/rpc/system/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde_json = "1" codec = { package = "parity-scale-codec", version = "3.0.0" } -jsonrpsee = { version = "0.15.1", features = ["server"] } +jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } futures = "0.3.21" log = "0.4.17" frame-system-rpc-runtime-api = { version = "4.0.0-dev", path = "../../../../frame/system/rpc/runtime-api" }