rpc v2: backpressure chainHead_v1_storage (#5741)

Close #5589

This PR makes it possible for `rpc_v2::Storage::query_iter_paginated` to
be "backpressured" which is achieved by having a channel where the
result is sent back and when this channel is "full" we pause the
iteration.

The chainHead_follow has an internal channel which doesn't represent the
actual connection and that is set to a very small number (16). Recall
that the JSON-RPC server has a dedicate buffer for each connection by
default of 64.

- Because `archive_storage` also depends on
`rpc_v2::Storage::query_iter_paginated` I had to tweak the method to
support limits as well. The reason is that archive_storage won't get
backpressured properly because it's not an subscription. (it would much
easier if it would be a subscription in rpc v2 spec because nothing
against querying huge amount storage keys)
- `query_iter_paginated` doesn't necessarily return the storage "in
order" such as
- `query_iter_paginated(vec![("key1", hash), ("key2", value)], ...)`
could return them in arbitrary order because it's wrapped in
FuturesUnordered but I could change that if we want to process it
inorder (it's slower)
- there is technically no limit on the number of storage queries in each
`chainHead_v1_storage call` rather than the rpc max message limit which
10MB and only allowed to max 16 calls `chainHead_v1_x` concurrently
(this should be fine)

- Iterate over 10 accounts on westend-dev -> ~2-3x faster
- Fetch 1024 storage values (i.e, not descedant values) -> ~50x faster
- Fetch 1024 descendant values -> ~500x faster

The reason for this is because as Josep explained in the issue is that
one is only allowed query five storage items per call and clients has
make lots of calls to drive it forward..

---------

Co-authored-by: command-bot <>
Co-authored-by: James Wilson <james@jsdw.me>

Cargo.toml

@@ -1313,7 +1313,7 @@ tikv-jemalloc-ctl = { version = "0.5.0" }
 tikv-jemallocator = { version = "0.5.0" }
 time = { version = "0.3" }
 tiny-keccak = { version = "2.0.2" }
-tokio = { version = "1.37.0", default-features = false }
+tokio = { version = "1.40.0", default-features = false }
 tokio-retry = { version = "0.3.0" }
 tokio-stream = { version = "0.1.14" }
 tokio-test = { version = "0.4.2" }

prdoc/pr_5741.prdoc

@@ -0,0 +1,25 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: make RPC endpoint `chainHead_v1_storage` faster
+
+doc:
+  - audience: Node Operator
+    description: |
+      The RPC endpoint `chainHead_v1_storage` now relies solely on backpressure to
+      determine how quickly to serve back values instead of handing back a fixed number
+      of entries and then expecting the client to ask for more. This should improve the 
+      throughput for bigger storage queries significantly.
+
+      Benchmarks using subxt on localhost:
+        - Iterate over 10 accounts on westend-dev -> ~2-3x faster
+        - Fetch 1024 storage values (i.e, not descedant values) -> ~50x faster
+        - Fetch 1024 descendant values -> ~500x faster
+
+crates: 
+  - name: sc-rpc-spec-v2
+    bump: major
+  - name: sc-rpc-server
+    bump: patch
+  - name: sc-service
+    bump: major

substrate/client/rpc-servers/src/lib.rs

@@ -224,8 +224,9 @@ where
 							),
 						};
 
-						let rpc_middleware =
-							RpcServiceBuilder::new().option_layer(middleware_layer.clone());
+						let rpc_middleware = RpcServiceBuilder::new()
+							.rpc_logger(1024)
+							.option_layer(middleware_layer.clone());
 						let mut svc = service_builder
 							.set_rpc_middleware(rpc_middleware)
 							.build(methods, stop_handle);

substrate/client/rpc-spec-v2/Cargo.toml

@@ -17,28 +17,18 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 jsonrpsee = { workspace = true, features = ["client-core", "macros", "server-core"] }
-sc-chain-spec.workspace = true
-sc-chain-spec.default-features = true
-sc-transaction-pool-api.workspace = true
-sc-transaction-pool-api.default-features = true
-sp-core.workspace = true
-sp-core.default-features = true
-sp-runtime.workspace = true
-sp-runtime.default-features = true
-sp-api.workspace = true
-sp-api.default-features = true
-sp-rpc.workspace = true
-sp-rpc.default-features = true
-sp-blockchain.workspace = true
-sp-blockchain.default-features = true
-sp-version.workspace = true
-sp-version.default-features = true
-sc-client-api.workspace = true
-sc-client-api.default-features = true
-sc-utils.workspace = true
-sc-utils.default-features = true
-sc-rpc.workspace = true
-sc-rpc.default-features = true
+# Internal chain structures for "chain_spec".
+sc-chain-spec = { workspace = true, default-features = true }
+# Pool for submitting extrinsics required by "transaction"
+sc-transaction-pool-api = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-runtime = { workspace = true, default-features = true }
+sp-api = { workspace = true, default-features = true }
+sp-rpc = { workspace = true, default-features = true }
+sp-blockchain = { workspace = true, default-features = true }
+sp-version = { workspace = true, default-features = true }
+sc-client-api = { workspace = true, default-features = true }
+sc-rpc = { workspace = true, default-features = true }
 codec = { workspace = true, default-features = true }
 thiserror = { workspace = true }
 serde = { workspace = true, default-features = true }
@@ -57,14 +47,16 @@ schnellru = { workspace = true }
 jsonrpsee = { workspace = true, features = ["server", "ws-client"] }
 serde_json = { workspace = true, default-features = true }
 tokio = { features = ["macros"], workspace = true, default-features = true }
-substrate-test-runtime-client = { path = "../../test-utils/runtime/client" }
-substrate-test-runtime = { path = "../../test-utils/runtime" }
-substrate-test-runtime-transaction-pool = { path = "../../test-utils/runtime/transaction-pool" }
-sp-consensus = { default-features = true, path = "../../primitives/consensus/common" }
-sp-externalities = { default-features = true, path = "../../primitives/externalities" }
-sp-maybe-compressed-blob = { default-features = true, path = "../../primitives/maybe-compressed-blob" }
-sc-block-builder = { default-features = true, path = "../block-builder" }
-sc-service = { features = ["test-helpers"], default-features = true, path = "../service" }
+substrate-test-runtime-client = { workspace = true }
+substrate-test-runtime = { workspace = true }
+substrate-test-runtime-transaction-pool = { workspace = true }
+sp-consensus = { workspace = true, default-features = true }
+sp-externalities = { workspace = true, default-features = true }
+sp-maybe-compressed-blob = { workspace = true, default-features = true }
+sc-block-builder = { workspace = true, default-features = true }
+sc-service = { features = ["test-helpers"], workspace = true, default-features = true }
+sc-rpc = { workspace = true, default-features = true, features = ["test-helpers"] }
 assert_matches = { workspace = true }
 pretty_assertions = { workspace = true }
-sc-transaction-pool = { default-features = true, path = "../transaction-pool" }
+sc-transaction-pool = { workspace = true, default-features = true }
+sc-utils = { workspace = true, default-features = true }

substrate/client/rpc-spec-v2/src/archive/archive.rs

@@ -275,6 +275,7 @@ where
 			self.storage_max_descendant_responses,
 			self.storage_max_queried_items,
 		);
+
 		Ok(storage_client.handle_query(hash, items, child_trie))
 	}
 }