feat(stateless_validation): Limit size of outgoing receipts to keep s…

…ize of source_receipt_proofs under control (#11492) This is a basic fix for: #11295 The problem is that the size of `source_receipt_proofs` could be really large in some scenarios. If all 6 shards send a 4MB outgoing receipt to shard 0, then `source_receipt_proofs` for shard 0 will be of size 6 * 4MB = 24MB. That's way too big, the network probably won't be able to distribute that in time. And as we add more shards to the network, the effect will get worse and worse. This fix deals with the problem by allowing only one chosen shard to send large receipts to the other shard. All other shards are only allowed to send ~100kB of receipts. So instead of 6 shards sending 4MB, we end up with 5 shards sending 100kB and one shard sending 4MB, which adds up to 4.5MB, much more manageable. The mapping of "who is able to send a lot of outgoing receipts to whom" changes with every block height: ![image](https://github.com/near/nearcore/assets/149345204/3b571d7c-da24-4cd9-ad8f-19686fc0f055) In this example at block height 2: * shard 0 can send: * 100kB of receipts to shard 0 * 100kB of receipts to shard 1 * 4.5MB of receipts to shard 2 * shard 1 can send: * 4.5MB of receipts to shard 0 * 100kB of receipts to shard 1 * 100kB of receipts to shard 2 * shard 2 can send: * 100kB of receipts to shard 0 * 4.5MB of receipts to shard 1 * 100kB of receipts to shard 2 At every height a receiving shard will receive large receipts from only one shard, so the size of `source_receipt_proofs` stays under control. The mapping changes, so every large receipt will eventually be sent out when the mapping allows to send it to the destination. The new limits are: * `outgoing_receipts_usual_size_limit`: 102_400 (100kiB) * `outgoing_receipts_big_size_limit`: 4_718_592 (4.5MiB - a bit larger than the 4MiB receipt limit to make sure that 4MiB receipts can get through) ### Flaws This is a basic solution which has some flaws. It limits the witness size, but it affects throughput in certain scenarios. It can serve as a starting point for further improvements, something that we can get in before the code freeze. * Flaw 1: big receipts can block small receipts Shard tries to send outgoing receipts in the order in which they were created. When a large receipt is at the front of the queue, the shard won't send anything until it can send out this large receipt. This means that the shard might not send out anything for a few blocks. This could be fixed by having a separate queue for large outgoing receipts. * Flaw 2: missing chunks When a chunk is missing, the next chunk receives receipts from two block heights. This means that it could receive two 4MB receipts. This could be fixed by disallowing sending large receipts to shard that just had missing chunks ### TODO The implementation is pretty much ready, I should probably write some tests, but for now I have to do other stuff. Posting the PR as is for now.
near · Jun 7, 2024 · 43f5a90 · 43f5a90
1 parent be7cb74
commit 43f5a90
Show file tree

Hide file tree

Showing 71 changed files with 424 additions and 138 deletions.
diff --git a/chain/client/src/tests/process_blocks.rs b/chain/client/src/tests/process_blocks.rs
@@ -255,6 +255,8 @@ fn test_bad_congestion_info_corrupt_buffered_receipts_bytes() {
     test_bad_congestion_info_impl(BadCongestionInfoMode::CorruptBufferedReceiptsBytes);
 }
 
+// TODO(congestion_control) validate allowed shard
+#[ignore]
 #[test]
 fn test_bad_congestion_info_corrupt_allowed_shard() {
     test_bad_congestion_info_impl(BadCongestionInfoMode::CorruptAllowedShard);

diff --git a/chain/jsonrpc/res/rpc_errors_schema.json b/chain/jsonrpc/res/rpc_errors_schema.json
@@ -735,6 +735,14 @@
         "method_name": ""
       }
     },
+    "ReceiptSizeExceeded": {
+      "name": "ReceiptSizeExceeded",
+      "subtypes": [],
+      "props": {
+        "limit": "",
+        "size": ""
+      }
+    },
     "ReceiptValidationError": {
       "name": "ReceiptValidationError",
       "subtypes": [
@@ -744,7 +752,8 @@
         "InvalidDataReceiverId",
         "ReturnedValueLengthExceeded",
         "NumberInputDataDependenciesExceeded",
-        "ActionsValidation"
+        "ActionsValidation",
+        "ReceiptSizeExceeded"
       ],
       "props": {}
     },

diff --git a/core/parameters/res/runtime_configs/87.yaml b/core/parameters/res/runtime_configs/87.yaml
@@ -2,6 +2,7 @@
 # State Witness
 
 max_transaction_size: {old: 4_194_304, new: 1_572_864}
+max_receipt_size: {old: 999_999_999_999_999, new: 4_194_304}
 combined_transactions_size_limit: {old: 999_999_999_999_999, new: 2_097_152}
 new_transactions_validation_state_size_soft_limit: {old: 999_999_999_999_999, new: 572_864}
 
@@ -70,3 +71,8 @@ reject_tx_congestion_threshold: {
   new : { numerator: 25, denominator: 100 }
 }
 
+# 100 kiB
+outgoing_receipts_usual_size_limit: {old: 999_999_999_999_999, new: 102_400}
+
+# 4.5 MiB
+outgoing_receipts_big_size_limit: {old: 999_999_999_999_999, new: 4_718_592}
diff --git a/core/parameters/res/runtime_configs/parameters.snap b/core/parameters/res/runtime_configs/parameters.snap
@@ -8,6 +8,8 @@ main_storage_proof_size_soft_limit       999_999_999_999_999
 per_receipt_storage_proof_size_limit     999_999_999_999_999
 new_transactions_validation_state_size_soft_limit 999_999_999_999_999
 combined_transactions_size_limit         999_999_999_999_999
+outgoing_receipts_usual_size_limit       999_999_999_999_999
+outgoing_receipts_big_size_limit         999_999_999_999_999
 min_allowed_top_level_account_length                      65
 registrar_account_id                    registrar
 storage_amount_per_byte                 10000000000000000000
@@ -163,6 +165,7 @@ max_arguments_length                               4_194_304
 max_length_returned_data                           4_194_304
 max_contract_size                                  4_194_304
 max_transaction_size                               4_194_304
+max_receipt_size                         999_999_999_999_999
 max_length_storage_key                                 2_048
 max_length_storage_value                           4_194_304
 max_promises_per_function_call_action                  1_024

diff --git a/core/parameters/res/runtime_configs/parameters.yaml b/core/parameters/res/runtime_configs/parameters.yaml
@@ -17,6 +17,8 @@ main_storage_proof_size_soft_limit: 999_999_999_999_999
 per_receipt_storage_proof_size_limit: 999_999_999_999_999
 combined_transactions_size_limit: 999_999_999_999_999
 new_transactions_validation_state_size_soft_limit: 999_999_999_999_999
+outgoing_receipts_usual_size_limit: 999_999_999_999_999
+outgoing_receipts_big_size_limit: 999_999_999_999_999
 
 # Account creation config
 min_allowed_top_level_account_length: 32
@@ -201,6 +203,7 @@ max_arguments_length: 4_194_304
 max_length_returned_data: 4_194_304
 max_contract_size: 4_194_304
 max_transaction_size: 4_194_304
+max_receipt_size: 999_999_999_999_999
 max_length_storage_key: 4_194_304
 max_length_storage_value: 4_194_304
 max_promises_per_function_call_action: 1_024

diff --git a/core/parameters/res/runtime_configs/parameters_testnet.yaml b/core/parameters/res/runtime_configs/parameters_testnet.yaml
@@ -13,6 +13,8 @@ main_storage_proof_size_soft_limit: 999_999_999_999_999
 per_receipt_storage_proof_size_limit: 999_999_999_999_999
 combined_transactions_size_limit: 999_999_999_999_999
 new_transactions_validation_state_size_soft_limit: 999_999_999_999_999
+outgoing_receipts_usual_size_limit: 999_999_999_999_999
+outgoing_receipts_big_size_limit: 999_999_999_999_999
 
 # Account creation config
 min_allowed_top_level_account_length: 0
@@ -198,6 +200,7 @@ max_arguments_length: 4_194_304
 max_length_returned_data: 4_194_304
 max_contract_size: 4_194_304
 max_transaction_size: 4_194_304
+max_receipt_size: 999_999_999_999_999
 max_length_storage_key: 4_194_304
 max_length_storage_value: 4_194_304
 max_promises_per_function_call_action: 1_024

diff --git a/core/parameters/src/config.rs b/core/parameters/src/config.rs
@@ -179,6 +179,16 @@ pub struct CongestionControlConfig {
     /// How much congestion a shard can tolerate before it stops all shards from
     /// accepting new transactions with the receiver set to the congested shard.
     pub reject_tx_congestion_threshold: f64,
+
+    /// The standard size limit for outgoing receipts aimed at a single shard.
+    /// This limit is pretty small to keep the size of source_receipt_proofs under control.
+    /// It limits the total sum of outgoing receipts, not individual receipts.
+    pub outgoing_receipts_usual_size_limit: u64,
+
+    /// Large size limit for outgoing receipts to a shard, used when it's safe
+    /// to send a lot of receipts without making the state witness too large.
+    /// It limits the total sum of outgoing receipts, not individual receipts.
+    pub outgoing_receipts_big_size_limit: u64,
 }
 
 // The Eq cannot be automatically derived for this class because it contains a
@@ -204,6 +214,8 @@ impl CongestionControlConfig {
             max_tx_gas: max_value,
             min_tx_gas: max_value,
             reject_tx_congestion_threshold: 2.0,
+            outgoing_receipts_usual_size_limit: max_value,
+            outgoing_receipts_big_size_limit: max_value,
         }
     }
 }

diff --git a/core/parameters/src/parameter.rs b/core/parameters/src/parameter.rs
@@ -32,6 +32,14 @@ pub enum Parameter {
     /// A witness contains transactions from both the previous chunk and the current one.
     /// This parameter limits the sum of sizes of transactions from both of those chunks.
     CombinedTransactionsSizeLimit,
+    /// The standard size limit for outgoing receipts aimed at a single shard.
+    /// This limit is pretty small to keep the size of source_receipt_proofs under control.
+    /// It limits the total sum of outgoing receipts, not individual receipts.
+    OutgoingReceiptsUsualSizeLimit,
+    /// Large size limit for outgoing receipts to a shard, used when it's safe
+    /// to send a lot of receipts without making the state witness too large.
+    /// It limits the total sum of outgoing receipts, not individual receipts.
+    OutgoingReceiptsBigSizeLimit,
 
     // Account creation config
     MinAllowedTopLevelAccountLength,
@@ -153,6 +161,7 @@ pub enum Parameter {
     MaxLengthReturnedData,
     MaxContractSize,
     MaxTransactionSize,
+    MaxReceiptSize,
     MaxLengthStorageKey,
     MaxLengthStorageValue,
     MaxPromisesPerFunctionCallAction,
@@ -247,6 +256,7 @@ impl Parameter {
             Parameter::MaxLengthReturnedData,
             Parameter::MaxContractSize,
             Parameter::MaxTransactionSize,
+            Parameter::MaxReceiptSize,
             Parameter::MaxLengthStorageKey,
             Parameter::MaxLengthStorageValue,
             Parameter::MaxPromisesPerFunctionCallAction,

diff --git a/core/parameters/src/parameter_table.rs b/core/parameters/src/parameter_table.rs
@@ -363,6 +363,9 @@ fn get_congestion_control_config(
             let rational: Rational32 = params.get(Parameter::RejectTxCongestionThreshold)?;
             *rational.numer() as f64 / *rational.denom() as f64
         },
+        outgoing_receipts_usual_size_limit: params
+            .get(Parameter::OutgoingReceiptsUsualSizeLimit)?,
+        outgoing_receipts_big_size_limit: params.get(Parameter::OutgoingReceiptsBigSizeLimit)?,
     };
     Ok(congestion_control_config)
 }

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__0.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__0.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -231,7 +232,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__129.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__129.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 1572864,
+      "max_receipt_size": 4194304,
       "max_length_storage_key": 2048,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -233,7 +234,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 1000000000000000,
     "max_tx_gas": 500000000000000,
     "min_tx_gas": 20000000000000,
-    "reject_tx_congestion_threshold": 0.25
+    "reject_tx_congestion_threshold": 0.25,
+    "outgoing_receipts_usual_size_limit": 102400,
+    "outgoing_receipts_big_size_limit": 4718592
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 3000000,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__138.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__138.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 1572864,
+      "max_receipt_size": 4194304,
       "max_length_storage_key": 2048,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -233,7 +234,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 1000000000000000,
     "max_tx_gas": 500000000000000,
     "min_tx_gas": 20000000000000,
-    "reject_tx_congestion_threshold": 0.25
+    "reject_tx_congestion_threshold": 0.25,
+    "outgoing_receipts_usual_size_limit": 102400,
+    "outgoing_receipts_big_size_limit": 4718592
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 3000000,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__35.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__35.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -231,7 +232,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__42.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__42.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -231,7 +232,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__46.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__46.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -231,7 +232,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__48.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__48.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -231,7 +232,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__49.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__49.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -232,7 +233,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__50.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__50.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -232,7 +233,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__52.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__52.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 4194304,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -232,7 +233,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__53.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__53.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 2048,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -233,7 +234,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,

diff --git a/core/parameters/src/snapshots/near_parameters__config_store__tests__55.json.snap b/core/parameters/src/snapshots/near_parameters__config_store__tests__55.json.snap
@@ -206,6 +206,7 @@ expression: config_view
       "max_length_returned_data": 4194304,
       "max_contract_size": 4194304,
       "max_transaction_size": 4194304,
+      "max_receipt_size": 999999999999999,
       "max_length_storage_key": 2048,
       "max_length_storage_value": 4194304,
       "max_promises_per_function_call_action": 1024,
@@ -233,7 +234,9 @@ expression: config_view
     "allowed_shard_outgoing_gas": 9223372036854775807,
     "max_tx_gas": 9223372036854775807,
     "min_tx_gas": 9223372036854775807,
-    "reject_tx_congestion_threshold": 1.0
+    "reject_tx_congestion_threshold": 1.0,
+    "outgoing_receipts_usual_size_limit": 999999999999999,
+    "outgoing_receipts_big_size_limit": 999999999999999
   },
   "witness_config": {
     "main_storage_proof_size_soft_limit": 999999999999999,