From e38ed545887fd001c6c71562453a347e2e304415 Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Thu, 5 Jan 2023 12:27:12 +0000 Subject: [PATCH 1/5] refactor: Store parameters as YAML --- Cargo.lock | 11 +++++----- Cargo.toml | 1 + core/primitives/Cargo.toml | 1 + .../{parameters.txt => parameters.yaml} | 2 +- ...rs_testnet.txt => parameters_testnet.yaml} | 2 +- core/primitives/src/runtime/config_store.rs | 4 ++-- .../primitives/src/runtime/parameter_table.rs | 22 ++++++++++++++----- 7 files changed, 28 insertions(+), 15 deletions(-) rename core/primitives/res/runtime_configs/{parameters.txt => parameters.yaml} (99%) rename core/primitives/res/runtime_configs/{parameters_testnet.txt => parameters_testnet.yaml} (99%) diff --git a/Cargo.lock b/Cargo.lock index b78479e1b53..62d2c0ce5e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2252,12 +2252,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", - "hashbrown 0.11.2", + "hashbrown 0.12.1", "serde", ] @@ -3396,6 +3396,7 @@ dependencies = [ "reed-solomon-erasure", "serde", "serde_json", + "serde_yaml", "smart-default", "strum", "thiserror", @@ -5229,9 +5230,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ "indexmap", "ryu", diff --git a/Cargo.toml b/Cargo.toml index a28f130a534..69bad9d1353 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,6 +170,7 @@ serde = { version = "1.0.136", features = ["alloc", "derive", "rc"] } serde_ignored = "0.1" serde_json = "1.0.68" serde_repr = "0.1.8" +serde_yaml = "0.8.26" sha2 = "0.10" sha3 = "0.10" shell-escape = "0.1.5" diff --git a/core/primitives/Cargo.toml b/core/primitives/Cargo.toml index 1a249b991b1..2a6f756c41a 100644 --- a/core/primitives/Cargo.toml +++ b/core/primitives/Cargo.toml @@ -29,6 +29,7 @@ rand.workspace = true reed-solomon-erasure.workspace = true serde.workspace = true serde_json.workspace = true +serde_yaml.workspace = true smart-default.workspace = true stdx.workspace = true strum.workspace = true diff --git a/core/primitives/res/runtime_configs/parameters.txt b/core/primitives/res/runtime_configs/parameters.yaml similarity index 99% rename from core/primitives/res/runtime_configs/parameters.txt rename to core/primitives/res/runtime_configs/parameters.yaml index c4ab8633ab6..eacbc558b06 100644 --- a/core/primitives/res/runtime_configs/parameters.txt +++ b/core/primitives/res/runtime_configs/parameters.yaml @@ -10,7 +10,7 @@ pessimistic_gas_price_inflation_denominator: 100 # Account creation config min_allowed_top_level_account_length: 32 -registrar_account_id: registrar +registrar_account_id: "registrar" # Storage usage config storage_amount_per_byte: 100_000_000_000_000_000_000 diff --git a/core/primitives/res/runtime_configs/parameters_testnet.txt b/core/primitives/res/runtime_configs/parameters_testnet.yaml similarity index 99% rename from core/primitives/res/runtime_configs/parameters_testnet.txt rename to core/primitives/res/runtime_configs/parameters_testnet.yaml index 158cce82351..84c72591272 100644 --- a/core/primitives/res/runtime_configs/parameters_testnet.txt +++ b/core/primitives/res/runtime_configs/parameters_testnet.yaml @@ -6,7 +6,7 @@ pessimistic_gas_price_inflation_denominator: 100 # Account creation config min_allowed_top_level_account_length: 0 -registrar_account_id: registrar +registrar_account_id: "registrar" # Storage usage config storage_amount_per_byte: 100_000_000_000_000_000_000 diff --git a/core/primitives/src/runtime/config_store.rs b/core/primitives/src/runtime/config_store.rs index 46bdc750095..748a4e6c5dc 100644 --- a/core/primitives/src/runtime/config_store.rs +++ b/core/primitives/src/runtime/config_store.rs @@ -13,7 +13,7 @@ macro_rules! include_config { /// The base config file with all initial parameter values defined. /// Later version are calculated by applying diffs to this base. -static BASE_CONFIG: &str = include_config!("parameters.txt"); +static BASE_CONFIG: &str = include_config!("parameters.yaml"); /// Stores pairs of protocol versions for which runtime config was updated and /// the file containing the diffs in bytes. @@ -31,7 +31,7 @@ static CONFIG_DIFFS: &[(ProtocolVersion, &str)] = &[ ]; /// Testnet parameters for versions <= 29, which (incorrectly) differed from mainnet parameters -pub static INITIAL_TESTNET_CONFIG: &str = include_config!("parameters_testnet.txt"); +pub static INITIAL_TESTNET_CONFIG: &str = include_config!("parameters_testnet.yaml"); /// Stores runtime config for each protocol version where it was updated. #[derive(Debug)] diff --git a/core/primitives/src/runtime/parameter_table.rs b/core/primitives/src/runtime/parameter_table.rs index d040df0d4be..f1cbf67e3da 100644 --- a/core/primitives/src/runtime/parameter_table.rs +++ b/core/primitives/src/runtime/parameter_table.rs @@ -4,6 +4,7 @@ use near_primitives_core::parameter::{FeeParameter, Parameter}; use near_primitives_core::runtime::fees::{RuntimeFeesConfig, StorageUsageConfig}; use num_rational::Rational; use serde::de::DeserializeOwned; +use serde::Deserialize; use serde_json::json; use std::any::Any; use std::collections::BTreeMap; @@ -29,6 +30,8 @@ pub(crate) enum InvalidConfigError { NoSeparator(usize, String), #[error("intermediate JSON created by parser does not match `RuntimeConfig`")] WrongStructure(#[source] serde_json::Error), + #[error("could not parse YAML that defines the structure of the config")] + InvalidYaml(#[source] serde_yaml::Error), #[error("config diff expected to contain old value `{1}` for parameter `{0}`")] OldValueExists(Parameter, String), #[error( @@ -46,12 +49,19 @@ pub(crate) enum InvalidConfigError { impl std::str::FromStr for ParameterTable { type Err = InvalidConfigError; fn from_str(arg: &str) -> Result { - let parameters = txt_to_key_values(arg) - .map(|result| { - let (typed_key, value) = result?; - Ok((typed_key, parse_parameter_txt_value(value.trim())?)) + let yaml_map: BTreeMap = + serde_yaml::from_str(arg).map_err(|err| InvalidConfigError::InvalidYaml(err))?; + + let parameters = yaml_map + .iter() + .map(|(key, value)| { + let typed_key: Parameter = key + .parse() + .map_err(|err| InvalidConfigError::UnknownParameter(err, key.to_owned()))?; + Ok((typed_key, parse_parameter_txt_value(value)?)) }) .collect::, _>>()?; + Ok(ParameterTable { parameters }) } } @@ -487,7 +497,7 @@ max_memory_pages: 512 fn test_parameter_table_no_key() { assert_matches!( check_invalid_parameter_table(": 100", &[]), - InvalidConfigError::UnknownParameter(_, _) + InvalidConfigError::InvalidYaml(_) ); } @@ -503,7 +513,7 @@ max_memory_pages: 512 fn test_parameter_table_wrong_separator() { assert_matches!( check_invalid_parameter_table("wasm_regular_op_cost=100", &[]), - InvalidConfigError::NoSeparator(1, _) + InvalidConfigError::InvalidYaml(_) ); } From 1161f00226ff2ac991df2f0f0b180d1c63d4292c Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Mon, 9 Jan 2023 09:20:18 +0000 Subject: [PATCH 2/5] fix: Remove duplicate definition of `stack_limiter_version` The parameters.txt contained the same parameter twice, which in my understanding is a noop, so removing a duplicate definition --- core/primitives/res/runtime_configs/parameters.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/core/primitives/res/runtime_configs/parameters.yaml b/core/primitives/res/runtime_configs/parameters.yaml index eacbc558b06..f6224056621 100644 --- a/core/primitives/res/runtime_configs/parameters.yaml +++ b/core/primitives/res/runtime_configs/parameters.yaml @@ -156,5 +156,4 @@ max_length_storage_key: 4_194_304 max_length_storage_value: 4_194_304 max_promises_per_function_call_action: 1_024 max_number_input_data_dependencies: 128 -stack_limiter_version: 0 account_id_validity_rules_version: 0 From dc938fa7dfbb9ec52ead47df06c4dd448015a710 Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Mon, 9 Jan 2023 10:27:34 +0000 Subject: [PATCH 3/5] refactor: Migrate parameter diff configs to YAML --- core/primitives/res/runtime_configs/42.txt | 1 - core/primitives/res/runtime_configs/42.yaml | 1 + core/primitives/res/runtime_configs/48.txt | 8 -- core/primitives/res/runtime_configs/48.yaml | 8 ++ core/primitives/res/runtime_configs/49.txt | 2 - core/primitives/res/runtime_configs/49.yaml | 2 + core/primitives/res/runtime_configs/50.txt | 1 - core/primitives/res/runtime_configs/50.yaml | 1 + core/primitives/res/runtime_configs/52.txt | 2 - core/primitives/res/runtime_configs/52.yaml | 2 + core/primitives/res/runtime_configs/53.txt | 4 - core/primitives/res/runtime_configs/53.yaml | 4 + core/primitives/res/runtime_configs/57.txt | 1 - core/primitives/res/runtime_configs/57.yaml | 1 + core/primitives/src/runtime/config_store.rs | 14 +-- .../primitives/src/runtime/parameter_table.rs | 105 ++++++++---------- 16 files changed, 72 insertions(+), 85 deletions(-) delete mode 100644 core/primitives/res/runtime_configs/42.txt create mode 100644 core/primitives/res/runtime_configs/42.yaml delete mode 100644 core/primitives/res/runtime_configs/48.txt create mode 100644 core/primitives/res/runtime_configs/48.yaml delete mode 100644 core/primitives/res/runtime_configs/49.txt create mode 100644 core/primitives/res/runtime_configs/49.yaml delete mode 100644 core/primitives/res/runtime_configs/50.txt create mode 100644 core/primitives/res/runtime_configs/50.yaml delete mode 100644 core/primitives/res/runtime_configs/52.txt create mode 100644 core/primitives/res/runtime_configs/52.yaml delete mode 100644 core/primitives/res/runtime_configs/53.txt create mode 100644 core/primitives/res/runtime_configs/53.yaml delete mode 100644 core/primitives/res/runtime_configs/57.txt create mode 100644 core/primitives/res/runtime_configs/57.yaml diff --git a/core/primitives/res/runtime_configs/42.txt b/core/primitives/res/runtime_configs/42.txt deleted file mode 100644 index 875589ae8f4..00000000000 --- a/core/primitives/res/runtime_configs/42.txt +++ /dev/null @@ -1 +0,0 @@ -storage_amount_per_byte: 100_000_000_000_000_000_000 -> 10_000_000_000_000_000_000 diff --git a/core/primitives/res/runtime_configs/42.yaml b/core/primitives/res/runtime_configs/42.yaml new file mode 100644 index 00000000000..8f79cc5cd19 --- /dev/null +++ b/core/primitives/res/runtime_configs/42.yaml @@ -0,0 +1 @@ +storage_amount_per_byte: { old: 100_000_000_000_000_000_000, new: 10_000_000_000_000_000_000 } diff --git a/core/primitives/res/runtime_configs/48.txt b/core/primitives/res/runtime_configs/48.txt deleted file mode 100644 index 830213d8917..00000000000 --- a/core/primitives/res/runtime_configs/48.txt +++ /dev/null @@ -1,8 +0,0 @@ -wasm_regular_op_cost: 3_856_371 -> 2_207_874 -wasm_ecrecover_base: 3_365_369_625_000 -> 278_821_988_457 -data_receipt_creation_base_send_sir: 4_697_339_419_375 -> 36_486_732_312 -data_receipt_creation_base_send_not_sir: 4_697_339_419_375 -> 36_486_732_312 -data_receipt_creation_base_execution: 4_697_339_419_375 -> 36_486_732_312 -data_receipt_creation_per_byte_send_sir: 59_357_464 -> 17_212_011 -data_receipt_creation_per_byte_send_not_sir: 59_357_464 -> 17_212_011 -data_receipt_creation_per_byte_execution: 59_357_464 -> 17_212_011 diff --git a/core/primitives/res/runtime_configs/48.yaml b/core/primitives/res/runtime_configs/48.yaml new file mode 100644 index 00000000000..bb6807967a3 --- /dev/null +++ b/core/primitives/res/runtime_configs/48.yaml @@ -0,0 +1,8 @@ +wasm_regular_op_cost: { old: 3_856_371, new: 2_207_874 } +wasm_ecrecover_base: { old: 3_365_369_625_000, new: 278_821_988_457 } +data_receipt_creation_base_send_sir: { old: 4_697_339_419_375, new: 36_486_732_312 } +data_receipt_creation_base_send_not_sir: { old: 4_697_339_419_375, new: 36_486_732_312 } +data_receipt_creation_base_execution: { old: 4_697_339_419_375, new: 36_486_732_312 } +data_receipt_creation_per_byte_send_sir: { old: 59_357_464, new: 17_212_011 } +data_receipt_creation_per_byte_send_not_sir: { old: 59_357_464, new: 17_212_011 } +data_receipt_creation_per_byte_execution: { old: 59_357_464, new: 17_212_011 } diff --git a/core/primitives/res/runtime_configs/49.txt b/core/primitives/res/runtime_configs/49.txt deleted file mode 100644 index 141d0c82aa1..00000000000 --- a/core/primitives/res/runtime_configs/49.txt +++ /dev/null @@ -1,2 +0,0 @@ -wasm_regular_op_cost: 2_207_874 -> 822_756 -max_functions_number_per_contract: 10_000 diff --git a/core/primitives/res/runtime_configs/49.yaml b/core/primitives/res/runtime_configs/49.yaml new file mode 100644 index 00000000000..c88d0db7997 --- /dev/null +++ b/core/primitives/res/runtime_configs/49.yaml @@ -0,0 +1,2 @@ +wasm_regular_op_cost: { old: 2_207_874, new: 822_756 } +max_functions_number_per_contract: { new: 10_000 } diff --git a/core/primitives/res/runtime_configs/50.txt b/core/primitives/res/runtime_configs/50.txt deleted file mode 100644 index 807d57581dc..00000000000 --- a/core/primitives/res/runtime_configs/50.txt +++ /dev/null @@ -1 +0,0 @@ -stack_limiter_version: 0 -> 1 diff --git a/core/primitives/res/runtime_configs/50.yaml b/core/primitives/res/runtime_configs/50.yaml new file mode 100644 index 00000000000..9c266cff266 --- /dev/null +++ b/core/primitives/res/runtime_configs/50.yaml @@ -0,0 +1 @@ +stack_limiter_version: { old: 0, new: 1 } diff --git a/core/primitives/res/runtime_configs/52.txt b/core/primitives/res/runtime_configs/52.txt deleted file mode 100644 index e5f54a0351a..00000000000 --- a/core/primitives/res/runtime_configs/52.txt +++ /dev/null @@ -1,2 +0,0 @@ -max_gas_burnt: 200_000_000_000_000 -> 300_000_000_000_000 -max_gas_burnt_view: 200_000_000_000_000 -> 300_000_000_000_000 diff --git a/core/primitives/res/runtime_configs/52.yaml b/core/primitives/res/runtime_configs/52.yaml new file mode 100644 index 00000000000..cbd07cdaf62 --- /dev/null +++ b/core/primitives/res/runtime_configs/52.yaml @@ -0,0 +1,2 @@ +max_gas_burnt: { old: 200_000_000_000_000, new: 300_000_000_000_000 } +max_gas_burnt_view: { old: 200_000_000_000_000, new: 300_000_000_000_000 } diff --git a/core/primitives/res/runtime_configs/53.txt b/core/primitives/res/runtime_configs/53.txt deleted file mode 100644 index 13bd07041f3..00000000000 --- a/core/primitives/res/runtime_configs/53.txt +++ /dev/null @@ -1,4 +0,0 @@ -action_deploy_contract_per_byte_execution: 6_812_999 -> 64_572_944 -wasmer2_stack_limit: 204_800 -max_length_storage_key: 4_194_304 -> 2_048 -max_locals_per_contract: 1_000_000 diff --git a/core/primitives/res/runtime_configs/53.yaml b/core/primitives/res/runtime_configs/53.yaml new file mode 100644 index 00000000000..e322b4507cf --- /dev/null +++ b/core/primitives/res/runtime_configs/53.yaml @@ -0,0 +1,4 @@ +action_deploy_contract_per_byte_execution: { old: 6_812_999, new: 64_572_944 } +wasmer2_stack_limit: { new: 204_800 } +max_length_storage_key: { old: 4_194_304, new: 2_048 } +max_locals_per_contract: { new: 1_000_000 } diff --git a/core/primitives/res/runtime_configs/57.txt b/core/primitives/res/runtime_configs/57.txt deleted file mode 100644 index ad517496ca9..00000000000 --- a/core/primitives/res/runtime_configs/57.txt +++ /dev/null @@ -1 +0,0 @@ -account_id_validity_rules_version: 0 -> 1 diff --git a/core/primitives/res/runtime_configs/57.yaml b/core/primitives/res/runtime_configs/57.yaml new file mode 100644 index 00000000000..436ee3e0807 --- /dev/null +++ b/core/primitives/res/runtime_configs/57.yaml @@ -0,0 +1 @@ +account_id_validity_rules_version: { old: 0, new: 1 } diff --git a/core/primitives/src/runtime/config_store.rs b/core/primitives/src/runtime/config_store.rs index 748a4e6c5dc..15d0cb185c0 100644 --- a/core/primitives/src/runtime/config_store.rs +++ b/core/primitives/src/runtime/config_store.rs @@ -18,16 +18,16 @@ static BASE_CONFIG: &str = include_config!("parameters.yaml"); /// Stores pairs of protocol versions for which runtime config was updated and /// the file containing the diffs in bytes. static CONFIG_DIFFS: &[(ProtocolVersion, &str)] = &[ - (42, include_config!("42.txt")), - (48, include_config!("48.txt")), - (49, include_config!("49.txt")), - (50, include_config!("50.txt")), + (42, include_config!("42.yaml")), + (48, include_config!("48.yaml")), + (49, include_config!("49.yaml")), + (50, include_config!("50.yaml")), // max_gas_burnt increased to 300 TGas - (52, include_config!("52.txt")), + (52, include_config!("52.yaml")), // Increased deployment costs, increased wasmer2 stack_limit, added limiting of contract locals, // set read_cached_trie_node cost, decrease storage key limit - (53, include_config!("53.txt")), - (57, include_config!("57.txt")), + (53, include_config!("53.yaml")), + (57, include_config!("57.yaml")), ]; /// Testnet parameters for versions <= 29, which (incorrectly) differed from mainnet parameters diff --git a/core/primitives/src/runtime/parameter_table.rs b/core/primitives/src/runtime/parameter_table.rs index f1cbf67e3da..8909e4bb55c 100644 --- a/core/primitives/src/runtime/parameter_table.rs +++ b/core/primitives/src/runtime/parameter_table.rs @@ -4,7 +4,6 @@ use near_primitives_core::parameter::{FeeParameter, Parameter}; use near_primitives_core::runtime::fees::{RuntimeFeesConfig, StorageUsageConfig}; use num_rational::Rational; use serde::de::DeserializeOwned; -use serde::Deserialize; use serde_json::json; use std::any::Any; use std::collections::BTreeMap; @@ -26,8 +25,6 @@ pub(crate) enum InvalidConfigError { UnknownParameter(#[source] strum::ParseError, String), #[error("could not parse `{1}` as a value")] ValueParseError(#[source] serde_json::Error, String), - #[error("expected a `:` separator between name and value of a parameter `{1}` on line {0}")] - NoSeparator(usize, String), #[error("intermediate JSON created by parser does not match `RuntimeConfig`")] WrongStructure(#[source] serde_json::Error), #[error("could not parse YAML that defines the structure of the config")] @@ -216,58 +213,45 @@ impl ParameterTable { } } +/// Represents YAML values supported by parameter diff config. +#[derive(serde::Deserialize, Clone, Debug)] +struct ParameterDiffValue { + old: Option, + new: Option, +} + impl std::str::FromStr for ParameterTableDiff { type Err = InvalidConfigError; fn from_str(arg: &str) -> Result { - let parameters = txt_to_key_values(arg) - .map(|result| { - let (typed_key, value) = result?; - if let Some((before, after)) = value.split_once("->") { - Ok(( - typed_key, - ( - parse_parameter_txt_value(before.trim())?, - parse_parameter_txt_value(after.trim())?, - ), - )) + let yaml_map: BTreeMap = + serde_yaml::from_str(arg).map_err(|err| InvalidConfigError::InvalidYaml(err))?; + + let parameters = yaml_map + .iter() + .map(|(key, value)| { + let typed_key: Parameter = key + .parse() + .map_err(|err| InvalidConfigError::UnknownParameter(err, key.to_owned()))?; + + let old_value = if let Some(s) = &value.old { + parse_parameter_txt_value(s)? } else { - Ok(( - typed_key, - (serde_json::Value::Null, parse_parameter_txt_value(value.trim())?), - )) - } + serde_json::Value::Null + }; + + let new_value = if let Some(s) = &value.new { + parse_parameter_txt_value(s)? + } else { + serde_json::Value::Null + }; + + Ok((typed_key, (old_value, new_value))) }) .collect::, _>>()?; Ok(ParameterTableDiff { parameters }) } } -fn txt_to_key_values( - arg: &str, -) -> impl Iterator> { - arg.lines() - .enumerate() - .filter_map(|(nr, line)| { - // ignore comments and empty lines - let trimmed = line.trim(); - if trimmed.starts_with("#") || trimmed.is_empty() { - None - } else { - Some((nr, trimmed)) - } - }) - .map(|(nr, trimmed)| { - let (key, value) = trimmed - .split_once(":") - .ok_or(InvalidConfigError::NoSeparator(nr + 1, trimmed.to_owned()))?; - let typed_key: Parameter = key - .trim() - .parse() - .map_err(|err| InvalidConfigError::UnknownParameter(err, key.to_owned()))?; - Ok((typed_key, value)) - }) -} - /// Parses a value from the custom format for runtime parameter definitions. /// /// A value can be a positive integer or a string, both written without quotes. @@ -370,17 +354,17 @@ storage_num_extra_bytes_record : 40 static DIFF_0: &str = r#" # Comment line -registrar_account_id: registrar -> near -min_allowed_top_level_account_length: 32 -> 32_000 -wasm_regular_op_cost: 3_856_371 +registrar_account_id: { old: "registrar", new: "near" } +min_allowed_top_level_account_length: { old: 32, new: 32_000 } +wasm_regular_op_cost: { new: 3_856_371 } "#; static DIFF_1: &str = r#" # Comment line -registrar_account_id: near -> registrar -storage_num_extra_bytes_record: 40 -> 77 -wasm_regular_op_cost: 3_856_371 -> 0 -max_memory_pages: 512 +registrar_account_id: { old: "near", new: "registrar" } +storage_num_extra_bytes_record: { old: 40, new: 77 } +wasm_regular_op_cost: { old: 3_856_371, new: 0 } +max_memory_pages: { new: 512 } "#; // Tests synthetic small example configurations. For tests with "real" @@ -462,7 +446,7 @@ max_memory_pages: 512 #[test] fn test_parameter_table_with_empty_value() { - let diff_with_empty_value = "min_allowed_top_level_account_length: 32 -> "; + let diff_with_empty_value = "min_allowed_top_level_account_length: { old: 32 }"; check_parameter_table( BASE_0, &[diff_with_empty_value], @@ -488,7 +472,10 @@ max_memory_pages: 512 #[test] fn test_parameter_table_invalid_key_in_diff() { assert_matches!( - check_invalid_parameter_table("wasm_regular_op_cost: 100", &["invalid_key: 100"]), + check_invalid_parameter_table( + "wasm_regular_op_cost: 100", + &["invalid_key: { new: 100 }"] + ), InvalidConfigError::UnknownParameter(_, _) ); } @@ -505,7 +492,7 @@ max_memory_pages: 512 fn test_parameter_table_no_key_in_diff() { assert_matches!( check_invalid_parameter_table("wasm_regular_op_cost: 100", &[": 100"]), - InvalidConfigError::UnknownParameter(_, _) + InvalidConfigError::InvalidYaml(_) ); } @@ -524,7 +511,7 @@ max_memory_pages: 512 "wasm_regular_op_cost: 100", &["wasm_regular_op_cost=100"] ), - InvalidConfigError::NoSeparator(1, _) + InvalidConfigError::InvalidYaml(_) ); } @@ -533,7 +520,7 @@ max_memory_pages: 512 assert_matches!( check_invalid_parameter_table( "min_allowed_top_level_account_length: 3_200_000_000", - &["min_allowed_top_level_account_length: 3_200_000 -> 1_600_000"] + &["min_allowed_top_level_account_length: { old: 3_200_000, new: 1_600_000 }"] ), InvalidConfigError::WrongOldValue( Parameter::MinAllowedTopLevelAccountLength, @@ -551,7 +538,7 @@ max_memory_pages: 512 assert_matches!( check_invalid_parameter_table( "min_allowed_top_level_account_length: 3_200_000_000", - &["min_allowed_top_level_account_length: 1_600_000"] + &["min_allowed_top_level_account_length: { new: 1_600_000 }"] ), InvalidConfigError::OldValueExists(Parameter::MinAllowedTopLevelAccountLength, expected) => { assert_eq!(expected, "3200000000"); @@ -564,7 +551,7 @@ max_memory_pages: 512 assert_matches!( check_invalid_parameter_table( "min_allowed_top_level_account_length: 3_200_000_000", - &["wasm_regular_op_cost: 3_200_000 -> 1_600_000"] + &["wasm_regular_op_cost: { old: 3_200_000, new: 1_600_000 }"] ), InvalidConfigError::NoOldValueExists(Parameter::WasmRegularOpCost, found) => { assert_eq!(found, "3200000"); From 031250887040ed3d11d3ec9467402b0d6f5f4f68 Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Tue, 10 Jan 2023 11:19:38 +0000 Subject: [PATCH 4/5] Workaround bugs in old `serde_yaml` version --- core/primitives/src/runtime/parameter_table.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/primitives/src/runtime/parameter_table.rs b/core/primitives/src/runtime/parameter_table.rs index 8909e4bb55c..a61ba17dd1d 100644 --- a/core/primitives/src/runtime/parameter_table.rs +++ b/core/primitives/src/runtime/parameter_table.rs @@ -46,6 +46,11 @@ pub(crate) enum InvalidConfigError { impl std::str::FromStr for ParameterTable { type Err = InvalidConfigError; fn from_str(arg: &str) -> Result { + // TODO(#8320): Remove this after migration to `serde_yaml` 0.9 that supports empty strings. + if arg.is_empty() { + return Ok(ParameterTable { parameters: BTreeMap::new() }); + } + let yaml_map: BTreeMap = serde_yaml::from_str(arg).map_err(|err| InvalidConfigError::InvalidYaml(err))?; @@ -484,7 +489,8 @@ max_memory_pages: { new: 512 } fn test_parameter_table_no_key() { assert_matches!( check_invalid_parameter_table(": 100", &[]), - InvalidConfigError::InvalidYaml(_) + // TODO(#8320): This must be invalid YAML after migration to `serde_yaml` 0.9. + InvalidConfigError::UnknownParameter(_, _) ); } From 4f95735e9a93e199b54543cb6f2b4b76c9786c8a Mon Sep 17 00:00:00 2001 From: Andrei Kashin Date: Mon, 9 Jan 2023 12:30:32 +0000 Subject: [PATCH 5/5] doc: Point to new YAML parameters in docs --- core/primitives/res/README.md | 10 +++++----- docs/architecture/gas/parameter_definition.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/primitives/res/README.md b/core/primitives/res/README.md index f532fa0d16d..e0a824c5979 100644 --- a/core/primitives/res/README.md +++ b/core/primitives/res/README.md @@ -4,18 +4,18 @@ Stores resource data which is part of the protocol stable enough to be moved out ### `runtime_configs` -All parameter value to configure the runtime are defined in `parameters.txt`. +All parameter value to configure the runtime are defined in `parameters.yaml`. Parameters added or changed in protocol upgrades are defined in differential -config files with a naming scheme like `V.txt`, where `V` is the new version. +config files with a naming scheme like `V.yaml`, where `V` is the new version. The content of the base configuration file is one flat list of typed keys and untyped values. Key names are defined in `core/primitives-core/src/parameter.rs`. The format of the differential files is slightly different. Inserting new -parameters uses the same syntax as the base configuration file: `key: value`. -Parameters that change are specified like this: `key: old_value -> new_value`. +parameters uses the following syntax: `key: { new: value }`. +Parameters that change are specified like this: `key: { old: old_value, new: new_value }`. Removing a previously defined parameter for a new version is done as follows: -`key: old_value ->`. This causes the parameter value to be undefined in newer +`key: { old: old_value }`. This causes the parameter value to be undefined in newer versions which generally means the default value is used to fill in the `RuntimeConfig` object. diff --git a/docs/architecture/gas/parameter_definition.md b/docs/architecture/gas/parameter_definition.md index ee76ec8ff6b..3aa403cf866 100644 --- a/docs/architecture/gas/parameter_definition.md +++ b/docs/architecture/gas/parameter_definition.md @@ -1,7 +1,7 @@ # Parameter Definitions Gas parameters are a subset of runtime parameters that are defined in -[core/primitives/res/runtime_configs/parameters.txt](https://github.com/near/nearcore/blob/d0dc37bf81f7e7bde9c560403b085fae04108659/core/primitives/res/runtime_configs/parameters.txt). +[core/primitives/res/runtime_configs/parameters.yaml](https://github.com/near/nearcore/blob/d0dc37bf81f7e7bde9c560403b085fae04108659/core/primitives/res/runtime_configs/parameters.yaml). IMPORTANT: This is not the final list of parameters, it contains the base values which can be overwritten per protocol version. For example, [53.txt](https://github.com/near/nearcore/blob/d0dc37bf81f7e7bde9c560403b085fae04108659/core/primitives/res/runtime_configs/53.txt) @@ -57,7 +57,7 @@ parameter. ### Necessary Code Changes Adding the parameter in code involves several steps. -1. Define the parameter by adding it to the list in `core/primitives/res/runtime_configs/parameters.txt.` +1. Define the parameter by adding it to the list in `core/primitives/res/runtime_configs/parameters.yaml.` 2. Update the Rust view of parameters by adding a variant to `enum Parameter` in `core/primitives-core/src/parameter.rs`. In the same file, update `enum FeeParameter` if you add an action cost or update `ext_costs()` @@ -77,7 +77,7 @@ Adding the parameter in code involves several steps. behind a feature flag. Add the feature to the `Cargo.toml` of each crate touched in step 3 and 4 and hide the code behind `#[cfg(feature = "protocol_feature_MY_NEW_FEATURE")]`. Do not hide code in step 2 so that - non-nightly builds can still read `parameters.txt`. Also add your feature as + non-nightly builds can still read `parameters.yaml`. Also add your feature as a dependency on `nightly` in `core/primitives/Cargo.toml` to make sure it gets included when compiling for nightly. After that, check `cargo check` and `cargo test --no-run` with and without `features=nightly`.