Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Allow separate max-heap-pages for offchain (non-consensus) execution context. #8885

Closed
wants to merge 34 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e6d90db
Working draft with some consideration about names.
kianenigma May 22, 2021
ca07e34
Master.into()
kianenigma Jun 28, 2021
e70e4db
Fix all tests
kianenigma Jun 28, 2021
1f7fabd
fix some line width
kianenigma Jun 28, 2021
98897fa
Fix
kianenigma Jun 28, 2021
9ce9263
Update primitives/state-machine/src/lib.rs
kianenigma Jul 2, 2021
8630537
Master.into()
kianenigma Jul 2, 2021
f6a185c
New version
kianenigma Jul 3, 2021
fbe4c81
Merge branch 'kiz-ocw-max-heap-pages' of github.com:paritytech/substr…
kianenigma Jul 3, 2021
79f6cf9
Add log
kianenigma Jul 4, 2021
8a133b1
Update client/executor/src/wasm_runtime.rs
kianenigma Jul 5, 2021
04d96f0
Master.into()
kianenigma Jul 8, 2021
2eca696
Update client/executor/src/wasm_runtime.rs
kianenigma Jul 8, 2021
b96ecab
Update client/service/src/builder.rs
kianenigma Jul 8, 2021
2be3e7d
Update client/service/src/client/wasm_override.rs
kianenigma Jul 8, 2021
5d3e8a3
Update client/service/test/src/client/light.rs
kianenigma Jul 8, 2021
d688a1b
Fix build
kianenigma Jul 8, 2021
0487c35
Master.into()
kianenigma Jul 21, 2021
c922f29
massive master merge
kianenigma Jul 29, 2021
2d7b5d8
again, merrge master.
kianenigma Jul 29, 2021
7c87ab1
Update client/finality-grandpa/src/lib.rs
kianenigma Jul 29, 2021
ce48421
Update frame/system/src/lib.rs
kianenigma Jul 29, 2021
573cc75
Update primitives/storage/src/lib.rs
kianenigma Jul 29, 2021
5918d2d
Update primitives/storage/src/lib.rs
kianenigma Jul 29, 2021
e3d9f50
Update test-utils/test-runner/src/lib.rs
kianenigma Jul 29, 2021
12dde90
Update client/light/src/call_executor.rs
kianenigma Jul 29, 2021
b0d1f66
Add log
kianenigma Jul 29, 2021
209fee5
Merge branch 'kiz-ocw-max-heap-pages' of github.com:paritytech/substr…
kianenigma Jul 29, 2021
e35326c
Try something new
kianenigma Jul 31, 2021
db1f7a3
A new approach
kianenigma Jul 31, 2021
a4fb6c6
Remove the stale heap pages param
kianenigma Jul 31, 2021
2242bd5
Merge branch 'master' of github.com:paritytech/substrate into kiz-ocw…
kianenigma Aug 1, 2021
249286b
Master.into()
kianenigma Aug 5, 2021
946cfde
fmt it properly
kianenigma Aug 5, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion bin/node/executor/tests/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ pub fn from_block_number(n: u32) -> Header {
}

pub fn executor() -> NativeExecutor<Executor> {
NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8)
NativeExecutor::new(WasmExecutionMethod::Interpreted, 8)
}

pub fn executor_call<
Expand All @@ -117,6 +117,7 @@ pub fn executor_call<
let heap_pages = t.storage(sp_core::storage::well_known_keys::HEAP_PAGES);
let runtime_code = RuntimeCode {
code_fetcher: &sp_core::traits::WrappedRuntimeCode(code.as_slice().into()),
context: sp_core::traits::CodeContext::Consensus,
hash: sp_core::blake2_256(&code).to_vec(),
heap_pages: heap_pages.and_then(|hp| Decode::decode(&mut &hp[..]).ok()),
};
Expand Down
36 changes: 19 additions & 17 deletions bin/node/testing/src/bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ use node_runtime::{
};
use sc_block_builder::BlockBuilderProvider;
use sc_client_api::{
execution_extensions::{ExecutionExtensions, ExecutionStrategies},
execution_extensions::{ExecutionConfig, ExecutionConfigs, ExecutionExtensions},
BlockBackend, ExecutionStrategy,
};
use sc_client_db::PruningMode;
Expand Down Expand Up @@ -390,11 +390,11 @@ impl BenchDb {
let backend = sc_service::new_db_backend(db_config).expect("Should not fail");
let client = sc_service::new_client(
backend.clone(),
NativeExecutor::new(WasmExecutionMethod::Compiled, None, 8),
NativeExecutor::new(WasmExecutionMethod::Compiled, 8),
&keyring.generate_genesis(),
None,
None,
ExecutionExtensions::new(profile.into_execution_strategies(), None, None),
ExecutionExtensions::new(profile.into_execution_configs(), None, None),
Box::new(task_executor.clone()),
None,
None,
Expand Down Expand Up @@ -588,7 +588,7 @@ impl BenchKeyring {
}
}

/// Profile for exetion strategies.
/// Profile for execution configurations.
#[derive(Clone, Copy, Debug)]
pub enum Profile {
/// As native as possible.
Expand All @@ -598,21 +598,23 @@ pub enum Profile {
}

impl Profile {
fn into_execution_strategies(self) -> ExecutionStrategies {
fn into_execution_configs(self) -> ExecutionConfigs {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
fn into_execution_configs(self) -> ExecutionConfigs {
fn into_offchain_execution_configs(self) -> ExecutionConfigs {

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why?

Copy link
Contributor

@cheme cheme Jul 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Code is calling new_offchain only, so just precising.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually shouldn't it be new_offchain only for 'offchain_worker'?

match self {
Profile::Wasm => ExecutionStrategies {
syncing: ExecutionStrategy::AlwaysWasm,
importing: ExecutionStrategy::AlwaysWasm,
block_construction: ExecutionStrategy::AlwaysWasm,
offchain_worker: ExecutionStrategy::AlwaysWasm,
other: ExecutionStrategy::AlwaysWasm,
Profile::Wasm => ExecutionConfigs {
syncing: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm),
importing: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm),
block_construction: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm),
offchain_worker: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm),
other: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm),
},
Profile::Native => ExecutionStrategies {
syncing: ExecutionStrategy::NativeElseWasm,
importing: ExecutionStrategy::NativeElseWasm,
block_construction: ExecutionStrategy::NativeElseWasm,
offchain_worker: ExecutionStrategy::NativeElseWasm,
other: ExecutionStrategy::NativeElseWasm,
Profile::Native => ExecutionConfigs {
syncing: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm),
importing: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm),
block_construction: ExecutionConfig::new_offchain(
ExecutionStrategy::NativeElseWasm,
),
offchain_worker: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm),
other: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm),
},
}
}
Expand Down
4 changes: 2 additions & 2 deletions client/api/src/call_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use sc_executor::{NativeVersion, RuntimeVersion};
use sp_core::NativeOrEncoded;
use sp_externalities::Extensions;
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
use sp_state_machine::{ExecutionManager, ExecutionStrategy, OverlayedChanges, StorageProof};
use sp_state_machine::{ExecutionConfig, ExecutionManager, OverlayedChanges, StorageProof};
use std::{cell::RefCell, panic::UnwindSafe, result};

use crate::execution_extensions::ExecutionExtensions;
Expand Down Expand Up @@ -57,7 +57,7 @@ pub trait CallExecutor<B: BlockT> {
id: &BlockId<B>,
method: &str,
call_data: &[u8],
strategy: ExecutionStrategy,
config: ExecutionConfig,
extensions: Option<Extensions>,
) -> Result<Vec<u8>, sp_blockchain::Error>;

Expand Down
68 changes: 34 additions & 34 deletions client/api/src/execution_extensions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,33 +32,33 @@ use sp_core::{
use sp_externalities::Extensions;
use sp_keystore::{KeystoreExt, SyncCryptoStorePtr};
use sp_runtime::{generic::BlockId, traits};
pub use sp_state_machine::ExecutionStrategy;
use sp_state_machine::{DefaultHandler, ExecutionManager};
pub use sp_state_machine::{ExecutionConfig, ExecutionStrategy};
use std::sync::{Arc, Weak};

/// Execution strategies settings.
/// Execution configurations, per operation type.
#[derive(Debug, Clone)]
pub struct ExecutionStrategies {
/// Execution strategy used when syncing.
pub syncing: ExecutionStrategy,
/// Execution strategy used when importing blocks.
pub importing: ExecutionStrategy,
/// Execution strategy used when constructing blocks.
pub block_construction: ExecutionStrategy,
/// Execution strategy used for offchain workers.
pub offchain_worker: ExecutionStrategy,
/// Execution strategy used in other cases.
pub other: ExecutionStrategy,
pub struct ExecutionConfigs {
/// Execution config used when syncing.
pub syncing: ExecutionConfig,
/// Execution config used when importing blocks.
pub importing: ExecutionConfig,
/// Execution config used when constructing blocks.
pub block_construction: ExecutionConfig,
/// Execution config used for offchain workers.
pub offchain_worker: ExecutionConfig,
/// Execution config used in other cases.
pub other: ExecutionConfig,
}

impl Default for ExecutionStrategies {
fn default() -> ExecutionStrategies {
ExecutionStrategies {
syncing: ExecutionStrategy::NativeElseWasm,
importing: ExecutionStrategy::NativeElseWasm,
block_construction: ExecutionStrategy::AlwaysWasm,
offchain_worker: ExecutionStrategy::NativeWhenPossible,
other: ExecutionStrategy::NativeElseWasm,
impl Default for ExecutionConfigs {
fn default() -> ExecutionConfigs {
ExecutionConfigs {
kianenigma marked this conversation as resolved.
Show resolved Hide resolved
syncing: ExecutionStrategy::NativeElseWasm.in_consensus(),
importing: ExecutionStrategy::NativeElseWasm.in_consensus(),
block_construction: ExecutionStrategy::AlwaysWasm.in_consensus(),
offchain_worker: ExecutionStrategy::NativeWhenPossible.in_offchain(),
other: ExecutionStrategy::NativeElseWasm.in_consensus(),
}
}
}
Expand Down Expand Up @@ -93,7 +93,7 @@ impl<T: offchain::DbExternalities + Clone + Sync + Send + 'static> DbExternaliti
/// and is responsible for producing a correct `Extensions` object.
/// for each call, based on required `Capabilities`.
pub struct ExecutionExtensions<Block: traits::Block> {
strategies: ExecutionStrategies,
configs: ExecutionConfigs,
keystore: Option<SyncCryptoStorePtr>,
offchain_db: Option<Box<dyn DbExternalitiesFactory>>,
// FIXME: these two are only RwLock because of https://github.com/paritytech/substrate/issues/4587
Expand All @@ -109,7 +109,7 @@ pub struct ExecutionExtensions<Block: traits::Block> {
impl<Block: traits::Block> Default for ExecutionExtensions<Block> {
fn default() -> Self {
Self {
strategies: Default::default(),
configs: Default::default(),
keystore: None,
offchain_db: None,
transaction_pool: RwLock::new(None),
Expand All @@ -119,26 +119,26 @@ impl<Block: traits::Block> Default for ExecutionExtensions<Block> {
}

impl<Block: traits::Block> ExecutionExtensions<Block> {
/// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`.
/// Create new `ExecutionExtensions` given a `keystore` and `ExecutionConfigs`.
pub fn new(
strategies: ExecutionStrategies,
configs: ExecutionConfigs,
keystore: Option<SyncCryptoStorePtr>,
offchain_db: Option<Box<dyn DbExternalitiesFactory>>,
) -> Self {
let transaction_pool = RwLock::new(None);
let extensions_factory = Box::new(());
Self {
strategies,
configs,
keystore,
offchain_db,
extensions_factory: RwLock::new(extensions_factory),
transaction_pool,
}
}

/// Get a reference to the execution strategies.
pub fn strategies(&self) -> &ExecutionStrategies {
&self.strategies
/// Get a reference to the execution configs.
pub fn configs(&self) -> &ExecutionConfigs {
&self.configs
}

/// Set the new extensions_factory
Expand Down Expand Up @@ -207,12 +207,12 @@ impl<Block: traits::Block> ExecutionExtensions<Block> {
context: ExecutionContext,
) -> (ExecutionManager<DefaultHandler<R, E>>, Extensions) {
let manager = match context {
ExecutionContext::BlockConstruction => self.strategies.block_construction.get_manager(),
ExecutionContext::Syncing => self.strategies.syncing.get_manager(),
ExecutionContext::Importing => self.strategies.importing.get_manager(),
ExecutionContext::BlockConstruction => self.configs.block_construction.get_manager(),
ExecutionContext::Syncing => self.configs.syncing.get_manager(),
ExecutionContext::Importing => self.configs.importing.get_manager(),
ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() =>
self.strategies.offchain_worker.get_manager(),
ExecutionContext::OffchainCall(_) => self.strategies.other.get_manager(),
self.configs.offchain_worker.get_manager(),
ExecutionContext::OffchainCall(_) => self.configs.other.get_manager(),
};

(manager, self.extensions(at, context))
Expand Down
2 changes: 1 addition & 1 deletion client/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ pub use proof_provider::*;
pub use sp_blockchain as blockchain;
pub use sp_blockchain::HeaderBackend;

pub use sp_state_machine::{ExecutionStrategy, StorageProof};
pub use sp_state_machine::{ExecutionConfig, ExecutionStrategy, StorageProof};
pub use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey};

/// Usage Information Provider interface
Expand Down
1 change: 1 addition & 0 deletions client/cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ sc-tracing = { version = "4.0.0-dev", path = "../tracing" }
chrono = "0.4.10"
serde = "1.0.126"
thiserror = "1.0.21"
paste = "1.0.5"

[target.'cfg(not(target_os = "unknown"))'.dependencies]
rpassword = "5.0.0"
Expand Down
34 changes: 22 additions & 12 deletions client/cli/src/arg_enums.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
// NOTE: we allow missing docs here because arg_enum! creates the function variants without doc
#![allow(missing_docs)]

use sp_core::traits::CodeContext;
use structopt::clap::arg_enum;

/// How to execute Wasm runtime code.
Expand Down Expand Up @@ -260,15 +261,24 @@ impl Into<sc_network::config::SyncMode> for SyncMode {
}
}

/// Default value for the `--execution-syncing` parameter.
pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm;
/// Default value for the `--execution-import-block` parameter.
pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::NativeElseWasm;
/// Default value for the `--execution-import-block` parameter when the node is a validator.
pub const DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR: ExecutionStrategy = ExecutionStrategy::Wasm;
/// Default value for the `--execution-block-construction` parameter.
pub const DEFAULT_EXECUTION_BLOCK_CONSTRUCTION: ExecutionStrategy = ExecutionStrategy::Wasm;
/// Default value for the `--execution-offchain-worker` parameter.
pub const DEFAULT_EXECUTION_OFFCHAIN_WORKER: ExecutionStrategy = ExecutionStrategy::Native;
/// Default value for the `--execution-other` parameter.
pub const DEFAULT_EXECUTION_OTHER: ExecutionStrategy = ExecutionStrategy::Native;
macro_rules! generate_config_const {
($( $role:ident => $strategy:ident, $context:ident ),*) => {
paste::paste! {
$(
pub const [<DEFAULT_STRATEGY_ $role:snake:upper>]: ExecutionStrategy
= ExecutionStrategy::$strategy;
pub const [<DEFAULT_CODE_CONTEXT_ $role:snake:upper>]: CodeContext
= CodeContext::$context;
)*
}
};
}

generate_config_const!(
syncing => NativeElseWasm, Consensus,
import_block => NativeElseWasm, Consensus,
import_block_validator => Wasm, Consensus,
block_construction => Wasm, Consensus,
offchain_worker => Native, Offchain,
other => Native, Consensus
);
Loading