Statically register host WASM functions (#10394)

* Statically register host WASM functions

* Fix `substrate-test-client` compilation

* Move `ExtendedHostFunctions` to `sp-wasm-interface`

* Fix `sp-runtime-interface` tests' compilation

* Fix `sc-executor-wasmtime` tests' compilation

* Use `runtime_interface` macro in `test-runner`

* Fix `sc-executor` tests' compilation

* Reformatting/`rustfmt`

* Add an extra comment regarding the `H` generic arg in `create_runtime`

* Even more `rustfmt`

* Depend on `wasmtime` without default features in `sp-wasm-interface`

* Bump version of `sp-wasm-interface` to 4.0.1

* Bump `sp-wasm-interface` in `Cargo.lock` too

* Bump all of the `sp-wasm-interface` requirements to 4.0.1

Maybe this will appease cargo-unleash?

* Revert "Bump all of the `sp-wasm-interface` requirements to 4.0.1"

This reverts commit 0f7ccf8.

* Make `cargo-unleash` happy (maybe)

* Use `cargo-unleash` to bump the crates' versions

* Align to review comments

client/allocator/Cargo.toml

@@ -15,6 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 sp-core = { version = "4.1.0-dev", path = "../../primitives/core" }
-sp-wasm-interface = { version = "4.0.0", path = "../../primitives/wasm-interface" }
+sp-wasm-interface = { version = "4.1.0-dev", path = "../../primitives/wasm-interface" }
 log = "0.4.11"
 thiserror = "1.0.30"

client/executor/Cargo.toml

@@ -24,8 +24,8 @@ sp-panic-handler = { version = "4.0.0-dev", path = "../../primitives/panic-handl
 wasmi = "0.9.1"
 lazy_static = "1.4.0"
 sp-api = { version = "4.0.0-dev", path = "../../primitives/api" }
-sp-wasm-interface = { version = "4.0.0", path = "../../primitives/wasm-interface" }
-sp-runtime-interface = { version = "4.0.0", path = "../../primitives/runtime-interface" }
+sp-wasm-interface = { version = "4.1.0-dev", path = "../../primitives/wasm-interface" }
+sp-runtime-interface = { version = "4.1.0-dev", path = "../../primitives/runtime-interface" }
 sp-externalities = { version = "0.10.0", path = "../../primitives/externalities" }
 sc-executor-common = { version = "0.10.0-dev", path = "common" }
 sc-executor-wasmi = { version = "0.10.0-dev", path = "wasmi" }

client/executor/common/Cargo.toml

@@ -20,7 +20,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0" }
 wasmi = "0.9.1"
 sp-core = { version = "4.1.0-dev", path = "../../../primitives/core" }
 sc-allocator = { version = "4.1.0-dev", path = "../../allocator" }
-sp-wasm-interface = { version = "4.0.0", path = "../../../primitives/wasm-interface" }
+sp-wasm-interface = { version = "4.1.0-dev", path = "../../../primitives/wasm-interface" }
 sp-maybe-compressed-blob = { version = "4.1.0-dev", path = "../../../primitives/maybe-compressed-blob" }
 sp-serializer = { version = "4.0.0-dev", path = "../../../primitives/serializer" }
 thiserror = "1.0.30"

client/executor/runtime-test/src/lib.rs

@@ -332,6 +332,14 @@ sp_core::wasm_export_functions! {
    fn test_panic_in_spawned() {
 	   sp_tasks::spawn(tasks::panicker, vec![]).join();
    }
+
+	fn test_return_i8() -> i8 {
+		-66
+	}
+
+	fn test_take_i8(value: i8) {
+		assert_eq!(value, -66);
+	}
 }
 
 #[cfg(not(feature = "std"))]

client/executor/src/integration_tests/mod.rs

@@ -34,7 +34,6 @@ use sp_core::{
 use sp_runtime::traits::BlakeTwo256;
 use sp_state_machine::TestExternalities as CoreTestExternalities;
 use sp_trie::{trie_types::Layout, TrieConfiguration};
-use sp_wasm_interface::HostFunctions as _;
 use std::sync::Arc;
 use tracing_subscriber::layer::SubscriberExt;
 
@@ -124,14 +123,8 @@ fn call_in_wasm<E: Externalities>(
 	execution_method: WasmExecutionMethod,
 	ext: &mut E,
 ) -> Result<Vec<u8>, String> {
-	let executor = crate::WasmExecutor::new(
-		execution_method,
-		Some(1024),
-		HostFunctions::host_functions(),
-		8,
-		None,
-		2,
-	);
+	let executor =
+		crate::WasmExecutor::<HostFunctions>::new(execution_method, Some(1024), 8, None, 2);
 	executor.uncached_call(
 		RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
 		ext,
@@ -475,10 +468,9 @@ test_wasm_execution!(should_trap_when_heap_exhausted);
 fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
 	let mut ext = TestExternalities::default();
 
-	let executor = crate::WasmExecutor::new(
+	let executor = crate::WasmExecutor::<HostFunctions>::new(
 		wasm_method,
 		Some(17), // `17` is the initial number of pages compiled into the binary.
-		HostFunctions::host_functions(),
 		8,
 		None,
 		2,
@@ -501,11 +493,10 @@ fn mk_test_runtime(wasm_method: WasmExecutionMethod, pages: u64) -> Arc<dyn Wasm
 	let blob = RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..])
 		.expect("failed to create a runtime blob out of test runtime");
 
-	crate::wasm_runtime::create_wasm_runtime_with_code(
+	crate::wasm_runtime::create_wasm_runtime_with_code::<HostFunctions>(
 		wasm_method,
 		pages,
 		blob,
-		HostFunctions::host_functions(),
 		true,
 		None,
 	)
@@ -589,10 +580,9 @@ fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) {
 
 test_wasm_execution!(parallel_execution);
 fn parallel_execution(wasm_method: WasmExecutionMethod) {
-	let executor = std::sync::Arc::new(crate::WasmExecutor::new(
+	let executor = std::sync::Arc::new(crate::WasmExecutor::<HostFunctions>::new(
 		wasm_method,
 		Some(1024),
-		HostFunctions::host_functions(),
 		8,
 		None,
 		2,
@@ -763,11 +753,10 @@ fn memory_is_cleared_between_invocations(wasm_method: WasmExecutionMethod) {
 	 )
 	)"#).unwrap();
 
-	let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
+	let runtime = crate::wasm_runtime::create_wasm_runtime_with_code::<HostFunctions>(
 		wasm_method,
 		1024,
 		RuntimeBlob::uncompress_if_needed(&binary[..]).unwrap(),
-		HostFunctions::host_functions(),
 		true,
 		None,
 	)
@@ -780,3 +769,22 @@ fn memory_is_cleared_between_invocations(wasm_method: WasmExecutionMethod) {
 	let res = instance.call_export("returns_no_bss_mutable_static", &[0]).unwrap();
 	assert_eq!(1, u64::decode(&mut &res[..]).unwrap());
 }
+
+test_wasm_execution!(return_i8);
+fn return_i8(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+
+	assert_eq!(
+		call_in_wasm("test_return_i8", &[], wasm_method, &mut ext).unwrap(),
+		(-66_i8).encode()
+	);
+}
+
+test_wasm_execution!(take_i8);
+fn take_i8(wasm_method: WasmExecutionMethod) {
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+
+	call_in_wasm("test_take_i8", &(-66_i8).encode(), wasm_method, &mut ext).unwrap();
+}

client/executor/src/lib.rs

@@ -67,17 +67,15 @@ mod tests {
 	use sc_executor_common::runtime_blob::RuntimeBlob;
 	use sc_runtime_test::wasm_binary_unwrap;
 	use sp_io::TestExternalities;
-	use sp_wasm_interface::HostFunctions;
 
 	#[test]
 	fn call_in_interpreted_wasm_works() {
 		let mut ext = TestExternalities::default();
 		let mut ext = ext.ext();
 
-		let executor = WasmExecutor::new(
+		let executor = WasmExecutor::<sp_io::SubstrateHostFunctions>::new(
 			WasmExecutionMethod::Interpreted,
 			Some(8),
-			sp_io::SubstrateHostFunctions::host_functions(),
 			8,
 			None,
 			2,

client/executor/src/native_executor.rs

@@ -24,6 +24,7 @@ use crate::{
 
 use std::{
 	collections::HashMap,
+	marker::PhantomData,
 	panic::{AssertUnwindSafe, UnwindSafe},
 	path::PathBuf,
 	result,
@@ -46,7 +47,7 @@ use sp_core::{
 use sp_externalities::ExternalitiesExt as _;
 use sp_tasks::new_async_externalities;
 use sp_version::{GetNativeVersion, NativeVersion, RuntimeVersion};
-use sp_wasm_interface::{Function, HostFunctions};
+use sp_wasm_interface::{ExtendedHostFunctions, HostFunctions};
 
 /// Default num of pages for the heap
 const DEFAULT_HEAP_PAGES: u64 = 2048;
@@ -91,22 +92,36 @@ pub trait NativeExecutionDispatch: Send + Sync {
 
 /// An abstraction over Wasm code executor. Supports selecting execution backend and
 /// manages runtime cache.
-#[derive(Clone)]
-pub struct WasmExecutor {
+pub struct WasmExecutor<H> {
 	/// Method used to execute fallback Wasm code.
 	method: WasmExecutionMethod,
 	/// The number of 64KB pages to allocate for Wasm execution.
 	default_heap_pages: u64,
-	/// The host functions registered with this instance.
-	host_functions: Arc<Vec<&'static dyn Function>>,
 	/// WASM runtime cache.
 	cache: Arc<RuntimeCache>,
 	/// The path to a directory which the executor can leverage for a file cache, e.g. put there
 	/// compiled artifacts.
 	cache_path: Option<PathBuf>,
+
+	phantom: PhantomData<H>,
+}
+
+impl<H> Clone for WasmExecutor<H> {
+	fn clone(&self) -> Self {
+		Self {
+			method: self.method,
+			default_heap_pages: self.default_heap_pages,
+			cache: self.cache.clone(),
+			cache_path: self.cache_path.clone(),
+			phantom: self.phantom,
+		}
+	}
 }
 
-impl WasmExecutor {
+impl<H> WasmExecutor<H>
+where
+	H: HostFunctions,
+{
 	/// Create new instance.
 	///
 	/// # Parameters
@@ -127,21 +142,20 @@ impl WasmExecutor {
 	pub fn new(
 		method: WasmExecutionMethod,
 		default_heap_pages: Option<u64>,
-		host_functions: Vec<&'static dyn Function>,
 		max_runtime_instances: usize,
 		cache_path: Option<PathBuf>,
 		runtime_cache_size: u8,
 	) -> Self {
 		WasmExecutor {
 			method,
 			default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES),
-			host_functions: Arc::new(host_functions),
 			cache: Arc::new(RuntimeCache::new(
 				max_runtime_instances,
 				cache_path.clone(),
 				runtime_cache_size,
 			)),
 			cache_path,
+			phantom: PhantomData,
 		}
 	}
 
@@ -173,12 +187,11 @@ impl WasmExecutor {
 			AssertUnwindSafe<&mut dyn Externalities>,
 		) -> Result<Result<R>>,
 	{
-		match self.cache.with_instance(
+		match self.cache.with_instance::<H, _, _>(
 			runtime_code,
 			ext,
 			self.method,
 			self.default_heap_pages,
-			&*self.host_functions,
 			allow_missing_host_functions,
 			|module, instance, version, ext| {
 				let module = AssertUnwindSafe(module);
@@ -208,11 +221,10 @@ impl WasmExecutor {
 		export_name: &str,
 		call_data: &[u8],
 	) -> std::result::Result<Vec<u8>, String> {
-		let module = crate::wasm_runtime::create_wasm_runtime_with_code(
+		let module = crate::wasm_runtime::create_wasm_runtime_with_code::<H>(
 			self.method,
 			self.default_heap_pages,
 			runtime_blob,
-			self.host_functions.to_vec(),
 			allow_missing_host_functions,
 			self.cache_path.as_deref(),
 		)
@@ -235,7 +247,10 @@ impl WasmExecutor {
 	}
 }
 
-impl sp_core::traits::ReadRuntimeVersion for WasmExecutor {
+impl<H> sp_core::traits::ReadRuntimeVersion for WasmExecutor<H>
+where
+	H: HostFunctions,
+{
 	fn read_runtime_version(
 		&self,
 		wasm_code: &[u8],
@@ -269,7 +284,10 @@ impl sp_core::traits::ReadRuntimeVersion for WasmExecutor {
 	}
 }
 
-impl CodeExecutor for WasmExecutor {
+impl<H> CodeExecutor for WasmExecutor<H>
+where
+	H: HostFunctions,
+{
 	type Error = Error;
 
 	fn call<
@@ -299,7 +317,10 @@ impl CodeExecutor for WasmExecutor {
 	}
 }
 
-impl RuntimeVersionOf for WasmExecutor {
+impl<H> RuntimeVersionOf for WasmExecutor<H>
+where
+	H: HostFunctions,
+{
 	fn runtime_version(
 		&self,
 		ext: &mut dyn Externalities,
@@ -313,13 +334,17 @@ impl RuntimeVersionOf for WasmExecutor {
 
 /// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence
 /// and dispatch to native code when possible, falling back on `WasmExecutor` when not.
-pub struct NativeElseWasmExecutor<D> {
+pub struct NativeElseWasmExecutor<D>
+where
+	D: NativeExecutionDispatch,
+{
 	/// Dummy field to avoid the compiler complaining about us not using `D`.
 	_dummy: std::marker::PhantomData<D>,
 	/// Native runtime version info.
 	native_version: NativeVersion,
 	/// Fallback wasm executor.
-	wasm: WasmExecutor,
+	wasm:
+		WasmExecutor<ExtendedHostFunctions<sp_io::SubstrateHostFunctions, D::ExtendHostFunctions>>,
 }
 
 impl<D: NativeExecutionDispatch> NativeElseWasmExecutor<D> {
@@ -337,24 +362,9 @@ impl<D: NativeExecutionDispatch> NativeElseWasmExecutor<D> {
 		max_runtime_instances: usize,
 		runtime_cache_size: u8,
 	) -> Self {
-		let extended = D::ExtendHostFunctions::host_functions();
-		let mut host_functions = sp_io::SubstrateHostFunctions::host_functions()
-			.into_iter()
-			// filter out any host function overrides provided.
-			.filter(|host_fn| {
-				extended
-					.iter()
-					.find(|ext_host_fn| host_fn.name() == ext_host_fn.name())
-					.is_none()
-			})
-			.collect::<Vec<_>>();
-
-		// Add the custom host functions provided by the user.
-		host_functions.extend(extended);
 		let wasm_executor = WasmExecutor::new(
 			fallback_method,
 			default_heap_pages,
-			host_functions,
 			max_runtime_instances,
 			None,
 			runtime_cache_size,
@@ -645,8 +655,21 @@ mod tests {
 			8,
 			2,
 		);
+
+		fn extract_host_functions<H>(
+			_: &WasmExecutor<H>,
+		) -> Vec<&'static dyn sp_wasm_interface::Function>
+		where
+			H: HostFunctions,
+		{
+			H::host_functions()
+		}
+
 		my_interface::HostFunctions::host_functions().iter().for_each(|function| {
-			assert_eq!(executor.wasm.host_functions.iter().filter(|f| f == &function).count(), 2);
+			assert_eq!(
+				extract_host_functions(&executor.wasm).iter().filter(|f| f == &function).count(),
+				2
+			);
 		});
 
 		my_interface::say_hello_world("hey");