fix: explicitly enable wasi on smartengine uses (#3988)

* fix: explicitly use wasi on smartengine uses

remove feature flag, wasi is always on if the engine is on

* chore: build alignment to wasm32-wasi

* chore: disable cli smoke test for dev-cli + stable cluster

* chore: restore versions on dep paths

* fix: repeated line

* fix: remove redundant rustup target add

build_smartmodules already adds it

* chore: one last missed version

.github/workflows/ci.yml

@@ -936,6 +936,9 @@ jobs:
         exclude:
           - cluster_version:  stable
             cli_version: stable
+          # temporary disable with known error w/ 0.11.7
+          - cluster_version:  stable
+            cli_version: dev
 
     steps:
       - uses: actions/checkout@v4

Cargo.toml

@@ -165,28 +165,28 @@ k8-diff = { version = "0.1.2" }
 trybuild = { branch = "check_option", git = "https://github.com/infinyon/trybuild" }
 
 # Internal fluvio dependencies
-fluvio = { version = "0.22.0", path = "crates/fluvio" }
+fluvio = { version = "0.22.2", path = "crates/fluvio" }
 fluvio-auth = { path = "crates/fluvio-auth" }
 fluvio-channel = { path = "crates/fluvio-channel" }
 fluvio-cli-common = { path = "crates/fluvio-cli-common"}
-fluvio-compression = { version = "0.3", path = "crates/fluvio-compression" }
+fluvio-compression = { version = "0.3.2", path = "crates/fluvio-compression" }
 fluvio-connector-package = { path = "crates/fluvio-connector-package/" }
 fluvio-controlplane = { path = "crates/fluvio-controlplane" }
-fluvio-controlplane-metadata = { version = "0.28.0", default-features = false, path = "crates/fluvio-controlplane-metadata" }
+fluvio-controlplane-metadata = { version = "0.28.1", default-features = false, path = "crates/fluvio-controlplane-metadata" }
 fluvio-extension-common = { path = "crates/fluvio-extension-common", default-features = false }
 fluvio-hub-util = { path = "crates/fluvio-hub-util" }
-fluvio-package-index = { version = "0.7.0", path = "crates/fluvio-package-index", default-features = false }
-fluvio-protocol = { version = "0.10.12", path = "crates/fluvio-protocol" }
+fluvio-package-index = { version = "0.7.6", path = "crates/fluvio-package-index", default-features = false }
+fluvio-protocol = { version = "0.10.13", path = "crates/fluvio-protocol" }
 fluvio-sc-schema = { version = "0.24.0", path = "crates/fluvio-sc-schema", default-features = false }
 fluvio-service = { path = "crates/fluvio-service" }
-fluvio-smartengine = {  version = "0.7.0", path = "crates/fluvio-smartengine", default-features = false }
-fluvio-smartmodule = { version = "0.7.0", path = "crates/fluvio-smartmodule", default-features = false }
-fluvio-socket = { version = "0.14.3", path = "crates/fluvio-socket", default-features = false }
-fluvio-spu-schema = { version = "0.14.5", path = "crates/fluvio-spu-schema", default-features  = false }
+fluvio-smartengine = {  version = "0.7.12", path = "crates/fluvio-smartengine", default-features = false }
+fluvio-smartmodule = { version = "0.7.3", path = "crates/fluvio-smartmodule", default-features = false }
+fluvio-socket = { version = "0.14.8", path = "crates/fluvio-socket", default-features = false }
+fluvio-spu-schema = { version = "0.14.7", path = "crates/fluvio-spu-schema", default-features  = false }
 fluvio-storage = { path = "crates/fluvio-storage" }
-fluvio-stream-dispatcher = { version = "0.13.0", path = "crates/fluvio-stream-dispatcher" }
-fluvio-stream-model = { version = "0.11.0", path = "crates/fluvio-stream-model", default-features = false }
-fluvio-types = { version = "0.4.4", path = "crates/fluvio-types", default-features = false }
+fluvio-stream-dispatcher = { version = "0.13.2", path = "crates/fluvio-stream-dispatcher" }
+fluvio-stream-model = { version = "0.11.2", path = "crates/fluvio-stream-model", default-features = false }
+fluvio-types = { version = "0.4.6", path = "crates/fluvio-types", default-features = false }
 fluvio-kv-storage = { path = "crates/fluvio-kv-storage", default-features = false }
 
 # Used to make eyre faster on debug builds

crates/cargo-builder/src/cargo.rs

@@ -132,6 +132,7 @@ mod test {
     use std::ffi::OsStr;
 
     use super::*;
+    const WASM_TARGET: &str = "wasm32-wasi";
 
     #[test]
     fn test_builder_default() {
@@ -164,7 +165,7 @@ mod test {
     fn test_builder_target() {
         let config = Cargo::build()
             .package("foo")
-            .target("wasm32-unknown-unknown")
+            .target(WASM_TARGET)
             .build()
             .expect("should build");
 
@@ -183,7 +184,7 @@ mod test {
                 "-p",
                 "foo",
                 "--target",
-                "wasm32-unknown-unknown"
+                WASM_TARGET
             ]
         );
     }
@@ -192,7 +193,7 @@ mod test {
     fn test_builder_extra_args() {
         let config = Cargo::build()
             .package("foo")
-            .target("wasm32-unknown-unknown")
+            .target(WASM_TARGET)
             .extra_arguments(vec![
                 "--benches".to_string(),
                 "--no-default-features".to_string(),
@@ -215,7 +216,7 @@ mod test {
                 "-p",
                 "foo",
                 "--target",
-                "wasm32-unknown-unknown",
+                WASM_TARGET,
                 "--benches",
                 "--no-default-features"
             ]
@@ -226,7 +227,7 @@ mod test {
     fn test_builder_complains_extra_args_target() {
         let config = Cargo::build()
             .package("foo")
-            .target("wasm32-unknown-unknown")
+            .target(WASM_TARGET)
             .extra_arguments(vec![
                 "--benches".to_string(),
                 "--no-default-features".to_string(),

crates/cargo-builder/src/package.rs

@@ -186,5 +186,9 @@ mod tests {
             .target_wasm32_path()
             .unwrap()
             .ends_with("wasm32-unknown-unknown/release-lto/cargo_builder.wasm"));
+        assert!(package_info
+            .target_wasm32_wasi_path()
+            .unwrap()
+            .ends_with("wasm32-wasi/release-lto/cargo_builder.wasm"));
     }
 }

crates/fluvio-connector-common/Cargo.toml

@@ -10,7 +10,6 @@ description = "Fluvio Connector Common SDK"
 [features]
 default = []
 derive = ["fluvio-connector-derive"]
-wasi = ["fluvio-smartengine/wasi"]
 
 [[test]]
 name = "derive-test"
@@ -33,10 +32,10 @@ fluvio-future = { workspace = true, features = ["subscriber"] }
 fluvio-connector-package = { workspace = true  }
 fluvio-connector-derive = { path = "../fluvio-connector-derive/", optional = true }
 fluvio-sc-schema = { workspace = true }
-fluvio-smartengine = { workspace = true , features = [ "transformation"] }
+fluvio-smartengine = { workspace = true , features = [ "transformation", "engine"] }
 
 
 [dev-dependencies]
-trybuild = { version = "1.0" } # default workspace dep is forked and fails for this crate 
+trybuild = { version = "1.0" } # default workspace dep is forked and fails for this crate
 serde = { workspace = true, features = ["derive"]}
 fluvio = { workspace = true }

crates/fluvio-connector-common/src/consumer.rs

@@ -66,8 +66,12 @@ pub async fn consumer_stream_from_config(
     if let Some(smartmodules) = smartmodule_vec_from_config(config) {
         builder.smartmodule(smartmodules);
     }
-
-    let stream = fluvio.consumer_with_config(builder.build()?).await?;
+    tracing::info!("Building config");
+    let cfg = builder.build().map_err(|e| {
+        tracing::error!("Config build error: {e}");
+        e
+    })?;
+    let stream = fluvio.consumer_with_config(cfg).await?;
 
     Ok((fluvio, stream))
 }

crates/fluvio-smartengine/Cargo.toml

@@ -12,10 +12,9 @@ description = "The official Fluvio SmartEngine"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-engine = ["wasmtime"]
-wasi = ["wasmtime-wasi", "engine"]
+engine = ["wasmtime", "wasmtime-wasi"]
 transformation = ["serde_json", "serde_yaml", "humantime-serde"]
-default = ["engine", "wasi"]
+default = ["engine"]
 
 
 [dependencies]

crates/fluvio-smartengine/src/engine/fixture.rs

@@ -10,7 +10,7 @@ pub(crate) fn read_wasm_module(module_name: &str) -> Vec<u8> {
         .parent()
         .expect("fluvio")
         .join(format!(
-            "smartmodule/examples/target/wasm32-unknown-unknown/release/{module_name}.wasm"
+            "smartmodule/examples/target/wasm32-wasi/release/{module_name}.wasm"
         ));
     read_module_from_path(wasm_path)
 }

crates/fluvio-smartengine/src/engine/wasmtime/state.rs

@@ -18,7 +18,6 @@ pub struct WasmState(Store<Context>);
 
 pub struct Context {
     limiter: StoreResourceLimiter,
-    #[cfg(feature = "wasi")]
     wasi_ctx: wasmtime_wasi::WasiCtx,
 }
 
@@ -54,28 +53,6 @@ impl WasmState {
     }
 }
 
-#[cfg(not(feature = "wasi"))]
-impl WasmState {
-    pub(crate) fn new(engine: &Engine, limiter: StoreResourceLimiter) -> Self {
-        let mut s = Self(Store::new(engine, Context { limiter }));
-        s.0.limiter(|inner| &mut inner.limiter);
-        s.top_up_fuel();
-        s
-    }
-
-    pub(crate) fn instantiate<Params, Args>(
-        &mut self,
-        module: &Module,
-        host_fn: impl IntoFunc<<Self as AsContext>::Data, Params, Args>,
-    ) -> Result<Instance, Error> {
-        use wasmtime::Func;
-
-        let func = Func::wrap(&mut *self, host_fn);
-        Instance::new(self, module, &[func.into()])
-    }
-}
-
-#[cfg(feature = "wasi")]
 impl WasmState {
     pub(crate) fn new(engine: &Engine, limiter: StoreResourceLimiter) -> Self {
         let wasi_ctx = wasmtime_wasi::WasiCtxBuilder::new()

crates/fluvio-spu/Cargo.toml

@@ -66,7 +66,7 @@ fluvio-future = { workspace = true,features = [
     "openssl_tls",
     "zero_copy",
 ] }
-fluvio-smartengine = { workspace = true, optional = true }
+fluvio-smartengine = { workspace = true, optional = true, features = ["engine"] }
 fluvio-smartmodule = { workspace = true}
 fluvio-kv-storage = { workspace = true}
 

crates/fluvio-spu/src/services/public/tests/mod.rs

@@ -91,7 +91,7 @@ fn read_wasm_module(module_name: &str) -> Vec<u8> {
         .parent()
         .expect("fluvio")
         .join(format!(
-            "smartmodule/examples/target/wasm32-unknown-unknown/release/{module_name}.wasm"
+            "smartmodule/examples/target/wasm32-wasi/release/{module_name}.wasm"
         ));
     read_filter_from_path(wasm_path)
 }

crates/fluvio/src/consumer/mod.rs

@@ -316,7 +316,10 @@ where
                             let batch: Result<Batch, _> = raw_batch.try_into();
                             match batch {
                                 Ok(batch) => Ok(batch),
-                                Err(err) => Err(ErrorCode::Other(err.to_string())),
+                                Err(err) => {
+                                    tracing::error!("{err:?}");
+                                    Err(ErrorCode::Other(err.to_string()))
+                                }
                             }
                         });
                 let error = {

crates/smartmodule-development-kit/Cargo.toml

@@ -32,7 +32,7 @@ fluvio = { path = "../fluvio", default-features = false }
 fluvio-hub-util = { path = "../fluvio-hub-util" }
 fluvio-protocol = { path = "../fluvio-protocol", features=["record","api"] }
 fluvio-future = { workspace = true, features = ["subscriber"]}
-fluvio-smartengine = { path = "../fluvio-smartengine", features = ["transformation", "wasi"] }
+fluvio-smartengine = { path = "../fluvio-smartengine", features = ["transformation"] }
 fluvio-extension-common = { path = "../fluvio-extension-common", features = ["target"] }
 fluvio-controlplane-metadata = { path = "../fluvio-controlplane-metadata", features = ["smartmodule"] }
 fluvio-sc-schema = { path = "../fluvio-sc-schema" }

makefiles/check.mk

@@ -25,7 +25,7 @@ check-udeps: install-udeps
 	cargo +nightly udeps --all-targets
 
 install-deny:
-	cargo install --locked cargo-deny	
+	cargo install --locked cargo-deny
 
 check-crate-audit: install-deny
 	cargo deny check
@@ -39,19 +39,16 @@ run-all-unit-test: install_rustup_target
 	cargo test -p fluvio-storage $(BUILD_FLAGS)
 	cargo test -p fluvio-channel-cli $(BUILD_FLAGS)
 	cargo test -p fluvio-connector-derive $(BUILD_FLAGS)
-	cargo test -p fluvio-connector-common --all-features $(BUILD_FLAGS) 
-	cargo test -p fluvio-connector-package $(BUILD_FLAGS) 
+	cargo test -p fluvio-connector-common --all-features $(BUILD_FLAGS)
+	cargo test -p fluvio-connector-package $(BUILD_FLAGS)
 	cargo test -p fluvio-controlplane-metadata --features=smartmodule $(BUILD_FLAGS)
 	make test-all -C crates/fluvio-protocol
 
 run-integration-test: build_smartmodules install_rustup_target
 	cargo test  --lib --all-features $(BUILD_FLAGS) -p fluvio-spu -- --ignored --test-threads=1
 	cargo test  --lib --all-features $(BUILD_FLAGS) -p fluvio-socket -- --ignored --test-threads=1
 	cargo test  --lib --all-features $(BUILD_FLAGS) -p fluvio-service -- --ignored --test-threads=1
-
 	cargo test -p fluvio-smartengine -- --ignored --test-threads=1
-	rustup target add wasm32-wasi
-	cargo test  --features wasi -p fluvio-smartengine -- --ignored --test-threads=1
 
 run-smartmodule-test:	build_smartmodules
 	cargo test  -p fluvio-smartengine -- --ignored --nocapture
@@ -60,12 +57,12 @@ run-k8-test:	install_rustup_target k8-setup build_k8_image
 	cargo test --lib  -p fluvio-sc  -- --ignored --test-threads=1
 
 
-run-all-doc-test: install_rustup_target 
+run-all-doc-test: install_rustup_target
 	cargo test --all-features --doc  $(BUILD_FLAGS)
 
-run-client-doc-test: install_rustup_target 
+run-client-doc-test: install_rustup_target
 	cargo test --all-features --doc -p fluvio-cli $(BUILD_FLAGS)
-	cargo test --all-features --doc -p fluvio-cluster $(BUILD_FLAGS) 
+	cargo test --all-features --doc -p fluvio-cluster $(BUILD_FLAGS)
 	cargo test --all-features --doc -p fluvio $(BUILD_FLAGS)
 
 

smartmodule/cargo_template/README.md

@@ -50,5 +50,5 @@ topic to send data to.
 ```bash
 $ fluvio topic create smartmodule-test
 $ cargo build --release
-$ fluvio consume smartmodule-test -B --{{smartmodule-type}}="target/wasm32-unknown-unknown/release/{{project-name}}"
+$ fluvio consume smartmodule-test -B --{{smartmodule-type}}="target/wasm32-wasi/release/{{project-name}}"
 ```

smartmodule/examples/.cargo/config.toml

@@ -1,2 +1,2 @@
 [build]
-target = "wasm32-unknown-unknown"
+target = "wasm32-wasi"

smartmodule/examples/Makefile

@@ -1,7 +1,6 @@
 default: build
 
 build:
-	rustup target add wasm32-unknown-unknown
 	rustup target add wasm32-wasi
 	cargo build --release
 

smartmodule/examples/array_map_json_array/src/lib.rs

@@ -27,7 +27,7 @@
 //! ```text
 //! $ cd smartmodule/examples
 //! $ cargo build --release
-//! $ fluvio consume array-map-array -B --array-map=target/wasm32-unknown-unknown/release/fluvio_wasm_array_map_array.wasm
+//! $ fluvio consume array-map-array -B --array-map=target/wasm32-wasi/release/fluvio_wasm_array_map_array.wasm
 //! "Apple"
 //! "Banana"
 //! "Cranberry"

smartmodule/examples/array_map_json_array_with_timestamp/src/lib.rs

@@ -27,7 +27,7 @@
 //! ```text
 //! $ cd smartmodule/examples
 //! $ cargo build --release
-//! $ fluvio consume array-map-array -B --array-map=target/wasm32-unknown-unknown/release/fluvio_wasm_array_map_array.wasm
+//! $ fluvio consume array-map-array -B --array-map=target/wasm32-wasi/release/fluvio_wasm_array_map_array.wasm
 //! "Apple"
 //! "Banana"
 //! "Cranberry"

smartmodule/examples/array_map_json_object/src/lib.rs

@@ -45,7 +45,7 @@
 //! ```text
 //! $ cd smartmodule/examples
 //! $ cargo build --release
-//! $ fluvio consume array-map-object -B --key-value --array-map=target/wasm32-unknown-unknown/release/fluvio_wasm_array_map_object.wasm
+//! $ fluvio consume array-map-object -B --key-value --array-map=target/wasm32-wasi/release/fluvio_wasm_array_map_object.wasm
 //! [a] "Apple"
 //! [b] "Banana"
 //! [c] "Cranberry"

smartmodule/examples/filter_json/src/lib.rs

@@ -37,7 +37,7 @@
 //!
 //! ```text
 //! $ cargo build --release -p fluvio-smartmodule-filter-json
-//! $ fluvio consume application-logs -B --smart-stream=target/wasm32-unknown-unknown/release/fluvio-smartmodule-filter-json.wasm
+//! $ fluvio consume application-logs -B --smart-stream=target/wasm32-wasi/release/fluvio-smartmodule-filter-json.wasm
 //! {"level":"info","message":"Server listening on 0.0.0.0:8000"}
 //! {"level":"info","message":"Accepted incoming connection"}
 //! {"level":"warn","message":"Client dropped connection}

smartmodule/regex-filter/README.md

@@ -15,13 +15,13 @@ run regex at top level:
 
 positive:
 ```
-$ target/debug/fluvio sm test --input AA -e regex="[A-Z]" --wasm-file target/wasm32-unknown-unknown/release-lto/fluvio_sm_regex.wasm
+$ target/debug/fluvio sm test --input AA -e regex="[A-Z]" --wasm-file target/wasm32-wasi/release-lto/fluvio_sm_regex.wasm
 1 records
 AA
 ```
 
 negative:
 ```
-$ target/debug/fluvio sm test --input aa -e regex="[A-Z]" --wasm-file target/wasm32-unknown-unknown/release-lto/fluvio_sm_regex.wasm
+$ target/debug/fluvio sm test --input aa -e regex="[A-Z]" --wasm-file target/wasm32-wasi/release-lto/fluvio_sm_regex.wasm
 0 records
 ```

tests/cli/fluvio_smoke_tests/e2e-smartmodule-basic.bats

@@ -13,7 +13,7 @@ load "$TEST_HELPER_DIR"/bats-assert/load.bash
 setup_file() {
     # Compile the smartmodule examples
 #    pushd "$BATS_TEST_DIRNAME/../../.." && make build_smartmodules && popd
-    SMARTMODULE_BUILD_DIR="$BATS_TEST_DIRNAME/../../../smartmodule/examples/target/wasm32-unknown-unknown/release"
+    SMARTMODULE_BUILD_DIR="$BATS_TEST_DIRNAME/../../../smartmodule/examples/target/wasm32-wasi/release"
     export SMARTMODULE_BUILD_DIR
 
 }