From 1209bdbfb891800abf4977d11787b7fef20f5fec Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Wed, 21 Jun 2023 10:41:03 +0300 Subject: [PATCH 1/3] tests: add Wasm modules for testing OCI features. The first such moduels are a test to enumerate default devices in the container and a hello world application, which also gets the current working directory. This can be used to test seccomp filter. Signed-off-by: Ismo Puustinen --- .github/workflows/ci.yml | 4 ++++ Cargo.lock | 4 ++++ Cargo.toml | 1 + Makefile | 3 +++ .../.cargo/config.toml | 2 ++ test/wasi-modules-for-testing/Cargo.toml | 14 +++++++++++++ .../src/has_default_devices.rs | 21 +++++++++++++++++++ .../src/hello_world.rs | 10 +++++++++ 8 files changed, 59 insertions(+) create mode 100644 test/wasi-modules-for-testing/.cargo/config.toml create mode 100644 test/wasi-modules-for-testing/Cargo.toml create mode 100644 test/wasi-modules-for-testing/src/has_default_devices.rs create mode 100644 test/wasi-modules-for-testing/src/hello_world.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e32e911cd..a0ec2bb00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,10 @@ jobs: args: --all --verbose - name: Validate docs run: ./scripts/validate-docs.sh + - name: Setup rust-wasm target and compile test modules + run: | + rustup target add wasm32-wasi + make test/wasm-modules - name: Run tests run: | make test diff --git a/Cargo.lock b/Cargo.lock index e607ed2cc..8286685cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2961,6 +2961,10 @@ dependencies = [ "tar", ] +[[package]] +name = "wasi-modules-for-testing" +version = "0.1.0" + [[package]] name = "wasm-bindgen" version = "0.2.87" diff --git a/Cargo.toml b/Cargo.toml index 48e2eae8b..901fd1ce3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/oci-tar-builder", "crates/containerd-shim-wasmedge", "crates/containerd-shim-wasmtime", + "test/wasi-modules-for-testing", ] [workspace.package] diff --git a/Makefile b/Makefile index 16f06da3c..f40bf6404 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,9 @@ test/k8s: test/k8s/cluster test/k8s/clean: bin/kind bin/kind delete cluster --name $(KIND_CLUSTER_NAME) +test/wasm-modules: + cargo build -p wasi-modules-for-testing --target wasm32-wasi + .PHONY: bin/k3s bin/k3s: mkdir -p bin && \ diff --git a/test/wasi-modules-for-testing/.cargo/config.toml b/test/wasi-modules-for-testing/.cargo/config.toml new file mode 100644 index 000000000..6b77899cb --- /dev/null +++ b/test/wasi-modules-for-testing/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasi" diff --git a/test/wasi-modules-for-testing/Cargo.toml b/test/wasi-modules-for-testing/Cargo.toml new file mode 100644 index 000000000..b004cdcf0 --- /dev/null +++ b/test/wasi-modules-for-testing/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "wasi-modules-for-testing" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "hello-world" +path = "src/hello_world.rs" + +[[bin]] +name = "has-default-devices" +path = "src/has_default_devices.rs" + +[dependencies] diff --git a/test/wasi-modules-for-testing/src/has_default_devices.rs b/test/wasi-modules-for-testing/src/has_default_devices.rs new file mode 100644 index 000000000..6e5222246 --- /dev/null +++ b/test/wasi-modules-for-testing/src/has_default_devices.rs @@ -0,0 +1,21 @@ +use std::path::Path; + +fn main() { + // Runtime must supply at least the following files regardless of OCI devices setting: + let devices = vec![ + "/dev/null", + "/dev/zero", + "/dev/full", + "/dev/random", + "/dev/urandom", + "/dev/tty", + ]; + + for device in devices.iter() { + if Path::new(device).exists() { + println!("{} found", device); + } else { + panic!("{} not found", device); + } + } +} diff --git a/test/wasi-modules-for-testing/src/hello_world.rs b/test/wasi-modules-for-testing/src/hello_world.rs new file mode 100644 index 000000000..003415923 --- /dev/null +++ b/test/wasi-modules-for-testing/src/hello_world.rs @@ -0,0 +1,10 @@ +fn main() { + // Add current working dir request so that we have some known system call to + // test seccomp with. + let cwd = std::env::current_dir().unwrap(); + + println!( + "hello world, current working dir: {}", + cwd.to_string_lossy() + ); +} From c89099f4f0a7a781426b7524000e944398594c5d Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Wed, 21 Jun 2023 10:43:26 +0300 Subject: [PATCH 2/3] tests: add integration tests for OCI seccomp and devices support. Signed-off-by: Ismo Puustinen --- Cargo.lock | 20 +++ Cargo.toml | 1 + test/oci-tests/Cargo.toml | 31 +++++ test/oci-tests/tests/common/mod.rs | 100 +++++++++++++ test/oci-tests/tests/devices.rs | 53 +++++++ test/oci-tests/tests/seccomp.rs | 217 +++++++++++++++++++++++++++++ 6 files changed, 422 insertions(+) create mode 100644 test/oci-tests/Cargo.toml create mode 100644 test/oci-tests/tests/common/mod.rs create mode 100644 test/oci-tests/tests/devices.rs create mode 100644 test/oci-tests/tests/seccomp.rs diff --git a/Cargo.lock b/Cargo.lock index 8286685cb..92144a61c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,6 +1829,26 @@ dependencies = [ "tar", ] +[[package]] +name = "oci-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "containerd-shim-wasm", + "containerd-shim-wasmedge", + "containerd-shim-wasmtime", + "libc", + "oci-spec", + "pretty_assertions", + "serde", + "serde_json", + "serial_test", + "tempfile", + "wasmedge-sdk", + "wasmtime", +] + [[package]] name = "once_cell" version = "1.18.0" diff --git a/Cargo.toml b/Cargo.toml index 901fd1ce3..f143db25c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/containerd-shim-wasmedge", "crates/containerd-shim-wasmtime", "test/wasi-modules-for-testing", + "test/oci-tests", ] [workspace.package] diff --git a/test/oci-tests/Cargo.toml b/test/oci-tests/Cargo.toml new file mode 100644 index 000000000..fccd92867 --- /dev/null +++ b/test/oci-tests/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "oci-tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = { workspace = true } +chrono = { workspace = true } +containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm" } +containerd-shim-wasmedge = { path = "../../crates/containerd-shim-wasmedge" } +containerd-shim-wasmtime = { path = "../../crates/containerd-shim-wasmtime" } +libc = { workspace = true } +oci-spec = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tempfile = "3.0" +wasmedge-sdk = { version = "0.10.1", features = [ "standalone", "static" ] } +wasmtime = { version = "11.0", default-features = false, features = [ + 'cache', + 'wat', + 'jitdump', + 'parallel-compilation', + 'cranelift', + 'pooling-allocator', + 'vtune', +]} + +[dev-dependencies] +tempfile = "3.0" +pretty_assertions = "1" +serial_test = "*" \ No newline at end of file diff --git a/test/oci-tests/tests/common/mod.rs b/test/oci-tests/tests/common/mod.rs new file mode 100644 index 000000000..ae36cdc5b --- /dev/null +++ b/test/oci-tests/tests/common/mod.rs @@ -0,0 +1,100 @@ +use std::fs::{create_dir, read_to_string, File, OpenOptions}; +use std::io::prelude::*; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::channel; +use std::time::Duration; + +use libc::SIGKILL; +use oci_spec::runtime::Spec; +use serde::{Deserialize, Serialize}; +use tempfile::tempdir; + +use containerd_shim_wasm::sandbox::instance::Wait; +use containerd_shim_wasm::sandbox::{EngineGetter, Error, Instance, InstanceConfig}; + +#[derive(Serialize, Deserialize)] +struct Options { + root: Option, +} + +pub static WASM_FILENAME: &str = "./file.wasm"; + +pub(crate) fn get_external_wasm_module(name: String) -> Result, Error> { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let target = Path::new(manifest_dir) + .join("../../target/wasm32-wasi/debug") + .join(name.clone()); + std::fs::read(target).map_err(|e| { + Error::Others(format!( + "failed to read requested Wasm module ({}): {}. Perhaps you need to run 'make test/wasm-modules' first.", + name, e + )) + }) +} + +pub(crate) fn run_test_with_spec(spec: &Spec, bytes: &[u8]) -> Result<(String, u32), Error> +where + I: Instance + EngineGetter, + E: Sync + Send + Clone, +{ + let dir = tempdir().unwrap(); + create_dir(dir.path().join("rootfs"))?; + let rootdir = dir.path().join("runwasi"); + create_dir(&rootdir)?; + let opts = Options { + root: Some(rootdir), + }; + let opts_file = OpenOptions::new() + .read(true) + .create(true) + .truncate(true) + .write(true) + .open(dir.path().join("options.json"))?; + write!(&opts_file, "{}", serde_json::to_string(&opts)?)?; + + let wasm_path = dir.path().join("rootfs").join(WASM_FILENAME); + let mut f = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o755) + .open(wasm_path)?; + f.write_all(bytes)?; + + let stdout = File::create(dir.path().join("stdout"))?; + drop(stdout); + + spec.save(dir.path().join("config.json"))?; + + let mut cfg = InstanceConfig::new( + I::new_engine()?, + "test_namespace".into(), + "/containerd/address".into(), + ); + let cfg = cfg + .set_bundle(dir.path().to_str().unwrap().to_string()) + .set_stdout(dir.path().join("stdout").to_str().unwrap().to_string()); + + let wasi = I::new("test".to_string(), Some(cfg)); + + wasi.start()?; + + let (tx, rx) = channel(); + let waiter = Wait::new(tx); + wasi.wait(&waiter).unwrap(); + + let res = match rx.recv_timeout(Duration::from_secs(600)) { + Ok(res) => res, + Err(e) => { + wasi.kill(SIGKILL as u32).unwrap(); + return Err(Error::Others(format!( + "error waiting for module to finish: {0}", + e + ))); + } + }; + wasi.delete()?; + let output = read_to_string(dir.path().join("stdout"))?; + Ok((output, res.0)) +} diff --git a/test/oci-tests/tests/devices.rs b/test/oci-tests/tests/devices.rs new file mode 100644 index 000000000..b34b43267 --- /dev/null +++ b/test/oci-tests/tests/devices.rs @@ -0,0 +1,53 @@ +use std::borrow::Cow; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serial_test::serial; +use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; +use wasmedge_sdk::Vm as WasmEdgeVm; +use wasmtime::Engine as WasmtimeVm; + +use containerd_shim_wasm::function; +use containerd_shim_wasm::sandbox::testutil::{has_cap_sys_admin, run_test_with_sudo}; +use containerd_shim_wasm::sandbox::Error; + +use containerd_shim_wasmedge::instance::Wasi as WasmEdgeWasi; +use containerd_shim_wasmtime::instance::Wasi as WasmtimeWasi; + +#[derive(Serialize, Deserialize)] +struct Options { + root: Option, +} + +mod common; + +#[test] +#[serial] +fn test_has_default_devices() -> Result<(), Error> { + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo("test_has_default_devices"); + } + + let wasmbytes = common::get_external_wasm_module("has-default-devices.wasm".to_string())?; + + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![common::WASM_FILENAME.to_string()]) + .build()?, + ) + .build()?; + + let bytes = Cow::from(wasmbytes); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0, "error: {}", output); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0, "error: {}", output); + + Ok(()) +} diff --git a/test/oci-tests/tests/seccomp.rs b/test/oci-tests/tests/seccomp.rs new file mode 100644 index 000000000..af66cc768 --- /dev/null +++ b/test/oci-tests/tests/seccomp.rs @@ -0,0 +1,217 @@ +use std::borrow::Cow; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serial_test::serial; +use oci_spec::runtime::{ + LinuxBuilder, LinuxSeccompAction, LinuxSeccompBuilder, LinuxSyscallBuilder, ProcessBuilder, + RootBuilder, SpecBuilder, +}; +use wasmedge_sdk::Vm as WasmEdgeVm; +use wasmtime::Engine as WasmtimeVm; + +use containerd_shim_wasm::function; +use containerd_shim_wasm::sandbox::testutil::{has_cap_sys_admin, run_test_with_sudo}; +use containerd_shim_wasm::sandbox::Error; + +use containerd_shim_wasmedge::instance::Wasi as WasmEdgeWasi; +use containerd_shim_wasmtime::instance::Wasi as WasmtimeWasi; + +#[derive(Serialize, Deserialize)] +struct Options { + root: Option, +} + +mod common; + +#[test] +#[serial] +fn test_external_hello_world() -> Result<(), Error> { + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo("test_external_hello_world"); + } + + let wasmbytes = common::get_external_wasm_module("hello-world.wasm".to_string())?; + + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![common::WASM_FILENAME.to_string()]) + .build()?, + ) + .build()?; + + let bytes = Cow::from(wasmbytes); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + + Ok(()) +} + +#[test] +#[serial] +fn test_seccomp_hello_world_pass() -> Result<(), Error> { + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo("test_seccomp_hello_world_pass"); + } + + let wasmbytes = common::get_external_wasm_module("hello-world.wasm".to_string())?; + + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![common::WASM_FILENAME.to_string()]) + .build()?, + ) + .linux( + LinuxBuilder::default() + .seccomp( + LinuxSeccompBuilder::default() + .default_action(LinuxSeccompAction::ScmpActAllow) + .architectures(vec![oci_spec::runtime::Arch::ScmpArchNative]) + .syscalls(vec![LinuxSyscallBuilder::default() + .names(vec!["getcwd".to_string()]) + .action(LinuxSeccompAction::ScmpActAllow) + .build()?]) + .build()?, + ) + .build()?, + ) + .build()?; + + let bytes = Cow::from(wasmbytes); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + + let (output, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + + Ok(()) +} + +#[test] +#[serial] +fn test_seccomp_hello_world_fail() -> Result<(), Error> { + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo("test_seccomp_hello_world_fail"); + } + + let wasmbytes = common::get_external_wasm_module("hello-world.wasm".to_string())?; + + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![common::WASM_FILENAME.to_string()]) + .build()?, + ) + .linux( + LinuxBuilder::default() + .seccomp( + LinuxSeccompBuilder::default() + .default_action(LinuxSeccompAction::ScmpActAllow) + .architectures(vec![oci_spec::runtime::Arch::ScmpArchNative]) + .syscalls(vec![LinuxSyscallBuilder::default() + .names(vec!["sched_getaffinity".to_string(), "getcwd".to_string()]) // Do not allow sched_getaffinity() + .action(LinuxSeccompAction::ScmpActKill) + .build()?]) + .build()?, + ) + .build()?, + ) + .build()?; + + let bytes = Cow::from(wasmbytes); + + let (_, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_ne!(retval, 0); + + let (_, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_ne!(retval, 0); + + Ok(()) +} + +#[test] +#[serial] +#[ignore] +fn test_seccomp_hello_world_notify() -> Result<(), Error> { + // Test how seccomp works together with an external notification agent. + // Configure the external agent to use socket /tmp/seccomp-agent.socket + // and set it to either allow or decline (with error) "getcwd" system + // call. Then configure success_expected to true if allowed and false + // if declined. + + let success_expected = true; + + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo(function!()); + } + + let wasmbytes = common::get_external_wasm_module("hello-world.wasm".to_string())?; + + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![common::WASM_FILENAME.to_string()]) + .build()?, + ) + .linux( + LinuxBuilder::default() + .seccomp( + LinuxSeccompBuilder::default() + .default_action(LinuxSeccompAction::ScmpActAllow) + .architectures(vec![oci_spec::runtime::Arch::ScmpArchNative]) + .syscalls(vec![LinuxSyscallBuilder::default() + .names(vec!["getcwd".to_string()]) // getcwd() is checked from an external process + .action(LinuxSeccompAction::ScmpActNotify) + .build()?]) + .listener_path("/tmp/seccomp-agent.socket") + .build()?, + ) + .build()?, + ) + .build()?; + + let bytes = Cow::from(wasmbytes); + + if success_expected { + let (output, retval) = + common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + + let (output, retval) = + common::run_test_with_spec::(&spec, &bytes)?; + assert_eq!(retval, 0); + assert!(output.starts_with("hello world")); + } else { + let (_, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_ne!(retval, 0); + + let (_, retval) = common::run_test_with_spec::(&spec, &bytes)?; + assert_ne!(retval, 0); + } + + Ok(()) +} From ed7d60f4a5255ad670257cacee418b4959679d82 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Tue, 25 Jul 2023 11:41:58 +0300 Subject: [PATCH 3/3] wasmedge: add support for rootfs files. Enable WasmEdge modules to access the container filesystem. This includes: * all the files on the container image * /proc filesystem * /sys filesysten * parts of /dev filesystem Signed-off-by: Ismo Puustinen --- crates/containerd-shim-wasmedge/src/executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/containerd-shim-wasmedge/src/executor.rs b/crates/containerd-shim-wasmedge/src/executor.rs index 21ab3bf76..3d0648585 100644 --- a/crates/containerd-shim-wasmedge/src/executor.rs +++ b/crates/containerd-shim-wasmedge/src/executor.rs @@ -71,7 +71,7 @@ impl WasmEdgeExecutor { wasi_module.initialize( Some(args.iter().map(|s| s as &str).collect()), Some(envs.iter().map(|s| s as &str).collect()), - None, + Some(vec!["/:/"]), // map the container root filesystem to be available to the WASI module ); let vm = vm .register_module_from_file("main", cmd)