diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 160ba2b52..3679e2c86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,8 +79,19 @@ jobs: - uses: actions/checkout@v3 - name: setup rust-wasm target run: rustup target add wasm32-wasi + - name: Setup OCI runtime build env + run: | + sudo apt -y update + sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev - name: run - run: make test/k8s + run: | + make test/k8s + # only runs when the previous step fails + - name: inspect failed pods + if: failure() + run: | + kubectl get pods --all-namespaces + kubectl describe pods --all-namespaces - name: cleanup if: always() run: make test/k8s/clean diff --git a/Cargo.lock b/Cargo.lock index 4d20f7299..995e78c41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,12 +58,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "0.3.2" @@ -289,12 +283,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.0.79" @@ -350,33 +338,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "ciborium" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" - -[[package]] -name = "ciborium-ll" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "clang-sys" version = "1.6.1" @@ -492,23 +453,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "containerd-shim-benchmarks" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "containerd-shim-wasm", - "containerd-shim-wasmedge", - "containerd-shim-wasmtime", - "criterion", - "libc", - "oci-spec", - "serde", - "serde_json", - "tempfile", -] - [[package]] name = "containerd-shim-protos" version = "0.3.0" @@ -580,11 +524,14 @@ dependencies = [ "chrono", "containerd-shim", "containerd-shim-wasm", + "env_logger", "libc", + "libcontainer", "log", "nix 0.26.2", "oci-spec", "pretty_assertions", + "serde", "serde_json", "tempfile", "thiserror", @@ -735,42 +682,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -1323,12 +1234,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - [[package]] name = "hashbrown" version = "0.12.3" @@ -1900,12 +1805,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" - [[package]] name = "os_pipe" version = "1.1.4" @@ -2013,34 +1912,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "plotters" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" - -[[package]] -name = "plotters-svg" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" -dependencies = [ - "plotters-backend", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2452,15 +2323,6 @@ dependencies = [ "libc", ] -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2753,16 +2615,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -2972,16 +2824,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "walkdir" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3518,16 +3360,6 @@ dependencies = [ "wast 60.0.0", ] -[[package]] -name = "web-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "which" version = "4.4.0" diff --git a/Cargo.toml b/Cargo.toml index 3f761f484..3a02da362 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "crates/oci-tar-builder", "crates/containerd-shim-wasmedge", "crates/containerd-shim-wasmtime", - "benches/containerd-shim-benchmarks", ] [workspace.package] @@ -18,6 +17,7 @@ homepage = "https://github.com/containerd/runwasi" [workspace.dependencies] anyhow = "1.0" +containerd-shim-wasm = { path = "crates/containerd-shim-wasm" } serde = "1.0" serde_json = "1.0" env_logger = "0.10" diff --git a/Makefile b/Makefile index c38680842..9041b3e7d 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ fix: .PHONY: test test: - cargo test --all --verbose + RUST_LOG=trace cargo test --all --verbose -- --nocapture .PHONY: install install: diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index b12b68a1f..2a62b058b 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -42,4 +42,4 @@ rand = "0.8" [features] default = [] generate_bindings = ["ttrpc-codegen"] -generate_doc = [] +generate_doc = [] \ No newline at end of file diff --git a/crates/containerd-shim-wasm/src/sandbox/instance.rs b/crates/containerd-shim-wasm/src/sandbox/instance.rs index 4007edfb1..3cc2941df 100644 --- a/crates/containerd-shim-wasm/src/sandbox/instance.rs +++ b/crates/containerd-shim-wasm/src/sandbox/instance.rs @@ -19,7 +19,7 @@ pub struct InstanceConfig where E: Send + Sync + Clone, { - /// The wasm engine to use. + /// The WASI engine to use. /// This should be cheap to clone. engine: E, /// Optional stdin named pipe path. @@ -104,21 +104,27 @@ where } } -/// Represents a wasi module(s). +/// Represents a WASI module(s). /// Instance is a trait that gets implemented by consumers of this library. pub trait Instance { + /// The WASI engine type type E: Send + Sync + Clone; + /// Create a new instance fn new(id: String, cfg: Option<&InstanceConfig>) -> Self; + /// Start the instance /// The returned value should be a unique ID (such as a PID) for the instance. /// Nothing internally should be using this ID, but it is returned to containerd where a user may want to use it. fn start(&self) -> Result; + /// Send a signal to the instance fn kill(&self, signal: u32) -> Result<(), Error>; + /// Delete any reference to the instance /// This is called after the instance has exited. fn delete(&self) -> Result<(), Error>; + /// Set up waiting for the instance to exit /// The Wait struct is used to send the exit code and time back to the /// caller. The recipient is expected to call function diff --git a/crates/containerd-shim-wasm/src/sandbox/instance_utils.rs b/crates/containerd-shim-wasm/src/sandbox/instance_utils.rs new file mode 100644 index 000000000..14d046199 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/instance_utils.rs @@ -0,0 +1,83 @@ +//! Common utilities for the containerd shims. +use crate::sandbox::error::Error; +use anyhow::{bail, Context, Result}; +use std::{ + fs::{self, OpenOptions}, + io::ErrorKind, + os::fd::{IntoRawFd, RawFd}, + path::{Path, PathBuf}, +}; + +/// Return the root path for the instance. +/// +/// The root path is the path to the directory containing the container's state. +pub fn get_instance_root>( + root_path: P, + instance_id: &str, +) -> Result { + let instance_root = construct_instance_root(root_path, instance_id)?; + if !instance_root.exists() { + bail!("container {} does not exist.", instance_id) + } + Ok(instance_root) +} + +/// Checks if the container exists. +/// +/// The root path is the path to the directory containing the container's state. +pub fn instance_exists>(root_path: P, container_id: &str) -> Result { + let instance_root = construct_instance_root(root_path, container_id)?; + Ok(instance_root.exists()) +} + +/// containerd can send an empty path or a non-existant path +/// In both these cases we should just assume that the stdio stream was not setup (intentionally) +/// Any other error is a real error. +pub fn maybe_open_stdio(path: &str) -> Result, Error> { + if path.is_empty() { + return Ok(None); + } + match OpenOptions::new().read(true).write(true).open(path) { + Ok(f) => Ok(Some(f.into_raw_fd())), + Err(err) => match err.kind() { + ErrorKind::NotFound => Ok(None), + _ => Err(err.into()), + }, + } +} + +fn construct_instance_root>(root_path: P, container_id: &str) -> Result { + let root_path = fs::canonicalize(&root_path).with_context(|| { + format!( + "failed to canonicalize {} for container {}", + root_path.as_ref().display(), + container_id + ) + })?; + Ok(root_path.join(container_id)) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + + use tempfile::tempdir; + + use super::*; + + #[test] + fn test_maybe_open_stdio() -> Result<(), Error> { + let f = maybe_open_stdio("")?; + assert!(f.is_none()); + + let f = maybe_open_stdio("/some/nonexistent/path")?; + assert!(f.is_none()); + + let dir = tempdir()?; + let temp = File::create(dir.path().join("testfile"))?; + drop(temp); + let f = maybe_open_stdio(dir.path().join("testfile").as_path().to_str().unwrap())?; + assert!(f.is_some()); + Ok(()) + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 72d7ac2f3..7494815a7 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -6,6 +6,7 @@ pub mod cgroups; pub mod error; pub mod exec; pub mod instance; +pub mod instance_utils; pub mod manager; pub mod shim; diff --git a/crates/containerd-shim-wasmedge/Cargo.toml b/crates/containerd-shim-wasmedge/Cargo.toml index afc6e271d..5d8f2745f 100644 --- a/crates/containerd-shim-wasmedge/Cargo.toml +++ b/crates/containerd-shim-wasmedge/Cargo.toml @@ -4,8 +4,8 @@ version.workspace = true edition.workspace = true [dependencies] -containerd-shim ={ workspace = true } -containerd-shim-wasm = { path = "../containerd-shim-wasm" } +containerd-shim = { workspace = true } +containerd-shim-wasm = { workspace = true } log = { workspace = true } ttrpc = { workspace = true } wasmedge-sdk = "0.8.1" diff --git a/crates/containerd-shim-wasmedge/src/executor.rs b/crates/containerd-shim-wasmedge/src/executor.rs index 1158271bc..21ab3bf76 100644 --- a/crates/containerd-shim-wasmedge/src/executor.rs +++ b/crates/containerd-shim-wasmedge/src/executor.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use containerd_shim_wasm::sandbox::oci; use nix::unistd::{dup, dup2}; use oci_spec::runtime::Spec; @@ -22,84 +23,72 @@ pub struct WasmEdgeExecutor { impl Executor for WasmEdgeExecutor { fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> { // parse wasi parameters - let args = get_args(spec); + let args = oci::get_args(spec); if args.is_empty() { return Err(ExecutorError::InvalidArg); } + let vm = self + .prepare(args, spec) + .map_err(|err| ExecutorError::Other(format!("failed to prepare function: {}", err)))?; + + // TODO: How to get exit code? + // This was relatively straight forward in go, but wasi and wasmtime are totally separate things in rust + match vm.run_func(Some("main"), "_start", params!()) { + Ok(_) => std::process::exit(0), + Err(_) => std::process::exit(137), + }; + } + + fn can_handle(&self, _spec: &Spec) -> bool { + true + } + + fn name(&self) -> &'static str { + EXECUTOR_NAME + } +} + +impl WasmEdgeExecutor { + fn prepare(&self, args: &[String], spec: &Spec) -> anyhow::Result { let mut cmd = args[0].clone(); if let Some(stripped) = args[0].strip_prefix(std::path::MAIN_SEPARATOR) { cmd = stripped.to_string(); } let envs = env_to_wasi(spec); - - // create configuration with `wasi` option enabled let config = ConfigBuilder::new(CommonConfigOptions::default()) .with_host_registration_config(HostRegistrationConfigOptions::default().wasi(true)) .build() .map_err(|err| ExecutorError::Execution(err))?; - - // create a vm with the config settings let mut vm = VmBuilder::new() .with_config(config) .build() .map_err(|err| ExecutorError::Execution(err))?; - - // initialize the wasi module with the parsed parameters let wasi_module = vm .wasi_module_mut() .ok_or_else(|| anyhow::Error::msg("Not found wasi module")) .map_err(|err| ExecutorError::Execution(err.into()))?; - wasi_module.initialize( Some(args.iter().map(|s| s as &str).collect()), Some(envs.iter().map(|s| s as &str).collect()), None, ); - let vm = vm .register_module_from_file("main", cmd) .map_err(|err| ExecutorError::Execution(err))?; - if let Some(stdin) = self.stdin { - let _ = dup(STDIN_FILENO); - let _ = dup2(stdin, STDIN_FILENO); + dup(STDIN_FILENO)?; + dup2(stdin, STDIN_FILENO)?; } if let Some(stdout) = self.stdout { - let _ = dup(STDOUT_FILENO); - let _ = dup2(stdout, STDOUT_FILENO); + dup(STDOUT_FILENO)?; + dup2(stdout, STDOUT_FILENO)?; } if let Some(stderr) = self.stderr { - let _ = dup(STDERR_FILENO); - let _ = dup2(stderr, STDERR_FILENO); + dup(STDERR_FILENO)?; + dup2(stderr, STDERR_FILENO)?; } - - // TODO: How to get exit code? - // This was relatively straight forward in go, but wasi and wasmtime are totally separate things in rust - match vm.run_func(Some("main"), "_start", params!()) { - Ok(_) => std::process::exit(0), - Err(_) => std::process::exit(137), - }; - } - - fn can_handle(&self, _spec: &Spec) -> bool { - true - } - - fn name(&self) -> &'static str { - EXECUTOR_NAME - } -} - -fn get_args(spec: &Spec) -> &[String] { - let p = match spec.process() { - None => return &[], - Some(p) => p, - }; - - match p.args() { - None => &[], - Some(args) => args.as_slice(), + Ok(vm) } } diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index ee3bb0ede..33609343a 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,14 +1,18 @@ -use std::fs::{File, OpenOptions}; +use std::fs::File; use std::io::prelude::*; use std::io::ErrorKind; -use std::os::unix::io::{IntoRawFd, RawFd}; +use std::os::unix::io::RawFd; use std::sync::{Arc, Condvar, Mutex}; use std::thread; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::Context; +use anyhow::{anyhow, Result}; use chrono::{DateTime, Utc}; use containerd_shim_wasm::sandbox::error::Error; use containerd_shim_wasm::sandbox::instance::Wait; +use containerd_shim_wasm::sandbox::instance_utils::{ + get_instance_root, instance_exists, maybe_open_stdio, +}; use containerd_shim_wasm::sandbox::{EngineGetter, Instance, InstanceConfig}; use libc::{dup2, SIGINT, SIGKILL, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use log::{debug, error}; @@ -55,73 +59,6 @@ pub struct Wasi { rootdir: PathBuf, } -fn construct_container_root>(root_path: P, container_id: &str) -> Result { - let root_path = fs::canonicalize(&root_path).with_context(|| { - format!( - "failed to canonicalize {} for container {}", - root_path.as_ref().display(), - container_id - ) - })?; - Ok(root_path.join(container_id)) -} - -fn load_container>(root_path: P, container_id: &str) -> Result { - let container_root = construct_container_root(root_path, container_id)?; - if !container_root.exists() { - bail!("container {} does not exist.", container_id) - } - - Container::load(container_root) - .with_context(|| format!("could not load state for container {container_id}")) -} - -fn container_exists>(root_path: P, container_id: &str) -> Result { - let container_root = construct_container_root(root_path, container_id)?; - Ok(container_root.exists()) -} - -#[cfg(test)] -mod tests { - use std::fs::File; - - use tempfile::tempdir; - - use super::*; - - #[test] - fn test_maybe_open_stdio() -> Result<(), Error> { - let f = maybe_open_stdio("")?; - assert!(f.is_none()); - - let f = maybe_open_stdio("/some/nonexistent/path")?; - assert!(f.is_none()); - - let dir = tempdir()?; - let temp = File::create(dir.path().join("testfile"))?; - drop(temp); - let f = maybe_open_stdio(dir.path().join("testfile").as_path().to_str().unwrap())?; - assert!(f.is_some()); - Ok(()) - } -} - -/// containerd can send an empty path or a non-existant path -/// In both these cases we should just assume that the stdio stream was not setup (intentionally) -/// Any other error is a real error. -fn maybe_open_stdio(path: &str) -> Result, Error> { - if path.is_empty() { - return Ok(None); - } - match OpenOptions::new().read(true).write(true).open(path) { - Ok(f) => Ok(Some(f.into_raw_fd())), - Err(err) => match err.kind() { - ErrorKind::NotFound => Ok(None), - _ => Err(err.into()), - }, - } -} - pub fn reset_stdio() { unsafe { if STDIN_FD.is_some() { @@ -160,45 +97,6 @@ fn determine_rootdir>(bundle: P, namespace: String) -> Result Result<(), Error> { - let namespace = "test_namespace"; - let dir = tempdir()?; - let rootdir = dir.path().join("runwasi"); - let opts = Options { - root: Some(rootdir.clone()), - }; - let opts_file = OpenOptions::new() - .read(true) - .create(true) - .truncate(true) - .write(true) - .open(dir.path().join("options.json"))?; - write!(&opts_file, "{}", serde_json::to_string(&opts)?)?; - let root = determine_rootdir(dir.path(), namespace.into())?; - assert_eq!(root, rootdir.join(namespace)); - Ok(()) - } - - #[test] - fn test_determine_rootdir_without_options_file() -> Result<(), Error> { - let dir = tempdir()?; - let namespace = "test_namespace"; - let root = determine_rootdir(dir.path(), namespace.into())?; - assert!(root.is_absolute()); - assert_eq!( - root, - PathBuf::from(DEFAULT_CONTAINER_ROOT_DIR).join(namespace) - ); - Ok(()) - } -} - impl Instance for Wasi { type E = Vm; fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { @@ -248,6 +146,7 @@ impl Instance for Wasi { Ok(_) => 0, Err(e) => { if e == Errno::ECHILD { + log::info!("no child process"); 0 } else { panic!("waitpid failed: {}", e); @@ -272,7 +171,13 @@ impl Instance for Wasi { ))?, }; - let mut container = load_container(&self.rootdir, self.id.as_str())?; + let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; + let mut container = Container::load(container_root).with_context(|| { + format!( + "could not load state for container {id}", + id = self.id.as_str() + ) + })?; match container.kill(signal, true) { Ok(_) => Ok(()), Err(e) => { @@ -285,7 +190,7 @@ impl Instance for Wasi { } fn delete(&self) -> Result<(), Error> { - match container_exists(&self.rootdir, self.id.as_str()) { + match instance_exists(&self.rootdir, self.id.as_str()) { Ok(exists) => { if !exists { return Ok(()); @@ -296,7 +201,14 @@ impl Instance for Wasi { return Ok(()); } } - match load_container(&self.rootdir, self.id.as_str()) { + let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; + let container = Container::load(container_root).with_context(|| { + format!( + "could not load state for container {id}", + id = self.id.as_str() + ) + }); + match container { Ok(mut container) => container.delete(true).map_err(|err| { Error::Any(anyhow!("failed to delete container {}: {}", self.id, err)) })?, @@ -337,12 +249,33 @@ impl Wasi { } } +impl EngineGetter for Wasi { + type E = Vm; + fn new_engine() -> Result { + PluginManager::load(None).unwrap(); + let mut host_options = HostRegistrationConfigOptions::default(); + host_options = host_options.wasi(true); + #[cfg(all(target_os = "linux", feature = "wasi_nn", target_arch = "x86_64"))] + { + host_options = host_options.wasi_nn(true); + } + let config = ConfigBuilder::new(CommonConfigOptions::default()) + .with_host_registration_config(host_options) + .build() + .map_err(anyhow::Error::msg)?; + let vm = VmBuilder::new() + .with_config(config) + .build() + .map_err(anyhow::Error::msg)?; + Ok(vm) + } +} + #[cfg(test)] mod wasitest { use std::borrow::Cow; - use std::fs::{create_dir, read_to_string, File}; - use std::io::prelude::*; - use std::os::unix::fs::OpenOptionsExt; + use std::fs::{create_dir, read_to_string, File, OpenOptions}; + use std::os::unix::prelude::OpenOptionsExt; use std::sync::mpsc::channel; use std::time::Duration; @@ -523,24 +456,43 @@ mod wasitest { } } -impl EngineGetter for Wasi { - type E = Vm; - fn new_engine() -> Result { - PluginManager::load(None).unwrap(); - let mut host_options = HostRegistrationConfigOptions::default(); - host_options = host_options.wasi(true); - #[cfg(all(target_os = "linux", feature = "wasi_nn", target_arch = "x86_64"))] - { - host_options = host_options.wasi_nn(true); - } - let config = ConfigBuilder::new(CommonConfigOptions::default()) - .with_host_registration_config(host_options) - .build() - .map_err(anyhow::Error::msg)?; - let vm = VmBuilder::new() - .with_config(config) - .build() - .map_err(anyhow::Error::msg)?; - Ok(vm) +#[cfg(test)] +mod rootdirtest { + use std::fs::OpenOptions; + + use super::*; + use tempfile::tempdir; + + #[test] + fn test_determine_rootdir_with_options_file() -> Result<(), Error> { + let namespace = "test_namespace"; + let dir = tempdir()?; + let rootdir = dir.path().join("runwasi"); + let opts = Options { + root: Some(rootdir.clone()), + }; + let opts_file = OpenOptions::new() + .read(true) + .create(true) + .truncate(true) + .write(true) + .open(dir.path().join("options.json"))?; + write!(&opts_file, "{}", serde_json::to_string(&opts)?)?; + let root = determine_rootdir(dir.path(), namespace.into())?; + assert_eq!(root, rootdir.join(namespace)); + Ok(()) + } + + #[test] + fn test_determine_rootdir_without_options_file() -> Result<(), Error> { + let dir = tempdir()?; + let namespace = "test_namespace"; + let root = determine_rootdir(dir.path(), namespace.into())?; + assert!(root.is_absolute()); + assert_eq!( + root, + PathBuf::from(DEFAULT_CONTAINER_ROOT_DIR).join(namespace) + ); + Ok(()) } } diff --git a/crates/containerd-shim-wasmtime/Cargo.toml b/crates/containerd-shim-wasmtime/Cargo.toml index ccb583772..dfe12ed63 100644 --- a/crates/containerd-shim-wasmtime/Cargo.toml +++ b/crates/containerd-shim-wasmtime/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] containerd-shim = { workspace = true } -containerd-shim-wasm = { path = "../containerd-shim-wasm" } +containerd-shim-wasm = { workspace = true } log = { workspace = true } ttrpc = { workspace = true } @@ -33,11 +33,15 @@ oci-spec = { workspace = true, features = ["runtime"] } thiserror = { workspace = true } serde_json = { workspace = true } nix = { workspace = true } +libcontainer = { workspace = true } +serde = { workspace = true } +libc = { workspace = true } [dev-dependencies] tempfile = "3.0" libc = { workspace = true } pretty_assertions = "1" +env_logger = "0.10" [[bin]] name = "containerd-shim-wasmtime-v1" diff --git a/crates/containerd-shim-wasmtime/src/executor.rs b/crates/containerd-shim-wasmtime/src/executor.rs new file mode 100644 index 000000000..cf21ffeb4 --- /dev/null +++ b/crates/containerd-shim-wasmtime/src/executor.rs @@ -0,0 +1,113 @@ +use nix::unistd::{dup, dup2}; +use std::{fs::OpenOptions, os::fd::RawFd}; + +use anyhow::{anyhow, Context, Result}; +use containerd_shim_wasm::sandbox::oci; +use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use libcontainer::workload::{Executor, ExecutorError}; +use oci_spec::runtime::Spec; + +use wasmtime::{Engine, Linker, Module, Store}; +use wasmtime_wasi::WasiCtxBuilder; + +use crate::oci_wasmtime::{self, wasi_dir}; + +const EXECUTOR_NAME: &str = "wasmtime"; + +pub struct WasmtimeExecutor { + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, + pub engine: Engine, +} + +impl Executor for WasmtimeExecutor { + fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> { + let args = oci::get_args(spec); + if args.len() != 1 { + return Err(ExecutorError::InvalidArg); + } + + let (mut store, f) = self + .prepare(spec, args) + .map_err(|err| ExecutorError::Other(format!("failed to prepare function: {}", err)))?; + + log::info!("calling start function"); + match f.call(&mut store, &[], &mut []) { + Ok(_) => std::process::exit(0), + Err(_) => std::process::exit(137), + }; + } + + fn can_handle(&self, _spec: &Spec) -> bool { + true + } + + fn name(&self) -> &'static str { + EXECUTOR_NAME + } +} + +impl WasmtimeExecutor { + fn prepare( + &self, + spec: &Spec, + args: &[String], + ) -> anyhow::Result<(Store, wasmtime::Func)> { + // already in the cgroup + let env = oci_wasmtime::env_to_wasi(spec); + log::info!("setting up wasi"); + + let path = wasi_dir(".", OpenOptions::new().read(true))?; + let wasi_builder = WasiCtxBuilder::new() + .args(args)? + .envs(env.as_slice())? + .inherit_stdio() + .preopened_dir(path, "/")?; + + if let Some(stdin) = self.stdin { + dup(STDIN_FILENO)?; + dup2(stdin, STDIN_FILENO)?; + } + if let Some(stdout) = self.stdout { + dup(STDOUT_FILENO)?; + dup2(stdout, STDOUT_FILENO)?; + } + if let Some(stderr) = self.stderr { + dup(STDERR_FILENO)?; + dup2(stderr, STDERR_FILENO)?; + } + + log::info!("building wasi context"); + let wctx = wasi_builder.build(); + + log::info!("wasi context ready"); + let mut iterator = args + .first() + .context("args must have at least one argument.")? + .split('#'); + let mut cmd = iterator.next().unwrap().to_string(); + let stripped = cmd.strip_prefix(std::path::MAIN_SEPARATOR); + if let Some(strpd) = stripped { + cmd = strpd.to_string(); + } + let method = iterator.next().unwrap_or("_start"); + let mod_path = cmd; + + log::info!("loading module from file"); + let module = Module::from_file(&self.engine, mod_path)?; + let mut linker = Linker::new(&self.engine); + + wasmtime_wasi::add_to_linker(&mut linker, |s| s)?; + let mut store = Store::new(&self.engine, wctx); + + log::info!("instantiating instance"); + let instance = linker.instantiate(&mut store, &module)?; + + log::info!("getting start function"); + let start_func = instance + .get_func(&mut store, method) + .ok_or_else(|| anyhow!("module does not have a WASI start function".to_string()))?; + Ok((store, start_func)) + } +} diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 4fbaa7c9a..5b5be474b 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,26 +1,44 @@ -use std::fs::OpenOptions; -use std::path::Path; +use anyhow::Result; +use containerd_shim_wasm::sandbox::instance_utils::{ + get_instance_root, instance_exists, maybe_open_stdio, +}; +use libcontainer::container::builder::ContainerBuilder; +use libcontainer::container::{Container, ContainerStatus}; +use nix::errno::Errno; +use nix::sys::wait::waitid; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{ErrorKind, Read}; +use std::os::fd::RawFd; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Condvar, Mutex}; use std::thread; use anyhow::Context; use chrono::{DateTime, Utc}; use containerd_shim_wasm::sandbox::error::Error; -use containerd_shim_wasm::sandbox::exec; use containerd_shim_wasm::sandbox::instance::Wait; -use containerd_shim_wasm::sandbox::oci; use containerd_shim_wasm::sandbox::{EngineGetter, Instance, InstanceConfig}; -use log::{debug, error}; -use nix::sys::signal::SIGKILL; -use wasmtime::{Engine, Linker, Module, Store}; -use wasmtime_wasi::{sync::file::File as WasiFile, WasiCtx, WasiCtxBuilder}; +use libc::{dup2, STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use libc::{SIGINT, SIGKILL}; +use libcontainer::syscall::syscall::create_syscall; +use log::error; +use nix::sys::wait::{Id as WaitID, WaitPidFlag, WaitStatus}; -use super::error::WasmtimeError; -use super::oci_wasmtime; +use wasmtime::Engine; + +use crate::executor::WasmtimeExecutor; +use libcontainer::signal::Signal; + +static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd/wasmtime"; +type ExitCode = Arc<(Mutex)>>, Condvar)>; + +static mut STDIN_FD: Option = None; +static mut STDOUT_FD: Option = None; +static mut STDERR_FD: Option = None; -type ExitCode = (Mutex)>>, Condvar); pub struct Wasi { - exit_code: Arc, + exit_code: ExitCode, engine: wasmtime::Engine, stdin: String, @@ -28,253 +46,239 @@ pub struct Wasi { stderr: String, bundle: String, - pidfd: Arc>>, -} - -#[cfg(test)] -mod tests { - use std::fs::File; - - use tempfile::tempdir; - - use super::*; - - #[test] - fn test_maybe_open_stdio() -> Result<(), Error> { - let f = maybe_open_stdio("")?; - assert!(f.is_none()); - - let f = maybe_open_stdio("/some/nonexistent/path")?; - assert!(f.is_none()); + rootdir: PathBuf, - let dir = tempdir()?; - let temp = File::create(dir.path().join("testfile"))?; - drop(temp); - let f = maybe_open_stdio(dir.path().join("testfile").as_path().to_str().unwrap())?; - assert!(f.is_some()); - drop(f); - - Ok(()) - } + id: String, } -/// containerd can send an empty path or a non-existant path -/// In both these cases we should just assume that the stdio stream was not setup (intentionally) -/// Any other error is a real error. -pub fn maybe_open_stdio(path: &str) -> Result, Error> { - if path.is_empty() { - return Ok(None); - } - match oci_wasmtime::wasi_file(path, OpenOptions::new().read(true).write(true)) { - Ok(f) => Ok(Some(f)), - Err(err) => match err.kind() { - std::io::ErrorKind::NotFound => Ok(None), - _ => Err(err.into()), - }, +pub fn reset_stdio() { + unsafe { + if STDIN_FD.is_some() { + dup2(STDIN_FD.unwrap(), STDIN_FILENO); + } + if STDOUT_FD.is_some() { + dup2(STDOUT_FD.unwrap(), STDOUT_FILENO); + } + if STDERR_FD.is_some() { + dup2(STDERR_FD.unwrap(), STDERR_FILENO); + } } } -fn load_spec(bundle: String) -> Result { - let mut spec = oci::load(Path::new(&bundle).join("config.json").to_str().unwrap())?; - spec.canonicalize_rootfs(&bundle) - .map_err(|e| Error::Others(format!("error canonicalizing rootfs in spec: {}", e)))?; - Ok(spec) +#[derive(Serialize, Deserialize)] +struct Options { + root: Option, } -pub fn prepare_module( - engine: wasmtime::Engine, - spec: &oci::Spec, - stdin_path: String, - stdout_path: String, - stderr_path: String, -) -> Result<(WasiCtx, Module, String), WasmtimeError> { - debug!("opening rootfs"); - let rootfs = oci_wasmtime::get_rootfs(spec)?; - let args = oci::get_args(spec); - let env = oci_wasmtime::env_to_wasi(spec); - - debug!("setting up wasi"); - let mut wasi_builder = WasiCtxBuilder::new() - .args(args)? - .envs(env.as_slice())? - .preopened_dir(rootfs, "/")?; - - debug!("opening stdin"); - let stdin = maybe_open_stdio(&stdin_path).context("could not open stdin")?; - if let Some(sin) = stdin { - wasi_builder = wasi_builder.stdin(Box::new(sin)); - } - - debug!("opening stdout"); - let stdout = maybe_open_stdio(&stdout_path).context("could not open stdout")?; - if let Some(sout) = stdout { - wasi_builder = wasi_builder.stdout(Box::new(sout)); - } - - debug!("opening stderr"); - let stderr = maybe_open_stdio(&stderr_path).context("could not open stderr")?; - if let Some(serr) = stderr { - wasi_builder = wasi_builder.stderr(Box::new(serr)); - } - - debug!("building wasi context"); - let wctx = wasi_builder.build(); - debug!("wasi context ready"); - - let start = args[0].clone(); - let mut iterator = start.split('#'); - let mut cmd = iterator.next().unwrap().to_string(); - - let stripped = cmd.strip_prefix(std::path::MAIN_SEPARATOR); - if let Some(strpd) = stripped { - cmd = strpd.to_string(); - } - let method = iterator.next().unwrap_or("_start"); - - let mod_path = oci::get_root(spec).join(cmd); - debug!("loading module from file"); - let module = Module::from_file(&engine, mod_path) - .map_err(|err| Error::Others(format!("could not load module from file: {}", err)))?; - - Ok((wctx, module, method.to_string())) +fn determine_rootdir>(bundle: P, namespace: String) -> Result { + log::info!( + "determining rootdir for bundle: {}", + bundle.as_ref().display() + ); + let mut file = match File::open(bundle.as_ref().join("options.json")) { + Ok(f) => f, + Err(err) => match err.kind() { + ErrorKind::NotFound => { + return Ok(<&str as Into>::into(DEFAULT_CONTAINER_ROOT_DIR).join(namespace)) + } + _ => return Err(err.into()), + }, + }; + let mut data = String::new(); + file.read_to_string(&mut data)?; + let options: Options = serde_json::from_str(&data)?; + let path = options + .root + .unwrap_or(PathBuf::from(DEFAULT_CONTAINER_ROOT_DIR)) + .join(namespace); + log::info!("youki root path is: {}", path.display()); + Ok(path) } impl Instance for Wasi { type E = wasmtime::Engine; - fn new(_id: String, cfg: Option<&InstanceConfig>) -> Self { - let cfg = cfg.unwrap(); // TODO: handle error + fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { + // TODO: there are failure cases e.x. parsing cfg, loading spec, etc. + // thus should make `new` return `Result` instead of `Self` + log::info!("creating new instance: {}", id); + let cfg = cfg.unwrap(); + let bundle = cfg.get_bundle().unwrap_or_default(); + let rootdir = determine_rootdir(bundle.as_str(), cfg.get_namespace()).unwrap(); Wasi { + id, exit_code: Arc::new((Mutex::new(None), Condvar::new())), engine: cfg.get_engine(), stdin: cfg.get_stdin().unwrap_or_default(), stdout: cfg.get_stdout().unwrap_or_default(), stderr: cfg.get_stderr().unwrap_or_default(), - bundle: cfg.get_bundle().unwrap_or_default(), - pidfd: Arc::new(Mutex::new(None)), + bundle, + rootdir, } } fn start(&self) -> Result { - let engine = self.engine.clone(); - let stdin = self.stdin.clone(); - let stdout = self.stdout.clone(); - let stderr = self.stderr.clone(); - - debug!("starting instance"); - let mut linker = Linker::new(&engine); - - wasmtime_wasi::add_to_linker(&mut linker, |s| s) - .map_err(|err| Error::Others(format!("error adding to linker: {}", err)))?; + log::info!("starting instance: {}", self.id); + let engine: Engine = self.engine.clone(); - debug!("preparing module"); - let spec = load_spec(self.bundle.clone())?; + let mut container = self.build_container( + self.stdin.as_str(), + self.stdout.as_str(), + self.stderr.as_str(), + engine, + )?; - let m = prepare_module(engine.clone(), &spec, stdin, stdout, stderr) - .map_err(|e| Error::Others(format!("error setting up module: {}", e)))?; - - let mut store = Store::new(&engine, m.0); - - debug!("instantiating instance"); - let i = linker - .instantiate(&mut store, &m.1) - .map_err(|err| Error::Others(format!("error instantiating module: {}", err)))?; - - debug!("getting start function"); - let f = i.get_func(&mut store, &m.2).ok_or_else(|| { - Error::InvalidArgument("module does not have a wasi start function".to_string()) - })?; - - debug!("starting wasi instance"); - - let cg = oci::get_cgroup(&spec)?; - - oci::setup_cgroup(cg.as_ref(), &spec) - .map_err(|e| Error::Others(format!("error setting up cgroups: {}", e)))?; - - let res = unsafe { exec::fork(Some(cg.as_ref())) }?; - match res { - exec::Context::Parent(tid, pidfd) => { - let mut lr = self.pidfd.lock().unwrap(); - *lr = Some(pidfd.clone()); - - debug!("started wasi instance with tid {}", tid); - - let code = self.exit_code.clone(); - - let _ = thread::spawn(move || { - let (lock, cvar) = &*code; - let status = match pidfd.wait() { - Ok(status) => status, - Err(e) => { - error!("error waiting for pid {}: {}", tid, e); - cvar.notify_all(); - return; - } - }; - - debug!("wasi instance exited with status {}", status.status); - let mut ec = lock.lock().unwrap(); - *ec = Some((status.status, Utc::now())); - drop(ec); - cvar.notify_all(); - }); - Ok(tid) - } - exec::Context::Child => { - // child process - - // TODO: How to get exit code? - // This was relatively straight forward in go, but wasi and wasmtime are totally separate things in rust. - match f.call(&mut store, &[], &mut []) { - Ok(_) => std::process::exit(0), - Err(_) => std::process::exit(137), - }; - } - } + log::info!("created container: {}", self.id); + let code = self.exit_code.clone(); + let pid = container.pid().unwrap(); + + container + .start() + .map_err(|err| Error::Any(anyhow::anyhow!("failed to start container: {}", err)))?; + + thread::spawn(move || { + let (lock, cvar) = &*code; + + let status = match waitid(WaitID::Pid(pid), WaitPidFlag::WEXITED) { + Ok(WaitStatus::Exited(_, status)) => status, + Ok(WaitStatus::Signaled(_, sig, _)) => sig as i32, + Ok(_) => 0, + Err(e) => { + if e == Errno::ECHILD { + log::info!("no child process"); + 0 + } else { + panic!("waitpid failed: {}", e); + } + } + } as u32; + let mut ec = lock.lock().unwrap(); + *ec = Some((status, Utc::now())); + drop(ec); + cvar.notify_all(); + }); + + Ok(pid.as_raw() as u32) } fn kill(&self, signal: u32) -> Result<(), Error> { - if signal != SIGKILL as u32 { + log::info!("killing instance: {}", self.id); + if signal as i32 != SIGKILL && signal as i32 != SIGINT { return Err(Error::InvalidArgument( - "only SIGKILL is supported".to_string(), + "only SIGKILL and SIGINT are supported".to_string(), )); } - - let lr = self.pidfd.lock().unwrap(); - let fd = lr - .as_ref() - .ok_or_else(|| Error::FailedPrecondition("module is not running".to_string()))?; - fd.kill(signal as i32) + let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; + let mut container = Container::load(container_root).with_context(|| { + format!( + "could not load state for container {id}", + id = self.id.as_str() + ) + })?; + let signal = Signal::try_from(signal as i32) + .map_err(|err| Error::InvalidArgument(format!("invalid signal number: {}", err)))?; + match container.kill(signal, true) { + Ok(_) => Ok(()), + Err(e) => { + if container.status() == ContainerStatus::Stopped { + return Err(Error::Others("container not running".into())); + } + Err(Error::Others(e.to_string())) + } + } } fn delete(&self) -> Result<(), Error> { - let spec = match load_spec(self.bundle.clone()) { - Ok(spec) => spec, + log::info!("deleting instance: {}", self.id); + match instance_exists(&self.rootdir, self.id.as_str()) { + Ok(exists) => { + if !exists { + return Ok(()); + } + } Err(err) => { - error!("Could not load spec, skipping cgroup cleanup: {}", err); + error!("could not find the container, skipping cleanup: {}", err); return Ok(()); } - }; - let cg = oci::get_cgroup(&spec)?; - cg.delete()?; + } + let container_root = get_instance_root(&self.rootdir, self.id.as_str())?; + let container = Container::load(container_root).with_context(|| { + format!( + "could not load state for container {id}", + id = self.id.as_str() + ) + }); + match container { + Ok(mut container) => container.delete(true).map_err(|err| { + Error::Any(anyhow::anyhow!( + "failed to delete container {}: {}", + self.id, + err + )) + })?, + Err(err) => { + error!("could not find the container, skipping cleanup: {}", err); + return Ok(()); + } + } + Ok(()) } fn wait(&self, waiter: &Wait) -> Result<(), Error> { + log::info!("waiting for instance: {}", self.id); let code = self.exit_code.clone(); waiter.set_up_exit_code_wait(code) } } +impl Wasi { + fn build_container( + &self, + stdin: &str, + stdout: &str, + stderr: &str, + engine: Engine, + ) -> anyhow::Result { + let syscall = create_syscall(); + let stdin = maybe_open_stdio(stdin).context("could not open stdin")?; + let stdout = maybe_open_stdio(stdout).context("could not open stdout")?; + let stderr = maybe_open_stdio(stderr).context("could not open stderr")?; + + let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) + .with_executor(vec![Box::new(WasmtimeExecutor { + stdin, + stdout, + stderr, + engine, + })])? + .with_root_path(self.rootdir.clone())? + .as_init(&self.bundle) + .with_systemd(false) + .build()?; + Ok(container) + } +} + +impl EngineGetter for Wasi { + type E = wasmtime::Engine; + fn new_engine() -> Result { + Ok(Engine::default()) + } +} + #[cfg(test)] mod wasitest { - use std::fs::{create_dir, read_to_string, File}; + use std::fs::{create_dir, read_to_string, File, OpenOptions}; use std::io::prelude::*; + use std::os::unix::prelude::OpenOptionsExt; use std::sync::mpsc::channel; use std::time::Duration; + use containerd_shim_wasm::function; + use containerd_shim_wasm::sandbox::exec::has_cap_sys_admin; use containerd_shim_wasm::sandbox::instance::Wait; + use containerd_shim_wasm::sandbox::testutil::run_test_with_sudo; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; - use tempfile::tempdir; + use tempfile::{tempdir, TempDir}; use super::*; @@ -309,46 +313,29 @@ mod wasitest { "#.as_bytes(); #[test] - fn test_delete_after_create() { - let i = Wasi::new( - "".to_string(), - Some(&InstanceConfig::new( - Engine::default(), - "test_namespace".into(), - )), - ); - i.delete().unwrap(); + fn test_delete_after_create() -> Result<()> { + let dir = tempdir()?; + let cfg = prepare_cfg(&dir)?; + + let i = Wasi::new("".to_string(), Some(&cfg)); + i.delete()?; + reset_stdio(); + Ok(()) } #[test] fn test_wasi() -> Result<(), Error> { - let dir = tempdir()?; - create_dir(dir.path().join("rootfs"))?; - - let mut f = File::create(dir.path().join("rootfs/hello.wat"))?; - f.write_all(WASI_HELLO_WAT)?; - - let stdout = File::create(dir.path().join("stdout"))?; - drop(stdout); - - let spec = SpecBuilder::default() - .root(RootBuilder::default().path("rootfs").build()?) - .process( - ProcessBuilder::default() - .cwd("/") - .args(vec!["hello.wat".to_string()]) - .build()?, - ) - .build()?; - - spec.save(dir.path().join("config.json"))?; + if !has_cap_sys_admin() { + println!("running test with sudo: {}", function!()); + return run_test_with_sudo(function!()); + } + // start logging + let _ = env_logger::try_init(); - let mut cfg = InstanceConfig::new(Engine::default(), "test_namespace".into()); - let cfg = cfg - .set_bundle(dir.path().to_str().unwrap().to_string()) - .set_stdout(dir.path().join("stdout").to_str().unwrap().to_string()); + let dir = tempdir()?; + let cfg = prepare_cfg(&dir)?; - let wasi = Wasi::new("test".to_string(), Some(cfg)); + let wasi = Wasi::new("test".to_string(), Some(&cfg)); wasi.start()?; @@ -373,14 +360,52 @@ mod wasitest { wasi.delete()?; + reset_stdio(); Ok(()) } -} -impl EngineGetter for Wasi { - type E = wasmtime::Engine; - fn new_engine() -> Result { - let engine = Engine::default(); - Ok(engine) + fn prepare_cfg(dir: &TempDir) -> Result> { + create_dir(dir.path().join("rootfs"))?; + + let opts = Options { + root: Some(dir.path().join("runwasi")), + }; + let opts_file = OpenOptions::new() + .read(true) + .create(true) + .truncate(true) + .write(true) + .open(dir.path().join("options.json"))?; + write!(&opts_file, "{}", serde_json::to_string(&opts)?)?; + + let wasm_path = dir.path().join("rootfs/hello.wat"); + let mut f = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o755) + .open(wasm_path)?; + f.write_all(WASI_HELLO_WAT)?; + + let stdout = File::create(dir.path().join("stdout"))?; + let stderr = File::create(dir.path().join("stderr"))?; + drop(stdout); + drop(stderr); + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec!["./hello.wat".to_string()]) + .build()?, + ) + .build()?; + spec.save(dir.path().join("config.json"))?; + let mut cfg = InstanceConfig::new(Engine::default(), "test_namespace".into()); + let cfg = cfg + .set_bundle(dir.path().to_str().unwrap().to_string()) + .set_stdout(dir.path().join("stdout").to_str().unwrap().to_string()) + .set_stderr(dir.path().join("stderr").to_str().unwrap().to_string()); + Ok(cfg.to_owned()) } } diff --git a/crates/containerd-shim-wasmtime/src/lib.rs b/crates/containerd-shim-wasmtime/src/lib.rs index 48495d071..312731e44 100644 --- a/crates/containerd-shim-wasmtime/src/lib.rs +++ b/crates/containerd-shim-wasmtime/src/lib.rs @@ -1,3 +1,4 @@ pub mod error; +pub mod executor; pub mod instance; pub mod oci_wasmtime; diff --git a/test/k8s/Dockerfile b/test/k8s/Dockerfile index e95149eab..7cffedfbc 100644 --- a/test/k8s/Dockerfile +++ b/test/k8s/Dockerfile @@ -33,6 +33,7 @@ FROM scratch AS kind COPY --from=kind-fetch /root/kind /kind FROM kind-base +RUN apt-get update && apt-get install --no-install-recommends -y build-essential git clang pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev RUN <> /etc/containerd/config.toml