From ee2c24cfaaf691a50ca4b3322e990c2b5d09ea3d Mon Sep 17 00:00:00 2001 From: Zhang Tianyang Date: Wed, 10 Jan 2024 15:46:22 +0800 Subject: [PATCH] task: add early and late init calls Add early_init_call() and late_init_call() for VM initiation. Signed-off-by: Zhang Tianyang --- vmm/task/src/container.rs | 28 ++------------ vmm/task/src/main.rs | 79 +++++++++++++++++++++++++++++++++------ vmm/task/src/task.rs | 9 ++--- 3 files changed, 73 insertions(+), 43 deletions(-) diff --git a/vmm/task/src/container.rs b/vmm/task/src/container.rs index 786b198b..8ac89465 100644 --- a/vmm/task/src/container.rs +++ b/vmm/task/src/container.rs @@ -41,8 +41,8 @@ use containerd_shim::{ util::read_spec, ExitSignal, }; -use log::{debug, error, warn}; -use nix::{mount, mount::MsFlags, sys::signalfd::signal::kill, unistd::Pid}; +use log::{debug, error}; +use nix::{sys::signalfd::signal::kill, unistd::Pid}; use oci_spec::runtime::{LinuxResources, Process, Spec}; use runc::{options::GlobalOpts, Runc, Spawner}; use serde::Deserialize; @@ -52,9 +52,7 @@ use tokio::{ process::Command, sync::Mutex, }; -use vmm_common::{ - mount::get_mount_type, storage::Storage, ETC_RESOLV, KUASAR_STATE_DIR, RESOLV_FILENAME, -}; +use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR}; use crate::{ device::rescan_pci_bus, @@ -197,26 +195,6 @@ impl KuasarFactory { Self { sandbox } } - // create_sandbox will do some preparation to provide a livable environment for containers, - // such as adding guest hooks, setting kernel paras, preparing sandbox files and namespaces - pub fn create_sandbox(&self) -> Result<()> { - // Setup DNS, bind mount to /etc/resolv.conf - let dns_file = Path::new(KUASAR_STATE_DIR).join(RESOLV_FILENAME); - if dns_file.exists() { - mount::mount( - Some(&dns_file), - ETC_RESOLV, - Some("bind"), - MsFlags::MS_BIND, - None::<&str>, - )?; - } else { - warn!("unable to find DNS files in kuasar state dir"); - } - - Ok(()) - } - async fn do_create(&self, init: &mut InitProcess) -> Result<()> { let id = init.id.to_string(); let stdio = &init.stdio; diff --git a/vmm/task/src/main.rs b/vmm/task/src/main.rs index 9600d753..05c4505c 100644 --- a/vmm/task/src/main.rs +++ b/vmm/task/src/main.rs @@ -22,6 +22,7 @@ use containerd_shim::{ io_error, other, protos::{shim::shim_ttrpc_async::create_task, ttrpc::asynchronous::Server}, util::IntoOption, + Result, }; use futures::StreamExt; use lazy_static::lazy_static; @@ -35,7 +36,9 @@ use nix::{ unistd::Pid, }; use signal_hook_tokio::Signals; -use vmm_common::{api::sandbox_ttrpc::create_sandbox_service, mount::mount, KUASAR_STATE_DIR}; +use vmm_common::{ + api::sandbox_ttrpc::create_sandbox_service, mount::mount, ETC_RESOLV, KUASAR_STATE_DIR, +}; use crate::{ config::TaskConfig, @@ -67,6 +70,15 @@ pub struct StaticMount { options: Vec<&'static str>, } +const ENVS: [(&str, &str); 2] = [ + ("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/"), + ("XDG_RUNTIME_DIR", "/run"), +]; + +const KERNEL_PARAS: [(&str, &str); 0] = []; + +const SANDBOX_DNS_PATH: &str = "/run/kuasar/state/resolv.conf"; + lazy_static! { pub static ref VM_ROOTFS_MOUNTS: Vec = vec![ StaticMount { @@ -118,13 +130,17 @@ lazy_static! { dest: KUASAR_STATE_DIR, options: vec!["relatime", "nodev", "sync", "dirsync",] },]; + static ref DNS_MOUNTS: Vec = vec![StaticMount { + fstype: "bind", + src: SANDBOX_DNS_PATH, + dest: ETC_RESOLV, + options: vec!["bind"] + },]; } #[tokio::main] async fn main() { - std::env::set_var("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/"); - std::env::set_var("XDG_RUNTIME_DIR", "/run"); - init_vm_rootfs().await.unwrap(); + early_init_call().await.expect("early init call"); let config = TaskConfig::new().await.unwrap(); let log_level = LevelFilter::from_str(&config.log_level).unwrap(); env_logger::Builder::from_default_env() @@ -154,6 +170,8 @@ async fn main() { } } + late_init_call().await.expect("late init call"); + // Start ttrpc server let mut server = start_ttrpc_server() .await @@ -166,6 +184,22 @@ async fn main() { handle_signals(signals).await; } +// Do some initialization before everything starts. +// Such as setting envs, preparing cgroup mounts, setting kernel paras. +async fn early_init_call() -> Result<()> { + // Set environment variables from ENVS vector(ordered). + for (k, v) in ENVS.iter() { + std::env::set_var(k, v); + } + // Set kernel parameters from KERNEL_PARAS vector(ordered). + for (_path, _v) in KERNEL_PARAS.iter() { + // TODO: write kernel paras + } + + init_vm_rootfs().await?; + Ok(()) +} + async fn handle_signals(signals: Signals) { let mut signals = signals.fuse(); while let Some(sig) = signals.next().await { @@ -248,7 +282,7 @@ async fn handle_signals(signals: Signals) { } } -async fn init_vm_rootfs() -> containerd_shim::Result<()> { +async fn init_vm_rootfs() -> Result<()> { let mounts = VM_ROOTFS_MOUNTS.clone(); mount_static_mounts(mounts).await?; // has to mount /proc before find cgroup mounts @@ -258,15 +292,36 @@ async fn init_vm_rootfs() -> containerd_shim::Result<()> { // For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt tokio::fs::write("/sys/fs/cgroup/memory/memory.use_hierarchy", "1") .await - .unwrap(); + .map_err(io_error!(e, "failed to set cgroup hierarchy to 1")) +} + +// Continue to do initialization that depend on shared path. +// such as adding guest hook, preparing sandbox files and namespaces. +async fn late_init_call() -> Result<()> { + // Setup DNS, bind mount to /etc/resolv.conf + if Path::new(SANDBOX_DNS_PATH).exists() { + mount_static_mounts(DNS_MOUNTS.clone()).await?; + } else { + warn!( + "unable to find {} files in kuasar state dir", + SANDBOX_DNS_PATH + ); + } + Ok(()) } -async fn mount_static_mounts(mounts: Vec) -> containerd_shim::Result<()> { +async fn mount_static_mounts(mounts: Vec) -> Result<()> { for m in mounts { - tokio::fs::create_dir_all(Path::new(m.dest)) - .await - .map_err(io_error!(e, "failed to create {}: ", m.dest))?; + let dest = Path::new(m.dest); + // For DNS mounts, dest file already exist. + if !dest.exists() { + tokio::fs::create_dir_all(dest).await.map_err(io_error!( + e, + "failed to create {}: ", + m.dest + ))?; + } match mount( m.fstype.none_if(|x| x.is_empty()), m.src.none_if(|x| x.is_empty()), @@ -290,8 +345,8 @@ async fn mount_static_mounts(mounts: Vec) -> containerd_shim::Resul // start_ttrpc_server will create all the ttrpc service and register them to a server that // bind to vsock 1024 port. -async fn start_ttrpc_server() -> containerd_shim::Result { - let task = create_task_service().await?; +async fn start_ttrpc_server() -> Result { + let task = create_task_service().await; let task_service = create_task(Arc::new(Box::new(task))); let sandbox = SandboxService::new()?; diff --git a/vmm/task/src/task.rs b/vmm/task/src/task.rs index a93eaa59..da0c5854 100644 --- a/vmm/task/src/task.rs +++ b/vmm/task/src/task.rs @@ -35,14 +35,11 @@ use crate::{ sandbox::SandboxResources, }; -pub(crate) async fn create_task_service( -) -> containerd_shim::Result> { +pub(crate) async fn create_task_service() -> TaskService { let (tx, mut rx) = channel(128); let sandbox = Arc::new(Mutex::new(SandboxResources::new().await)); - let factory = KuasarFactory::new(sandbox); - factory.create_sandbox()?; let task = TaskService { - factory, + factory: KuasarFactory::new(sandbox), containers: Arc::new(Default::default()), namespace: "k8s.io".to_string(), exit: Arc::new(Default::default()), @@ -57,7 +54,7 @@ pub(crate) async fn create_task_service( debug!("received event {:?}", e); } }); - Ok(task) + task } async fn process_exits(s: Subscription, task: &TaskService) {