task: add early and late init calls

Add early_init_call() and late_init_call() for VM initiation.

Signed-off-by: Zhang Tianyang <burning9699@gmail.com>

vmm/task/src/container.rs

@@ -41,8 +41,8 @@ use containerd_shim::{
     util::read_spec,
     ExitSignal,
 };
-use log::{debug, error, warn};
-use nix::{mount, mount::MsFlags, sys::signalfd::signal::kill, unistd::Pid};
+use log::{debug, error};
+use nix::{sys::signalfd::signal::kill, unistd::Pid};
 use oci_spec::runtime::{LinuxResources, Process, Spec};
 use runc::{options::GlobalOpts, Runc, Spawner};
 use serde::Deserialize;
@@ -52,9 +52,7 @@ use tokio::{
     process::Command,
     sync::Mutex,
 };
-use vmm_common::{
-    mount::get_mount_type, storage::Storage, ETC_RESOLV, KUASAR_STATE_DIR, RESOLV_FILENAME,
-};
+use vmm_common::{mount::get_mount_type, storage::Storage, KUASAR_STATE_DIR};
 
 use crate::{
     device::rescan_pci_bus,
@@ -197,26 +195,6 @@ impl KuasarFactory {
         Self { sandbox }
     }
 
-    // create_sandbox will do some preparation to provide a livable environment for containers,
-    // such as adding guest hooks, setting kernel paras, preparing sandbox files and namespaces
-    pub fn create_sandbox(&self) -> Result<()> {
-        // Setup DNS, bind mount to /etc/resolv.conf
-        let dns_file = Path::new(KUASAR_STATE_DIR).join(RESOLV_FILENAME);
-        if dns_file.exists() {
-            mount::mount(
-                Some(&dns_file),
-                ETC_RESOLV,
-                Some("bind"),
-                MsFlags::MS_BIND,
-                None::<&str>,
-            )?;
-        } else {
-            warn!("unable to find DNS files in kuasar state dir");
-        }
-
-        Ok(())
-    }
-
     async fn do_create(&self, init: &mut InitProcess) -> Result<()> {
         let id = init.id.to_string();
         let stdio = &init.stdio;

vmm/task/src/main.rs

@@ -22,6 +22,7 @@ use containerd_shim::{
     io_error, other,
     protos::{shim::shim_ttrpc_async::create_task, ttrpc::asynchronous::Server},
     util::IntoOption,
+    Result,
 };
 use futures::StreamExt;
 use lazy_static::lazy_static;
@@ -35,7 +36,9 @@ use nix::{
     unistd::Pid,
 };
 use signal_hook_tokio::Signals;
-use vmm_common::{api::sandbox_ttrpc::create_sandbox_service, mount::mount, KUASAR_STATE_DIR};
+use vmm_common::{
+    api::sandbox_ttrpc::create_sandbox_service, mount::mount, ETC_RESOLV, KUASAR_STATE_DIR,
+};
 
 use crate::{
     config::TaskConfig,
@@ -67,6 +70,15 @@ pub struct StaticMount {
     options: Vec<&'static str>,
 }
 
+const ENVS: [(&str, &str); 2] = [
+    ("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/"),
+    ("XDG_RUNTIME_DIR", "/run"),
+];
+
+const KERNEL_PARAS: [(&str, &str); 0] = [];
+
+const SANDBOX_DNS_PATH: &str = "/run/kuasar/state/resolv.conf";
+
 lazy_static! {
     pub static ref VM_ROOTFS_MOUNTS: Vec<StaticMount> = vec![
         StaticMount {
@@ -118,13 +130,17 @@ lazy_static! {
         dest: KUASAR_STATE_DIR,
         options: vec!["relatime", "nodev", "sync", "dirsync",]
     },];
+    static ref DNS_MOUNTS: Vec<StaticMount> = vec![StaticMount {
+        fstype: "bind",
+        src: SANDBOX_DNS_PATH,
+        dest: ETC_RESOLV,
+        options: vec!["bind"]
+    },];
 }
 
 #[tokio::main]
 async fn main() {
-    std::env::set_var("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/");
-    std::env::set_var("XDG_RUNTIME_DIR", "/run");
-    init_vm_rootfs().await.unwrap();
+    early_init_call().await.expect("early init call");
     let config = TaskConfig::new().await.unwrap();
     let log_level = LevelFilter::from_str(&config.log_level).unwrap();
     env_logger::Builder::from_default_env()
@@ -154,6 +170,8 @@ async fn main() {
         }
     }
 
+    late_init_call().await.expect("late init call");
+
     // Start ttrpc server
     let mut server = start_ttrpc_server()
         .await
@@ -166,6 +184,22 @@ async fn main() {
     handle_signals(signals).await;
 }
 
+// Do some initialization before everything starts.
+// Such as setting envs, preparing cgroup mounts, setting kernel paras.
+async fn early_init_call() -> Result<()> {
+    // Set environment variables from ENVS vector(ordered).
+    for (k, v) in ENVS.iter() {
+        std::env::set_var(k, v);
+    }
+    // Set kernel parameters from KERNEL_PARAS vector(ordered).
+    for (_path, _v) in KERNEL_PARAS.iter() {
+        // TODO: write kernel paras
+    }
+
+    init_vm_rootfs().await?;
+    Ok(())
+}
+
 async fn handle_signals(signals: Signals) {
     let mut signals = signals.fuse();
     while let Some(sig) = signals.next().await {
@@ -248,7 +282,7 @@ async fn handle_signals(signals: Signals) {
     }
 }
 
-async fn init_vm_rootfs() -> containerd_shim::Result<()> {
+async fn init_vm_rootfs() -> Result<()> {
     let mounts = VM_ROOTFS_MOUNTS.clone();
     mount_static_mounts(mounts).await?;
     // has to mount /proc before find cgroup mounts
@@ -258,15 +292,36 @@ async fn init_vm_rootfs() -> containerd_shim::Result<()> {
     // For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
     tokio::fs::write("/sys/fs/cgroup/memory/memory.use_hierarchy", "1")
         .await
-        .unwrap();
+        .map_err(io_error!(e, "failed to set cgroup hierarchy to 1"))
+}
+
+// Continue to do initialization that depend on shared path.
+// such as adding guest hook, preparing sandbox files and namespaces.
+async fn late_init_call() -> Result<()> {
+    // Setup DNS, bind mount to /etc/resolv.conf
+    if Path::new(SANDBOX_DNS_PATH).exists() {
+        mount_static_mounts(DNS_MOUNTS.clone()).await?;
+    } else {
+        warn!(
+            "unable to find {} files in kuasar state dir",
+            SANDBOX_DNS_PATH
+        );
+    }
+
     Ok(())
 }
 
-async fn mount_static_mounts(mounts: Vec<StaticMount>) -> containerd_shim::Result<()> {
+async fn mount_static_mounts(mounts: Vec<StaticMount>) -> Result<()> {
     for m in mounts {
-        tokio::fs::create_dir_all(Path::new(m.dest))
-            .await
-            .map_err(io_error!(e, "failed to create {}: ", m.dest))?;
+        let dest = Path::new(m.dest);
+        // For DNS mounts, dest file already exist.
+        if !dest.exists() {
+            tokio::fs::create_dir_all(dest).await.map_err(io_error!(
+                e,
+                "failed to create {}: ",
+                m.dest
+            ))?;
+        }
         match mount(
             m.fstype.none_if(|x| x.is_empty()),
             m.src.none_if(|x| x.is_empty()),
@@ -290,8 +345,8 @@ async fn mount_static_mounts(mounts: Vec<StaticMount>) -> containerd_shim::Resul
 
 // start_ttrpc_server will create all the ttrpc service and register them to a server that
 // bind to vsock 1024 port.
-async fn start_ttrpc_server() -> containerd_shim::Result<Server> {
-    let task = create_task_service().await?;
+async fn start_ttrpc_server() -> Result<Server> {
+    let task = create_task_service().await;
     let task_service = create_task(Arc::new(Box::new(task)));
 
     let sandbox = SandboxService::new()?;

vmm/task/src/task.rs

@@ -35,14 +35,11 @@ use crate::{
     sandbox::SandboxResources,
 };
 
-pub(crate) async fn create_task_service(
-) -> containerd_shim::Result<TaskService<KuasarFactory, KuasarContainer>> {
+pub(crate) async fn create_task_service() -> TaskService<KuasarFactory, KuasarContainer> {
     let (tx, mut rx) = channel(128);
     let sandbox = Arc::new(Mutex::new(SandboxResources::new().await));
-    let factory = KuasarFactory::new(sandbox);
-    factory.create_sandbox()?;
     let task = TaskService {
-        factory,
+        factory: KuasarFactory::new(sandbox),
         containers: Arc::new(Default::default()),
         namespace: "k8s.io".to_string(),
         exit: Arc::new(Default::default()),
@@ -57,7 +54,7 @@ pub(crate) async fn create_task_service(
             debug!("received event {:?}", e);
         }
     });
-    Ok(task)
+    task
 }
 
 async fn process_exits(s: Subscription, task: &TaskService<KuasarFactory, KuasarContainer>) {