Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

vmm: support starting stratovirt lightweight vm sandbox #93

Merged
merged 1 commit into from
Nov 8, 2023
Merged

vmm: support starting stratovirt lightweight vm sandbox #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions vmm/sandbox/src/stratovirt/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ use sandbox_derive::{CmdLineParamSet, CmdLineParams};
use serde::{Deserialize, Serialize};

use crate::{
param::ToCmdLineParams, stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH,
vm::HypervisorCommonConfig,
device::Transport, param::ToCmdLineParams,
stratovirt::virtiofs::DEFAULT_VHOST_USER_FS_BIN_PATH, vm::HypervisorCommonConfig,
};

#[allow(dead_code)]
Expand All @@ -31,17 +31,15 @@ pub(crate) const MACHINE_TYPE_PC: &str = "pc";
#[allow(dead_code)]
pub(crate) const MACHINE_TYPE_VIRT: &str = "virt";
#[allow(dead_code)]
pub(crate) const MACHINE_TYPE_MICROVM: &str = "microvm";
#[allow(dead_code)]
pub(crate) const MACHINE_TYPE_PSERIES: &str = "pseries";
#[allow(dead_code)]
pub(crate) const MACHINE_TYPE_CCW_VIRTIO: &str = "s390-ccw-virtio";

const DEFAULT_STRATOVIRT_PATH: &str = "/usr/bin/stratovirt";
const DEFAULT_KERNEL_PARAMS: &str = "console=hvc0 console=hvc1 iommu=off panic=1 pcie_ports=native";

#[cfg(target_arch = "x86_64")]
const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda ro rootfstype=ext4";
#[cfg(target_arch = "aarch64")]
const ROOTFS_KERNEL_PARAMS: &str = " root=/dev/vda1 ro rootfstype=ext4";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is it different before? and is it unified now?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it is unified now, i used an unofficial image in arm before, which would divide a vda1 patition, now i use the image provided by stratovirt to start vm, it will not devide vda1.


#[derive(Clone, Debug, Deserialize)]
pub struct StratoVirtVMConfig {
Expand All @@ -62,8 +60,8 @@ impl Default for StratoVirtVMConfig {
fn default() -> Self {
Self {
common: Default::default(),
path: "stratovirt".to_string(),
machine_type: "virt".to_string(),
path: DEFAULT_STRATOVIRT_PATH.to_string(),
machine_type: MACHINE_TYPE_VIRT.to_string(),
virtiofsd_conf: VirtiofsdConfig {
path: DEFAULT_VHOST_USER_FS_BIN_PATH.to_string(),
},
Expand Down Expand Up @@ -134,9 +132,12 @@ impl StratoVirtVMConfig {
result.kernel.kernel_params.push_str(" debug task.debug");
}

result.global_params = vec![Global {
param: "pcie-root-port.fast-unplug=1".to_string(),
}];
let machine_array: Vec<_> = self.machine_type.split(',').collect();
if machine_array[0] != MACHINE_TYPE_MICROVM {
result.global_params = vec![Global {
param: "pcie-root-port.fast-unplug=1".to_string(),
}];
}

result.knobs = Knobs {
daemonize: true,
Expand All @@ -163,6 +164,16 @@ pub struct Machine {
pub options: Option<String>,
}

impl Machine {
pub fn transport(&self) -> Transport {
let machine_array: Vec<_> = self.r#type.split(',').collect();
match machine_array[0] {
MACHINE_TYPE_MICROVM => Transport::Mmio,
_ => Transport::Pci,
}
}
}

#[derive(CmdLineParamSet, Debug, Clone, Default, Serialize, Deserialize)]
pub struct Kernel {
#[param(key = "kernel")]
Expand Down
4 changes: 2 additions & 2 deletions vmm/sandbox/src/stratovirt/devices/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ pub struct VirtioBlockDevice {
pub readonly: Option<bool>,
#[property(param = "drive", generator = "crate::utils::bool_to_on_off")]
pub direct: Option<bool>,
#[property(param = "device")]
#[property(param = "device", predicate = "self.addr.len()>0")]
pub bus: Option<String>,
#[property(param = "device")]
#[property(param = "device", predicate = "self.addr.len()>0")]
pub addr: String,
}

Expand Down
6 changes: 3 additions & 3 deletions vmm/sandbox/src/stratovirt/devices/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ pub trait StratoVirtHotAttachable: Device + HotAttachable {}

impl<T> StratoVirtHotAttachable for T where T: Device + HotAttachable {}

pub fn create_pcie_root_bus() -> PcieRootBus {
pub fn create_pcie_root_bus() -> Option<PcieRootBus> {
let mut pcie_root_bus = PcieRootBus {
id: "pcie.0".to_string(),
bus: Bus {
Expand All @@ -85,7 +85,7 @@ pub fn create_pcie_root_bus() -> PcieRootBus {

// since pcie.0/0x0 addr is reserved, set slot 0 status is "SlotStatus::Occupied"
pcie_root_bus.bus.slots[0].status = SlotStatus::Occupied("reserved".to_string());
pcie_root_bus
Some(pcie_root_bus)
}

#[cfg(test)]
Expand All @@ -96,7 +96,7 @@ mod tests {
#[test]
fn test_create_pcie_root_bus() {
let pcie_root_bus = create_pcie_root_bus();
if let SlotStatus::Occupied(s) = &pcie_root_bus.bus.slots[0].status {
if let SlotStatus::Occupied(s) = &pcie_root_bus.unwrap().bus.slots[0].status {
assert_eq!(s, "reserved");
}
}
Expand Down
4 changes: 2 additions & 2 deletions vmm/sandbox/src/stratovirt/devices/rng.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ pub struct VirtioRngDevice {
pub(crate) max_bytes: Option<u32>,
#[property(param = "device")]
pub(crate) period: Option<u32>,
#[property(param = "device")]
#[property(param = "device", predicate = "self.addr.len()>0")]
pub(crate) bus: String,
#[property(param = "device")]
#[property(param = "device", predicate = "self.addr.len()>0")]
pub(crate) addr: String,
}

Expand Down
2 changes: 2 additions & 0 deletions vmm/sandbox/src/stratovirt/devices/serial.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ pub struct SerialDevice {
#[property(ignore_key)]
pub driver: String,
pub id: String,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub bus: String,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub addr: String,
#[property(ignore)]
pub transport: Transport,
Expand Down
2 changes: 2 additions & 0 deletions vmm/sandbox/src/stratovirt/devices/vhost_user_fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ pub struct VhostUserFs {
#[property(key = "chardev")]
pub chardev_id: String,
pub tag: String,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub bus: String,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub addr: String,
}

Expand Down
2 changes: 2 additions & 0 deletions vmm/sandbox/src/stratovirt/devices/vsock.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ pub struct VSockDevice {
pub id: String,
#[property(key = "guest-cid")]
pub context_id: u64,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub bus: String,
#[property(param = "device", predicate = "self.addr.len()>0")]
pub addr: String,
pub vhostfd: i32,
}
Expand Down
43 changes: 26 additions & 17 deletions vmm/sandbox/src/stratovirt/factory.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,8 @@ use super::devices::{
DEFAULT_SERIAL_DEVICE_ID, PCIE_ROOTPORT_CAPACITY,
};
use crate::{
device::Transport,
stratovirt::{
config::{QmpSocket, StratoVirtVMConfig},
config::{QmpSocket, StratoVirtVMConfig, MACHINE_TYPE_MICROVM},
devices::vsock::{find_context_id, VSockDevice},
StratoVirtVM,
},
Expand Down Expand Up @@ -82,21 +81,30 @@ impl VMFactory for StratoVirtVMFactory {
no_wait: true,
});

// create pcie.0 root bus object
vm.pcie_root_bus = create_pcie_root_bus();
let machine_clone = vm.config.machine.r#type.clone();
let machine_array: Vec<_> = machine_clone.split(',').collect();
if machine_array[0] != MACHINE_TYPE_MICROVM {
// create pcie.0 root bus object
vm.pcie_root_bus = create_pcie_root_bus();
}

let transport = vm.config.machine.transport();
// set virtio-rng device
let rng_device = VirtioRngDevice::new(
DEFAULT_RNG_DEVICE_ID,
"/dev/urandom",
Transport::Pci,
transport.clone(),
DEFAULT_PCIE_BUS,
);
vm.attach_to_pcie_rootbus(rng_device)?;
vm.attach_to_bus(rng_device)?;

// set console
let serial = SerialDevice::new(DEFAULT_SERIAL_DEVICE_ID, Transport::Pci, DEFAULT_PCIE_BUS);
vm.attach_to_pcie_rootbus(serial)?;
let serial = SerialDevice::new(
DEFAULT_SERIAL_DEVICE_ID,
transport.clone(),
DEFAULT_PCIE_BUS,
);
vm.attach_to_bus(serial)?;
let console_backend_chardev =
CharDevice::new("socket", DEFAULT_CONSOLE_CHARDEV_ID, &vm.console_socket);
vm.attach_device(console_backend_chardev);
Expand All @@ -106,22 +114,22 @@ impl VMFactory for StratoVirtVMFactory {

if vm.config.kernel.image.is_some() {
let mut image_device: VirtioBlockDevice = VirtioBlockDevice::new(
&Transport::Pci.to_driver(VIRTIO_BLK_DRIVER),
&transport.clone().to_driver(VIRTIO_BLK_DRIVER),
"rootfs",
"blk-0",
vm.config.kernel.image.clone(),
Some(true),
);
image_device.bus = Some(DEFAULT_PCIE_BUS.to_string());
vm.attach_to_pcie_rootbus(image_device)?;
vm.attach_to_bus(image_device)?;
}

// set vsock port as the rpc channel to agent
let (fd, cid) = find_context_id().await?;
let fd_index = vm.append_fd(fd);
let vhost_vsock_device =
VSockDevice::new(cid, Transport::Pci, DEFAULT_PCIE_BUS, fd_index as i32);
vm.attach_to_pcie_rootbus(vhost_vsock_device)?;
VSockDevice::new(cid, transport.clone(), DEFAULT_PCIE_BUS, fd_index as i32);
vm.attach_to_bus(vhost_vsock_device)?;
vm.agent_socket = format!("vsock://{}:1024", cid);

//share fs, stratovirt only support virtiofs share
Expand All @@ -133,22 +141,23 @@ impl VMFactory for StratoVirtVMFactory {
vm.attach_device(virtiofs_chardev);
let vhost_user_fs_device = VhostUserFs::new(
&format!("vhost-user-fs-{}", id),
Transport::Pci,
transport.clone(),
&chardev_id,
DEFAULT_MOUNT_TAG_NAME,
DEFAULT_PCIE_BUS,
);
vm.attach_to_pcie_rootbus(vhost_user_fs_device)?;
vm.attach_to_bus(vhost_user_fs_device)?;

// create virtiofs daemon
vm.create_vitiofs_daemon(
self.default_config.virtiofsd_conf.path.as_str(),
s.base_dir.as_str(),
share_fs_path.as_str(),
);

// set pcie-root-ports for hotplugging
vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?;
if machine_array[0] != MACHINE_TYPE_MICROVM {
// set pcie-root-ports for hotplugging
vm.create_pcie_root_ports(PCIE_ROOTPORT_CAPACITY)?;
}

Ok(vm)
}
Expand Down
33 changes: 22 additions & 11 deletions vmm/sandbox/src/stratovirt/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ pub struct StratoVirtVM {
client: Option<QmpClient>,
virtiofs_daemon: Option<VirtiofsDaemon>,
#[serde(skip)]
pcie_root_bus: PcieRootBus,
pcie_root_bus: Option<PcieRootBus>,
#[serde(skip)]
pcie_root_ports_pool: Option<PCIERootPorts>,
}
Expand Down Expand Up @@ -199,7 +199,7 @@ impl VM for StratoVirtVM {
.vhostfds(vec![])
.bus(Some(DEFAULT_PCIE_BUS.to_string()))
.build();
self.attach_to_pcie_rootbus(virtio_net_device)?;
self.attach_to_bus(virtio_net_device)?;
}
_ => {
todo!()
Expand Down Expand Up @@ -292,7 +292,7 @@ impl StratoVirtVM {
client: None,
virtiofs_daemon: None,
pcie_root_ports_pool: None,
pcie_root_bus: PcieRootBus::default(),
pcie_root_bus: None,
pids: Pids::default(),
}
}
Expand Down Expand Up @@ -435,7 +435,15 @@ impl StratoVirtVM {
}

fn get_empty_pcie_slot_index(&mut self) -> Result<usize> {
for (index, slot) in self.pcie_root_bus.bus.slots.iter_mut().enumerate() {
for (index, slot) in self
.pcie_root_bus
.as_mut()
.unwrap()
.bus
.slots
.iter_mut()
.enumerate()
{
if let SlotStatus::Empty = slot.status {
return Ok(index);
}
Expand All @@ -446,15 +454,18 @@ impl StratoVirtVM {
))
}

fn attach_to_pcie_rootbus<T: StratoVirtDevice + Sync + Send + 'static>(
fn attach_to_bus<T: StratoVirtDevice + Sync + Send + 'static>(
&mut self,
mut device: T,
) -> Result<()> {
// get the empty slot index
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now microVM support a new virtual bus typemmio, so attatch_to_pcie_rootbus func name is not accurate, maybe attach_to_bus is more clear.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

let slot_index = self.get_empty_pcie_slot_index()?;
// set the pcie slot status to Occupied
self.pcie_root_bus.bus.slots[slot_index].status = SlotStatus::Occupied(device.id());
device.set_device_addr(slot_index);
if self.pcie_root_bus.is_some() {
// get the empty slot index
let slot_index = self.get_empty_pcie_slot_index()?;
// set the pcie slot status to Occupied
self.pcie_root_bus.as_mut().unwrap().bus.slots[slot_index].status =
SlotStatus::Occupied(device.id());
device.set_device_addr(slot_index);
}
Comment on lines +461 to +468
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if self.pcie_root_bus.is_some() {
// get the empty slot index
let slot_index = self.get_empty_pcie_slot_index()?;
// set the pcie slot status to Occupied
self.pcie_root_bus.as_mut().unwrap().bus.slots[slot_index].status =
SlotStatus::Occupied(device.id());
device.set_device_addr(slot_index);
}
if let Some(&mut pcie_root_bus) = self.pcie_root_bus {
...
pcie_root_bus.bus.slots[slot_index].status =
SlotStatus::Occupied(device.id());
...
}

I am not sure does it work? :)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried but it did not work well...
move occurs because pcie_root_bus does not implement the Copy trait, but self.get_empty_pcie_slot_index needs to use it next

self.devices.push(Box::new(device));
Ok(())
}
Expand All @@ -480,7 +491,7 @@ impl StratoVirtVM {
None,
);
root_ports.push(root_port.clone());
self.attach_to_pcie_rootbus(root_port)?;
self.attach_to_bus(root_port)?;
}

self.pcie_root_ports_pool = Some(PCIERootPorts {
Expand Down
Loading