From 8cc09775e65b50b82c70b93c011e876b09bed760 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 3 Sep 2024 20:40:21 +0200 Subject: [PATCH] remove spinlock in interrupt handlers --- src/arch/aarch64/kernel/interrupts.rs | 83 ++++++++++++++------------- src/arch/aarch64/mod.rs | 6 -- src/arch/riscv64/kernel/interrupts.rs | 77 ++++++++++++------------- src/arch/riscv64/mod.rs | 6 -- src/arch/x86_64/kernel/interrupts.rs | 39 ++++++------- src/arch/x86_64/mod.rs | 8 --- src/drivers/fs/virtio_fs.rs | 7 +++ src/drivers/mmio.rs | 39 +++++++++++++ src/drivers/mod.rs | 30 ++++++++++ src/drivers/net/gem.rs | 14 ++--- src/drivers/net/mod.rs | 3 +- src/drivers/net/rtl8139.rs | 19 +++--- src/drivers/net/virtio/mmio.rs | 6 +- src/drivers/net/virtio/mod.rs | 8 +++ src/drivers/net/virtio/pci.rs | 1 + src/drivers/pci.rs | 80 +++++++++++++++++++++++++- src/drivers/virtio/transport/mmio.rs | 34 ++--------- src/drivers/virtio/transport/mod.rs | 11 ---- src/drivers/virtio/transport/pci.rs | 31 +--------- src/drivers/vsock/mod.rs | 7 +++ src/lib.rs | 2 +- 21 files changed, 297 insertions(+), 214 deletions(-) diff --git a/src/arch/aarch64/kernel/interrupts.rs b/src/arch/aarch64/kernel/interrupts.rs index 08c38e9465..e0d3f78efd 100644 --- a/src/arch/aarch64/kernel/interrupts.rs +++ b/src/arch/aarch64/kernel/interrupts.rs @@ -17,6 +17,11 @@ use crate::arch::aarch64::kernel::scheduler::State; use crate::arch::aarch64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; use crate::arch::aarch64::mm::{virtualmem, PhysAddr}; use crate::core_scheduler; +#[cfg(not(feature = "pci"))] +use crate::drivers::mmio::get_interrupt_handlers; +#[cfg(feature = "pci")] +use crate::drivers::pci::get_interrupt_handlers; +use crate::drivers::InterruptHandlerQueue; use crate::scheduler::{self, CoreId}; /// The ID of the first Private Peripheral Interrupt. @@ -27,13 +32,11 @@ const SPI_START: u8 = 32; /// Software-generated interrupt for rescheduling pub(crate) const SGI_RESCHED: u8 = 1; -type InterruptHandlerQueue = VecDeque; - /// Number of the timer interrupt static mut TIMER_INTERRUPT: u32 = 0; /// Possible interrupt handlers -static INTERRUPT_HANDLERS: InterruptSpinMutex> = - InterruptSpinMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); +static INTERRUPT_HANDLERS: OnceCell> = + OnceCell::new(); /// Driver for the Arm Generic Interrupt Controller version 3 (or 4). pub(crate) static mut GIC: OnceCell = OnceCell::new(); @@ -76,18 +79,34 @@ pub fn disable() { } } -#[allow(dead_code)] -pub(crate) fn irq_install_handler(irq_number: u8, handler: fn()) { - debug!("Install handler for interrupt {}", irq_number); - let irq_number = irq_number + SPI_START; - let mut guard = INTERRUPT_HANDLERS.lock(); - if let Some(queue) = guard.get_mut(&irq_number) { - queue.push_back(handler); - } else { - let mut queue = VecDeque::new(); - queue.push_back(handler); - guard.insert(irq_number, queue); +pub(crate) fn install_handlers() { + let mut handlers = get_interrupt_handlers(); + + fn timer_handler() { + debug!("Handle timer interrupt"); + + // disable timer + unsafe { + asm!( + "msr cntp_cval_el0, xzr", + "msr cntp_ctl_el0, xzr", + options(nostack, nomem), + ); + } } + + unsafe { + if let Some(queue) = handlers.get_mut(&(u8::try_from(TIMER_INTERRUPT).unwrap() + PPI_START)) + { + queue.push_back(timer_handler); + } else { + let mut queue = VecDeque::::new(); + queue.push_back(timer_handler); + handlers.insert(u8::try_from(TIMER_INTERRUPT).unwrap() + PPI_START, queue); + } + } + + INTERRUPT_HANDLERS.set(handlers).unwrap(); } #[no_mangle] @@ -98,9 +117,11 @@ pub(crate) extern "C" fn do_fiq(_state: &State) -> *mut usize { debug!("Receive fiq {}", vector); increment_irq_counter(vector); - if let Some(queue) = INTERRUPT_HANDLERS.lock().get(&vector) { - for handler in queue.iter() { - handler(); + if let Some(handlers) = INTERRUPT_HANDLERS.get() { + if let Some(queue) = handlers.get(&vector) { + for handler in queue.iter() { + handler(); + } } } crate::executor::run(); @@ -124,9 +145,11 @@ pub(crate) extern "C" fn do_irq(_state: &State) -> *mut usize { debug!("Receive interrupt {}", vector); increment_irq_counter(vector); - if let Some(queue) = INTERRUPT_HANDLERS.lock().get(&vector) { - for handler in queue.iter() { - handler(); + if let Some(handlers) = INTERRUPT_HANDLERS.get() { + if let Some(queue) = handlers.get(&vector) { + for handler in queue.iter() { + handler(); + } } } crate::executor::run(); @@ -280,24 +303,6 @@ pub(crate) fn init() { irq, irqtype, irqflags ); - fn timer_handler() { - debug!("Handle timer interrupt"); - - // disable timer - unsafe { - asm!( - "msr cntp_cval_el0, xzr", - "msr cntp_ctl_el0, xzr", - options(nostack, nomem), - ); - } - } - - let mut queue = VecDeque::::new(); - queue.push_back(timer_handler); - INTERRUPT_HANDLERS - .lock() - .insert(u8::try_from(irq).unwrap() + PPI_START, queue); IRQ_NAMES .lock() .insert(u8::try_from(irq).unwrap() + PPI_START, "Timer"); diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index f5717c958e..f6957c9355 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -10,9 +10,3 @@ pub(crate) fn memory_barrier() { asm!("dmb ish", options(nostack, nomem, preserves_flags),); } } - -pub fn init_drivers() { - // Initialize PCI Drivers - #[cfg(feature = "pci")] - crate::drivers::pci::init_drivers(); -} diff --git a/src/arch/riscv64/kernel/interrupts.rs b/src/arch/riscv64/kernel/interrupts.rs index 43cce75719..5e68eb317d 100644 --- a/src/arch/riscv64/kernel/interrupts.rs +++ b/src/arch/riscv64/kernel/interrupts.rs @@ -1,13 +1,17 @@ -use alloc::collections::VecDeque; use alloc::vec::Vec; use ahash::RandomState; use hashbrown::HashMap; -use hermit_sync::{InterruptSpinMutex, SpinMutex}; +use hermit_sync::{OnceCell, SpinMutex}; use riscv::asm::wfi; use riscv::register::{scause, sie, sip, sstatus, stval}; use trapframe::TrapFrame; +#[cfg(not(feature = "pci"))] +use crate::drivers::mmio::get_interrupt_handlers; +#[cfg(feature = "pci")] +use crate::drivers::pci::get_interrupt_handlers; +use crate::drivers::InterruptHandlerQueue; use crate::scheduler; /// base address of the PLIC, only one access at the same time is allowed @@ -19,9 +23,8 @@ static PLIC_CONTEXT: SpinMutex = SpinMutex::new(0x0); /// PLIC context for new interrupt handlers static CURRENT_INTERRUPTS: SpinMutex> = SpinMutex::new(Vec::new()); -type InterruptHandlerQueue = VecDeque; -static INTERRUPT_HANDLERS: InterruptSpinMutex> = - InterruptSpinMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); +static INTERRUPT_HANDLERS: OnceCell> = + OnceCell::new(); /// Init Interrupts pub fn install() { @@ -110,40 +113,32 @@ pub fn disable() { } /// Currently not needed because we use the trapframe crate -#[cfg(feature = "tcp")] -pub fn irq_install_handler(irq_number: u8, handler: fn()) { - unsafe { - let base_ptr = PLIC_BASE.lock(); - let context = PLIC_CONTEXT.lock(); - debug!( - "Install handler for interrupt {}, context {}", - irq_number, *context - ); - - let mut guard = INTERRUPT_HANDLERS.lock(); - if let Some(queue) = guard.get_mut(&irq_number) { - queue.push_back(handler); - } else { - let mut queue = VecDeque::new(); - queue.push_back(handler); - guard.insert(irq_number, queue); +pub(crate) fn install_handlers() { + let handlers = get_interrupt_handlers(); + + for irq_number in handlers.keys() { + unsafe { + let base_ptr = PLIC_BASE.lock(); + let context = PLIC_CONTEXT.lock(); + + // Set priority to 7 (highest on FU740) + let prio_address = *base_ptr + *irq_number as usize * 4; + core::ptr::write_volatile(prio_address as *mut u32, 1); + // Set Threshold to 0 (lowest) + let thresh_address = *base_ptr + 0x20_0000 + 0x1000 * (*context as usize); + core::ptr::write_volatile(thresh_address as *mut u32, 0); + // Enable irq for context + const PLIC_ENABLE_OFFSET: usize = 0x002000; + let enable_address = *base_ptr + + PLIC_ENABLE_OFFSET + + 0x80 * (*context as usize) + + ((*irq_number / 32) * 4) as usize; + debug!("enable_address {:x}", enable_address); + core::ptr::write_volatile(enable_address as *mut u32, 1 << (irq_number % 32)); } - - // Set priority to 7 (highest on FU740) - let prio_address = *base_ptr + irq_number as usize * 4; - core::ptr::write_volatile(prio_address as *mut u32, 1); - // Set Threshold to 0 (lowest) - let thresh_address = *base_ptr + 0x20_0000 + 0x1000 * (*context as usize); - core::ptr::write_volatile(thresh_address as *mut u32, 0); - // Enable irq for context - const PLIC_ENABLE_OFFSET: usize = 0x002000; - let enable_address = *base_ptr - + PLIC_ENABLE_OFFSET - + 0x80 * (*context as usize) - + ((irq_number / 32) * 4) as usize; - debug!("enable_address {:x}", enable_address); - core::ptr::write_volatile(enable_address as *mut u32, 1 << (irq_number % 32)); } + + INTERRUPT_HANDLERS.set(handlers).unwrap(); } // Derived from rCore: https://github.com/rcore-os/rCore @@ -205,9 +200,11 @@ fn external_handler() { } // Call handler - if let Some(queue) = INTERRUPT_HANDLERS.lock().get(&u8::try_from(irq).unwrap()) { - for handler in queue.iter() { - handler(); + if let Some(handlers) = INTERRUPT_HANDLERS.get() { + if let Some(queue) = handlers.get(&u8::try_from(irq).unwrap()) { + for handler in queue.iter() { + handler(); + } } } crate::executor::run(); diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index 3640db768e..c69a84dc95 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -7,9 +7,3 @@ pub mod mm; pub(crate) fn memory_barrier() { riscv::asm::sfence_vma_all() } - -pub fn init_drivers() { - // Initialize PCI Drivers - #[cfg(feature = "pci")] - crate::drivers::pci::init_drivers(); -} diff --git a/src/arch/x86_64/kernel/interrupts.rs b/src/arch/x86_64/kernel/interrupts.rs index f91ae6e554..be5397f7a4 100644 --- a/src/arch/x86_64/kernel/interrupts.rs +++ b/src/arch/x86_64/kernel/interrupts.rs @@ -1,10 +1,10 @@ -use alloc::collections::{BTreeMap, VecDeque}; +use alloc::collections::BTreeMap; use core::arch::asm; use core::sync::atomic::{AtomicU64, Ordering}; use ahash::RandomState; use hashbrown::HashMap; -use hermit_sync::{InterruptSpinMutex, InterruptTicketMutex}; +use hermit_sync::{InterruptSpinMutex, InterruptTicketMutex, OnceCell}; #[cfg(not(feature = "idle-poll"))] use x86_64::instructions::interrupts::enable_and_hlt; pub use x86_64::instructions::interrupts::{disable, enable}; @@ -16,11 +16,14 @@ use crate::arch::x86_64::kernel::core_local::{core_scheduler, increment_irq_coun use crate::arch::x86_64::kernel::{apic, processor}; use crate::arch::x86_64::mm::paging::{page_fault_handler, BasePageSize, PageSize}; use crate::arch::x86_64::swapgs; +#[cfg(not(feature = "pci"))] +use crate::drivers::mmio::get_interrupt_handlers; +#[cfg(feature = "pci")] +use crate::drivers::pci::get_interrupt_handlers; +use crate::drivers::InterruptHandlerQueue; use crate::scheduler::{self, CoreId}; -type InterruptHandlerQueue = VecDeque; -static IRQ_HANDLERS: InterruptSpinMutex> = - InterruptSpinMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); +static IRQ_HANDLERS: OnceCell> = OnceCell::new(); static IRQ_NAMES: InterruptTicketMutex> = InterruptTicketMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); @@ -158,18 +161,8 @@ pub(crate) fn install() { IRQ_NAMES.lock().insert(7, "FPU"); } -#[cfg(any(feature = "fuse", feature = "tcp", feature = "udp", feature = "vsock"))] -pub fn irq_install_handler(irq_number: u8, handler: fn()) { - debug!("Install handler for interrupt {}", irq_number); - - let mut guard = IRQ_HANDLERS.lock(); - if let Some(map) = guard.get_mut(&irq_number) { - map.push_back(handler); - } else { - let mut map = VecDeque::new(); - map.push_back(handler); - guard.insert(irq_number, map); - } +pub(crate) fn install_handlers() { + IRQ_HANDLERS.set(get_interrupt_handlers()).unwrap(); } fn handle_interrupt(stack_frame: ExceptionStackFrame, index: u8, _error_code: Option) { @@ -177,13 +170,13 @@ fn handle_interrupt(stack_frame: ExceptionStackFrame, index: u8, _error_code: Op use crate::arch::kernel::core_local::core_scheduler; use crate::scheduler::PerCoreSchedulerExt; - if let Some(map) = IRQ_HANDLERS.lock().get(&(index - 32)) { - debug!("received interrupt {index}"); - for handler in map.iter() { - handler(); + if let Some(handlers) = IRQ_HANDLERS.get() { + if let Some(map) = handlers.get(&(index - 32)) { + debug!("received interrupt {index}"); + for handler in map.iter() { + handler(); + } } - } else { - warn!("received unhandled interrupt {index}"); } apic::eoi(); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 2998938989..5a7af85f96 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -32,11 +32,3 @@ pub(crate) fn memory_barrier() { asm!("mfence", options(nostack, nomem, preserves_flags),); } } - -pub fn init_drivers() { - // Initialize PCI Drivers - #[cfg(feature = "pci")] - crate::drivers::pci::init_drivers(); - #[cfg(all(not(feature = "pci"), any(feature = "tcp", feature = "udp")))] - crate::arch::x86_64::kernel::mmio::init_drivers(); -} diff --git a/src/drivers/fs/virtio_fs.rs b/src/drivers/fs/virtio_fs.rs index 9a10dc4f5f..af6909daf2 100644 --- a/src/drivers/fs/virtio_fs.rs +++ b/src/drivers/fs/virtio_fs.rs @@ -20,6 +20,7 @@ use crate::drivers::virtio::virtqueue::split::SplitVq; use crate::drivers::virtio::virtqueue::{ AvailBufferToken, BufferElem, BufferType, Virtq, VqIndex, VqSize, }; +use crate::drivers::Driver; use crate::fs::fuse::{self, FuseInterface, Rsp, RspHeader}; use crate::mm::device_alloc::DeviceAlloc; @@ -195,6 +196,12 @@ impl FuseInterface for VirtioFsDriver { } } +impl Driver for VirtioFsDriver { + fn get_interrupt_number(&self) -> InterruptLine { + self.irq + } +} + /// Error module of virtios filesystem driver. pub mod error { /// Network filesystem error enum. diff --git a/src/drivers/mmio.rs b/src/drivers/mmio.rs index 642d528ee6..df8a757f4a 100644 --- a/src/drivers/mmio.rs +++ b/src/drivers/mmio.rs @@ -1,2 +1,41 @@ +#[cfg(any(feature = "tcp", feature = "udp"))] +use alloc::collections::VecDeque; + +use ahash::RandomState; +use hashbrown::HashMap; + #[cfg(any(feature = "tcp", feature = "udp"))] pub(crate) use crate::arch::kernel::mmio::get_network_driver; +#[cfg(any(feature = "tcp", feature = "udp"))] +use crate::drivers::net::NetworkDriver; +#[cfg(any(feature = "tcp", feature = "udp"))] +use crate::drivers::Driver; +use crate::drivers::{InterruptHandlerQueue, InterruptLine}; + +pub(crate) fn get_interrupt_handlers() -> HashMap +{ + #[allow(unused_mut)] + let mut handlers: HashMap = + HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0)); + + #[cfg(any(feature = "tcp", feature = "udp"))] + if let Some(drv) = get_network_driver() { + fn network_handler() { + if let Some(driver) = get_network_driver() { + driver.lock().handle_interrupt(); + } + } + + let irq_number = drv.lock().get_interrupt_number(); + + if let Some(map) = handlers.get_mut(&irq_number) { + map.push_back(network_handler); + } else { + let mut map: InterruptHandlerQueue = VecDeque::new(); + map.push_back(network_handler); + handlers.insert(irq_number, map); + } + } + + handlers +} diff --git a/src/drivers/mod.rs b/src/drivers/mod.rs index 580171c434..4f0a6d6f01 100644 --- a/src/drivers/mod.rs +++ b/src/drivers/mod.rs @@ -17,6 +17,15 @@ pub mod virtio; #[cfg(feature = "vsock")] pub mod vsock; +use alloc::collections::VecDeque; + +#[cfg(feature = "pci")] +pub(crate) use pci_types::InterruptLine; +#[cfg(not(feature = "pci"))] +pub(crate) type InterruptLine = u8; + +pub(crate) type InterruptHandlerQueue = VecDeque; + /// A common error module for drivers. /// [DriverError](error::DriverError) values will be /// passed on to higher layers. @@ -97,3 +106,24 @@ pub mod error { } } } + +/// A trait to determine general driver information +#[allow(dead_code)] +pub(crate) trait Driver { + /// Returns the interrupt number of the device + fn get_interrupt_number(&self) -> InterruptLine; +} + +pub(crate) fn init() { + // Initialize PCI Drivers + #[cfg(feature = "pci")] + crate::drivers::pci::init(); + #[cfg(all( + not(feature = "pci"), + target_arch = "x86_64", + any(feature = "tcp", feature = "udp") + ))] + crate::arch::x86_64::kernel::mmio::init_drivers(); + + crate::arch::interrupts::install_handlers(); +} diff --git a/src/drivers/net/gem.rs b/src/drivers/net/gem.rs index 065b624460..0aeff53ab4 100644 --- a/src/drivers/net/gem.rs +++ b/src/drivers/net/gem.rs @@ -25,6 +25,7 @@ use crate::drivers::error::DriverError; use crate::drivers::net::NetworkDriver; #[cfg(all(any(feature = "tcp", feature = "udp"), feature = "pci"))] use crate::drivers::pci as hardware; +use crate::drivers::{Driver, InterruptLine}; use crate::executor::device::{RxToken, TxToken}; //Base address of the control registers @@ -398,8 +399,12 @@ impl NetworkDriver for GEMDriver { // handle incoming packets todo!(); } + } +} - //increment_irq_counter((32 + self.irq).into()); +impl Driver for GEMDriver { + fn get_interrupt_number(&self) -> InterruptLine { + self.irq } } @@ -679,13 +684,6 @@ pub fn init_device( // Configure Interrupts debug!("Install interrupt handler for GEM"); - fn network_handler() { - if let Some(driver) = hardware::get_network_driver() { - driver.lock().handle_interrupt() - } - } - - irq_install_handler(irq, network_handler); (*gem).int_enable.write(Interrupts::FRAMERX::SET); // + Interrupts::TXCOMPL::SET // Enable the Controller (again?) diff --git a/src/drivers/net/mod.rs b/src/drivers/net/mod.rs index d4f72652a0..0cff2854af 100644 --- a/src/drivers/net/mod.rs +++ b/src/drivers/net/mod.rs @@ -9,10 +9,11 @@ use smoltcp::phy::ChecksumCapabilities; #[allow(unused_imports)] use crate::arch::kernel::core_local::*; +use crate::drivers::Driver; use crate::executor::device::{RxToken, TxToken}; /// A trait for accessing the network interface -pub(crate) trait NetworkDriver { +pub(crate) trait NetworkDriver: Driver { /// Returns smoltcp's checksum capabilities fn get_checksums(&self) -> ChecksumCapabilities { ChecksumCapabilities::default() diff --git a/src/drivers/net/rtl8139.rs b/src/drivers/net/rtl8139.rs index 8fad45280c..9f9ec2b8f9 100644 --- a/src/drivers/net/rtl8139.rs +++ b/src/drivers/net/rtl8139.rs @@ -8,15 +8,14 @@ use core::mem; use pci_types::{Bar, CommandRegister, InterruptLine, MAX_BARS}; use x86::io::*; -use crate::arch::kernel::core_local::increment_irq_counter; use crate::arch::kernel::interrupts::*; use crate::arch::mm::paging::virt_to_phys; use crate::arch::mm::VirtAddr; use crate::arch::pci::PciConfigRegion; use crate::drivers::error::DriverError; use crate::drivers::net::NetworkDriver; -use crate::drivers::pci as hardware; use crate::drivers::pci::PciDevice; +use crate::drivers::Driver; use crate::executor::device::{RxToken, TxToken}; /// size of the receive buffer @@ -319,8 +318,6 @@ impl NetworkDriver for RTL8139Driver { } fn handle_interrupt(&mut self) { - increment_irq_counter(32 + self.irq); - let isr_contents = unsafe { inw(self.iobase + ISR) }; if (isr_contents & ISR_TOK) == ISR_TOK { @@ -348,6 +345,12 @@ impl NetworkDriver for RTL8139Driver { } } +impl Driver for RTL8139Driver { + fn get_interrupt_number(&self) -> InterruptLine { + self.irq + } +} + impl RTL8139Driver { // Tells driver, that buffer is consumed and can be deallocated fn consume_current_buffer(&mut self) { @@ -570,14 +573,6 @@ pub(crate) fn init_device( // Install interrupt handler for RTL8139 debug!("Install interrupt handler for RTL8139 at {}", irq); - - fn network_handler() { - if let Some(driver) = hardware::get_network_driver() { - driver.lock().handle_interrupt() - } - } - - irq_install_handler(irq, network_handler); add_irq_name(irq, "rtl8139_net"); Ok(RTL8139Driver { diff --git a/src/drivers/net/virtio/mmio.rs b/src/drivers/net/virtio/mmio.rs index ff73ef3b8e..595302f03c 100644 --- a/src/drivers/net/virtio/mmio.rs +++ b/src/drivers/net/virtio/mmio.rs @@ -14,12 +14,14 @@ use crate::drivers::net::virtio::{CtrlQueue, NetDevCfg, RxQueues, TxQueues, Virt use crate::drivers::virtio::error::{VirtioError, VirtioNetError}; use crate::drivers::virtio::transport::mmio::{ComCfg, IsrStatus, NotifCfg}; use crate::drivers::virtio::virtqueue::Virtq; +use crate::drivers::InterruptLine; // Backend-dependent interface for Virtio network driver impl VirtioNetDriver { pub fn new( dev_id: u16, mut registers: VolatileRef<'static, DeviceRegisters>, + irq: InterruptLine, ) -> Result { let dev_cfg_raw: &'static virtio::net::Config = unsafe { &*registers @@ -58,6 +60,7 @@ impl VirtioNetDriver { send_vqs, num_vqs: 0, mtu, + irq, checksums: ChecksumCapabilities::default(), }) } @@ -77,8 +80,9 @@ impl VirtioNetDriver { pub fn init( dev_id: u16, registers: VolatileRef<'static, DeviceRegisters>, + irq: InterruptLine, ) -> Result { - if let Ok(mut drv) = VirtioNetDriver::new(dev_id, registers) { + if let Ok(mut drv) = VirtioNetDriver::new(dev_id, registers, irq) { match drv.init_dev() { Err(error_code) => Err(VirtioError::NetDriver(error_code)), _ => { diff --git a/src/drivers/net/virtio/mod.rs b/src/drivers/net/virtio/mod.rs index 4c39d12b8b..0b8030366d 100644 --- a/src/drivers/net/virtio/mod.rs +++ b/src/drivers/net/virtio/mod.rs @@ -34,6 +34,7 @@ use crate::drivers::virtio::virtqueue::split::SplitVq; use crate::drivers::virtio::virtqueue::{ AvailBufferToken, BufferElem, BufferType, UsedBufferToken, Virtq, VqIndex, VqSize, }; +use crate::drivers::{Driver, InterruptLine}; use crate::executor::device::{RxToken, TxToken}; use crate::mm::device_alloc::DeviceAlloc; @@ -247,6 +248,7 @@ pub(crate) struct VirtioNetDriver { pub(super) num_vqs: u16, pub(super) mtu: u16, + pub(super) irq: InterruptLine, pub(super) checksums: ChecksumCapabilities, } @@ -408,6 +410,12 @@ impl NetworkDriver for VirtioNetDriver { } } +impl Driver for VirtioNetDriver { + fn get_interrupt_number(&self) -> InterruptLine { + self.irq + } +} + // Backend-independent interface for Virtio network driver impl VirtioNetDriver { #[cfg(feature = "pci")] diff --git a/src/drivers/net/virtio/pci.rs b/src/drivers/net/virtio/pci.rs index a477b6509e..b3584743c2 100644 --- a/src/drivers/net/virtio/pci.rs +++ b/src/drivers/net/virtio/pci.rs @@ -98,6 +98,7 @@ impl VirtioNetDriver { send_vqs, num_vqs: 0, mtu, + irq: device.get_irq().unwrap(), checksums: ChecksumCapabilities::default(), }) } diff --git a/src/drivers/pci.rs b/src/drivers/pci.rs index a00f4a8637..ed9ff8477d 100644 --- a/src/drivers/pci.rs +++ b/src/drivers/pci.rs @@ -1,8 +1,11 @@ #![allow(dead_code)] +use alloc::collections::VecDeque; use alloc::vec::Vec; use core::fmt; +use ahash::RandomState; +use hashbrown::HashMap; use hermit_sync::without_interrupts; #[cfg(any(feature = "tcp", feature = "udp", feature = "fuse", feature = "vsock"))] use hermit_sync::InterruptTicketMutex; @@ -20,6 +23,8 @@ use crate::drivers::fs::virtio_fs::VirtioFsDriver; use crate::drivers::net::rtl8139::{self, RTL8139Driver}; #[cfg(all(not(feature = "rtl8139"), any(feature = "tcp", feature = "udp")))] use crate::drivers::net::virtio::VirtioNetDriver; +#[cfg(any(feature = "tcp", feature = "udp"))] +use crate::drivers::net::NetworkDriver; #[cfg(any( all(any(feature = "tcp", feature = "udp"), not(feature = "rtl8139")), feature = "fuse", @@ -34,6 +39,8 @@ use crate::drivers::virtio::transport::pci as pci_virtio; use crate::drivers::virtio::transport::pci::VirtioDriver; #[cfg(feature = "vsock")] use crate::drivers::vsock::VirtioVsockDriver; +#[allow(unused_imports)] +use crate::drivers::{Driver, InterruptHandlerQueue}; pub(crate) static mut PCI_DEVICES: Vec> = Vec::new(); static mut PCI_DRIVERS: Vec = Vec::new(); @@ -350,6 +357,57 @@ impl PciDriver { _ => None, } } + + fn get_interrupt_handler(&self) -> (InterruptLine, fn()) { + #[allow(unreachable_patterns)] + match self { + #[cfg(feature = "vsock")] + Self::VirtioVsock(drv) => { + fn vsock_handler() { + if let Some(driver) = get_vsock_driver() { + driver.lock().handle_interrupt(); + } + } + + let irq_number = drv.lock().get_interrupt_number(); + + (irq_number, vsock_handler) + } + #[cfg(all(feature = "rtl8139", any(feature = "tcp", feature = "udp")))] + Self::RTL8139Net(drv) => { + fn rtl8139_handler() { + if let Some(driver) = get_network_driver() { + driver.lock().handle_interrupt(); + } + } + + let irq_number = drv.lock().get_interrupt_number(); + + (irq_number, rtl8139_handler) + } + #[cfg(all(not(feature = "rtl8139"), any(feature = "tcp", feature = "udp")))] + Self::VirtioNet(drv) => { + fn network_handler() { + if let Some(driver) = get_network_driver() { + driver.lock().handle_interrupt(); + } + } + + let irq_number = drv.lock().get_interrupt_number(); + + (irq_number, network_handler) + } + #[cfg(feature = "fuse")] + Self::VirtioFs(drv) => { + fn fuse_handler() {} + + let irq_number = drv.lock().get_interrupt_number(); + + (irq_number, fuse_handler) + } + _ => todo!(), + } + } } pub(crate) fn register_driver(drv: PciDriver) { @@ -358,6 +416,26 @@ pub(crate) fn register_driver(drv: PciDriver) { } } +pub(crate) fn get_interrupt_handlers() -> HashMap +{ + let mut handlers: HashMap = + HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0)); + + for drv in unsafe { PCI_DRIVERS.iter() } { + let (irq_number, handler) = drv.get_interrupt_handler(); + + if let Some(map) = handlers.get_mut(&irq_number) { + map.push_back(handler); + } else { + let mut map: InterruptHandlerQueue = VecDeque::new(); + map.push_back(handler); + handlers.insert(irq_number, map); + } + } + + handlers +} + #[cfg(all(not(feature = "rtl8139"), any(feature = "tcp", feature = "udp")))] pub(crate) fn get_network_driver() -> Option<&'static InterruptTicketMutex> { unsafe { PCI_DRIVERS.iter().find_map(|drv| drv.get_network_driver()) } @@ -382,7 +460,7 @@ pub(crate) fn get_filesystem_driver() -> Option<&'static InterruptTicketMutex { vq_index: u16, @@ -355,7 +352,7 @@ pub(crate) enum VirtioDriver { #[allow(unused_variables)] pub(crate) fn init_device( registers: VolatileRef<'static, DeviceRegisters>, - irq_no: u8, + irq_no: InterruptLine, ) -> Result { let dev_id: u16 = 0; @@ -369,21 +366,11 @@ pub(crate) fn init_device( // Verify the device-ID to find the network card match registers.as_ptr().device_id().read() { #[cfg(any(feature = "tcp", feature = "udp"))] - virtio::Id::Net => match VirtioNetDriver::init(dev_id, registers) { + virtio::Id::Net => match VirtioNetDriver::init(dev_id, registers, irq_no) { Ok(virt_net_drv) => { info!("Virtio network driver initialized."); - - fn network_handler() { - use crate::drivers::net::NetworkDriver; - if let Some(driver) = hardware::get_network_driver() { - driver.lock().handle_interrupt() - } - } - - irq_install_handler(irq_no, network_handler); - #[cfg(not(target_arch = "riscv64"))] - add_irq_name(irq_no, "virtio"); + crate::arch::interrupts::add_irq_name(irq_no, "virtio"); Ok(VirtioDriver::Network(virt_net_drv)) } @@ -393,20 +380,11 @@ pub(crate) fn init_device( } }, #[cfg(feature = "vsock")] - virtio::Id::Vsock => match VirtioVsockDriver::init(dev_id, registers) { + virtio::Id::Vsock => match VirtioVsockDriver::init(dev_id, registers, irq_no) { Ok(virt_net_drv) => { info!("Virtio sock driver initialized."); - - fn vsock_handler() { - if let Some(driver) = hardware::get_vsock_driver() { - driver.lock().handle_interrupt(); - } - } - - irq_install_handler(irq_no, vsock_handler); #[cfg(not(target_arch = "riscv64"))] - add_irq_name(irq_no, "virtio"); - let _ = VIRTIO_IRQ.try_insert(irq_no); + crate::arch::interrupts::add_irq_name(irq_no, "virtio"); Ok(VirtioDriver::Vsock(virt_vsock_drv)) } diff --git a/src/drivers/virtio/transport/mod.rs b/src/drivers/virtio/transport/mod.rs index 278481e8d1..50df4b30c9 100644 --- a/src/drivers/virtio/transport/mod.rs +++ b/src/drivers/virtio/transport/mod.rs @@ -8,14 +8,3 @@ pub mod mmio; #[cfg(feature = "pci")] pub mod pci; - -#[cfg(all( - any(feature = "vsock", feature = "tcp", feature = "udp"), - not(feature = "pci") -))] -use crate::arch::kernel::mmio as hardware; -#[cfg(all( - any(feature = "vsock", feature = "tcp", feature = "udp"), - feature = "pci" -))] -use crate::drivers::pci as hardware; diff --git a/src/drivers/virtio/transport/pci.rs b/src/drivers/virtio/transport/pci.rs index c7698bb9e5..c003a0e69f 100644 --- a/src/drivers/virtio/transport/pci.rs +++ b/src/drivers/virtio/transport/pci.rs @@ -16,11 +16,6 @@ use virtio::{le16, le32, DeviceStatus}; use volatile::access::ReadOnly; use volatile::{VolatilePtr, VolatileRef}; -#[cfg(all( - not(feature = "rtl8139"), - any(feature = "tcp", feature = "udp", feature = "vsock") -))] -use crate::arch::kernel::interrupts::*; use crate::arch::memory_barrier; use crate::arch::mm::PhysAddr; use crate::arch::pci::PciConfigRegion; @@ -32,11 +27,6 @@ use crate::drivers::net::virtio::VirtioNetDriver; use crate::drivers::pci::error::PciError; use crate::drivers::pci::PciDevice; use crate::drivers::virtio::error::VirtioError; -#[cfg(all( - not(feature = "rtl8139"), - any(feature = "tcp", feature = "udp", feature = "vsock") -))] -use crate::drivers::virtio::transport::hardware; #[cfg(feature = "vsock")] use crate::drivers::vsock::VirtioVsockDriver; @@ -962,16 +952,7 @@ pub(crate) fn init_device( let irq = device.get_irq().unwrap(); info!("Install virtio interrupt handler at line {}", irq); - - fn network_handler() { - use crate::drivers::net::NetworkDriver; - if let Some(driver) = hardware::get_network_driver() { - driver.lock().handle_interrupt() - } - } - - irq_install_handler(irq, network_handler); - add_irq_name(irq, "virtio"); + crate::arch::interrupts::add_irq_name(irq, "virtio"); Ok(drv) } @@ -980,15 +961,7 @@ pub(crate) fn init_device( let irq = device.get_irq().unwrap(); info!("Install virtio interrupt handler at line {}", irq); - - fn vsock_handler() { - if let Some(driver) = hardware::get_vsock_driver() { - driver.lock().handle_interrupt(); - } - } - - irq_install_handler(irq, vsock_handler); - add_irq_name(irq, "virtio"); + crate::arch::interrupts::add_irq_name(irq, "virtio"); Ok(drv) } diff --git a/src/drivers/vsock/mod.rs b/src/drivers/vsock/mod.rs index 7ab72e3008..5b20deae59 100644 --- a/src/drivers/vsock/mod.rs +++ b/src/drivers/vsock/mod.rs @@ -22,6 +22,7 @@ use crate::drivers::virtio::virtqueue::{ }; #[cfg(feature = "pci")] use crate::drivers::vsock::pci::VsockDevCfgRaw; +use crate::drivers::Driver; use crate::mm::device_alloc::DeviceAlloc; fn fill_queue( @@ -291,6 +292,12 @@ pub(crate) struct VirtioVsockDriver { pub(super) send_vq: TxQueue, } +impl Driver for VirtioVsockDriver { + fn get_interrupt_number(&self) -> InterruptLine { + self.irq + } +} + impl VirtioVsockDriver { #[cfg(feature = "pci")] pub fn get_dev_id(&self) -> u16 { diff --git a/src/lib.rs b/src/lib.rs index 04efe7b6ac..0ec171e61d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,7 +139,7 @@ extern "C" fn initd(_arg: usize) { } // Initialize Drivers - arch::init_drivers(); + drivers::init(); crate::executor::init(); // Initialize MMIO Drivers if on riscv64