From b6c3a8d5623b82b93113759a9003dc60c9fc59d2 Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Thu, 30 Nov 2023 15:10:40 +0100 Subject: [PATCH] refactor: move riscv specific code to `arch` mod In the future we want to port this to other architectures beside riscv, to make this easier its helpful to have architecture-specific code cleanly separated. --- crates/kernel/src/arch/backtrace.rs | 103 +++++++++++ crates/kernel/src/{ => arch}/interrupt.rs | 0 crates/kernel/src/arch/mod.rs | 19 ++ crates/kernel/src/{ => arch}/paging/entry.rs | 6 +- crates/kernel/src/{ => arch}/paging/flush.rs | 6 +- crates/kernel/src/arch/paging/mapper.rs | 143 +++++++++++++++ crates/kernel/src/arch/paging/mod.rs | 113 ++++++++++++ crates/kernel/src/arch/paging/table.rs | 125 ++++++++++++++ crates/kernel/src/arch/start.rs | 128 ++++++++++++++ crates/kernel/src/{ => arch}/trap.rs | 50 +++--- crates/kernel/src/backtrace.rs | 104 ++--------- crates/kernel/src/error.rs | 8 + crates/kernel/src/logger.rs | 13 +- crates/kernel/src/main.rs | 51 ++---- crates/kernel/src/paging/frame_alloc.rs | 10 +- crates/kernel/src/paging/mapper.rs | 94 ---------- crates/kernel/src/paging/mod.rs | 133 +------------- crates/kernel/src/paging/table.rs | 51 ------ crates/kernel/src/panic.rs | 22 +++ crates/kernel/src/start.rs | 172 ------------------- 20 files changed, 735 insertions(+), 616 deletions(-) create mode 100644 crates/kernel/src/arch/backtrace.rs rename crates/kernel/src/{ => arch}/interrupt.rs (100%) create mode 100644 crates/kernel/src/arch/mod.rs rename crates/kernel/src/{ => arch}/paging/entry.rs (92%) rename crates/kernel/src/{ => arch}/paging/flush.rs (66%) create mode 100644 crates/kernel/src/arch/paging/mapper.rs create mode 100644 crates/kernel/src/arch/paging/mod.rs create mode 100644 crates/kernel/src/arch/paging/table.rs create mode 100644 crates/kernel/src/arch/start.rs rename crates/kernel/src/{ => arch}/trap.rs (87%) delete mode 100644 crates/kernel/src/paging/mapper.rs delete mode 100644 crates/kernel/src/paging/table.rs create mode 100644 crates/kernel/src/panic.rs delete mode 100644 crates/kernel/src/start.rs diff --git a/crates/kernel/src/arch/backtrace.rs b/crates/kernel/src/arch/backtrace.rs new file mode 100644 index 00000000..2bea6533 --- /dev/null +++ b/crates/kernel/src/arch/backtrace.rs @@ -0,0 +1,103 @@ +use core::arch::asm; +use core::ops; +use gimli::{Register, RiscV}; + +// The LLVM source (https://llvm.org/doxygen/RISCVFrameLowering_8cpp_source.html) +// specify that only ra (x1) and saved registers (x8-x9, x18-x27) are used for +// frame unwinding info, plus sp (x2) for the CFA, so we only need to save those. +// If this causes issues down the line it should be trivial to change this to capture the full context. +#[derive(Debug, Clone)] +pub struct Context { + pub ra: usize, + pub sp: usize, + pub s: [usize; 12], +} + +impl Context { + // Load bearing inline don't remove + // TODO figure out why this is and remove + #[inline(always)] + pub fn capture() -> Self { + let (ra, sp, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11); + unsafe { + asm!( + "mv {}, ra", + "mv {}, sp", + "mv {}, s0", + "mv {}, s1", + "mv {}, s2", + "mv {}, s3", + "mv {}, s4", + "mv {}, s5", + "mv {}, s6", + "mv {}, s7", + "mv {}, s8", + "mv {}, s9", + "mv {}, s10", + "mv {}, s11", + out(reg) ra, + out(reg) sp, + out(reg) s0, + out(reg) s1, + out(reg) s2, + out(reg) s3, + out(reg) s4, + out(reg) s5, + out(reg) s6, + out(reg) s7, + out(reg) s8, + out(reg) s9, + out(reg) s10, + out(reg) s11, + ) + } + + Self { + ra, + sp, + s: [s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11], + } + } + + pub fn return_address(&self) -> usize { + self.ra + } + + pub fn set_return_address(&mut self, ra: usize) { + self.ra = ra; + } + + pub fn stack_pointer(&self) -> usize { + self.sp + } + + pub fn set_stack_pointer(&mut self, sp: usize) { + self.sp = sp; + } +} + +impl ops::Index for Context { + type Output = usize; + + fn index(&self, index: Register) -> &Self::Output { + match index { + RiscV::RA => &self.ra, + RiscV::SP => &self.sp, + Register(reg @ 8..=9) => &self.s[reg as usize - 8], + Register(reg @ 18..=27) => &self.s[reg as usize - 16], + reg => panic!("unsupported register {reg:?}"), + } + } +} + +impl ops::IndexMut for Context { + fn index_mut(&mut self, index: Register) -> &mut Self::Output { + match index { + RiscV::RA => &mut self.ra, + RiscV::SP => &mut self.sp, + Register(reg @ 8..=9) => &mut self.s[reg as usize - 8], + Register(reg @ 18..=27) => &mut self.s[reg as usize - 16], + reg => panic!("unsupported register {reg:?}"), + } + } +} diff --git a/crates/kernel/src/interrupt.rs b/crates/kernel/src/arch/interrupt.rs similarity index 100% rename from crates/kernel/src/interrupt.rs rename to crates/kernel/src/arch/interrupt.rs diff --git a/crates/kernel/src/arch/mod.rs b/crates/kernel/src/arch/mod.rs new file mode 100644 index 00000000..a4fe7619 --- /dev/null +++ b/crates/kernel/src/arch/mod.rs @@ -0,0 +1,19 @@ +use core::arch::asm; + +pub mod backtrace; +pub mod interrupt; +pub mod paging; +mod start; +pub mod trap; + +pub const PAGE_SIZE: usize = 4096; + +pub const STACK_SIZE_PAGES: usize = 25; + +pub fn halt() -> ! { + unsafe { + loop { + asm!("wfi"); + } + } +} diff --git a/crates/kernel/src/paging/entry.rs b/crates/kernel/src/arch/paging/entry.rs similarity index 92% rename from crates/kernel/src/paging/entry.rs rename to crates/kernel/src/arch/paging/entry.rs index 0a94b070..ee054053 100644 --- a/crates/kernel/src/paging/entry.rs +++ b/crates/kernel/src/arch/paging/entry.rs @@ -1,4 +1,4 @@ -use super::PhysicalAddress; +use crate::paging::PhysicalAddress; use bitflags::bitflags; use core::fmt; use core::fmt::Formatter; @@ -31,11 +31,11 @@ impl Entry { } pub fn set_address(&mut self, adress: PhysicalAddress) { - self.0 |= adress.0 >> 2; + self.0 |= adress.as_raw() >> 2; } pub fn address(&self) -> PhysicalAddress { - PhysicalAddress((self.0 & !0x3ff) << 2) + unsafe { PhysicalAddress::new((self.0 & !0x3ff) << 2) } } } diff --git a/crates/kernel/src/paging/flush.rs b/crates/kernel/src/arch/paging/flush.rs similarity index 66% rename from crates/kernel/src/paging/flush.rs rename to crates/kernel/src/arch/paging/flush.rs index 7a78165c..debbd2c9 100644 --- a/crates/kernel/src/paging/flush.rs +++ b/crates/kernel/src/arch/paging/flush.rs @@ -1,4 +1,4 @@ -use super::VirtualAddress; +use crate::paging::VirtualAddress; use core::mem; pub struct Flush { @@ -6,14 +6,14 @@ pub struct Flush { } impl Flush { - pub(super) fn new(virt: VirtualAddress) -> Self { + pub(crate) fn new(virt: VirtualAddress) -> Self { Self { virt } } pub fn flush(self) { // TODO check if this is necessary & make SBI call instead unsafe { - riscv::asm::sfence_vma(0, self.virt.0); + riscv::asm::sfence_vma(0, self.virt.as_raw()); } } pub unsafe fn ignore(self) { diff --git a/crates/kernel/src/arch/paging/mapper.rs b/crates/kernel/src/arch/paging/mapper.rs new file mode 100644 index 00000000..49addbe5 --- /dev/null +++ b/crates/kernel/src/arch/paging/mapper.rs @@ -0,0 +1,143 @@ +use super::entry::PageFlags; +use super::flush::Flush; +use super::table::Table; +use crate::arch::paging::MAX_LEVEL; +use crate::arch::PAGE_SIZE; +use crate::paging::frame_alloc::FrameAllocator; +use crate::paging::{PhysicalAddress, VirtualAddress}; +use crate::Error; +use riscv::register::satp; +use riscv::register::satp::Mode; + +pub struct Mapper { + root_table: PhysicalAddress, + allocator: FrameAllocator, +} + +impl Mapper { + pub fn new(mut allocator: FrameAllocator) -> crate::Result { + let root_table = allocator.allocate_frame()?; + + Ok(Self { + root_table, + allocator, + }) + } + + pub fn from_active(allocator: FrameAllocator) -> Self { + let root_table = unsafe { PhysicalAddress::new(satp::read().ppn() << 12) }; + + Self { + root_table, + allocator, + } + } + + pub fn activate(&self) -> crate::Result<()> { + unsafe { + // we have to access these addresses as the table is not mapped + // so after activating the page table, we can't access it anymore + // TODO: this is a bit of a hack, we should probably map the table + let start = self.root_table().lowest_mapped_address().unwrap(); + let end = self.root_table().highest_mapped_address().unwrap(); + + let ppn = self.root_table().address().as_raw() >> 12; + satp::set(Mode::Sv39, 0, ppn); + + // the most brutal approach to this, probably not necessary + // this will take a hammer to the page tables and synchronize *everything* + sbicall::rfence::sfence_vma(0, -1isize as usize, start.as_raw(), end.as_raw())?; + } + + Ok(()) + } + + pub fn allocator(&self) -> &FrameAllocator { + &self.allocator + } + + pub fn root_table(&self) -> Table { + Table::from_address(self.root_table, MAX_LEVEL) + } + + pub fn map_identity( + &mut self, + phys: PhysicalAddress, + flags: PageFlags, + ) -> crate::Result { + let virt = unsafe { VirtualAddress::new(phys.as_raw()) }; + self.map(virt, phys, flags, 0) + } + + pub fn map( + &mut self, + virt: VirtualAddress, + phys: PhysicalAddress, + flags: PageFlags, + level: usize, + ) -> crate::Result { + assert_eq!( + phys.as_raw() % PAGE_SIZE, + 0, + "can only map to aligned physical addresses {:#x?}", + phys.as_raw() + ); + + // Make sure that Read, Write, or Execute have been provided + // otherwise, we'll leak memory and always create a page fault. + assert!(flags.intersects(PageFlags::READ | PageFlags::WRITE | PageFlags::EXECUTE)); + + let mut table = self.root_table(); + + for i in (level..=MAX_LEVEL).rev() { + let entry = &mut table[virt]; + + if i == level { + entry.set_flags(flags | PageFlags::VALID); + entry.set_address(phys); + return Ok(Flush::new(virt)); + } else { + if !entry.flags().contains(PageFlags::VALID) { + let frame = self.allocator.allocate_frame()?; + entry.set_flags(PageFlags::VALID); + entry.set_address(frame); + } + + table = Table::from_address(entry.address(), i - 1); + } + } + + Err(Error::VirtualAddressTooLarge(virt)) + } + + pub fn virt_to_phys(&self, virt: VirtualAddress) -> crate::Result { + let mut table = self.root_table(); + + for i in (0..=MAX_LEVEL).rev() { + let entry = &table[virt]; + + if entry + .flags() + .intersects(PageFlags::EXECUTE | PageFlags::READ) + { + let addr = entry.address(); + let off_mask = (1 << 12) - 1; + let pgoff = virt.as_raw() & off_mask; + + unsafe { + return Ok(PhysicalAddress::new(addr.as_raw() & !off_mask | pgoff)); + } + } else { + // PTE is pointer to next level page table + assert!( + entry.flags().contains(PageFlags::VALID), + "invalid page table entry {entry:?} for virt {:#x?}", + virt.as_raw() + ); + table = Table::from_address(entry.address(), i - 1); + } + } + + Err(Error::VirtualAddressNotMapped(virt)) + } +} diff --git a/crates/kernel/src/arch/paging/mod.rs b/crates/kernel/src/arch/paging/mod.rs new file mode 100644 index 00000000..b912b048 --- /dev/null +++ b/crates/kernel/src/arch/paging/mod.rs @@ -0,0 +1,113 @@ +use crate::arch; +use crate::arch::paging::entry::PageFlags; +use crate::arch::paging::mapper::Mapper; +use crate::board_info::BoardInfo; +use crate::paging::frame_alloc::FrameAllocator; +use crate::paging::PhysicalAddress; +use core::ops::Range; +use core::ptr::addr_of; +use riscv::register::satp; +use riscv::register::satp::Mode; + +mod entry; +mod flush; +mod mapper; +mod table; + +pub const MAX_LEVEL: usize = 2; // Sv39 + +extern "C" { + static __text_start: u8; + static __text_end: u8; + static __stack_start: u8; + static __rodata_start: u8; + static __rodata_end: u8; +} + +/// Initialize virtual memory management +/// +/// This will set up the page table, identity map the kernel and stack, and enable paging. +/// +/// TODO lift this out of the arch module +pub fn init(board_info: &BoardInfo) -> crate::Result<()> { + let stack_start = unsafe { addr_of!(__stack_start) as usize }; + let text_start = unsafe { addr_of!(__text_start) as usize }; + let text_end = unsafe { addr_of!(__text_end) as usize }; + let rodata_start = unsafe { addr_of!(__rodata_start) as usize }; + let rodata_end = unsafe { addr_of!(__rodata_end) as usize }; + + let stack_region = unsafe { + let start = PhysicalAddress::new(stack_start); + + start..start.add(arch::STACK_SIZE_PAGES * arch::PAGE_SIZE * board_info.cpus) + }; + let kernel_region = + unsafe { PhysicalAddress::new(text_start)..PhysicalAddress::new(stack_start) }; + let text_region = unsafe { PhysicalAddress::new(text_start)..PhysicalAddress::new(text_end) }; + let rodata_region = + unsafe { PhysicalAddress::new(rodata_start)..PhysicalAddress::new(rodata_end) }; + + // Step 1: collect memory regions + // these are all the addresses we can use for allocation + // which should get this info from the DTB ideally + let regions = [stack_region.end..board_info.memory.end]; + + // Step 2: initialize allocator + let allocator = unsafe { FrameAllocator::new(®ions) }; + + // Step 4: create mapper + let mut mapper = Mapper::new(allocator)?; + + // helper function to identity map a region + let mut identity_map_range = |region: Range| -> crate::Result<()> { + let len = region.end.as_raw() - region.start.as_raw(); + + for i in 0..len / arch::PAGE_SIZE { + let phys = region.start.add(i * arch::PAGE_SIZE); + + let flags = if text_region.contains(&phys) { + PageFlags::READ | PageFlags::EXECUTE + } else if rodata_region.contains(&phys) { + PageFlags::READ + } else { + PageFlags::READ | PageFlags::WRITE + }; + + let flush = mapper.map_identity(phys, flags)?; + + unsafe { + flush.ignore(); // no flushing during init + } + } + + Ok(()) + }; + + // Step 4: map kernel + log::debug!("mapping kernel region: {:?}", kernel_region); + identity_map_range(kernel_region.clone())?; + + // Step 5: map stack + log::debug!("mapping stack region: {:?}", stack_region); + identity_map_range(stack_region.clone())?; + + // Step 6: map MMIO regions (UART) + log::debug!("mapping mmio region: {:?}", board_info.serial.mmio_regs); + identity_map_range(align_range(board_info.serial.mmio_regs.clone()))?; + + mapper.root_table().print_table(); + + // Step 7: enable paging + log::debug!("enabling paging... {:?}", mapper.root_table().address()); + mapper.activate()?; + log::debug!("paging enabled"); + + Ok(()) +} + +fn align_range(range: Range) -> Range { + let start = range.start.as_raw() & !(arch::PAGE_SIZE - 1); + let end = (range.end.as_raw() + arch::PAGE_SIZE - 1) & !(arch::PAGE_SIZE - 1); + + unsafe { PhysicalAddress::new(start)..PhysicalAddress::new(end) } +} diff --git a/crates/kernel/src/arch/paging/table.rs b/crates/kernel/src/arch/paging/table.rs new file mode 100644 index 00000000..9ee3b286 --- /dev/null +++ b/crates/kernel/src/arch/paging/table.rs @@ -0,0 +1,125 @@ +use super::entry::{Entry, PageFlags}; +use super::PhysicalAddress; +use crate::paging::VirtualAddress; +use core::{mem, ops}; + +pub struct Table { + /// the physical address of this table + /// [Entry; 512] + phys: PhysicalAddress, + level: usize, +} + +impl Table { + pub fn from_address(phys: PhysicalAddress, level: usize) -> Self { + Self { phys, level } + } + + pub fn address(&self) -> PhysicalAddress { + self.phys + } + + pub fn lowest_mapped_address(&self) -> Option { + self.lowest_mapped_address_inner(unsafe { VirtualAddress::new(0) }) + } + + pub fn lowest_mapped_address_inner(&self, acc: VirtualAddress) -> Option { + for i in 0..512 { + let entry = &self[i]; + let virt = + unsafe { VirtualAddress::new(acc.as_raw() | (i & 0x1ff) << (self.level * 9 + 12)) }; + + if entry + .flags() + .intersects(PageFlags::READ | PageFlags::EXECUTE) + { + return Some(virt); + } else if entry.flags().contains(PageFlags::VALID) { + return Table::from_address(entry.address(), self.level - 1) + .lowest_mapped_address_inner(virt); + } + } + + None + } + + pub fn highest_mapped_address(&self) -> Option { + self.highest_mapped_address_inner(unsafe { VirtualAddress::new(0) }) + } + + fn highest_mapped_address_inner(&self, acc: VirtualAddress) -> Option { + for i in (0..512).rev() { + let entry = &self[i]; + let virt = + unsafe { VirtualAddress::new(acc.as_raw() | (i & 0x1ff) << (self.level * 9 + 12)) }; + + if entry + .flags() + .intersects(PageFlags::READ | PageFlags::EXECUTE) + { + return Some(virt); + } else if entry.flags().contains(PageFlags::VALID) { + return Table::from_address(entry.address(), self.level - 1) + .highest_mapped_address_inner(virt); + } + } + + None + } + + fn index_of_virt(&self, virt: VirtualAddress) -> usize { + (virt.as_raw() >> (self.level * 9 + 12)) & 0x1ff + } + + pub fn print_table(&self) { + let padding = match self.level { + 0 => 8, + 1 => 4, + _ => 0, + }; + + for i in 0..512 { + let entry = &self[i]; + + if entry + .flags() + .intersects(PageFlags::READ | PageFlags::EXECUTE) + { + log::debug!("{:^padding$}{}:{i} is a leaf", "", self.level); + } else if entry.flags().contains(PageFlags::VALID) { + log::debug!("{:^padding$}{}:{i} is a table node", "", self.level); + Table::from_address(entry.address(), self.level - 1).print_table(); + } + } + } +} + +impl ops::Index for Table { + type Output = Entry; + + fn index(&self, index: usize) -> &Self::Output { + let ptr = self.phys.add(index * mem::size_of::()).as_raw() as *const Entry; + unsafe { &*(ptr) } + } +} + +impl ops::IndexMut for Table { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + let ptr = self.phys.add(index * mem::size_of::()).as_raw() as *mut Entry; + unsafe { &mut *(ptr) } + } +} + +impl ops::Index for Table { + type Output = Entry; + + fn index(&self, index: VirtualAddress) -> &Self::Output { + self.index(self.index_of_virt(index)) + } +} + +impl ops::IndexMut for Table { + fn index_mut(&mut self, index: VirtualAddress) -> &mut Self::Output { + self.index_mut(self.index_of_virt(index)) + } +} diff --git a/crates/kernel/src/arch/start.rs b/crates/kernel/src/arch/start.rs new file mode 100644 index 00000000..6b906528 --- /dev/null +++ b/crates/kernel/src/arch/start.rs @@ -0,0 +1,128 @@ +use crate::arch; +use crate::board_info::BoardInfo; +use core::arch::asm; +use core::ptr::addr_of_mut; + +/// Sets the harts stack pointer to the top of the stack. +/// +/// Since all stacks are laid out sequentially in memory, starting at the `__stack_start` symbol, +/// we can calculate the stack pointer for each hart by adding the stack size multiplied by the +/// hart ID to the `__stack_start` symbol. +/// +/// Therefore the hart ID essentially acts as an index into the stack area. +/// +/// # Safety +/// +/// The caller must ensure the hart ID is passed in `a0`. +#[naked] +unsafe extern "C" fn set_stack_pointer() { + asm!( + "la sp, __stack_start", // set the stack pointer to the bottom of the stack + "li t0, {stack_size}", // load the stack size + "addi t1, a0, 1", // add one to the hart id so that we add at least one stack size (stack grows from the top downwards) + "mul t0, t0, t1", // multiply the stack size by the hart id to get the offset + "add sp, sp, t0", // add the offset from sp to get the harts stack pointer + "ret", + + stack_size = const arch::STACK_SIZE_PAGES * arch::PAGE_SIZE, + options(noreturn) + ) +} + +// #[naked] +// unsafe extern "C" fn allocate_trap_frame() { +// asm!( +// "addi sp, sp, -{trap_frame_size}", +// "csrrw x0, sscratch, sp", // sscratch points to the trap frame +// "ret", +// trap_frame_size = const mem::size_of::(), +// options(noreturn) +// ) +// } + +/// This is the boot harts entry point into the kernel. +/// It is the first function that is called after OpenSBI has set up the environment. +/// +/// Because we want to jump into Rust as soon as possible, we only set up the stack +/// pointer and move on. +#[link_section = ".text.start"] +#[no_mangle] +#[naked] +unsafe extern "C" fn _start() -> ! { + asm!( + ".option push", + ".option norelax", + " la gp, __global_pointer$", + ".option pop", + "call {set_stack_pointer}", + // "call {allocate_trap_frame}", + + "jal zero, {start_rust}", // jump into Rust + + set_stack_pointer = sym set_stack_pointer, + // allocate_trap_frame = sym allocate_trap_frame, + start_rust = sym start, + options(noreturn) + ) +} + +/// This is the entry point for all other harts that aren't the boot hart. +/// This function is called after initializing the kernel by the boot hart through HSM `start_hart`. +/// +/// As with the boot hart, we only set up the stack pointer and jump into Rust. +/// But since all global state has already been initialized by the boot hart, and hart-local +/// state will be set up in `kmain` there is no `start_hart` function, we directly move on to `kmain`. +#[no_mangle] +#[naked] +unsafe extern "C" fn _start_hart() -> ! { + asm!( + ".option push", + ".option norelax", + " la gp, __global_pointer$", + ".option pop", + "call {set_stack_pointer}", + // "call {allocate_trap_frame}", + + "jal zero, {start_rust}", // jump into Rust + + set_stack_pointer = sym set_stack_pointer, + // allocate_trap_frame = sym allocate_trap_frame, + start_rust = sym crate::kmain, + options(noreturn) + ) +} + +/// This is the init function of the kernel. +/// +/// This function will take care of initializing all global state (not per-hart state) +/// such as parsing `BoardInfo`, initializing the logger, and setting up the kernel heap. +/// +/// It will then start all other harts and jump into `kmain`. +extern "C" fn start(hartid: usize, opaque: *const u8) -> ! { + extern "C" { + static mut __bss_start: u64; + static mut __bss_end: u64; + } + unsafe { + let mut ptr = addr_of_mut!(__bss_start); + let end = addr_of_mut!(__bss_end); + while ptr < end { + ptr.write_volatile(0); + ptr = ptr.offset(1); + } + } + + let board_info = BoardInfo::from_raw(opaque).unwrap(); + + crate::logger::init(&board_info, 38400).unwrap(); + + arch::paging::init(&board_info).unwrap(); + + for hart in 0..board_info.cpus { + if hart != hartid { + sbicall::hsm::start_hart(hart, _start_hart as usize, 0).unwrap(); + } + } + + crate::kmain(hartid) +} diff --git a/crates/kernel/src/trap.rs b/crates/kernel/src/arch/trap.rs similarity index 87% rename from crates/kernel/src/trap.rs rename to crates/kernel/src/arch/trap.rs index f9115bea..711a4542 100644 --- a/crates/kernel/src/trap.rs +++ b/crates/kernel/src/arch/trap.rs @@ -3,6 +3,16 @@ use core::marker::PhantomPinned; use riscv::register::stvec::TrapMode; use riscv::register::{scause, sie, sstatus, stvec}; +pub fn init() -> crate::Result<()> { + unsafe { + stvec::write(trap_vec as _, TrapMode::Vectored); + sie::set_stimer(); + sstatus::set_sie(); + } + + Ok(()) +} + /// This struct keeps the harts state during a trap so we can restore it later. /// /// Currently we only save the `t` and `a` registers as well as the `ra` register. @@ -17,14 +27,6 @@ pub struct TrapFrame { pub _pinned: PhantomPinned, } -pub fn init() { - unsafe { - stvec::write(trap_vec as _, TrapMode::Vectored); - sie::set_stimer(); - sstatus::set_sie(); - } -} - #[naked] pub unsafe extern "C" fn trap_vec() { // When in vectored mode @@ -34,22 +36,22 @@ pub unsafe extern "C" fn trap_vec() { // We can use this to direct some traps that don't need // expensive SBI call handling to cheaper handlers (like timers) asm! ( - ".align 2", - ".option push", - ".option norvc", - "j {default}", // exception - "j {default}", // supervisor software interrupt - "j {default}", // reserved - "j {default}", // reserved - "j {default}", // reserved - "j {default}", // supervisor timer interrupt - "j {default}", // reserved - "j {default}", // reserved - "j {default}", // reserved - "j {default}", // supervisor external interrupt - ".option pop", - default = sym default_trap_entry, - options(noreturn) + ".align 2", + ".option push", + ".option norvc", + "j {default}", // exception + "j {default}", // supervisor software interrupt + "j {default}", // reserved + "j {default}", // reserved + "j {default}", // reserved + "j {default}", // supervisor timer interrupt + "j {default}", // reserved + "j {default}", // reserved + "j {default}", // reserved + "j {default}", // supervisor external interrupt + ".option pop", + default = sym default_trap_entry, + options(noreturn) ) } diff --git a/crates/kernel/src/backtrace.rs b/crates/kernel/src/backtrace.rs index 760d2a69..66971f73 100644 --- a/crates/kernel/src/backtrace.rs +++ b/crates/kernel/src/backtrace.rs @@ -1,16 +1,16 @@ -use core::arch::asm; +use crate::arch::backtrace as arch; use core::ptr::addr_of; -use core::{ops, slice}; +use core::slice; use gimli::{ BaseAddresses, CfaRule, EhFrame, EndianSlice, FrameDescriptionEntry, NativeEndian, Register, - RegisterRule, RiscV, UnwindContext, UnwindSection, UnwindTableRow, + RegisterRule, UnwindContext, UnwindSection, UnwindTableRow, }; // Load bearing inline don't remove // TODO figure out why this is and remove #[inline(always)] pub fn trace(mut cb: F) { - let mut ctx = Context::capture(); + let mut ctx = arch::Context::capture(); extern "C" { static __eh_frame_start: u8; @@ -46,90 +46,6 @@ pub fn trace(mut cb: F) { } } -// The LLVM source (https://llvm.org/doxygen/RISCVFrameLowering_8cpp_source.html) -// specify that only ra (x1) and saved registers (x8-x9, x18-x27) are used for -// frame unwinding info, plus sp (x2) for the CFA, so we only need to save those. -// If this causes issues down the line it should be trivial to change this to capture the full context. -#[derive(Debug, Clone)] -pub struct Context { - pub ra: usize, - pub sp: usize, - pub s: [usize; 12], -} - -impl Context { - // Load bearing inline don't remove - // TODO figure out why this is and remove - #[inline(always)] - pub fn capture() -> Self { - let (ra, sp, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11); - unsafe { - asm!( - "mv {}, ra", - "mv {}, sp", - "mv {}, s0", - "mv {}, s1", - "mv {}, s2", - "mv {}, s3", - "mv {}, s4", - "mv {}, s5", - "mv {}, s6", - "mv {}, s7", - "mv {}, s8", - "mv {}, s9", - "mv {}, s10", - "mv {}, s11", - out(reg) ra, - out(reg) sp, - out(reg) s0, - out(reg) s1, - out(reg) s2, - out(reg) s3, - out(reg) s4, - out(reg) s5, - out(reg) s6, - out(reg) s7, - out(reg) s8, - out(reg) s9, - out(reg) s10, - out(reg) s11, - ) - } - - Self { - ra, - sp, - s: [s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11], - } - } -} - -impl ops::Index for Context { - type Output = usize; - - fn index(&self, index: Register) -> &Self::Output { - match index { - RiscV::RA => &self.ra, - RiscV::SP => &self.sp, - Register(reg @ 8..=9) => &self.s[reg as usize - 8], - Register(reg @ 18..=27) => &self.s[reg as usize - 16], - reg => panic!("unsupported register {reg:?}"), - } - } -} - -impl ops::IndexMut for Context { - fn index_mut(&mut self, index: Register) -> &mut Self::Output { - match index { - RiscV::RA => &mut self.ra, - RiscV::SP => &mut self.sp, - Register(reg @ 8..=9) => &mut self.s[reg as usize - 8], - Register(reg @ 18..=27) => &mut self.s[reg as usize - 16], - reg => panic!("unsupported register {reg:?}"), - } - } -} - pub struct Frame<'a> { // the program counter pc: usize, @@ -141,12 +57,12 @@ pub struct Frame<'a> { impl<'a> Frame<'a> { fn from_context( - ctx: &Context, + ctx: &arch::Context, eh_frame: &'a EhFrame>, bases: &BaseAddresses, unwinder: &'a mut UnwindContext, StoreOnStack>, ) -> Option { - let ra = ctx[RiscV::RA]; + let ra = ctx.return_address(); if ra == 0 { return None; @@ -162,13 +78,13 @@ impl<'a> Frame<'a> { Some(Frame { pc: ra, - sp: ctx[RiscV::SP], + sp: ctx.stack_pointer(), fde, row, }) } - pub fn unwind(&self, ctx: &Context) -> Context { + pub fn unwind(&self, ctx: &arch::Context) -> arch::Context { let row = &self.row; let mut new_ctx = ctx.clone(); @@ -179,8 +95,8 @@ impl<'a> Frame<'a> { CfaRule::Expression(_) => panic!("DWARF expressions are unsupported"), }; - new_ctx[RiscV::SP] = cfa as _; - new_ctx[RiscV::RA] = 0; + new_ctx.set_stack_pointer(cfa); + new_ctx.set_return_address(0); for (reg, rule) in row.registers() { let value = match *rule { diff --git a/crates/kernel/src/error.rs b/crates/kernel/src/error.rs index 004484ef..8a441f5f 100644 --- a/crates/kernel/src/error.rs +++ b/crates/kernel/src/error.rs @@ -1,3 +1,5 @@ +use crate::paging::VirtualAddress; + #[derive(Debug, onlyerror::Error)] pub enum Error { #[error("failed to parse Device Tree Blob")] @@ -6,4 +8,10 @@ pub enum Error { MissingBordInfo(&'static str), #[error("SBI call failed: {0}")] SBI(#[from] sbicall::Error), + #[error("virtual address {0:?} is too large to be mapped")] + VirtualAddressTooLarge(VirtualAddress), + #[error("virtual address {0:?} is not mapped")] + VirtualAddressNotMapped(VirtualAddress), + #[error("out of memory")] + OutOfMemory, } diff --git a/crates/kernel/src/logger.rs b/crates/kernel/src/logger.rs index c1d8c4be..2d6136a4 100644 --- a/crates/kernel/src/logger.rs +++ b/crates/kernel/src/logger.rs @@ -1,4 +1,5 @@ -use crate::board_info::Serial; +use crate::arch; +use crate::board_info::BoardInfo; use crate::sync::Mutex; use log::{Metadata, Record}; use uart_16550::SerialPort; @@ -7,11 +8,11 @@ static LOGGER: Logger = Logger::empty(); struct Logger(Mutex>); -pub fn init(serial_info: &Serial, baud_rate: u32) { +pub fn init(board_info: &BoardInfo, baud_rate: u32) -> crate::Result<()> { let uart = unsafe { SerialPort::new( - serial_info.mmio_regs.start.as_raw(), - serial_info.clock_frequency, + board_info.serial.mmio_regs.start.as_raw(), + board_info.serial.clock_frequency, baud_rate, ) }; @@ -19,6 +20,8 @@ pub fn init(serial_info: &Serial, baud_rate: u32) { log::set_logger(&LOGGER).unwrap(); log::set_max_level(log::LevelFilter::Trace); + + Ok(()) } impl Logger { @@ -39,7 +42,7 @@ impl log::Log for Logger { // disable interrupts while we hold the uart lock // otherwise we might deadlock if we try to log from the trap handler // TODO maybe replace this with a reentrant mutex - crate::interrupt::without(|| { + arch::interrupt::without(|| { let mut uart = self.0.lock(); // don't panic if we accidentally log before the logger is initialized // logs are not that important anyway diff --git a/crates/kernel/src/main.rs b/crates/kernel/src/main.rs index a8638019..1238b69e 100644 --- a/crates/kernel/src/main.rs +++ b/crates/kernel/src/main.rs @@ -1,61 +1,30 @@ #![no_std] #![no_main] -#![feature(naked_functions, asm_const, error_in_core, allocator_api)] -#![feature(c_unwind)] +#![feature(naked_functions, asm_const, error_in_core)] +mod arch; mod backtrace; mod board_info; mod error; -mod interrupt; mod logger; mod paging; -mod start; +mod panic; mod sync; -mod trap; - -use core::arch::asm; -use core::sync::atomic::{AtomicBool, Ordering}; -use error::Error; +pub use error::Error; pub type Result = core::result::Result; /// This is the main function of the kernel. /// /// After performing arch & board specific initialization, all harts will end up in this function. /// This function should set up hart-local state, and then ?. It should never return. -fn kmain(hartid: usize) -> ! { - log::info!("Hello world from hart {hartid}!"); - - trap::init(); - - // sbi::time::set_timer(2_000_000).unwrap(); +pub fn kmain(hartid: usize) -> ! { + // arch-agnostic initialization + // per-hart initialization - loop { - unsafe { - asm!("wfi"); - } - } -} - -static PANICKING: AtomicBool = AtomicBool::new(false); - -#[panic_handler] -fn panic(info: &core::panic::PanicInfo) -> ! { - log::error!("KERNEL PANIC {}", info); + log::info!("Hello world from hart {hartid}!"); - // if we panic in the backtrace, prevent us from spinning into an infinite panic loop - if !PANICKING.swap(true, Ordering::AcqRel) { - log::error!("un-symbolized stack trace:"); - let mut count = 0; - backtrace::trace(|frame| { - count += 1; - log::debug!("{:<2}- {:#x?}", count, frame.symbol_address()); - }); - } + arch::trap::init().unwrap(); - loop { - unsafe { - asm!("wfi"); - } - } + todo!() } diff --git a/crates/kernel/src/paging/frame_alloc.rs b/crates/kernel/src/paging/frame_alloc.rs index 915021e4..17489ab9 100644 --- a/crates/kernel/src/paging/frame_alloc.rs +++ b/crates/kernel/src/paging/frame_alloc.rs @@ -1,4 +1,5 @@ -use super::{PhysicalAddress, PAGE_SIZE}; +use crate::arch::PAGE_SIZE; +use crate::paging::PhysicalAddress; use core::ops::Range; pub struct FrameAllocator { @@ -12,8 +13,11 @@ impl FrameAllocator { } } - pub fn allocate_frame(&mut self) -> Option { - self.free_frame_list.pop().map(|frame| frame.as_ptr()) + pub fn allocate_frame(&mut self) -> crate::Result { + self.free_frame_list + .pop() + .map(|frame| frame.as_ptr()) + .ok_or(crate::Error::OutOfMemory) } pub fn deallocate_frame(&mut self, phys: PhysicalAddress) { diff --git a/crates/kernel/src/paging/mapper.rs b/crates/kernel/src/paging/mapper.rs deleted file mode 100644 index 1b761971..00000000 --- a/crates/kernel/src/paging/mapper.rs +++ /dev/null @@ -1,94 +0,0 @@ -use super::entry::PageFlags; -use super::flush::Flush; -use super::frame_alloc::FrameAllocator; -use super::table::Table; -use super::{PhysicalAddress, VirtualAddress, MAX_LEVEL}; - -pub struct Mapper { - root_table: PhysicalAddress, - allocator: FrameAllocator, -} - -impl Mapper { - pub fn new(mut allocator: FrameAllocator) -> Self { - let root_table = allocator.allocate_frame().unwrap(); - - Self { - root_table, - allocator, - } - } - - pub fn allocator(&self) -> &FrameAllocator { - &self.allocator - } - - pub fn root_table(&self) -> Table { - Table::from_address(self.root_table) - } - - pub fn map( - &mut self, - virt: VirtualAddress, - phys: PhysicalAddress, - flags: PageFlags, - level: usize, - ) -> Option { - // Make sure that Read, Write, or Execute have been provided - // otherwise, we'll leak memory and always create a page fault. - assert!(flags.intersects(PageFlags::READ | PageFlags::WRITE | PageFlags::EXECUTE)); - - let mut table = self.root_table(); - - for i in (level..=MAX_LEVEL).rev() { - let entry = table.get_entry_mut(virt.vpn(i)); - - if i == level { - entry.set_flags(flags | PageFlags::VALID); - entry.set_address(phys); - return Some(Flush::new(virt)); - } else { - if !entry.flags().contains(PageFlags::VALID) { - let frame = self.allocator.allocate_frame().unwrap(); - entry.set_flags(PageFlags::VALID); - entry.set_address(frame); - } - - table = Table::from_address(entry.address()); - } - } - - None - } - - pub fn virt_to_phys(&self, virt: VirtualAddress) -> Result { - let mut table = self.root_table(); - - for i in (0..=MAX_LEVEL).rev() { - let entry = table.get_entry(virt.vpn(i)); - - if entry - .flags() - .intersects(PageFlags::EXECUTE | PageFlags::READ) - { - // PTE is a leaf - - let addr = entry.address(); - let off_mask = (1 << 12) - 1; - let pgoff = virt.0 & off_mask; - - return Ok(PhysicalAddress(addr.0 & !off_mask | pgoff)); - } else { - // PTE is pointer to next level page table - assert!( - entry.flags().contains(PageFlags::VALID), - "invalid page table entry {entry:?} for virt {:#x?}", - virt.0 - ); - table = Table::from_address(entry.address()); - } - } - - Err(virt) - } -} diff --git a/crates/kernel/src/paging/mod.rs b/crates/kernel/src/paging/mod.rs index 4354cf88..2ef0c31c 100644 --- a/crates/kernel/src/paging/mod.rs +++ b/crates/kernel/src/paging/mod.rs @@ -1,24 +1,7 @@ -use crate::board_info::BoardInfo; -use crate::paging::entry::PageFlags; -use crate::paging::frame_alloc::FrameAllocator; -use crate::paging::mapper::Mapper; -use crate::start::STACK_SIZE_PAGES; use core::fmt; use core::fmt::Formatter; -use core::ops::Range; -use core::ptr::addr_of; -use riscv::register::satp; -use riscv::register::satp::Mode; -// use spin::Once; -mod entry; -mod flush; -mod frame_alloc; -mod mapper; -mod table; - -pub const PAGE_SIZE: usize = 4096; -pub const MAX_LEVEL: usize = 2; // Sv39 +pub mod frame_alloc; /// A physical address. #[derive(Copy, Clone, PartialOrd, PartialEq)] @@ -49,8 +32,12 @@ impl fmt::Debug for PhysicalAddress { pub struct VirtualAddress(usize); impl VirtualAddress { - pub fn vpn(&self, level: usize) -> usize { - (self.0 >> (level * 9 + 12)) & 0x1ff + pub unsafe fn new(addr: usize) -> Self { + Self(addr) + } + + pub fn as_raw(&self) -> usize { + self.0 } } @@ -61,109 +48,3 @@ impl fmt::Debug for VirtualAddress { .finish() } } - -// static MAPPER: Once = Once::new(); - -extern "C" { - static __text_start: u8; - static __text_end: u8; - static __stack_start: u8; - static __rodata_start: u8; - static __rodata_end: u8; -} - -pub fn init(board_info: &BoardInfo) { - let stack_start = unsafe { addr_of!(__stack_start) as usize }; - let text_start = unsafe { addr_of!(__text_start) as usize }; - let text_end = unsafe { addr_of!(__text_end) as usize }; - let rodata_start = unsafe { addr_of!(__rodata_start) as usize }; - let rodata_end = unsafe { addr_of!(__rodata_end) as usize }; - - let stack_region = unsafe { - let start = PhysicalAddress::new(stack_start); - - start..start.add(STACK_SIZE_PAGES * PAGE_SIZE * board_info.cpus) - }; - let kernel_region = - unsafe { PhysicalAddress::new(text_start)..PhysicalAddress::new(stack_start) }; - let text_region = unsafe { PhysicalAddress::new(text_start)..PhysicalAddress::new(text_end) }; - let rodata_region = - unsafe { PhysicalAddress::new(rodata_start)..PhysicalAddress::new(rodata_end) }; - - // Step 1: collect memory regions - // these are all the addresses we can use for allocation - // which should get this info from the DTB ideally - let regions = [stack_region.end..board_info.memory.end]; - - // Step 2: initialize allocator - let allocator = unsafe { FrameAllocator::new(®ions) }; - - // Step 4: create mapper - let mut mapper = Mapper::new(allocator); - - let mut identity_map = |region: Range| { - let len = region.end.0 - region.start.0; - - for i in 0..len / PAGE_SIZE { - let phys = region.start.add(i * PAGE_SIZE); - let virt = VirtualAddress(phys.0); - - let flags = if text_region.contains(&phys) { - PageFlags::READ | PageFlags::EXECUTE - } else if rodata_region.contains(&phys) { - PageFlags::READ - } else { - PageFlags::READ | PageFlags::WRITE - }; - - let flush = mapper.map(virt, phys, flags, 0).unwrap(); - unsafe { - flush.ignore(); // no flushing during init - } - } - }; - - // Step 4: map kernel - log::debug!("mapping kernel region: {:?}", kernel_region); - identity_map(kernel_region.clone()); - // Step 5: map stack - log::debug!("mapping stack region: {:?}", stack_region); - identity_map(stack_region.clone()); - // Step 6: map MMIO regions (UART) - log::debug!("mapping mmio region: {:?}", board_info.serial.mmio_regs); - identity_map(align_range(board_info.serial.mmio_regs.clone())); - - mapper.root_table().print_table(2); - - let addr = PhysicalAddress(0x10000005); - let virt = VirtualAddress(addr.0); - let phys = mapper.virt_to_phys(virt).unwrap(); - assert_eq!(phys, addr); - - let addr = stack_region.start.add(400); - let virt = VirtualAddress(addr.0); - let phys = mapper.virt_to_phys(virt).unwrap(); - assert_eq!(phys, addr); - - // Step 7: enable paging - log::debug!("enabling paging... {:?}", mapper.root_table().address()); - unsafe { - let ppn = mapper.root_table().address().as_raw() >> 12; - satp::set(Mode::Sv39, 0, ppn); - // the most brutal approach to this, probably not necessary - // this will take a hammer to the page tables and synchronize *everything* - sbicall::rfence::sfence_vma(0, -1isize as usize, 0, board_info.memory.end.as_raw()) - .unwrap(); - } - log::debug!("paging enabled"); - - // Step 7: set global mapper - // MAPPER.call_once(|| mapper); -} - -fn align_range(range: Range) -> Range { - let start = range.start.0 & !(PAGE_SIZE - 1); - let end = (range.end.0 + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); - - PhysicalAddress(start)..PhysicalAddress(end) -} diff --git a/crates/kernel/src/paging/table.rs b/crates/kernel/src/paging/table.rs deleted file mode 100644 index 2e1f351e..00000000 --- a/crates/kernel/src/paging/table.rs +++ /dev/null @@ -1,51 +0,0 @@ -use super::entry::{Entry, PageFlags}; -use super::PhysicalAddress; -use core::mem; - -pub struct Table { - /// the physical address of this table - /// [Entry; 512] - phys: PhysicalAddress, -} - -impl Table { - pub fn from_address(phys: PhysicalAddress) -> Self { - Self { phys } - } - - pub fn address(&self) -> PhysicalAddress { - self.phys - } - - pub fn get_entry(&self, vpn: usize) -> &Entry { - let ptr = self.phys.add(vpn * mem::size_of::()).0 as *const Entry; - unsafe { &*(ptr) } - } - - pub fn get_entry_mut(&mut self, vpn: usize) -> &mut Entry { - let ptr = self.phys.add(vpn * mem::size_of::()).0 as *mut Entry; - unsafe { &mut *(ptr) } - } - - pub fn print_table(&self, level: usize) { - let padding = match level { - 0 => 8, - 1 => 4, - _ => 0, - }; - - for i in 0..512 { - let entry = self.get_entry(i); - - if entry - .flags() - .intersects(PageFlags::READ | PageFlags::EXECUTE) - { - log::debug!("{:^padding$}{level}:{i} is a leaf", ""); - } else if entry.flags().contains(PageFlags::VALID) { - log::debug!("{:^padding$}{level}:{i} is a table node", "",); - Table::from_address(entry.address()).print_table(level - 1); - } - } - } -} diff --git a/crates/kernel/src/panic.rs b/crates/kernel/src/panic.rs new file mode 100644 index 00000000..8f32045b --- /dev/null +++ b/crates/kernel/src/panic.rs @@ -0,0 +1,22 @@ +use crate::{arch, backtrace}; +use core::panic::PanicInfo; +use core::sync::atomic::{AtomicBool, Ordering}; + +static PANICKING: AtomicBool = AtomicBool::new(false); + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + log::error!("KERNEL PANIC {}", info); + + // if we panic in the backtrace, prevent us from spinning into an infinite panic loop + if !PANICKING.swap(true, Ordering::AcqRel) { + log::error!("un-symbolized stack trace:"); + let mut count = 0; + backtrace::trace(|frame| { + count += 1; + log::debug!("{:<2}- {:#x?}", count, frame.symbol_address()); + }); + } + + arch::halt(); +} diff --git a/crates/kernel/src/start.rs b/crates/kernel/src/start.rs deleted file mode 100644 index 09cf82e5..00000000 --- a/crates/kernel/src/start.rs +++ /dev/null @@ -1,172 +0,0 @@ -use crate::board_info::BoardInfo; -use crate::logger; -use crate::paging::PAGE_SIZE; -use crate::trap::TrapFrame; -use core::arch::asm; -use core::mem; -use core::ptr::{addr_of, addr_of_mut}; - -pub const STACK_SIZE_PAGES: usize = 25; - -/// Sets the harts stack pointer to the top of the stack. -/// -/// Since all stacks are laid out sequentially in memory, starting at the `__stack_start` symbol, -/// we can calculate the stack pointer for each hart by adding the stack size multiplied by the -/// hart ID to the `__stack_start` symbol. -/// -/// Therefore the hart ID essentially acts as an index into the stack area. -/// -/// # Safety -/// -/// The caller must ensure the hart ID is passed in `a0`. -#[naked] -unsafe extern "C" fn set_stack_pointer() { - asm!( - "la sp, __stack_start", // set the stack pointer to the bottom of the stack - "li t0, {stack_size}", // load the stack size - "addi t1, a0, 1", // add one to the hart id so that we add at least one stack size (stack grows from the top downwards) - "mul t0, t0, t1", // multiply the stack size by the hart id to get the offset - "add sp, sp, t0", // add the offset from sp to get the harts stack pointer - "ret", - - stack_size = const STACK_SIZE_PAGES * PAGE_SIZE, - options(noreturn) - ) -} - -#[naked] -unsafe extern "C" fn allocate_trap_frame() { - asm!( - "addi sp, sp, -{trap_frame_size}", - "csrrw x0, sscratch, sp", // sscratch points to the trap frame - "ret", - trap_frame_size = const mem::size_of::(), - options(noreturn) - ) -} - -/// This is the boot harts entry point into the kernel. -/// It is the first function that is called after OpenSBI has set up the environment. -/// -/// Because we want to jump into Rust as soon as possible, we only set up the stack -/// pointer and move on. -#[link_section = ".text.start"] -#[no_mangle] -#[naked] -unsafe extern "C" fn _start() -> ! { - asm!( - ".option push", - ".option norelax", - " la gp, __global_pointer$", - ".option pop", - "call {set_stack_pointer}", - "call {allocate_trap_frame}", - - "jal zero, {start_rust}", // jump into Rust - - set_stack_pointer = sym set_stack_pointer, - allocate_trap_frame = sym allocate_trap_frame, - start_rust = sym start, - options(noreturn) - ) -} - -/// This is the entry point for all other harts that aren't the boot hart. -/// This function is called after initializing the kernel by the boot hart through HSM `start_hart`. -/// -/// As with the boot hart, we only set up the stack pointer and jump into Rust. -/// But since all global state has already been initialized by the boot hart, and hart-local -/// state will be set up in `kmain` there is no `start_hart` function, we directly move on to `kmain`. -#[no_mangle] -#[naked] -unsafe extern "C" fn _start_hart() -> ! { - asm!( - ".option push", - ".option norelax", - " la gp, __global_pointer$", - ".option pop", - "call {set_stack_pointer}", - "call {allocate_trap_frame}", - - "jal zero, {start_rust}", // jump into Rust - - set_stack_pointer = sym set_stack_pointer, - allocate_trap_frame = sym allocate_trap_frame, - start_rust = sym crate::kmain, - options(noreturn) - ) -} - -/// This is the init function of the kernel. -/// -/// This function will take care of initializing all global state (not per-hart state) -/// such as parsing `BoardInfo`, initializing the logger, and setting up the kernel heap. -/// -/// It will then start all other harts and jump into `kmain`. -extern "C" fn start(hartid: usize, opaque: *const u8) -> ! { - extern "C" { - static mut __bss_start: u64; - static mut __bss_end: u64; - } - unsafe { - let mut ptr = addr_of_mut!(__bss_start); - let end = addr_of_mut!(__bss_end); - while ptr < end { - ptr.write_volatile(0); - ptr = ptr.offset(1); - } - } - - let board_info = BoardInfo::from_raw(opaque).unwrap(); - - logger::init(&board_info.serial, 38400); - - print_debug_info(&board_info); - - unsafe { - let dtb = dtb_parser::Dtb::from_raw(opaque).unwrap(); - let mut uart = uart_16550::SerialPort::new( - board_info.serial.mmio_regs.start.as_raw(), - board_info.serial.clock_frequency, - 38400, - ); - dtb.walk(&mut dtb_parser::DebugVisitor::new(&mut uart)) - .unwrap(); - } - - crate::paging::init(&board_info); - - for hart in 0..board_info.cpus { - if hart != hartid { - sbicall::hsm::start_hart(hart, _start_hart as usize, 0).unwrap(); - } - } - - crate::kmain(hartid) -} - -fn print_debug_info(board_info: &BoardInfo) { - extern "C" { - static __text_start: u8; - static __text_end: u8; - static __stack_start: u8; - } - - let text_start = unsafe { addr_of!(__text_start) }; - let text_end = unsafe { addr_of!(__text_end) }; - - log::debug!("text area {:?}", text_start..text_end); - - let stack_start = unsafe { addr_of!(__stack_start) }; - - log::debug!( - "stack area {:?}", - stack_start..unsafe { stack_start.add(STACK_SIZE_PAGES * PAGE_SIZE * board_info.cpus) } - ); - - for cpu in 0..board_info.cpus { - let start = unsafe { stack_start.add(STACK_SIZE_PAGES * PAGE_SIZE * cpu) }; - let end = unsafe { stack_start.add(STACK_SIZE_PAGES * PAGE_SIZE * (cpu + 1)) }; - log::debug!("stack for hart {cpu}: {:?}", start..end); - } -}