Merge pull request #1243 from stlankes/mwait

add option to use a busy waiting within the idle loop

Cargo.toml

@@ -69,6 +69,7 @@ common-os = []
 nostd = []
 semihosting = ["dep:semihosting"]
 shell = ["simple-shell"]
+idle-poll = []
 
 [dependencies]
 hermit-macro = { path = "hermit-macro" }

src/arch/x86_64/kernel/apic.rs

@@ -837,8 +837,11 @@ pub fn ipi_tlb_flush() {
 /// Send an inter-processor interrupt to wake up a CPU Core that is in a HALT state.
 #[allow(unused_variables)]
 pub fn wakeup_core(core_id_to_wakeup: CoreId) {
-	#[cfg(feature = "smp")]
-	if core_id_to_wakeup != core_id() {
+	#[cfg(all(feature = "smp", not(feature = "idle-poll")))]
+	if core_id_to_wakeup != core_id()
+		&& !crate::processor::supports_mwait()
+		&& crate::scheduler::take_core_hlt_state(core_id_to_wakeup)
+	{
 		without_interrupts(|| {
 			let apic_ids = CPU_LOCAL_APIC_IDS.lock();
 			let local_apic_id = apic_ids[core_id_to_wakeup as usize];

src/arch/x86_64/kernel/core_local.rs

@@ -2,6 +2,8 @@ use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::arch::asm;
 use core::cell::{Cell, RefCell, RefMut};
+#[cfg(feature = "smp")]
+use core::sync::atomic::AtomicBool;
 use core::sync::atomic::Ordering;
 use core::{mem, ptr};
 
@@ -32,6 +34,8 @@ pub(crate) struct CoreLocal {
 	irq_statistics: &'static IrqStatistics,
 	/// Queue of async tasks
 	async_tasks: RefCell<Vec<AsyncTask>>,
+	#[cfg(feature = "smp")]
+	pub hlt: AtomicBool,
 	/// Queues to handle incoming requests from the other cores
 	#[cfg(feature = "smp")]
 	pub scheduler_input: InterruptTicketMutex<SchedulerInput>,
@@ -59,6 +63,8 @@ impl CoreLocal {
 			irq_statistics,
 			async_tasks: RefCell::new(Vec::new()),
 			#[cfg(feature = "smp")]
+			hlt: AtomicBool::new(false),
+			#[cfg(feature = "smp")]
 			scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()),
 		};
 		let this = if core_id == 0 {

src/arch/x86_64/kernel/interrupts.rs

@@ -5,7 +5,9 @@ use core::sync::atomic::{AtomicU64, Ordering};
 use ahash::RandomState;
 use hashbrown::HashMap;
 use hermit_sync::{InterruptSpinMutex, InterruptTicketMutex};
-pub use x86_64::instructions::interrupts::{disable, enable, enable_and_hlt as enable_and_wait};
+#[cfg(not(feature = "idle-poll"))]
+use x86_64::instructions::interrupts::enable_and_hlt;
+pub use x86_64::instructions::interrupts::{disable, enable};
 use x86_64::set_general_handler;
 #[cfg(any(feature = "fuse", feature = "tcp", feature = "udp"))]
 use x86_64::structures::idt;
@@ -34,6 +36,49 @@ pub(crate) fn load_idt() {
 	}
 }
 
+#[inline]
+pub(crate) fn enable_and_wait() {
+	#[cfg(feature = "idle-poll")]
+	unsafe {
+		asm!("pause", options(nomem, nostack, preserves_flags));
+	}
+
+	#[cfg(not(feature = "idle-poll"))]
+	if crate::processor::supports_mwait() {
+		let addr = core_scheduler().get_priority_bitmap() as *const _ as *const u8;
+
+		unsafe {
+			if crate::processor::supports_clflush() {
+				core::arch::x86_64::_mm_clflush(addr);
+			}
+
+			asm!(
+				"monitor",
+				in("rax") addr,
+				in("rcx") 0,
+				in("rdx") 0,
+				options(readonly, nostack, preserves_flags)
+			);
+
+			// The maximum waiting time is an implicit 64-bit timestamp-counter value
+			// stored in the EDX:EBX register pair.
+			// Test timeout by changing "b" => (0xffffffff) or "d"((wakeup >> 32) + 1)
+			// ECX bit 31 indicate whether timeout feature is used
+			// EAX [0:3] indicate sub C-state; [4:7] indicate C-states e.g., 0=>C1, 1=>C2 ...
+			asm!(
+				"sti; mwait",
+				in("rax") 0x2,
+				in("rcx") 0 /* break on interrupt flag */,
+				options(readonly, nostack, preserves_flags)
+			);
+		}
+	} else {
+		#[cfg(feature = "smp")]
+		crate::CoreLocal::get().hlt.store(true, Ordering::Relaxed);
+		enable_and_hlt();
+	}
+}
+
 pub(crate) fn install() {
 	let mut idt = IDT.lock();
 

src/arch/x86_64/kernel/processor.rs

@@ -55,6 +55,8 @@ struct Features {
 	supports_tsc_deadline: bool,
 	supports_x2apic: bool,
 	supports_xsave: bool,
+	supports_mwait: bool,
+	supports_clflush: bool,
 	run_on_hypervisor: bool,
 	supports_fsgs: bool,
 	supports_rdtscp: bool,
@@ -85,6 +87,8 @@ static FEATURES: Lazy<Features> = Lazy::new(|| {
 		supports_tsc_deadline: feature_info.has_tsc_deadline(),
 		supports_x2apic: feature_info.has_x2apic(),
 		supports_xsave: feature_info.has_xsave(),
+		supports_mwait: feature_info.has_monitor_mwait(),
+		supports_clflush: feature_info.has_clflush(),
 		run_on_hypervisor: feature_info.has_hypervisor(),
 		supports_fsgs: extended_feature_info.has_fsgsbase(),
 		supports_rdtscp: extend_processor_identifiers.has_rdtscp(),
@@ -587,6 +591,9 @@ impl fmt::Display for CpuFeaturePrinter {
 		if self.feature_info.has_monitor_mwait() {
 			write!(f, "MWAIT ")?;
 		}
+		if self.extend_processor_identifiers.has_monitorx_mwaitx() {
+			write!(f, "MWAITX ")?;
+		}
 		if self.feature_info.has_clflush() {
 			write!(f, "CLFLUSH ")?;
 		}
@@ -602,7 +609,6 @@ impl fmt::Display for CpuFeaturePrinter {
 		if self.feature_info.has_hypervisor() {
 			write!(f, "HYPERVISOR ")?;
 		}
-
 		if self.extended_feature_info.has_avx2() {
 			write!(f, "AVX2 ")?;
 		}
@@ -996,6 +1002,16 @@ pub fn supports_xsave() -> bool {
 	FEATURES.supports_xsave
 }
 
+#[inline]
+pub fn supports_mwait() -> bool {
+	FEATURES.supports_mwait
+}
+
+#[inline]
+pub fn supports_clflush() -> bool {
+	FEATURES.supports_clflush
+}
+
 #[inline]
 pub fn supports_fsgs() -> bool {
 	FEATURES.supports_fsgs

src/scheduler/mod.rs

@@ -9,6 +9,8 @@ use alloc::vec::Vec;
 use core::cell::RefCell;
 use core::future::{self, Future};
 use core::ptr;
+#[cfg(all(target_arch = "x86_64", feature = "smp"))]
+use core::sync::atomic::AtomicBool;
 use core::sync::atomic::{AtomicI32, AtomicU32, Ordering};
 use core::task::ready;
 use core::task::Poll::Ready;
@@ -38,6 +40,8 @@ static NO_TASKS: AtomicU32 = AtomicU32::new(0);
 #[cfg(feature = "smp")]
 static SCHEDULER_INPUTS: SpinMutex<Vec<&InterruptTicketMutex<SchedulerInput>>> =
 	SpinMutex::new(Vec::new());
+#[cfg(all(target_arch = "x86_64", feature = "smp"))]
+static CORE_HLT_STATE: SpinMutex<Vec<&AtomicBool>> = SpinMutex::new(Vec::new());
 /// Map between Task ID and Queue of waiting tasks
 static WAITING_TASKS: InterruptTicketMutex<BTreeMap<TaskId, VecDeque<TaskHandle>>> =
 	InterruptTicketMutex::new(BTreeMap::new());
@@ -615,6 +619,13 @@ impl PerCoreScheduler {
 		without_interrupts(|| self.current_task.borrow().prio)
 	}
 
+	/// Returns reference to prio_bitmap
+	#[allow(dead_code)]
+	#[inline]
+	pub fn get_priority_bitmap(&self) -> &u64 {
+		self.ready_queue.get_priority_bitmap()
+	}
+
 	#[cfg(target_arch = "x86_64")]
 	pub fn set_current_kernel_stack(&self) {
 		use x86_64::VirtAddr;
@@ -731,11 +742,14 @@ impl PerCoreScheduler {
 			crate::executor::run();
 
 			// do housekeeping
+			#[cfg(all(any(target_arch = "x86_64", target_arch = "riscv64"), feature = "smp"))]
+			core_scheduler.check_input();
 			core_scheduler.cleanup_tasks();
 
 			if core_scheduler.ready_queue.is_empty() {
 				if backoff.is_completed() {
 					interrupts::enable_and_wait();
+					backoff.reset();
 				} else {
 					interrupts::enable();
 					backoff.snooze();
@@ -920,9 +934,19 @@ pub(crate) fn add_current_core() {
 			core_id.try_into().unwrap(),
 			&CoreLocal::get().scheduler_input,
 		);
+		#[cfg(target_arch = "x86_64")]
+		CORE_HLT_STATE
+			.lock()
+			.insert(core_id.try_into().unwrap(), &CoreLocal::get().hlt);
 	}
 }
 
+#[inline]
+#[cfg(all(target_arch = "x86_64", feature = "smp"))]
+pub(crate) fn take_core_hlt_state(core_id: CoreId) -> bool {
+	CORE_HLT_STATE.lock()[usize::try_from(core_id).unwrap()].swap(false, Ordering::Acquire)
+}
+
 #[inline]
 #[cfg(feature = "smp")]
 fn get_scheduler_input(core_id: CoreId) -> &'static InterruptTicketMutex<SchedulerInput> {

src/scheduler/task.rs

@@ -13,6 +13,7 @@ use core::ops::DerefMut;
 use core::{cmp, fmt};
 
 use ahash::RandomState;
+use crossbeam_utils::CachePadded;
 use hashbrown::HashMap;
 use hermit_sync::OnceCell;
 
@@ -160,7 +161,7 @@ impl Eq for TaskHandle {}
 #[derive(Default)]
 pub(crate) struct TaskHandlePriorityQueue {
 	queues: [Option<VecDeque<TaskHandle>>; NO_PRIORITIES],
-	prio_bitmap: u64,
+	prio_bitmap: CachePadded<u64>,
 }
 
 impl TaskHandlePriorityQueue {
@@ -172,13 +173,13 @@ impl TaskHandlePriorityQueue {
 				None, None, None, None, None, None, None, None, None, None, None, None, None, None,
 				None, None, None,
 			],
-			prio_bitmap: 0,
+			prio_bitmap: CachePadded::new(0),
 		}
 	}
 
 	/// Checks if the queue is empty.
 	pub fn is_empty(&self) -> bool {
-		self.prio_bitmap == 0
+		self.prio_bitmap.into_inner() == 0
 	}
 
 	/// Checks if the given task is in the queue. Returns `true` if the task
@@ -193,7 +194,7 @@ impl TaskHandlePriorityQueue {
 		let i = task.priority.into() as usize;
 		//assert!(i < NO_PRIORITIES, "Priority {} is too high", i);
 
-		self.prio_bitmap |= (1 << i) as u64;
+		*self.prio_bitmap |= (1 << i) as u64;
 		if let Some(queue) = &mut self.queues[i] {
 			queue.push_back(task);
 		} else {
@@ -208,7 +209,7 @@ impl TaskHandlePriorityQueue {
 			let task = queue.pop_front();
 
 			if queue.is_empty() {
-				self.prio_bitmap &= !(1 << queue_index as u64);
+				*self.prio_bitmap &= !(1 << queue_index as u64);
 			}
 
 			task
@@ -219,7 +220,7 @@ impl TaskHandlePriorityQueue {
 
 	/// Pop the task handle with the highest priority from the queue
 	pub fn pop(&mut self) -> Option<TaskHandle> {
-		if let Some(i) = msb(self.prio_bitmap) {
+		if let Some(i) = msb(self.prio_bitmap.into_inner()) {
 			return self.pop_from_queue(i as usize);
 		}
 
@@ -245,7 +246,7 @@ impl TaskHandlePriorityQueue {
 			}
 
 			if queue.is_empty() {
-				self.prio_bitmap &= !(1 << queue_index as u64);
+				*self.prio_bitmap &= !(1 << queue_index as u64);
 			}
 		}
 
@@ -317,6 +318,13 @@ impl PriorityTaskQueue {
 		self.prio_bitmap == 0
 	}
 
+	/// Returns reference to prio_bitmap
+	#[allow(dead_code)]
+	#[inline]
+	pub fn get_priority_bitmap(&self) -> &u64 {
+		&self.prio_bitmap
+	}
+
 	/// Pop the task with the highest priority from the queue
 	pub fn pop(&mut self) -> Option<Rc<RefCell<Task>>> {
 		if let Some(i) = msb(self.prio_bitmap) {