From 51dcf50e9c9a8615f812de99a54fd2100136d040 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 3 Aug 2023 09:46:09 +0000 Subject: [PATCH 01/25] Supporting moving immix (wip) --- src/plan/generational/gc_work.rs | 21 ++- src/plan/generational/mod.rs | 6 +- src/plan/global.rs | 90 ++++----- src/plan/immix/gc_work.rs | 2 + src/plan/immix/global.rs | 4 +- src/plan/mod.rs | 34 ++-- src/plan/tracing.rs | 8 +- src/policy/copyspace.rs | 2 + src/policy/gc_work.rs | 1 + src/policy/immix/immixspace.rs | 18 +- src/policy/immix/mod.rs | 4 +- src/policy/markcompactspace.rs | 3 +- src/scheduler/gc_work.rs | 299 ++++++++++++++++++++++++------ src/scheduler/scheduler.rs | 4 +- src/scheduler/work.rs | 1 + src/scheduler/work_bucket.rs | 2 + src/util/finalizable_processor.rs | 6 +- src/util/reference_processor.rs | 9 +- src/vm/scanning.rs | 6 +- 19 files changed, 369 insertions(+), 151 deletions(-) diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index 0ebd617ff6..5e3876aaf7 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -1,7 +1,7 @@ use atomic::Ordering; use crate::plan::PlanTraceObject; -use crate::scheduler::{gc_work::*, GCWork, GCWorker}; +use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage}; use crate::util::ObjectReference; use crate::vm::edge_shape::{Edge, MemorySlice}; use crate::vm::*; @@ -25,8 +25,13 @@ impl + PlanTraceObject> ProcessEdg type VM = VM; type ScanObjectsWorkType = PlanScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref().unwrap(); Self { plan, base } } @@ -51,7 +56,7 @@ impl + PlanTraceObject> ProcessEdg nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - PlanScanObjects::new(self.plan, nodes, false, roots) + PlanScanObjects::new(self.plan, nodes, false, roots, self.bucket) } } @@ -106,7 +111,7 @@ impl GCWork for ProcessModBuf { // Scan objects in the modbuf and forward pointers let modbuf = std::mem::take(&mut self.modbuf); GCWork::do_work( - &mut ScanObjects::::new(modbuf, false, false), + &mut ScanObjects::::new(modbuf, false, false, WorkBucketStage::Closure), worker, mmtk, ) @@ -144,7 +149,11 @@ impl GCWork for ProcessRegionModBuf { } } // Forward entries - GCWork::do_work(&mut E::new(edges, false, mmtk), worker, mmtk) + GCWork::do_work( + &mut E::new(edges, false, mmtk, WorkBucketStage::Closure), + worker, + mmtk, + ) } } } diff --git a/src/plan/generational/mod.rs b/src/plan/generational/mod.rs index 29882ba5f3..b0758cb288 100644 --- a/src/plan/generational/mod.rs +++ b/src/plan/generational/mod.rs @@ -21,9 +21,9 @@ use super::mutator_context::ReservedAllocators; pub mod barrier; /// Generational copying (GenCopy) -pub mod copying; -/// Generational immix (GenImmix) -pub mod immix; +// pub mod copying; +// /// Generational immix (GenImmix) +// pub mod immix; // Common generational code diff --git a/src/plan/global.rs b/src/plan/global.rs index 1340fffac2..513bc64a7f 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -40,29 +40,30 @@ pub fn create_mutator( mmtk: &'static MMTK, ) -> Box> { Box::new(match *mmtk.options.plan { - PlanSelector::NoGC => crate::plan::nogc::mutator::create_nogc_mutator(tls, &*mmtk.plan), - PlanSelector::SemiSpace => { - crate::plan::semispace::mutator::create_ss_mutator(tls, &*mmtk.plan) - } - PlanSelector::GenCopy => { - crate::plan::generational::copying::mutator::create_gencopy_mutator(tls, mmtk) - } - PlanSelector::GenImmix => { - crate::plan::generational::immix::mutator::create_genimmix_mutator(tls, mmtk) - } - PlanSelector::MarkSweep => { - crate::plan::marksweep::mutator::create_ms_mutator(tls, &*mmtk.plan) - } + // PlanSelector::NoGC => crate::plan::nogc::mutator::create_nogc_mutator(tls, &*mmtk.plan), + // PlanSelector::SemiSpace => { + // crate::plan::semispace::mutator::create_ss_mutator(tls, &*mmtk.plan) + // } + // PlanSelector::GenCopy => { + // crate::plan::generational::copying::mutator::create_gencopy_mutator(tls, mmtk) + // } + // PlanSelector::GenImmix => { + // crate::plan::generational::immix::mutator::create_genimmix_mutator(tls, mmtk) + // } + // PlanSelector::MarkSweep => { + // crate::plan::marksweep::mutator::create_ms_mutator(tls, &*mmtk.plan) + // } PlanSelector::Immix => crate::plan::immix::mutator::create_immix_mutator(tls, &*mmtk.plan), - PlanSelector::PageProtect => { - crate::plan::pageprotect::mutator::create_pp_mutator(tls, &*mmtk.plan) - } - PlanSelector::MarkCompact => { - crate::plan::markcompact::mutator::create_markcompact_mutator(tls, &*mmtk.plan) - } - PlanSelector::StickyImmix => { - crate::plan::sticky::immix::mutator::create_stickyimmix_mutator(tls, mmtk) - } + // PlanSelector::PageProtect => { + // crate::plan::pageprotect::mutator::create_pp_mutator(tls, &*mmtk.plan) + // } + // PlanSelector::MarkCompact => { + // crate::plan::markcompact::mutator::create_markcompact_mutator(tls, &*mmtk.plan) + // } + // PlanSelector::StickyImmix => { + // crate::plan::sticky::immix::mutator::create_stickyimmix_mutator(tls, mmtk) + // } + _ => { todo!() }, }) } @@ -84,31 +85,32 @@ pub fn create_plan( let gc_trigger = args.gc_trigger.clone(); let plan = match plan { - PlanSelector::NoGC => { - Box::new(crate::plan::nogc::NoGC::new(args)) as Box> - } - PlanSelector::SemiSpace => { - Box::new(crate::plan::semispace::SemiSpace::new(args)) as Box> - } - PlanSelector::GenCopy => Box::new(crate::plan::generational::copying::GenCopy::new(args)) - as Box>, - PlanSelector::GenImmix => Box::new(crate::plan::generational::immix::GenImmix::new(args)) - as Box>, - PlanSelector::MarkSweep => { - Box::new(crate::plan::marksweep::MarkSweep::new(args)) as Box> - } + // PlanSelector::NoGC => { + // Box::new(crate::plan::nogc::NoGC::new(args)) as Box> + // } + // PlanSelector::SemiSpace => { + // Box::new(crate::plan::semispace::SemiSpace::new(args)) as Box> + // } + // PlanSelector::GenCopy => Box::new(crate::plan::generational::copying::GenCopy::new(args)) + // as Box>, + // PlanSelector::GenImmix => Box::new(crate::plan::generational::immix::GenImmix::new(args)) + // as Box>, + // PlanSelector::MarkSweep => { + // Box::new(crate::plan::marksweep::MarkSweep::new(args)) as Box> + // } PlanSelector::Immix => { Box::new(crate::plan::immix::Immix::new(args)) as Box> } - PlanSelector::PageProtect => { - Box::new(crate::plan::pageprotect::PageProtect::new(args)) as Box> - } - PlanSelector::MarkCompact => { - Box::new(crate::plan::markcompact::MarkCompact::new(args)) as Box> - } - PlanSelector::StickyImmix => { - Box::new(crate::plan::sticky::immix::StickyImmix::new(args)) as Box> - } + // PlanSelector::PageProtect => { + // Box::new(crate::plan::pageprotect::PageProtect::new(args)) as Box> + // } + // PlanSelector::MarkCompact => { + // Box::new(crate::plan::markcompact::MarkCompact::new(args)) as Box> + // } + // PlanSelector::StickyImmix => { + // Box::new(crate::plan::sticky::immix::StickyImmix::new(args)) as Box> + // } + _ => { todo!() } }; // We have created Plan in the heap, and we won't explicitly move it. diff --git a/src/plan/immix/gc_work.rs b/src/plan/immix/gc_work.rs index 0696086fc1..33e225f239 100644 --- a/src/plan/immix/gc_work.rs +++ b/src/plan/immix/gc_work.rs @@ -2,6 +2,7 @@ use super::global::Immix; use crate::policy::gc_work::TraceKind; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; +use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; pub(super) struct ImmixGCWorkContext( std::marker::PhantomData, @@ -12,4 +13,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = Immix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; + type ImmovableProcessEdges = PlanProcessEdges, TRACE_KIND_IMMOVABLE>; } diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index d9c2682f50..03e66fec47 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -173,8 +173,8 @@ impl Immix { /// to schedule a full heap collection. A plan must call set_collection_kind and set_gc_status before this method. pub(crate) fn schedule_immix_full_heap_collection< PlanType: Plan, - FastContext: 'static + GCWorkContext, - DefragContext: 'static + GCWorkContext, + FastContext: Send + 'static + GCWorkContext, + DefragContext: Send + 'static + GCWorkContext, >( plan: &'static DefragContext::PlanType, immix_space: &ImmixSpace, diff --git a/src/plan/mod.rs b/src/plan/mod.rs index 15c16182fa..afcfce513d 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -40,17 +40,17 @@ pub use plan_constraints::DEFAULT_PLAN_CONSTRAINTS; mod tracing; pub use tracing::{ObjectQueue, ObjectsClosure, VectorObjectQueue, VectorQueue}; -/// Generational plans (with a copying nursery) +// /// Generational plans (with a copying nursery) mod generational; -/// Sticky plans (using sticky marks for generational behaviors without a copying nursery) -mod sticky; +// /// Sticky plans (using sticky marks for generational behaviors without a copying nursery) +// mod sticky; mod immix; -mod markcompact; -mod marksweep; -mod nogc; -mod pageprotect; -mod semispace; +// mod markcompact; +// mod marksweep; +// mod nogc; +// mod pageprotect; +// mod semispace; pub(crate) use generational::global::is_nursery_gc; pub(crate) use generational::global::GenerationalPlan; @@ -58,12 +58,14 @@ pub(crate) use generational::global::GenerationalPlan; // Expose plan constraints as public. Though a binding can get them from plan.constraints(), // it is possible for performance reasons that they want the constraints as constants. -pub use generational::copying::GENCOPY_CONSTRAINTS; -pub use generational::immix::GENIMMIX_CONSTRAINTS; +// pub use generational::copying::GENCOPY_CONSTRAINTS; +// pub use generational::immix::GENIMMIX_CONSTRAINTS; pub use immix::IMMIX_CONSTRAINTS; -pub use markcompact::MARKCOMPACT_CONSTRAINTS; -pub use marksweep::MS_CONSTRAINTS; -pub use nogc::NOGC_CONSTRAINTS; -pub use pageprotect::PP_CONSTRAINTS; -pub use semispace::SS_CONSTRAINTS; -pub use sticky::immix::STICKY_IMMIX_CONSTRAINTS; + +// use crate::vm::VMBinding; +// pub use markcompact::MARKCOMPACT_CONSTRAINTS; +// pub use marksweep::MS_CONSTRAINTS; +// pub use nogc::NOGC_CONSTRAINTS; +// pub use pageprotect::PP_CONSTRAINTS; +// pub use semispace::SS_CONSTRAINTS; +// pub use sticky::immix::STICKY_IMMIX_CONSTRAINTS; diff --git a/src/plan/tracing.rs b/src/plan/tracing.rs index 1f3b4e6dc3..3c3df5dfc2 100644 --- a/src/plan/tracing.rs +++ b/src/plan/tracing.rs @@ -75,13 +75,15 @@ impl ObjectQueue for VectorQueue { pub struct ObjectsClosure<'a, E: ProcessEdgesWork> { buffer: VectorQueue>, worker: &'a mut GCWorker, + bucket: WorkBucketStage, } impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> { - pub fn new(worker: &'a mut GCWorker) -> Self { + pub fn new(worker: &'a mut GCWorker, bucket: WorkBucketStage) -> Self { Self { buffer: VectorQueue::new(), worker, + bucket, } } @@ -89,8 +91,8 @@ impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> { let buf = self.buffer.take(); if !buf.is_empty() { self.worker.add_work( - WorkBucketStage::Closure, - E::new(buf, false, self.worker.mmtk), + self.bucket, + E::new(buf, false, self.worker.mmtk, self.bucket), ); } } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 11bc3baace..87d3ccb16b 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -1,5 +1,6 @@ use crate::plan::{ObjectQueue, VectorObjectQueue}; use crate::policy::copy_context::PolicyCopyContext; +use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; @@ -126,6 +127,7 @@ impl crate::policy::gc_work::PolicyTraceObject for CopySpace< copy: Option, worker: &mut GCWorker, ) -> ObjectReference { + assert!(KIND != TRACE_KIND_IMMOVABLE, "Copyspace does not support immovable trace."); self.trace_object(queue, object, copy, worker) } diff --git a/src/policy/gc_work.rs b/src/policy/gc_work.rs index 15dabd7f96..674844f5f5 100644 --- a/src/policy/gc_work.rs +++ b/src/policy/gc_work.rs @@ -3,6 +3,7 @@ pub(crate) type TraceKind = u8; pub const DEFAULT_TRACE: u8 = u8::MAX; +pub const TRACE_KIND_IMMOVABLE: u8 = DEFAULT_TRACE - 1; use crate::plan::ObjectQueue; use crate::scheduler::GCWorker; diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index c13e4ca76c..f6735ca422 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -1,7 +1,7 @@ use super::line::*; use super::{block::*, defrag::Defrag}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_IMMOVABLE}; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; @@ -191,7 +191,9 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace copy: Option, worker: &mut GCWorker, ) -> ObjectReference { - if KIND == TRACE_KIND_DEFRAG { + if KIND == TRACE_KIND_IMMOVABLE { + self.trace_object_without_moving(queue, object) + } else if KIND == TRACE_KIND_DEFRAG { if Block::containing::(object).is_defrag_source() { debug_assert!(self.in_defrag()); debug_assert!( @@ -226,7 +228,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace fn may_move_objects() -> bool { if KIND == TRACE_KIND_DEFRAG { true - } else if KIND == TRACE_KIND_FAST { + } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_IMMOVABLE{ false } else { unreachable!() @@ -618,11 +620,11 @@ impl ImmixSpace { } else if self.is_marked(object) { // We won the forwarding race but the object is already marked so we clear the // forwarding status and return the unmoved object - debug_assert!( - nursery_collection || self.defrag.space_exhausted() || self.is_pinned(object), - "Forwarded object is the same as original object {} even though it should have been copied", - object, - ); + // debug_assert!( + // nursery_collection || self.defrag.space_exhausted() || self.is_pinned(object), + // "Forwarded object is the same as original object {} even though it should have been copied", + // object, + // ); ForwardingWord::clear_forwarding_bits::(object); object } else { diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index fb9f0846f9..fb96053c07 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -18,11 +18,11 @@ pub const BLOCK_ONLY: bool = false; pub const DEFRAG: bool = !cfg!(feature = "immix_non_moving"); // defrag if we are allowed to move. /// Make every GC a defragment GC. (for debugging) -pub const STRESS_DEFRAG: bool = false; +pub const STRESS_DEFRAG: bool = true; /// Mark every allocated block as defragmentation source before GC. (for debugging) /// Set both this and `STRESS_DEFRAG` to true to make Immix move as many objects as possible. -pub const DEFRAG_EVERY_BLOCK: bool = false; +pub const DEFRAG_EVERY_BLOCK: bool = true; /// If Immix is used as a nursery space, do we prefer copy? pub const PREFER_COPY_ON_NURSERY_GC: bool = !cfg!(feature = "immix_non_moving"); // copy nursery objects if we are allowed to move. diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index ecc7b2035d..8f08899dba 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -1,7 +1,7 @@ use super::sft::SFT; use super::space::{CommonSpace, Space}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_IMMOVABLE}; use crate::policy::sft::GCWorkerMutRef; use crate::scheduler::GCWorker; use crate::util::alloc::allocator::align_allocation_no_fill; @@ -128,6 +128,7 @@ impl crate::policy::gc_work::PolicyTraceObject for MarkCompac _copy: Option, _worker: &mut GCWorker, ) -> ObjectReference { + assert!(KIND != TRACE_KIND_IMMOVABLE, "MarkCompact does not support immovable trace."); if KIND == TRACE_KIND_MARK { self.trace_mark_object(queue, object) } else if KIND == TRACE_KIND_FORWARD { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 8cdef9a04e..edf3118f6c 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -169,29 +169,29 @@ impl GCWork for ReleaseCollector { /// /// TODO: Smaller work granularity #[derive(Default)] -pub struct StopMutators(PhantomData); +pub struct StopMutators(PhantomData); -impl StopMutators { +impl StopMutators { pub fn new() -> Self { Self(PhantomData) } } -impl GCWork for StopMutators { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for StopMutators { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("stop_all_mutators start"); mmtk.plan.base().prepare_for_stack_scanning(); - ::VMCollection::stop_all_mutators(worker.tls, |mutator| { - mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanStackRoot::(mutator)); + ::VMCollection::stop_all_mutators(worker.tls, |mutator| { + mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanStackRoot::(mutator)); }); trace!("stop_all_mutators end"); mmtk.scheduler.notify_mutators_paused(mmtk); - if ::VMScanning::SCAN_MUTATORS_IN_SAFEPOINT { + if ::VMScanning::SCAN_MUTATORS_IN_SAFEPOINT { // Prepare mutators if necessary // FIXME: This test is probably redundant. JikesRVM requires to call `prepare_mutator` once after mutators are paused if !mmtk.plan.base().stacks_prepared() { - for mutator in ::VMActivePlan::mutators() { - ::VMCollection::prepare_mutator( + for mutator in ::VMActivePlan::mutators() { + ::VMCollection::prepare_mutator( worker.tls, mutator.get_tls(), mutator, @@ -199,17 +199,17 @@ impl GCWork for StopMutators { } } // Scan mutators - if ::VMScanning::SINGLE_THREAD_MUTATOR_SCANNING { + if ::VMScanning::SINGLE_THREAD_MUTATOR_SCANNING { mmtk.scheduler.work_buckets[WorkBucketStage::Prepare] - .add(ScanStackRoots::::new()); + .add(ScanStackRoots::::new()); } else { - for mutator in ::VMActivePlan::mutators() { + for mutator in ::VMActivePlan::mutators() { mmtk.scheduler.work_buckets[WorkBucketStage::Prepare] - .add(ScanStackRoot::(mutator)); + .add(ScanStackRoot::(mutator)); } } } - mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanVMSpecificRoots::::new()); + mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanVMSpecificRoots::::new()); } } @@ -315,7 +315,7 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC let mmtk = worker.mmtk; // Prepare the underlying ProcessEdgesWork - let mut process_edges_work = E::new(vec![], false, mmtk); + let mut process_edges_work = E::new(vec![], false, mmtk, self.stage); // FIXME: This line allows us to omit the borrowing lifetime of worker. // We should refactor ProcessEdgesWork so that it uses `worker` locally, not as a member. process_edges_work.set_worker(worker); @@ -432,36 +432,36 @@ impl GCWork for VMPostForwarding { } #[derive(Default)] -pub struct ScanStackRoots(PhantomData); +pub struct ScanStackRoots(PhantomData); -impl ScanStackRoots { +impl ScanStackRoots { pub fn new() -> Self { Self(PhantomData) } } -impl GCWork for ScanStackRoots { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ScanStackRoots { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStackRoots"); - let factory = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - ::VMScanning::scan_roots_in_all_mutator_threads(worker.tls, factory); - ::VMScanning::notify_initial_thread_scan_complete(false, worker.tls); - for mutator in ::VMActivePlan::mutators() { + let factory = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + ::VMScanning::scan_roots_in_all_mutator_threads(worker.tls, factory); + ::VMScanning::notify_initial_thread_scan_complete(false, worker.tls); + for mutator in ::VMActivePlan::mutators() { mutator.flush(); } mmtk.plan.common().base.set_gc_status(GcStatus::GcProper); } } -pub struct ScanStackRoot(pub &'static mut Mutator); +pub struct ScanStackRoot(pub &'static mut Mutator); -impl GCWork for ScanStackRoot { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ScanStackRoot { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStackRoot for mutator {:?}", self.0.get_tls()); let base = &mmtk.plan.base(); - let mutators = ::VMActivePlan::number_of_mutators(); - let factory = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - ::VMScanning::scan_roots_in_mutator_thread( + let mutators = ::VMActivePlan::number_of_mutators(); + let factory = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + ::VMScanning::scan_roots_in_mutator_thread( worker.tls, unsafe { &mut *(self.0 as *mut _) }, factory, @@ -469,7 +469,7 @@ impl GCWork for ScanStackRoot { self.0.flush(); if mmtk.plan.base().inform_stack_scanned(mutators) { - ::VMScanning::notify_initial_thread_scan_complete( + ::VMScanning::notify_initial_thread_scan_complete( false, worker.tls, ); base.set_gc_status(GcStatus::GcProper); @@ -478,19 +478,20 @@ impl GCWork for ScanStackRoot { } #[derive(Default)] -pub struct ScanVMSpecificRoots(PhantomData); +pub struct ScanVMSpecificRoots(PhantomData); -impl ScanVMSpecificRoots { +impl ScanVMSpecificRoots { pub fn new() -> Self { Self(PhantomData) } } -impl GCWork for ScanVMSpecificRoots { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ScanVMSpecificRoots { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStaticRoots"); - let factory = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - ::VMScanning::scan_vm_specific_roots(worker.tls, factory); + let factory // = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); + = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } } @@ -502,6 +503,7 @@ pub struct ProcessEdgesBase { // Because a copying gc will dereference this pointer at least once for every object copy. worker: *mut GCWorker, pub roots: bool, + pub bucket: WorkBucketStage, } unsafe impl Send for ProcessEdgesBase {} @@ -509,7 +511,12 @@ unsafe impl Send for ProcessEdgesBase {} impl ProcessEdgesBase { // Requires an MMTk reference. Each plan-specific type that uses ProcessEdgesBase can get a static plan reference // at creation. This avoids overhead for dynamic dispatch or downcasting plan for each object traced. - pub fn new(edges: Vec, roots: bool, mmtk: &'static MMTK) -> Self { + pub fn new( + edges: Vec, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { #[cfg(feature = "extreme_assertions")] if crate::util::edge_logger::should_check_duplicate_edges(&*mmtk.plan) { for edge in &edges { @@ -523,6 +530,7 @@ impl ProcessEdgesBase { mmtk, worker: std::ptr::null_mut(), roots, + bucket, } } pub fn set_worker(&mut self, worker: &mut GCWorker) { @@ -570,7 +578,12 @@ pub trait ProcessEdgesWork: const OVERWRITE_REFERENCE: bool = true; const SCAN_OBJECTS_IMMEDIATELY: bool = true; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self; + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self; /// Trace an MMTk object. The implementation should forward this call to the policy-specific /// `trace_object()` methods, depending on which space this object is in. @@ -599,7 +612,8 @@ pub trait ProcessEdgesWork: // Executing these work packets now can remarkably reduce the global synchronization time. self.worker().do_work(work_packet); } else { - self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(work_packet); + debug_assert!(self.bucket != WorkBucketStage::Unconstrained); + self.mmtk.scheduler.work_buckets[self.bucket].add(work_packet); } } @@ -667,8 +681,13 @@ impl ProcessEdgesWork for SFTProcessEdges { type VM = VM; type ScanObjectsWorkType = ScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); Self { base } } @@ -688,7 +707,7 @@ impl ProcessEdgesWork for SFTProcessEdges { } fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, self.bucket) } } @@ -703,17 +722,19 @@ impl Clone for ProcessEdgesWorkRootsWorkFactory { } impl RootsWorkFactory> for ProcessEdgesWorkRootsWorkFactory { - fn create_process_edge_roots_work(&mut self, edges: Vec>) { + fn create_process_edge_roots_work(&mut self, edges: Vec>, movable: bool) { + debug_assert!(movable); crate::memory_manager::add_work_packet( self.mmtk, WorkBucketStage::Closure, - E::new(edges, true, self.mmtk), + E::new(edges, true, self.mmtk, WorkBucketStage::Closure), ); } - fn create_process_node_roots_work(&mut self, nodes: Vec) { + fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool) { // We want to use E::create_scan_work. - let process_edges_work = E::new(vec![], true, self.mmtk); + debug_assert!(movable); + let process_edges_work = E::new(vec![], true, self.mmtk, WorkBucketStage::Closure); let work = process_edges_work.create_scan_work(nodes, true); crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, work); } @@ -725,6 +746,64 @@ impl ProcessEdgesWorkRootsWorkFactory { } } +struct ImmovableProcessEdgesWorkRootsWorkFactory< + VM: VMBinding, + E: ProcessEdgesWork, + I: ProcessEdgesWork, +> { + mmtk: &'static MMTK, + phantom: PhantomData<(E, I)>, +} + +impl, I: ProcessEdgesWork> Clone + for ImmovableProcessEdgesWorkRootsWorkFactory +{ + fn clone(&self) -> Self { + Self { + mmtk: self.mmtk, + phantom: PhantomData, + } + } +} + +impl, I: ProcessEdgesWork> + RootsWorkFactory> for ImmovableProcessEdgesWorkRootsWorkFactory +{ + fn create_process_edge_roots_work(&mut self, edges: Vec>, movable: bool) { + if movable { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::Closure, + E::new(edges, true, self.mmtk, WorkBucketStage::Closure), + ); + } else { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::ImmovableClosure, + I::new(edges, true, self.mmtk, WorkBucketStage::ImmovableClosure), + ); + } + } + + fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool) { + // We want to use E::create_scan_work. + if movable { + // let process_edges_work = E::new(vec![], true, self.mmtk, WorkBucketStage::Closure); + // let work = process_edges_work.create_scan_root_node_work(nodes, true); + + crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, ProcessRootNode::::new(nodes, WorkBucketStage::Closure)); + } else { + crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::ImmovableClosure, ProcessRootNode::::new(nodes, WorkBucketStage::ImmovableClosure)); + } + } +} + +impl, I: ProcessEdgesWork> ImmovableProcessEdgesWorkRootsWorkFactory { + fn new(mmtk: &'static MMTK) -> Self { + Self { mmtk, phantom : PhantomData } + } +} + impl Deref for SFTProcessEdges { type Target = ProcessEdgesBase; fn deref(&self) -> &Self::Target { @@ -754,6 +833,8 @@ pub trait ScanObjectsWork: GCWork + Sized { /// the objects in this packet. fn make_another(&self, buffer: Vec) -> Self; + fn get_bucket(&self) -> WorkBucketStage; + /// The common code for ScanObjects and PlanScanObjects. fn do_work_common( &self, @@ -762,6 +843,7 @@ pub trait ScanObjectsWork: GCWork + Sized { mmtk: &'static MMTK<::VM>, ) { let tls = worker.tls; + assert!(!self.roots()); #[cfg(feature = "sanity")] { @@ -784,7 +866,7 @@ pub trait ScanObjectsWork: GCWork + Sized { // objects which are traced for the first time. let scanned_root_objects = self.roots().then(|| { // We create an instance of E to use its `trace_object` method and its object queue. - let mut process_edges_work = Self::E::new(vec![], false, mmtk); + let mut process_edges_work = Self::E::new(vec![], false, mmtk, self.get_bucket()); process_edges_work.set_worker(worker); for object in buffer.iter().copied() { @@ -808,7 +890,7 @@ pub trait ScanObjectsWork: GCWork + Sized { // Then scan those objects for edges. let mut scan_later = vec![]; { - let mut closure = ObjectsClosure::::new(worker); + let mut closure = ObjectsClosure::::new(worker, self.get_bucket()); for object in objects_to_scan.iter().copied() { if ::VMScanning::support_edge_enqueuing(tls, object) { trace!("Scan object (edge) {}", object); @@ -830,7 +912,7 @@ pub trait ScanObjectsWork: GCWork + Sized { // If any object does not support edge-enqueuing, we process them now. if !scan_later.is_empty() { let object_tracer_context = ProcessEdgesWorkTracerContext:: { - stage: WorkBucketStage::Closure, + stage: self.get_bucket(), phantom_data: PhantomData, }; @@ -863,15 +945,22 @@ pub struct ScanObjects { concurrent: bool, roots: bool, phantom: PhantomData, + bucket: WorkBucketStage, } impl ScanObjects { - pub fn new(buffer: Vec, concurrent: bool, roots: bool) -> Self { + pub fn new( + buffer: Vec, + concurrent: bool, + roots: bool, + bucket: WorkBucketStage, + ) -> Self { Self { buffer, concurrent, roots, phantom: PhantomData, + bucket, } } } @@ -883,12 +972,29 @@ impl> ScanObjectsWork for ScanOb self.roots } + fn get_bucket(&self) -> WorkBucketStage { + self.bucket + } + fn post_scan_object(&self, _object: ObjectReference) { + // use std::fs::OpenOptions; + // use std::io::Write; + + // let mut file = OpenOptions::new() + // .write(true) + // .append(true) + // .create(true) + // .open("/home/eduardo/mmtk-julia/scanned_objs.log") + // .unwrap(); + + // if let Err(e) = writeln!(file, "Processed object {} at bucket {:?}", object, self.bucket) { + // eprintln!("Couldn't write to file: {}", e); + // } // Do nothing. } fn make_another(&self, buffer: Vec) -> Self { - Self::new(buffer, self.concurrent, false) + Self::new(buffer, self.concurrent, false, self.bucket) } } @@ -922,8 +1028,13 @@ impl + Plan, const KIND: TraceKin type VM = VM; type ScanObjectsWorkType = PlanScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref::

().unwrap(); Self { plan, base } } @@ -933,7 +1044,7 @@ impl + Plan, const KIND: TraceKin nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - PlanScanObjects::::new(self.plan, nodes, false, roots) + PlanScanObjects::::new(self.plan, nodes, false, roots, self.bucket) } fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { @@ -982,6 +1093,7 @@ pub struct PlanScanObjects + PlanTraceO concurrent: bool, roots: bool, phantom: PhantomData, + bucket: WorkBucketStage, } impl + PlanTraceObject> PlanScanObjects { @@ -990,6 +1102,7 @@ impl + PlanTraceObject> PlanScan buffer: Vec, concurrent: bool, roots: bool, + bucket: WorkBucketStage, ) -> Self { Self { plan, @@ -997,6 +1110,7 @@ impl + PlanTraceObject> PlanScan concurrent, roots, phantom: PhantomData, + bucket, } } } @@ -1010,12 +1124,29 @@ impl + PlanTraceObject> ScanObje self.roots } + fn get_bucket(&self) -> WorkBucketStage { + self.bucket + } + fn post_scan_object(&self, object: ObjectReference) { + // use std::fs::OpenOptions; + // use std::io::Write; + + // let mut file = OpenOptions::new() + // .write(true) + // .append(true) + // .create(true) + // .open("/home/eduardo/mmtk-julia/scanned_objs.log") + // .unwrap(); + + // if let Err(e) = writeln!(file, "Processed object {} at bucket {:?}", object, self.bucket) { + // eprintln!("Couldn't write to file: {}", e); + // } self.plan.post_scan_object(object); } fn make_another(&self, buffer: Vec) -> Self { - Self::new(self.plan, buffer, self.concurrent, false) + Self::new(self.plan, buffer, self.concurrent, false, self.bucket) } } @@ -1028,3 +1159,61 @@ impl + PlanTraceObject> GCWork, E2: ProcessEdgesWork> { + phantom : PhantomData<(VM, E1, E2)>, + roots : Vec, + bucket: WorkBucketStage +} + +impl, E2: ProcessEdgesWork> ProcessRootNode { + pub fn new(nodes : Vec, bucket: WorkBucketStage) -> Self { + Self { + phantom : PhantomData, + roots : nodes, + bucket + } + } +} + +impl, E2: ProcessEdgesWork> GCWork for ProcessRootNode { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + trace!("ProcessRootNode"); + + #[cfg(feature = "sanity")] + { + if !mmtk.plan.is_in_sanity() { + mmtk.sanity_checker + .lock() + .unwrap() + .add_root_nodes(self.roots.copied()); + } + } + + let scanned_root_objects = { + // We create an instance of E to use its `trace_object` method and its object queue. + let mut process_edges_work = E1::new(vec![], true, mmtk, self.bucket); + process_edges_work.set_worker(worker); + + for object in self.roots.iter().copied() { + let new_object = process_edges_work.trace_object(object); + debug_assert_eq!( + object, new_object, + "Object moved while tracing root unmovable root object: {} -> {}", + object, new_object + ); + } + + // This contains root objects that are visited the first time. + // It is sufficient to only scan these objects. + process_edges_work.nodes.take() + }; + + let process_edges_work = E2::new(vec![], false, mmtk, self.bucket); + let work = process_edges_work.create_scan_work(scanned_root_objects, false); + crate::memory_manager::add_work_packet(mmtk, self.bucket, work); + + + trace!("ProcessRootNode End"); + } +} diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 2cf17533b4..4e3f20e153 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -108,7 +108,7 @@ impl GCWorkScheduler { } /// Schedule all the common work packets - pub fn schedule_common_work + 'static>( + pub fn schedule_common_work + 'static + Send>( &self, plan: &'static C::PlanType, ) { @@ -116,7 +116,7 @@ impl GCWorkScheduler { use crate::scheduler::gc_work::*; // Stop & scan mutators (mutator scanning can happen before STW) self.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::::new()); + .add(StopMutators::::new()); // Prepare global/collectors/mutators self.work_buckets[WorkBucketStage::Prepare].add(Prepare::::new(plan)); diff --git a/src/scheduler/work.rs b/src/scheduler/work.rs index 2621fcedc0..489a1da67a 100644 --- a/src/scheduler/work.rs +++ b/src/scheduler/work.rs @@ -51,4 +51,5 @@ pub trait GCWorkContext { // We should use SFTProcessEdges as the default value for this associate type. However, this requires // `associated_type_defaults` which has not yet been stablized. type ProcessEdgesWorkType: ProcessEdgesWork; + type ImmovableProcessEdges: ProcessEdgesWork; } diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index 8569469b2e..0a877fd9ed 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -232,6 +232,8 @@ pub enum WorkBucketStage { /// Clear the VO bit metadata. Mainly used by ImmixSpace. #[cfg(feature = "vo_bit")] ClearVOBits, + /// Compute the transtive closure following only strong references, making sure that objects do not move + ImmovableClosure, /// Compute the transtive closure following only strong references. Closure, /// Handle Java-style soft references, and potentially expand the transitive closure. diff --git a/src/util/finalizable_processor.rs b/src/util/finalizable_processor.rs index 72545083a8..13ad0e0d1a 100644 --- a/src/util/finalizable_processor.rs +++ b/src/util/finalizable_processor.rs @@ -1,6 +1,6 @@ use crate::plan::is_nursery_gc; use crate::scheduler::gc_work::ProcessEdgesWork; -use crate::scheduler::{GCWork, GCWorker}; +use crate::scheduler::{GCWork, GCWorker, WorkBucketStage}; use crate::util::ObjectReference; use crate::util::VMWorkerThread; use crate::vm::Finalizable; @@ -147,7 +147,7 @@ impl GCWork for Finalization { finalizable_processor.ready_for_finalize.len() ); - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::FinalRefClosure); w.set_worker(worker); finalizable_processor.scan(worker.tls, &mut w, is_nursery_gc(&*mmtk.plan)); debug!( @@ -170,7 +170,7 @@ impl GCWork for ForwardFinalization { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("Forward finalization"); let mut finalizable_processor = mmtk.finalizable_processor.lock().unwrap(); - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::FinalizableForwarding); w.set_worker(worker); finalizable_processor.forward_candidate(&mut w, is_nursery_gc(&*mmtk.plan)); diff --git a/src/util/reference_processor.rs b/src/util/reference_processor.rs index 219cbba0d8..ea37c93c68 100644 --- a/src/util/reference_processor.rs +++ b/src/util/reference_processor.rs @@ -6,6 +6,7 @@ use std::vec::Vec; use crate::plan::is_nursery_gc; use crate::scheduler::ProcessEdgesWork; +use crate::scheduler::WorkBucketStage; use crate::util::ObjectReference; use crate::util::VMWorkerThread; use crate::vm::ReferenceGlue; @@ -478,7 +479,7 @@ use std::marker::PhantomData; pub struct SoftRefProcessing(PhantomData); impl GCWork for SoftRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::SoftRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_soft_refs(&mut w, mmtk); w.flush(); @@ -494,7 +495,7 @@ impl SoftRefProcessing { pub struct WeakRefProcessing(PhantomData); impl GCWork for WeakRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::WeakRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_weak_refs(&mut w, mmtk); w.flush(); @@ -510,7 +511,7 @@ impl WeakRefProcessing { pub struct PhantomRefProcessing(PhantomData); impl GCWork for PhantomRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::PhantomRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_phantom_refs(&mut w, mmtk); w.flush(); @@ -526,7 +527,7 @@ impl PhantomRefProcessing { pub struct RefForwarding(PhantomData); impl GCWork for RefForwarding { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::RefForwarding); w.set_worker(worker); mmtk.reference_processors.forward_refs(&mut w, mmtk); w.flush(); diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 3a262f7542..bcd3d87f02 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -103,7 +103,8 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `edges`: A vector of edges. - fn create_process_edge_roots_work(&mut self, edges: Vec); + /// * `movable`: Indicates whether the trace from the roots can transitively move objects + fn create_process_edge_roots_work(&mut self, edges: Vec, movable: bool); /// Create work packets to handle nodes pointed by root edges. /// @@ -116,7 +117,8 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_node_roots_work(&mut self, nodes: Vec); + /// * `movable`: Indicates whether the trace from the roots can transitively move objects + fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool); } /// VM-specific methods for scanning roots/objects. From f4cb3380e72ad3ef6aedda85dc81bcf53f326249 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 9 Aug 2023 04:53:48 +0000 Subject: [PATCH 02/25] Removing print to file statements --- src/scheduler/gc_work.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index edf3118f6c..9c904ef9c6 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -977,19 +977,6 @@ impl> ScanObjectsWork for ScanOb } fn post_scan_object(&self, _object: ObjectReference) { - // use std::fs::OpenOptions; - // use std::io::Write; - - // let mut file = OpenOptions::new() - // .write(true) - // .append(true) - // .create(true) - // .open("/home/eduardo/mmtk-julia/scanned_objs.log") - // .unwrap(); - - // if let Err(e) = writeln!(file, "Processed object {} at bucket {:?}", object, self.bucket) { - // eprintln!("Couldn't write to file: {}", e); - // } // Do nothing. } @@ -1129,19 +1116,6 @@ impl + PlanTraceObject> ScanObje } fn post_scan_object(&self, object: ObjectReference) { - // use std::fs::OpenOptions; - // use std::io::Write; - - // let mut file = OpenOptions::new() - // .write(true) - // .append(true) - // .create(true) - // .open("/home/eduardo/mmtk-julia/scanned_objs.log") - // .unwrap(); - - // if let Err(e) = writeln!(file, "Processed object {} at bucket {:?}", object, self.bucket) { - // eprintln!("Couldn't write to file: {}", e); - // } self.plan.post_scan_object(object); } From fa68296accaf520eb00858fa748a96850b5810c8 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 03:52:57 +0000 Subject: [PATCH 03/25] Fixing incompatibilities from other plans to support immovable trace --- src/plan/generational/copying/gc_work.rs | 4 +- src/plan/generational/immix/gc_work.rs | 3 + src/plan/generational/mod.rs | 4 +- src/plan/global.rs | 90 ++++----- src/plan/immix/gc_work.rs | 2 +- src/plan/markcompact/gc_work.rs | 16 +- src/plan/markcompact/global.rs | 4 +- src/plan/marksweep/gc_work.rs | 1 + src/plan/mod.rs | 30 ++- src/plan/pageprotect/gc_work.rs | 3 +- src/plan/semispace/gc_work.rs | 3 +- src/plan/sticky/immix/gc_work.rs | 3 + src/policy/copyspace.rs | 5 +- src/policy/immix/immixspace.rs | 2 +- src/policy/markcompactspace.rs | 5 +- src/scheduler/gc_work.rs | 243 ++++++++++++----------- src/scheduler/scheduler.rs | 3 +- src/vm/scanning.rs | 23 ++- 18 files changed, 241 insertions(+), 203 deletions(-) diff --git a/src/plan/generational/copying/gc_work.rs b/src/plan/generational/copying/gc_work.rs index e94b43d71b..04b1cf5095 100644 --- a/src/plan/generational/copying/gc_work.rs +++ b/src/plan/generational/copying/gc_work.rs @@ -3,13 +3,14 @@ use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::vm::*; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; pub struct GenCopyNurseryGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkContext { type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type ImmovableProcessEdges = UnsupportedProcessEdges; } pub struct GenCopyGCWorkContext(std::marker::PhantomData); @@ -17,4 +18,5 @@ impl crate::scheduler::GCWorkContext for GenCopyGCWorkContext type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type ImmovableProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/generational/immix/gc_work.rs b/src/plan/generational/immix/gc_work.rs index ea6b220935..af3d425f17 100644 --- a/src/plan/generational/immix/gc_work.rs +++ b/src/plan/generational/immix/gc_work.rs @@ -1,6 +1,7 @@ use super::global::GenImmix; use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; @@ -9,6 +10,7 @@ impl crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkCon type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type ImmovableProcessEdges = GenNurseryProcessEdges; } pub(super) struct GenImmixMatureGCWorkContext( @@ -20,4 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; + type ImmovableProcessEdges = PlanProcessEdges, TRACE_KIND_IMMOVABLE>; } diff --git a/src/plan/generational/mod.rs b/src/plan/generational/mod.rs index 412537d3ba..571e5672c1 100644 --- a/src/plan/generational/mod.rs +++ b/src/plan/generational/mod.rs @@ -22,9 +22,9 @@ use super::mutator_context::ReservedAllocators; pub mod barrier; /// Generational copying (GenCopy) -// pub mod copying; +pub mod copying; // /// Generational immix (GenImmix) -// pub mod immix; +pub mod immix; // Common generational code diff --git a/src/plan/global.rs b/src/plan/global.rs index df104020bc..5d9f26ca33 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -40,30 +40,29 @@ pub fn create_mutator( mmtk: &'static MMTK, ) -> Box> { Box::new(match *mmtk.options.plan { - // PlanSelector::NoGC => crate::plan::nogc::mutator::create_nogc_mutator(tls, &*mmtk.plan), - // PlanSelector::SemiSpace => { - // crate::plan::semispace::mutator::create_ss_mutator(tls, &*mmtk.plan) - // } - // PlanSelector::GenCopy => { - // crate::plan::generational::copying::mutator::create_gencopy_mutator(tls, mmtk) - // } - // PlanSelector::GenImmix => { - // crate::plan::generational::immix::mutator::create_genimmix_mutator(tls, mmtk) - // } - // PlanSelector::MarkSweep => { - // crate::plan::marksweep::mutator::create_ms_mutator(tls, &*mmtk.plan) - // } + PlanSelector::NoGC => crate::plan::nogc::mutator::create_nogc_mutator(tls, &*mmtk.plan), + PlanSelector::SemiSpace => { + crate::plan::semispace::mutator::create_ss_mutator(tls, &*mmtk.plan) + } + PlanSelector::GenCopy => { + crate::plan::generational::copying::mutator::create_gencopy_mutator(tls, mmtk) + } + PlanSelector::GenImmix => { + crate::plan::generational::immix::mutator::create_genimmix_mutator(tls, mmtk) + } + PlanSelector::MarkSweep => { + crate::plan::marksweep::mutator::create_ms_mutator(tls, &*mmtk.plan) + } PlanSelector::Immix => crate::plan::immix::mutator::create_immix_mutator(tls, &*mmtk.plan), - // PlanSelector::PageProtect => { - // crate::plan::pageprotect::mutator::create_pp_mutator(tls, &*mmtk.plan) - // } - // PlanSelector::MarkCompact => { - // crate::plan::markcompact::mutator::create_markcompact_mutator(tls, &*mmtk.plan) - // } - // PlanSelector::StickyImmix => { - // crate::plan::sticky::immix::mutator::create_stickyimmix_mutator(tls, mmtk) - // } - _ => { todo!() }, + PlanSelector::PageProtect => { + crate::plan::pageprotect::mutator::create_pp_mutator(tls, &*mmtk.plan) + } + PlanSelector::MarkCompact => { + crate::plan::markcompact::mutator::create_markcompact_mutator(tls, &*mmtk.plan) + } + PlanSelector::StickyImmix => { + crate::plan::sticky::immix::mutator::create_stickyimmix_mutator(tls, mmtk) + } }) } @@ -85,32 +84,31 @@ pub fn create_plan( let gc_trigger = args.gc_trigger.clone(); let plan = match plan { - // PlanSelector::NoGC => { - // Box::new(crate::plan::nogc::NoGC::new(args)) as Box> - // } - // PlanSelector::SemiSpace => { - // Box::new(crate::plan::semispace::SemiSpace::new(args)) as Box> - // } - // PlanSelector::GenCopy => Box::new(crate::plan::generational::copying::GenCopy::new(args)) - // as Box>, - // PlanSelector::GenImmix => Box::new(crate::plan::generational::immix::GenImmix::new(args)) - // as Box>, - // PlanSelector::MarkSweep => { - // Box::new(crate::plan::marksweep::MarkSweep::new(args)) as Box> - // } + PlanSelector::NoGC => { + Box::new(crate::plan::nogc::NoGC::new(args)) as Box> + } + PlanSelector::SemiSpace => { + Box::new(crate::plan::semispace::SemiSpace::new(args)) as Box> + } + PlanSelector::GenCopy => Box::new(crate::plan::generational::copying::GenCopy::new(args)) + as Box>, + PlanSelector::GenImmix => Box::new(crate::plan::generational::immix::GenImmix::new(args)) + as Box>, + PlanSelector::MarkSweep => { + Box::new(crate::plan::marksweep::MarkSweep::new(args)) as Box> + } PlanSelector::Immix => { Box::new(crate::plan::immix::Immix::new(args)) as Box> } - // PlanSelector::PageProtect => { - // Box::new(crate::plan::pageprotect::PageProtect::new(args)) as Box> - // } - // PlanSelector::MarkCompact => { - // Box::new(crate::plan::markcompact::MarkCompact::new(args)) as Box> - // } - // PlanSelector::StickyImmix => { - // Box::new(crate::plan::sticky::immix::StickyImmix::new(args)) as Box> - // } - _ => { todo!() } + PlanSelector::PageProtect => { + Box::new(crate::plan::pageprotect::PageProtect::new(args)) as Box> + } + PlanSelector::MarkCompact => { + Box::new(crate::plan::markcompact::MarkCompact::new(args)) as Box> + } + PlanSelector::StickyImmix => { + Box::new(crate::plan::sticky::immix::StickyImmix::new(args)) as Box> + } }; // We have created Plan in the heap, and we won't explicitly move it. diff --git a/src/plan/immix/gc_work.rs b/src/plan/immix/gc_work.rs index 33e225f239..f39374e567 100644 --- a/src/plan/immix/gc_work.rs +++ b/src/plan/immix/gc_work.rs @@ -1,8 +1,8 @@ use super::global::Immix; use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; -use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; pub(super) struct ImmixGCWorkContext( std::marker::PhantomData, diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index 22c615589b..8fdbc1918c 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -53,12 +53,13 @@ impl GCWork for UpdateReferences { // scheduler.work_buckets[WorkBucketStage::RefForwarding] // .add(ScanStackRoots::>::new()); for mutator in VM::VMActivePlan::mutators() { - mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] - .add(ScanStackRoot::>(mutator)); + mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots].add(ScanStackRoot::< + MarkCompactForwardingGCWorkContext, + >(mutator)); } mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] - .add(ScanVMSpecificRoots::>::new()); + .add(ScanVMSpecificRoots::>::new()); } } @@ -95,4 +96,13 @@ impl crate::scheduler::GCWorkContext for MarkCompactGCWorkContext type VM = VM; type PlanType = MarkCompact; type ProcessEdgesWorkType = MarkingProcessEdges; + type ImmovableProcessEdges = UnsupportedProcessEdges; +} + +pub struct MarkCompactForwardingGCWorkContext(std::marker::PhantomData); +impl crate::scheduler::GCWorkContext for MarkCompactForwardingGCWorkContext { + type VM = VM; + type PlanType = MarkCompact; + type ProcessEdgesWorkType = ForwardingProcessEdges; + type ImmovableProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/markcompact/global.rs b/src/plan/markcompact/global.rs index 96117e6a39..6bd5f300ff 100644 --- a/src/plan/markcompact/global.rs +++ b/src/plan/markcompact/global.rs @@ -1,8 +1,8 @@ -use super::gc_work::MarkCompactGCWorkContext; use super::gc_work::{ CalculateForwardingAddress, Compact, ForwardingProcessEdges, MarkingProcessEdges, UpdateReferences, }; +use super::gc_work::{MarkCompactForwardingGCWorkContext, MarkCompactGCWorkContext}; use crate::plan::global::CommonPlan; use crate::plan::global::GcStatus; use crate::plan::global::{BasePlan, CreateGeneralPlanArgs, CreateSpecificPlanArgs}; @@ -97,7 +97,7 @@ impl Plan for MarkCompact { // Stop & scan mutators (mutator scanning can happen before STW) scheduler.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::>::new()); + .add(StopMutators::>::new()); // Prepare global/collectors/mutators scheduler.work_buckets[WorkBucketStage::Prepare] diff --git a/src/plan/marksweep/gc_work.rs b/src/plan/marksweep/gc_work.rs index 90ed731dd6..96c325c695 100644 --- a/src/plan/marksweep/gc_work.rs +++ b/src/plan/marksweep/gc_work.rs @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for MSGCWorkContext { type VM = VM; type PlanType = MarkSweep; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type ImmovableProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/mod.rs b/src/plan/mod.rs index afcfce513d..6e93d7d919 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -43,14 +43,14 @@ pub use tracing::{ObjectQueue, ObjectsClosure, VectorObjectQueue, VectorQueue}; // /// Generational plans (with a copying nursery) mod generational; // /// Sticky plans (using sticky marks for generational behaviors without a copying nursery) -// mod sticky; +mod sticky; mod immix; -// mod markcompact; -// mod marksweep; -// mod nogc; -// mod pageprotect; -// mod semispace; +mod markcompact; +mod marksweep; +mod nogc; +mod pageprotect; +mod semispace; pub(crate) use generational::global::is_nursery_gc; pub(crate) use generational::global::GenerationalPlan; @@ -58,14 +58,12 @@ pub(crate) use generational::global::GenerationalPlan; // Expose plan constraints as public. Though a binding can get them from plan.constraints(), // it is possible for performance reasons that they want the constraints as constants. -// pub use generational::copying::GENCOPY_CONSTRAINTS; -// pub use generational::immix::GENIMMIX_CONSTRAINTS; +pub use generational::copying::GENCOPY_CONSTRAINTS; +pub use generational::immix::GENIMMIX_CONSTRAINTS; pub use immix::IMMIX_CONSTRAINTS; - -// use crate::vm::VMBinding; -// pub use markcompact::MARKCOMPACT_CONSTRAINTS; -// pub use marksweep::MS_CONSTRAINTS; -// pub use nogc::NOGC_CONSTRAINTS; -// pub use pageprotect::PP_CONSTRAINTS; -// pub use semispace::SS_CONSTRAINTS; -// pub use sticky::immix::STICKY_IMMIX_CONSTRAINTS; +pub use markcompact::MARKCOMPACT_CONSTRAINTS; +pub use marksweep::MS_CONSTRAINTS; +pub use nogc::NOGC_CONSTRAINTS; +pub use pageprotect::PP_CONSTRAINTS; +pub use semispace::SS_CONSTRAINTS; +pub use sticky::immix::STICKY_IMMIX_CONSTRAINTS; diff --git a/src/plan/pageprotect/gc_work.rs b/src/plan/pageprotect/gc_work.rs index c962eae190..f2cd389f75 100644 --- a/src/plan/pageprotect/gc_work.rs +++ b/src/plan/pageprotect/gc_work.rs @@ -1,6 +1,6 @@ use super::global::PageProtect; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; use crate::vm::VMBinding; pub struct PPGCWorkContext(std::marker::PhantomData); @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for PPGCWorkContext { type VM = VM; type PlanType = PageProtect; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type ImmovableProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/semispace/gc_work.rs b/src/plan/semispace/gc_work.rs index 9f02c73eea..cbf0b75e3c 100644 --- a/src/plan/semispace/gc_work.rs +++ b/src/plan/semispace/gc_work.rs @@ -1,6 +1,6 @@ use super::global::SemiSpace; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; use crate::vm::VMBinding; pub struct SSGCWorkContext(std::marker::PhantomData); @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for SSGCWorkContext { type VM = VM; type PlanType = SemiSpace; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type ImmovableProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/sticky/immix/gc_work.rs b/src/plan/sticky/immix/gc_work.rs index 0afdc44ee0..dc50345198 100644 --- a/src/plan/sticky/immix/gc_work.rs +++ b/src/plan/sticky/immix/gc_work.rs @@ -1,4 +1,5 @@ use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; use crate::scheduler::gc_work::PlanProcessEdges; use crate::{plan::generational::gc_work::GenNurseryProcessEdges, vm::VMBinding}; @@ -9,6 +10,7 @@ impl crate::scheduler::GCWorkContext for StickyImmixNurseryGCWork type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type ImmovableProcessEdges = GenNurseryProcessEdges; } pub struct StickyImmixMatureGCWorkContext( @@ -20,4 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = PlanProcessEdges; + type ImmovableProcessEdges = PlanProcessEdges; } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 87d3ccb16b..e97a663959 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -127,7 +127,10 @@ impl crate::policy::gc_work::PolicyTraceObject for CopySpace< copy: Option, worker: &mut GCWorker, ) -> ObjectReference { - assert!(KIND != TRACE_KIND_IMMOVABLE, "Copyspace does not support immovable trace."); + assert!( + KIND != TRACE_KIND_IMMOVABLE, + "Copyspace does not support immovable trace." + ); self.trace_object(queue, object, copy, worker) } diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index f6735ca422..5010198572 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -228,7 +228,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace fn may_move_objects() -> bool { if KIND == TRACE_KIND_DEFRAG { true - } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_IMMOVABLE{ + } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_IMMOVABLE { false } else { unreachable!() diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 8f08899dba..28ed7c5f47 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -128,7 +128,10 @@ impl crate::policy::gc_work::PolicyTraceObject for MarkCompac _copy: Option, _worker: &mut GCWorker, ) -> ObjectReference { - assert!(KIND != TRACE_KIND_IMMOVABLE, "MarkCompact does not support immovable trace."); + assert!( + KIND != TRACE_KIND_IMMOVABLE, + "MarkCompact does not support immovable trace." + ); if KIND == TRACE_KIND_MARK { self.trace_mark_object(queue, object) } else if KIND == TRACE_KIND_FORWARD { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 2695fef070..adb1972e28 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -477,7 +477,11 @@ impl ScanStackRoots { impl GCWork for ScanStackRoots { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStackRoots"); - let factory = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + let factory = ProcessEdgesWorkRootsWorkFactory::< + C::VM, + C::ProcessEdgesWorkType, + C::ImmovableProcessEdges, + >::new(mmtk); ::VMScanning::scan_roots_in_all_mutator_threads(worker.tls, factory); ::VMScanning::notify_initial_thread_scan_complete(false, worker.tls); for mutator in ::VMActivePlan::mutators() { @@ -494,7 +498,11 @@ impl GCWork for ScanStackRoot { trace!("ScanStackRoot for mutator {:?}", self.0.get_tls()); let base = &mmtk.plan.base(); let mutators = ::VMActivePlan::number_of_mutators(); - let factory = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + let factory = ProcessEdgesWorkRootsWorkFactory::< + C::VM, + C::ProcessEdgesWorkType, + C::ImmovableProcessEdges, + >::new(mmtk); ::VMScanning::scan_roots_in_mutator_thread( worker.tls, unsafe { &mut *(self.0 as *mut _) }, @@ -524,7 +532,7 @@ impl GCWork for ScanVMSpecificRoots fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStaticRoots"); let factory // = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - = ImmovableProcessEdgesWorkRootsWorkFactory::::new(mmtk); + = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } } @@ -749,43 +757,7 @@ impl ProcessEdgesWork for SFTProcessEdges { ScanObjects::::new(nodes, false, roots, self.bucket) } } - -struct ProcessEdgesWorkRootsWorkFactory { - mmtk: &'static MMTK, -} - -impl Clone for ProcessEdgesWorkRootsWorkFactory { - fn clone(&self) -> Self { - Self { mmtk: self.mmtk } - } -} - -impl RootsWorkFactory> for ProcessEdgesWorkRootsWorkFactory { - fn create_process_edge_roots_work(&mut self, edges: Vec>, movable: bool) { - debug_assert!(movable); - crate::memory_manager::add_work_packet( - self.mmtk, - WorkBucketStage::Closure, - E::new(edges, true, self.mmtk, WorkBucketStage::Closure), - ); - } - - fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool) { - // We want to use E::create_scan_work. - debug_assert!(movable); - let process_edges_work = E::new(vec![], true, self.mmtk, WorkBucketStage::Closure); - let work = process_edges_work.create_scan_work(nodes, true); - crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, work); - } -} - -impl ProcessEdgesWorkRootsWorkFactory { - fn new(mmtk: &'static MMTK) -> Self { - Self { mmtk } - } -} - -struct ImmovableProcessEdgesWorkRootsWorkFactory< +struct ProcessEdgesWorkRootsWorkFactory< VM: VMBinding, E: ProcessEdgesWork, I: ProcessEdgesWork, @@ -795,7 +767,7 @@ struct ImmovableProcessEdgesWorkRootsWorkFactory< } impl, I: ProcessEdgesWork> Clone - for ImmovableProcessEdgesWorkRootsWorkFactory + for ProcessEdgesWorkRootsWorkFactory { fn clone(&self) -> Self { Self { @@ -806,40 +778,49 @@ impl, I: ProcessEdgesWork> } impl, I: ProcessEdgesWork> - RootsWorkFactory> for ImmovableProcessEdgesWorkRootsWorkFactory + RootsWorkFactory> for ProcessEdgesWorkRootsWorkFactory { - fn create_process_edge_roots_work(&mut self, edges: Vec>, movable: bool) { - if movable { - crate::memory_manager::add_work_packet( - self.mmtk, - WorkBucketStage::Closure, - E::new(edges, true, self.mmtk, WorkBucketStage::Closure), - ); - } else { - crate::memory_manager::add_work_packet( - self.mmtk, - WorkBucketStage::ImmovableClosure, - I::new(edges, true, self.mmtk, WorkBucketStage::ImmovableClosure), - ); - } + fn create_process_edge_roots_work(&mut self, edges: Vec>) { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::Closure, + E::new(edges, true, self.mmtk, WorkBucketStage::Closure), + ); } - fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool) { - // We want to use E::create_scan_work. - if movable { - // let process_edges_work = E::new(vec![], true, self.mmtk, WorkBucketStage::Closure); - // let work = process_edges_work.create_scan_root_node_work(nodes, true); - - crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, ProcessRootNode::::new(nodes, WorkBucketStage::Closure)); - } else { - crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::ImmovableClosure, ProcessRootNode::::new(nodes, WorkBucketStage::ImmovableClosure)); - } + fn create_process_node_roots_work(&mut self, nodes: Vec) { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::Closure, + ProcessRootNode::::new(nodes, WorkBucketStage::Closure), + ); + } + + fn create_immovable_process_edge_roots_work(&mut self, edges: Vec>) { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::ImmovableClosure, + I::new(edges, true, self.mmtk, WorkBucketStage::ImmovableClosure), + ); + } + + fn create_immovable_process_node_roots_work(&mut self, nodes: Vec) { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::ImmovableClosure, + ProcessRootNode::::new(nodes, WorkBucketStage::ImmovableClosure), + ); } } -impl, I: ProcessEdgesWork> ImmovableProcessEdgesWorkRootsWorkFactory { +impl, I: ProcessEdgesWork> + ProcessEdgesWorkRootsWorkFactory +{ fn new(mmtk: &'static MMTK) -> Self { - Self { mmtk, phantom : PhantomData } + Self { + mmtk, + phantom: PhantomData, + } } } @@ -879,52 +860,13 @@ pub trait ScanObjectsWork: GCWork + Sized { &self, buffer: &[ObjectReference], worker: &mut GCWorker<::VM>, - mmtk: &'static MMTK<::VM>, + _mmtk: &'static MMTK<::VM>, ) { let tls = worker.tls; assert!(!self.roots()); - #[cfg(feature = "sanity")] - { - if self.roots() && !mmtk.plan.is_in_sanity() { - mmtk.sanity_checker - .lock() - .unwrap() - .add_root_nodes(buffer.to_vec()); - } - } - - // If this is a root packet, the objects in this packet will have not been traced, yet. - // - // This step conceptually traces the edges from root slots to the objects they point to. - // However, VMs that deliver root objects instead of root edges are incapable of updating - // root slots. Like processing an edge, we call `trace_object` on those objects, and - // assert the GC doesn't move those objects because we cannot store back to the slots. - // - // If this is a root packet, the `scanned_root_objects` variable will hold those root - // objects which are traced for the first time. - let scanned_root_objects = self.roots().then(|| { - // We create an instance of E to use its `trace_object` method and its object queue. - let mut process_edges_work = Self::E::new(vec![], false, mmtk, self.get_bucket()); - process_edges_work.set_worker(worker); - - for object in buffer.iter().copied() { - let new_object = process_edges_work.trace_object(object); - debug_assert_eq!( - object, new_object, - "Object moved while tracing root unmovable root object: {} -> {}", - object, new_object - ); - } - - // This contains root objects that are visited the first time. - // It is sufficient to only scan these objects. - process_edges_work.nodes.take() - }); - - // If it is a root packet, scan the nodes that are first scanned; - // otherwise, scan the nodes in the buffer. - let objects_to_scan = scanned_root_objects.as_deref().unwrap_or(buffer); + // Scan the nodes in the buffer. + let objects_to_scan = buffer; // Then scan those objects for edges. let mut scan_later = vec![]; @@ -1180,23 +1122,28 @@ impl + PlanTraceObject> GCWork, E2: ProcessEdgesWork> { - phantom : PhantomData<(VM, E1, E2)>, - roots : Vec, - bucket: WorkBucketStage +struct ProcessRootNode, E2: ProcessEdgesWork> +{ + phantom: PhantomData<(VM, E1, E2)>, + roots: Vec, + bucket: WorkBucketStage, } -impl, E2: ProcessEdgesWork> ProcessRootNode { - pub fn new(nodes : Vec, bucket: WorkBucketStage) -> Self { +impl, E2: ProcessEdgesWork> + ProcessRootNode +{ + pub fn new(nodes: Vec, bucket: WorkBucketStage) -> Self { Self { - phantom : PhantomData, - roots : nodes, - bucket + phantom: PhantomData, + roots: nodes, + bucket, } } } -impl, E2: ProcessEdgesWork> GCWork for ProcessRootNode { +impl, E2: ProcessEdgesWork> GCWork + for ProcessRootNode +{ fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ProcessRootNode"); @@ -1210,6 +1157,15 @@ impl, E2: ProcessEdgesWork> GC } } + // Because this is a root packet, the objects in this packet will have not been traced, yet. + // + // This step conceptually traces the edges from root slots to the objects they point to. + // However, VMs that deliver root objects instead of root edges are incapable of updating + // root slots. Like processing an edge, we call `trace_object` on those objects, and + // assert the GC doesn't move those objects because we cannot store back to the slots. + // + // The `scanned_root_objects` variable will hold those root + // objects which are traced for the first time and we create work for scanning those roots. let scanned_root_objects = { // We create an instance of E to use its `trace_object` method and its object queue. let mut process_edges_work = E1::new(vec![], true, mmtk, self.bucket); @@ -1233,7 +1189,52 @@ impl, E2: ProcessEdgesWork> GC let work = process_edges_work.create_scan_work(scanned_root_objects, false); crate::memory_manager::add_work_packet(mmtk, self.bucket, work); - trace!("ProcessRootNode End"); } } + +// // create UnsupportedProcessEdge +#[derive(Default)] +pub struct UnsupportedProcessEdges { + phantom: PhantomData, +} + +impl Deref for UnsupportedProcessEdges { + type Target = ProcessEdgesBase; + fn deref(&self) -> &Self::Target { + panic!("unsupported!") + } +} + +impl DerefMut for UnsupportedProcessEdges { + fn deref_mut(&mut self) -> &mut Self::Target { + panic!("unsupported!") + } +} + +impl ProcessEdgesWork for UnsupportedProcessEdges { + type VM = VM; + + type ScanObjectsWorkType = ScanObjects; + + fn new( + _edges: Vec>, + _roots: bool, + _mmtk: &'static MMTK, + _bucket: WorkBucketStage, + ) -> Self { + panic!("unsupported!") + } + + fn trace_object(&mut self, _object: ObjectReference) -> ObjectReference { + panic!("unsupported!") + } + + fn create_scan_work( + &self, + _nodes: Vec, + _roots: bool, + ) -> Self::ScanObjectsWorkType { + panic!("unsupported!") + } +} diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index e210786c69..ca5ec47588 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -115,8 +115,7 @@ impl GCWorkScheduler { use crate::plan::Plan; use crate::scheduler::gc_work::*; // Stop & scan mutators (mutator scanning can happen before STW) - self.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::::new()); + self.work_buckets[WorkBucketStage::Unconstrained].add(StopMutators::::new()); // Prepare global/collectors/mutators self.work_buckets[WorkBucketStage::Prepare].add(Prepare::::new(plan)); diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index bcd3d87f02..e64cb8dfc3 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -103,8 +103,7 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `edges`: A vector of edges. - /// * `movable`: Indicates whether the trace from the roots can transitively move objects - fn create_process_edge_roots_work(&mut self, edges: Vec, movable: bool); + fn create_process_edge_roots_work(&mut self, edges: Vec); /// Create work packets to handle nodes pointed by root edges. /// @@ -117,8 +116,24 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - /// * `movable`: Indicates whether the trace from the roots can transitively move objects - fn create_process_node_roots_work(&mut self, nodes: Vec, movable: bool); + fn create_process_node_roots_work(&mut self, nodes: Vec); + + /// Create work packets to handle root edges. + /// + /// The work packet must not update the edges transitively found from any of the root edges. + /// + /// Arguments: + /// * `edges`: A vector of edges. + fn create_immovable_process_edge_roots_work(&mut self, edges: Vec); + + /// Create work packets to handle nodes pointed by root edges. + /// + /// Similar to `create_process_node_roots_work`, this work packet won't move root objects, but also will + /// not move any object in their transitive closure. + /// + /// Arguments: + /// * `nodes`: A vector of references to objects pointed by root edges. + fn create_immovable_process_node_roots_work(&mut self, nodes: Vec); } /// VM-specific methods for scanning roots/objects. From b1db9788ef864f661ffa39288297bfdb998d55a7 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 04:51:37 +0000 Subject: [PATCH 04/25] Adding option for not moving during nursery GCs for sticky immix --- Cargo.toml | 4 ++++ src/policy/immix/mod.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 204027bf7c..b0c08aa833 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,10 @@ object_pinning = [] # Disable any object copying in Immix. This makes Immix a non-moving policy. immix_non_moving = [] +# Disable any object copying in nursery GC for Sticky Immix. +sticky_immix_non_moving_nursery = [] + + # Reduce block size for ImmixSpace. This mitigates fragmentation when defrag is disabled. immix_smaller_block = [] # Zero the unmarked lines after a GC cycle in immix. This helps debug untraced objects. diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index fb96053c07..76848c15a6 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -25,7 +25,7 @@ pub const STRESS_DEFRAG: bool = true; pub const DEFRAG_EVERY_BLOCK: bool = true; /// If Immix is used as a nursery space, do we prefer copy? -pub const PREFER_COPY_ON_NURSERY_GC: bool = !cfg!(feature = "immix_non_moving"); // copy nursery objects if we are allowed to move. +pub const PREFER_COPY_ON_NURSERY_GC: bool = !cfg!(feature = "immix_non_moving") || !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. /// In some cases/settings, Immix may never move objects. /// Currently we only have two cases where we move objects: 1. defrag, 2. nursery copy. From a349cc7e9ca116ea1d1eb2ae8ceaca54b12e6ec0 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 04:51:58 +0000 Subject: [PATCH 05/25] Minor. --- src/policy/immix/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index 76848c15a6..81482d5bbd 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -25,7 +25,8 @@ pub const STRESS_DEFRAG: bool = true; pub const DEFRAG_EVERY_BLOCK: bool = true; /// If Immix is used as a nursery space, do we prefer copy? -pub const PREFER_COPY_ON_NURSERY_GC: bool = !cfg!(feature = "immix_non_moving") || !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. +pub const PREFER_COPY_ON_NURSERY_GC: bool = + !cfg!(feature = "immix_non_moving") || !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. /// In some cases/settings, Immix may never move objects. /// Currently we only have two cases where we move objects: 1. defrag, 2. nursery copy. From b1b31cf6f08fcb06deacdcdc93b5031dc0d21871 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 04:56:17 +0000 Subject: [PATCH 06/25] Fixing formatting and tests --- src/scheduler/gc_work.rs | 7 +++++-- tests/test_roots_work_factory.rs | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index adb1972e28..ba9323c4a6 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -531,8 +531,11 @@ impl ScanVMSpecificRoots { impl GCWork for ScanVMSpecificRoots { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStaticRoots"); - let factory // = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); + let factory = ProcessEdgesWorkRootsWorkFactory::< + C::VM, + C::ProcessEdgesWorkType, + C::ImmovableProcessEdges, + >::new(mmtk); ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } } diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index 15a95ca227..3338251954 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -69,6 +69,14 @@ impl RootsWorkFactory

for MockFactory { fn create_process_node_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } + + fn create_immovable_process_edge_roots_work(&mut self, edges: Vec
) { + unimplemented!(); + } + + fn create_immovable_process_node_roots_work(&mut self, nodes: Vec) { + unimplemented!(); + } } #[test] From 2089e0a2adaa4eeb60d9adf57d1c74faeb6f0546 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 05:30:06 +0000 Subject: [PATCH 07/25] Fixing sanity GC --- docs/tutorial/code/mygc_semispace/gc_work.rs | 2 +- src/scheduler/gc_work.rs | 2 +- src/util/sanity/sanity_checker.rs | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/tutorial/code/mygc_semispace/gc_work.rs b/docs/tutorial/code/mygc_semispace/gc_work.rs index 3eee9db448..c43529ff1b 100644 --- a/docs/tutorial/code/mygc_semispace/gc_work.rs +++ b/docs/tutorial/code/mygc_semispace/gc_work.rs @@ -74,7 +74,7 @@ impl ProcessEdgesWork for MyGCProcessEdges { } fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, self.bucket) } } // ANCHOR_END: mygc_process_edges_impl diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 180af1cb07..b8fac1ee10 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -1108,7 +1108,7 @@ impl, E2: ProcessEdgesWork mmtk.sanity_checker .lock() .unwrap() - .add_root_nodes(self.roots.copied()); + .add_root_nodes(self.roots.clone()); } } diff --git a/src/util/sanity/sanity_checker.rs b/src/util/sanity/sanity_checker.rs index 4cb0a19a88..1fd03f1790 100644 --- a/src/util/sanity/sanity_checker.rs +++ b/src/util/sanity/sanity_checker.rs @@ -89,14 +89,14 @@ impl GCWork for ScheduleSanityGC

{ let sanity_checker = mmtk.sanity_checker.lock().unwrap(); for roots in &sanity_checker.root_edges { scheduler.work_buckets[WorkBucketStage::Closure].add( - SanityGCProcessEdges::::new(roots.clone(), true, mmtk), + SanityGCProcessEdges::::new(roots.clone(), true, mmtk, WorkBucketStage::Closure), ); } for roots in &sanity_checker.root_nodes { scheduler.work_buckets[WorkBucketStage::Closure].add(ScanObjects::< SanityGCProcessEdges, >::new( - roots.clone(), false, true + roots.clone(), false, true, WorkBucketStage::Closure )); } } @@ -187,9 +187,9 @@ impl ProcessEdgesWork for SanityGCProcessEdges { type ScanObjectsWorkType = ScanObjects; const OVERWRITE_REFERENCE: bool = false; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { + fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK, bucket: WorkBucketStage) -> Self { Self { - base: ProcessEdgesBase::new(edges, roots, mmtk), + base: ProcessEdgesBase::new(edges, roots, mmtk, bucket), // ..Default::default() } } @@ -238,6 +238,6 @@ impl ProcessEdgesWork for SanityGCProcessEdges { nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, WorkBucketStage::Closure) } } From a37fb5604cedf0e015782eceeebc824f924a1b2b Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 06:12:18 +0000 Subject: [PATCH 08/25] Fix warning in tests --- tests/test_roots_work_factory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index 3338251954..74fe9cd60c 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -70,11 +70,11 @@ impl RootsWorkFactory

for MockFactory { unimplemented!(); } - fn create_immovable_process_edge_roots_work(&mut self, edges: Vec
) { + fn create_immovable_process_edge_roots_work(&mut self, _edges: Vec
) { unimplemented!(); } - fn create_immovable_process_node_roots_work(&mut self, nodes: Vec) { + fn create_immovable_process_node_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } } From 325d61b831f582bdaf69ce2a9d5476482aedb9c8 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 06:37:48 +0000 Subject: [PATCH 09/25] Applying cargo fmt --- src/plan/markcompact/gc_work.rs | 5 +++-- src/scheduler/gc_work.rs | 4 ++-- src/util/sanity/sanity_checker.rs | 19 ++++++++++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index a6fa13502f..e0fdec4e81 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -50,8 +50,9 @@ impl GCWork for UpdateReferences { .get_and_clear_worker_live_bytes(); for mutator in VM::VMActivePlan::mutators() { - mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] - .add(ScanMutatorRoots::>(mutator)); + mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots].add(ScanMutatorRoots::< + MarkCompactForwardingGCWorkContext, + >(mutator)); } mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index b8fac1ee10..38158ee0b5 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -443,9 +443,9 @@ impl GCWork for VMPostForwarding { } } -pub struct ScanMutatorRoots(pub &'static mut Mutator); +pub struct ScanMutatorRoots(pub &'static mut Mutator); -impl GCWork for ScanMutatorRoots { +impl GCWork for ScanMutatorRoots { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanMutatorRoots for mutator {:?}", self.0.get_tls()); let base = &mmtk.plan.base(); diff --git a/src/util/sanity/sanity_checker.rs b/src/util/sanity/sanity_checker.rs index 1fd03f1790..991fa3865f 100644 --- a/src/util/sanity/sanity_checker.rs +++ b/src/util/sanity/sanity_checker.rs @@ -89,14 +89,22 @@ impl GCWork for ScheduleSanityGC

{ let sanity_checker = mmtk.sanity_checker.lock().unwrap(); for roots in &sanity_checker.root_edges { scheduler.work_buckets[WorkBucketStage::Closure].add( - SanityGCProcessEdges::::new(roots.clone(), true, mmtk, WorkBucketStage::Closure), + SanityGCProcessEdges::::new( + roots.clone(), + true, + mmtk, + WorkBucketStage::Closure, + ), ); } for roots in &sanity_checker.root_nodes { scheduler.work_buckets[WorkBucketStage::Closure].add(ScanObjects::< SanityGCProcessEdges, >::new( - roots.clone(), false, true, WorkBucketStage::Closure + roots.clone(), + false, + true, + WorkBucketStage::Closure, )); } } @@ -187,7 +195,12 @@ impl ProcessEdgesWork for SanityGCProcessEdges { type ScanObjectsWorkType = ScanObjects; const OVERWRITE_REFERENCE: bool = false; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK, bucket: WorkBucketStage) -> Self { + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { Self { base: ProcessEdgesBase::new(edges, roots, mmtk, bucket), // ..Default::default() From b054fd2fbc4b767165102411983dfd6917311503 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 10 Aug 2023 10:50:40 +0000 Subject: [PATCH 10/25] Fixing docs code --- docs/tutorial/code/mygc_semispace/gc_work.rs | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/tutorial/code/mygc_semispace/gc_work.rs b/docs/tutorial/code/mygc_semispace/gc_work.rs index c43529ff1b..7e91ff5ac7 100644 --- a/docs/tutorial/code/mygc_semispace/gc_work.rs +++ b/docs/tutorial/code/mygc_semispace/gc_work.rs @@ -1,6 +1,6 @@ // ANCHOR: imports use super::global::MyGC; -use crate::scheduler::gc_work::*; +use crate::scheduler::{gc_work::*, WorkBucketStage}; use crate::vm::VMBinding; use std::ops::{Deref, DerefMut}; // ANCHOR_END: imports @@ -11,24 +11,26 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = SFTProcessEdges; + type ImmovableProcessEdges = UnsupportedProcessEdges; } // ANCHOR_END: workcontext_sft // ANCHOR: workcontext_plan -use crate::scheduler::gc_work::PlanProcessEdges; use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; pub struct MyGCWorkContext2(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for MyGCWorkContext2 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type ImmovableProcessEdges = UnsupportedProcessEdges; } // ANCHOR: workcontext_plan -use crate::util::ObjectReference; +use crate::policy::space::Space; use crate::util::copy::CopySemantics; +use crate::util::ObjectReference; use crate::MMTK; -use crate::policy::space::Space; // ANCHOR: mygc_process_edges pub struct MyGCProcessEdges { @@ -38,12 +40,17 @@ pub struct MyGCProcessEdges { // ANCHOR_END: mygc_process_edges // ANCHOR: mygc_process_edges_impl -impl ProcessEdgesWork for MyGCProcessEdges { +impl ProcessEdgesWork for MyGCProcessEdges { type VM = VM; type ScanObjectsWorkType = ScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref::>().unwrap(); Self { base, plan } } @@ -100,5 +107,6 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext3 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = MyGCProcessEdges; + type ImmovableProcessEdges = UnsupportedProcessEdges; } // ANCHOR: workcontext_mygc From 407047b76e7248e3f6d5e04e19b7f67ea554d4a4 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 16 Aug 2023 02:00:06 +0000 Subject: [PATCH 11/25] Renaming immovable => transitive pin (TP); Adding NodeRootsTrace to handle node roots that should be kept it place but their children may move --- docs/tutorial/code/mygc_semispace/gc_work.rs | 6 +++--- src/plan/generational/copying/gc_work.rs | 4 ++-- src/plan/generational/immix/gc_work.rs | 6 +++--- src/plan/immix/gc_work.rs | 4 ++-- src/plan/markcompact/gc_work.rs | 4 ++-- src/plan/marksweep/gc_work.rs | 2 +- src/plan/pageprotect/gc_work.rs | 2 +- src/plan/semispace/gc_work.rs | 2 +- src/plan/sticky/immix/gc_work.rs | 6 +++--- src/policy/copyspace.rs | 6 +++--- src/policy/gc_work.rs | 2 +- src/policy/immix/immixspace.rs | 6 +++--- src/policy/markcompactspace.rs | 6 +++--- src/scheduler/gc_work.rs | 22 +++++++++++--------- src/scheduler/work.rs | 2 +- src/scheduler/work_bucket.rs | 7 +++++-- src/vm/scanning.rs | 4 ++-- tests/test_roots_work_factory.rs | 4 ++-- 18 files changed, 50 insertions(+), 45 deletions(-) diff --git a/docs/tutorial/code/mygc_semispace/gc_work.rs b/docs/tutorial/code/mygc_semispace/gc_work.rs index 7e91ff5ac7..1b643f23a6 100644 --- a/docs/tutorial/code/mygc_semispace/gc_work.rs +++ b/docs/tutorial/code/mygc_semispace/gc_work.rs @@ -11,7 +11,7 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = SFTProcessEdges; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR_END: workcontext_sft @@ -23,7 +23,7 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext2 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR: workcontext_plan @@ -107,6 +107,6 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext3 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = MyGCProcessEdges; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR: workcontext_mygc diff --git a/src/plan/generational/copying/gc_work.rs b/src/plan/generational/copying/gc_work.rs index 04b1cf5095..226ddab715 100644 --- a/src/plan/generational/copying/gc_work.rs +++ b/src/plan/generational/copying/gc_work.rs @@ -10,7 +10,7 @@ impl crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkCont type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = GenNurseryProcessEdges; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } pub struct GenCopyGCWorkContext(std::marker::PhantomData); @@ -18,5 +18,5 @@ impl crate::scheduler::GCWorkContext for GenCopyGCWorkContext type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/generational/immix/gc_work.rs b/src/plan/generational/immix/gc_work.rs index af3d425f17..cc15160488 100644 --- a/src/plan/generational/immix/gc_work.rs +++ b/src/plan/generational/immix/gc_work.rs @@ -1,7 +1,7 @@ use super::global::GenImmix; use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::policy::gc_work::TraceKind; -use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; @@ -10,7 +10,7 @@ impl crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkCon type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; - type ImmovableProcessEdges = GenNurseryProcessEdges; + type TPProcessEdges = GenNurseryProcessEdges; } pub(super) struct GenImmixMatureGCWorkContext( @@ -22,5 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; - type ImmovableProcessEdges = PlanProcessEdges, TRACE_KIND_IMMOVABLE>; + type TPProcessEdges = PlanProcessEdges, TRACE_KIND_TRANSITIVE_PIN>; } diff --git a/src/plan/immix/gc_work.rs b/src/plan/immix/gc_work.rs index f39374e567..e3200f8811 100644 --- a/src/plan/immix/gc_work.rs +++ b/src/plan/immix/gc_work.rs @@ -1,6 +1,6 @@ use super::global::Immix; use crate::policy::gc_work::TraceKind; -use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; @@ -13,5 +13,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = Immix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; - type ImmovableProcessEdges = PlanProcessEdges, TRACE_KIND_IMMOVABLE>; + type TPProcessEdges = PlanProcessEdges, TRACE_KIND_TRANSITIVE_PIN>; } diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index e0fdec4e81..6a3d78bf00 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -93,7 +93,7 @@ impl crate::scheduler::GCWorkContext for MarkCompactGCWorkContext type VM = VM; type PlanType = MarkCompact; type ProcessEdgesWorkType = MarkingProcessEdges; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } pub struct MarkCompactForwardingGCWorkContext(std::marker::PhantomData); @@ -101,5 +101,5 @@ impl crate::scheduler::GCWorkContext for MarkCompactForwardingGCW type VM = VM; type PlanType = MarkCompact; type ProcessEdgesWorkType = ForwardingProcessEdges; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/marksweep/gc_work.rs b/src/plan/marksweep/gc_work.rs index 96c325c695..20db8b38c1 100644 --- a/src/plan/marksweep/gc_work.rs +++ b/src/plan/marksweep/gc_work.rs @@ -8,5 +8,5 @@ impl crate::scheduler::GCWorkContext for MSGCWorkContext { type VM = VM; type PlanType = MarkSweep; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/pageprotect/gc_work.rs b/src/plan/pageprotect/gc_work.rs index f2cd389f75..34d4206c47 100644 --- a/src/plan/pageprotect/gc_work.rs +++ b/src/plan/pageprotect/gc_work.rs @@ -8,5 +8,5 @@ impl crate::scheduler::GCWorkContext for PPGCWorkContext { type VM = VM; type PlanType = PageProtect; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/semispace/gc_work.rs b/src/plan/semispace/gc_work.rs index cbf0b75e3c..1d4e633c36 100644 --- a/src/plan/semispace/gc_work.rs +++ b/src/plan/semispace/gc_work.rs @@ -8,5 +8,5 @@ impl crate::scheduler::GCWorkContext for SSGCWorkContext { type VM = VM; type PlanType = SemiSpace; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type ImmovableProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/sticky/immix/gc_work.rs b/src/plan/sticky/immix/gc_work.rs index dc50345198..3c7b6f973a 100644 --- a/src/plan/sticky/immix/gc_work.rs +++ b/src/plan/sticky/immix/gc_work.rs @@ -1,5 +1,5 @@ use crate::policy::gc_work::TraceKind; -use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; use crate::{plan::generational::gc_work::GenNurseryProcessEdges, vm::VMBinding}; @@ -10,7 +10,7 @@ impl crate::scheduler::GCWorkContext for StickyImmixNurseryGCWork type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; - type ImmovableProcessEdges = GenNurseryProcessEdges; + type TPProcessEdges = GenNurseryProcessEdges; } pub struct StickyImmixMatureGCWorkContext( @@ -22,5 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = PlanProcessEdges; - type ImmovableProcessEdges = PlanProcessEdges; + type TPProcessEdges = PlanProcessEdges; } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index e97a663959..82c0f50ffa 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -1,6 +1,6 @@ use crate::plan::{ObjectQueue, VectorObjectQueue}; use crate::policy::copy_context::PolicyCopyContext; -use crate::policy::gc_work::TRACE_KIND_IMMOVABLE; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; @@ -128,8 +128,8 @@ impl crate::policy::gc_work::PolicyTraceObject for CopySpace< worker: &mut GCWorker, ) -> ObjectReference { assert!( - KIND != TRACE_KIND_IMMOVABLE, - "Copyspace does not support immovable trace." + KIND != TRACE_KIND_TRANSITIVE_PIN, + "Copyspace does not support transitive pin trace." ); self.trace_object(queue, object, copy, worker) } diff --git a/src/policy/gc_work.rs b/src/policy/gc_work.rs index 674844f5f5..389618b1e0 100644 --- a/src/policy/gc_work.rs +++ b/src/policy/gc_work.rs @@ -3,7 +3,7 @@ pub(crate) type TraceKind = u8; pub const DEFAULT_TRACE: u8 = u8::MAX; -pub const TRACE_KIND_IMMOVABLE: u8 = DEFAULT_TRACE - 1; +pub const TRACE_KIND_TRANSITIVE_PIN: u8 = DEFAULT_TRACE - 1; use crate::plan::ObjectQueue; use crate::scheduler::GCWorker; diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 5010198572..9368467045 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -1,7 +1,7 @@ use super::line::*; use super::{block::*, defrag::Defrag}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::{TraceKind, TRACE_KIND_IMMOVABLE}; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_TRANSITIVE_PIN}; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; @@ -191,7 +191,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace copy: Option, worker: &mut GCWorker, ) -> ObjectReference { - if KIND == TRACE_KIND_IMMOVABLE { + if KIND == TRACE_KIND_TRANSITIVE_PIN { self.trace_object_without_moving(queue, object) } else if KIND == TRACE_KIND_DEFRAG { if Block::containing::(object).is_defrag_source() { @@ -228,7 +228,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace fn may_move_objects() -> bool { if KIND == TRACE_KIND_DEFRAG { true - } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_IMMOVABLE { + } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_TRANSITIVE_PIN { false } else { unreachable!() diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 28ed7c5f47..b551ec2868 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -1,7 +1,7 @@ use super::sft::SFT; use super::space::{CommonSpace, Space}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::{TraceKind, TRACE_KIND_IMMOVABLE}; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_TRANSITIVE_PIN}; use crate::policy::sft::GCWorkerMutRef; use crate::scheduler::GCWorker; use crate::util::alloc::allocator::align_allocation_no_fill; @@ -129,8 +129,8 @@ impl crate::policy::gc_work::PolicyTraceObject for MarkCompac _worker: &mut GCWorker, ) -> ObjectReference { assert!( - KIND != TRACE_KIND_IMMOVABLE, - "MarkCompact does not support immovable trace." + KIND != TRACE_KIND_TRANSITIVE_PIN, + "MarkCompact does not support transitive pin trace." ); if KIND == TRACE_KIND_MARK { self.trace_mark_object(queue, object) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 38158ee0b5..efc476f977 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -453,7 +453,7 @@ impl GCWork for ScanMutatorRoots { let factory = ProcessEdgesWorkRootsWorkFactory::< C::VM, C::ProcessEdgesWorkType, - C::ImmovableProcessEdges, + C::TPProcessEdges, >::new(mmtk); ::VMScanning::scan_roots_in_mutator_thread( worker.tls, @@ -486,7 +486,7 @@ impl GCWork for ScanVMSpecificRoots let factory = ProcessEdgesWorkRootsWorkFactory::< C::VM, C::ProcessEdgesWorkType, - C::ImmovableProcessEdges, + C::TPProcessEdges, >::new(mmtk); ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } @@ -744,26 +744,28 @@ impl, I: ProcessEdgesWork> } fn create_process_node_roots_work(&mut self, nodes: Vec) { + // Will process roots within the NodeRootsTrace bucket + // And put work in the Closure bucket crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::Closure, + WorkBucketStage::NodeRootsTrace, ProcessRootNode::::new(nodes, WorkBucketStage::Closure), ); } - fn create_immovable_process_edge_roots_work(&mut self, edges: Vec>) { + fn create_process_tp_edge_roots_work(&mut self, edges: Vec>) { crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::ImmovableClosure, - I::new(edges, true, self.mmtk, WorkBucketStage::ImmovableClosure), + WorkBucketStage::TPClosure, + I::new(edges, true, self.mmtk, WorkBucketStage::TPClosure), ); } - fn create_immovable_process_node_roots_work(&mut self, nodes: Vec) { + fn create_process_tp_node_roots_work(&mut self, nodes: Vec) { crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::ImmovableClosure, - ProcessRootNode::::new(nodes, WorkBucketStage::ImmovableClosure), + WorkBucketStage::TPClosure, + ProcessRootNode::::new(nodes, WorkBucketStage::TPClosure), ); } } @@ -1123,7 +1125,7 @@ impl, E2: ProcessEdgesWork // objects which are traced for the first time and we create work for scanning those roots. let scanned_root_objects = { // We create an instance of E to use its `trace_object` method and its object queue. - let mut process_edges_work = E1::new(vec![], true, mmtk, self.bucket); + let mut process_edges_work = E1::new(vec![], true, mmtk, WorkBucketStage::NodeRootsTrace); process_edges_work.set_worker(worker); for object in self.roots.iter().copied() { diff --git a/src/scheduler/work.rs b/src/scheduler/work.rs index aae85aebe7..861ed491dc 100644 --- a/src/scheduler/work.rs +++ b/src/scheduler/work.rs @@ -56,5 +56,5 @@ pub trait GCWorkContext { // We should use SFTProcessEdges as the default value for this associate type. However, this requires // `associated_type_defaults` which has not yet been stablized. type ProcessEdgesWorkType: ProcessEdgesWork; - type ImmovableProcessEdges: ProcessEdgesWork; + type TPProcessEdges: ProcessEdgesWork; } diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index 0a877fd9ed..a909c075a4 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -232,8 +232,11 @@ pub enum WorkBucketStage { /// Clear the VO bit metadata. Mainly used by ImmixSpace. #[cfg(feature = "vo_bit")] ClearVOBits, - /// Compute the transtive closure following only strong references, making sure that objects do not move - ImmovableClosure, + /// Compute the transtive closure starting from roots following only strong references, transitively pinning (TP) objects. + /// No objects in this closure are allow to move. + TPClosure, + /// Trace node roots. Node roots must not move, but their children may. To ensure correctness, these must be processed after TPClosure + NodeRootsTrace, /// Compute the transtive closure following only strong references. Closure, /// Handle Java-style soft references, and potentially expand the transitive closure. diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 5be32049e2..0b807d5596 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -124,7 +124,7 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `edges`: A vector of edges. - fn create_immovable_process_edge_roots_work(&mut self, edges: Vec); + fn create_process_tp_edge_roots_work(&mut self, edges: Vec); /// Create work packets to handle nodes pointed by root edges. /// @@ -133,7 +133,7 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_immovable_process_node_roots_work(&mut self, nodes: Vec); + fn create_process_tp_node_roots_work(&mut self, nodes: Vec); } /// VM-specific methods for scanning roots/objects. diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index 74fe9cd60c..b586985313 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -70,11 +70,11 @@ impl RootsWorkFactory

for MockFactory { unimplemented!(); } - fn create_immovable_process_edge_roots_work(&mut self, _edges: Vec
) { + fn create_process_tp_edge_roots_work(&mut self, _edges: Vec
) { unimplemented!(); } - fn create_immovable_process_node_roots_work(&mut self, _nodes: Vec) { + fn create_process_tp_node_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } } From 8921ed7fc0f54227e0e38288ab960601707f0b4c Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 16 Aug 2023 02:04:10 +0000 Subject: [PATCH 12/25] Run cargo fmt --- src/scheduler/gc_work.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index efc476f977..692545356b 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -1125,7 +1125,8 @@ impl, E2: ProcessEdgesWork // objects which are traced for the first time and we create work for scanning those roots. let scanned_root_objects = { // We create an instance of E to use its `trace_object` method and its object queue. - let mut process_edges_work = E1::new(vec![], true, mmtk, WorkBucketStage::NodeRootsTrace); + let mut process_edges_work = + E1::new(vec![], true, mmtk, WorkBucketStage::NodeRootsTrace); process_edges_work.set_worker(worker); for object in self.roots.iter().copied() { From 48dde0783407ec16fa6fc048391758aca38aba93 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 23 Aug 2023 04:53:37 +0000 Subject: [PATCH 13/25] Turning off stress copying as default; fixing logic to prefer copy on nursery GC, allowing moving on full heap but disabling it for the nursery GC --- src/policy/immix/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index 81482d5bbd..32c94261e9 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -18,15 +18,15 @@ pub const BLOCK_ONLY: bool = false; pub const DEFRAG: bool = !cfg!(feature = "immix_non_moving"); // defrag if we are allowed to move. /// Make every GC a defragment GC. (for debugging) -pub const STRESS_DEFRAG: bool = true; +pub const STRESS_DEFRAG: bool = false; /// Mark every allocated block as defragmentation source before GC. (for debugging) /// Set both this and `STRESS_DEFRAG` to true to make Immix move as many objects as possible. -pub const DEFRAG_EVERY_BLOCK: bool = true; +pub const DEFRAG_EVERY_BLOCK: bool = false; /// If Immix is used as a nursery space, do we prefer copy? pub const PREFER_COPY_ON_NURSERY_GC: bool = - !cfg!(feature = "immix_non_moving") || !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. + !cfg!(feature = "immix_non_moving") && !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. /// In some cases/settings, Immix may never move objects. /// Currently we only have two cases where we move objects: 1. defrag, 2. nursery copy. From b41474614525500cf7c93e314674575fbad0bef1 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 24 Aug 2023 04:27:24 +0000 Subject: [PATCH 14/25] Fixed naming of node roots => pinned roots, and tp => tpinned/tpinning --- src/scheduler/gc_work.rs | 20 ++++++++++---------- src/scheduler/work_bucket.rs | 6 +++--- src/vm/scanning.rs | 13 ++++++------- tests/test_roots_work_factory.rs | 6 +++--- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 692545356b..182132dd81 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -621,7 +621,7 @@ pub trait ProcessEdgesWork: /// Create an object-scanning work packet to be used for this ProcessEdgesWork. /// /// `roots` indicates if we are creating a packet for root scanning. It is only true when this - /// method is called to handle `RootsWorkFactory::create_process_node_roots_work`. + /// method is called to handle `RootsWorkFactory::create_process_pinned_roots_work`. fn create_scan_work( &self, nodes: Vec, @@ -743,29 +743,29 @@ impl, I: ProcessEdgesWork> ); } - fn create_process_node_roots_work(&mut self, nodes: Vec) { + fn create_process_pinned_roots_work(&mut self, nodes: Vec) { // Will process roots within the NodeRootsTrace bucket // And put work in the Closure bucket crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::NodeRootsTrace, + WorkBucketStage::PinnedRootsTrace, ProcessRootNode::::new(nodes, WorkBucketStage::Closure), ); } - fn create_process_tp_edge_roots_work(&mut self, edges: Vec>) { + fn create_process_tpinned_edge_roots_work(&mut self, edges: Vec>) { crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::TPClosure, - I::new(edges, true, self.mmtk, WorkBucketStage::TPClosure), + WorkBucketStage::TPinningClosure, + I::new(edges, true, self.mmtk, WorkBucketStage::TPinningClosure), ); } - fn create_process_tp_node_roots_work(&mut self, nodes: Vec) { + fn create_process_tpinned_roots_work(&mut self, nodes: Vec) { crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::TPClosure, - ProcessRootNode::::new(nodes, WorkBucketStage::TPClosure), + WorkBucketStage::TPinningClosure, + ProcessRootNode::::new(nodes, WorkBucketStage::TPinningClosure), ); } } @@ -1126,7 +1126,7 @@ impl, E2: ProcessEdgesWork let scanned_root_objects = { // We create an instance of E to use its `trace_object` method and its object queue. let mut process_edges_work = - E1::new(vec![], true, mmtk, WorkBucketStage::NodeRootsTrace); + E1::new(vec![], true, mmtk, WorkBucketStage::PinnedRootsTrace); process_edges_work.set_worker(worker); for object in self.roots.iter().copied() { diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index a909c075a4..1c0ef3037d 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -234,9 +234,9 @@ pub enum WorkBucketStage { ClearVOBits, /// Compute the transtive closure starting from roots following only strong references, transitively pinning (TP) objects. /// No objects in this closure are allow to move. - TPClosure, - /// Trace node roots. Node roots must not move, but their children may. To ensure correctness, these must be processed after TPClosure - NodeRootsTrace, + TPinningClosure, + /// Trace node roots. Node roots must not move, but their children may. To ensure correctness, these must be processed after TPinningClosure + PinnedRootsTrace, /// Compute the transtive closure following only strong references. Closure, /// Handle Java-style soft references, and potentially expand the transitive closure. diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 299fc19c95..187767285c 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -107,16 +107,15 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// Create work packets to handle nodes pointed by root edges. /// - /// The work packet cannot update root edges, therefore it cannot move the objects. This - /// method can only be used by GC algorithms that never moves objects, or GC algorithms that - /// supports object pinning. + /// The work packet cannot update root edges, therefore it cannot move the objects + /// i.e. they will be pinned for the duration of the GC /// /// This method is useful for conservative stack scanning, or VMs that cannot update some /// of the root edges. /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_node_roots_work(&mut self, nodes: Vec); + fn create_process_pinned_roots_work(&mut self, nodes: Vec); /// Create work packets to handle root edges. /// @@ -124,16 +123,16 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// Arguments: /// * `edges`: A vector of edges. - fn create_process_tp_edge_roots_work(&mut self, edges: Vec); + fn create_process_tpinned_edge_roots_work(&mut self, edges: Vec); /// Create work packets to handle nodes pointed by root edges. /// - /// Similar to `create_process_node_roots_work`, this work packet won't move root objects, but also will + /// Similar to `create_process_pinned_roots_work`, this work packet won't move root objects, but also will /// not move any object in their transitive closure. /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_tp_node_roots_work(&mut self, nodes: Vec); + fn create_process_tpinned_roots_work(&mut self, nodes: Vec); } /// VM-specific methods for scanning roots/objects. diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index b586985313..7c259601b9 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -66,15 +66,15 @@ impl RootsWorkFactory
for MockFactory { } } - fn create_process_node_roots_work(&mut self, _nodes: Vec) { + fn create_process_pinned_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } - fn create_process_tp_edge_roots_work(&mut self, _edges: Vec
) { + fn create_process_tpinned_edge_roots_work(&mut self, _edges: Vec
) { unimplemented!(); } - fn create_process_tp_node_roots_work(&mut self, _nodes: Vec) { + fn create_process_tpinned_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } } From a33fc303052e2f329cd513ad24606a8ba5de1f4a Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 4 Sep 2023 12:20:41 +1000 Subject: [PATCH 15/25] Apply suggestions from code review (comments, and renaming) Co-authored-by: Yi Lin Co-authored-by: Kunshan Wang --- Cargo.toml | 4 +++- src/plan/generational/mod.rs | 2 +- src/plan/mod.rs | 4 ++-- src/scheduler/work_bucket.rs | 6 +++--- src/vm/scanning.rs | 17 +++++++++-------- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b7c205031f..c4d8237a9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,9 @@ object_pinning = [] # Disable any object copying in Immix. This makes Immix a non-moving policy. immix_non_moving = [] -# Disable any object copying in nursery GC for Sticky Immix. +# Disable any object copying in nursery GC for Sticky Immix while allowing other kinds of copying. +# `immix_non_moving` disables all kinds of copying in Immix, so this feature is not needed +# if `immix_non_moving` is in use. sticky_immix_non_moving_nursery = [] diff --git a/src/plan/generational/mod.rs b/src/plan/generational/mod.rs index 571e5672c1..96c25c1e0f 100644 --- a/src/plan/generational/mod.rs +++ b/src/plan/generational/mod.rs @@ -23,7 +23,7 @@ use super::mutator_context::ReservedAllocators; pub mod barrier; /// Generational copying (GenCopy) pub mod copying; -// /// Generational immix (GenImmix) +/// Generational immix (GenImmix) pub mod immix; // Common generational code diff --git a/src/plan/mod.rs b/src/plan/mod.rs index 6e93d7d919..15c16182fa 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -40,9 +40,9 @@ pub use plan_constraints::DEFAULT_PLAN_CONSTRAINTS; mod tracing; pub use tracing::{ObjectQueue, ObjectsClosure, VectorObjectQueue, VectorQueue}; -// /// Generational plans (with a copying nursery) +/// Generational plans (with a copying nursery) mod generational; -// /// Sticky plans (using sticky marks for generational behaviors without a copying nursery) +/// Sticky plans (using sticky marks for generational behaviors without a copying nursery) mod sticky; mod immix; diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index 1c0ef3037d..b461471081 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -232,11 +232,11 @@ pub enum WorkBucketStage { /// Clear the VO bit metadata. Mainly used by ImmixSpace. #[cfg(feature = "vo_bit")] ClearVOBits, - /// Compute the transtive closure starting from roots following only strong references, transitively pinning (TP) objects. + /// Compute the transtive closure starting from transitively pinning (TP) roots following only strong references. /// No objects in this closure are allow to move. TPinningClosure, - /// Trace node roots. Node roots must not move, but their children may. To ensure correctness, these must be processed after TPinningClosure - PinnedRootsTrace, + /// Trace (non-transitively) pinning roots. Objects pointed by those roots must not move, but their children may. To ensure correctness, these must be processed after TPinningClosure + PinningRootsTrace, /// Compute the transtive closure following only strong references. Closure, /// Handle Java-style soft references, and potentially expand the transitive closure. diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 187767285c..2fbc3645a9 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -105,17 +105,18 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// * `edges`: A vector of edges. fn create_process_edge_roots_work(&mut self, edges: Vec); - /// Create work packets to handle nodes pointed by root edges. + /// Create work packets to handle non-transitively pinning roots. /// - /// The work packet cannot update root edges, therefore it cannot move the objects - /// i.e. they will be pinned for the duration of the GC + /// The work packet will prevent the objects in `nodes` from moving, + /// i.e. they will be pinned for the duration of the GC. + /// But it will not prevent the children of those objects from moving. /// /// This method is useful for conservative stack scanning, or VMs that cannot update some /// of the root edges. /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_pinned_roots_work(&mut self, nodes: Vec); + fn create_process_pinning_roots_work(&mut self, nodes: Vec); /// Create work packets to handle root edges. /// @@ -125,14 +126,14 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// * `edges`: A vector of edges. fn create_process_tpinned_edge_roots_work(&mut self, edges: Vec); - /// Create work packets to handle nodes pointed by root edges. + /// Create work packets to handle transitively pinning (TP) roots. /// - /// Similar to `create_process_pinned_roots_work`, this work packet won't move root objects, but also will - /// not move any object in their transitive closure. + /// Similar to `create_process_pinning_roots_work`, this work packet will not move objects in `nodes`. + /// Unlike ``create_process_pinning_roots_work`, no objects in the transitive closure of `nodes` will be moved, either. /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_tpinned_roots_work(&mut self, nodes: Vec); + fn create_process_tpinning_roots_work(&mut self, nodes: Vec); } /// VM-specific methods for scanning roots/objects. From 7f96d1581bebe66d08ea82f1fd9207653ae354f3 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 4 Sep 2023 12:24:03 +1000 Subject: [PATCH 16/25] Apply suggestions from code review (more comments) Co-authored-by: Kunshan Wang Co-authored-by: Yi Lin --- src/scheduler/gc_work.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 182132dd81..71c0ec6e4a 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -621,7 +621,7 @@ pub trait ProcessEdgesWork: /// Create an object-scanning work packet to be used for this ProcessEdgesWork. /// /// `roots` indicates if we are creating a packet for root scanning. It is only true when this - /// method is called to handle `RootsWorkFactory::create_process_pinned_roots_work`. + /// method is called to handle `RootsWorkFactory::create_process_pinning_roots_work`. fn create_scan_work( &self, nodes: Vec, @@ -743,7 +743,7 @@ impl, I: ProcessEdgesWork> ); } - fn create_process_pinned_roots_work(&mut self, nodes: Vec) { + fn create_process_pinning_roots_work(&mut self, nodes: Vec) { // Will process roots within the NodeRootsTrace bucket // And put work in the Closure bucket crate::memory_manager::add_work_packet( @@ -1151,7 +1151,8 @@ impl, E2: ProcessEdgesWork } } -// // create UnsupportedProcessEdge +/// A `ProcessEdgesWork` type that panics when any of its method is used. +/// This is currently used for plans that do not support transitively pinning. #[derive(Default)] pub struct UnsupportedProcessEdges { phantom: PhantomData, From b23898b6389c1b3c0d4a3e318691971803b1575c Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 24 Aug 2023 17:44:07 +0800 Subject: [PATCH 17/25] GCWorkContext should always have 'static and Send --- src/plan/immix/global.rs | 4 ++-- src/scheduler/gc_work.rs | 10 +++++----- src/scheduler/scheduler.rs | 5 +---- src/scheduler/work.rs | 7 ++++++- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index 03e66fec47..82c4d5e8a0 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -173,8 +173,8 @@ impl Immix { /// to schedule a full heap collection. A plan must call set_collection_kind and set_gc_status before this method. pub(crate) fn schedule_immix_full_heap_collection< PlanType: Plan, - FastContext: Send + 'static + GCWorkContext, - DefragContext: Send + 'static + GCWorkContext, + FastContext: GCWorkContext, + DefragContext: GCWorkContext, >( plan: &'static DefragContext::PlanType, immix_space: &ImmixSpace, diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 71c0ec6e4a..db380b2e52 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -42,7 +42,7 @@ impl Prepare { } } -impl GCWork for Prepare { +impl GCWork for Prepare { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("Prepare Global"); // We assume this is the only running work packet that accesses plan at the point of execution @@ -110,7 +110,7 @@ impl Release { } } -impl GCWork for Release { +impl GCWork for Release { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("Release Global"); @@ -187,7 +187,7 @@ impl StopMutators { } } -impl GCWork for StopMutators { +impl GCWork for StopMutators { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("stop_all_mutators start"); mmtk.plan.base().prepare_for_stack_scanning(); @@ -445,7 +445,7 @@ impl GCWork for VMPostForwarding { pub struct ScanMutatorRoots(pub &'static mut Mutator); -impl GCWork for ScanMutatorRoots { +impl GCWork for ScanMutatorRoots { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanMutatorRoots for mutator {:?}", self.0.get_tls()); let base = &mmtk.plan.base(); @@ -480,7 +480,7 @@ impl ScanVMSpecificRoots { } } -impl GCWork for ScanVMSpecificRoots { +impl GCWork for ScanVMSpecificRoots { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStaticRoots"); let factory = ProcessEdgesWorkRootsWorkFactory::< diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index ca5ec47588..1ebaca9b88 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -108,10 +108,7 @@ impl GCWorkScheduler { } /// Schedule all the common work packets - pub fn schedule_common_work + 'static + Send>( - &self, - plan: &'static C::PlanType, - ) { + pub fn schedule_common_work>(&self, plan: &'static C::PlanType) { use crate::plan::Plan; use crate::scheduler::gc_work::*; // Stop & scan mutators (mutator scanning can happen before STW) diff --git a/src/scheduler/work.rs b/src/scheduler/work.rs index 861ed491dc..6018eed0a3 100644 --- a/src/scheduler/work.rs +++ b/src/scheduler/work.rs @@ -50,7 +50,12 @@ use crate::plan::Plan; /// needs this trait to schedule different work packets. For certain plans, /// they may need to provide several types that implement this trait, e.g. one for /// nursery GC, one for mature GC. -pub trait GCWorkContext { +/// +/// Note: Because `GCWorkContext` is often used as parameters of implementations of `GCWork`, we +/// let GCWorkContext require `Send + 'static`. Since `GCWorkContext` is just a group of +/// associated types, its implementations should not have any actual fields other than +/// `PhantomData`, and will automatically have `Send + 'static`. +pub trait GCWorkContext: Send + 'static { type VM: VMBinding; type PlanType: Plan; // We should use SFTProcessEdges as the default value for this associate type. However, this requires From 1b40a94ad6feaab13aabce5a80980efecf8bf695 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Sep 2023 01:50:48 +0000 Subject: [PATCH 18/25] Addressing comments in the review --- src/plan/generational/immix/gc_work.rs | 6 ++--- src/plan/marksweep/gc_work.rs | 2 +- src/plan/pageprotect/gc_work.rs | 4 +-- src/policy/copyspace.rs | 2 +- src/policy/immix/immixspace.rs | 5 ---- src/policy/markcompactspace.rs | 2 +- src/scheduler/gc_work.rs | 37 +++++++++++--------------- src/vm/scanning.rs | 8 ------ 8 files changed, 24 insertions(+), 42 deletions(-) diff --git a/src/plan/generational/immix/gc_work.rs b/src/plan/generational/immix/gc_work.rs index cc15160488..14f46a0958 100644 --- a/src/plan/generational/immix/gc_work.rs +++ b/src/plan/generational/immix/gc_work.rs @@ -1,8 +1,8 @@ use super::global::GenImmix; use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::policy::gc_work::TraceKind; -use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::UnsupportedProcessEdges; use crate::vm::VMBinding; pub struct GenImmixNurseryGCWorkContext(std::marker::PhantomData); @@ -10,7 +10,7 @@ impl crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkCon type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; - type TPProcessEdges = GenNurseryProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } pub(super) struct GenImmixMatureGCWorkContext( @@ -22,5 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; - type TPProcessEdges = PlanProcessEdges, TRACE_KIND_TRANSITIVE_PIN>; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/marksweep/gc_work.rs b/src/plan/marksweep/gc_work.rs index 20db8b38c1..604e355759 100644 --- a/src/plan/marksweep/gc_work.rs +++ b/src/plan/marksweep/gc_work.rs @@ -8,5 +8,5 @@ impl crate::scheduler::GCWorkContext for MSGCWorkContext { type VM = VM; type PlanType = MarkSweep; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type TPProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/pageprotect/gc_work.rs b/src/plan/pageprotect/gc_work.rs index 34d4206c47..4e68c4afb2 100644 --- a/src/plan/pageprotect/gc_work.rs +++ b/src/plan/pageprotect/gc_work.rs @@ -1,6 +1,6 @@ use super::global::PageProtect; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; +use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; pub struct PPGCWorkContext(std::marker::PhantomData); @@ -8,5 +8,5 @@ impl crate::scheduler::GCWorkContext for PPGCWorkContext { type VM = VM; type PlanType = PageProtect; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; - type TPProcessEdges = UnsupportedProcessEdges; + type TPProcessEdges = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 82c0f50ffa..0756d6d68f 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -127,7 +127,7 @@ impl crate::policy::gc_work::PolicyTraceObject for CopySpace< copy: Option, worker: &mut GCWorker, ) -> ObjectReference { - assert!( + debug_assert!( KIND != TRACE_KIND_TRANSITIVE_PIN, "Copyspace does not support transitive pin trace." ); diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 9368467045..c555ff0fc1 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -620,11 +620,6 @@ impl ImmixSpace { } else if self.is_marked(object) { // We won the forwarding race but the object is already marked so we clear the // forwarding status and return the unmoved object - // debug_assert!( - // nursery_collection || self.defrag.space_exhausted() || self.is_pinned(object), - // "Forwarded object is the same as original object {} even though it should have been copied", - // object, - // ); ForwardingWord::clear_forwarding_bits::(object); object } else { diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index b551ec2868..494330e34d 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -128,7 +128,7 @@ impl crate::policy::gc_work::PolicyTraceObject for MarkCompac _copy: Option, _worker: &mut GCWorker, ) -> ObjectReference { - assert!( + debug_assert!( KIND != TRACE_KIND_TRANSITIVE_PIN, "MarkCompact does not support transitive pin trace." ); diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index db380b2e52..b75db7cfae 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -744,24 +744,16 @@ impl, I: ProcessEdgesWork> } fn create_process_pinning_roots_work(&mut self, nodes: Vec) { - // Will process roots within the NodeRootsTrace bucket + // Will process roots within the PinningRootsTrace bucket // And put work in the Closure bucket crate::memory_manager::add_work_packet( self.mmtk, - WorkBucketStage::PinnedRootsTrace, + WorkBucketStage::PinningRootsTrace, ProcessRootNode::::new(nodes, WorkBucketStage::Closure), ); } - fn create_process_tpinned_edge_roots_work(&mut self, edges: Vec>) { - crate::memory_manager::add_work_packet( - self.mmtk, - WorkBucketStage::TPinningClosure, - I::new(edges, true, self.mmtk, WorkBucketStage::TPinningClosure), - ); - } - - fn create_process_tpinned_roots_work(&mut self, nodes: Vec) { + fn create_process_tpinning_roots_work(&mut self, nodes: Vec) { crate::memory_manager::add_work_packet( self.mmtk, WorkBucketStage::TPinningClosure, @@ -820,7 +812,7 @@ pub trait ScanObjectsWork: GCWork + Sized { _mmtk: &'static MMTK<::VM>, ) { let tls = worker.tls; - assert!(!self.roots()); + debug_assert!(!self.roots()); // Scan the nodes in the buffer. let objects_to_scan = buffer; @@ -1079,15 +1071,18 @@ impl + PlanTraceObject> GCWork, E2: ProcessEdgesWork> -{ - phantom: PhantomData<(VM, E1, E2)>, +/// This creates work for processing pinning roots. In particular it traces the objects in these roots using I, +/// but creates the work to scan these objects using E. This is necessary to guarantee that these objects do not move +/// (`I` should trace them without moving) as we do not have the information about the edges pointing to them. + +struct ProcessRootNode, E: ProcessEdgesWork> { + phantom: PhantomData<(VM, I, E)>, roots: Vec, bucket: WorkBucketStage, } -impl, E2: ProcessEdgesWork> - ProcessRootNode +impl, E: ProcessEdgesWork> + ProcessRootNode { pub fn new(nodes: Vec, bucket: WorkBucketStage) -> Self { Self { @@ -1098,8 +1093,8 @@ impl, E2: ProcessEdgesWork } } -impl, E2: ProcessEdgesWork> GCWork - for ProcessRootNode +impl, E: ProcessEdgesWork> GCWork + for ProcessRootNode { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ProcessRootNode"); @@ -1126,7 +1121,7 @@ impl, E2: ProcessEdgesWork let scanned_root_objects = { // We create an instance of E to use its `trace_object` method and its object queue. let mut process_edges_work = - E1::new(vec![], true, mmtk, WorkBucketStage::PinnedRootsTrace); + I::new(vec![], true, mmtk, WorkBucketStage::PinningRootsTrace); process_edges_work.set_worker(worker); for object in self.roots.iter().copied() { @@ -1143,7 +1138,7 @@ impl, E2: ProcessEdgesWork process_edges_work.nodes.take() }; - let process_edges_work = E2::new(vec![], false, mmtk, self.bucket); + let process_edges_work = E::new(vec![], false, mmtk, self.bucket); let work = process_edges_work.create_scan_work(scanned_root_objects, false); crate::memory_manager::add_work_packet(mmtk, self.bucket, work); diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 2fbc3645a9..b57c28cf54 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -118,14 +118,6 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// * `nodes`: A vector of references to objects pointed by root edges. fn create_process_pinning_roots_work(&mut self, nodes: Vec); - /// Create work packets to handle root edges. - /// - /// The work packet must not update the edges transitively found from any of the root edges. - /// - /// Arguments: - /// * `edges`: A vector of edges. - fn create_process_tpinned_edge_roots_work(&mut self, edges: Vec); - /// Create work packets to handle transitively pinning (TP) roots. /// /// Similar to `create_process_pinning_roots_work`, this work packet will not move objects in `nodes`. From 57f298fda1f72fc72cc38b4523e8ae1e43460fe7 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Sep 2023 02:07:03 +0000 Subject: [PATCH 19/25] Applying cargo fmt --- src/scheduler/gc_work.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 866665011a..c5562f0f16 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -454,10 +454,10 @@ impl GCWork for ScanMutatorRoots { let base = mmtk.get_plan().base(); let mutators = ::VMActivePlan::number_of_mutators(); let factory = ProcessEdgesWorkRootsWorkFactory::< - C::VM, - C::ProcessEdgesWorkType, - C::TPProcessEdges, - >::new(mmtk); + C::VM, + C::ProcessEdgesWorkType, + C::TPProcessEdges, + >::new(mmtk); ::VMScanning::scan_roots_in_mutator_thread( worker.tls, unsafe { &mut *(self.0 as *mut _) }, From 5d2cab466bf8ed2e0a6d6c2ad01825a998354a94 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Sep 2023 02:21:31 +0000 Subject: [PATCH 20/25] Fixing code inside conditional compilation --- src/scheduler/gc_work.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index c5562f0f16..ebf45d5c77 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -1104,7 +1104,7 @@ impl, E: ProcessEdgesWork> #[cfg(feature = "sanity")] { - if !mmtk.plan.is_in_sanity() { + if !mmtk.get_plan().is_in_sanity() { mmtk.sanity_checker .lock() .unwrap() From 73ab10a30b8093a6079fc54c43c007a69b01f0b2 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Sep 2023 02:39:54 +0000 Subject: [PATCH 21/25] Fixing tests --- tests/test_roots_work_factory.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index 7c259601b9..4a8b64b511 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -66,15 +66,11 @@ impl RootsWorkFactory
for MockFactory { } } - fn create_process_pinned_roots_work(&mut self, _nodes: Vec) { + fn create_process_pinning_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } - fn create_process_tpinned_edge_roots_work(&mut self, _edges: Vec
) { - unimplemented!(); - } - - fn create_process_tpinned_roots_work(&mut self, _nodes: Vec) { + fn create_process_tpinning_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } } From 4773952883f8b27f68272d48427f927b22aa98eb Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Tue, 5 Sep 2023 03:11:58 +0000 Subject: [PATCH 22/25] Fixing formatting in macros. --- macros/src/has_spaces_impl.rs | 3 ++- macros/src/plan_trace_object_impl.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/macros/src/has_spaces_impl.rs b/macros/src/has_spaces_impl.rs index dbf6da6ed3..71286e415d 100644 --- a/macros/src/has_spaces_impl.rs +++ b/macros/src/has_spaces_impl.rs @@ -12,7 +12,8 @@ pub(crate) fn derive(input: DeriveInput) -> TokenStream2 { let syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), .. - }) = input.data else { + }) = input.data + else { abort_call_site!("`#[derive(HasSpaces)]` only supports structs with named fields."); }; diff --git a/macros/src/plan_trace_object_impl.rs b/macros/src/plan_trace_object_impl.rs index b5241bfd04..7be2e44c9a 100644 --- a/macros/src/plan_trace_object_impl.rs +++ b/macros/src/plan_trace_object_impl.rs @@ -12,7 +12,8 @@ pub(crate) fn derive(input: DeriveInput) -> TokenStream2 { let syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), .. - }) = input.data else { + }) = input.data + else { abort_call_site!("`#[derive(PlanTraceObject)]` only supports structs with named fields."); }; From a91453fc7ecfc8a8c7ba440970c43f825473ad43 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 6 Sep 2023 04:35:54 +0000 Subject: [PATCH 23/25] Fixing bug when replacing MarkingProcessEdges in markcompact/global.rs --- src/plan/markcompact/global.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plan/markcompact/global.rs b/src/plan/markcompact/global.rs index d8cabe5cc6..4e5cc8c972 100644 --- a/src/plan/markcompact/global.rs +++ b/src/plan/markcompact/global.rs @@ -92,7 +92,7 @@ impl Plan for MarkCompact { // Stop & scan mutators (mutator scanning can happen before STW) scheduler.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::>::new()); + .add(StopMutators::>::new()); // Prepare global/collectors/mutators scheduler.work_buckets[WorkBucketStage::Prepare] From d3fa97760213898cd9fdd894ac35abdacb824168 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 6 Sep 2023 04:58:10 +0000 Subject: [PATCH 24/25] Fixing import that failed cargo clippy --- src/plan/markcompact/global.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plan/markcompact/global.rs b/src/plan/markcompact/global.rs index 4e5cc8c972..32da425ea3 100644 --- a/src/plan/markcompact/global.rs +++ b/src/plan/markcompact/global.rs @@ -1,8 +1,8 @@ +use super::gc_work::MarkCompactGCWorkContext; use super::gc_work::{ CalculateForwardingAddress, Compact, ForwardingProcessEdges, MarkingProcessEdges, UpdateReferences, }; -use super::gc_work::{MarkCompactForwardingGCWorkContext, MarkCompactGCWorkContext}; use crate::plan::global::CommonPlan; use crate::plan::global::GcStatus; use crate::plan::global::{BasePlan, CreateGeneralPlanArgs, CreateSpecificPlanArgs}; From f18406442098ab2b30cbc79f3300aa52502dec1a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 7 Sep 2023 11:58:41 +0800 Subject: [PATCH 25/25] Fix dependency replacing script Remove all keys that may specify the location of a dependency since they may conflict. --- .github/scripts/replace-mmtk-dep.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/scripts/replace-mmtk-dep.py b/.github/scripts/replace-mmtk-dep.py index da811db5dd..3eadb5c577 100644 --- a/.github/scripts/replace-mmtk-dep.py +++ b/.github/scripts/replace-mmtk-dep.py @@ -19,10 +19,15 @@ mmtk_node = toml_data["dependencies"]["mmtk"] -print("Deleting dependencies.mmtk.git") -if "git" in mmtk_node: - del mmtk_node["git"] - +# These keys may specify the locations of the dependency. Remove them. +for key in ["git", "branch", "version", "registry"]: + if key in mmtk_node: + print("Deleting dependencies.mmtk.{}".format(key)) + del mmtk_node[key] + else: + print("Key dependencies.mmtk.{} does not exist. Ignored.".format(key)) + +# Use mmtk-core from the specified local directory. mmtk_repo_path = os.path.realpath(args.mmtk_core_path) print("Setting dependencies.mmtk.path to {}".format(mmtk_repo_path)) mmtk_node["path"] = mmtk_repo_path