diff --git a/Cargo.toml b/Cargo.toml index 22233eb428..3507446732 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,9 +50,11 @@ rustversion = "1.0" rayon-core = "=1.12.1" # We can remove this dependency when we use MSRV 1.80+ spin = "0.9.5" static_assertions = "1.1.0" -strum = "0.27.1" -strum_macros = "0.27.1" -sysinfo = "0.33.1" +strum = "0.26.2" +strum_macros = "0.26.2" +sysinfo = "0.30.9" +crabgrind = { version = "0.1.12", optional = true } + [dev-dependencies] paste = "1.0.8" @@ -253,6 +255,9 @@ eager_sweeping = [] # normal heap range, we will have to use chunk-based SFT table. Turning on this feature will use a different SFT map implementation on 64bits, # and will affect all the plans in the build. Please be aware of the consequence, and this is only meant to be experimental use. malloc_mark_sweep = [] +# Enable Valgrind support in MMTk. This will invoke Valgrind interfaces on allocation and deallocation. At the moment only MarkSweep +# space is supported +crabgrind = ["dep:crabgrind", "vo_bit"] # Group:nonmovingspace immortal_as_nonmoving = [] diff --git a/src/plan/global.rs b/src/plan/global.rs index 5cb6ce9bb9..23346e93cc 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -115,6 +115,8 @@ pub fn create_plan( plan.for_each_space(&mut |s| { sft_map.notify_space_creation(s.as_sft()); s.initialize_sft(sft_map); + // after SFT is initialized, we can also initialize mempool tracking + s.get_page_resource().track(); }); plan diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index ddef8d6bc1..8deaafa5d0 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -12,6 +12,7 @@ use crate::util::metadata; use crate::util::object_enum::ClosureObjectEnumerator; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; +use crate::util::track::track_free; use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; @@ -352,10 +353,7 @@ impl LargeObjectSpace { let sweep = |object: ObjectReference| { #[cfg(feature = "vo_bit")] crate::util::metadata::vo_bit::unset_vo_bit(object); - // Clear log bits for dead objects to prevent a new nursery object having the unlog bit set - if self.clear_log_bit_on_sweep { - VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); - } + track_free(object.to_object_start::(), 0 /* TODO: Size */); self.pr .release_pages(get_super_page(object.to_object_start::())); }; diff --git a/src/policy/marksweepspace/native_ms/block.rs b/src/policy/marksweepspace/native_ms/block.rs index 5bef9a3e52..707d6944b3 100644 --- a/src/policy/marksweepspace/native_ms/block.rs +++ b/src/policy/marksweepspace/native_ms/block.rs @@ -297,6 +297,13 @@ impl Block { if !VM::VMObjectModel::LOCAL_MARK_BIT_SPEC .is_marked::(potential_object, Ordering::SeqCst) { + #[cfg(feature = "crabgrind")] + { + let vo_bit = crate::util::metadata::vo_bit::is_vo_bit_set(potential_object); + if vo_bit { + crabgrind::memcheck::alloc::free(cell.to_mut_ptr(), 0); + } + } // clear VO bit if it is ever set. It is possible that the VO bit is never set for this cell (i.e. there was no object in this cell before this GC), // we unset the bit anyway. #[cfg(feature = "vo_bit")] @@ -304,6 +311,7 @@ impl Block { unsafe { cell.store::
(last); } + last = cell; } cell += cell_size; @@ -365,7 +373,14 @@ impl Block { "{:?} Free cell: {}, last cell in freelist is {}", self, cell, last ); - + #[cfg(feature = "crabgrind")] + { + let vo_bit = + crate::util::metadata::vo_bit::is_vo_bit_set(potential_object_ref); + if vo_bit { + crabgrind::memcheck::alloc::free(cell.to_mut_ptr(), 0); + } + } // Clear VO bit: we don't know where the object reference actually is, so we bulk zero the cell. #[cfg(feature = "vo_bit")] crate::util::metadata::vo_bit::bzero_vo_bit(cell, cell_size); @@ -376,6 +391,7 @@ impl Block { cell.store::
(last); } last = cell; + cell += cell_size; debug_assert_eq!(cursor, cell); } diff --git a/src/policy/space.rs b/src/policy/space.rs index a80dc3f973..7c8992e547 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -6,6 +6,7 @@ use crate::util::metadata::side_metadata::{ SideMetadataContext, SideMetadataSanity, SideMetadataSpec, }; use crate::util::object_enum::ObjectEnumerator; +use crate::util::track::track_mempool_alloc; use crate::util::Address; use crate::util::ObjectReference; @@ -247,7 +248,6 @@ pub trait Space: 'static + SFT + Sync + Downcast { self.common().descriptor ); } - debug!("Space.acquire(), returned = {}", res.start); res.start } diff --git a/src/util/alloc/bumpallocator.rs b/src/util/alloc/bumpallocator.rs index 9c70fba5c1..8c8fdff36c 100644 --- a/src/util/alloc/bumpallocator.rs +++ b/src/util/alloc/bumpallocator.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use crate::util::track::track_malloc; use crate::util::Address; use crate::util::alloc::Allocator; @@ -116,13 +117,16 @@ impl Allocator for BumpAllocator { self.bump_pointer.cursor, self.bump_pointer.limit ); + track_malloc(result, size, false); result } } fn alloc_slow_once(&mut self, size: usize, align: usize, offset: usize) -> Address { trace!("alloc_slow"); - self.acquire_block(size, align, offset, false) + let block = self.acquire_block(size, align, offset, false); + track_malloc(block, size, false); + block } /// Slow path for allocation if precise stress testing has been enabled. diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index b443fe0c6a..f92f01aafc 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -6,6 +6,7 @@ use crate::policy::marksweepspace::native_ms::*; use crate::util::alloc::allocator; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; +use crate::util::track::track_malloc; use crate::util::Address; use crate::util::VMThread; use crate::vm::VMBinding; @@ -76,6 +77,7 @@ impl Allocator for FreeListAllocator { size, align, offset, cell, cell_size, res + size, cell + cell_size ); } + return res; } } @@ -179,6 +181,8 @@ impl FreeListAllocator { } } + track_malloc(cell, cell_size, true); + cell } diff --git a/src/util/alloc/large_object_allocator.rs b/src/util/alloc/large_object_allocator.rs index 89b5526c2e..1e5297acfc 100644 --- a/src/util/alloc/large_object_allocator.rs +++ b/src/util/alloc/large_object_allocator.rs @@ -4,6 +4,7 @@ use crate::policy::largeobjectspace::LargeObjectSpace; use crate::policy::space::Space; use crate::util::alloc::{allocator, Allocator}; use crate::util::opaque_pointer::*; +use crate::util::track::track_malloc; use crate::util::Address; use crate::vm::VMBinding; @@ -42,7 +43,10 @@ impl Allocator for LargeObjectAllocator { let cell: Address = self.alloc_slow(size, align, offset); // We may get a null ptr from alloc due to the VM being OOM if !cell.is_zero() { - allocator::align_allocation::(cell, align, offset) + let result = allocator::align_allocation::(cell, align, offset); + + track_malloc(result, size, true); + result } else { cell } diff --git a/src/util/heap/blockpageresource.rs b/src/util/heap/blockpageresource.rs index 7e40a25219..e500be2253 100644 --- a/src/util/heap/blockpageresource.rs +++ b/src/util/heap/blockpageresource.rs @@ -9,6 +9,7 @@ use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::linear_scan::Region; use crate::util::opaque_pointer::*; use crate::util::rust_util::zeroed_alloc::new_zeroed_vec; +use crate::util::track::{track_mempool, track_mempool_alloc, track_mempool_free, untrack_mempool}; use crate::vm::*; use atomic::Ordering; use spin::RwLock; @@ -30,6 +31,10 @@ pub struct BlockPageResource { } impl PageResource for BlockPageResource { + fn track(&self) { + track_mempool(self, 0, false); + } + fn common(&self) -> &CommonPageResource { self.flpr.common() } @@ -58,6 +63,12 @@ impl PageResource for BlockPageResource { } } +impl Drop for BlockPageResource { + fn drop(&mut self) { + untrack_mempool(self); + } +} + impl BlockPageResource { /// Block granularity in pages const LOG_PAGES: usize = B::LOG_BYTES - LOG_BYTES_IN_PAGE as usize; @@ -136,6 +147,7 @@ impl BlockPageResource { self.block_queue.add_global_array(array); // Finish slow-allocation self.commit_pages(reserved_pages, required_pages, tls); + track_mempool_alloc(self, first_block, required_pages * BYTES_IN_PAGE); Result::Ok(PRAllocResult { start: first_block, pages: required_pages, @@ -156,6 +168,7 @@ impl BlockPageResource { // Fast allocate from the blocks list if let Some(block) = self.block_queue.pop() { self.commit_pages(reserved_pages, required_pages, tls); + track_mempool_alloc(self, block.start(), required_pages * BYTES_IN_PAGE); return Result::Ok(PRAllocResult { start: block.start(), pages: required_pages, @@ -170,6 +183,7 @@ impl BlockPageResource { let pages = 1 << Self::LOG_PAGES; debug_assert!(pages as usize <= self.common().accounting.get_committed_pages()); self.common().accounting.release(pages as _); + track_mempool_free(self, block.start()); self.block_queue.push(block) } diff --git a/src/util/heap/externalpageresource.rs b/src/util/heap/externalpageresource.rs index 3c8a8d3c2f..b0ee43efdf 100644 --- a/src/util/heap/externalpageresource.rs +++ b/src/util/heap/externalpageresource.rs @@ -28,6 +28,10 @@ pub struct ExternalPages { } impl PageResource for ExternalPageResource { + fn track(&self) { + /* cannot track external pages reliably? */ + } + fn common(&self) -> &CommonPageResource { &self.common } diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index 1c9dabf460..b5e107df22 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -17,6 +17,7 @@ use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::memory; use crate::util::opaque_pointer::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; +use crate::util::track::{track_mempool, track_mempool_alloc, track_mempool_free, untrack_mempool}; use crate::vm::*; use std::marker::PhantomData; @@ -41,6 +42,10 @@ struct FreeListPageResourceSync { } impl PageResource for FreeListPageResource { + fn track(&self) { + track_mempool(self, 0, false); + } + fn common(&self) -> &CommonPageResource { &self.common } @@ -134,6 +139,9 @@ impl PageResource for FreeListPageResource { } } }; + + track_mempool_alloc(self, rtn, conversions::pages_to_bytes(required_pages)); + Result::Ok(PRAllocResult { start: rtn, pages: required_pages, @@ -344,6 +352,7 @@ impl FreeListPageResource { } self.common.accounting.release(pages as _); + track_mempool_free(self, first); let freed = sync.free_list.free(page_offset as _, true); sync.pages_currently_on_freelist += pages as usize; if !self.common.contiguous { @@ -389,3 +398,9 @@ impl FreeListPageResource { } } } + +impl Drop for FreeListPageResource { + fn drop(&mut self) { + untrack_mempool(self); + } +} diff --git a/src/util/heap/monotonepageresource.rs b/src/util/heap/monotonepageresource.rs index c30cb010cc..6ec2bfd2b6 100644 --- a/src/util/heap/monotonepageresource.rs +++ b/src/util/heap/monotonepageresource.rs @@ -2,7 +2,8 @@ use super::layout::vm_layout::{BYTES_IN_CHUNK, PAGES_IN_CHUNK}; use crate::policy::space::required_chunks; use crate::util::address::Address; use crate::util::constants::BYTES_IN_PAGE; -use crate::util::conversions::*; +use crate::util::conversions::{self, *}; +use crate::util::track::{track_mempool, track_mempool_alloc}; use std::ops::Range; use std::sync::{Mutex, MutexGuard}; @@ -45,6 +46,10 @@ pub enum MonotonePageResourceConditional { Discontiguous, } impl PageResource for MonotonePageResource { + fn track(&self) { + track_mempool(self, 0, true); + } + fn common(&self) -> &CommonPageResource { &self.common } @@ -149,7 +154,7 @@ impl PageResource for MonotonePageResource { sync.current_chunk = chunk_align_down(sync.cursor); } self.commit_pages(reserved_pages, required_pages, tls); - + track_mempool_alloc(self, rtn, conversions::pages_to_bytes(required_pages)); Result::Ok(PRAllocResult { start: rtn, pages: required_pages, diff --git a/src/util/heap/pageresource.rs b/src/util/heap/pageresource.rs index e41eb5dd76..ad513ce390 100644 --- a/src/util/heap/pageresource.rs +++ b/src/util/heap/pageresource.rs @@ -10,6 +10,9 @@ use crate::util::heap::PageAccounting; use crate::vm::VMBinding; pub trait PageResource: 'static { + /// Track this page resource for memory tools like Valgrind. + fn track(&self); + /// Allocate pages from this resource. /// Simply bump the cursor, and fail if we hit the sentinel. /// Return The start of the first page if successful, zero on failure. diff --git a/src/util/memory.rs b/src/util/memory.rs index 9a9d4d16dc..f19e0470df 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -353,6 +353,21 @@ pub fn handle_mmap_error( /// This function is currently left empty for non-linux, and should be implemented in the future. /// As the function is only used for assertions, MMTk will still run even if we never panic. pub(crate) fn panic_if_unmapped(_start: Address, _size: usize, _anno: &MmapAnnotation) { + #[cfg(feature = "crabgrind")] + { + use crabgrind::memcheck::Error; + let result = crabgrind::memcheck::is_defined(_start.to_mut_ptr(), _size); + match result { + Ok(_) => panic!("{} of size {} is not mapped", _start, _size), + Err(err) => match err { + Error::NotAddressable(addr) => { + panic!("Address {addr:x} is not addressable, start={_start}"); + } + + _ => (), + }, + } + } #[cfg(target_os = "linux")] { let flags = MMAP_FLAGS; diff --git a/src/util/metadata/vo_bit/helper.rs b/src/util/metadata/vo_bit/helper.rs index 4dd06459de..622bd1945f 100644 --- a/src/util/metadata/vo_bit/helper.rs +++ b/src/util/metadata/vo_bit/helper.rs @@ -29,6 +29,7 @@ use crate::{ util::{ linear_scan::Region, metadata::{vo_bit, MetadataSpec}, + track::{track_free, tracking_enabled}, ObjectReference, }, vm::{ObjectModel, VMBinding}, @@ -184,6 +185,18 @@ pub(crate) fn on_object_forwarded(new_object: ObjectReference) { } pub(crate) fn on_region_swept(region: &R, is_occupied: bool) { + if tracking_enabled() { + let mut cursor = region.start(); + while cursor < region.end() { + if let Some(object) = vo_bit::is_vo_bit_set_for_addr(cursor) { + if object.is_live() { + track_free(object.to_object_start::(), 0); + } + } + cursor += VM::MIN_ALIGNMENT; + } + } + match strategy::() { VOBitUpdateStrategy::ClearAndReconstruct => { // Do nothing. The VO bit metadata is already reconstructed. diff --git a/src/util/mod.rs b/src/util/mod.rs index d22c29a2e3..1dce4dc1df 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -68,6 +68,7 @@ pub(crate) mod sanity; pub(crate) mod slot_logger; /// Utils for collecting statistics. pub(crate) mod statistics; +pub(crate) mod track; /// A treadmill implementation. pub(crate) mod treadmill; diff --git a/src/util/track.rs b/src/util/track.rs new file mode 100644 index 0000000000..8a204b8510 --- /dev/null +++ b/src/util/track.rs @@ -0,0 +1,150 @@ +//! Track memory ranges for tools like Valgrind address sanitizer, or other memory checkers. + +use crate::util::Address; + +pub fn tracking_enabled() -> bool { + #[cfg(feature = "crabgrind")] + { + crabgrind::run_mode() != crabgrind::RunMode::Native + } + + #[cfg(not(feature = "crabgrind"))] + { + false + } +} + +pub fn track_malloc(p: Address, size: usize, zero: bool) { + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::alloc::malloc(p.to_mut_ptr(), size, 0, zero) + } + + #[cfg(not(feature = "crabgrind"))] + { + let _ = p; + let _ = size; + let _ = zero; + } +} + +pub fn track_free(p: Address, size: usize) { + let _ = size; + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::alloc::free(p.to_mut_ptr(), 0); + } + #[cfg(not(feature = "crabgrind"))] + { + let _ = p; + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum MemState { + NoAccess, + Undefined, + Defined, + DefinedIfAddressable, +} + +pub fn track_mem(p: Address, size: usize, state: MemState) { + #[cfg(feature = "crabgrind")] + { + let state = match state { + MemState::Defined => crabgrind::memcheck::MemState::Defined, + MemState::DefinedIfAddressable => crabgrind::memcheck::MemState::DefinedIfAddressable, + MemState::NoAccess => crabgrind::memcheck::MemState::NoAccess, + MemState::Undefined => crabgrind::memcheck::MemState::Undefined, + }; + + crabgrind::memcheck::mark_mem(p.to_mut_ptr(), size, state); + } + + #[cfg(not(feature = "crabgrind"))] + { + let _ = p; + let _ = size; + let _ = state; + } +} + +/// Track a memory pool. Read [Memory Pools](https://valgrind.org/docs/manual/mc-manual.html#mc-manual.mempools) +/// of valgrind for more information. +/// +/// # Parameters +/// +/// - `pool`: The memory pool to track. +/// - `redzone`: Redzone in between chunks. +/// - `is_zeroed`: Whether the memory pool is zeroed. +pub fn track_mempool(pool: &T, redzone: usize, is_zeroed: bool) { + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::mempool::create( + Address::from_ref(pool).to_mut_ptr(), + redzone, + is_zeroed, + Some(crabgrind::memcheck::mempool::AUTO_FREE | crabgrind::memcheck::mempool::METAPOOL), + ); + } + + #[cfg(not(feature = "crabgrind"))] + { + let _ = pool; + let _ = redzone; + let _ = is_zeroed; + } +} + +/// Untrack a memory pool. This destroys the memory pool in the memory checker. +pub fn untrack_mempool(pool: &T) { + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::mempool::destroy(Address::from_ref(pool).to_mut_ptr()); + } + #[cfg(not(feature = "crabgrind"))] + { + let _ = pool; + } +} + +/// Associate a piece of memory with a memory pool. +/// +/// # Parameters +/// - `pool`: The memory pool to associate with. +/// - `addr`: The address of the memory to associate. +/// - `size`: The size of the memory to associate. +pub fn track_mempool_alloc(pool: &T, addr: Address, size: usize) { + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::mempool::alloc( + Address::from_ptr(pool as *const T as *const u8).to_mut_ptr(), + addr.to_mut_ptr(), + size, + ); + } + + #[cfg(not(feature = "crabgrind"))] + { + let _ = pool; + let _ = addr; + let _ = size; + } +} + +/// Disassociate a piece of memory with a memory pool. +/// +/// # Parameters +/// - `pool`: The memory pool to disassociate with. +/// - `addr`: The address of the memory to disassociate. +pub fn track_mempool_free(pool: &T, addr: Address) { + #[cfg(feature = "crabgrind")] + { + crabgrind::memcheck::mempool::free(Address::from_ref(pool).to_mut_ptr(), addr.to_mut_ptr()); + } + #[cfg(not(feature = "crabgrind"))] + { + let _ = pool; + let _ = addr; + } +}