From 3f3c04a0896cb21dd82263010109322b61ccf1a8 Mon Sep 17 00:00:00 2001 From: Chia-hung Duan Date: Wed, 18 Dec 2024 23:50:11 +0000 Subject: [PATCH 1/3] [scudo] Add the record of number of attempted page release This also removes the `RangesReleased` which doesn't give much insight to whether we should adjust the heuristic of doing page release. --- compiler-rt/lib/scudo/standalone/primary32.h | 25 +++++++++-------- compiler-rt/lib/scudo/standalone/primary64.h | 28 ++++++++++---------- compiler-rt/lib/scudo/standalone/release.h | 8 ------ 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index 654b129d9f547..2f37cc96c0d40 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -387,7 +387,7 @@ template class SizeClassAllocator32 { struct ReleaseToOsInfo { uptr BytesInFreeListAtLastCheckpoint; - uptr RangesReleased; + uptr NumReleaseAttempted; uptr LastReleasedBytes; u64 LastReleaseAtNs; }; @@ -880,14 +880,14 @@ template class SizeClassAllocator32 { BytesInFreeList - Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint; } const uptr AvailableChunks = Sci->AllocatedUser / BlockSize; - Str->append(" %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " - "inuse: %6zu avail: %6zu releases: %6zu last released: %6zuK " - "latest pushed bytes: %6zuK\n", - ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10, - Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks, - InUse, AvailableChunks, Sci->ReleaseInfo.RangesReleased, - Sci->ReleaseInfo.LastReleasedBytes >> 10, - PushedBytesDelta >> 10); + Str->append( + " %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " + "inuse: %6zu avail: %6zu release count: %6zu last released: %6zuK " + "latest pushed bytes: %6zuK\n", + ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10, + Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks, InUse, + AvailableChunks, Sci->ReleaseInfo.NumReleaseAttempted, + Sci->ReleaseInfo.LastReleasedBytes >> 10, PushedBytesDelta >> 10); } void getSizeClassFragmentationInfo(SizeClassInfo *Sci, uptr ClassId, @@ -972,6 +972,10 @@ template class SizeClassAllocator32 { const uptr Base = First * RegionSize; const uptr NumberOfRegions = Last - First + 1U; + // The following steps contribute to the majority time spent in page + // releasing thus we increment the counter here. + ++Sci->ReleaseInfo.NumReleaseAttempted; + // ==================================================================== // // 2. Mark the free blocks and we can tell which pages are in-use by // querying `PageReleaseContext`. @@ -991,9 +995,8 @@ template class SizeClassAllocator32 { }; releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - if (Recorder.getReleasedRangesCount() > 0) { + if (Recorder.getReleasedBytes() > 0) { Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes; } diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index e382e01f5d8e5..c99463d3946b0 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -530,7 +530,7 @@ template class SizeClassAllocator64 { struct ReleaseToOsInfo { uptr BytesInFreeListAtLastCheckpoint; - uptr RangesReleased; + uptr NumReleaseAttempted; uptr LastReleasedBytes; // The minimum size of pushed blocks to trigger page release. uptr TryReleaseThreshold; @@ -1141,17 +1141,18 @@ template class SizeClassAllocator64 { BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint; } const uptr TotalChunks = Region->MemMapInfo.AllocatedUser / BlockSize; - Str->append( - "%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " - "inuse: %6zu total: %6zu releases: %6zu last " - "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n", - Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), - Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, - Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, - Region->ReleaseInfo.RangesReleased, - Region->ReleaseInfo.LastReleasedBytes >> 10, - RegionPushedBytesDelta >> 10, Region->RegionBeg, - getRegionBaseByClassId(ClassId)); + Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " + "inuse: %6zu total: %6zu releases: %6zu last " + "release count: %6zuK latest pushed bytes: %6zuK region: 0x%zx " + "(0x%zx)\n", + Region->Exhausted ? "E" : " ", ClassId, + getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, + Region->FreeListInfo.PoppedBlocks, + Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, + Region->ReleaseInfo.NumReleaseAttempted, + Region->ReleaseInfo.LastReleasedBytes >> 10, + RegionPushedBytesDelta >> 10, Region->RegionBeg, + getRegionBaseByClassId(ClassId)); } void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, @@ -1322,7 +1323,7 @@ template class SizeClassAllocator64 { Context.getReleaseOffset()); auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - if (Recorder.getReleasedRangesCount() > 0) { + if (Recorder.getReleasedBytes() > 0) { // This is the case that we didn't hit the release threshold but it has // been past a certain period of time. Thus we try to release some pages // and if it does release some additional pages, it's hint that we are @@ -1342,7 +1343,6 @@ template class SizeClassAllocator64 { } Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); } Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h index 6353dafde3160..7a4912ec0b5a6 100644 --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -22,8 +22,6 @@ template class RegionReleaseRecorder { RegionReleaseRecorder(MemMapT *RegionMemMap, uptr Base, uptr Offset = 0) : RegionMemMap(RegionMemMap), Base(Base), Offset(Offset) {} - uptr getReleasedRangesCount() const { return ReleasedRangesCount; } - uptr getReleasedBytes() const { return ReleasedBytes; } uptr getBase() const { return Base; } @@ -33,12 +31,10 @@ template class RegionReleaseRecorder { void releasePageRangeToOS(uptr From, uptr To) { const uptr Size = To - From; RegionMemMap->releasePagesToOS(getBase() + Offset + From, Size); - ReleasedRangesCount++; ReleasedBytes += Size; } private: - uptr ReleasedRangesCount = 0; uptr ReleasedBytes = 0; MemMapT *RegionMemMap = nullptr; uptr Base = 0; @@ -52,8 +48,6 @@ class ReleaseRecorder { ReleaseRecorder(uptr Base, uptr Offset = 0, MapPlatformData *Data = nullptr) : Base(Base), Offset(Offset), Data(Data) {} - uptr getReleasedRangesCount() const { return ReleasedRangesCount; } - uptr getReleasedBytes() const { return ReleasedBytes; } uptr getBase() const { return Base; } @@ -62,12 +56,10 @@ class ReleaseRecorder { void releasePageRangeToOS(uptr From, uptr To) { const uptr Size = To - From; releasePagesToOS(Base, From + Offset, Size, Data); - ReleasedRangesCount++; ReleasedBytes += Size; } private: - uptr ReleasedRangesCount = 0; uptr ReleasedBytes = 0; // The starting address to release. Note that we may want to combine (Base + // Offset) as a new Base. However, the Base is retrieved from From f0c51be972dae55a583cb1fd76507effc5d24830 Mon Sep 17 00:00:00 2001 From: Chia-hung Duan Date: Thu, 19 Dec 2024 00:34:36 +0000 Subject: [PATCH 2/3] Address review comment: Rename the variable --- compiler-rt/lib/scudo/standalone/primary32.h | 8 +++---- compiler-rt/lib/scudo/standalone/primary64.h | 24 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index 2f37cc96c0d40..b0bc8b7592fca 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -387,7 +387,7 @@ template class SizeClassAllocator32 { struct ReleaseToOsInfo { uptr BytesInFreeListAtLastCheckpoint; - uptr NumReleaseAttempted; + uptr NumReleasesAttempted; uptr LastReleasedBytes; u64 LastReleaseAtNs; }; @@ -882,11 +882,11 @@ template class SizeClassAllocator32 { const uptr AvailableChunks = Sci->AllocatedUser / BlockSize; Str->append( " %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " - "inuse: %6zu avail: %6zu release count: %6zu last released: %6zuK " + "inuse: %6zu avail: %6zu release attempted: %6zu last released: %6zuK " "latest pushed bytes: %6zuK\n", ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10, Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks, InUse, - AvailableChunks, Sci->ReleaseInfo.NumReleaseAttempted, + AvailableChunks, Sci->ReleaseInfo.NumReleasesAttempted, Sci->ReleaseInfo.LastReleasedBytes >> 10, PushedBytesDelta >> 10); } @@ -974,7 +974,7 @@ template class SizeClassAllocator32 { // The following steps contribute to the majority time spent in page // releasing thus we increment the counter here. - ++Sci->ReleaseInfo.NumReleaseAttempted; + ++Sci->ReleaseInfo.NumReleasesAttempted; // ==================================================================== // // 2. Mark the free blocks and we can tell which pages are in-use by diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index c99463d3946b0..9228e85a86099 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -1141,18 +1141,18 @@ template class SizeClassAllocator64 { BytesInFreeList - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint; } const uptr TotalChunks = Region->MemMapInfo.AllocatedUser / BlockSize; - Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " - "inuse: %6zu total: %6zu releases: %6zu last " - "release count: %6zuK latest pushed bytes: %6zuK region: 0x%zx " - "(0x%zx)\n", - Region->Exhausted ? "E" : " ", ClassId, - getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, - Region->FreeListInfo.PoppedBlocks, - Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, - Region->ReleaseInfo.NumReleaseAttempted, - Region->ReleaseInfo.LastReleasedBytes >> 10, - RegionPushedBytesDelta >> 10, Region->RegionBeg, - getRegionBaseByClassId(ClassId)); + Str->append( + "%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " + "inuse: %6zu total: %6zu releases: %6zu last " + "release attempted: %6zuK latest pushed bytes: %6zuK region: 0x%zx " + "(0x%zx)\n", + Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), + Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, + Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, + Region->ReleaseInfo.NumReleaseAttempted, + Region->ReleaseInfo.LastReleasedBytes >> 10, + RegionPushedBytesDelta >> 10, Region->RegionBeg, + getRegionBaseByClassId(ClassId)); } void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, From d2d381e8982e10bb16b1ffd7d1353193f04a04bc Mon Sep 17 00:00:00 2001 From: Chia-hung Duan Date: Thu, 19 Dec 2024 00:46:53 +0000 Subject: [PATCH 3/3] Further minor fixes --- compiler-rt/lib/scudo/standalone/primary32.h | 2 +- compiler-rt/lib/scudo/standalone/primary64.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index b0bc8b7592fca..596c48f004b22 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -882,7 +882,7 @@ template class SizeClassAllocator32 { const uptr AvailableChunks = Sci->AllocatedUser / BlockSize; Str->append( " %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " - "inuse: %6zu avail: %6zu release attempted: %6zu last released: %6zuK " + "inuse: %6zu avail: %6zu releases attempted: %6zu last released: %6zuK " "latest pushed bytes: %6zuK\n", ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10, Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks, InUse, diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index 9228e85a86099..2b520ceb33148 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -530,7 +530,7 @@ template class SizeClassAllocator64 { struct ReleaseToOsInfo { uptr BytesInFreeListAtLastCheckpoint; - uptr NumReleaseAttempted; + uptr NumReleasesAttempted; uptr LastReleasedBytes; // The minimum size of pushed blocks to trigger page release. uptr TryReleaseThreshold; @@ -1144,12 +1144,12 @@ template class SizeClassAllocator64 { Str->append( "%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " "inuse: %6zu total: %6zu releases: %6zu last " - "release attempted: %6zuK latest pushed bytes: %6zuK region: 0x%zx " + "releases attempted: %6zuK latest pushed bytes: %6zuK region: 0x%zx " "(0x%zx)\n", Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, - Region->ReleaseInfo.NumReleaseAttempted, + Region->ReleaseInfo.NumReleasesAttempted, Region->ReleaseInfo.LastReleasedBytes >> 10, RegionPushedBytesDelta >> 10, Region->RegionBeg, getRegionBaseByClassId(ClassId));