From a1a39b905f130181f874a02d7e96756861114c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 11:47:36 +0200 Subject: [PATCH 1/6] perf(gc): classifier differential-verify mode for the valid-pointer set (#6179 groundwork) PERRY_GC_VERIFY_CLASSIFIER=1 asserts, on every positive contains() query against a CENSUS-BUILT set, that the page-metadata classifier (current_heap_header_for_user_ptr) accepts the object - the SUPERSET property that must hold before the exact BTreeSet can be skipped on precise cycles. Reverse divergence (classifier accepts, census lacks) is expected: objects born after the census; safe over-approximation under allocate-black. Verification carve-outs, each principled: - built_by_census: tests fabricating sets bypass verification. - CLASSIFIER_VERIFY_SUPPRESSED: test-only page-metadata wipes (old_arena_page_index_clear_for_tests) make real objects unclassifiable; production never clears the index. - FORWARDED headers: censused-then-moved objects are rejected by the plausible-header check BY DESIGN (the copying machinery owns the redirect). Full suite under verification: 1227/0. --- crates/perry-runtime/src/arena/page_meta.rs | 3 + crates/perry-runtime/src/gc/trace.rs | 62 ++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index aea0872527..5bb0c4d71a 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -836,6 +836,9 @@ pub(crate) fn old_page_clear_dirty(page: usize) { #[cfg(test)] pub(crate) fn old_arena_page_index_clear_for_tests() { + // Wiping page metadata makes real old-arena objects unclassifiable — + // stand the #6179 differential verifier down for this thread's test. + crate::gc::CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.set(true)); OLD_GEN_PAGE_OBJECTS.with(|index| index.borrow_mut().clear()); } diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 9e178a71cc..0e6b682c46 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1,5 +1,28 @@ use super::*; +thread_local! { + /// Set by test-only helpers that wipe page metadata for isolation + /// (`old_arena_page_index_clear_for_tests`): real objects become + /// unclassifiable in that synthetic state, so the differential verifier + /// must stand down for the rest of the thread's test. + pub(crate) static CLASSIFIER_VERIFY_SUPPRESSED: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +/// #6179: differential-verification mode for the page-metadata classifier. +pub(super) fn classifier_verify_enabled() -> bool { + if CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.get()) { + return false; + } + static CACHED: std::sync::OnceLock = std::sync::OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_VERIFY_CLASSIFIER").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + thread_local! { pub(super) static MARK_SEEDS: std::cell::UnsafeCell> = const { std::cell::UnsafeCell::new(Vec::new()) }; @@ -35,6 +58,12 @@ pub(crate) struct ValidPointerSet { /// building the pointer set so evacuation policy Stage 1 doesn't /// need a second full arena walk on low-pressure cycles. pub(super) tenured_nursery_bytes: usize, + /// True only for sets produced by the production census walk + /// (`ValidPointerSetBuilder::finish`). The #6179 differential verifier + /// runs only on census-built sets: tests fabricate sets with synthetic + /// addresses, and classifying those dereferences headers that don't + /// exist (panics inside no-unwind scanner contexts → poisoned globals). + pub(super) built_by_census: bool, } impl ValidPointerSet { @@ -46,6 +75,7 @@ impl ValidPointerSet { range_min: usize::MAX, range_max: 0, tenured_nursery_bytes: 0, + built_by_census: false, } } @@ -114,12 +144,41 @@ impl ValidPointerSet { #[inline] pub(crate) fn contains(&self, ptr: &usize) -> bool { if !self.maybe_contains(*ptr) { + // Range-rejected candidates skip the differential check: the + // classifier legitimately accepts objects born after the census + // (allocate-black keeps them safe), which can lie outside the + // censused address range. return false; } // Exact lookup. The B-tree insert path is bounded during // `BuildValidPointerSet`, so a tiny GC step cannot trigger a // heap-sized hash-table rebuild. - self.lookup_set.contains(ptr) + let exact = self.lookup_set.contains(ptr); + // #6179 differential verification (PERRY_GC_VERIFY_CLASSIFIER=1): + // before the exact set can be replaced by page-metadata + // classification on precise cycles, the classifier must be proven a + // SUPERSET of the census on every query — a censused object the + // classifier rejects would be un-markable, i.e. swept live. The + // other direction (classifier accepts, census lacks) is expected: + // objects allocated after the census walk; over-approximation is + // safe (bounded floating garbage under allocate-black). + if exact && self.built_by_census && classifier_verify_enabled() { + let heur = super::barrier::current_heap_header_for_user_ptr(*ptr, None).is_some(); + // Censused-then-MOVED is a legitimate divergence: the plausible- + // header check rejects FORWARDED headers by design (the object + // lives at its new address; the copying machinery owns the + // redirect). + let forwarded = !heur + && unsafe { + let header = header_from_user_ptr(*ptr as *const u8); + (*header).gc_flags & GC_FLAG_FORWARDED != 0 + }; + assert!( + heur || forwarded, + "classifier rejected censused object {ptr:#x} — page metadata / header heuristic disagrees with the exact valid-pointer set" + ); + } + exact } /// Issue #73: interior-pointer lookup. Given a scanned word, find @@ -276,6 +335,7 @@ impl ValidPointerSetBuilder { } pub(super) fn finish(mut self) -> ValidPointerSet { + self.set.built_by_census = true; if self.phase != ValidPointerSetBuildPhase::Done { while !self.step(usize::MAX) {} } From 4952c0d3f10d85d2881bb719442ffa3b992dffbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 12:02:26 +0200 Subject: [PATCH 2/6] perf(gc): budgeted cycles skip the exact valid-pointer census (#6179) Budgeted (precise, non-moving) cycles no longer build the O(heap) BTreeSet census held across the whole sliced cycle - membership resolves through classifier_valid_object_start: the UNION of exact malloc-registry membership and a plausible arena header on an arena-classified page. Union, not the exclusive generation branch of current_heap_header_for_user_ptr: malloc allocations can sit inside ranges the page metadata attributes to an arena generation, and the exclusive branch left live malloc roots unmarked. Safety anchors: - classifier-mode cycles NEVER conservative-scan (traced conservative words cannot tolerate heuristic false positives) - even if a ManualGcScanGuard appears mid-cycle via drain-before-manual-gc. - maybe_contains passes everything through in classifier mode (no census, no range; call sites prefilter separately). - Synchronous cycles keep the exact set: they force the conservative scan, which requires exactness. - The census walk still runs for budgeted cycles (evacuation policy reads tenured_nursery_bytes); only the set inserts are skipped. Differentially verified: PERRY_GC_VERIFY_CLASSIFIER=1 across the full suite asserts the classifier is a superset of the census on every census-built set. 1227/0 with and without verification. --- crates/perry-runtime/src/gc/cycle.rs | 26 ++++++++++-- crates/perry-runtime/src/gc/trace.rs | 61 +++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 9e1c882faf..d25cd08d5a 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -562,6 +562,15 @@ impl RootScanCycleState { if budget == 0 { return false; } + // #6179: classifier-mode (budgeted) cycles have no exact set + // and traced conservative words cannot tolerate heuristic + // false positives — never conservative-scan here, even if a + // ManualGcScanGuard appears mid-cycle (drain-before-manual-gc + // finishing a parked cycle under the guard). + if valid_ptrs.classifier_mode { + self.subphase = RootScanSubphase::MutableSlots; + return false; + } let conservative_scan_decision = conservative_stack_scan_decision(); // #5029: minors retain old-gen conservative discoveries // pin-only (no trace) — see try_mark_conservative_word. @@ -1050,9 +1059,20 @@ impl GcCycleState { fn step_build_valid_pointer_set(&mut self, budget: GcWorkBudget) { let phase_start = trace_phase_start(&self.trace); - let builder = self - .valid_builder - .get_or_insert_with(ValidPointerSetBuilder::new); + let builder = self.valid_builder.get_or_insert_with(|| { + // #6179: budgeted cycles are precise (no conservative scan, + // non-moving) — skip the O(heap) exact census and resolve + // membership via the page-metadata classifier, which the + // differential mode proves is a census superset. Synchronous + // cycles keep the exact set: they force the conservative + // scan, whose TRACED roots cannot tolerate a heuristic + // false positive. + if self.progress_kind.is_budgeted() { + ValidPointerSetBuilder::new_classifier() + } else { + ValidPointerSetBuilder::new() + } + }); if !builder.step(budget.work_units) { trace_phase_record(&mut self.trace, "build_valid_pointer_set", phase_start); return; diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 0e6b682c46..ef4ed4cbff 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -9,6 +9,28 @@ thread_local! { const { std::cell::Cell::new(false) }; } +/// #6179 membership classifier: is `addr` a plausible live GC object start? +/// UNION of the two backends — exact membership in the malloc registry OR a +/// plausible arena header on an arena-classified page — deliberately NOT the +/// exclusive generation branch of `current_heap_header_for_user_ptr`: malloc +/// allocations can sit inside address ranges the page metadata attributes to +/// an arena generation, and an exclusive branch then never consults the +/// (exact) malloc registry, leaving live malloc roots unmarked. +#[inline] +pub(super) fn classifier_valid_object_start(addr: usize) -> bool { + if addr < GC_HEADER_SIZE + 0x1000 { + return false; + } + let header = unsafe { header_from_user_ptr(addr as *const u8) }; + if super::gc_malloc_header_is_tracked(header) { + return true; + } + !matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Unknown + ) && unsafe { super::barrier::plausible_arena_user_ptr_header(header).is_some() } +} + /// #6179: differential-verification mode for the page-metadata classifier. pub(super) fn classifier_verify_enabled() -> bool { if CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.get()) { @@ -64,6 +86,13 @@ pub(crate) struct ValidPointerSet { /// addresses, and classifying those dereferences headers that don't /// exist (panics inside no-unwind scanner contexts → poisoned globals). pub(super) built_by_census: bool, + /// #6179: budgeted (precise, non-moving, no-conservative-scan) cycles + /// skip the O(heap) exact census entirely; membership routes through the + /// page-metadata classifier (differentially verified as a superset of + /// the census across the whole suite). Classifier-mode cycles must NEVER + /// run a conservative stack scan: conservative words are traced, and a + /// heuristic false positive would trace garbage. + pub(super) classifier_mode: bool, } impl ValidPointerSet { @@ -76,6 +105,7 @@ impl ValidPointerSet { range_max: 0, tenured_nursery_bytes: 0, built_by_census: false, + classifier_mode: false, } } @@ -83,6 +113,9 @@ impl ValidPointerSet { /// order — `ValidPointerSetBuilder` does so via `ArenaObjectCursor` /// in address order. pub(super) fn push_arena(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode + } if let Some(previous) = self .current_arena_run .last() @@ -101,6 +134,9 @@ impl ValidPointerSet { } pub(super) fn push_malloc(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode + } self.lookup_set.insert(ptr); self.record_pointer_range(ptr); } @@ -139,10 +175,23 @@ impl ValidPointerSet { /// and we skip the binary search. #[inline(always)] pub(crate) fn maybe_contains(&self, ptr: usize) -> bool { + // #6179 classifier mode: no census, so no address range — the + // prefilter must pass everything through to `contains()` (call sites + // check maybe_contains SEPARATELY before contains and would + // otherwise reject every root). + if self.classifier_mode { + return true; + } ptr >= self.range_min && ptr <= self.range_max } #[inline] pub(crate) fn contains(&self, ptr: &usize) -> bool { + if self.classifier_mode { + // No census was built: classify live. FORWARDED rejects are + // correct here (budgeted cycles are non-moving; a forwarded + // header can only be pre-existing stale state). + return classifier_valid_object_start(*ptr); + } if !self.maybe_contains(*ptr) { // Range-rejected candidates skip the differential check: the // classifier legitimately accepts objects born after the census @@ -163,7 +212,7 @@ impl ValidPointerSet { // objects allocated after the census walk; over-approximation is // safe (bounded floating garbage under allocate-black). if exact && self.built_by_census && classifier_verify_enabled() { - let heur = super::barrier::current_heap_header_for_user_ptr(*ptr, None).is_some(); + let heur = classifier_valid_object_start(*ptr); // Censused-then-MOVED is a legitimate divergence: the plausible- // header check rejects FORWARDED headers by design (the object // lives at its new address; the copying machinery owns the @@ -263,6 +312,16 @@ pub(super) struct ValidPointerSetBuilderSnapshot { } impl ValidPointerSetBuilder { + /// #6179: budgeted-cycle flavor — the census walk still runs (it feeds + /// tenured_nursery_bytes for evacuation policy) but nothing is inserted + /// into the exact set; membership resolves via the classifier. Saves the + /// O(heap) BTreeSet held across the whole sliced cycle. + pub(super) fn new_classifier() -> Self { + let mut b = Self::new(); + b.set.classifier_mode = true; + b + } + pub(super) fn new() -> Self { Self { set: ValidPointerSet::new(), From 4e711bad6ed4f8af031dd1ee174de16fbdaf9154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 12:15:39 +0200 Subject: [PATCH 3/6] =?UTF-8?q?perf(gc):=20RSS=20floor=20=E2=80=94=20pacin?= =?UTF-8?q?g=20gain=201/32=20+=20allocator=20trim=20on=20budgeted=20cycles?= =?UTF-8?q?=20(#6180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurement redirect: removing the exact census barely moved RSS (929 vs 921 MB on the churn benchmark) - the ~1.8x gap over the synchronous collector is pacing equilibrium (floating garbage across the debt window + allocate-black retention) and allocator pages never returned to the OS, not the BTreeSet. Two levers: - GC_ASSIST_DEBT_BYTES_PER_WORK_UNIT 64 -> 32: doubles the pacing gain, halving equilibrium debt and with it the per-cycle floating-garbage window. - run_malloc_trim no longer skips budgeted cycles (2026-07-09 audit finding): a long-lived incremental process never returned freed allocator pages. Trim runs at Reclaim, outside the atomic tail. Telemetry/reclaim tests updated: budgeted trim outcome is platform- dependent (executed on glibc, unsupported elsewhere), never the old ordinary_budgeted skip. --- crates/perry-runtime/src/gc/cycle.rs | 12 +++++------ crates/perry-runtime/src/gc/progress.rs | 2 +- .../src/gc/tests/incremental_sweep_reclaim.rs | 21 ++++++++++--------- .../src/gc/tests/telemetry_verifier.rs | 13 ++++++++---- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index d25cd08d5a..6afb52768e 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -749,13 +749,11 @@ fn record_test_malloc_trim_call() { } fn run_malloc_trim(progress_kind: GcProgressKind) -> MallocTrimOutcome { - if progress_kind.is_budgeted() { - return MallocTrimOutcome { - status: AllocatorMaintenanceStatus::Skipped, - reason: AllocatorMaintenanceReason::OrdinaryBudgeted, - elapsed_us: 0, - }; - } + // #6179/#6180 RSS floor: budgeted cycles are the DEFAULT-path collector + // once incremental graduates — skipping allocator trim there meant a + // long-lived incremental process never returned freed allocator pages to + // the OS (2026-07-09 audit finding). Trim runs at Reclaim, outside the + // atomic tail, and is itself bounded allocator maintenance. #[cfg(target_env = "gnu")] { diff --git a/crates/perry-runtime/src/gc/progress.rs b/crates/perry-runtime/src/gc/progress.rs index 969dce2e66..d4c45d34dd 100644 --- a/crates/perry-runtime/src/gc/progress.rs +++ b/crates/perry-runtime/src/gc/progress.rs @@ -24,7 +24,7 @@ pub const GC_MUTATOR_ASSIST_SOFT_PAUSE_US: u64 = 500; /// ~its trigger step and RSS lands near parity. When the collector is /// keeping up (debt ≈ 0) the budget stays at the base, so low-latency /// workloads never see the scaled assists. -pub const GC_ASSIST_DEBT_BYTES_PER_WORK_UNIT: u64 = 64; +pub const GC_ASSIST_DEBT_BYTES_PER_WORK_UNIT: u64 = 32; /// Runtime-visible classification for GC progress. /// diff --git a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs index 1acc25da1e..93dc8e43a4 100644 --- a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs @@ -543,7 +543,7 @@ fn budgeted_reclaim_slices_conservative_pin_cleanup() { } #[test] -fn budgeted_reclaim_skips_process_malloc_trim() { +fn budgeted_reclaim_runs_process_malloc_trim() { let _trace_guard = TestGcTraceCaptureGuard::force_enabled(); let _guard = CopyingNurseryTestGuard::new(1); let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); @@ -571,13 +571,14 @@ fn budgeted_reclaim_skips_process_malloc_trim() { assert_eq!(js_shadow_slot_get(0) & POINTER_MASK, live as u64); let event = take_test_last_gc_trace_json().expect("budgeted reclaim should emit trace JSON"); - assert_eq!( - event["allocator_maintenance"]["malloc_trim"]["status"].as_str(), - Some("skipped") - ); - assert_eq!( - event["allocator_maintenance"]["malloc_trim"]["reason"].as_str(), - Some("ordinary_budgeted") - ); - assert_eq!(event["phase_us"]["malloc_trim"].as_u64(), Some(0)); + // #6180 RSS floor: budgeted cycles now RUN allocator trim (the audit + // found long-lived incremental processes never returned freed allocator + // pages to the OS). Outcome is platform-dependent (executed on glibc, + // unsupported elsewhere) — never the old budgeted skip. + let trim = &event["allocator_maintenance"]["malloc_trim"]; + assert_ne!(trim["reason"].as_str(), Some("ordinary_budgeted")); + assert!(matches!( + trim["status"].as_str(), + Some("executed") | Some("unsupported") + )); } diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 8f008ce663..eed2245ce2 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -80,16 +80,21 @@ fn assert_budgeted_ordinary_trace(event: &serde_json::Value, expected_kind: &str Some(true) ); let malloc_trim = &event["allocator_maintenance"]["malloc_trim"]; - assert_eq!(malloc_trim["status"].as_str(), Some("skipped")); - assert_eq!(malloc_trim["reason"].as_str(), Some("ordinary_budgeted")); + // #6180 RSS floor: budgeted cycles now RUN allocator trim (previously + // skipped with reason ordinary_budgeted). The outcome is platform- + // dependent: executed on glibc, unsupported elsewhere — but never the + // old budgeted skip. + assert_ne!(malloc_trim["reason"].as_str(), Some("ordinary_budgeted")); + assert!(matches!( + malloc_trim["status"].as_str(), + Some("executed") | Some("unsupported") + )); assert_eq!(malloc_trim["progress_kind"].as_str(), Some(expected_kind)); assert_eq!(malloc_trim["class"].as_str(), Some("ordinary_budgeted")); assert_eq!( malloc_trim["ordinary_pause_stats_include"].as_bool(), Some(false) ); - assert_eq!(malloc_trim["elapsed_us"].as_u64(), Some(0)); - assert_eq!(event["phase_us"]["malloc_trim"].as_u64(), Some(0)); for (index, step) in event["pause_steps"] .as_array() .expect("ordinary trace should include pause steps") From 3fe5d8464d2659db68623c7ee5da52b8928b2bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 12:28:23 +0200 Subject: [PATCH 4/6] =?UTF-8?q?perf(gc):=20darwin=20allocator=20trim=20?= =?UTF-8?q?=E2=80=94=20malloc=5Fzone=5Fpressure=5Frelief=20at=20Reclaim=20?= =?UTF-8?q?(#6180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budgeted-trim lever only worked on glibc; darwin returned Unsupported, so macOS incremental processes never returned clean allocator pages (B2 churn RSS pinned at ~928MB with healthy pacing). NULL zone = all zones, goal 0 = release everything releasable. Same placement as glibc malloc_trim: Reclaim, outside the atomic tail. Tests updated for the platform matrix. --- crates/perry-runtime/src/gc/cycle.rs | 27 +++++++++++++++++-- .../src/gc/tests/incremental_sweep_reclaim.rs | 7 +++-- .../src/gc/tests/telemetry_verifier.rs | 3 ++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 6afb52768e..f0a692c1d7 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -743,7 +743,7 @@ pub(super) fn test_malloc_trim_call_count() -> usize { TEST_MALLOC_TRIM_CALLS.with(Cell::get) } -#[cfg(all(test, target_env = "gnu"))] +#[cfg(all(test, any(target_env = "gnu", target_os = "macos")))] fn record_test_malloc_trim_call() { TEST_MALLOC_TRIM_CALLS.with(|calls| calls.set(calls.get().saturating_add(1))); } @@ -771,7 +771,30 @@ fn run_malloc_trim(progress_kind: GcProgressKind) -> MallocTrimOutcome { }; } - #[cfg(not(target_env = "gnu"))] + #[cfg(target_os = "macos")] + { + #[cfg(test)] + record_test_malloc_trim_call(); + + // Darwin counterpart of glibc's malloc_trim: ask every malloc zone + // to return clean pages to the OS. Bounded allocator maintenance — + // same placement (Reclaim, outside the atomic tail). + unsafe extern "C" { + fn malloc_zone_pressure_relief(zone: *mut core::ffi::c_void, goal: usize) -> usize; + } + let start = Instant::now(); + unsafe { + // NULL zone = all zones; goal 0 = release as much as possible. + malloc_zone_pressure_relief(core::ptr::null_mut(), 0); + } + return MallocTrimOutcome { + status: AllocatorMaintenanceStatus::Executed, + reason: AllocatorMaintenanceReason::ExplicitOrEmergency, + elapsed_us: start.elapsed().as_micros() as u64, + }; + } + + #[cfg(not(any(target_env = "gnu", target_os = "macos")))] { MallocTrimOutcome { status: AllocatorMaintenanceStatus::Unsupported, diff --git a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs index 93dc8e43a4..0470d304bc 100644 --- a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs @@ -561,10 +561,9 @@ fn budgeted_reclaim_runs_process_malloc_trim() { let completed = complete_budgeted_gc_cycle(); assert_eq!(completed.status, JS_GC_STEP_STATUS_COMPLETED); - assert_eq!( - test_malloc_trim_call_count(), - 0, - "ordinary budgeted reclaim must not invoke process-wide malloc_trim" + assert!( + test_malloc_trim_call_count() >= 1, + "budgeted reclaim must invoke allocator trim (#6180 RSS floor)" ); assert!(gc_collection_count() > before); assert_eq!(tracked_malloc_headers_matching(&dead_headers), 0); diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index eed2245ce2..81b5c1a7c6 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -294,7 +294,8 @@ fn emergency_full_trace_is_excluded_from_ordinary_pause_stats() { malloc_trim["ordinary_pause_stats_include"].as_bool(), Some(false) ); - if cfg!(target_env = "gnu") { + if cfg!(any(target_env = "gnu", target_os = "macos")) { + // glibc malloc_trim / darwin malloc_zone_pressure_relief (#6180). assert_eq!(malloc_trim["status"].as_str(), Some("executed")); assert_eq!( malloc_trim["reason"].as_str(), From ab625780aa8c79109b7bf67332b05f685c42a47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 12:34:13 +0200 Subject: [PATCH 5/6] =?UTF-8?q?fix(gc):=20budgeted=20sweeps=20must=20not?= =?UTF-8?q?=20age-bump=20=E2=80=94=20allocate-black=20is=20not=20survival?= =?UTF-8?q?=20(#6180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whole-cycle allocate-black (#6224 soundness) marks every mid-cycle birth; the sweep age-bump read MARKED as survived, so dead churn tenured into old-gen two cycles later where minors never reclaim it. Measured on the churn benchmark: ~700 MB of tenured garbage (928 MB RSS vs 524 MB synchronous, identical 203-block arena commitment). Budgeted sweeps skip aging; promotion under incremental flows through the copied-minor path, which ages genuinely-surviving objects only. --- crates/perry-runtime/src/gc/cycle.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index f0a692c1d7..559950c977 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1586,7 +1586,18 @@ impl GcCycleState { if let Some(minor) = self.minor.as_ref() { let targeted_old_blocks = (minor.evacuation.old_page_moved_bytes > 0) .then(|| minor.old_page_source_blocks.block_indices.clone()); - (true, false, targeted_old_blocks, minor.malloc_sweep_due) + // Budgeted cycles must NOT age-bump: whole-cycle + // allocate-black (#6224 soundness) marks every mid-cycle + // birth, and the age-bump reads MARKED as "survived" — + // dead churn then tenures into old-gen two cycles later, + // where minors never reclaim it (measured: ~700 MB of + // tenured garbage on a churn loop, 928 MB RSS vs the + // synchronous collector's 524 MB with identical arena + // block counts). Promotion under incremental happens via + // the copied-minor path, which runs between budgeted + // cycles and ages genuinely-surviving objects only. + let age_bump = !self.progress_kind.is_budgeted(); + (age_bump, false, targeted_old_blocks, minor.malloc_sweep_due) } else { (false, true, None, true) }; From 6e838b14ee9ebbd55299c471f77a7569c85380b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 10 Jul 2026 12:41:52 +0200 Subject: [PATCH 6/6] perf(gc): weak processing skips the whole-heap walk when no weak holders exist (#6180) The AtomicFinalize WeakProcessing pass walks the entire arena solely to FIND weak holders; the #6182 registry already tracks every live one. Empty registry (the common case) -> the O(heap) pass is a no-op, removing the single largest atomic-finalize cost for weakref-free programs. Registry-iteration for the nonzero case is the tracked refinement. --- crates/perry-runtime/src/weakref.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/perry-runtime/src/weakref.rs b/crates/perry-runtime/src/weakref.rs index 008d9220a0..4cfc122234 100644 --- a/crates/perry-runtime/src/weakref.rs +++ b/crates/perry-runtime/src/weakref.rs @@ -855,6 +855,14 @@ pub(crate) fn process_weak_targets_after_mark( minor_only: bool, enqueue_callbacks: bool, ) { + // #6180 pause floor: the whole-heap walk below exists only to FIND weak + // holders (WeakRef / FinalizationRegistry / WeakMap-entry objects). The + // #6182 registry tracks every live holder — if none exist (the common + // case), the entire O(heap) pass is a no-op. This is the single largest + // atomic-finalize cost for weakref-free programs. + if !weak_target_holders_allocated() { + return; + } let liveness = FullCycleLiveness { valid_ptrs, minor_only,