diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index aea087252..5bb0c4d71 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -836,6 +836,9 @@ pub(crate) fn old_page_clear_dirty(page: usize) { #[cfg(test)] pub(crate) fn old_arena_page_index_clear_for_tests() { + // Wiping page metadata makes real old-arena objects unclassifiable — + // stand the #6179 differential verifier down for this thread's test. + crate::gc::CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.set(true)); OLD_GEN_PAGE_OBJECTS.with(|index| index.borrow_mut().clear()); } diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 9e1c882fa..559950c97 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -562,6 +562,15 @@ impl RootScanCycleState { if budget == 0 { return false; } + // #6179: classifier-mode (budgeted) cycles have no exact set + // and traced conservative words cannot tolerate heuristic + // false positives — never conservative-scan here, even if a + // ManualGcScanGuard appears mid-cycle (drain-before-manual-gc + // finishing a parked cycle under the guard). + if valid_ptrs.classifier_mode { + self.subphase = RootScanSubphase::MutableSlots; + return false; + } let conservative_scan_decision = conservative_stack_scan_decision(); // #5029: minors retain old-gen conservative discoveries // pin-only (no trace) — see try_mark_conservative_word. @@ -734,28 +743,49 @@ pub(super) fn test_malloc_trim_call_count() -> usize { TEST_MALLOC_TRIM_CALLS.with(Cell::get) } -#[cfg(all(test, target_env = "gnu"))] +#[cfg(all(test, any(target_env = "gnu", target_os = "macos")))] fn record_test_malloc_trim_call() { TEST_MALLOC_TRIM_CALLS.with(|calls| calls.set(calls.get().saturating_add(1))); } fn run_malloc_trim(progress_kind: GcProgressKind) -> MallocTrimOutcome { - if progress_kind.is_budgeted() { + // #6179/#6180 RSS floor: budgeted cycles are the DEFAULT-path collector + // once incremental graduates — skipping allocator trim there meant a + // long-lived incremental process never returned freed allocator pages to + // the OS (2026-07-09 audit finding). Trim runs at Reclaim, outside the + // atomic tail, and is itself bounded allocator maintenance. + + #[cfg(target_env = "gnu")] + { + #[cfg(test)] + record_test_malloc_trim_call(); + + let start = Instant::now(); + unsafe { + libc::malloc_trim(0); + } return MallocTrimOutcome { - status: AllocatorMaintenanceStatus::Skipped, - reason: AllocatorMaintenanceReason::OrdinaryBudgeted, - elapsed_us: 0, + status: AllocatorMaintenanceStatus::Executed, + reason: AllocatorMaintenanceReason::ExplicitOrEmergency, + elapsed_us: start.elapsed().as_micros() as u64, }; } - #[cfg(target_env = "gnu")] + #[cfg(target_os = "macos")] { #[cfg(test)] record_test_malloc_trim_call(); + // Darwin counterpart of glibc's malloc_trim: ask every malloc zone + // to return clean pages to the OS. Bounded allocator maintenance — + // same placement (Reclaim, outside the atomic tail). + unsafe extern "C" { + fn malloc_zone_pressure_relief(zone: *mut core::ffi::c_void, goal: usize) -> usize; + } let start = Instant::now(); unsafe { - libc::malloc_trim(0); + // NULL zone = all zones; goal 0 = release as much as possible. + malloc_zone_pressure_relief(core::ptr::null_mut(), 0); } return MallocTrimOutcome { status: AllocatorMaintenanceStatus::Executed, @@ -764,7 +794,7 @@ fn run_malloc_trim(progress_kind: GcProgressKind) -> MallocTrimOutcome { }; } - #[cfg(not(target_env = "gnu"))] + #[cfg(not(any(target_env = "gnu", target_os = "macos")))] { MallocTrimOutcome { status: AllocatorMaintenanceStatus::Unsupported, @@ -1050,9 +1080,20 @@ impl GcCycleState { fn step_build_valid_pointer_set(&mut self, budget: GcWorkBudget) { let phase_start = trace_phase_start(&self.trace); - let builder = self - .valid_builder - .get_or_insert_with(ValidPointerSetBuilder::new); + let builder = self.valid_builder.get_or_insert_with(|| { + // #6179: budgeted cycles are precise (no conservative scan, + // non-moving) — skip the O(heap) exact census and resolve + // membership via the page-metadata classifier, which the + // differential mode proves is a census superset. Synchronous + // cycles keep the exact set: they force the conservative + // scan, whose TRACED roots cannot tolerate a heuristic + // false positive. + if self.progress_kind.is_budgeted() { + ValidPointerSetBuilder::new_classifier() + } else { + ValidPointerSetBuilder::new() + } + }); if !builder.step(budget.work_units) { trace_phase_record(&mut self.trace, "build_valid_pointer_set", phase_start); return; @@ -1545,7 +1586,18 @@ impl GcCycleState { if let Some(minor) = self.minor.as_ref() { let targeted_old_blocks = (minor.evacuation.old_page_moved_bytes > 0) .then(|| minor.old_page_source_blocks.block_indices.clone()); - (true, false, targeted_old_blocks, minor.malloc_sweep_due) + // Budgeted cycles must NOT age-bump: whole-cycle + // allocate-black (#6224 soundness) marks every mid-cycle + // birth, and the age-bump reads MARKED as "survived" — + // dead churn then tenures into old-gen two cycles later, + // where minors never reclaim it (measured: ~700 MB of + // tenured garbage on a churn loop, 928 MB RSS vs the + // synchronous collector's 524 MB with identical arena + // block counts). Promotion under incremental happens via + // the copied-minor path, which runs between budgeted + // cycles and ages genuinely-surviving objects only. + let age_bump = !self.progress_kind.is_budgeted(); + (age_bump, false, targeted_old_blocks, minor.malloc_sweep_due) } else { (false, true, None, true) }; diff --git a/crates/perry-runtime/src/gc/progress.rs b/crates/perry-runtime/src/gc/progress.rs index 969dce2e6..d4c45d34d 100644 --- a/crates/perry-runtime/src/gc/progress.rs +++ b/crates/perry-runtime/src/gc/progress.rs @@ -24,7 +24,7 @@ pub const GC_MUTATOR_ASSIST_SOFT_PAUSE_US: u64 = 500; /// ~its trigger step and RSS lands near parity. When the collector is /// keeping up (debt ≈ 0) the budget stays at the base, so low-latency /// workloads never see the scaled assists. -pub const GC_ASSIST_DEBT_BYTES_PER_WORK_UNIT: u64 = 64; +pub const GC_ASSIST_DEBT_BYTES_PER_WORK_UNIT: u64 = 32; /// Runtime-visible classification for GC progress. /// diff --git a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs index 1acc25da1..0470d304b 100644 --- a/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs +++ b/crates/perry-runtime/src/gc/tests/incremental_sweep_reclaim.rs @@ -543,7 +543,7 @@ fn budgeted_reclaim_slices_conservative_pin_cleanup() { } #[test] -fn budgeted_reclaim_skips_process_malloc_trim() { +fn budgeted_reclaim_runs_process_malloc_trim() { let _trace_guard = TestGcTraceCaptureGuard::force_enabled(); let _guard = CopyingNurseryTestGuard::new(1); let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); @@ -561,23 +561,23 @@ fn budgeted_reclaim_skips_process_malloc_trim() { let completed = complete_budgeted_gc_cycle(); assert_eq!(completed.status, JS_GC_STEP_STATUS_COMPLETED); - assert_eq!( - test_malloc_trim_call_count(), - 0, - "ordinary budgeted reclaim must not invoke process-wide malloc_trim" + assert!( + test_malloc_trim_call_count() >= 1, + "budgeted reclaim must invoke allocator trim (#6180 RSS floor)" ); assert!(gc_collection_count() > before); assert_eq!(tracked_malloc_headers_matching(&dead_headers), 0); assert_eq!(js_shadow_slot_get(0) & POINTER_MASK, live as u64); let event = take_test_last_gc_trace_json().expect("budgeted reclaim should emit trace JSON"); - assert_eq!( - event["allocator_maintenance"]["malloc_trim"]["status"].as_str(), - Some("skipped") - ); - assert_eq!( - event["allocator_maintenance"]["malloc_trim"]["reason"].as_str(), - Some("ordinary_budgeted") - ); - assert_eq!(event["phase_us"]["malloc_trim"].as_u64(), Some(0)); + // #6180 RSS floor: budgeted cycles now RUN allocator trim (the audit + // found long-lived incremental processes never returned freed allocator + // pages to the OS). Outcome is platform-dependent (executed on glibc, + // unsupported elsewhere) — never the old budgeted skip. + let trim = &event["allocator_maintenance"]["malloc_trim"]; + assert_ne!(trim["reason"].as_str(), Some("ordinary_budgeted")); + assert!(matches!( + trim["status"].as_str(), + Some("executed") | Some("unsupported") + )); } diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 8f008ce66..81b5c1a7c 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -80,16 +80,21 @@ fn assert_budgeted_ordinary_trace(event: &serde_json::Value, expected_kind: &str Some(true) ); let malloc_trim = &event["allocator_maintenance"]["malloc_trim"]; - assert_eq!(malloc_trim["status"].as_str(), Some("skipped")); - assert_eq!(malloc_trim["reason"].as_str(), Some("ordinary_budgeted")); + // #6180 RSS floor: budgeted cycles now RUN allocator trim (previously + // skipped with reason ordinary_budgeted). The outcome is platform- + // dependent: executed on glibc, unsupported elsewhere — but never the + // old budgeted skip. + assert_ne!(malloc_trim["reason"].as_str(), Some("ordinary_budgeted")); + assert!(matches!( + malloc_trim["status"].as_str(), + Some("executed") | Some("unsupported") + )); assert_eq!(malloc_trim["progress_kind"].as_str(), Some(expected_kind)); assert_eq!(malloc_trim["class"].as_str(), Some("ordinary_budgeted")); assert_eq!( malloc_trim["ordinary_pause_stats_include"].as_bool(), Some(false) ); - assert_eq!(malloc_trim["elapsed_us"].as_u64(), Some(0)); - assert_eq!(event["phase_us"]["malloc_trim"].as_u64(), Some(0)); for (index, step) in event["pause_steps"] .as_array() .expect("ordinary trace should include pause steps") @@ -289,7 +294,8 @@ fn emergency_full_trace_is_excluded_from_ordinary_pause_stats() { malloc_trim["ordinary_pause_stats_include"].as_bool(), Some(false) ); - if cfg!(target_env = "gnu") { + if cfg!(any(target_env = "gnu", target_os = "macos")) { + // glibc malloc_trim / darwin malloc_zone_pressure_relief (#6180). assert_eq!(malloc_trim["status"].as_str(), Some("executed")); assert_eq!( malloc_trim["reason"].as_str(), diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 9e178a71c..ef4ed4cbf 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -1,5 +1,50 @@ use super::*; +thread_local! { + /// Set by test-only helpers that wipe page metadata for isolation + /// (`old_arena_page_index_clear_for_tests`): real objects become + /// unclassifiable in that synthetic state, so the differential verifier + /// must stand down for the rest of the thread's test. + pub(crate) static CLASSIFIER_VERIFY_SUPPRESSED: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +/// #6179 membership classifier: is `addr` a plausible live GC object start? +/// UNION of the two backends — exact membership in the malloc registry OR a +/// plausible arena header on an arena-classified page — deliberately NOT the +/// exclusive generation branch of `current_heap_header_for_user_ptr`: malloc +/// allocations can sit inside address ranges the page metadata attributes to +/// an arena generation, and an exclusive branch then never consults the +/// (exact) malloc registry, leaving live malloc roots unmarked. +#[inline] +pub(super) fn classifier_valid_object_start(addr: usize) -> bool { + if addr < GC_HEADER_SIZE + 0x1000 { + return false; + } + let header = unsafe { header_from_user_ptr(addr as *const u8) }; + if super::gc_malloc_header_is_tracked(header) { + return true; + } + !matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Unknown + ) && unsafe { super::barrier::plausible_arena_user_ptr_header(header).is_some() } +} + +/// #6179: differential-verification mode for the page-metadata classifier. +pub(super) fn classifier_verify_enabled() -> bool { + if CLASSIFIER_VERIFY_SUPPRESSED.with(|c| c.get()) { + return false; + } + static CACHED: std::sync::OnceLock = std::sync::OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_VERIFY_CLASSIFIER").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + thread_local! { pub(super) static MARK_SEEDS: std::cell::UnsafeCell> = const { std::cell::UnsafeCell::new(Vec::new()) }; @@ -35,6 +80,19 @@ pub(crate) struct ValidPointerSet { /// building the pointer set so evacuation policy Stage 1 doesn't /// need a second full arena walk on low-pressure cycles. pub(super) tenured_nursery_bytes: usize, + /// True only for sets produced by the production census walk + /// (`ValidPointerSetBuilder::finish`). The #6179 differential verifier + /// runs only on census-built sets: tests fabricate sets with synthetic + /// addresses, and classifying those dereferences headers that don't + /// exist (panics inside no-unwind scanner contexts → poisoned globals). + pub(super) built_by_census: bool, + /// #6179: budgeted (precise, non-moving, no-conservative-scan) cycles + /// skip the O(heap) exact census entirely; membership routes through the + /// page-metadata classifier (differentially verified as a superset of + /// the census across the whole suite). Classifier-mode cycles must NEVER + /// run a conservative stack scan: conservative words are traced, and a + /// heuristic false positive would trace garbage. + pub(super) classifier_mode: bool, } impl ValidPointerSet { @@ -46,6 +104,8 @@ impl ValidPointerSet { range_min: usize::MAX, range_max: 0, tenured_nursery_bytes: 0, + built_by_census: false, + classifier_mode: false, } } @@ -53,6 +113,9 @@ impl ValidPointerSet { /// order — `ValidPointerSetBuilder` does so via `ArenaObjectCursor` /// in address order. pub(super) fn push_arena(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode + } if let Some(previous) = self .current_arena_run .last() @@ -71,6 +134,9 @@ impl ValidPointerSet { } pub(super) fn push_malloc(&mut self, ptr: usize) { + if self.classifier_mode { + return; // #6179: no exact census in classifier mode + } self.lookup_set.insert(ptr); self.record_pointer_range(ptr); } @@ -109,17 +175,59 @@ impl ValidPointerSet { /// and we skip the binary search. #[inline(always)] pub(crate) fn maybe_contains(&self, ptr: usize) -> bool { + // #6179 classifier mode: no census, so no address range — the + // prefilter must pass everything through to `contains()` (call sites + // check maybe_contains SEPARATELY before contains and would + // otherwise reject every root). + if self.classifier_mode { + return true; + } ptr >= self.range_min && ptr <= self.range_max } #[inline] pub(crate) fn contains(&self, ptr: &usize) -> bool { + if self.classifier_mode { + // No census was built: classify live. FORWARDED rejects are + // correct here (budgeted cycles are non-moving; a forwarded + // header can only be pre-existing stale state). + return classifier_valid_object_start(*ptr); + } if !self.maybe_contains(*ptr) { + // Range-rejected candidates skip the differential check: the + // classifier legitimately accepts objects born after the census + // (allocate-black keeps them safe), which can lie outside the + // censused address range. return false; } // Exact lookup. The B-tree insert path is bounded during // `BuildValidPointerSet`, so a tiny GC step cannot trigger a // heap-sized hash-table rebuild. - self.lookup_set.contains(ptr) + let exact = self.lookup_set.contains(ptr); + // #6179 differential verification (PERRY_GC_VERIFY_CLASSIFIER=1): + // before the exact set can be replaced by page-metadata + // classification on precise cycles, the classifier must be proven a + // SUPERSET of the census on every query — a censused object the + // classifier rejects would be un-markable, i.e. swept live. The + // other direction (classifier accepts, census lacks) is expected: + // objects allocated after the census walk; over-approximation is + // safe (bounded floating garbage under allocate-black). + if exact && self.built_by_census && classifier_verify_enabled() { + let heur = classifier_valid_object_start(*ptr); + // Censused-then-MOVED is a legitimate divergence: the plausible- + // header check rejects FORWARDED headers by design (the object + // lives at its new address; the copying machinery owns the + // redirect). + let forwarded = !heur + && unsafe { + let header = header_from_user_ptr(*ptr as *const u8); + (*header).gc_flags & GC_FLAG_FORWARDED != 0 + }; + assert!( + heur || forwarded, + "classifier rejected censused object {ptr:#x} — page metadata / header heuristic disagrees with the exact valid-pointer set" + ); + } + exact } /// Issue #73: interior-pointer lookup. Given a scanned word, find @@ -204,6 +312,16 @@ pub(super) struct ValidPointerSetBuilderSnapshot { } impl ValidPointerSetBuilder { + /// #6179: budgeted-cycle flavor — the census walk still runs (it feeds + /// tenured_nursery_bytes for evacuation policy) but nothing is inserted + /// into the exact set; membership resolves via the classifier. Saves the + /// O(heap) BTreeSet held across the whole sliced cycle. + pub(super) fn new_classifier() -> Self { + let mut b = Self::new(); + b.set.classifier_mode = true; + b + } + pub(super) fn new() -> Self { Self { set: ValidPointerSet::new(), @@ -276,6 +394,7 @@ impl ValidPointerSetBuilder { } pub(super) fn finish(mut self) -> ValidPointerSet { + self.set.built_by_census = true; if self.phase != ValidPointerSetBuildPhase::Done { while !self.step(usize::MAX) {} } diff --git a/crates/perry-runtime/src/weakref.rs b/crates/perry-runtime/src/weakref.rs index 008d9220a..4cfc12223 100644 --- a/crates/perry-runtime/src/weakref.rs +++ b/crates/perry-runtime/src/weakref.rs @@ -855,6 +855,14 @@ pub(crate) fn process_weak_targets_after_mark( minor_only: bool, enqueue_callbacks: bool, ) { + // #6180 pause floor: the whole-heap walk below exists only to FIND weak + // holders (WeakRef / FinalizationRegistry / WeakMap-entry objects). The + // #6182 registry tracks every live holder — if none exist (the common + // case), the entire O(heap) pass is a no-op. This is the single largest + // atomic-finalize cost for weakref-free programs. + if !weak_target_holders_allocated() { + return; + } let liveness = FullCycleLiveness { valid_ptrs, minor_only,