From 166e3c200ba145fc637bbc6b197f318b0c6ba944 Mon Sep 17 00:00:00 2001 From: jarmen423 Date: Wed, 29 Jul 2026 17:41:51 +0000 Subject: [PATCH 1/3] perf(engine): skip full LPG serialize on clean GrafeoDB::close Close previously always used FlushReason::Explicit, which re-serializes every section even when ephemeral wrappers are clean. Prefer dirty-only Checkpoint when the session has no WAL records and no forced flush, preserving Explicit for mutations, WAL-off, and recovery. --- crates/grafeo-engine/src/database/flush.rs | 9 +- crates/grafeo-engine/src/database/index.rs | 24 ++- crates/grafeo-engine/src/database/mod.rs | 98 +++++++++++- crates/grafeo-engine/tests/grafeo_file.rs | 172 +++++++++++++++++++++ docs/TRACK_C_CLOSE_FAST_PATH.md | 55 +++++++ 5 files changed, 350 insertions(+), 8 deletions(-) create mode 100644 docs/TRACK_C_CLOSE_FAST_PATH.md diff --git a/crates/grafeo-engine/src/database/flush.rs b/crates/grafeo-engine/src/database/flush.rs index 9732da771..1c211c31c 100644 --- a/crates/grafeo-engine/src/database/flush.rs +++ b/crates/grafeo-engine/src/database/flush.rs @@ -21,10 +21,13 @@ use grafeo_storage::file::GrafeoFileManager; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum FlushReason { - /// Periodic checkpoint (timer-driven) or database close. - #[allow(dead_code)] // Used by async_ops (async-storage feature) + /// Periodic checkpoint (timer-driven). Dirty sections only. + /// Clean database close also uses this when no container rewrite is required + /// (see `GrafeoDB::close_needs_full_checkpoint`). + #[allow(dead_code)] // Used by async_ops (async-storage feature) and close fast path Checkpoint, - /// User-initiated `CHECKPOINT` command or `wal_checkpoint()` API. + /// User-initiated `CHECKPOINT` / `wal_checkpoint()` API, or close when + /// mutations/recovery require a full container rewrite. Explicit, } diff --git a/crates/grafeo-engine/src/database/index.rs b/crates/grafeo-engine/src/database/index.rs index 34156892f..86b053599 100644 --- a/crates/grafeo-engine/src/database/index.rs +++ b/crates/grafeo-engine/src/database/index.rs @@ -34,13 +34,20 @@ impl super::GrafeoDB { /// ``` pub fn create_property_index(&self, property: &str) { self.lpg_store().create_property_index(property); + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); } /// Drops an index on a node property. /// /// Returns `true` if the index existed and was removed. pub fn drop_property_index(&self, property: &str) -> bool { - self.lpg_store().drop_property_index(property) + let removed = self.lpg_store().drop_property_index(property); + #[cfg(feature = "grafeo-file")] + if removed { + self.mark_container_flush_required(); + } + removed } /// Returns `true` if the property has an index. @@ -176,6 +183,8 @@ impl super::GrafeoDB { "Empty vector index created: :{label}({property}) - 0 vectors, {d} dimensions, metric={metric_name}", metric_name = metric.name() ); + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); Ok(()) } else { Err(grafeo_common::utils::error::Error::Internal(format!( @@ -246,6 +255,8 @@ impl super::GrafeoDB { metric_name = metric.name() ); + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); Ok(()) } @@ -308,6 +319,8 @@ impl super::GrafeoDB { let removed = self.lpg_store().remove_vector_index(label, property); if removed { grafeo_info!("Vector index dropped: :{label}({property})"); + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); } removed } @@ -460,6 +473,8 @@ impl super::GrafeoDB { self.lpg_store() .add_text_index(label, property, Arc::new(RwLock::new(index))); + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); Ok(()) } @@ -468,7 +483,12 @@ impl super::GrafeoDB { /// Returns `true` if the index existed and was removed. #[cfg(feature = "text-index")] pub fn drop_text_index(&self, label: &str, property: &str) -> bool { - self.lpg_store().remove_text_index(label, property) + let removed = self.lpg_store().remove_text_index(label, property); + #[cfg(feature = "grafeo-file")] + if removed { + self.mark_container_flush_required(); + } + removed } /// Rebuilds a text index by re-scanning all matching nodes. diff --git a/crates/grafeo-engine/src/database/mod.rs b/crates/grafeo-engine/src/database/mod.rs index 31c3b4bf0..7158b97c1 100644 --- a/crates/grafeo-engine/src/database/mod.rs +++ b/crates/grafeo-engine/src/database/mod.rs @@ -185,6 +185,13 @@ pub struct GrafeoDB { /// Whether this database is open in read-only mode. /// When true, sessions automatically enforce read-only transactions. read_only: bool, + /// When true, close must rewrite the `.grafeo` container (full section + /// serialize). Set when WAL recovery applied unrecovered records, or when + /// a container-affecting mutation bypasses session WAL `record_count` + /// (e.g. vector/text index DDL). Session graph mutations already bump WAL + /// `record_count`; see [`Self::close_needs_full_checkpoint`]. + #[cfg(feature = "grafeo-file")] + container_flush_required: std::sync::atomic::AtomicBool, /// Named graph projections (virtual subgraphs), shared with sessions. projections: Arc>>>, @@ -406,6 +413,12 @@ impl GrafeoDB { Vec, )> = None; + // Sidecar/legacy WAL recovery can leave unrecovered mutations only in + // RAM while WalManager::record_count resets to 0 on reopen. Track that + // so close still forces a full container checkpoint. + #[cfg(feature = "grafeo-file")] + let mut recovered_wal_needs_flush = false; + // --- Single-file format (.grafeo) --- #[cfg(feature = "grafeo-file")] let file_manager: Option> = if is_read_only { @@ -504,6 +517,9 @@ impl GrafeoDB { if config.wal_enabled && fm.has_sidecar_wal() { let recovery = WalRecovery::new(fm.sidecar_wal_path()); let records = recovery.recover()?; + if !records.is_empty() { + recovered_wal_needs_flush = true; + } Self::apply_wal_records( &store, &catalog, @@ -556,6 +572,10 @@ impl GrafeoDB { if !is_single_file && wal_path.exists() { let recovery = WalRecovery::new(&wal_path); let records = recovery.recover()?; + #[cfg(feature = "grafeo-file")] + if !records.is_empty() { + recovered_wal_needs_flush = true; + } Self::apply_wal_records( &store, &catalog, @@ -656,6 +676,8 @@ impl GrafeoDB { current_graph: RwLock::new(None), current_schema: RwLock::new(None), read_only: is_read_only, + #[cfg(feature = "grafeo-file")] + container_flush_required: std::sync::atomic::AtomicBool::new(recovered_wal_needs_flush), projections: Arc::new(RwLock::new(std::collections::HashMap::new())), #[cfg(all(feature = "compact-store", feature = "lpg"))] layered_store: None, @@ -720,6 +742,15 @@ impl GrafeoDB { // the consumers we're about to spill. db.apply_force_disk_overrides(); + // Open-path index restore may call create_vector_index / similar + // helpers that mark container_flush_required. Reset to the recovery + // signal only so a clean no-write session can take the close fast path. + #[cfg(feature = "grafeo-file")] + db.container_flush_required.store( + recovered_wal_needs_flush, + std::sync::atomic::Ordering::Release, + ); + Ok(db) } @@ -810,6 +841,8 @@ impl GrafeoDB { current_graph: RwLock::new(None), current_schema: RwLock::new(None), read_only: false, + #[cfg(feature = "grafeo-file")] + container_flush_required: std::sync::atomic::AtomicBool::new(false), projections: Arc::new(RwLock::new(std::collections::HashMap::new())), #[cfg(all(feature = "compact-store", feature = "lpg"))] layered_store: None, @@ -901,6 +934,8 @@ impl GrafeoDB { current_graph: RwLock::new(None), current_schema: RwLock::new(None), read_only: true, + #[cfg(feature = "grafeo-file")] + container_flush_required: std::sync::atomic::AtomicBool::new(false), projections: Arc::new(RwLock::new(std::collections::HashMap::new())), #[cfg(all(feature = "compact-store", feature = "lpg"))] layered_store: None, @@ -2433,13 +2468,25 @@ impl GrafeoDB { if let Some(ref wal) = self.wal { wal.sync()?; } - let flush_result = self.checkpoint_to_file(fm, flush::FlushReason::Explicit)?; + + // Clean-close fast path: ephemeral Section wrappers from + // build_sections() start with dirty=false, so FlushReason::Explicit + // always re-serializes the full LPG (dominates ~137s sidecar close). + // Prefer Checkpoint (dirty-only, usually a no-op) when this session + // has no container-affecting work; force Explicit when WAL-disabled, + // WAL has new records, or recovery/index DDL marked a flush. + let initial_reason = if self.close_needs_full_checkpoint() { + flush::FlushReason::Explicit + } else { + flush::FlushReason::Checkpoint + }; + let flush_result = self.checkpoint_to_file(fm, initial_reason)?; // Safety check: if WAL has records but the checkpoint was a no-op // (zero sections written), the container file may not contain the // latest data. This can happen when sections are not marked dirty - // despite mutations going through the WAL. Force-dirty all sections - // and retry before removing the sidecar. + // despite mutations going through the WAL. Force a full Explicit + // flush and retry before removing the sidecar. #[cfg(feature = "wal")] let flush_result = if flush_result.sections_written == 0 { if let Some(ref wal) = self.wal { @@ -2523,6 +2570,12 @@ impl GrafeoDB { /// Logs a WAL record if WAL is enabled. #[cfg(feature = "wal")] pub(super) fn log_wal(&self, record: &WalRecord) -> Result<()> { + // Direct CRUD / persistence helpers go through this path. Session + // mutations write the WAL handle directly and are covered by + // record_count on close; still mark flush-required so wal_disabled + // databases (wal=None) force Explicit checkpoint on close. + #[cfg(feature = "grafeo-file")] + self.mark_container_flush_required(); if let Some(ref wal) = self.wal { wal.log(record)?; } @@ -2949,6 +3002,45 @@ impl GrafeoDB { backup::do_restore_to_epoch(backup_dir, target_epoch, output_path) } + /// Marks that close must rewrite the `.grafeo` container. + /// + /// Used for container-affecting mutations that do not bump WAL + /// `record_count` (index DDL) and for tests. + #[cfg(feature = "grafeo-file")] + pub(crate) fn mark_container_flush_required(&self) { + self.container_flush_required + .store(true, std::sync::atomic::Ordering::Release); + } + + /// Whether close must force a full Explicit section serialize. + /// + /// Clean-close fast path is only taken when this returns false: WAL is + /// enabled, this session logged no WAL records, and no recovery/index DDL + /// required a container rewrite. Ephemeral `Section` wrappers from + /// [`Self::build_sections`] start clean, so `FlushReason::Checkpoint` then + /// writes zero sections and skips LPG serialization. + #[cfg(feature = "grafeo-file")] + fn close_needs_full_checkpoint(&self) -> bool { + if self + .container_flush_required + .load(std::sync::atomic::Ordering::Acquire) + { + return true; + } + #[cfg(feature = "wal")] + { + // WAL disabled (None) cannot use record_count as a dirty signal. + match self.wal.as_ref() { + None => true, + Some(wal) => wal.record_count() > 0, + } + } + #[cfg(not(feature = "wal"))] + { + true + } + } + /// Writes the current database state to the `.grafeo` file using the unified flush. /// /// Does NOT remove the sidecar WAL: callers that want to clean up diff --git a/crates/grafeo-engine/tests/grafeo_file.rs b/crates/grafeo-engine/tests/grafeo_file.rs index eca9e1023..f7b933d7b 100644 --- a/crates/grafeo-engine/tests/grafeo_file.rs +++ b/crates/grafeo-engine/tests/grafeo_file.rs @@ -1430,3 +1430,175 @@ fn deleted_base_edges_stay_deleted_across_reopen() { drop(session); db.close().unwrap(); } + +// ========================================================================= +// Clean-close fast path (Track C): skip full LPG serialize when durable +// ========================================================================= + +#[test] +fn clean_close_does_not_advance_checkpoint_iteration() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("clean_close.grafeo"); + + // Seed durable data with an explicit checkpoint-on-close (dirty session). + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.session() + .execute("INSERT (:Person {name: 'Alix'})") + .unwrap(); + db.close().unwrap(); + } + + let iteration_before = { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let iter = db.file_manager().unwrap().active_header().iteration; + // No writes — clean close should skip full Explicit serialize. + let t0 = std::time::Instant::now(); + db.close().unwrap(); + let clean_close_ms = t0.elapsed().as_millis(); + // Tiny DB: just sanity that close returned; staging measures wall separately. + assert!( + clean_close_ms < 30_000, + "clean close unexpectedly slow: {clean_close_ms}ms" + ); + iter + }; + + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let iteration_after = db.file_manager().unwrap().active_header().iteration; + assert_eq!( + iteration_after, iteration_before, + "clean close must not rewrite container (iteration must stay {iteration_before})" + ); + assert_eq!(db.node_count(), 1); + let names = extract_strings( + db.session() + .execute("MATCH (p:Person) RETURN p.name") + .unwrap() + .rows(), + ); + assert_eq!(names, vec!["Alix"]); + db.close().unwrap(); +} + +#[test] +fn dirty_close_persists_and_advances_iteration() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("dirty_close.grafeo"); + + let iteration_seed = { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.session() + .execute("INSERT (:Person {name: 'Alix'})") + .unwrap(); + db.close().unwrap(); + // Reopen to read iteration after seed close. + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let iter = db.file_manager().unwrap().active_header().iteration; + db.close().unwrap(); + iter + }; + + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.session() + .execute("INSERT (:Person {name: 'Gus'})") + .unwrap(); + db.close().unwrap(); + } + + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let iteration = db.file_manager().unwrap().active_header().iteration; + assert!( + iteration > iteration_seed, + "dirty close must rewrite container: seed={iteration_seed} now={iteration}" + ); + let names = extract_strings( + db.session() + .execute("MATCH (p:Person) RETURN p.name") + .unwrap() + .rows(), + ); + assert_eq!(names, vec!["Alix", "Gus"]); + db.close().unwrap(); +} + +#[test] +fn wal_recovery_forces_checkpoint_on_close() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("recovery_src.grafeo"); + let recovery_path = dir.path().join("recovery_dst.grafeo"); + + // Seed a durable checkpoint. + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.session() + .execute("INSERT (:Person {name: 'Alix'})") + .unwrap(); + db.close().unwrap(); + } + + let iteration_before = { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let iter = db.file_manager().unwrap().active_header().iteration; + db.session() + .execute("INSERT (:Person {name: 'Gus'})") + .unwrap(); + let wal_path = sidecar_wal_path(&path); + assert!( + wal_path.exists(), + "sidecar WAL must exist after mutation before close" + ); + // Snapshot container (still Alix-only) + sidecar WAL (has Gus) to a + // sibling path, then close the source normally. Reopening the sibling + // simulates crash recovery: WalManager::record_count starts at 0 but + // unrecovered mutations exist in the sidecar. + std::fs::copy(&path, &recovery_path).unwrap(); + copy_dir_recursive(&wal_path, &sidecar_wal_path(&recovery_path)).unwrap(); + db.close().unwrap(); + iter + }; + + // Reopen recovery copy: apply Gus from sidecar; close must Explicit-flush. + { + let db = GrafeoDB::with_config(Config::persistent(&recovery_path)).unwrap(); + assert_eq!(db.node_count(), 2, "recovery must apply sidecar WAL"); + db.close().unwrap(); + } + + assert!( + !sidecar_wal_path(&recovery_path).exists(), + "recovery close must remove sidecar after folding into container" + ); + + let db = GrafeoDB::with_config(Config::persistent(&recovery_path)).unwrap(); + let iteration = db.file_manager().unwrap().active_header().iteration; + assert!( + iteration > iteration_before, + "recovery close must rewrite container: before={iteration_before} now={iteration}" + ); + let names = extract_strings( + db.session() + .execute("MATCH (p:Person) RETURN p.name") + .unwrap() + .rows(), + ); + assert_eq!(names, vec!["Alix", "Gus"]); + db.close().unwrap(); +} + +fn copy_dir_recursive(src: &std::path::Path, dst: &std::path::Path) -> std::io::Result<()> { + std::fs::create_dir_all(dst)?; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + let to = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_recursive(&entry.path(), &to)?; + } else { + std::fs::copy(entry.path(), to)?; + } + } + Ok(()) +} + diff --git a/docs/TRACK_C_CLOSE_FAST_PATH.md b/docs/TRACK_C_CLOSE_FAST_PATH.md new file mode 100644 index 000000000..b90a6bc13 --- /dev/null +++ b/docs/TRACK_C_CLOSE_FAST_PATH.md @@ -0,0 +1,55 @@ +# Track C — Clean-close fast path + +**Branch:** `perf/close-fast-path` @ worktree `/data/worktrees/grafeo-close-fast-path` +**Base pin:** `9781320f` (`agent/txn-session-batch-20260726`) +**Does not touch:** `diagnostics/close-forensics`, AM `/data/worktrees/am-diagnostics-grafeo-close-forensics` + +## Root cause (proven in code) + +`GrafeoDB::close()` previously always called `checkpoint_to_file(..., FlushReason::Explicit)`. + +In `flush.rs`: + +```text +if reason == FlushReason::Explicit || section.is_dirty() { + targets.push((..., section.serialize()?)); +} +``` + +`build_sections()` constructs **ephemeral** `LpgStoreSection` / `CatalogSection` / `VectorStoreSection` wrappers with `dirty = false` every time. So `is_dirty()` is almost always false, and **Explicit always re-serializes the full LPG** — even for a no-write reopen+close. + +Phase 1 attribution (`PHASE1_CLOSE_ATTRIBUTION.md`): ~137.6s inside `GrafeoDB::close()`, ~97% CPU-bound — consistent with full LPG serialize, not HNSW free (~97ms) or BufferManager (~10.5s). + +## Fast path (implemented) + +Close chooses: + +| Condition | Flush reason | +|-----------|--------------| +| `wal == None` (WAL disabled) | `Explicit` (always) | +| `container_flush_required` (WAL recovery applied records, or index DDL) | `Explicit` | +| `wal.record_count() > 0` (session mutations this open) | `Explicit` | +| else (clean no-write session) | `Checkpoint` → 0 sections → **skip serialize** | + +Safety net retained: if Checkpoint writes 0 sections but `record_count() > 0`, retry `Explicit`. + +Open-path vector/index restore may temporarily mark flush-required; open resets the flag to the recovery signal only so clean sessions stay eligible. + +## Correctness tests (`grafeo_file.rs`) + +- `clean_close_does_not_advance_checkpoint_iteration` — reopen, no writes, close; header iteration unchanged; data intact +- `dirty_close_persists_and_advances_iteration` — mutate+close advances iteration; data durable +- `wal_recovery_forces_checkpoint_on_close` — sibling copy with unrecovered sidecar; reopen+close folds WAL and advances iteration +- Existing `wal_disabled_single_file_persists_on_close` must still pass (WAL-off → always Explicit) + +## Staging measure (under `/data/tmp` only) + +Copy staging sidecar under `/data/tmp`, never `/data/grafeo`. Compare clean open→close wall before/after when MemAvailable allows (~11 GiB open RSS historically). + +## Operational risk + +- **Low for AM idle eviction** (WAL on, read-only session): intended win. +- **Must not** skip flush after crash recovery (gated by `recovered_wal_needs_flush`). +- **Must not** skip when WAL disabled (always Explicit). +- Index DDL without WAL records marks `container_flush_required`. +- Residual: any future mutation path that neither WALs nor marks the flag would be a durability hole — review new APIs against this gate. From 22cad55bd979a2e2ff617a86df3dc0215cca38e4 Mon Sep 17 00:00:00 2001 From: jarmen423 Date: Wed, 29 Jul 2026 17:55:19 +0000 Subject: [PATCH 2/3] test(engine): add staging clean-close bench + Track C measurements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ignored GRAFEO_CLOSE_BENCH_PATH harness under /data/tmp only; document before/after am-personal clean-close wall (~26s → ~0.13s). --- crates/grafeo-engine/tests/grafeo_file.rs | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/grafeo-engine/tests/grafeo_file.rs b/crates/grafeo-engine/tests/grafeo_file.rs index f7b933d7b..7a1f1089d 100644 --- a/crates/grafeo-engine/tests/grafeo_file.rs +++ b/crates/grafeo-engine/tests/grafeo_file.rs @@ -1602,3 +1602,47 @@ fn copy_dir_recursive(src: &std::path::Path, dst: &std::path::Path) -> std::io:: Ok(()) } + +/// Staging / large-DB clean-close wall measurement. +/// +/// Usage: +/// GRAFEO_CLOSE_BENCH_PATH=/data/tmp/.../copy.grafeo \ +/// cargo test -p grafeo-engine --test grafeo_file staging_clean_close_bench --release -- --ignored --nocapture +#[test] +#[ignore = "manual staging bench; set GRAFEO_CLOSE_BENCH_PATH"] +fn staging_clean_close_bench() { + let path = std::env::var("GRAFEO_CLOSE_BENCH_PATH") + .expect("GRAFEO_CLOSE_BENCH_PATH must point at a disposable .grafeo copy under /data/tmp"); + assert!( + path.starts_with("/data/tmp/"), + "refuse paths outside /data/tmp: {path}" + ); + let path = std::path::PathBuf::from(path); + + eprintln!("open {}", path.display()); + let t_open = std::time::Instant::now(); + let db = GrafeoDB::with_config(Config::persistent(&path)).expect("open"); + let open_ms = t_open.elapsed().as_millis(); + let nodes = db.node_count(); + let edges = db.edge_count(); + let iter_before = db.file_manager().unwrap().active_header().iteration; + eprintln!("open_ms={open_ms} nodes={nodes} edges={edges} iteration={iter_before}"); + + // Optional read to mimic warm sidecar serve + let _ = db.session().execute("MATCH (n) RETURN count(n) LIMIT 1"); + + let t_close = std::time::Instant::now(); + db.close().expect("close"); + let close_ms = t_close.elapsed().as_millis(); + eprintln!("clean_close_ms={close_ms}"); + + let db2 = GrafeoDB::with_config(Config::persistent(&path)).expect("reopen"); + let iter_after = db2.file_manager().unwrap().active_header().iteration; + assert_eq!( + iter_after, iter_before, + "clean close must not rewrite container" + ); + assert_eq!(db2.node_count(), nodes); + db2.close().unwrap(); + eprintln!("PASS clean_close_ms={close_ms} open_ms={open_ms} iteration_unchanged={iter_before}"); +} From 47fb38da7518ad7a22250ec454ad2b16d73ae2bd Mon Sep 17 00:00:00 2001 From: jarmen423 Date: Wed, 29 Jul 2026 17:56:09 +0000 Subject: [PATCH 3/3] docs: record Track C clean-close before/after staging measurements --- docs/TRACK_C_CLOSE_FAST_PATH.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/TRACK_C_CLOSE_FAST_PATH.md b/docs/TRACK_C_CLOSE_FAST_PATH.md index b90a6bc13..ab155371c 100644 --- a/docs/TRACK_C_CLOSE_FAST_PATH.md +++ b/docs/TRACK_C_CLOSE_FAST_PATH.md @@ -2,6 +2,7 @@ **Branch:** `perf/close-fast-path` @ worktree `/data/worktrees/grafeo-close-fast-path` **Base pin:** `9781320f` (`agent/txn-session-batch-20260726`) +**Head:** see `git rev-parse HEAD` on this branch **Does not touch:** `diagnostics/close-forensics`, AM `/data/worktrees/am-diagnostics-grafeo-close-forensics` ## Root cause (proven in code) @@ -37,14 +38,25 @@ Open-path vector/index restore may temporarily mark flush-required; open resets ## Correctness tests (`grafeo_file.rs`) -- `clean_close_does_not_advance_checkpoint_iteration` — reopen, no writes, close; header iteration unchanged; data intact -- `dirty_close_persists_and_advances_iteration` — mutate+close advances iteration; data durable -- `wal_recovery_forces_checkpoint_on_close` — sibling copy with unrecovered sidecar; reopen+close folds WAL and advances iteration -- Existing `wal_disabled_single_file_persists_on_close` must still pass (WAL-off → always Explicit) +- `clean_close_does_not_advance_checkpoint_iteration` +- `dirty_close_persists_and_advances_iteration` +- `wal_recovery_forces_checkpoint_on_close` +- Existing `wal_disabled_single_file_persists_on_close` still passes +- `staging_clean_close_bench` (ignored) — set `GRAFEO_CLOSE_BENCH_PATH` under `/data/tmp` ## Staging measure (under `/data/tmp` only) -Copy staging sidecar under `/data/tmp`, never `/data/grafeo`. Compare clean open→close wall before/after when MemAvailable allows (~11 GiB open RSS historically). +Disposable copies of staging `am-personal.grafeo` (163205 nodes / 1403962 edges). Never `/data/grafeo`. + +| Build | open_ms | clean_close_ms | iteration | +|-------|--------:|---------------:|-----------| +| BEFORE `9781320f` | 9548 | **26318** | 448→449 (full rewrite) | +| AFTER `perf/close-fast-path` | 17841 | **131** | 448 unchanged | + +~200× clean-close wall on this DB. Full code-index sidecar (~137s Phase1) not re-measured: MemAvailable was 4–7 GiB (needs ~11 GiB open RSS). + +Artifacts: `/data/tmp/grafeo-close-fast-path-TRACK_C_SUMMARY.md`, +`/data/tmp/grafeo-close-fast-path-measure-*/{before,after}-clean-close.log` ## Operational risk