From 759ced0d3f7eea333d23ba7b3e3b26cacdd77484 Mon Sep 17 00:00:00 2001 From: Bryan Russett Date: Tue, 4 Aug 2026 10:01:16 -0400 Subject: [PATCH] perf(codex): stream rollout metadata instead of parsing every line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session listing (p list codex, the share picker, bare resume) called read_metadata on every rollout, which serde-parsed every line of every file — a minute-plus silent stall on a multi-gigabyte sessions tree. read_metadata is now one streaming pass: JSON-parse a bounded head (session_meta, first timestamps, first user prompt — all at the top of this append-only log) plus the final line (newest timestamp), and otherwise just count non-empty lines. 3.9 GB / 421 sessions: ~80 s -> raw-I/O speed. Documented trades: a prompt buried past the head budget reports None; line_count counts non-empty lines, not parsed ones. toolpath-codex 0.6.2. Also picks up the pre-existing rustfmt drift in paths.rs that fails cargo fmt --check on main under the pinned 1.94.0 toolchain. --- CHANGELOG.md | 17 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- crates/toolpath-codex/Cargo.toml | 2 +- crates/toolpath-codex/src/io.rs | 243 ++++++++++++++++++++++++++--- crates/toolpath-codex/src/paths.rs | 5 +- site/_data/crates.json | 2 +- 7 files changed, 250 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75f31478..30f42cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to the Toolpath workspace are documented here. +## Codex session listing no longer parses every byte — 2026-08-04 + +Listing Codex sessions read every rollout file end to end through +serde. On a real multi-gigabyte `~/.codex/sessions` tree that turned +every listing surface — `p list codex`, the `share` picker, and the +upcoming bare `resume` picker — into a minute-plus silent stall. + +- **toolpath-codex** (0.6.2): `read_metadata` becomes a single + streaming pass. It JSON-parses only a bounded head (session_meta, + first timestamps, the first user prompt — all of which live at the + top of this append-only log) plus the final line (newest timestamp), + and otherwise just counts lines. A 3.9 GB / 421-session tree drops + from ~80 s to raw-I/O speed. Deliberate trades, documented on the + method: a first prompt buried past the head budget reports as + `None`, and `line_count` counts non-empty lines rather than + successfully parsed ones. + ## Projected Claude sessions are resumable again — 2026-07-30 Two fixes found by live-resuming a projected session against the real diff --git a/Cargo.lock b/Cargo.lock index d9b355b7..4ceeda10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4090,7 +4090,7 @@ dependencies = [ [[package]] name = "toolpath-codex" -version = "0.6.1" +version = "0.6.2" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index ec3e2606..9a5c8866 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ toolpath-convo = { version = "0.11.1", path = "crates/toolpath-convo" } toolpath-git = { version = "0.6.0", path = "crates/toolpath-git" } toolpath-claude = { version = "0.12.2", path = "crates/toolpath-claude", default-features = false } toolpath-gemini = { version = "0.6.1", path = "crates/toolpath-gemini", default-features = false } -toolpath-codex = { version = "0.6.1", path = "crates/toolpath-codex" } +toolpath-codex = { version = "0.6.2", path = "crates/toolpath-codex" } toolpath-copilot = { version = "0.1.0", path = "crates/toolpath-copilot" } toolpath-opencode = { version = "0.5.0", path = "crates/toolpath-opencode" } toolpath-cursor = { version = "0.2.0", path = "crates/toolpath-cursor" } diff --git a/crates/toolpath-codex/Cargo.toml b/crates/toolpath-codex/Cargo.toml index 693636e3..75138402 100644 --- a/crates/toolpath-codex/Cargo.toml +++ b/crates/toolpath-codex/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-codex" -version = "0.6.1" +version = "0.6.2" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" diff --git a/crates/toolpath-codex/src/io.rs b/crates/toolpath-codex/src/io.rs index 25441a9a..e1054a75 100644 --- a/crates/toolpath-codex/src/io.rs +++ b/crates/toolpath-codex/src/io.rs @@ -78,21 +78,134 @@ impl ConvoIO { RolloutReader::read_session(path) } - /// Cheap per-file metadata: parses the session_meta line + walks - /// the file for first/last timestamps. + /// Cheap per-file metadata: a single streaming pass that + /// JSON-parses only the head of the file (session_meta, first + /// timestamps, first user prompt all live there in this + /// append-only log) plus the final line (last timestamp), and + /// otherwise just counts lines. Multi-gigabyte session trees made + /// the previous parse-every-line approach a minute-plus stall in + /// every session-listing surface (`p list codex`, `share`, bare + /// `resume`). + /// + /// Bounded-head consequences, deliberate: `first_user_message` is + /// `None` if the first prompt appears after the head budget, and + /// `line_count` counts non-empty lines (unparseable ones + /// included) rather than successfully parsed ones. pub fn read_metadata>(&self, path: P) -> Result { + use crate::types::{ResponseItem, RolloutLine}; + use std::io::BufRead; + + // Parse at most this many non-empty head lines looking for + // session_meta / timestamps / the first user prompt. Real + // sessions surface the prompt within the first dozen lines + // (after session_meta, turn_context, and injected context). + const HEAD_PARSE_BUDGET: usize = 100; + let path = path.as_ref(); - // Full parse is simplest; rollout files are small (typical - // session 200-300 KB). If that becomes a bottleneck we'd peek - // the first line plus `stat` for mtime. - let session = RolloutReader::read_session(path)?; - - let meta_line = session.items().find_map(|item| match item { - RolloutItem::SessionMeta(m) => Some(m), - _ => None, + if !path.exists() { + return Err(crate::error::ConvoError::SessionNotFound( + path.display().to_string(), + )); + } + + let file = std::fs::File::open(path)?; + let mut reader = std::io::BufReader::with_capacity(1 << 20, file); + let mut raw = String::new(); + let mut last_nonempty = String::new(); + + let mut line_count = 0usize; + let mut head_parsed = 0usize; + let mut first_line_meta_id: Option = None; + let mut meta: Option> = None; + let mut started_at = None; + let mut last_ts = None; + let mut first_user: Option = None; + let mut first_user_fallback: Option = None; + + loop { + raw.clear(); + match reader.read_line(&mut raw) { + Ok(0) => break, + Ok(_) => {} + Err(e) => { + eprintln!( + "Warning: IO error reading {} after line {}: {}", + path.display(), + line_count, + e + ); + break; + } + } + let trimmed = raw.trim(); + if trimmed.is_empty() { + continue; + } + line_count += 1; + + let still_hunting = meta.is_none() || started_at.is_none() || first_user.is_none(); + if still_hunting && head_parsed < HEAD_PARSE_BUDGET { + head_parsed += 1; + if let Ok(line) = serde_json::from_str::(trimmed) { + if let Some(ts) = line.parsed_timestamp() { + if started_at.is_none_or(|s| ts < s) { + started_at = Some(ts); + } + if last_ts.is_none_or(|l| ts > l) { + last_ts = Some(ts); + } + } + if line_count == 1 && line.kind == "session_meta" { + first_line_meta_id = line + .payload + .get("id") + .and_then(|v| v.as_str()) + .map(str::to_string); + } + if first_user.is_none() + && line.kind == "event_msg" + && line.payload.get("type").and_then(|v| v.as_str()) == Some("user_message") + && let Some(msg) = line.payload.get("message").and_then(|v| v.as_str()) + && !msg.is_empty() + { + first_user = Some(msg.to_string()); + } + match line.item() { + RolloutItem::SessionMeta(m) if meta.is_none() => meta = Some(m), + RolloutItem::ResponseItem(ResponseItem::Message(m)) + if m.role == "user" && first_user_fallback.is_none() => + { + let t = m.text(); + if !t.is_empty() { + first_user_fallback = Some(t); + } + } + _ => {} + } + } + } + std::mem::swap(&mut last_nonempty, &mut raw); + } + + // The tail line carries the newest timestamp in an + // append-only log; parse just that one. + if let Ok(line) = serde_json::from_str::(last_nonempty.trim()) + && let Some(ts) = line.parsed_timestamp() + && last_ts.is_none_or(|l| ts > l) + { + last_ts = Some(ts); + } + + // Same id rule as RolloutReader::derive_session_id: the first + // line's session_meta payload wins, else the filename stem. + let id = first_line_meta_id.unwrap_or_else(|| { + path.file_stem() + .and_then(|s| s.to_str()) + .map(|stem| crate::paths::session_id_from_stem(stem).to_string()) + .unwrap_or_else(|| "unknown".to_string()) }); - let (cwd, cli_version, git_branch, git_commit) = match &meta_line { + let (cwd, cli_version, git_branch, git_commit) = match &meta { Some(m) => ( Some(m.cwd.clone()), Some(m.cli_version.clone()), @@ -103,16 +216,16 @@ impl ConvoIO { }; Ok(SessionMetadata { - id: session.id.clone(), - file_path: session.file_path.clone(), - started_at: session.started_at(), - last_activity: session.last_activity(), + id, + file_path: path.to_path_buf(), + started_at, + last_activity: last_ts, cwd, cli_version, - first_user_message: session.first_user_text(), + first_user_message: first_user.or(first_user_fallback), git_branch, git_commit, - line_count: session.lines.len(), + line_count, }) } @@ -173,7 +286,11 @@ mod tests { let (_t, io) = setup(); let day = io.resolver().sessions_root().unwrap().join("2026/04/21"); fs::create_dir_all(&day).unwrap(); - fs::write(day.join("rollout-2026-04-21T09-00-00-bbb.jsonl"), "not json").unwrap(); + fs::write( + day.join("rollout-2026-04-21T09-00-00-bbb.jsonl"), + "not json", + ) + .unwrap(); let ids = io.list_session_ids().unwrap(); assert_eq!(ids.len(), 2); @@ -220,4 +337,94 @@ mod tests { let io = ConvoIO::with_resolver(PathResolver::new().with_codex_dir(&codex)); assert!(io.list_sessions().unwrap().is_empty()); } + + /// Write a rollout body into the fixture tree and return its path. + fn write_rollout(io: &ConvoIO, name: &str, body: &str) -> PathBuf { + let day = io.resolver().sessions_root().unwrap().join("2026/04/22"); + fs::create_dir_all(&day).unwrap(); + let path = day.join(name); + fs::write(&path, body).unwrap(); + path + } + + #[test] + fn metadata_last_activity_comes_from_tail_line() { + let (_t, io) = setup(); + let body = [ + r#"{"timestamp":"2026-04-22T10:00:00.000Z","type":"session_meta","payload":{"id":"019dtail-bbb","cwd":"/tmp/p","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}"#, + r#"{"timestamp":"2026-04-22T10:00:01.000Z","type":"event_msg","payload":{"type":"user_message","message":"first prompt"}}"#, + r#"{"timestamp":"2026-04-22T11:30:00.000Z","type":"event_msg","payload":{"type":"task_complete"}}"#, + ] + .join("\n"); + let path = write_rollout(&io, "rollout-2026-04-22T10-00-00-019dtail-bbb.jsonl", &body); + let m = io.read_metadata(&path).unwrap(); + assert_eq!(m.id, "019dtail-bbb"); + assert_eq!( + m.started_at + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "2026-04-22T10:00:00Z" + ); + assert_eq!( + m.last_activity + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "2026-04-22T11:30:00Z" + ); + assert_eq!(m.first_user_message.as_deref(), Some("first prompt")); + } + + /// line_count counts non-empty lines; blank and unparseable lines + /// don't abort the scan (the count deliberately includes junk — + /// it approximates message_count, nothing more). + #[test] + fn metadata_line_count_counts_nonempty_lines_tolerantly() { + let (_t, io) = setup(); + let body = [ + r#"{"timestamp":"2026-04-22T10:00:00.000Z","type":"session_meta","payload":{"id":"019djunk-ccc","cwd":"/tmp/p","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}"#, + "", + r#"{"not json"#, + r#"{"timestamp":"2026-04-22T10:00:02.000Z","type":"event_msg","payload":{"type":"user_message","message":"hi"}}"#, + ] + .join("\n"); + let path = write_rollout(&io, "rollout-2026-04-22T10-00-00-019djunk-ccc.jsonl", &body); + let m = io.read_metadata(&path).unwrap(); + assert_eq!(m.line_count, 3); + assert_eq!(m.first_user_message.as_deref(), Some("hi")); + } + + /// The head-parse budget bounds the prompt hunt: a first user + /// message buried past the budget yields None rather than a full + /// parse of the file. + #[test] + fn metadata_first_user_none_beyond_head_budget() { + let (_t, io) = setup(); + let mut lines = vec![ + r#"{"timestamp":"2026-04-22T10:00:00.000Z","type":"session_meta","payload":{"id":"019ddeep-ddd","cwd":"/tmp/p","originator":"codex-tui","cli_version":"0.118.0","source":"cli"}}"#.to_string(), + ]; + for i in 0..150 { + lines.push(format!( + r#"{{"timestamp":"2026-04-22T10:00:01.000Z","type":"event_msg","payload":{{"type":"task_started","n":{i}}}}}"# + )); + } + lines.push( + r#"{"timestamp":"2026-04-22T10:05:00.000Z","type":"event_msg","payload":{"type":"user_message","message":"buried"}}"#.to_string(), + ); + let path = write_rollout( + &io, + "rollout-2026-04-22T10-00-00-019ddeep-ddd.jsonl", + &lines.join("\n"), + ); + let m = io.read_metadata(&path).unwrap(); + assert_eq!(m.first_user_message, None); + assert_eq!(m.line_count, 152); + // The tail line still supplies last_activity even though the + // head budget was exhausted long before it. + assert_eq!( + m.last_activity + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "2026-04-22T10:05:00Z" + ); + } } diff --git a/crates/toolpath-codex/src/paths.rs b/crates/toolpath-codex/src/paths.rs index 610b2638..c047a0af 100644 --- a/crates/toolpath-codex/src/paths.rs +++ b/crates/toolpath-codex/src/paths.rs @@ -141,7 +141,10 @@ impl PathResolver { /// isn't stem-shaped or the file isn't at its dated path (the caller /// falls back to the tree walk). fn rollout_file_for_stem(&self, session_id: &str) -> Result> { - let Some(date) = session_id.strip_prefix("rollout-").and_then(|r| r.get(..10)) else { + let Some(date) = session_id + .strip_prefix("rollout-") + .and_then(|r| r.get(..10)) + else { return Ok(None); }; let parts: Vec<&str> = date.split('-').collect(); diff --git a/site/_data/crates.json b/site/_data/crates.json index 4d282a97..7c364f20 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -49,7 +49,7 @@ }, { "name": "toolpath-codex", - "version": "0.6.1", + "version": "0.6.2", "description": "Derive from Codex CLI rollout files", "docs": "https://docs.rs/toolpath-codex", "crate": "https://crates.io/crates/toolpath-codex",