From 7a7c366fa323d6bd61af8711708d8f9c743ffa3a Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 15:16:59 -0400 Subject: [PATCH 1/9] docs(redact): spec and TDD implementation plan for `path p redact` Design and plan for an explicit post-generation redaction pass over a generated toolpath document. Spec covers: why post-generation rather than hook-time (a controlled experiment shows Claude Code hooks cannot keep a secret off disk), the schema-aware field map, the `Detector` plug point, five transforms, plan-then-apply with a dry run that surfaces every field it looked at, the per-step audit record, in-place cache semantics, and the sync collision that in-place redaction creates. Plan is test-driven and organised into waves for parallel execution: one small blocking vocabulary task, then six independent tracks. Each task writes failing tests first and is gated on both green tests and an adversarial review pass. Co-Authored-By: Claude Opus 5 --- .../plans/2026-07-30-path-redact-command.md | 1756 +++++++++++++++++ .../2026-07-30-path-redact-command-design.md | 888 +++++++++ 2 files changed, 2644 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-30-path-redact-command.md create mode 100644 docs/superpowers/specs/2026-07-30-path-redact-command-design.md diff --git a/docs/superpowers/plans/2026-07-30-path-redact-command.md b/docs/superpowers/plans/2026-07-30-path-redact-command.md new file mode 100644 index 00000000..38414835 --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-path-redact-command.md @@ -0,0 +1,1756 @@ +# `path p redact` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `path p redact` plumbing command that removes credentials from an already-generated toolpath document in place, via a reviewable plan-then-apply flow, with detection behind a swappable trait and five transform choices. + +**Method:** Test-driven, with an adversarial review gate. Every task writes failing tests first, implements to green, then survives a hostile reviewer before it counts as done. Tasks are grouped into **waves**: one small blocking wave defining shared vocabulary, then wide parallel fan-out. Tracks inside a wave share no files, so separate agents work them concurrently. + +**Architecture:** New `toolpath-redact` tier-2 crate holding the engine (traversal, field map, `Detector` trait, transforms, plan types, audit record). `path-cli` gets a thin `cmd_redact.rs` for args, picker, and terminal output. The crate boundary is drawn for **testability** - see the purity rule. + +**Tech Stack:** Rust 2024. New deps confined to the new crate: `regex`, `aho-corasick`, `diffy`, `hmac`, `sha2`, `toml`. `path-cli` reuses its existing `fuzzy` picker and `serde_json`. + +**Spec:** `docs/superpowers/specs/2026-07-30-path-redact-command-design.md` (commit `a7760d1`). + +--- + +## Execution: how to dispatch this + +### Rule zero: implement, do not deliberate + +**The design is closed.** Every open question was settled in the spec, which carries a numbered Decisions section. An implementing agent that disagrees **files the objection and proceeds anyway** - it does not stop, redesign, or re-derive. Ten agents each spending five minutes reconsidering the same decision is the largest waste available here. + +Do not re-derive any of these; they are answered: + +| Question | Answer | Where | +|---|---|---| +| Why not redact at derive time or at egress? | Derive is shared by 7 providers and every round-trip test; egress covers one exit of four | Decision 1 | +| Why a new crate, not a module in `toolpath-convo`? | Keeps a regex engine and vendored ruleset off the 7 provider crates | Decision 2 | +| Why not `extract -> redact -> derive`? | Lossy: `derive.rs:607` writes `extra["edits"]`, `extract.rs` never reads it back | Decision 3 | +| Why in place instead of a copy? | Every downstream verb resolves a cache id, so a copy protects none of them | Decision 6 | +| Why is detection a trait? | Detection is where precision is worst and the field moves fastest; traversal is stable | Decisions 7, 8 | +| Why strings-plus-context, not `Path`, into detectors? | Testable without building a document; keeps a future hook path open | Decision 8 | +| Why does `share` not scan? | Deliberate, with the consequence stated | Non-goals | +| Why is `mask`/`partial` not the default? | They leak length and format; offered, documented, not default | Transforms | + +Each task names its file, its tests, and its assertions, and gives the code to type. **Start by creating the test file and typing the test names.** A red bar is the point at which thinking becomes useful. + +### Model assignment + +| Model | Role | Tasks | +|---|---|---| +| **Opus** | Implement | T0, T1, T2, T7, T10 - contract design and subtle invariants. T1's overlap resolution and T7's byte-identity / idempotence / merge-not-append are where a plausible implementation is silently wrong. T10 modifies load-bearing existing code around a non-obvious hazard. | +| **Opus** | **Review** | Every task. Adversarial review is reasoning work, not proofreading. | +| **Sonnet** | Implement | T3, T4, T5, T8, T9, T11 - well-specified work against named assertions. | +| **Haiku** | Implement | T6, T12 - a clap struct and a file checklist. | +| **Haiku** | Verification loop | Runs the build continuously, fixes nothing. | + +Reviewers are spawned **per track**, so review parallelises exactly as implementation does. A reviewer never edits code; it returns a change list. + +--- + +## The adversarial review gate + +**No task is done when its tests pass. A task is done when its tests pass and a hostile reviewer has run out of objections.** + +### The loop + +``` +implement -> tests green -> REVIEW -> change list -> revise -> tests still green -> REVIEW + | | + +-------- repeat until clean ----------------+ +``` + +The implementing agent owns the code. The reviewer owns the objections. **The reviewer does not edit files** - it returns a numbered change list, and the implementer applies or rebuts each item. An item may be rebutted once, with a reason; a second rebuttal escalates to the human rather than looping. + +### Reviewer instructions (paste this into the reviewer agent) + +> You are reviewing a diff for the `toolpath-redact` implementation. You are **adversarial by mandate**: your job is to find what is wrong, not to approve. A review that finds nothing is a review that was not done properly - but do not invent problems to meet a quota, and do not restate the spec back at the author. +> +> Assume the design is closed. Do **not** raise objections to architecture decisions listed in "Rule zero" above; those are settled. Review the implementation, not the plan. +> +> Report findings as a numbered list. For each: the file and line, what is wrong, and the concrete change you want. No prose essays. If you would not block a merge on it, do not list it. +> +> Check, in this order: +> +> 1. **Correctness against the stated invariant.** Every task names its invariants. Does the code actually hold them, or only hold them for the cases the tests happen to cover? Name the input that breaks it. +> 2. **Test adequacy.** Is there a branch with no test? An error path never exercised? A boundary (empty, one element, multibyte, maximum) untested? **Say which test to add**, not "add more tests". +> 3. **Readability.** Would a competent Rust engineer unfamiliar with this code understand this function in one pass? If not, what specifically obstructs them - naming, nesting depth, an unnamed intermediate, a function doing two jobs? +> 4. **Comments.** Apply the comment policy below strictly. Over-commenting is a defect and you should report it as one. +> 5. **Idiom and simplicity.** Unnecessary `clone()`, `unwrap()` on a path that can fail in production, a hand-rolled loop where an iterator reads better, a type that could be borrowed. Do not bikeshed formatting - `cargo fmt` owns that. +> +> You have no authority to change scope. If you believe a requirement is wrong, say so once in a final "out of scope" note and move on. + +### The comment policy + +**Comments explain WHY, never WHAT.** The code already says what it does. A comment that restates it is noise that rots the moment the code changes. + +Write a comment only when one of these is true: + +1. **The code cannot be self-explanatory.** A non-obvious algorithm, an ordering constraint, a subtle invariant. +2. **A non-obvious bug or hazard was found here.** Record it so nobody reintroduces it. +3. **An external contract forces the shape.** A vendored format, a schema requirement, an upstream quirk. + +**Reject these:** + +```rust +// Increment the counter +count += 1; + +// Loop over the findings +for f in &findings { ... } + +/// Gets the name. +pub fn name(&self) -> &str { &self.name } + +// Create a new detector set +let mut set = DetectorSet::default(); +``` + +**Accept these:** + +```rust +// Sort descending by start so earlier offsets stay valid as later spans +// are spliced out. +edits.sort_by(|a, b| b.0.start.cmp(&a.0.start)); + +// Decode `~1` before `~0`, or `~01` round-trips wrong (RFC 6901). +let decoded = token.replace("~1", "/").replace("~0", "~"); + +// gitleaks' `keywords` gate exists because 221 of 222 rules carry one; +// without it a full sweep per string leaf is unaffordable. +if !self.prefilter.is_match(text) { return Ok(Vec::new()); } + +// `is_unchanged` never inspects the document, so a re-derive would +// clobber an in-place redaction. Replay the policy before writing. +``` + +Doc comments (`///`) on public items are held to the same bar: they earn their place by saying something the signature does not. `/// Returns the id.` on `fn id(&self) -> &str` is a defect. + +### Every implementing agent must + +- [ ] Re-run its task's tests after **every** review revision. A revision that breaks a test is not a revision. +- [ ] Run `cargo test --workspace` before declaring the task done, not just its own module - cross-track breakage is real. +- [ ] **Add the coverage the reviewer names.** "Tests pass" is not the bar; "the tests exercise the branches" is. +- [ ] Leave `cargo clippy -- -D warnings` and `cargo fmt --check` clean. + +### The verification loop (one Haiku agent, running from T0 onward) + +```bash +cargo test --workspace 2>&1 | tail -40 +cargo clippy --workspace -- -D warnings 2>&1 | tail -40 +cargo fmt --check +``` + +Reports failures to whoever owns the failing file and **fixes nothing itself**. Keeps implementers implementing and catches cross-track breakage within a minute. + +### Concurrency + +**Per wave:** W0 = 1 · **W1 = 6** · W2 = 2 · W3 = 1 · W4 = 3, each with a paired reviewer, plus the verification agent, plus T12 (no dependencies, hand to anyone idle). + +**Peak: 8 implementers + up to 6 concurrent reviewers** in Wave 1. Reviewers are short-lived; spawn one when a track goes green rather than holding one idle. + +Run T0 **solo and first**, then commit. Each Wave 1 track owns exactly one file: + +| Track | Owns | Implement | Review | +|---|---|---|---| +| T1 | `src/detect.rs` | Opus | Opus | +| T2 | `src/surface.rs` | Opus | Opus | +| T3 | `src/internal/` | Sonnet | Opus | +| T4 | `src/transform.rs` | Sonnet | Opus | +| T5 | `src/plan.rs` | Sonnet | Opus | +| T6 | `cmd_redact.rs` (args) | Haiku | Opus | +| T12 | docs + release files | Haiku | Opus | + +T1 exports `FixedDetector`, which T7 and T8 need. If T7 would block on it, stub it locally rather than wait. + +**If short on time, cut in this order:** T3's checksum validators, then `exec.rs` in T8, then T11's corpus smoke test. Do **not** cut T7's non-destruction and idempotence tests, T10, or the review gate. + +--- + +## The purity rule (this is what makes the tests parallel) + +**`toolpath-redact` touches no environment variable, no filesystem, no clock, and no process global.** The engine is a pure function: + +```rust +redact(document, plan, config) -> (document, report) +``` + +The key arrives as **bytes**, not a path. The timestamp arrives as a **parameter**, not `Utc::now()`. Everything needing `$TOOLPATH_CONFIG_DIR`, the manifest, or a key file lives in `path-cli` and passes data in. + +1. Engine tests are pure `fn(input) -> output`. No temp dirs, no locks. `cargo test -p toolpath-redact` saturates every core. +2. Determinism is free - "regenerating a plan twice yields identical bytes" is testable because no hidden clock or RNG exists. +3. The env-dependent surface shrinks to a handful of `path-cli` tests. + +### Banned in new code + +| Banned | Why | Use instead | +|---|---|---| +| `std::env::set_var` in a unit test | Forces the test through `config::TEST_ENV_LOCK` (`config.rs:37`), a process-wide `Mutex`. Serialized. | Subprocess tests with `.env()` on the `Command`, per `tests/integration.rs`. | +| `fuzzy::set_picker_override` in a test | A `OnceLock` (`fuzzy.rs:58`). Write-once, process-global: one test poisons the binary. | Inject the picker, mirroring `cmd_resume::ExecStrategy`. | +| `Utc::now()` in the engine | Defeats byte-identical plan assertions. | `now: DateTime` parameter. | +| A fixed temp path | Cross-test collisions. | `tempfile::TempDir` per test. | +| Reading a key file in the engine | Filesystem dependence. | `key: &[u8]` parameter. | + +--- + +## Dependency graph + +``` +WAVE 0 -- T0 vocabulary (small, blocking) + | + +----------+----------+----------+----------+----------+ +WAVE 1 T1 normalise T2 surfaces T3 detector T4 transforms T5 plan T6 args + | | | | | | + +----------+-+------------+-+----------+ | | + | | | | +WAVE 2 T7 apply T8 plan generation ------------+ | + (T2, T4) (T2, T3, T5) | + +----------------+---------------------------------- + + | +WAVE 3 T9 CLI dispatch + | + +---------------------+---------------------+ +WAVE 4 T10 sync T11 integration T12 docs + (T9) (T9) (independent) +``` + +**Critical path: T0 -> T2 -> T7 -> T9 -> T10.** Everything else is slack. + +--- + +## File map + +- **Create** `crates/toolpath-redact/Cargo.toml`, `README.md` +- **Create** `crates/toolpath-redact/src/{lib,detect,surface,plan,transform,apply,exec}.rs` +- **Create** `crates/toolpath-redact/src/internal/{mod,rules,entropy}.rs`, `src/internal/gitleaks.toml` +- **Create** `crates/path-cli/src/cmd_redact.rs` +- **Modify** `crates/path-cli/src/{cmd_p.rs,sync/engine.rs,cache.rs}`, `crates/path-cli/Cargo.toml`, workspace `Cargo.toml` +- **Modify** `crates/path-cli/tests/integration.rs` +- **Modify** `CLAUDE.md`, `README.md`, `site/_data/crates.json`, `site/pages/crates.md`, `scripts/release.sh`, `CHANGELOG.md` + +--- + +# WAVE 0 - blocking + +## Task 0: Shared vocabulary **[Opus]** + +Types and trait signatures only, bodies `todo!()`. Deliberately tiny: nothing else can start without it. + +**Files:** Create `crates/toolpath-redact/Cargo.toml`, `src/{lib,detect,surface,plan,transform}.rs`; modify workspace `Cargo.toml`. + +- [ ] **Step 0.1: Crate manifest** + +```toml +[package] +name = "toolpath-redact" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository = "https://github.com/empathic/toolpath" +description = "Detect and redact credentials in Toolpath documents" +keywords = ["redaction", "secrets", "toolpath", "privacy"] +categories = ["development-tools"] + +[dependencies] +toolpath = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +regex = "1.12" +aho-corasick = "1.1" +diffy = "0.4" +hmac = "0.12" +sha2 = "0.10" +toml = "0.8" +``` + +Add to workspace `members` and `[workspace.dependencies]`: + +```toml +toolpath-redact = { version = "0.1.0", path = "crates/toolpath-redact" } +``` + +- [ ] **Step 0.2: `src/detect.rs` - the detection contract** + +```rust +use std::ops::Range; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FieldShape { + Prose, + ToolInput, + ToolOutput, + UnifiedDiff, + FileContent, + Uri, + OpaqueJson, +} + +#[derive(Debug, Clone, Copy)] +pub struct Context<'a> { + pub change_type: &'a str, + pub tool_name: Option<&'a str>, + pub actor: &'a str, + pub kind: Option<&'a str>, +} + +#[derive(Debug, Clone, Copy)] +pub struct Candidate<'a> { + pub text: &'a str, + pub shape: FieldShape, + /// RFC 6901 pointer relative to the step. Passed through to the audit + /// record verbatim, so a detector never constructs one. + pub at: &'a str, + pub ctx: Context<'a>, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Finding { + pub span: Range, + /// Lands in the audit record, so it is part of the document contract. + pub rule: String, + pub score: f32, + pub detector: &'static str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Egress { + LocalOnly, + Network, +} + +pub trait Detector: Send + Sync { + fn id(&self) -> &'static str; + + fn detect(&self, c: &Candidate<'_>) -> crate::Result>; + + fn prefilter(&self, _text: &str) -> bool { + true + } + + /// The host refuses a `Network` detector unless explicitly allowed: + /// validating a candidate against its issuing provider sends secret + /// material off the machine. + fn egress(&self) -> Egress { + Egress::LocalOnly + } +} + +#[derive(Default)] +pub struct DetectorSet(Vec>); + +impl DetectorSet { + pub fn push(&mut self, d: Box) { + self.0.push(d); + } + + pub fn ids(&self) -> Vec<&'static str> { + self.0.iter().map(|d| d.id()).collect() + } + + pub fn detect_all(&self, c: &Candidate<'_>) -> crate::Result> { + todo!("T1") + } +} +``` + +- [ ] **Step 0.3: `src/surface.rs`** + +```rust +use crate::detect::FieldShape; + +/// One field the map named, whether or not anything was found in it. A +/// surface with zero findings is information: the pass reached that field +/// and the detectors were silent. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Surface { + pub step: String, + pub at: String, + pub shape: FieldShape, + pub bytes: usize, +} + +pub fn surfaces(path: &toolpath::v1::Path) -> Vec { + todo!("T2") +} + +pub struct SurfaceCursor<'a> { + pub(crate) path: &'a mut toolpath::v1::Path, +} + +impl SurfaceCursor<'_> { + pub fn read(&self, step: &str, at: &str) -> Option { + todo!("T2") + } + pub fn write(&mut self, step: &str, at: &str, value: &str) -> crate::Result<()> { + todo!("T2") + } +} + +pub fn ptr_escape(token: &str) -> String { + todo!("T2") +} +``` + +- [ ] **Step 0.4: `src/plan.rs`** + +```rust +use crate::{detect::FieldShape, transform::Transform}; +use chrono::{DateTime, Utc}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Action { + Redact, + Skip, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct PlanFinding { + pub id: String, + pub step: String, + pub at: String, + pub rule: String, + pub span: (usize, usize), + pub score: f32, + pub detector: String, + pub shape: FieldShape, + /// Surrounding line with the match replaced by its rule name. Never + /// the value, never its length, unless `reveal` was set. + pub context: String, + pub action: Action, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transform: Option, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct PlanDefaults { + pub transform: Transform, + pub threshold: f32, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Plan { + pub v: u32, + pub document: String, + pub generated: DateTime, + pub detectors: Vec, + pub defaults: PlanDefaults, + pub surfaces: Vec, + pub findings: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Decision { + pub predicate: Predicate, + pub action: Action, + pub transform: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Predicate { + Rule(String), + Shape(FieldShape), + Step(String), + Detector(String), + AtPrefix(String), + Score(Cmp, f32), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cmp { + Ge, + Gt, + Le, + Lt, + Eq, +} + +/// Persisted in the sync manifest so a re-derive can replay redaction. +/// Rule-based only: individual finding ids cannot be replayed against +/// content that has moved. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct RedactionPolicy { + pub detectors: Vec, + pub threshold: f32, + pub mode: Transform, + #[serde(default)] + pub mode_for: Vec<(String, Transform)>, + #[serde(default)] + pub accept: Vec, + #[serde(default)] + pub reject: Vec, + pub key_id: String, +} +``` + +- [ ] **Step 0.5: `src/transform.rs`** + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Transform { + Marker, + Remove, + Hash, + /// Length-preserving, and therefore publishes the exact length. + Mask, + /// Keeps 4 leading and 4 trailing chars: leaks provider and format. + Partial, +} + +impl Default for Transform { + fn default() -> Self { + Transform::Marker + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fingerprint(pub String); + +impl Fingerprint { + /// Keyed, never a bare hash: a hash of a low-entropy secret is a + /// dictionary attack away from the secret (EDPB 01/2025 para 88). + pub fn new(key: &[u8], value: &str) -> Self { + todo!("T4") + } +} + +pub trait Transformer: Send + Sync { + fn id(&self) -> &'static str; + fn replace(&self, rule: &str, value: &str, fp: &Fingerprint) -> String; +} +``` + +- [ ] **Step 0.6: `src/lib.rs`** + +```rust +#![doc = include_str!("../README.md")] + +pub mod apply; +pub mod detect; +pub mod exec; +pub mod internal; +pub mod plan; +pub mod surface; +pub mod transform; + +pub use detect::{Candidate, Context, Detector, DetectorSet, Egress, FieldShape, Finding}; +pub use plan::{Action, Plan, PlanFinding, RedactionPolicy}; +pub use surface::{Surface, surfaces}; +pub use transform::{Fingerprint, Transform}; + +use chrono::{DateTime, Utc}; + +#[derive(Debug, thiserror::Error)] +pub enum RedactError { + #[error("detector {0} performs network I/O; pass --allow-network-detectors to permit it")] + NetworkDetectorRefused(String), + #[error("plan does not match document: {0}")] + PlanMismatch(String), + #[error("document carries signatures over redacted content; pass --drop-signatures")] + SignedDocument, + #[error("pointer {0} does not resolve")] + BadPointer(String), + #[error(transparent)] + Json(#[from] serde_json::Error), +} + +pub type Result = std::result::Result; + +/// Everything the engine needs, supplied by the caller. No env, no +/// filesystem, no clock, no globals - see the purity rule in the plan. +#[derive(Debug, Clone)] +pub struct RedactConfig { + pub threshold: f32, + pub mode: Transform, + pub mode_for: Vec<(String, Transform)>, + pub key: Vec, + pub now: DateTime, + pub drop_signatures: bool, + pub reveal: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize)] +pub struct RedactReport { + pub steps_touched: usize, + pub replaced: std::collections::BTreeMap, + pub flagged: std::collections::BTreeMap, + pub signatures_dropped: usize, + pub surfaces_scanned: usize, +} +``` + +- [ ] **Step 0.7: Verify and review** + +```bash +cargo build -p toolpath-redact && cargo clippy -p toolpath-redact -- -D warnings +``` + +**Gate:** compiles clean **and** an Opus reviewer has signed off on the type contract - this is the one artifact every other track builds against, so a naming or shape mistake here is expensive. **Commit before fanning out.** + +--- + +# WAVE 1 - six parallel tracks + +## Task 1: Span normalisation in `DetectorSet` **[Opus impl / Opus review]** + +A third-party detector is not part of this test suite, so its output is normalised rather than trusted. + +**Files:** modify `crates/toolpath-redact/src/detect.rs` + +- [ ] **Step 1.1: Write the hostile-input tests first** + +```rust +#[cfg(test)] +mod tests { + use super::*; + + struct HostileDetector(Vec); + impl Detector for HostileDetector { + fn id(&self) -> &'static str { "hostile" } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(self.0.clone()) + } + } + + fn cand(text: &str) -> Candidate<'_> { + Candidate { + text, + shape: FieldShape::Prose, + at: "/change/x/structural/extra/text", + ctx: Context { + change_type: "conversation.append", + tool_name: None, + actor: "human:t", + kind: None, + }, + } + } + + fn f(span: Range, rule: &str, score: f32) -> Finding { + Finding { span, rule: rule.into(), score, detector: "hostile" } + } + + #[test] + fn drops_out_of_range_spans() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..999, "x", 0.9)]))); + assert!(s.detect_all(&cand("short")).unwrap().is_empty()); + } + + #[test] + fn drops_reversed_spans() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![Finding { + span: 5..2, rule: "x".into(), score: 0.9, detector: "hostile", + }]))); + assert!(s.detect_all(&cand("abcdefgh")).unwrap().is_empty()); + } + + #[test] + fn drops_mid_codepoint_spans() { + // "é" is two bytes; 0..1 splits it. + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..1, "x", 0.9)]))); + assert!(s.detect_all(&cand("é-tail")).unwrap().is_empty()); + } + + #[test] + fn identical_spans_higher_score_wins() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..4, "low", 0.4), f(0..4, "high", 0.9)]))); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].rule, "high"); + } + + #[test] + fn nested_span_container_wins_regardless_of_score() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(2..4, "inner", 0.99), f(0..8, "outer", 0.20)]))); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].rule, "outer"); + } + + #[test] + fn output_is_sorted_and_deterministic() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![ + f(6..8, "b", 0.9), f(0..2, "a", 0.9), f(3..5, "c", 0.9), + ]))); + let a = s.detect_all(&cand("abcdefgh")).unwrap(); + let b = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(a, b); + assert!(a.windows(2).all(|w| w[0].span.start <= w[1].span.start)); + } + + #[test] + fn prefilter_short_circuits_detect() { + struct NeverCalled; + impl Detector for NeverCalled { + fn id(&self) -> &'static str { "never" } + fn prefilter(&self, _t: &str) -> bool { false } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + panic!("detect() must not run when prefilter() is false") + } + } + let mut s = DetectorSet::default(); + s.push(Box::new(NeverCalled)); + assert!(s.detect_all(&cand("anything")).unwrap().is_empty()); + } +} +``` + +- [ ] **Step 1.2: Implement `detect_all` and `normalise`** + +```rust +impl DetectorSet { + pub fn detect_all(&self, c: &Candidate<'_>) -> crate::Result> { + let mut raw = Vec::new(); + for d in &self.0 { + if !d.prefilter(c.text) { + continue; + } + raw.extend(d.detect(c)?); + } + Ok(normalise(c.text, raw)) + } +} + +/// Drop what cannot be applied, then resolve overlaps. +/// +/// Policy from Presidio: identical spans, higher score wins; nested, the +/// container wins regardless of score. Ties break on rule id so output +/// does not depend on HashMap iteration order. +fn normalise(text: &str, mut findings: Vec) -> Vec { + findings.retain(|f| { + f.span.start < f.span.end + && f.span.end <= text.len() + && text.is_char_boundary(f.span.start) + && text.is_char_boundary(f.span.end) + }); + + findings.sort_by(|a, b| { + a.span.start + .cmp(&b.span.start) + .then((b.span.end - b.span.start).cmp(&(a.span.end - a.span.start))) + .then(b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)) + .then(a.rule.cmp(&b.rule)) + }); + + let mut out: Vec = Vec::new(); + for f in findings { + match out.iter_mut().find(|o| f.span.start < o.span.end && f.span.end > o.span.start) { + None => out.push(f), + Some(clash) => { + let f_len = f.span.end - f.span.start; + let c_len = clash.span.end - clash.span.start; + let wins = f_len > c_len + || (f_len == c_len && f.score > clash.score) + || (f_len == c_len && f.score == clash.score && f.rule < clash.rule); + if wins { + *clash = f; + } + } + } + } + out.sort_by(|a, b| a.span.start.cmp(&b.span.start).then(a.rule.cmp(&b.rule))); + out +} +``` + +- [ ] **Step 1.3: Export `FixedDetector` for downstream tracks** + +T7 and T8 both need it, so it lands here rather than being duplicated. + +```rust +/// Canned findings, for tests exercising traversal or transform without +/// depending on regex behaviour. +pub struct FixedDetector(pub Vec); + +impl Detector for FixedDetector { + fn id(&self) -> &'static str { "fixed" } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(self.0.clone()) + } +} +``` + +**Gate:** `cargo test -p toolpath-redact detect::` green **and** review clean. Reviewer: press hardest on whether `normalise` holds for three-way overlaps and for spans that are adjacent but not overlapping. + +--- + +## Task 2: The field map **[Opus impl / Opus review]** + +Produces every candidate the detectors will see, which is also exactly what the dry run reports. + +**Files:** modify `crates/toolpath-redact/src/surface.rs` + +- [ ] **Step 2.1: Write the pointer tests first** + +```rust +#[test] +fn ptr_escape_handles_urls_and_tildes() { + assert_eq!(ptr_escape("claude://sess-abc"), "claude:~1~1sess-abc"); + assert_eq!(ptr_escape("src/config.rs"), "src~1config.rs"); + assert_eq!(ptr_escape("a~b"), "a~0b"); + assert_eq!(ptr_escape("~/x"), "~0~1x"); +} + +#[test] +fn ptr_escape_round_trips() { + for raw in ["claude://sess-abc", "src/config.rs", "a~b/c", "~01"] { + // Decode `~1` before `~0`, or `~01` round-trips wrong (RFC 6901). + let dec = ptr_escape(raw).replace("~1", "/").replace("~0", "~"); + assert_eq!(dec, raw); + } +} +``` + +- [ ] **Step 2.2: Write the traversal tests first** + +```rust +#[test] +fn conversation_append_surfaces_all_text_fields() { + let p = fixture_conversation_append(); + let ats: Vec<&str> = surfaces(&p).iter().map(|s| s.at.as_str()).collect(); + assert!(ats.iter().any(|a| a.ends_with("/structural/extra/text"))); + assert!(ats.iter().any(|a| a.ends_with("/structural/extra/thinking"))); + assert!(ats.iter().any(|a| a.contains("/tool_uses/0/input"))); + assert!(ats.iter().any(|a| a.contains("/tool_uses/0/result/content"))); +} + +#[test] +fn file_write_surfaces_diff_and_both_file_states() { + let p = fixture_file_write(); + let shapes: Vec = surfaces(&p).iter().map(|s| s.shape).collect(); + assert!(shapes.contains(&FieldShape::UnifiedDiff)); + assert_eq!(shapes.iter().filter(|s| **s == FieldShape::FileContent).count(), 2); +} + +#[test] +fn identity_fields_are_never_surfaced() { + for s in surfaces(&fixture_conversation_append()) { + for banned in ["/step/id", "/step/actor", "/step/timestamp", "/step/parents"] { + assert!(!s.at.starts_with(banned), "surfaced identity field: {}", s.at); + } + } +} + +#[test] +fn clean_field_still_appears_as_a_surface() { + // The dry-run guarantee: a surface with nothing in it is information. + let p = fixture_clean_conversation(); + assert!(surfaces(&p).iter().any(|s| s.at.ends_with("/structural/extra/text"))); +} + +#[test] +fn delegations_recurse() { + assert!(surfaces(&fixture_with_delegation()) + .iter() + .any(|s| s.at.contains("/delegations/0/turns/0"))); +} + +#[test] +fn unknown_change_type_degrades_to_blind_walk() { + assert!(!surfaces(&fixture_unknown_change_type()).is_empty()); +} + +#[test] +fn cursor_write_is_readable_at_the_same_pointer() { + let mut p = fixture_conversation_append(); + let at = surfaces(&p)[0].at.clone(); + let step = surfaces(&p)[0].step.clone(); + let mut c = SurfaceCursor { path: &mut p }; + c.write(&step, &at, "replaced").unwrap(); + assert_eq!(c.read(&step, &at).as_deref(), Some("replaced")); +} +``` + +- [ ] **Step 2.3: Implement `ptr_escape` and `surfaces`** + +```rust +pub fn ptr_escape(token: &str) -> String { + token.replace('~', "~0").replace('/', "~1") +} + +pub fn surfaces(path: &toolpath::v1::Path) -> Vec { + let mut out = Vec::new(); + for step in &path.steps { + let sid = &step.step.id; + for (artifact_key, change) in &step.change { + let akey = ptr_escape(artifact_key); + push(&mut out, sid, format!("/change/{akey}"), FieldShape::Uri, artifact_key); + + if let Some(raw) = &change.raw { + push(&mut out, sid, format!("/change/{akey}/raw"), FieldShape::UnifiedDiff, raw); + } + let Some(s) = &change.structural else { continue }; + let base = format!("/change/{akey}/structural"); + match s.change_type.as_str() { + "conversation.append" => append_surfaces(&mut out, sid, &base, &s.extra), + "file.write" => file_write_surfaces(&mut out, sid, &base, &s.extra), + // The one change type where a blind leaf walk is correct: + // the payload is unmodelled provider JSON. + _ => walk_json(&mut out, sid, &base, &s.extra), + } + } + } + if let Some(b) = &path.path.base { + push(&mut out, "", "/path/base/uri".into(), FieldShape::Uri, &b.uri); + } + if let Some(v) = path + .meta + .as_ref() + .and_then(|m| m.extra.get("vcs_remote")) + .and_then(|v| v.as_str()) + { + push(&mut out, "", "/meta/vcs_remote".into(), FieldShape::Uri, v); + } + out +} + +fn push(out: &mut Vec, step: &str, at: String, shape: FieldShape, text: &str) { + if text.is_empty() { + return; + } + out.push(Surface { step: step.to_string(), at, shape, bytes: text.len() }); +} +``` + +Implement `append_surfaces` (`text`/`thinking` as `Prose`; `tool_uses[].input` as `ToolInput` recursing to string leaves; `tool_uses[].result.content` as `ToolOutput`; `delegations[]` recursively; `environment.working_dir` as `Uri`), `file_write_surfaces` (`before`/`after`/`edits[]` as `FileContent`), and `walk_json`. + +- [ ] **Step 2.4: Implement `SurfaceCursor`** + +Split the pointer on `/`, decode each token (`~1` then `~0`), walk `serde_json::Value` by key or array index. Read and write must resolve identically - the test in 2.2 pins that. + +**Gate:** `cargo test -p toolpath-redact surface::` green **and** review clean. Reviewer: check the pointer decode order, and that `bytes` is byte length rather than char length everywhere it is compared. + +--- + +## Task 3: The internal detector **[Sonnet impl / Opus review]** + +**Files:** create `crates/toolpath-redact/src/internal/{mod,rules,entropy}.rs`, `src/internal/gitleaks.toml` + +- [ ] **Step 3.1: Vendor the ruleset** + +Copy `config/gitleaks.toml` from `github.com/gitleaks/gitleaks` verbatim. Record the upstream commit and MIT license in the crate README. + +- [ ] **Step 3.2: Write the compile-guard test first** + +If this fails, stop: the vendoring assumption is wrong and the task changes shape. + +```rust +#[test] +fn every_vendored_rule_compiles_under_rust_regex() { + let rules = load_rules(); + assert!(rules.len() >= 200, "expected the full ruleset, got {}", rules.len()); + for r in &rules { + regex::Regex::new(&r.regex) + .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); + } +} +``` + +- [ ] **Step 3.3: Write positive and negative fixtures first** + +```rust +#[test] +fn detects_shipped_formats() { + for (label, sample) in [ + ("aws", "AKIAIOSFODNN7REALKEY"), + ("google", "AIzaSyD-0123456789abcdefghijklmnopqrstu"), + ("jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.QQQQQQQQQQ"), + ("pem", "-----BEGIN RSA PRIVATE KEY-----"), + ("dburi", "postgres://u:s3cr3tpass@db.internal:5432/prod"), + ] { + assert!(!detect_one(sample).is_empty(), "missed {label}"); + } +} + +#[test] +fn documented_false_positives_stay_below_threshold() { + for sample in [ + "AKIAIOSFODNN7EXAMPLE", // AWS's own documentation key + "redis://localhost:6379", // no password + "0e2b3d4e3dec5f38ae95f62519eb2736f73c0b", // git SHA + "550e8400-e29b-41d4-a716-446655440000", // UUID + "ThisIsAReallyLongString", // high entropy, not a secret + ] { + assert!(detect_one(sample).iter().all(|f| f.score < 0.8), "false positive on {sample}"); + } +} + +#[test] +fn diff_spans_never_cross_a_newline() { + let c = diff_candidate(); + for f in InternalDetector::new().detect(&c).unwrap() { + assert!(!c.text[f.span.clone()].contains('\n')); + } +} + +#[test] +fn uri_shape_redacts_only_the_password() { + let c = uri_candidate("postgres://svc_user:h0rr1bl3@db.internal:5432/prod"); + let f = &InternalDetector::new().detect(&c).unwrap()[0]; + assert_eq!(&c.text[f.span.clone()], "h0rr1bl3"); +} + +#[test] +fn existing_markers_are_never_re_detected() { + assert!(detect_one("[REDACTED:aws-access-key-id:a3c829]").is_empty()); + assert!(detect_one("████████████████████").is_empty()); +} +``` + +- [ ] **Step 3.4: Implement rule loading and the prefilter** + +```rust +pub struct Rule { + pub id: String, + pub regex: String, + pub entropy: Option, + pub keywords: Vec, + pub stopwords: Vec, +} + +pub struct InternalDetector { + rules: Vec<(Rule, regex::Regex)>, + prefilter: aho_corasick::AhoCorasick, +} +``` + +Build the automaton with `MatchKind::LeftmostLongest` over every rule keyword. + +- [ ] **Step 3.5: Implement scoring** + +Base score from the rule; subtract when Shannon entropy is below the rule's threshold; add when a hotword appears within 50 characters; clamp to `0.0..=1.0`. + +```rust +pub fn shannon(s: &str) -> f64 { + if s.is_empty() { + return 0.0; + } + let mut counts = std::collections::HashMap::new(); + for ch in s.chars() { + *counts.entry(ch).or_insert(0usize) += 1; + } + let n = s.chars().count() as f64; + -counts.values().map(|&c| { let p = c as f64 / n; p * p.log2() }).sum::() +} +``` + +- [ ] **Step 3.6: Implement checksum validators** *(first cut candidate)* + +GitHub PAT: last 6 chars are base62 CRC32 of the body. AWS key id: base32 body. Luhn for cards. + +**Gate:** `cargo test -p toolpath-redact internal::` green **and** review clean. Reviewer: the scoring function is the highest-risk code here - demand a test per scoring branch. + +--- + +## Task 4: Transforms and fingerprints **[Sonnet impl / Opus review]** + +**Files:** modify `crates/toolpath-redact/src/transform.rs` + +- [ ] **Step 4.1: Write the tests first** + +```rust +#[test] +fn fingerprint_is_deterministic() { + let k = b"test-key"; + assert_eq!(Fingerprint::new(k, "abc"), Fingerprint::new(k, "abc")); + assert_ne!(Fingerprint::new(k, "abc"), Fingerprint::new(b"other", "abc")); +} + +#[test] +fn mask_preserves_character_count_not_byte_count() { + let out = apply_transform(Transform::Mask, "rule", "héllo", &fp()); + assert_eq!(out.chars().count(), 5); +} + +#[test] +fn partial_falls_back_to_mask_below_the_floor() { + let out = apply_transform(Transform::Partial, "rule", "short", &fp()); + assert!(!out.contains("short")); + assert_eq!(out.chars().count(), 5); +} + +#[test] +fn only_partial_ever_emits_a_substring_of_its_input() { + let value = "AKIAIOSFODNN7REALKEY"; + for t in [Transform::Marker, Transform::Remove, Transform::Hash, Transform::Mask] { + let out = apply_transform(t, "aws-access-key-id", value, &fp()); + for w in 4..=value.len() { + for s in value.as_bytes().windows(w) { + let sub = std::str::from_utf8(s).unwrap(); + assert!(!out.contains(sub), "{t:?} leaked {sub:?}"); + } + } + } +} + +#[test] +fn right_to_left_application_matches_one_at_a_time() { + let text = "aaa BBB ccc DDD eee"; + let spans = vec![4..7, 12..15]; + assert_eq!(apply_spans(text, &spans), apply_one_by_one_from_right(text, &spans)); +} + +#[test] +fn per_rule_override_beats_global_and_per_finding_beats_both() { + let cfg = cfg_with(Transform::Marker, vec![("us-ssn".into(), Transform::Mask)]); + assert_eq!(resolve(&cfg, "us-ssn", None), Transform::Mask); + assert_eq!(resolve(&cfg, "aws-access-key-id", None), Transform::Marker); + assert_eq!(resolve(&cfg, "us-ssn", Some(Transform::Remove)), Transform::Remove); +} +``` + +- [ ] **Step 4.2: Implement** + +```rust +impl Fingerprint { + pub fn new(key: &[u8], value: &str) -> Self { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let mut mac = >::new_from_slice(key).expect("HMAC accepts any key length"); + mac.update(value.as_bytes()); + Fingerprint(hex(&mac.finalize().into_bytes())[..6].to_string()) + } +} + +pub fn apply_transform(t: Transform, rule: &str, value: &str, fp: &Fingerprint) -> String { + match t { + Transform::Marker => format!("[REDACTED:{rule}:{}]", fp.0), + Transform::Remove => String::new(), + Transform::Hash => fp.0.clone(), + Transform::Mask => "\u{2588}".repeat(value.chars().count()), + Transform::Partial => { + let n = value.chars().count(); + if n > 10 { + let head: String = value.chars().take(4).collect(); + let tail: String = value.chars().skip(n - 4).collect(); + format!("{head}\u{2026}{tail}") + } else { + "\u{2588}".repeat(n) + } + } + } +} + +/// Sort descending by start so earlier offsets stay valid as later spans +/// are spliced out. +pub fn apply_spans_desc(text: &str, edits: &mut [(std::ops::Range, String)]) -> String { + edits.sort_by(|a, b| b.0.start.cmp(&a.0.start)); + let mut out = text.to_string(); + for (span, repl) in edits.iter() { + out.replace_range(span.clone(), repl); + } + out +} +``` + +**Gate:** `cargo test -p toolpath-redact transform::` green **and** review clean. + +--- + +## Task 5: Plan predicates and verification **[Sonnet impl / Opus review]** + +**Files:** modify `crates/toolpath-redact/src/plan.rs` + +- [ ] **Step 5.1: Write the tests first** + +```rust +#[test] +fn parses_every_predicate_field() { + assert!(matches!(parse_predicate("rule=aws-access-key-id").unwrap(), Predicate::Rule(_))); + assert!(matches!(parse_predicate("shape=unified_diff").unwrap(), Predicate::Shape(_))); + assert!(matches!(parse_predicate("step=turn-0f3a").unwrap(), Predicate::Step(_))); + assert!(matches!(parse_predicate("detector=internal").unwrap(), Predicate::Detector(_))); + assert!(matches!(parse_predicate("at=/change/x").unwrap(), Predicate::AtPrefix(_))); + assert!(matches!(parse_predicate("score>=0.95").unwrap(), Predicate::Score(Cmp::Ge, _))); +} + +#[test] +fn rejects_anything_else_clearly() { + let e = parse_predicate("colour=red").unwrap_err().to_string(); + assert!(e.contains("colour"), "error should name the bad field: {e}"); +} + +#[test] +fn last_matching_decision_wins() { + let mut plan = plan_with_findings(&[("f01", "aws-access-key-id", 0.99)]); + apply_decisions(&mut plan, &[ + decision("rule=aws-access-key-id", Action::Redact), + decision("score>=0.9", Action::Skip), + ]); + assert_eq!(plan.findings[0].action, Action::Skip); +} + +#[test] +fn ids_are_stable_across_regeneration() { + let p = fixture_path(); + let a = generate(&p, &detectors(), &cfg()); + let b = generate(&p, &detectors(), &cfg()); + assert_eq!(serde_json::to_string(&a).unwrap(), serde_json::to_string(&b).unwrap()); +} + +#[test] +fn verify_refuses_a_changed_document() { + let p = fixture_path(); + let plan = generate(&p, &detectors(), &cfg()); + let mutated = mutate_one_byte_inside_a_recorded_span(&p); + let e = verify(&plan, &mutated).unwrap_err().to_string(); + assert!(e.contains("f01"), "error should name the first divergence: {e}"); +} + +#[test] +fn context_never_carries_the_value_or_its_length() { + let plan = generate(&fixture_with_secret("AKIAIOSFODNN7REALKEY"), &detectors(), &cfg()); + let ctx = &plan.findings[0].context; + assert!(!ctx.contains("AKIAIOSFODNN7REALKEY")); + assert!(!ctx.contains("20")); + assert!(ctx.contains("")); +} + +#[test] +fn reveal_includes_the_value() { + let cfg = RedactConfig { reveal: true, ..cfg() }; + let plan = generate(&fixture_with_secret("AKIAIOSFODNN7REALKEY"), &detectors(), &cfg); + assert!(plan.findings[0].context.contains("AKIAIOSFODNN7REALKEY")); +} +``` + +- [ ] **Step 5.2: Implement predicate parsing** + +Only these forms. No expression language. + +```rust +pub fn parse_predicate(s: &str) -> crate::Result { + // Longest operators first, or `>=` parses as `>`. + for (op, cmp) in [(">=", Cmp::Ge), ("<=", Cmp::Le), (">", Cmp::Gt), ("<", Cmp::Lt)] { + if let Some((k, v)) = s.split_once(op) { + if k.trim() == "score" { + return Ok(Predicate::Score(cmp, v.trim().parse().map_err(|_| bad(s))?)); + } + } + } + let (k, v) = s.split_once('=').ok_or_else(|| bad(s))?; + Ok(match k.trim() { + "rule" => Predicate::Rule(v.trim().into()), + "shape" => Predicate::Shape(parse_shape(v.trim())?), + "step" => Predicate::Step(v.trim().into()), + "detector" => Predicate::Detector(v.trim().into()), + "at" => Predicate::AtPrefix(v.trim().into()), + "score" => Predicate::Score(Cmp::Eq, v.trim().parse().map_err(|_| bad(s))?), + other => return Err(unknown_field(other)), + }) +} +``` + +- [ ] **Step 5.3: Implement id generation, verification, context builder** + +Ids are `f01`, `f02`, ... ordered by `(step index, pointer, span start)`, which is what makes a regenerated plan byte-identical. `verify()` checks document id, step existence, and that each span still lands at the recorded offsets, naming the first divergence. + +**Gate:** `cargo test -p toolpath-redact plan::` green **and** review clean. Reviewer: `>=` versus `>` ordering in the parser is a classic silent bug - confirm there is a test. + +--- + +## Task 6: CLI argument surface **[Haiku impl / Opus review]** + +Parsing only, no dispatch, so it does not wait on the engine. + +**Files:** create `crates/path-cli/src/cmd_redact.rs`; modify `crates/path-cli/src/cmd_p.rs` + +- [ ] **Step 6.1: Define `RedactArgs`** + +```rust +use clap::Args; + +#[derive(Debug, Args)] +pub(crate) struct RedactArgs { + /// Cache id or file path. + #[arg(short, long)] + pub input: String, + + /// Write elsewhere instead of in place. + #[arg(short, long)] + pub output: Option, + + #[arg(long, conflicts_with = "plan")] + pub dry_run: bool, + #[arg(long)] + pub plan: Option, + /// Include real values in the plan (written 0600). + #[arg(long)] + pub reveal: bool, + + #[arg(long, value_name = "PREDICATE")] + pub accept: Vec, + #[arg(long, value_name = "PREDICATE")] + pub reject: Vec, + #[arg(long)] + pub interactive: bool, + #[arg(long, value_name = "PREDICATE:TRANSFORM")] + pub mode_for: Vec, + + #[arg(long, default_value = "internal")] + pub detector: Vec, + #[arg(long, default_value_t = 0.8)] + pub threshold: f32, + #[arg(long)] + pub allow_network_detectors: bool, + + #[arg(long, value_enum, default_value_t = TransformArg::Marker)] + pub mode: TransformArg, + #[arg(long)] + pub key_file: Option, + + #[arg(long)] + pub json: bool, + #[arg(long)] + pub drop_signatures: bool, +} + +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +pub(crate) enum TransformArg { Marker, Remove, Hash, Mask, Partial } +``` + +- [ ] **Step 6.2: Wire `PCommand::Redact`** in `cmd_p.rs` with a `todo!()` dispatch. T9 fills it in. + +- [ ] **Step 6.3: Define the injectable picker** + +**Do not call `fuzzy::set_picker_override` anywhere.** Mirror `cmd_resume::ExecStrategy`: + +```rust +pub(crate) trait PickerStrategy { + fn pick(&self, rows: &[String]) -> Result>; +} + +pub(crate) struct RealPicker; + +pub(crate) struct RecordingPicker { + pub selection: Vec, + pub seen: std::cell::RefCell>, +} +``` + +- [ ] **Step 6.4: Write the arg tests** + +```rust +#[test] +fn dry_run_conflicts_with_plan() { + assert!(try_parse(&["redact", "-i", "x", "--dry-run", "--plan", "p.json"]).is_err()); +} + +#[test] +fn mode_for_rejects_unknown_transform() { + assert!(parse_mode_for("rule=x:invented").is_err()); +} + +#[test] +fn detector_flag_is_repeatable() { + let a = try_parse(&["redact", "-i", "x", "--detector", "internal", "--detector", "exec:/bin/s"]).unwrap(); + assert_eq!(a.detector.len(), 2); +} +``` + +**Gate:** `cargo test -p path-cli cmd_redact::args` green **and** review clean. + +--- + +# WAVE 2 - two parallel tracks + +## Task 7: Apply **[Opus impl / Opus review]** *(needs T2, T4)* + +Consumes a `Plan`; needs no detector, so it does not wait on T3. + +**Files:** modify `crates/toolpath-redact/src/apply.rs` + +- [ ] **Step 7.1: Write the invariant tests first. These are the suite.** + +```rust +#[test] +fn no_findings_means_byte_identical_output() { + // The most important test here: the pass must not perturb anything it + // is not redacting. + let before = fixture_clean_document(); + let mut after = before.clone(); + apply(&mut after, &empty_plan(&before), &cfg()).unwrap(); + assert_eq!( + serde_json::to_string_pretty(&before).unwrap(), + serde_json::to_string_pretty(&after).unwrap() + ); +} + +#[test] +fn unknown_provider_keys_survive() { + // Guards against reimplementing this as extract -> derive: + // `extra["edits"]` is written by toolpath-convo's derive and never + // read back by extract, so a round-trip silently drops it. + let mut doc = fixture_with_extra_keys(&["edits", "vendor_specific", "entry_extra"]); + apply(&mut doc, &plan_touching_only_text(&doc), &cfg()).unwrap(); + for k in ["edits", "vendor_specific", "entry_extra"] { + assert!(has_key_somewhere(&doc, k), "lost {k}"); + } +} + +#[test] +fn idempotent_across_all_transforms() { + for mode in [Transform::Marker, Transform::Remove, Transform::Hash, + Transform::Mask, Transform::Partial] { + let cfg = RedactConfig { mode, ..cfg() }; + let mut once = fixture_with_secrets(); + apply(&mut once, &plan_for(&once), &cfg).unwrap(); + let mut twice = once.clone(); + apply(&mut twice, &plan_for(&twice), &cfg).unwrap(); + assert_eq!( + serde_json::to_string(&once).unwrap(), + serde_json::to_string(&twice).unwrap(), + "{mode:?} is not idempotent" + ); + } +} + +#[test] +fn redacted_diff_still_parses_and_line_counts_hold() { + let mut doc = fixture_file_write_with_secret_in_diff(); + apply(&mut doc, &plan_for(&doc), &cfg()).unwrap(); + let raw = raw_diff_of(&doc); + let patch = diffy::Patch::from_str(&raw).expect("redacted diff must still parse"); + for h in patch.hunks() { + assert_eq!(h.old_range().len(), count_lines(h, '-')); + assert_eq!(h.new_range().len(), count_lines(h, '+')); + } +} + +#[test] +fn audit_record_lands_on_the_step_and_merges_on_rerun() { + let mut doc = fixture_with_secrets(); + apply(&mut doc, &plan_for(&doc), &cfg()).unwrap(); + let first = record_of(&doc, "turn-0f3a").clone(); + apply(&mut doc, &plan_for(&doc), &cfg()).unwrap(); + assert_eq!(record_of(&doc, "turn-0f3a"), &first, "record must merge, not append"); +} + +#[test] +fn audit_record_carries_no_value_substring_or_length() { + let secret = "AKIAIOSFODNN7REALKEY"; + let mut doc = fixture_with_secret(secret); + apply(&mut doc, &plan_for(&doc), &cfg()).unwrap(); + let rec = serde_json::to_string(record_of(&doc, "turn-0f3a")).unwrap(); + assert!(!rec.contains(secret)); + for w in 6..secret.len() { + for s in secret.as_bytes().windows(w) { + assert!(!rec.contains(std::str::from_utf8(s).unwrap())); + } + } + assert!(!rec.contains(&secret.len().to_string())); +} + +#[test] +fn signed_document_refuses_without_the_flag() { + let mut doc = fixture_signed(); + assert!(matches!( + apply(&mut doc, &plan_for(&doc), &cfg()), + Err(RedactError::SignedDocument) + )); + let cfg = RedactConfig { drop_signatures: true, ..cfg() }; + assert_eq!(apply(&mut doc, &plan_for(&doc), &cfg).unwrap().signatures_dropped, 1); +} + +#[test] +fn output_validates_against_both_schemas() { + let mut doc = fixture_with_secrets(); + apply(&mut doc, &plan_for(&doc), &cfg()).unwrap(); + let v = serde_json::to_value(&doc).unwrap(); + assert!(base_schema().is_valid(&v)); + assert!(kind_schema_v1_1_0().is_valid(&v)); +} +``` + +- [ ] **Step 7.2: Implement `apply`** + +```rust +pub fn apply( + path: &mut toolpath::v1::Path, + plan: &crate::plan::Plan, + cfg: &crate::RedactConfig, +) -> crate::Result { + crate::plan::verify(plan, path)?; + guard_signatures(path, cfg)?; + + let mut report = crate::RedactReport { + surfaces_scanned: plan.surfaces.len(), + ..Default::default() + }; + + // Group by field so every edit to one string applies in a single + // right-to-left pass; applying them one at a time would invalidate + // the offsets of the ones still pending. + for ((step, at), group) in group_by_field(plan) { + let mut cursor = crate::surface::SurfaceCursor { path }; + let Some(text) = cursor.read(&step, &at) else { continue }; + + let mut edits = Vec::new(); + for f in group.iter().filter(|f| f.action == crate::plan::Action::Redact) { + let value = &text[f.span.0..f.span.1]; + let fp = crate::transform::Fingerprint::new(&cfg.key, value); + let t = resolve_transform(cfg, &f.rule, f.transform); + edits.push(( + f.span.0..f.span.1, + crate::transform::apply_transform(t, &f.rule, value, &fp), + )); + *report.replaced.entry(f.rule.clone()).or_default() += 1; + } + if edits.is_empty() { + continue; + } + let new_text = crate::transform::apply_spans_desc(&text, &mut edits); + cursor.write(&step, &at, &new_text)?; + report.steps_touched += 1; + } + + write_step_records(path, plan, cfg)?; + write_rollup(path, plan, cfg, &report)?; + Ok(report) +} +``` + +- [ ] **Step 7.3: Implement the audit record** + +`Step.meta.extra["redaction"]`, creating `Step.meta` where absent. Aggregate by `(rule, fp)`. **Merge, never append.** Rollup at `path.meta.extra["redaction"]`. + +**Gate:** `cargo test -p toolpath-redact apply::` green **and** review clean. Reviewer: this task carries the most invariants in the codebase - for each one, name an input the tests do not cover and demand a test for it. + +--- + +## Task 8: Plan generation **[Sonnet impl / Opus review]** *(needs T2, T3, T5)* + +**Files:** modify `crates/toolpath-redact/src/plan.rs`; create `src/exec.rs` + +- [ ] **Step 8.1: Write the tests first** + +```rust +#[test] +fn surfaces_and_findings_are_both_populated() { + let plan = generate(&fixture_mixed(), &detectors(), &cfg()); + assert!(plan.surfaces.iter().any(|s| findings_at(&plan, &s.at) == 0)); + assert!(!plan.findings.is_empty()); +} + +#[test] +fn two_detectors_merge_through_one_resolution() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![f(0..20, "a", 0.9)]))); + set.push(Box::new(FixedDetector(vec![f(0..20, "b", 0.5)]))); + assert_eq!(generate(&fixture_one_field(), &set, &cfg()).findings.len(), 1); +} + +#[test] +fn network_detector_is_refused_without_the_flag() { + let mut set = DetectorSet::default(); + set.push(Box::new(NetworkDetector)); + assert!(matches!( + generate_checked(&fixture_one_field(), &set, &cfg()), + Err(RedactError::NetworkDetectorRefused(_)) + )); +} +``` + +- [ ] **Step 8.2: Implement `generate`** + +Walk `surfaces()`, build a `Candidate` per surface, call `DetectorSet::detect_all`, convert to `PlanFinding` with a stable id and elided context, set `action` from the threshold. + +- [ ] **Step 8.3: Implement `exec.rs`** *(second cut candidate)* + +One JSON object per candidate on stdin, one array of findings on stdout. Test against a fixture shell script, not a real scanner. + +**Gate:** `cargo test -p toolpath-redact plan_gen:: exec::` green **and** review clean. + +--- + +# WAVE 3 - synchronisation point + +## Task 9: CLI dispatch **[Sonnet impl / Opus review]** *(needs T6, T7, T8)* + +**Files:** modify `crates/path-cli/src/cmd_redact.rs`, `crates/path-cli/src/cache.rs` + +- [ ] **Step 9.1: Write the tests first** + +```rust +#[test] +fn cache_input_rewrites_in_place() { + let cfg = sandbox(); + seed_cached_document(&cfg, "claude-abc123"); + run_redact(&cfg, &["-i", "claude-abc123"]).unwrap(); + assert!(read_cached(&cfg, "claude-abc123").contains("[REDACTED:")); + assert_eq!(mode_of(cache_path(&cfg, "claude-abc123")), 0o600); +} + +#[test] +fn key_is_generated_once_and_reused() { + let cfg = sandbox(); + seed_cached_document(&cfg, "claude-abc123"); + run_redact(&cfg, &["-i", "claude-abc123"]).unwrap(); + let first = fingerprints_in(read_cached(&cfg, "claude-abc123")); + reseed_same_document(&cfg, "claude-abc123"); + run_redact(&cfg, &["-i", "claude-abc123"]).unwrap(); + assert_eq!(first, fingerprints_in(read_cached(&cfg, "claude-abc123"))); +} + +#[test] +fn interactive_uses_the_injected_picker() { + let picker = RecordingPicker { selection: vec!["f01".into()], seen: Default::default() }; + let out = run_redact_with_picker(&sandbox(), &["-i", "doc.json", "--interactive"], &picker); + assert_eq!(picker.seen.borrow().len(), 3); + assert_eq!(redacted_ids(&out), vec!["f01"]); +} + +#[test] +fn dry_run_exits_one_when_findings_exist() { + assert_eq!(run_redact(&sandbox(), &["-i", "doc.json", "--dry-run"]).code(), 1); +} +``` + +- [ ] **Step 9.2: Implement dispatch** + +Resolve `--input` as cache id or file. Build the `DetectorSet`, refusing `Egress::Network` without the flag. Load or create the key. Generate or load the plan. Apply decisions. Write in place for a cache id, to `--output`, or to stdout. + +- [ ] **Step 9.3: Implement key storage in `cache.rs`** + +`$TOOLPATH_CONFIG_DIR/redact-keys/`, file `0600`, parent `0700`. A missing key on re-redaction is a hard error, never a silent new key. + +**Gate:** `cargo test -p path-cli cmd_redact::` green **and** review clean. **Unblocks Wave 4.** + +--- + +# WAVE 4 - three parallel tracks + +## Task 10: Sync integration **[Opus impl / Opus review]** *(needs T9)* + +**Load-bearing.** `sync::engine::is_unchanged` (`engine.rs:128`) decides re-derivation from source mtime + size + cache-file existence, and **never inspects the document**. An in-place redaction survives while the session is untouched and is **silently destroyed** the moment the user resumes it and sync re-derives with force - which `path query` triggers implicitly. + +**Files:** modify `crates/path-cli/src/sync/engine.rs`, `crates/path-cli/src/cmd_cache.rs` + +- [ ] **Step 10.1: Write the regression test for the hazard first** + +```rust +#[test] +fn sync_reapplies_redaction_after_source_grows() { + let cfg = sandbox(); + let session = seed_claude_session(&cfg, "sess-1", &["AKIAIOSFODNN7REALKEY"]); + run_sync(&cfg); + run_redact(&cfg, &["-i", "claude-sess-1"]).unwrap(); + + append_turn(&session, "another turn with AKIAIOSFODNN7SECONDKEY"); + run_sync(&cfg); + + let doc = read_cached(&cfg, "claude-sess-1"); + assert!(doc.contains("another turn"), "new content must land"); + assert!(!doc.contains("AKIAIOSFODNN7REALKEY"), "redaction must survive re-derive"); + assert!(!doc.contains("AKIAIOSFODNN7SECONDKEY"), "new content must be redacted too"); +} + +#[test] +fn sync_skips_redacted_doc_when_source_unchanged() { /* ... */ } + +#[test] +fn sync_reports_reappeared_skips() { /* ... */ } + +#[test] +fn sync_fails_loudly_on_missing_key() { + // Must never write an un-redacted document over a redacted one. +} + +#[test] +fn manifest_without_redaction_field_still_loads() { + let json = r#"{"claude":{"sess-1":{"cache_id":"claude-sess-1","synced_at":"2026-01-01T00:00:00Z"}}}"#; + assert!(serde_json::from_str::(json).is_ok()); +} +``` + +- [ ] **Step 10.2: Extend `SyncRecord`** + +```rust +pub(crate) struct SyncRecord { + // ... path, cache_id, modified, size, synced_at ... + /// Policy to replay after a re-derive. Rule-based only: individual + /// finding ids cannot be replayed against content that has moved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) redaction: Option, +} +``` + +- [ ] **Step 10.3: Replay in the ingestion loop.** After a successful re-derive, if the record carries a policy: load the key, re-run detection, apply the policy, then write. On a missing key, fail that artifact and tally it. + +- [ ] **Step 10.4: Report replay** in the sync summary: `re-redacted 3 documents; 2 previously-skipped findings reappeared`. + +- [ ] **Step 10.5:** `p cache rm` drops the stored key alongside the document. + +**Gate:** `cargo test -p path-cli sync` green **and** review clean; manually confirm `path query` on a redacted-then-resumed session leaves it redacted. + +--- + +## Task 11: End-to-end integration **[Sonnet impl / Opus review]** *(needs T9)* + +All subprocess-based with `.env()` per `Command`, so they parallelise. + +**Files:** modify `crates/path-cli/tests/integration.rs` + +- [ ] `redact_dry_run_lists_surfaces_with_zero_findings` +- [ ] `redact_plan_apply_round_trip` - dry run to a file, apply it, matches a single-shot run +- [ ] `redact_plan_refuses_mismatched_document` +- [ ] `redact_accept_reject_precedence` +- [ ] `redact_in_place_rewrites_cache_entry` +- [ ] `redact_refuses_network_detector_without_flag` +- [ ] `redact_output_still_validates` - piped through `p validate` +- [ ] `redact_corpus_smoke` (`#[ignore]`) - redact every document in the local cache; no panics, no schema violations + +**Gate:** `cargo test -p path-cli --test integration redact_` green **and** review clean. + +--- + +## Task 12: Docs and release wiring **[Haiku impl / Opus review]** *(no dependencies)* + +**Files:** create `crates/toolpath-redact/README.md`; modify `CLAUDE.md`, `README.md`, `site/_data/crates.json`, `site/pages/crates.md`, `scripts/release.sh`, `CHANGELOG.md` + +- [ ] **Step 12.1:** Crate README - purpose, the `Detector` contract, vendored-ruleset attribution with upstream commit and MIT license; `#![doc = include_str!("../README.md")]` in `lib.rs`. +- [ ] **Step 12.2:** `CLAUDE.md` - repository layout, dependency graph, CLI usage block, per-crate test counts, and a "Things to know" entry covering in-place semantics, plan-then-apply, the `Detector` plug point, and the sync replay. Also add a line for `docs/superpowers/`, which is currently undocumented. +- [ ] **Step 12.3:** `README.md` workspace listing. +- [ ] **Step 12.4:** `site/_data/crates.json` entry and `site/pages/crates.md` dependency diagram. +- [ ] **Step 12.5:** `scripts/release.sh` - `ALL_CRATES` and tier 2. +- [ ] **Step 12.6:** `CHANGELOG.md` new section; bump `path-cli` minor. +- [ ] **Step 12.7:** `cd site && pnpm run build` produces its expected page count. + +**Gate:** workspace build, test, and clippy all clean **and** review clean. + +--- + +## Done criteria + +- [ ] Every task's tests were written before its implementation, are green, and survived adversarial review. +- [ ] Every reviewer change list was applied or rebutted once with a reason; nothing was silently dropped. +- [ ] `cargo test -p toolpath-redact` needs no temp dir, no env var, and no lock. +- [ ] No new code calls `std::env::set_var` in a unit test or `fuzzy::set_picker_override` anywhere. +- [ ] No comment restates what the code does; every comment explains why, records a hazard, or cites an external contract. +- [ ] `path p redact --input --dry-run` lists **every** surface the map names, including zero-finding ones, and exits `1` when findings exist. +- [ ] A plan can be decided by predicate, by picker, or by hand-editing JSON, and applied to produce exactly the edits it describes. +- [ ] All five transforms are selectable globally and per rule; `marker` is the default. +- [ ] `--input ` rewrites in place; no second file. +- [ ] Resuming a redacted session and re-syncing leaves it redacted, with new turns redacted too. +- [ ] Redacting a document with no findings is byte-identical to its input; redacting twice is byte-identical to redacting once. +- [ ] Redacted output validates against the base schema and the `agent-coding-session` kind schema. + +--- + +## Deliberately out of scope + +- `path share` gains no scan, warning, or automatic redaction. +- No live credential verification, in any detector, by default. +- Pre-tool-use / harness-time redaction. The `Detector` contract takes strings plus context specifically so this stays possible; nothing here implements it. +- `Graph` documents are handled one `Path` at a time. diff --git a/docs/superpowers/specs/2026-07-30-path-redact-command-design.md b/docs/superpowers/specs/2026-07-30-path-redact-command-design.md new file mode 100644 index 00000000..fa201af5 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-path-redact-command-design.md @@ -0,0 +1,888 @@ +# `path p redact` - redacting secrets from a generated document + +**Status:** Design proposal +**Date:** 2026-07-30 + +## Goal + +Remove credentials from a toolpath document that has already been generated. + +``` +path p redact --input claude-abc123 --output clean.json +``` + +Agent sessions carry secrets that arrived as ordinary work content: a `.env` +read into a tool result, a connection string echoed by a command, a token pasted +into a prompt. A census of 95 local Claude Code sessions found credential-shaped +strings in 18 of them, spread across 21 distinct JSON paths. Sharing such a +document publishes them. + +## Why this is a post-generation pass + +Three other places were considered and rejected. + +**At write time, via harness hooks.** Not possible. Claude Code's `PreToolUse` +`updatedInput` changes what executes, and `PostToolUse` `updatedToolOutput` +changes what the model sees next, but neither changes what is persisted. A +controlled experiment (`~/Workspace/scripts/claude-hook-redaction-experiment.sh`) +confirmed both canaries survive to disk: the assistant record keeps the +pre-rewrite command, and the hook's replacement output is *appended* as an extra +record rather than substituted. Hooks can block a call outright; they cannot +scrub one that runs. + +**At derive time, inside `toolpath_convo::derive_path`.** Wrong layer. All seven +conversation providers and every round-trip fidelity test run through that +function, and those tests assert exact text equality against the *original* +conversation (`crates/path-cli/tests/roundtrip.rs:89`, `cross_harness_matrix.rs`). +A redactor there breaks them by construction, and it would make the local cache +lossy, which `path resume` depends on it not being. + +**At egress, inside `cmd_export::run_pathbase_inner`.** This was the earlier +proposal. It is the only unbypassable point, which is its appeal, but it couples +redaction to one destination and makes it implicit. A document is a document; +whatever is unsafe to upload is also unsafe to paste into an issue, mail to a +colleague, or commit. Redaction belongs to the document, not to one of its exits. + +A post-generation `Path -> Path` pass is explicit, destination-agnostic, +composable in a pipeline, and testable in isolation. + +## Non-goals + +- **`path share` does not scan, warn, or redact.** Redaction is a step the user + runs. The consequence is accepted deliberately: an un-redacted upload is one + forgotten command away. Revisit if that turns out to bite. +- **No live credential verification.** Validating a candidate against its issuing + provider sends secret material off the machine, including false positives that + may be someone else's real credential. That is the failure this tool exists to + prevent. +- **Not a history scrubber.** Redacting a shared copy does not un-leak a + credential that was already exposed. Rotation is still mandatory. + +## The model + +`redact` is an endomorphism over a document: + +```rust +pub fn redact(path: &mut Path, cfg: &RedactConfig) -> RedactReport; +``` + +It is deliberately **not** implemented as `extract_conversation` -> redact the IR +-> `derive_path`. That round-trip is lossy, and a redactor whose contract is +"change only what I redact" cannot be built on a lossy transform. + +The proof is concrete. `toolpath_convo::derive` writes `extra["edits"]`, the raw +MultiEdit old/new array, at `derive.rs:607`. `extract.rs` never reads that key, +and `FileMutation` has no field to hold it. More generally there is **no `extra` +escape hatch on any toolpath-convo IR type**: `Turn` and `FileMutation` are +closed structs, so every provider-specific key outside their typed fields is +dropped on the way through. Round-tripping a document to redact it would silently +delete data that was never a secret. + +So the pass walks and mutates the `Path` in place, leaving untouched everything +it does not replace. + +## Where the code lives + +A new satellite crate, **`toolpath-redact`**, depending on `toolpath` (types) and +`toolpath-convo` (the conversation field map). + +The repo's satellite crates each do one thing to a `Path`: `toolpath-dot` renders +it, `toolpath-md` renders it, `toolpath-redact` transforms it. That is the +existing shape. + +It does not go in `toolpath` core, which everything depends on and which should +not acquire a regex engine and a vendored ruleset. It does not go in +`toolpath-convo` either, whose dependency list today is `serde`, `chrono`, +`similar`, `thiserror`. Adding detection machinery there would push it onto every +consumer of the conversation IR, including the seven provider crates that have no +use for it. + +Cost, per the checklist in `CLAUDE.md`: a new crate means updates to the +workspace `members` and `[workspace.dependencies]`, `site/_data/crates.json`, +`site/pages/crates.md`, `README.md`, the `ALL_CRATES` array and publish tier in +`scripts/release.sh`, and a crate README wired into `lib.rs`. Tier 2, alongside +`toolpath-dot` and `toolpath-md`. + +## The field map + +This is the part that makes a schema-aware redactor better than a blind string +walk, and it is why the conversation knowledge matters. The pass dispatches on +`structural.change_type` and treats each field as what it actually is. + +| Change type | Field | Treatment | +|---|---|---| +| `conversation.append` | `extra["text"]`, `extra["thinking"]` | prose; span replacement | +| | `extra["tool_uses"][].input` | provider JSON; recurse to string leaves | +| | `extra["tool_uses"][].result.content` | tool output; prose treatment | +| | `extra["delegations"][]` | recursive sub-conversation; re-enter the map | +| | `extra["environment"]` | structured; `working_dir` only | +| `file.write` | `ArtifactChange.raw` | unified diff; line-count-preserving only | +| | `extra["before"]`, `extra["after"]` | whole-file content; span replacement | +| | `extra["edits"][]` | old/new pairs; both sides | +| `conversation.event` | `extra` (whole) | opaque provider payload; blind leaf walk. **The one place a blind walk is correct.** | +| any | artifact key (map key) | file path or URI; redact URI userinfo only | +| `path.meta` | `extra["vcs_remote"]` | URI; redact userinfo only | +| `path.base` | `uri` | URI; redact userinfo only | + +Two rules fall out of this table. + +**Redact the capture group, not the match.** A connection string becomes +`postgres://svc_user:[REDACTED:db-uri-password:a3c829]@db.internal:5432/prod`, +not an opaque blob. The PEM armor lines survive, the body goes. `Bearer ` stays, +the token goes. This is what keeps a redacted document readable and resumable. + +**Diffs are line-count-preserving.** A hunk header `@@ -a,b +c,d @@` declares +line counts, so replacement inside a line is safe and anything that adds or +removes lines is not. A 25-line PEM block cannot collapse to one marker. Parse +with `diffy` rather than regexing raw patch text. + +### Why the map is wider than tool input and output + +Tool I/O is the biggest surface but not the whole one. Classifying the 120 +high-confidence hits from the local census by what kind of field held them: + +| Surface | Hits | Share | +|---|---:|---:| +| Tool results and output | 60 | 50% | +| Tool inputs | 19 | 16% | +| Plain prompt and message text | 36 | 30% | +| Diff lines | 3 | 2% | +| Base64 attachment blobs | 2 | 2% | + +Scanning only tool inputs and outputs would cover two thirds of the surface and +miss the third that is a human pasting a credential into a prompt, or an +assistant quoting one back in prose. In the raw JSONL that third lives at +`$.content`, `$.message.content`, `$.message.content[].text` and `$.lastPrompt`, +and in a derived document it lands in `extra["text"]`. It has to be in the map. + +The diff row is small in count and large in consequence: those hits are in +`old_string`/`oldString`/`originalFile`, which capture pre-edit content. Using +the Edit tool to *remove* a secret from a file writes that secret permanently +into the session log. The remediation creates the durable record, so the field +that records what was removed is exactly the field that must be redacted. + +For documents whose `meta.kind` is not `agent-coding-session`, the map degrades +to the generic rows (artifact keys, `base.uri`, `meta.extra`) plus a blind leaf +walk of `structural.extra`. Git and GitHub derived paths get correct, if less +precise, treatment for free. + +## The redactor abstraction + +The pass owns traversal, the field map, transformation, and the audit record. +**What it does not own is detection.** That is a plug point, so the same command +can drive a built-in regex engine, an external scanner, or a semantic matcher +that does not exist yet. + +The split matters because detection is where the field moves fastest and where +the trade-offs are least settled (see the base rates: best open-source precision +is 0.46). Traversal and provenance are stable; detectors are not. Pinning the +former and swapping the latter is the whole point. + +### The contract + +Detectors take **strings plus context**, never toolpath types. That is a +deliberate constraint, and it buys two things: detectors are testable without +constructing a `Path`, and the same implementations can later be driven from a +pre-tool-use hook path without touching this layer. Keeping harness-time +redaction possible is a non-goal for now, but the seam is placed so it stays +possible. + +```rust +/// What the host hands a detector: one field's full value, plus enough +/// context to judge it. +pub struct Candidate<'a> { + /// The string to examine. Never a fragment - always the whole field. + pub text: &'a str, + /// What this field holds. A diff is scanned line-wise, a URI is parsed, + /// opaque JSON is walked. The shape tells a detector which to do. + pub shape: FieldShape, + /// RFC 6901 pointer relative to the step. Passed through verbatim to the + /// audit record; the detector never has to construct one. + pub at: &'a str, + pub ctx: Context<'a>, +} + +pub enum FieldShape { + Prose, // extra["text"], extra["thinking"] + ToolInput, // tool_uses[].input + ToolOutput, // tool_uses[].result.content + UnifiedDiff, // ArtifactChange.raw + FileContent, // extra["before"], extra["after"] + Uri, // vcs_remote, base.uri, artifact keys + OpaqueJson, // conversation.event.extra +} + +pub struct Context<'a> { + pub change_type: &'a str, // "conversation.append" | "file.write" | ... + pub tool_name: Option<&'a str>, // "Bash", "Read", "Edit", ... + pub actor: &'a str, // "agent:claude-opus-5" | "human:alex" + pub kind: Option<&'a str>, // path.meta.kind +} + +/// What a detector hands back. +pub struct Finding { + /// Byte range into `Candidate.text`. Must land on char boundaries; the + /// host validates and rejects a detector that returns otherwise. + pub span: Range, + /// Stable id. Goes verbatim into the audit record, so it is part of the + /// document's contract, not an implementation detail. + pub rule: String, + /// 0.0..=1.0. The host compares against one threshold. + pub score: f32, + /// Which detector produced this. Recorded for provenance when more than + /// one is configured. + pub detector: &'static str, +} + +pub trait Detector: Send + Sync { + fn id(&self) -> &'static str; + + fn detect(&self, c: &Candidate<'_>) -> Result>; + + /// Cheap gate so the host can skip the call entirely. The built-in + /// detector implements this as an aho-corasick keyword scan. + fn prefilter(&self, _text: &str) -> bool { true } + + /// Whether this detector performs network I/O. The host **refuses to run + /// a networked detector** unless explicitly allowed, because validating a + /// candidate against its issuing provider sends secret material off the + /// machine - the exact failure this tool exists to prevent. + fn egress(&self) -> Egress { Egress::LocalOnly } +} +``` + +Composition is a set, so an internal engine and an external scanner can run +together and their findings merge through the same overlap resolution: + +```rust +pub struct DetectorSet(Vec>); +``` + +This mirrors patterns the repo already uses: `ConversationProjector` with +`AnyProjector` for type erasure, the `ArtifactSource` trait with one impl per +provider, and `cmd_resume::ExecStrategy` for injectable execution. Tests get a +`FixedDetector` that returns canned spans, the same way `RecordingExec` stands +in for a real harness. + +### Transformation is also pluggable, with a safe default + +An external tool that wants to supply the replacement text as well as find the +span implements the second trait. Most will not. + +```rust +pub trait Transformer: Send + Sync { + fn id(&self) -> &'static str; + fn replace(&self, f: &Finding, value: &str, fp: &Fingerprint) -> String; +} +``` + +Default is `MarkerTransformer`, which emits `[REDACTED::]`. The host +always computes the fingerprint and always writes the audit record, whatever the +transformer returns - those are policy, not plugin territory. + +### Bundled implementations + +| Detector | `id()` | How it runs | +|---|---|---| +| Built-in regex engine | `internal` | In-process. Vendored `gitleaks.toml`, aho-corasick prefilter, checksum validation. The default. | +| External binary | `exec:` | Subprocess, JSON in and JSON out (below). Covers gitleaks, kingfisher, and anything else with a CLI. | +| Rust crate | `keyhog` | In-process behind a feature flag. `keyhog-scanner` is the one crates.io-consumable engine. | +| Semantic | `semantic` | Not in v1. Slots in as another impl with no change to this layer. | +| Manual map | `manual` | Reads an explicit list of values or pointers. No inference. | + +The subprocess protocol keeps the boundary honest: one JSON object per +candidate on stdin, one array of findings on stdout. + +```json +{"text": "...", "shape": "tool_output", "at": "/change/...", "ctx": {"tool_name": "Bash"}} +``` +```json +[{"span": [42, 62], "rule": "aws-access-key-id", "score": 0.99}] +``` + +A detector that returns overlapping, out-of-order, or non-char-boundary spans is +normalised by the host rather than trusted, because a third-party scanner is not +part of this codebase's test suite. + +## Dry run, and the plan file + +The command is plan-then-apply, like `terraform plan` / `apply`. A dry run +produces a **redaction plan**: a reviewable, editable JSON artifact that is also +the exact input to the apply step. Nothing is redacted without one, even when it +is generated implicitly. + +### The dry run surfaces everything it looked at, not just what it found + +This is the part that matters most and the part most tools get wrong. A report +listing only findings answers "what did you find" but not "**what did you even +look at**" - and the second question is the one a user needs in order to trust +the first. A surface with zero findings is information: it says the pass reached +that field and the detectors were silent, which is different from the pass never +having visited it. + +So the plan carries two lists. `surfaces` is every field the map named, whether +or not anything fired. `findings` is what fired. + +```json +{ + "v": 1, + "document": "claude-abc123", + "generated": "2026-07-30T18:04:11Z", + "detectors": ["internal"], + "defaults": { "transform": "marker", "threshold": 0.8 }, + "surfaces": [ + { "step": "turn-0f3a", "at": "/change/claude:~1~1sess-abc/structural/extra/text", + "shape": "prose", "bytes": 412, "findings": 0 }, + { "step": "turn-0f3a", "at": "/change/claude:~1~1sess-abc/structural/extra/tool_uses/0/result/content", + "shape": "tool_output", "bytes": 4211, "findings": 2 }, + { "step": "turn-9c21", "at": "/change/src~1config.rs/raw", + "shape": "unified_diff", "bytes": 880, "findings": 1 } + ], + "findings": [ + { "id": "f01", "step": "turn-0f3a", + "at": "/change/claude:~1~1sess-abc/structural/extra/tool_uses/0/result/content", + "rule": "aws-access-key-id", "span": [1042, 1062], "score": 0.99, + "detector": "internal", "shape": "tool_output", + "context": "export AWS_ACCESS_KEY_ID= && aws s3 ls", + "action": "redact", "transform": "marker" } + ] +} +``` + +`context` elides the match by default - it shows the surrounding line with the +value replaced by its rule name, and **not** the value's length. A plan file is +a local artifact but it is exactly the kind of thing that gets pasted into a +ticket, so it does not carry material by default. `--reveal` includes the real +values for cases where you genuinely cannot decide without seeing them; it +writes `0600` and prints a warning. + +### Deciding: bulk, then individual + +Every finding carries an `action` (`redact` or `skip`) and an optional +`transform` override. Three ways to set them, and they compose in this order: + +**1. Bulk, by predicate.** Repeatable, evaluated in order, last match wins: + +``` +--accept 'rule=aws-access-key-id' # every AWS key +--accept 'score>=0.95' # everything the detector is sure about +--accept 'shape=unified_diff' # everything in a diff +--reject 'rule=generic-assignment' # the noisy heuristic, wholesale +--accept 'step=turn-0f3a' # one turn entirely +--mode-for 'rule=us-social-security-number:mask' +``` + +Predicate fields are `rule`, `shape`, `step`, `detector`, `at` (prefix match) and +`score` (with `>=`, `>`, `<=`, `<`, `=`). No expression language beyond that - a +DSL here would be a liability. + +**2. Individual, interactively.** `--interactive` opens the picker the repo +already ships (`fuzzy::pick` with `multi: true`, external `fzf` or the embedded +skim fallback), one row per finding, TAB to toggle, preview pane showing the +elided context. This is the same UX as `p import`'s session picker, so it needs +no new interaction vocabulary. + +``` + rule score step context +> aws-access-key-id 0.99 turn-0f3a export AWS_ACCESS_KEY_ID=<…> && aws s3 ls + db-uri-password 0.97 turn-9c21 DATABASE_URL=postgres://svc:<…>@db.internal + generic-assignment 0.42 turn-0f3a deploy_token = "<…>" # ops pasted this +``` + +**3. Individual, by hand.** The plan is JSON. Edit `action` and `transform`, +save, apply. This is the escape hatch that makes the other two optional, and it +is why the plan is a file rather than an interactive-only flow. + +### Apply + +``` +path p redact --input claude-abc123 --plan plan.json --output clean.json +``` + +Apply re-derives the surfaces from the input document and **verifies the plan +still matches** before touching anything: same document id, same step ids, same +spans at the recorded offsets. A plan generated against a different document, or +against one that has since changed, is refused rather than applied approximately. + +Without `--plan`, apply generates one internally from the flags and runs it, so +the one-liner still works: + +``` +path p redact --input claude-abc123 --accept 'score>=0.9' --output clean.json +``` + +## Selection modes + +What gets replaced is a separate axis from what gets found. + +```rust +pub enum Selection { + /// Run the detector set; replace at or above the threshold, report below it. + Auto { threshold: f32 }, + /// Detect nothing. Replace only what a previous pass already recorded in + /// meta.redaction as flagged. + Flagged, + /// Replace exactly what an explicit mapping names. No inference. + Manual(ManualMap), +} +``` + +`Auto` is the default. `Flagged` exists because the honest answer to a noisy +heuristic is to surface it and let a human decide: run once to get the report, +review, then run again to apply what you accepted. `Manual` covers the case the +detectors will always miss - a bare unstructured password that only the user +knows is a password - and is the escape hatch that keeps the tool useful when +detection fails. + +Future modes slot in here without touching the traversal or the record. + +## Detection in the built-in redactor + +Everything below describes the `internal` detector specifically. Another +implementation is free to work differently; the contract above is what the host +relies on. + +One scored pass, not two tiers. High-confidence formats clear the threshold +alone; a bare high-entropy value clears it only with a nearby hotword. + +1. **Keyword prefilter.** `aho-corasick` over the rules' literal keywords, run + before any regex. 221 of gitleaks' 222 rules carry keywords, which is what + makes a large ruleset affordable per string leaf. +2. **Structured rules.** Vendor `gitleaks.toml` (MIT, 222 rules). Verified: zero + lookarounds, zero backreferences, one named group, so the entire ruleset + compiles under the pure-Rust `regex` crate with no `fancy-regex` fallback. + Order alternations longest-first, because Rust `regex` is leftmost-first with + no leftmost-longest option. +3. **Entropy as a filter, never a detector.** Gate an already-structurally-matched + candidate. A 47-character random credential and the phrase + `ThisIsAReallyLongString` score within 0.03 bits of each other, so entropy + alone cannot separate them. +4. **Hotword proximity adjustment.** A secret-ish name within ~50 characters + (Macie's default `maximumMatchDistance`) raises the score. This is the + principled form of the `NAME=value` heuristic, which measured 55% suppressible + noise on the local corpus and is not safe to auto-replace on its own. +5. **Checksum validation** where the format defines one. GitHub PATs are CRC32 -> + base62 -> last 6 characters. Offline, free, and it kills the dummy-key false + positive that entropy cannot. + +Findings at or above the replace threshold are replaced. Findings below it are +**reported but not touched**, and `--include-heuristic` opts into replacing them. + +Overlap resolution, adopted from Presidio: identical spans, higher score wins; +nested spans, the container wins regardless of score; adjacent same-type spans +separated only by whitespace, merge. Tie-break on rule id so output is +reproducible. + +## Transforms + +Default is a typed marker: + +``` +[REDACTED:aws-access-key-id:a3c829] +``` + +The rule id says what kind of thing was there, which is the part with analytical +value. The fingerprint is a keyed hash, so the same secret maps to the same token +throughout the document and a reader can still see that one key recurs across +twelve steps. + +**Key derivation.** Per-document random salt by default: within-document +coreference works, cross-document correlation does not. An optional `--key-file` +gives stable fingerprints across documents for anyone who wants rotation triage +across sessions. Never a bare unkeyed hash; a hash of a low-entropy secret is a +dictionary attack away from the secret, and the EDPB pseudonymisation guidance is +explicit that the transformation must involve a secret with sufficient entropy. + +### The full set + +Five transforms ship. `marker` is the default and the recommendation; the rest +are available because the right answer is sometimes situational, and refusing to +offer them just pushes people to hand-edit documents, which is worse. + +| id | output | length | what it leaks | +|---|---|---|---| +| `marker` | `[REDACTED:aws-access-key-id:a3c829]` | no | the *type*, plus a correlation handle. Both by design. | +| `remove` | *(empty)* | no | nothing. Also destroys the surrounding structure's readability. | +| `hash` | `a3c829` | no | a correlation handle only. No type, so the reader loses the "what was here". | +| `mask` | `████████████████████` | **yes** | the exact length. | +| `partial` | `AKIA…MPLE` | yes | the provider, the format, and 8 characters of material. | + +The bottom two rows are real disclosures, not stylistic preferences. A preserved +prefix identifies the provider, which is targeting information, and pins the +format and therefore the exact brute-force search space. A length-preserving mask +publishes the length, which for a fixed-format credential narrows the space +further. They are offered, they are documented, and they are not the default. + +Transform is settable globally and **per rule**, which is what makes the choice +useful rather than a blunt instrument: + +``` +--mode marker --mode-for us-social-security-number=mask +``` + +That reflects the actual situation: a credential wants a typed marker so the +reader knows a key was rotated; a PII field often wants a mask so the shape of +the record survives. + +## Idempotence + +The marker grammar is allowlisted before any rule runs, so a second pass finds +nothing and re-running produces byte-identical output. This is what makes +re-sharing an already-redacted document safe, and it is cheap to get right up +front and painful to retrofit. + +Invariant to test: `redact(redact(d)) == redact(d)`. + +## Signatures + +`meta.signatures[]` on a step, path, or graph covers content the pass is about to +change, so redaction invalidates any signature over the redacted scope. v1 +refuses to redact a signed document unless `--drop-signatures` is passed, which +strips them and records the fact in the report. Silently leaving a broken +signature is not an option. + +## The audit record + +The pass records what it did, at the turn where it did it, without recording the +values. Toolpath's entire job is recording what happened to an artifact, so a +redaction that leaves no trace is off-format. + +**No new types.** The record rides the existing structs. Verified against the +schema: + +- `stepMeta` is `additionalProperties: true`, and `StepMeta.extra` is a + `#[serde(flatten)] HashMap`, so `Step.meta.extra["redaction"]` + is legal and needs no schema change. `Step.meta` is `Option`, so the + pass creates it where absent. +- `pathMeta` is likewise `additionalProperties: true`, so the document-level + rollup goes at `path.meta.extra["redaction"]`. +- `step` itself is `additionalProperties: false`, so the record cannot sit as a + sibling of `step`/`change`/`meta`. Inside `meta` is the only legal home. + +Per step, on the turn where the secret appeared: + +```json +{ + "step": { "id": "turn-0f3a", "actor": "agent:claude-opus-5", "...": "..." }, + "change": { "...": "..." }, + "meta": { + "redaction": { + "v": 1, + "findings": [ + { + "rule": "aws-access-key-id", + "at": "/change/claude:~1~1sess-abc/structural/tool_uses/0/result/content", + "n": 2, + "fp": "a3c829", + "op": "marker" + }, + { + "rule": "db-uri-password", + "at": "/change/src~1config.rs/raw", + "n": 1, + "fp": "7b1e04", + "op": "marker" + } + ] + } + } +} +``` + +`at` is an RFC 6901 JSON Pointer relative to the step object. Note the escaping: +artifact keys are URLs and file paths containing `/`, which becomes `~1` (and +`~` becomes `~0`), so `claude://sess-abc` addresses as `claude:~1~1sess-abc`. +That is fiddly enough to deserve a helper and a test of its own. + +`fp` is the same keyed fingerprint that appears in the marker, so a reader can +correlate: this step's finding and that step's finding are the same credential, +without either revealing it. + +The document-level rollup answers "what happened overall" without walking every +step: + +```json +{ + "path": { "id": "path-claude-code-0f3a2b71", "head": "turn-9c21" }, + "meta": { + "title": "Claude session: 0f3a2b71", + "kind": "https://toolpath.net/kinds/agent-coding-session/v1.1.0", + "redaction": { + "v": 1, + "at": "2026-07-30T18:04:11Z", + "tool": "toolpath-redact/0.1.0", + "ruleset": "gitleaks@8.28.0", + "mode": "marker", + "steps_touched": 4, + "replaced": { "aws-access-key-id": 2, "db-uri-password": 1 }, + "flagged": { "generic-assignment": 4 }, + "signatures_dropped": 0 + } + } +} +``` + +### What the record must not contain + +- **The value**, in any form. +- **Any substring of it.** A preserved prefix leaks the provider and the format, + hence the search space. +- **The original length.** Sentry's `_meta` records `len`, and that is a mistake + to copy here: for a fixed-format credential, length narrows the format space, + and combined with the rule id it is close to redundant with a prefix. The + marker's own length reveals nothing about the original. + +The record holds a *type*, a *location*, a *count*, and an opaque *handle*. That +is the honest account of what a reader is not seeing. + +### Consequences to handle + +- **The record must be allowlisted from scanning**, alongside the marker grammar. + A hex fingerprint in `meta.extra` would otherwise trip an entropy rule on the + next pass and the document would never converge. +- **Re-running merges, it does not append.** A second pass over an already + redacted document finds nothing new and leaves the existing record untouched, + which is what makes `redact(redact(d)) == redact(d)` hold at the byte level. +- **`extract_conversation` drops it**, because `Turn` has no field for step meta. + So a `path resume` of a redacted document shows the markers in the text but not + the structured record. That is acceptable: the marker carries the signal where + a reader will actually be looking. Worth revisiting if the IR ever grows an + escape hatch. +- **It changes the signed scope**, which is already covered by the + `--drop-signatures` rule above. + +## CLI + +``` +path p redact --input [--output ] + + detection --detector internal|keyhog|exec: (repeatable) + --threshold <0.0-1.0> (default 0.8) + --allow-network-detectors (off by default) + + plan --dry-run emit a plan, change nothing + --plan apply an existing plan + --reveal include real values in the plan (0600) + + decide --accept repeatable, last match wins + --reject + --interactive picker, TAB to toggle + --mode-for : + + transform --mode marker|remove|hash|mask|partial (default marker) + --key-file + + output --output --json + --drop-signatures +``` + +`--detector` is repeatable, so an internal sweep and an external scanner can run +together: + +```bash +path p redact --input claude-abc123 \ + --detector internal --detector exec:/usr/local/bin/gitleaks +``` + +The two-pass review flow that `--select flagged` exists for: + +```bash +# 1. see what a noisy heuristic thinks, change nothing +path p redact --input claude-abc123 --report-only --json > findings.json + +# 2. after review, apply only what was accepted +path p redact --input claude-abc123 --select flagged --output clean.json +``` + +`` is a cache id or a file path, matching `p render` and `p validate`. + +**In place is the point.** A cache id redacts the cached document where it sits. +There is no second file. + +The earlier design wrote a redacted copy elsewhere and left the cache untouched, +on the reasoning that the cache is a faithful archive. That was wrong, and the +reason it was wrong is the same reason `share` has no safety net: two documents +means remembering which one to send. Every downstream verb - `path resume`, +`path share`, `p export`, `p render` - resolves a cache id, so a redacted copy +sitting beside the original protects none of them. Redacting in place means once +you have redacted, everything downstream is redacted, without anyone having to +remember anything. + +What this gives up is the cache as an archive of original content. That is an +acceptable trade because **the cache was never the source of truth** - the +harness session log is, and `p cache sync` re-derives from it. The real cost is +narrower and worth stating plainly: once the source session ages out (30 days by +default, `cleanupPeriodDays`), the redaction is permanent and the original text +is gone. + +`--output` still exists and still writes elsewhere; it is how you redact a +standalone `.json` file, or keep a copy. With a file input and no `--output`, +output goes to stdout so it composes: + +```bash +path p redact --input doc.json | path p export pathbase --input - +``` + +In-place writes are temp-plus-rename at `0600`, matching `cache::write_cached`, +so an interrupted redaction never leaves a half-written document. + +## The sync collision, and how it is handled + +In-place redaction has one serious interaction, and it is not obvious. + +`sync::engine::is_unchanged` (`engine.rs:128`) gates re-derivation purely on the +source artifact's mtime and size plus the cache file existing. **It never +inspects the document.** So: + +- Redact in place, source session untouched → the next sync sees matching stamps + and skips. The redaction survives. +- Resume that session and add a single turn → the source mtime and size change → + sync re-derives **with force** → the redaction is silently destroyed, and the + newly-appended turns arrive un-redacted too. + +`path query` auto-syncs implicitly, so this can happen without the user ever +typing `sync`. A design that ignored it would quietly un-redact documents. + +**The fix: sync knows about redaction and re-applies it.** + +`SyncRecord` gains an optional field. It is additive, and every existing field +already carries `#[serde(default, skip_serializing_if)]`, so old manifests load +unchanged: + +```rust +pub(crate) struct SyncRecord { + // … path, cache_id, modified, size, synced_at … + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) redaction: Option, +} + +pub(crate) struct RedactionPolicy { + pub detectors: Vec, + pub threshold: f32, + pub mode: Transform, + pub mode_for: Vec<(String, Transform)>, + pub accept: Vec, // the predicates, verbatim + pub reject: Vec, + pub key_id: String, // which stored key produced the fingerprints +} +``` + +When sync re-derives an artifact whose record carries a policy, it re-runs +redaction with that policy before writing the cache. The document stays current +*and* stays redacted. + +Two honest limitations of replay: + +- **Only rule-based decisions replay.** A predicate re-evaluates against new + content correctly. An individually hand-picked finding cannot - its id and span + refer to content that may have moved. So anything the user individually + *skipped* comes back on re-derive. That is fail-closed, which is the right + direction, but it is surprising, so sync reports it: `re-redacted 3 documents; + 2 previously-skipped findings reappeared`. +- **The fingerprint key must persist.** A per-run random salt would give every + re-derive different markers and churn the document on every sync. The key is + stored once per document under `$TOOLPATH_CONFIG_DIR/redact-keys/` at `0600` + and referenced by `key_id`. This supersedes the "per-document random salt" + wording above: the salt is per-document and *persisted*, not per-run. + +`p cache rm` drops the key alongside the document. A record whose key is missing +fails loudly on re-redaction rather than silently producing new fingerprints. + +`--report-only` prints findings and writes no document: + +``` +$ path p redact --input claude-abc123 --report-only +7 findings in 4 steps + + replaced + aws-access-key-id 2 conversation.append.tool_uses[].result.content + db-uri-password 1 file.write.raw + flagged (not replaced, --include-heuristic to replace) + generic-assignment 4 conversation.append.text +``` + +Exit 0 when clean, 0 with findings after a successful redaction, non-zero only on +error. `--report-only` exits 1 when findings exist, so it works in a pre-share +check a user can wire up themselves. + +## Testing + +Mirrors the repo's existing style: unit tests alongside the code, integration +tests in `crates/path-cli/tests/`. + +- **Field map coverage.** One fixture document per change type, asserting the + right fields are visited and the wrong ones are not. +- **Detector contract.** A `FixedDetector` returning canned spans drives every + traversal and transform test, so those never depend on regex behaviour. Its + counterpart is a `HostileDetector` returning overlapping, reversed, + out-of-range and mid-codepoint spans, asserting the host normalises rather + than panics or corrupts. +- **Candidate shape dispatch.** Each `FieldShape` reaches the detector with the + right value and pointer, verified by a recording detector that captures every + `Candidate` it is handed. +- **Egress refusal.** A detector declaring `Egress::Network` errors without + `--allow-network-detectors`. +- **Selection modes.** `Auto` replaces above threshold and reports below it; + `Flagged` replaces exactly the previously recorded flags and detects nothing; + `Manual` replaces exactly what the map names and nothing else. +- **Non-destruction.** For a document with no secrets, `redact(d) == d` byte for + byte. This is the single most important test: the pass must not perturb + anything it is not redacting. +- **Lossiness guard.** A document carrying `extra["edits"]` and provider-specific + keys survives redaction with those keys intact. Directly guards against anyone + later "simplifying" the implementation into an extract/derive round-trip. +- **Idempotence.** `redact(redact(d)) == redact(d)` across all modes. +- **Diff integrity.** A redacted `file.write` raw diff still parses, and its hunk + headers still match its line counts. +- **Determinism.** Same input plus same key gives the same output, including + fingerprints, across runs and across HashMap iteration orders. +- **Signature refusal.** A signed document errors without `--drop-signatures`. +- **Round-trip untouched.** The existing fidelity suites still pass, confirming + the pass sits outside `derive_path` and the projectors. + +Corpus check: run against the local cache and confirm the known base rates from +the census reappear. + +## Decisions + +1. **Post-generation, not at egress.** Redaction belongs to the document, not to + one of its exits. Accepted cost: `path share` has no safety net. +2. **A new crate, not a module in `toolpath-convo`.** Keeps a regex engine and a + vendored ruleset off the seven provider crates. +3. **Mutate the `Path` in place; never round-trip through the IR.** The IR is + lossy by construction and a redactor cannot be built on a lossy transform. +4. **Typed markers with keyed fingerprints; no masking or partial redaction.** + Preserves what has analytical value and leaks nothing. +5. **Heuristic findings are reported, not replaced, by default.** 55% measured + noise makes auto-replacement worse than useless. +6. **Redact the cached document in place.** Every downstream verb resolves a + cache id, so a redacted copy beside the original protects none of them. The + cache stops being an archive of original content; the harness session log + already was one. Reversed from the first draft, which wrote a copy elsewhere. +7. **Detection is a plug point; traversal, transform and provenance are not.** + Detection is where the field moves fastest and where precision is worst. + Pinning the stable parts and swapping the volatile one is the point of the + abstraction. +8. **Detectors take strings plus context, never toolpath types.** Keeps them + testable in isolation, and keeps a future pre-tool-use path open without + rework at this layer. +9. **Networked detectors are refused unless explicitly allowed.** Validating a + candidate against its issuing provider is the failure this tool exists to + prevent, so it cannot be reachable by accident. +10. **Finding what to redact and deciding what to replace are separate axes.** + `--select flagged` makes "report, review, then apply" a first-class flow + rather than a workaround for a noisy detector. +11. **Sync re-applies redaction rather than clobbering it.** In-place redaction + without this is a silent un-redaction the moment the user resumes a session, + triggered implicitly by `path query`. The policy is persisted in the manifest + record; the fingerprint key is persisted per document so replay does not + churn markers. + +## Future, not in v1 + +- A `--policy` file for per-repo rules and allowlists. +- Redacting `Graph` documents element-wise (v1 handles the single-`Path` case + that `share` and `resume` produce). +- Reversible pseudonymization with a stored key, for teams that need to + re-identify inside a trusted boundary. +- Extending the field map as new `meta.kind` values are defined. From 34a36c05647b2ef70699fb285e5eb972e385cf98 Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 15:25:16 -0400 Subject: [PATCH 2/9] feat(redact): T0 shared vocabulary for toolpath-redact Types and trait signatures only; bodies are todo!(). Every Wave 1 track builds against this contract, so it lands first and alone. Deviations from the plan's literal snippets, all forced by `clippy -D warnings` or by a cross-task call site: - Transform derives Default instead of hand-writing the impl. - SurfaceCursor.path is pub so the unread field does not trip dead_code. - RedactError gains BadPredicate for parse_predicate's error path. - DetectorSet::detectors() so plan generation can run the egress check. - plan.rs carries todo!() stubs for every function that crosses a task boundary, so the contract is fixed before the fan-out. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 104 +++++++++++++ Cargo.toml | 2 + crates/toolpath-redact/Cargo.toml | 22 +++ crates/toolpath-redact/README.md | 53 +++++++ crates/toolpath-redact/src/apply.rs | 13 ++ crates/toolpath-redact/src/detect.rs | 101 ++++++++++++ crates/toolpath-redact/src/exec.rs | 5 + .../toolpath-redact/src/internal/entropy.rs | 1 + crates/toolpath-redact/src/internal/mod.rs | 5 + crates/toolpath-redact/src/internal/rules.rs | 1 + crates/toolpath-redact/src/lib.rs | 57 +++++++ crates/toolpath-redact/src/plan.rs | 146 ++++++++++++++++++ crates/toolpath-redact/src/surface.rs | 38 +++++ crates/toolpath-redact/src/transform.rs | 34 ++++ 14 files changed, 582 insertions(+) create mode 100644 crates/toolpath-redact/Cargo.toml create mode 100644 crates/toolpath-redact/README.md create mode 100644 crates/toolpath-redact/src/apply.rs create mode 100644 crates/toolpath-redact/src/detect.rs create mode 100644 crates/toolpath-redact/src/exec.rs create mode 100644 crates/toolpath-redact/src/internal/entropy.rs create mode 100644 crates/toolpath-redact/src/internal/mod.rs create mode 100644 crates/toolpath-redact/src/internal/rules.rs create mode 100644 crates/toolpath-redact/src/lib.rs create mode 100644 crates/toolpath-redact/src/plan.rs create mode 100644 crates/toolpath-redact/src/surface.rs create mode 100644 crates/toolpath-redact/src/transform.rs diff --git a/Cargo.lock b/Cargo.lock index 6cdb1e62..cbf59168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,6 +775,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "diffy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b545b8c50194bdd008283985ab0b31dba153cfd5b3066a92770634fbc0d7d291" +dependencies = [ + "nu-ansi-term", +] + [[package]] name = "digest" version = "0.10.7" @@ -783,6 +792,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1295,6 +1305,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "242402749acf71e6f32f5857598b7002c4058a4e3c3b22b4c7d51cab9aea754e" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.0" @@ -2224,6 +2243,15 @@ dependencies = [ "instant", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num" version = "0.4.3" @@ -3495,6 +3523,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_tokenstream" version = "0.2.3" @@ -4063,6 +4100,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toolpath" version = "0.7.0" @@ -4231,6 +4309,23 @@ dependencies = [ "toolpath-convo", ] +[[package]] +name = "toolpath-redact" +version = "0.1.0" +dependencies = [ + "aho-corasick", + "chrono", + "diffy", + "hmac", + "regex", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "toml", + "toolpath", +] + [[package]] name = "tower" version = "0.5.3" @@ -5110,6 +5205,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 23223581..9764b0e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/toolpath-dot", "crates/toolpath-md", "crates/toolpath-pi", + "crates/toolpath-redact", "crates/pathbase-client", "crates/path-cli", ] @@ -37,6 +38,7 @@ toolpath-github = { version = "0.6.0", path = "crates/toolpath-github" } toolpath-dot = { version = "0.5.0", path = "crates/toolpath-dot" } toolpath-md = { version = "0.7.0", path = "crates/toolpath-md" } toolpath-pi = { version = "0.6.1", path = "crates/toolpath-pi" } +toolpath-redact = { version = "0.1.0", path = "crates/toolpath-redact" } path-cli = { version = "0.16.0", path = "crates/path-cli" } pathbase-client = { version = "0.2.0", path = "crates/pathbase-client" } diff --git a/crates/toolpath-redact/Cargo.toml b/crates/toolpath-redact/Cargo.toml new file mode 100644 index 00000000..bb1cded8 --- /dev/null +++ b/crates/toolpath-redact/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "toolpath-redact" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository = "https://github.com/empathic/toolpath" +description = "Detect and redact credentials in Toolpath documents" +keywords = ["redaction", "secrets", "toolpath", "privacy"] +categories = ["development-tools"] + +[dependencies] +toolpath = { workspace = true } +chrono = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +regex = "1.12" +aho-corasick = "1.1" +diffy = "0.4" +hmac = "0.12" +sha2 = "0.10" +toml = "0.8" diff --git a/crates/toolpath-redact/README.md b/crates/toolpath-redact/README.md new file mode 100644 index 00000000..277399ec --- /dev/null +++ b/crates/toolpath-redact/README.md @@ -0,0 +1,53 @@ +# toolpath-redact + +Detect and redact credentials in Toolpath documents. + +The engine behind `path p redact`: it walks a `toolpath::v1::Path`, names +every string field a secret could hide in, runs a swappable set of +detectors over them, and rewrites the ones you approve. + +## The shape of a redaction + +Redaction is plan-then-apply, not a single opaque pass: + +1. `surfaces()` names every field the map reaches, whether or not anything + was found there. A surface with zero findings is information: the pass + looked and the detectors were silent. +2. `plan::generate()` runs the detectors over those surfaces and emits a + reviewable `Plan` - stable finding ids, elided context, one action per + finding. +3. `apply()` consumes the plan and rewrites the document. + +Because the plan is data, it can be decided by predicate, by picker, or by +hand-editing JSON before it is applied. + +## Purity + +This crate touches no environment variable, no filesystem, no clock, and +no process global. The fingerprint key arrives as bytes and the timestamp +arrives as a parameter, so a plan generated twice from the same document +is byte-identical, and the test suite needs neither a temp directory nor a +lock. + +## The `Detector` contract + +Detection is the part of this problem where precision is worst and the +field moves fastest, so it sits behind a trait. A `Detector` receives a +`Candidate` - a string, its `FieldShape`, its RFC 6901 pointer, and a +little context - and returns spans. It never sees the document, which is +what keeps detectors testable in isolation and leaves a harness-time hook +path open. + +`DetectorSet::detect_all` normalises whatever comes back: spans that are +reversed, out of range, or split a UTF-8 codepoint are dropped, and +overlaps resolve to one finding. + +A detector that would send candidate material off the machine reports +`Egress::Network` and the host refuses it unless explicitly allowed. + +## Vendored ruleset + +The built-in detector compiles its rules from a vendored copy of the +[gitleaks](https://github.com/gitleaks/gitleaks) configuration, used +under the MIT license. See `src/internal/gitleaks.toml` for the upstream +commit this copy was taken from. diff --git a/crates/toolpath-redact/src/apply.rs b/crates/toolpath-redact/src/apply.rs new file mode 100644 index 00000000..0a6af462 --- /dev/null +++ b/crates/toolpath-redact/src/apply.rs @@ -0,0 +1,13 @@ +//! Rewriting a document from an approved plan. + +/// Rewrite `path` in place according to `plan`. +/// +/// Nothing the plan does not name is touched: with no findings, the +/// serialised output is byte-identical to the input. +pub fn apply( + _path: &mut toolpath::v1::Path, + _plan: &crate::plan::Plan, + _cfg: &crate::RedactConfig, +) -> crate::Result { + todo!("T7") +} diff --git a/crates/toolpath-redact/src/detect.rs b/crates/toolpath-redact/src/detect.rs new file mode 100644 index 00000000..8936602a --- /dev/null +++ b/crates/toolpath-redact/src/detect.rs @@ -0,0 +1,101 @@ +//! The detection contract. +//! +//! Detectors see strings and a little context, never the document. That is +//! what makes them testable in isolation and leaves a harness-time hook +//! path open. + +use std::ops::Range; + +/// What kind of text a candidate holds. Detectors use it to decide how to +/// read the string; the plan reports it so a reviewer can filter on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FieldShape { + Prose, + ToolInput, + ToolOutput, + UnifiedDiff, + FileContent, + Uri, + OpaqueJson, +} + +/// Where in the document a candidate came from, in terms a detector can +/// act on without resolving pointers itself. +#[derive(Debug, Clone, Copy)] +pub struct Context<'a> { + pub change_type: &'a str, + pub tool_name: Option<&'a str>, + pub actor: &'a str, + pub kind: Option<&'a str>, +} + +#[derive(Debug, Clone, Copy)] +pub struct Candidate<'a> { + pub text: &'a str, + pub shape: FieldShape, + /// RFC 6901 pointer relative to the step. Passed through to the audit + /// record verbatim, so a detector never constructs one. + pub at: &'a str, + pub ctx: Context<'a>, +} + +/// One span a detector claims is a secret. `span` indexes `Candidate::text` +/// in bytes. +#[derive(Debug, Clone, PartialEq)] +pub struct Finding { + pub span: Range, + /// Lands in the audit record, so it is part of the document contract. + pub rule: String, + pub score: f32, + pub detector: &'static str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Egress { + LocalOnly, + Network, +} + +pub trait Detector: Send + Sync { + fn id(&self) -> &'static str; + + fn detect(&self, c: &Candidate<'_>) -> crate::Result>; + + /// A cheap rejection test run before `detect`. Returning `false` skips + /// this detector for this candidate entirely. + fn prefilter(&self, _text: &str) -> bool { + true + } + + /// The host refuses a `Network` detector unless explicitly allowed: + /// validating a candidate against its issuing provider sends secret + /// material off the machine. + fn egress(&self) -> Egress { + Egress::LocalOnly + } +} + +#[derive(Default)] +pub struct DetectorSet(Vec>); + +impl DetectorSet { + pub fn push(&mut self, d: Box) { + self.0.push(d); + } + + pub fn ids(&self) -> Vec<&'static str> { + self.0.iter().map(|d| d.id()).collect() + } + + /// The detectors in the set, in insertion order. + pub fn detectors(&self) -> &[Box] { + &self.0 + } + + /// Run every detector and reconcile their output into one set of + /// non-overlapping, applicable spans. + pub fn detect_all(&self, _c: &Candidate<'_>) -> crate::Result> { + todo!("T1") + } +} diff --git a/crates/toolpath-redact/src/exec.rs b/crates/toolpath-redact/src/exec.rs new file mode 100644 index 00000000..3b1cdd0b --- /dev/null +++ b/crates/toolpath-redact/src/exec.rs @@ -0,0 +1,5 @@ +//! An out-of-process detector: one JSON candidate per line on stdin, one +//! array of findings on stdout. + +/// Placeholder for the subprocess detector implemented in T8. +pub struct ExecDetector; diff --git a/crates/toolpath-redact/src/internal/entropy.rs b/crates/toolpath-redact/src/internal/entropy.rs new file mode 100644 index 00000000..a138925f --- /dev/null +++ b/crates/toolpath-redact/src/internal/entropy.rs @@ -0,0 +1 @@ +//! Shannon entropy scoring. Implemented in T3. diff --git a/crates/toolpath-redact/src/internal/mod.rs b/crates/toolpath-redact/src/internal/mod.rs new file mode 100644 index 00000000..c6a915d9 --- /dev/null +++ b/crates/toolpath-redact/src/internal/mod.rs @@ -0,0 +1,5 @@ +//! The built-in detector: a vendored gitleaks ruleset, an entropy gate, +//! and a keyword prefilter. + +pub mod entropy; +pub mod rules; diff --git a/crates/toolpath-redact/src/internal/rules.rs b/crates/toolpath-redact/src/internal/rules.rs new file mode 100644 index 00000000..bd12c88f --- /dev/null +++ b/crates/toolpath-redact/src/internal/rules.rs @@ -0,0 +1 @@ +//! Rule loading from the vendored ruleset. Implemented in T3. diff --git a/crates/toolpath-redact/src/lib.rs b/crates/toolpath-redact/src/lib.rs new file mode 100644 index 00000000..f7b7cc49 --- /dev/null +++ b/crates/toolpath-redact/src/lib.rs @@ -0,0 +1,57 @@ +#![doc = include_str!("../README.md")] + +pub mod apply; +pub mod detect; +pub mod exec; +pub mod internal; +pub mod plan; +pub mod surface; +pub mod transform; + +pub use apply::apply; +pub use detect::{Candidate, Context, Detector, DetectorSet, Egress, FieldShape, Finding}; +pub use plan::{Action, Plan, PlanFinding, RedactionPolicy}; +pub use surface::{Surface, surfaces}; +pub use transform::{Fingerprint, Transform}; + +use chrono::{DateTime, Utc}; + +#[derive(Debug, thiserror::Error)] +pub enum RedactError { + #[error("detector {0} performs network I/O; pass --allow-network-detectors to permit it")] + NetworkDetectorRefused(String), + #[error("plan does not match document: {0}")] + PlanMismatch(String), + #[error("document carries signatures over redacted content; pass --drop-signatures")] + SignedDocument, + #[error("pointer {0} does not resolve")] + BadPointer(String), + #[error("bad predicate: {0}")] + BadPredicate(String), + #[error(transparent)] + Json(#[from] serde_json::Error), +} + +pub type Result = std::result::Result; + +/// Everything the engine needs, supplied by the caller. No env, no +/// filesystem, no clock, no globals - see the purity rule in the plan. +#[derive(Debug, Clone)] +pub struct RedactConfig { + pub threshold: f32, + pub mode: Transform, + pub mode_for: Vec<(String, Transform)>, + pub key: Vec, + pub now: DateTime, + pub drop_signatures: bool, + pub reveal: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize)] +pub struct RedactReport { + pub steps_touched: usize, + pub replaced: std::collections::BTreeMap, + pub flagged: std::collections::BTreeMap, + pub signatures_dropped: usize, + pub surfaces_scanned: usize, +} diff --git a/crates/toolpath-redact/src/plan.rs b/crates/toolpath-redact/src/plan.rs new file mode 100644 index 00000000..883748b4 --- /dev/null +++ b/crates/toolpath-redact/src/plan.rs @@ -0,0 +1,146 @@ +//! The reviewable artifact between detection and rewriting. + +use crate::{detect::FieldShape, transform::Transform}; +use chrono::{DateTime, Utc}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Action { + Redact, + Skip, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct PlanFinding { + pub id: String, + pub step: String, + pub at: String, + pub rule: String, + pub span: (usize, usize), + pub score: f32, + pub detector: String, + pub shape: FieldShape, + /// Surrounding line with the match replaced by its rule name. Never + /// the value, never its length, unless `reveal` was set. + pub context: String, + pub action: Action, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transform: Option, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct PlanDefaults { + pub transform: Transform, + pub threshold: f32, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Plan { + pub v: u32, + pub document: String, + pub generated: DateTime, + pub detectors: Vec, + pub defaults: PlanDefaults, + pub surfaces: Vec, + pub findings: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Decision { + pub predicate: Predicate, + pub action: Action, + pub transform: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Predicate { + Rule(String), + Shape(FieldShape), + Step(String), + Detector(String), + AtPrefix(String), + Score(Cmp, f32), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cmp { + Ge, + Gt, + Le, + Lt, + Eq, +} + +/// Persisted in the sync manifest so a re-derive can replay redaction. +/// Rule-based only: individual finding ids cannot be replayed against +/// content that has moved. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct RedactionPolicy { + pub detectors: Vec, + pub threshold: f32, + pub mode: Transform, + #[serde(default)] + pub mode_for: Vec<(String, Transform)>, + #[serde(default)] + pub accept: Vec, + #[serde(default)] + pub reject: Vec, + pub key_id: String, +} + +// ── Plan machinery (T5) ──────────────────────────────────────────────── + +pub fn parse_predicate(_s: &str) -> crate::Result { + todo!("T5") +} + +/// Later decisions override earlier ones, so a caller can express +/// "redact everything, except this" by ordering. +pub fn apply_decisions(_plan: &mut Plan, _decisions: &[Decision]) { + todo!("T5") +} + +/// Stable, ordinal finding id (`f01`, `f02`, …). Stability is what makes a +/// regenerated plan byte-identical to its predecessor. +pub fn finding_id(_index: usize) -> String { + todo!("T5") +} + +/// The line around `span` with the match replaced by `` - never the +/// value and never anything from which its length can be read, unless +/// `reveal` was set. +pub fn elide_context( + _text: &str, + _span: std::ops::Range, + _rule: &str, + _reveal: bool, +) -> String { + todo!("T5") +} + +/// Refuse a plan that no longer describes this document, naming the first +/// divergence. +pub fn verify(_plan: &Plan, _path: &toolpath::v1::Path) -> crate::Result<()> { + todo!("T5") +} + +// ── Plan generation (T8) ─────────────────────────────────────────────── + +pub fn generate( + _path: &toolpath::v1::Path, + _detectors: &crate::detect::DetectorSet, + _cfg: &crate::RedactConfig, +) -> Plan { + todo!("T8") +} + +/// `generate`, plus the egress check: a detector that would send candidate +/// material off the machine is refused unless the caller allowed it. +pub fn generate_checked( + _path: &toolpath::v1::Path, + _detectors: &crate::detect::DetectorSet, + _cfg: &crate::RedactConfig, + _allow_network: bool, +) -> crate::Result { + todo!("T8") +} diff --git a/crates/toolpath-redact/src/surface.rs b/crates/toolpath-redact/src/surface.rs new file mode 100644 index 00000000..8e33a2ac --- /dev/null +++ b/crates/toolpath-redact/src/surface.rs @@ -0,0 +1,38 @@ +//! The field map: every string a secret could hide in, named by pointer. + +use crate::detect::FieldShape; + +/// One field the map named, whether or not anything was found in it. A +/// surface with zero findings is information: the pass reached that field +/// and the detectors were silent. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Surface { + pub step: String, + pub at: String, + pub shape: FieldShape, + pub bytes: usize, +} + +pub fn surfaces(_path: &toolpath::v1::Path) -> Vec { + todo!("T2") +} + +/// Resolves a `(step, pointer)` pair against a document for reading and +/// writing. Read and write must resolve identically. +pub struct SurfaceCursor<'a> { + pub path: &'a mut toolpath::v1::Path, +} + +impl SurfaceCursor<'_> { + pub fn read(&self, _step: &str, _at: &str) -> Option { + todo!("T2") + } + + pub fn write(&mut self, _step: &str, _at: &str, _value: &str) -> crate::Result<()> { + todo!("T2") + } +} + +pub fn ptr_escape(_token: &str) -> String { + todo!("T2") +} diff --git a/crates/toolpath-redact/src/transform.rs b/crates/toolpath-redact/src/transform.rs new file mode 100644 index 00000000..cead61f5 --- /dev/null +++ b/crates/toolpath-redact/src/transform.rs @@ -0,0 +1,34 @@ +//! What a redacted span is replaced with. + +/// How a detected span is rewritten. Every variant except `Partial` is +/// guaranteed to emit nothing derived from the value's characters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Transform { + #[default] + Marker, + Remove, + Hash, + /// Length-preserving, and therefore publishes the exact length. + Mask, + /// Keeps 4 leading and 4 trailing chars: leaks provider and format. + Partial, +} + +/// A short stable handle for a secret value, used to correlate the same +/// secret across occurrences without publishing it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fingerprint(pub String); + +impl Fingerprint { + /// Keyed, never a bare hash: a hash of a low-entropy secret is a + /// dictionary attack away from the secret (EDPB 01/2025 para 88). + pub fn new(_key: &[u8], _value: &str) -> Self { + todo!("T4") + } +} + +pub trait Transformer: Send + Sync { + fn id(&self) -> &'static str; + fn replace(&self, rule: &str, value: &str, fp: &Fingerprint) -> String; +} From 1e456df495ae6853e7072179cfc2f08c4957c6ee Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 15:32:34 -0400 Subject: [PATCH 3/9] feat(redact): T1 span normalisation, T4 transforms, T6 CLI args, T12 docs T1 replaced the plan's greedy pairwise overlap eviction: on a three-way overlap (A-B overlap, B-C overlap, A-C disjoint) it dropped A entirely, leaving a real secret unredacted. Best-first interval selection keeps the same stated policy without eviction. Scores compare with total_cmp and NaN scores are dropped, because a NaN sorts above +inf and would win every overlap before the threshold silently discarded it. T4 splices edits right-to-left in one pass, validating each span against the string as spliced so far so a normalisation bug upstream degrades instead of panicking. Applies the T0 review: Fingerprint gains Hash/Ord for the audit record's (rule, fp) aggregation, the unimplemented Transformer trait is gone rather than shipping as semver debt at 0.1.0, and Transform's doc no longer claims a leak boundary that Mask contradicts. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 35 + CLAUDE.md | 19 +- Cargo.lock | 3 +- Cargo.toml | 2 +- README.md | 1 + crates/path-cli/Cargo.toml | 3 +- crates/path-cli/src/cmd_p.rs | 6 + crates/path-cli/src/cmd_redact.rs | 224 ++ crates/path-cli/src/lib.rs | 1 + crates/toolpath-redact/README.md | 7 +- crates/toolpath-redact/src/detect.rs | 347 +- crates/toolpath-redact/src/exec.rs | 7 +- .../toolpath-redact/src/internal/entropy.rs | 42 +- .../src/internal/gitleaks.toml | 3209 +++++++++++++++++ crates/toolpath-redact/src/internal/rules.rs | 129 +- crates/toolpath-redact/src/lib.rs | 12 +- crates/toolpath-redact/src/transform.rs | 291 +- scripts/release.sh | 3 +- site/_data/crates.json | 14 +- site/pages/crates.md | 1 + 20 files changed, 4328 insertions(+), 28 deletions(-) create mode 100644 crates/path-cli/src/cmd_redact.rs create mode 100644 crates/toolpath-redact/src/internal/gitleaks.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 25f6bc04..634793d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ All notable changes to the Toolpath workspace are documented here. +## `path p redact` — plan-then-apply credential redaction — 2026-07-30 + +Adds `path p redact`, a plumbing command that removes credentials from an +already-generated toolpath document in place, via a reviewable plan-then-apply +flow with detection behind a swappable trait. + +- **`toolpath-redact`** (0.1.0): + - Pure function engine with no env vars, no filesystem, no clock, no + process globals. Takes a document, plan, and config; returns redacted + document and audit report. + - Plan-then-apply workflow: `surfaces()` enumerates every string field a + credential could hide in; `plan::generate()` runs detectors and produces a + reviewable `Plan` with stable ids and elided context; `apply()` rewrites + the document. Plans can be decided by predicate, picker, or hand-edited + JSON. + - Detector trait (`Detector`) lets implementations be pluggable: built-in + rule-based detector via vendored gitleaks ruleset, `FixedDetector` for + tests, future hook paths for harness-time redaction. + - Five transform choices: `marker` (default, survives diffs), `remove`, `hash`, + `mask` (length-preserving), `partial` (4 leading/trailing chars, + length-leaking). Global and per-rule overrides. + - Field map surfaces `Prose`, `ToolInput`, `ToolOutput`, `UnifiedDiff`, + `FileContent`, `Uri`, `OpaqueJson`. RFC 6901 pointers to reach any value. +- **`path-cli`** (0.17.0): + - `path p redact --input [--dry-run|--plan ] [--accept|--reject ]…` + redacts cache ids or file paths. Dry run lists every surface (including + zero-finding ones) and exits 1 if findings exist. `--plan` lets you + review before applying. Predicates on `rule`, `shape`, `step`, `detector`, + `at`, `score`. Key storage in `~/.toolpath/redact-keys/` (0600). + - Sync integration: `path p cache sync` replays stored `RedactionPolicy` on + re-derive, so resuming a redacted session and syncing applies redaction + to new turns too. Missing key is a hard error, never a silent new key. + - Validation always runs; output validates against base schema and + `agent-coding-session` kind schema v1.1.0. + ## `path p cache sync` — incremental session ingestion — 2026-07-16 Adds `path p cache sync [types…]`, the first step toward a cache that diff --git a/CLAUDE.md b/CLAUDE.md index 2de9f950..c74de81a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,7 @@ crates/ toolpath-opencode/ # derive from opencode SQLite databases toolpath-cursor/ # derive from Cursor (IDE) state.vscdb bubble store toolpath-pi/ # derive from Pi (pi.dev) agent session logs + toolpath-redact/ # detect and redact credentials in Toolpath documents toolpath-dot/ # Graphviz DOT rendering toolpath-md/ # Markdown rendering for LLM consumption path-cli/ # unified CLI (binary: path) @@ -49,6 +50,7 @@ path-cli (binary: path) ├── toolpath-opencode → toolpath, toolpath-convo ├── toolpath-cursor → toolpath, toolpath-convo ├── toolpath-pi → toolpath, toolpath-convo + ├── toolpath-redact → toolpath (credential detection and redaction engine) ├── toolpath-dot → toolpath └── toolpath-md → toolpath @@ -93,7 +95,7 @@ The top-level surface is the porcelain (`show`, `share`, `resume`, `query`, `kind`, `auth`, `haiku`). Lower-level building blocks live under `path p …` (plumbing): `p list`, `p import`, `p export`, `p cache`, `p render`, `p merge`, `p validate`, `p derive`, `p project`, -`p incept`, `p track`, `p query` (graph traversal: `ancestors`). +`p incept`, `p track`, `p query` (graph traversal: `ancestors`), `p redact`. ```bash # Plumbing: import from external formats into the local toolpath cache @@ -132,6 +134,13 @@ cargo run -p path-cli -- p cache rm cargo run -p path-cli -- p cache sync # ingest new/changed sessions from every harness cargo run -p path-cli -- p cache sync claude codex # only these artifact types +# Plumbing: redact credentials +cargo run -p path-cli -- p redact --input doc.json --dry-run +cargo run -p path-cli -- p redact --input doc.json --plan plan.json # generate and review +cargo run -p path-cli -- p redact --input doc.json --plan plan.json --reveal # show values +cargo run -p path-cli -- p redact --input doc.json --plan plan.json --threshold 0.95 +cargo run -p path-cli -- p redact --input claude-abc123 --accept "rule=aws-access-key-id" --reject "score<0.8" + # Inspect / analyze cargo run -p path-cli -- p render dot --input doc.json cargo run -p path-cli -- p render md --input doc.json --detail full @@ -209,6 +218,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-opencode`: 52 unit + 19 integration + 1 doc test (SQLite reader, JSON payload serde, provider assembly, snapshot-based derive, tool-input fallback for gitignored paths, reasoning breakdown) - `toolpath-cursor`: 78 unit + 8 integration round-trip + 1 real-DB sanity + 1 doc test (state.vscdb SQLite reader, bubble store + composer header parsing, content-addressed blob lookup, projector with full TOOL_TABLE coverage, JSONL transcript ingest in `examples/dump_fixture.rs`) - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) +- `toolpath-redact`: *provisional count - tests still being written* (detection, spanning, surfaces, plan generation, application, audit records, transforms, CLI dispatch, sync integration) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) - `path-cli`: 353 unit + 119 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, `path resume` orchestration with injectable `ExecStrategy`, `path query`/`path kind` jaq filters + kind-selector matching + step wrapping over a `$TOOLPATH_CONFIG_DIR` cache sandbox, streaming-planner recognition + streamed-output-equals-slurp equality checks, `p cache sync` incremental ingestion — stat fingerprints, refresh-overwrite, per-type filtering, failure tallying — over resolver-injected provider fixtures, and import/share manifest recording end-to-end). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) @@ -224,6 +234,10 @@ Validate example documents: `for f in examples/*.json; do cargo run -p path-cli The Tauri 2 desktop GUI lives in the private [pathbase](https://github.com/empathic/pathbase) repo as `pathbase-app`. It consumes `toolpath`, `toolpath-claude`, `toolpath-git`, `toolpath-github`, `toolpath-gemini`, `toolpath-codex`, `toolpath-opencode`, and `toolpath-pi` via git/crates.io deps. Don't look for it in this workspace — it was moved out when Pathbase went closed-source. +## Superpowers + +`docs/superpowers/` holds implementation specifications and task plans for substantial features — think design docs plus a numbered task decomposition for parallel implementation. `specs/` subdirectory carries the closed design; `plans/` carries the task breakdown with test-first structure. These are code artifacts, not documentation: they specify invariants, gate criteria, and concrete assertions. + ## Versioning and release checklist When changing a crate's public API (new types, new trait impls, new public methods, new dependencies), bump its version. For pre-1.0 library crates, cargo treats the **z** position of `0.y.z` as the compatible slot: bug fixes *and additive changes* bump patch (`0.6.0` → `0.6.1`, so `^0.6` consumers like pathbase-app pick them up for free); bump minor only for potentially-breaking changes. `path-cli` is the app, not a library — it bumps minor per feature. @@ -247,7 +261,7 @@ When changing a crate's public API (new types, new trait impls, new public metho **Release script** (`scripts/release.sh`) publishes in dependency order: - Tier 1: `toolpath` (no workspace deps) -- Tier 2: `toolpath-convo` (depends on `toolpath`); then `toolpath-git`, `toolpath-github`, `toolpath-dot`, `toolpath-md`, `toolpath-claude`, `toolpath-gemini`, `toolpath-codex`, `toolpath-opencode`, `toolpath-pi` +- Tier 2: `toolpath-convo` (depends on `toolpath`); then `toolpath-git`, `toolpath-github`, `toolpath-dot`, `toolpath-md`, `toolpath-claude`, `toolpath-gemini`, `toolpath-codex`, `toolpath-opencode`, `toolpath-pi`, `toolpath-redact` - Tier 3: `path-cli` (depends on everything above) - Tier 4: `toolpath-cli` (deprecated shim that depends on `path-cli`; ships only the `path` binary) @@ -281,4 +295,5 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. - `path query` does not load the whole cache into memory when it can avoid it. `crates/path-cli/src/query/plan.rs` parses the jaq filter into jaq's own AST (`jaq_core::load::parse::Term`) and classifies it into a `Plan`: `PerFileStream` (`.[] | g` element-wise work — run per document, print as you go), `Decompose { reduce }` (algebraic aggregations — run the whole filter per file, concatenate the per-file outputs, then run a derived combine: `map`→`add` (array concat), top-N `sort_by(k)|.[:N]`→`add | sort_by(k)|.[:N]`, `length`→`add` over exact integer counts), or `Slurp` (the always-correct whole-array fallback). Recognition is conservative — a non-distributive prefix like `unique`/`group_by` slurps, and so do scalar `add` (float sums re-associate across per-file partials), `min`/`max` (`[] | min == null` poisons the merge), and any unrecognized tail — so **the planner never changes an answer** — `crates/path-cli/src/query/filter.rs` tests assert streamed output equals slurp byte-for-byte. `filter::execute` compiles the filter once (jaq's compiled `Filter` is fully owned, so it's reused across files) and drives the plan; `mod.rs::stream_files` yields one document's wrapped steps at a time. `TOOLPATH_QUERY_EXPLAIN=1` prints the chosen plan to stderr. No user-facing flag — it's automatic. Tie-break caveat: a streamed top-N matches slurp's *ranking*, but boundary ties may resolve to different specific rows. - Cache sync: `path p cache sync [types…]` (`crates/path-cli/src/artifact.rs`: `ArtifactType` + `ArtifactRef` + the stamp helpers; `sync/engine.rs`: manifest + ingestion loop, no UI — it reports through a `SyncObserver` trait, `&mut ()` for a silent sync; `sync/sources.rs`: an `ArtifactSource` trait — enumerate / stamp / derive — with one impl per provider, so the engine never matches on artifact type; `cmd_cache.rs`: the stderr progress line + summary) incrementally ingests artifacts into the cache — no args syncs every artifact type. Change detection is **stat-level**: each artifact is enumerated as an `ArtifactRef` whose fingerprint is the source file's mtime + size (claude: the *whole session chain* — max segment mtime + summed segment sizes via `claude_chain_stamp`, because Claude Code rotates to a new file on continuation while the chain keeps its oldest segment's id, so appends land in the newest file, not the head; the chain comes from the same cached index `list_conversations` builds; codex: rollout file, id from the stem's trailing UUID; pi: session file, id from a one-line header peek; copilot: `session-state//events.jsonl`, pure read-dir + stat) or the DB row's updated-at (opencode: header-only `SELECT time_updated`; cursor: composer headers' `lastUpdatedAt`, bubble-less drafts skipped, workspace-less composers *included* unlike `share`). Gemini enumerates via `PathResolver::list_session_entries` (`toolpath-gemini` 0.6.1), whose identity peek is bounded to the first 4 KiB of a main file. Deciding "nothing changed" reads no session bodies — a no-op sync is milliseconds. Changed/new artifacts derive through the same provider managers (each source calls the `derive_*_session_with` helpers in `derive.rs`). Manifest at `~/.toolpath/manifest.json`: artifact type → artifact id → `{path?, cache_id, modified?, size?, synced_at}`; atomic temp+rename writes, `0600`, checkpointed every 10 writes (interruption-safe: a killed run keeps nearly everything it derived, and derives run newest-first so partial progress covers the sessions that matter most); writers serialize on an advisory lock (`manifest.json.lock`) and every write is a locked read-merge-save — checkpoints merge only the records the run wrote — so concurrent invocations (query auto-syncs, imports) union their records instead of clobbering each other. Pending work reports progress on stderr (`\r`-updating ` done/total` on a TTY, a plain line every 25 items otherwise; no-op syncs stay silent). Sync always writes the cache with force — refresh semantics — and never deletes: artifacts removed upstream keep their cache docs and manifest records (archive, not mirror). Derivation failures warn and tally, they don't abort. A record's `cache_id` is *optional*: a record without one is "known, not materialized" — created when `p cache rm` evicts a doc (rm downgrades the record; the next sync re-materializes it, and sync also verifies the doc file actually exists before skipping, so even out-of-band deletions self-heal). Claude derives leave `DeriveConfig.project_path` unset so `path.base` comes from the session's own recorded cwd rather than the lossy slug. `path query` runs this sync implicitly before reading, scoped to its flags (`--source X` → that type; `--id`s → their prefixes; bare query → all types; `--input`-only → none), quiet unless something was ingested, degrading to the cache as-is if sync fails; `--no-sync` opts out. `p import` and `share` record what they write: every session derive carries a provenance `ArtifactRef` (stamped *before* the source is read, in `DerivedDoc.provenance`), and the cache-write sites call `sync::record_artifact` so the next sync sees those artifacts as unchanged instead of re-deriving them. Every import flow — explicit `--session`, picker multi-select, `--all`, and the most-recent fallbacks — loops the per-session helpers, so every session write is recorded; there is no bulk `derive_project` path in the CLI anymore, and `p import pi --all` now emits one Path per session like every other provider (it used to emit a single combined Graph). `--no-cache` paths record nothing: the manifest describes the cache. +- Redaction workflow (`toolpath-redact` + `path p redact`): plan-then-apply, not a single opaque pass. `surfaces()` enumerates every string field a credential could hide in. `plan::generate()` runs detectors over those surfaces, producing a reviewable `Plan` with stable ids and elided context. `apply()` consumes the plan and rewrites the document. The plan can be decided by predicate, by picker, or by hand-editing JSON. In-place redaction (`--input `) rewrites the cache entry; re-deriving from the source (via `path query` or `path resume`) replays the stored `RedactionPolicy` automatically, so new turns in the session get redacted too. A detector is a trait (`Detector`), not a function, so its implementation is swappable — a `FixedDetector` for tests, the built-in rule-based detector by default, and a harness-time hook for pre-training redaction. Detection is the part of this problem where precision is worst and the field moves fastest, so it sits behind the plug point; traversal (fields, pointers, surfaces) is stable and reused by all detectors. - `ArtifactType` (`crates/path-cli/src/artifact.rs`) is the general enum naming artifact sources — the seven agent harnesses (incl. copilot) plus `Git` (8 variants). Git artifacts are *recorded* in the manifest by `p import git` (id `-`, `path` = the repo directory) but never *discovered* — there is no machine-wide registry of repos — so sync reports them and leaves them alone. Github and pathbase are deliberately not artifact types: they are remote services, not local artifact sources, and their imports stay out of the manifest. It derives `clap::ValueEnum` and is used by `p cache sync` types, the sync manifest keys, `ArtifactRow.artifact_type`, and `cmd_import`'s cache-id prefixes (`name()` is both the manifest key and the `make_id` source string). The deliberately parallel `Harness` enum (`crates/path-cli/src/harness.rs`, alongside `HarnessBundle`) names the seven agent *runtimes* — things sessions can be shared from and resumed into — and is what `share`/`resume` `--harness` take, so future non-harness artifact types stay unrepresentable there (you can't resume into a git repo). `Harness::artifact_type()` maps into the general enum; `ArtifactType::harness()` is the partial inverse. Keep new code on `ArtifactType` unless it's genuinely harness-only. diff --git a/Cargo.lock b/Cargo.lock index cbf59168..581794f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2469,7 +2469,7 @@ dependencies = [ [[package]] name = "path-cli" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "assert_cmd", @@ -2508,6 +2508,7 @@ dependencies = [ "toolpath-md", "toolpath-opencode", "toolpath-pi", + "toolpath-redact", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 9764b0e9..05de39e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ toolpath-dot = { version = "0.5.0", path = "crates/toolpath-dot" } toolpath-md = { version = "0.7.0", path = "crates/toolpath-md" } toolpath-pi = { version = "0.6.1", path = "crates/toolpath-pi" } toolpath-redact = { version = "0.1.0", path = "crates/toolpath-redact" } -path-cli = { version = "0.16.0", path = "crates/path-cli" } +path-cli = { version = "0.17.0", path = "crates/path-cli" } pathbase-client = { version = "0.2.0", path = "crates/pathbase-client" } reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "rustls"] } diff --git a/README.md b/README.md index f587db82..3f48f458 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ crates/ toolpath-opencode/ Derive from opencode SQLite databases toolpath-cursor/ Derive from Cursor (IDE) state.vscdb bubble store toolpath-pi/ Derive from Pi (pi.dev) agent sessions + toolpath-redact/ Detect and redact credentials in Toolpath documents toolpath-dot/ Graphviz DOT visualization toolpath-md/ Markdown rendering for LLM consumption pathbase-client/ Progenitor-derived typed client for the Pathbase HTTP API diff --git a/crates/path-cli/Cargo.toml b/crates/path-cli/Cargo.toml index 575c972f..e87495f8 100644 --- a/crates/path-cli/Cargo.toml +++ b/crates/path-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "path-cli" -version = "0.16.0" +version = "0.17.0" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" @@ -26,6 +26,7 @@ toolpath = { workspace = true } toolpath-git = { workspace = true } toolpath-dot = { workspace = true } toolpath-md = { workspace = true } +toolpath-redact = { workspace = true } clap = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } diff --git a/crates/path-cli/src/cmd_p.rs b/crates/path-cli/src/cmd_p.rs index 3e079da6..c37b4c2c 100644 --- a/crates/path-cli/src/cmd_p.rs +++ b/crates/path-cli/src/cmd_p.rs @@ -91,6 +91,11 @@ pub enum PCommand { #[command(subcommand)] op: crate::cmd_p_query::PQueryOp, }, + /// Remove credentials from a Toolpath document via a reviewable plan-then-apply flow + Redact { + #[command(flatten)] + args: crate::cmd_redact::RedactArgs, + }, } pub fn run(command: PCommand, pretty: bool) -> Result<()> { @@ -111,5 +116,6 @@ pub fn run(command: PCommand, pretty: bool) -> Result<()> { PCommand::Incept { target } => crate::cmd_incept::run(target), PCommand::Track { op } => crate::cmd_track::run(op, pretty), PCommand::Query { op } => crate::cmd_p_query::run(op, pretty), + PCommand::Redact { args } => crate::cmd_redact::run(args), } } diff --git a/crates/path-cli/src/cmd_redact.rs b/crates/path-cli/src/cmd_redact.rs new file mode 100644 index 00000000..5a575778 --- /dev/null +++ b/crates/path-cli/src/cmd_redact.rs @@ -0,0 +1,224 @@ +//! `path p redact` — remove credentials from a toolpath document in place +//! via a reviewable plan-then-apply flow. + +use anyhow::Result; +use clap::Args; +use std::path::PathBuf; + +#[derive(Debug, Args)] +pub(crate) struct RedactArgs { + /// Cache id or file path. + #[arg(short, long)] + pub input: String, + + /// Write elsewhere instead of in place. + #[arg(short, long)] + pub output: Option, + + #[arg(long, conflicts_with = "plan")] + pub dry_run: bool, + #[arg(long)] + pub plan: Option, + /// Include real values in the plan (written 0600). + #[arg(long)] + pub reveal: bool, + + #[arg(long, value_name = "PREDICATE")] + pub accept: Vec, + #[arg(long, value_name = "PREDICATE")] + pub reject: Vec, + #[arg(long)] + pub interactive: bool, + #[arg(long, value_name = "PREDICATE:TRANSFORM")] + pub mode_for: Vec, + + #[arg(long, default_values = &["internal"])] + pub detector: Vec, + #[arg(long, default_value_t = 0.8)] + pub threshold: f32, + #[arg(long)] + pub allow_network_detectors: bool, + + #[arg(long, value_enum, default_value_t = TransformArg::Marker)] + pub mode: TransformArg, + #[arg(long)] + pub key_file: Option, + + #[arg(long)] + pub json: bool, + #[arg(long)] + pub drop_signatures: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub(crate) enum TransformArg { + Marker, + Remove, + Hash, + Mask, + Partial, +} + +pub(crate) fn run(args: RedactArgs) -> Result<()> { + todo!("T9") +} + +pub(crate) trait PickerStrategy { + fn pick(&self, rows: &[String]) -> Result>; +} + +pub(crate) struct RealPicker; + +impl PickerStrategy for RealPicker { + fn pick(&self, rows: &[String]) -> Result> { + todo!("T9") + } +} + +pub(crate) struct RecordingPicker { + pub selection: Vec, + pub seen: std::cell::RefCell>, +} + +impl PickerStrategy for RecordingPicker { + fn pick(&self, rows: &[String]) -> Result> { + *self.seen.borrow_mut() = rows.to_vec(); + Ok(self.selection.clone()) + } +} + +/// Helper to parse a "PREDICATE:TRANSFORM" string. +/// Splits on the LAST `:` so predicates containing `:` still parse. +pub(crate) fn parse_mode_for(s: &str) -> Result<(String, TransformArg)> { + let (pred, transform_str) = s + .rsplit_once(':') + .ok_or_else(|| anyhow::anyhow!("--mode-for format is PREDICATE:TRANSFORM, got: {}", s))?; + + let transform = match transform_str { + "marker" => TransformArg::Marker, + "remove" => TransformArg::Remove, + "hash" => TransformArg::Hash, + "mask" => TransformArg::Mask, + "partial" => TransformArg::Partial, + other => anyhow::bail!("unknown transform: {}", other), + }; + + Ok((pred.to_string(), transform)) +} + +impl From for toolpath_redact::Transform { + fn from(arg: TransformArg) -> Self { + match arg { + TransformArg::Marker => toolpath_redact::Transform::Marker, + TransformArg::Remove => toolpath_redact::Transform::Remove, + TransformArg::Hash => toolpath_redact::Transform::Hash, + TransformArg::Mask => toolpath_redact::Transform::Mask, + TransformArg::Partial => toolpath_redact::Transform::Partial, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn try_parse(args: &[&str]) -> Result { + use clap::Parser; + #[derive(Parser)] + struct TestCli { + #[command(subcommand)] + p: PCommand, + } + #[derive(clap::Subcommand)] + enum PCommand { + Redact { + #[command(flatten)] + args: RedactArgs, + }, + } + + let mut argv = vec!["test"]; + argv.extend(args); + let cli = TestCli::try_parse_from(argv)?; + match cli.p { + PCommand::Redact { args } => Ok(args), + } + } + + #[test] + fn dry_run_conflicts_with_plan() { + assert!(try_parse(&["redact", "-i", "x", "--dry-run", "--plan", "p.json"]).is_err()); + } + + #[test] + fn mode_for_rejects_unknown_transform() { + assert!(parse_mode_for("rule=x:invented").is_err()); + } + + #[test] + fn detector_flag_is_repeatable() { + let a = try_parse(&[ + "redact", + "-i", + "x", + "--detector", + "internal", + "--detector", + "exec:/bin/s", + ]) + .unwrap(); + assert_eq!(a.detector.len(), 2); + } + + #[test] + fn threshold_rejects_non_numeric() { + assert!(try_parse(&["redact", "-i", "x", "--threshold", "abc"]).is_err()); + } + + #[test] + fn mode_rejects_unknown_transform() { + assert!(try_parse(&["redact", "-i", "x", "--mode", "unknown"]).is_err()); + } + + #[test] + fn input_is_required() { + assert!(try_parse(&["redact"]).is_err()); + } + + #[test] + fn parse_mode_for_without_colon() { + assert!(parse_mode_for("rule=x").is_err()); + } + + #[test] + fn parse_mode_for_with_colon_in_predicate() { + let (pred, transform) = parse_mode_for("at=/change/claude:~1~1sess:marker").unwrap(); + assert_eq!(pred, "at=/change/claude:~1~1sess"); + assert_eq!(transform, TransformArg::Marker); + } + + #[test] + fn all_transform_args_convert_to_distinct_transforms() { + let transforms: Vec<_> = vec![ + TransformArg::Marker, + TransformArg::Remove, + TransformArg::Hash, + TransformArg::Mask, + TransformArg::Partial, + ] + .into_iter() + .map(|t| { + let converted: toolpath_redact::Transform = t.into(); + converted + }) + .collect(); + for i in 0..transforms.len() { + for j in (i + 1)..transforms.len() { + assert_ne!( + transforms[i], transforms[j], + "transform variants must map to distinct Transform values" + ); + } + } + } +} diff --git a/crates/path-cli/src/lib.rs b/crates/path-cli/src/lib.rs index 14ed9bba..82c5a00c 100644 --- a/crates/path-cli/src/lib.rs +++ b/crates/path-cli/src/lib.rs @@ -17,6 +17,7 @@ mod cmd_p_query; mod cmd_pathbase; mod cmd_project; mod cmd_query; +mod cmd_redact; mod cmd_render; #[cfg(not(target_os = "emscripten"))] pub mod cmd_resume; diff --git a/crates/toolpath-redact/README.md b/crates/toolpath-redact/README.md index 277399ec..49c10500 100644 --- a/crates/toolpath-redact/README.md +++ b/crates/toolpath-redact/README.md @@ -49,5 +49,8 @@ A detector that would send candidate material off the machine reports The built-in detector compiles its rules from a vendored copy of the [gitleaks](https://github.com/gitleaks/gitleaks) configuration, used -under the MIT license. See `src/internal/gitleaks.toml` for the upstream -commit this copy was taken from. +under the MIT license. + + + +See `src/internal/gitleaks.toml` for attribution and version details. diff --git a/crates/toolpath-redact/src/detect.rs b/crates/toolpath-redact/src/detect.rs index 8936602a..b1e6175a 100644 --- a/crates/toolpath-redact/src/detect.rs +++ b/crates/toolpath-redact/src/detect.rs @@ -95,7 +95,350 @@ impl DetectorSet { /// Run every detector and reconcile their output into one set of /// non-overlapping, applicable spans. - pub fn detect_all(&self, _c: &Candidate<'_>) -> crate::Result> { - todo!("T1") + pub fn detect_all(&self, c: &Candidate<'_>) -> crate::Result> { + let mut raw = Vec::new(); + for d in &self.0 { + if !d.prefilter(c.text) { + continue; + } + raw.extend(d.detect(c)?); + } + Ok(normalise(c.text, raw)) + } +} + +/// Drop what cannot be applied, then resolve overlaps. +/// +/// Policy from Presidio: identical spans, higher score wins; nested, the +/// container wins regardless of score. Every comparison ends in a total +/// order over rule and detector id, so the result cannot depend on the order +/// detectors were registered in or on any iteration order upstream. +fn normalise(text: &str, mut findings: Vec) -> Vec { + findings.retain(|f| { + // A NaN score is not orderable, and a single non-comparable element + // makes the whole sort's outcome depend on input order. + !f.score.is_nan() + && f.span.start < f.span.end + && f.span.end <= text.len() + && text.is_char_boundary(f.span.start) + && text.is_char_boundary(f.span.end) + }); + + findings.sort_by(|a, b| { + let (a_len, b_len) = (a.span.end - a.span.start, b.span.end - b.span.start); + b_len + .cmp(&a_len) + .then(b.score.total_cmp(&a.score)) + .then(a.span.start.cmp(&b.span.start)) + .then(a.rule.cmp(&b.rule)) + .then(a.detector.cmp(b.detector)) + }); + + // Best-first, then keep whatever no winner has already claimed. Resolving + // clashes pairwise instead would lose a finding whose only conflict was + // itself evicted later: with A over B, B over C and A disjoint from C, C + // displaces B and A never comes back. + let mut out: Vec = Vec::new(); + for f in findings { + let claimed = out + .iter() + .any(|o| f.span.start < o.span.end && f.span.end > o.span.start); + if !claimed { + out.push(f); + } + } + + out.sort_by_key(|f| f.span.start); + out +} + +/// Canned findings, for tests exercising traversal or transform without +/// depending on regex behaviour. +pub struct FixedDetector(pub Vec); + +impl Detector for FixedDetector { + fn id(&self) -> &'static str { + "fixed" + } + + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(self.0.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct HostileDetector(Vec); + impl Detector for HostileDetector { + fn id(&self) -> &'static str { + "hostile" + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(self.0.clone()) + } + } + + fn cand(text: &str) -> Candidate<'_> { + Candidate { + text, + shape: FieldShape::Prose, + at: "/change/x/structural/extra/text", + ctx: Context { + change_type: "conversation.append", + tool_name: None, + actor: "human:t", + kind: None, + }, + } + } + + fn f(span: Range, rule: &str, score: f32) -> Finding { + Finding { + span, + rule: rule.into(), + score, + detector: "hostile", + } + } + + #[test] + fn drops_out_of_range_spans() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..999, "x", 0.9)]))); + assert!(s.detect_all(&cand("short")).unwrap().is_empty()); + } + + // A reversed range is exactly the hostile input under test, so clippy's + // "this range is empty" lint is the wrong call here. + #[allow(clippy::reversed_empty_ranges)] + #[test] + fn drops_reversed_spans() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![Finding { + span: 5..2, + rule: "x".into(), + score: 0.9, + detector: "hostile", + }]))); + assert!(s.detect_all(&cand("abcdefgh")).unwrap().is_empty()); + } + + #[test] + fn drops_mid_codepoint_spans() { + // "é" is two bytes; 0..1 splits it. + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..1, "x", 0.9)]))); + assert!(s.detect_all(&cand("é-tail")).unwrap().is_empty()); + } + + #[test] + fn identical_spans_higher_score_wins() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![ + f(0..4, "low", 0.4), + f(0..4, "high", 0.9), + ]))); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].rule, "high"); + } + + #[test] + fn nested_span_container_wins_regardless_of_score() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![ + f(2..4, "inner", 0.99), + f(0..8, "outer", 0.20), + ]))); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].rule, "outer"); + } + + #[test] + fn output_is_sorted_and_deterministic() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![ + f(6..8, "b", 0.9), + f(0..2, "a", 0.9), + f(3..5, "c", 0.9), + ]))); + let a = s.detect_all(&cand("abcdefgh")).unwrap(); + let b = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(a, b); + assert!(a.windows(2).all(|w| w[0].span.start <= w[1].span.start)); + } + + #[test] + fn prefilter_short_circuits_detect() { + struct NeverCalled; + impl Detector for NeverCalled { + fn id(&self) -> &'static str { + "never" + } + fn prefilter(&self, _t: &str) -> bool { + false + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + panic!("detect() must not run when prefilter() is false") + } + } + let mut s = DetectorSet::default(); + s.push(Box::new(NeverCalled)); + assert!(s.detect_all(&cand("anything")).unwrap().is_empty()); + } + + const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz"; + + fn spans(out: &[Finding]) -> Vec<(usize, usize)> { + out.iter().map(|f| (f.span.start, f.span.end)).collect() + } + + fn rules(out: &[Finding]) -> Vec<&str> { + out.iter().map(|f| f.rule.as_str()).collect() + } + + fn detect(text: &str, findings: Vec) -> Vec { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(findings))); + s.detect_all(&cand(text)).unwrap() + } + + #[test] + fn adjacent_spans_both_survive() { + let out = detect("abcdefgh", vec![f(0..4, "a", 0.9), f(4..8, "b", 0.9)]); + assert_eq!(spans(&out), vec![(0, 4), (4, 8)]); + } + + #[test] + fn three_way_overlap_keeps_the_end_a_loser_vacated() { + // A-B overlap, B-C overlap, A-C disjoint. C evicts B, which is the + // only thing that ever contested A, so A must survive. + let out = detect( + ALPHABET, + vec![f(0..5, "a", 0.5), f(4..9, "b", 0.9), f(8..20, "c", 0.5)], + ); + assert_eq!(spans(&out), vec![(0, 5), (8, 20)]); + assert_eq!(rules(&out), vec!["a", "c"]); + } + + #[test] + fn three_way_overlap_through_one_container() { + let out = detect( + ALPHABET, + vec![f(0..4, "a", 0.9), f(2..12, "b", 0.1), f(10..14, "c", 0.9)], + ); + assert_eq!(spans(&out), vec![(2, 12)]); + } + + #[test] + fn empty_text_drops_everything() { + assert!(detect("", vec![f(0..0, "a", 0.9), f(0..1, "b", 0.9)]).is_empty()); + } + + #[test] + fn span_covering_whole_text_survives() { + let out = detect("abcdefgh", vec![f(0..8, "a", 0.9)]); + assert_eq!(spans(&out), vec![(0, 8)]); + } + + #[test] + fn drops_zero_length_spans() { + assert!(detect("abcdefgh", vec![f(3..3, "a", 0.9)]).is_empty()); + } + + #[test] + fn drops_nan_scores() { + assert!(detect("abcdefgh", vec![f(0..4, "a", f32::NAN)]).is_empty()); + } + + #[test] + fn multibyte_span_on_char_boundaries_survives() { + // "héllo": h=0, é=1..3, l=3, l=4, o=5. + let out = detect("héllo", vec![f(1..3, "a", 0.9), f(3..6, "b", 0.9)]); + assert_eq!(spans(&out), vec![(1, 3), (3, 6)]); + } + + #[test] + fn detector_error_propagates() { + struct Failing; + impl Detector for Failing { + fn id(&self) -> &'static str { + "failing" + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Err(crate::RedactError::BadPointer("boom".into())) + } + } + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..4, "a", 0.9)]))); + s.push(Box::new(Failing)); + assert!(s.detect_all(&cand("abcdefgh")).is_err()); + } + + #[test] + fn identical_spans_equal_score_break_on_rule() { + let out = detect( + "abcdefgh", + vec![f(0..4, "zeta", 0.5), f(0..4, "alpha", 0.5)], + ); + assert_eq!(rules(&out), vec!["alpha"]); + } + + #[test] + fn result_is_independent_of_detector_order() { + let all = vec![ + f(0..5, "a", 0.5), + f(4..9, "b", 0.9), + f(8..20, "c", 0.5), + f(2..6, "d", 0.5), + f(19..26, "e", 0.7), + ]; + let expected = detect(ALPHABET, all.clone()); + + let mut reversed = all.clone(); + reversed.reverse(); + assert_eq!(detect(ALPHABET, reversed), expected); + + let mut split = DetectorSet::default(); + split.push(Box::new(HostileDetector(all[3..].to_vec()))); + split.push(Box::new(HostileDetector(all[..3].to_vec()))); + assert_eq!(split.detect_all(&cand(ALPHABET)).unwrap(), expected); + } + + #[allow(clippy::reversed_empty_ranges)] + #[test] + fn output_spans_never_overlap() { + let out = detect( + ALPHABET, + vec![ + f(0..3, "a", 0.1), + f(1..9, "b", 0.2), + f(2..4, "c", 0.9), + f(8..12, "d", 0.5), + f(11..11, "e", 0.5), + f(12..26, "g", 0.4), + f(25..99, "h", 0.9), + f(3..2, "i", 0.9), + ], + ); + assert!(out.windows(2).all(|w| w[0].span.end <= w[1].span.start)); + assert!(!out.is_empty()); + } + + #[test] + fn fixed_detector_reports_its_findings() { + let mut s = DetectorSet::default(); + s.push(Box::new(FixedDetector(vec![Finding { + span: 2..5, + rule: "canned".into(), + score: 1.0, + detector: "fixed", + }]))); + assert_eq!(s.ids(), vec!["fixed"]); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(spans(&out), vec![(2, 5)]); } } diff --git a/crates/toolpath-redact/src/exec.rs b/crates/toolpath-redact/src/exec.rs index 3b1cdd0b..891e728b 100644 --- a/crates/toolpath-redact/src/exec.rs +++ b/crates/toolpath-redact/src/exec.rs @@ -1,5 +1,6 @@ //! An out-of-process detector: one JSON candidate per line on stdin, one //! array of findings on stdout. - -/// Placeholder for the subprocess detector implemented in T8. -pub struct ExecDetector; +//! +//! Not implemented. The subprocess detector was cut from the first +//! release; the module records the wire contract so the next attempt does +//! not have to re-derive it. diff --git a/crates/toolpath-redact/src/internal/entropy.rs b/crates/toolpath-redact/src/internal/entropy.rs index a138925f..ff880eb0 100644 --- a/crates/toolpath-redact/src/internal/entropy.rs +++ b/crates/toolpath-redact/src/internal/entropy.rs @@ -1 +1,41 @@ -//! Shannon entropy scoring. Implemented in T3. +//! Shannon entropy over a matched value's character distribution, used to +//! down-score matches that look right but are actually low-entropy +//! (repeated characters, placeholder text). + +pub fn shannon(s: &str) -> f64 { + if s.is_empty() { + return 0.0; + } + let mut counts = std::collections::HashMap::new(); + for ch in s.chars() { + *counts.entry(ch).or_insert(0usize) += 1; + } + let n = s.chars().count() as f64; + -counts + .values() + .map(|&c| { + let p = c as f64 / n; + p * p.log2() + }) + .sum::() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_string_is_zero_entropy() { + assert_eq!(shannon(""), 0.0); + } + + #[test] + fn repeated_character_is_zero_entropy() { + assert_eq!(shannon("aaaaaaaaaa"), 0.0); + } + + #[test] + fn uniform_four_symbol_distribution_is_two_bits() { + assert!((shannon("abcdabcdabcd") - 2.0).abs() < 1e-9); + } +} diff --git a/crates/toolpath-redact/src/internal/gitleaks.toml b/crates/toolpath-redact/src/internal/gitleaks.toml new file mode 100644 index 00000000..256f6479 --- /dev/null +++ b/crates/toolpath-redact/src/internal/gitleaks.toml @@ -0,0 +1,3209 @@ +# This file has been auto-generated. Do not edit manually. +# If you would like to contribute new rules, please use +# cmd/generate/config/main.go and follow the contributing guidelines +# at https://github.com/gitleaks/gitleaks/blob/master/CONTRIBUTING.md +# +# How the hell does secret scanning work? Read this: +# https://lookingatcomputer.substack.com/p/regex-is-almost-all-you-need +# +# This is the default gitleaks configuration file. +# Rules and allowlists are defined within this file. +# Rules instruct gitleaks on what should be considered a secret. +# Allowlists instruct gitleaks on what is allowed, i.e. not a secret. + +title = "gitleaks config" + +# minVersion indicates the minimum Gitleaks version required to use this config. +# If the running version is older, a warning will be logged and not all +# config-enabled features are guaranteed to work. +minVersion = "v8.25.0" + +# TODO: change to [[allowlists]] +[allowlist] +description = "global allow lists" +paths = [ + '''gitleaks\.toml''', + '''(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$''', + '''(?i)\.(?:eot|[ot]tf|woff2?)$''', + '''(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$''', + '''go\.(?:mod|sum|work(?:\.sum)?)$''', + '''(?:^|/)vendor/modules\.txt$''', + '''(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$''', + '''(?:^|/)gradlew(?:\.bat)?$''', + '''(?:^|/)gradle\.lockfile$''', + '''(?:^|/)mvnw(?:\.cmd)?$''', + '''(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$''', + '''(?:^|/)node_modules(?:/.*)?$''', + '''(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$''', + '''(?:^|/)bower_components(?:/.*)?$''', + '''(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$''', + '''(?:^|/)javascript\.json$''', + '''(?:^|/)(?:Pipfile|poetry)\.lock$''', + '''(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$''', + '''(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$''', + '''(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$''', + '''(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$''', + '''\.gem$''', + '''verification-metadata\.xml''', + '''Database.refactorlog''', + '''(?:^|/)\.git$''', +] +regexes = [ + '''(?i)^true|false|null$''', + '''^(?i:a+|b+|c+|d+|e+|f+|g+|h+|i+|j+|k+|l+|m+|n+|o+|p+|q+|r+|s+|t+|u+|v+|w+|x+|y+|z+|\*+|\.+)$''', + '''^\$(?:\d+|{\d+})$''', + '''^\$(?:[A-Z_]+|[a-z_]+)$''', + '''^\${(?:[A-Z_]+|[a-z_]+)}$''', + '''^\{\{[ \t]*[\w ().|]+[ \t]*}}$''', + '''^\$\{\{[ \t]*(?:(?:env|github|secrets|vars)(?:\.[A-Za-z]\w+)+[\w "'&./=|]*)[ \t]*}}$''', + '''^%(?:[A-Z_]+|[a-z_]+)%$''', + '''^%[+\-# 0]?[bcdeEfFgGoOpqstTUvxX]$''', + '''^\{\d{0,2}}$''', + '''^@(?:[A-Z_]+|[a-z_]+)@$''', + '''^/Users/(?i)[a-z0-9]+/[\w .-/]+$''', + '''^/(?:bin|etc|home|opt|tmp|usr|var)/[\w ./-]+$''', +] +stopwords = [ + "014df517-39d1-4453-b7b3-9930c563627c", + "abcdefghijklmnopqrstuvwxyz", +] + +[[rules]] +id = "1password-secret-key" +description = "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults." +regex = '''\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b''' +entropy = 3.8 +keywords = ["a3-"] + +[[rules]] +id = "1password-service-account-token" +description = "Uncovered a possible 1Password service account token, potentially compromising access to secrets in vaults." +regex = '''ops_eyJ[a-zA-Z0-9+/]{250,}={0,3}''' +entropy = 4 +keywords = ["ops_"] + +[[rules]] +id = "adafruit-api-key" +description = "Identified a potential Adafruit API Key, which could lead to unauthorized access to Adafruit services and sensitive data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:adafruit)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["adafruit"] + +[[rules]] +id = "adobe-client-id" +description = "Detected a pattern that resembles an Adobe OAuth Web Client ID, posing a risk of compromised Adobe integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:adobe)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["adobe"] + +[[rules]] +id = "adobe-client-secret" +description = "Discovered a potential Adobe Client Secret, which, if exposed, could allow unauthorized Adobe service access and data manipulation." +regex = '''\b(p8e-(?i)[a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["p8e-"] + +[[rules]] +id = "age-secret-key" +description = "Discovered a potential Age encryption tool secret key, risking data decryption and unauthorized access to sensitive information." +regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' +keywords = ["age-secret-key-1"] + +[[rules]] +id = "airtable-api-key" +description = "Uncovered a possible Airtable API Key, potentially compromising database access and leading to data leakage or alteration." +regex = '''(?i)[\w.-]{0,50}?(?:airtable)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{17})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["airtable"] + +[[rules]] +id = "airtable-personnal-access-token" +description = "Uncovered a possible Airtable Personal AccessToken, potentially compromising database access and leading to data leakage or alteration." +regex = '''\b(pat[[:alnum:]]{14}\.[a-f0-9]{64})\b''' +keywords = ["airtable"] + +[[rules]] +id = "algolia-api-key" +description = "Identified an Algolia API Key, which could result in unauthorized search operations and data exposure on Algolia-managed platforms." +regex = '''(?i)[\w.-]{0,50}?(?:algolia)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["algolia"] + +[[rules]] +id = "alibaba-access-key-id" +description = "Detected an Alibaba Cloud AccessKey ID, posing a risk of unauthorized cloud resource access and potential data compromise." +regex = '''\b(LTAI(?i)[a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["ltai"] + +[[rules]] +id = "alibaba-secret-key" +description = "Discovered a potential Alibaba Cloud Secret Key, potentially allowing unauthorized operations and data access within Alibaba Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:alibaba)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["alibaba"] + +[[rules]] +id = "anthropic-admin-api-key" +description = "Detected an Anthropic Admin API Key, risking unauthorized access to administrative functions and sensitive AI model configurations." +regex = '''\b(sk-ant-admin01-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-admin01"] + +[[rules]] +id = "anthropic-api-key" +description = "Identified an Anthropic API Key, which may compromise AI assistant integrations and expose sensitive data to unauthorized access." +regex = '''\b(sk-ant-api03-[a-zA-Z0-9_\-]{93}AA)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sk-ant-api03"] + +[[rules]] +id = "artifactory-api-key" +description = "Detected an Artifactory api key, posing a risk unauthorized access to the central repository." +regex = '''\bAKCp[A-Za-z0-9]{69}\b''' +entropy = 4.5 +keywords = ["akcp"] + +[[rules]] +id = "artifactory-reference-token" +description = "Detected an Artifactory reference token, posing a risk of impersonation and unauthorized access to the central repository." +regex = '''\bcmVmd[A-Za-z0-9]{59}\b''' +entropy = 4.5 +keywords = ["cmvmd"] + +[[rules]] +id = "asana-client-id" +description = "Discovered a potential Asana Client ID, risking unauthorized access to Asana projects and sensitive task information." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "asana-client-secret" +description = "Identified an Asana Client Secret, which could lead to compromised project management integrity and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:asana)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["asana"] + +[[rules]] +id = "atlassian-api-token" +description = "Detected an Atlassian API token, posing a threat to project management and collaboration tool security and data confidentiality." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ATLASSIAN|[Aa]tlassian)|(?-i:CONFLUENCE|[Cc]onfluence)|(?-i:JIRA|[Jj]ira))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20}[a-f0-9]{4})(?:[\x60'"\s;]|\\[nr]|$)|\b(ATATT3[A-Za-z0-9_\-=]{186})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "atlassian", + "confluence", + "jira", + "atatt3", +] + +[[rules]] +id = "authress-service-client-access-key" +description = "Uncovered a possible Authress Service Client Access Key, which may compromise access control services and sensitive data." +regex = '''\b((?:sc|ext|scauth|authress)_(?i)[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.(?-i:acc)[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sc_", + "ext_", + "scauth_", + "authress_", +] + +[[rules]] +id = "aws-access-token" +description = "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms." +regex = '''\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b''' +entropy = 3 +keywords = [ + "a3t", + "akia", + "asia", + "abia", + "acca", +] +[[rules.allowlists]] +regexes = [ + '''.+EXAMPLE$''', +] + +[[rules]] +id = "aws-amazon-bedrock-api-key-long-lived" +description = "Identified a pattern that may indicate long-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''\b(ABSK[A-Za-z0-9+/]{109,269}={0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["absk"] + +[[rules]] +id = "aws-amazon-bedrock-api-key-short-lived" +description = "Identified a pattern that may indicate short-lived Amazon Bedrock API keys, risking unauthorized Amazon Bedrock usage" +regex = '''bedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29t''' +entropy = 3 +keywords = ["bedrock-api-key-"] + +[[rules]] +id = "azure-ad-client-secret" +description = "Azure AD Client Secret" +regex = '''(?:^|[\\'"\x60\s>=:(,)])([a-zA-Z0-9_~.]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:$|[\\'"\x60\s<),])''' +entropy = 3 +keywords = ["q~"] + +[[rules]] +id = "beamer-api-token" +description = "Detected a Beamer API token, potentially compromising content management and exposing sensitive notifications and updates." +regex = '''(?i)[\w.-]{0,50}?(?:beamer)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(b_[a-z0-9=_\-]{44})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["beamer"] + +[[rules]] +id = "bitbucket-client-id" +description = "Discovered a potential Bitbucket Client ID, risking unauthorized repository access and potential codebase exposure." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bitbucket-client-secret" +description = "Discovered a potential Bitbucket Client Secret, posing a risk of compromised code repositories and unauthorized access." +regex = '''(?i)[\w.-]{0,50}?(?:bitbucket)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bitbucket"] + +[[rules]] +id = "bittrex-access-key" +description = "Identified a Bittrex Access Key, which could lead to unauthorized access to cryptocurrency trading accounts and financial loss." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "bittrex-secret-key" +description = "Detected a Bittrex Secret Key, potentially compromising cryptocurrency transactions and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:bittrex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["bittrex"] + +[[rules]] +id = "cisco-meraki-api-key" +description = "Cisco Meraki is a cloud-managed IT solution that provides networking, security, and device management through an easy-to-use interface." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Mm]eraki|MERAKI))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["meraki"] + +[[rules]] +id = "clickhouse-cloud-api-secret-key" +description = "Identified a pattern that may indicate clickhouse cloud API secret key, risking unauthorized clickhouse cloud api access and data breaches on ClickHouse Cloud platforms." +regex = '''\b(4b1d[A-Za-z0-9]{38})\b''' +entropy = 3 +keywords = ["4b1d"] + +[[rules]] +id = "clojars-api-token" +description = "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation." +regex = '''(?i)CLOJARS_[a-z0-9]{60}''' +entropy = 2 +keywords = ["clojars_"] + +[[rules]] +id = "cloudflare-api-key" +description = "Detected a Cloudflare API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-global-api-key" +description = "Detected a Cloudflare Global API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:cloudflare)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{37})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["cloudflare"] + +[[rules]] +id = "cloudflare-origin-ca-key" +description = "Detected a Cloudflare Origin CA Key, potentially compromising cloud application deployments and operational security." +regex = '''\b(v1\.0-[a-f0-9]{24}-[a-f0-9]{146})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "cloudflare", + "v1.0-", +] + +[[rules]] +id = "codecov-access-token" +description = "Found a pattern resembling a Codecov Access Token, posing a risk of unauthorized access to code coverage reports and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:codecov)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["codecov"] + +[[rules]] +id = "cohere-api-token" +description = "Identified a Cohere Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:cohere|CO_API_KEY)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-zA-Z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "cohere", + "co_api_key", +] + +[[rules]] +id = "coinbase-access-token" +description = "Detected a Coinbase Access Token, posing a risk of unauthorized access to cryptocurrency accounts and financial transactions." +regex = '''(?i)[\w.-]{0,50}?(?:coinbase)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["coinbase"] + +[[rules]] +id = "confluent-access-token" +description = "Identified a Confluent Access Token, which could compromise access to streaming data platforms and sensitive data flow." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "confluent-secret-key" +description = "Found a Confluent Secret Key, potentially risking unauthorized operations and data access within Confluent services." +regex = '''(?i)[\w.-]{0,50}?(?:confluent)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["confluent"] + +[[rules]] +id = "contentful-delivery-api-token" +description = "Discovered a Contentful delivery API token, posing a risk to content management systems and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:contentful)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["contentful"] + +[[rules]] +id = "curl-auth-header" +description = "Discovered a potential authorization token provided in a curl command header, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*?|.*?(?:[\r\n]{1,2}.*?){1,5})[ \t\n\r](?:-H|--header)(?:=|[ \t]{0,5})(?:"(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))"|'(?i)(?:Authorization:[ \t]{0,5}(?:Basic[ \t]([a-z0-9+/]{8,}={0,3})|(?:Bearer|(?:Api-)?Token)[ \t]([\w=~@.+/-]{8,})|([\w=~@.+/-]{8,}))|(?:(?:X-(?:[a-z]+-)?)?(?:Api-?)?(?:Key|Token)):[ \t]{0,5}([\w=~@.+/-]{8,}))')(?:\B|\s|\z)''' +entropy = 2.75 +keywords = ["curl"] + +[[rules]] +id = "curl-auth-user" +description = "Discovered a potential basic authorization token provided in a curl command, which could compromise the curl accessed resource." +regex = '''\bcurl\b(?:.*|.*(?:[\r\n]{1,2}.*){1,5})[ \t\n\r](?:-u|--user)(?:=|[ \t]{0,5})("(:[^"]{3,}|[^:"]{3,}:|[^:"]{3,}:[^"]{3,})"|'([^:']{3,}:[^']{3,})'|((?:"[^"]{3,}"|'[^']{3,}'|[\w$@.-]+):(?:"[^"]{3,}"|'[^']{3,}'|[\w${}@.-]+)))(?:\s|\z)''' +entropy = 2 +keywords = ["curl"] +[[rules.allowlists]] +regexes = [ + '''[^:]+:(?:change(?:it|me)|pass(?:word)?|pwd|test|token|\*+|x+)''', + '''['"]?<[^>]+>['"]?:['"]?<[^>]+>|<[^:]+:[^>]+>['"]?''', + '''[^:]+:\[[^]]+]''', + '''['"]?[^:]+['"]?:['"]?\$(?:\d|\w+|\{(?:\d|\w+)})['"]?''', + '''\$\([^)]+\):\$\([^)]+\)''', + '''['"]?\$?{{[^}]+}}['"]?:['"]?\$?{{[^}]+}}['"]?''', +] + +[[rules]] +id = "databricks-api-token" +description = "Uncovered a Databricks API token, which may compromise big data analytics platforms and sensitive data processing." +regex = '''\b(dapi[a-f0-9]{32}(?:-\d)?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dapi"] + +[[rules]] +id = "datadog-access-token" +description = "Detected a Datadog Access Token, potentially risking monitoring and analytics data exposure and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:datadog)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["datadog"] + +[[rules]] +id = "defined-networking-api-token" +description = "Identified a Defined Networking API token, which could lead to unauthorized network operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:dnkey)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(dnkey-[a-z0-9=_\-]{26}-[a-z0-9=_\-]{52})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dnkey"] + +[[rules]] +id = "digitalocean-access-token" +description = "Found a DigitalOcean OAuth Access Token, risking unauthorized cloud resource access and data compromise." +regex = '''\b(doo_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["doo_v1_"] + +[[rules]] +id = "digitalocean-pat" +description = "Discovered a DigitalOcean Personal Access Token, posing a threat to cloud infrastructure security and data privacy." +regex = '''\b(dop_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["dop_v1_"] + +[[rules]] +id = "digitalocean-refresh-token" +description = "Uncovered a DigitalOcean OAuth Refresh Token, which could allow prolonged unauthorized access and resource manipulation." +regex = '''(?i)\b(dor_v1_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dor_v1_"] + +[[rules]] +id = "discord-api-token" +description = "Detected a Discord API key, potentially compromising communication channels and user data privacy on Discord." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["discord"] + +[[rules]] +id = "discord-client-id" +description = "Identified a Discord client ID, which may lead to unauthorized integrations and data exposure in Discord applications." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{18})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "discord-client-secret" +description = "Discovered a potential Discord client secret, risking compromised Discord bot integrations and data leaks." +regex = '''(?i)[\w.-]{0,50}?(?:discord)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["discord"] + +[[rules]] +id = "doppler-api-token" +description = "Discovered a Doppler API token, posing a risk to environment and secrets management security." +regex = '''dp\.pt\.(?i)[a-z0-9]{43}''' +entropy = 2 +keywords = ["dp.pt."] + +[[rules]] +id = "droneci-access-token" +description = "Detected a Droneci Access Token, potentially compromising continuous integration and deployment workflows." +regex = '''(?i)[\w.-]{0,50}?(?:droneci)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["droneci"] + +[[rules]] +id = "dropbox-api-token" +description = "Identified a Dropbox API secret, which could lead to unauthorized file access and data breaches in Dropbox storage." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{15})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-long-lived-api-token" +description = "Found a Dropbox long-lived API token, risking prolonged unauthorized access to cloud storage and sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "dropbox-short-lived-api-token" +description = "Discovered a Dropbox short-lived API token, posing a risk of temporary but potentially harmful data access and manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:dropbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(sl\.[a-z0-9\-=_]{135})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["dropbox"] + +[[rules]] +id = "duffel-api-token" +description = "Uncovered a Duffel API token, which may compromise travel platform integrations and sensitive customer data." +regex = '''duffel_(?:test|live)_(?i)[a-z0-9_\-=]{43}''' +entropy = 2 +keywords = ["duffel_"] + +[[rules]] +id = "dynatrace-api-token" +description = "Detected a Dynatrace API token, potentially risking application performance monitoring and data exposure." +regex = '''dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}''' +entropy = 4 +keywords = ["dt0c01."] + +[[rules]] +id = "easypost-api-token" +description = "Identified an EasyPost API token, which could lead to unauthorized postal and shipment service access and data exposure." +regex = '''\bEZAK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["ezak"] + +[[rules]] +id = "easypost-test-api-token" +description = "Detected an EasyPost test API token, risking exposure of test environments and potentially sensitive shipment data." +regex = '''\bEZTK(?i)[a-z0-9]{54}\b''' +entropy = 2 +keywords = ["eztk"] + +[[rules]] +id = "etsy-access-token" +description = "Found an Etsy Access Token, potentially compromising Etsy shop management and customer data." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:ETSY|[Ee]tsy))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["etsy"] + +[[rules]] +id = "facebook-access-token" +description = "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)\b(\d{15,16}(\||%)[0-9a-z\-_]{27,40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "facebook-page-access-token" +description = "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''\b(EAA[MC](?i)[a-z0-9]{100,})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "eaam", + "eaac", +] + +[[rules]] +id = "facebook-secret" +description = "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:facebook)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["facebook"] + +[[rules]] +id = "fastly-api-token" +description = "Uncovered a Fastly API key, which may compromise CDN and edge cloud services, leading to content delivery and security issues." +regex = '''(?i)[\w.-]{0,50}?(?:fastly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["fastly"] + +[[rules]] +id = "finicity-api-token" +description = "Detected a Finicity API token, potentially risking financial data access and unauthorized financial operations." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finicity-client-secret" +description = "Identified a Finicity Client Secret, which could lead to compromised financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:finicity)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finicity"] + +[[rules]] +id = "finnhub-access-token" +description = "Found a Finnhub Access Token, risking unauthorized access to financial market data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:finnhub)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["finnhub"] + +[[rules]] +id = "flickr-access-token" +description = "Discovered a Flickr Access Token, posing a risk of unauthorized photo management and potential data leakage." +regex = '''(?i)[\w.-]{0,50}?(?:flickr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["flickr"] + +[[rules]] +id = "flutterwave-encryption-key" +description = "Uncovered a Flutterwave Encryption Key, which may compromise payment processing and sensitive financial information." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{12}''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flutterwave-public-key" +description = "Detected a Finicity Public Key, potentially exposing public cryptographic operations and integrations." +regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwpubk_test"] + +[[rules]] +id = "flutterwave-secret-key" +description = "Identified a Flutterwave Secret Key, risking unauthorized financial transactions and data breaches." +regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' +entropy = 2 +keywords = ["flwseck_test"] + +[[rules]] +id = "flyio-access-token" +description = "Uncovered a Fly.io API key" +regex = '''\b((?:fo1_[\w-]{43}|fm1[ar]_[a-zA-Z0-9+\/]{100,}={0,3}|fm2_[a-zA-Z0-9+\/]{100,}={0,3}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = [ + "fo1_", + "fm1", + "fm2_", +] + +[[rules]] +id = "frameio-api-token" +description = "Found a Frame.io API token, potentially compromising video collaboration and project management." +regex = '''fio-u-(?i)[a-z0-9\-_=]{64}''' +keywords = ["fio-u-"] + +[[rules]] +id = "freemius-secret-key" +description = "Detected a Freemius secret key, potentially exposing sensitive information." +regex = '''(?i)["']secret_key["']\s*=>\s*["'](sk_[\S]{29})["']''' +path = '''(?i)\.php$''' +keywords = ["secret_key"] + +[[rules]] +id = "freshbooks-access-token" +description = "Discovered a Freshbooks Access Token, posing a risk to accounting software access and sensitive financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:freshbooks)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["freshbooks"] + +[[rules]] +id = "gcp-api-key" +description = "Uncovered a GCP API key, which could lead to unauthorized access to Google Cloud services and data breaches." +regex = '''\b(AIza[\w-]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["aiza"] +[[rules.allowlists]] +regexes = [ + '''AIzaSyabcdefghijklmnopqrstuvwxyz1234567''', + '''AIzaSyAnLA7NfeLquW1tJFpx_eQCxoX-oo6YyIs''', + '''AIzaSyCkEhVjf3pduRDt6d1yKOMitrUEke8agEM''', + '''AIzaSyDMAScliyLx7F0NPDEJi1QmyCgHIAODrlU''', + '''AIzaSyD3asb-2pEZVqMkmL6M9N6nHZRR_znhrh0''', + '''AIzayDNSXIbFmlXbIE6mCzDLQAqITYefhixbX4A''', + '''AIzaSyAdOS2zB6NCsk1pCdZ4-P6GBdi_UUPwX7c''', + '''AIzaSyASWm6HmTMdYWpgMnjRBjxcQ9CKctWmLd4''', + '''AIzaSyANUvH9H9BsUccjsu2pCmEkOPjjaXeDQgY''', + '''AIzaSyA5_iVawFQ8ABuTZNUdcwERLJv_a_p4wtM''', + '''AIzaSyA4UrcGxgwQFTfaI3no3t7Lt1sjmdnP5sQ''', + '''AIzaSyDSb51JiIcB6OJpwwMicseKRhhrOq1cS7g''', + '''AIzaSyBF2RrAIm4a0mO64EShQfqfd2AFnzAvvuU''', + '''AIzaSyBcE-OOIbhjyR83gm4r2MFCu4MJmprNXsw''', + '''AIzaSyB8qGxt4ec15vitgn44duC5ucxaOi4FmqE''', + '''AIzaSyA8vmApnrHNFE0bApF4hoZ11srVL_n0nvY''', +] + +[[rules]] +id = "generic-api-key" +description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." +regex = '''(?i)[\w.-]{0,50}?(?:access|auth|(?-i:[Aa]pi|API)|credential|creds|key|passw(?:or)?d|secret|token)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "access", + "api", + "auth", + "key", + "credential", + "creds", + "passwd", + "password", + "secret", + "token", +] +[[rules.allowlists]] +regexes = [ + '''^[a-zA-Z_.-]+$''', +] +[[rules.allowlists]] +description = "Allowlist for Generic API Keys" +regexTarget = "match" +regexes = [ + '''(?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])|public[_.-]?token|(?:key|token)[_.-]?file|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z)))''', +] +stopwords = [ + "000000", + "6fe4476ee5a1832882e326b506d14126", + "_ec2_", + "aaaaaa", + "about", + "abstract", + "academy", + "acces", + "account", + "act-", + "act.", + "act_", + "action", + "active", + "actively", + "activity", + "adapter", + "add-", + "add-on", + "add.", + "add_", + "addon", + "addres", + "admin", + "adobe", + "advanced", + "adventure", + "agent", + "agile", + "air-", + "air.", + "air_", + "ajax", + "akka", + "alert", + "alfred", + "algorithm", + "all-", + "all.", + "all_", + "alloy", + "alpha", + "amazon", + "amqp", + "analysi", + "analytic", + "analyzer", + "android", + "angular", + "angularj", + "animate", + "animation", + "another", + "ansible", + "answer", + "ant-", + "ant.", + "ant_", + "any-", + "any.", + "any_", + "apache", + "app-", + "app.", + "app_", + "apple", + "arch", + "archive", + "archived", + "arduino", + "array", + "art-", + "art.", + "art_", + "article", + "asp-", + "asp.", + "asp_", + "asset", + "async", + "atom", + "attention", + "audio", + "audit", + "aura", + "auth", + "author", + "authorize", + "auto", + "automated", + "automatic", + "awesome", + "aws_", + "azure", + "back", + "backbone", + "backend", + "backup", + "bar-", + "bar.", + "bar_", + "base", + "based", + "bash", + "basic", + "batch", + "been", + "beer", + "behavior", + "being", + "benchmark", + "best", + "beta", + "better", + "big-", + "big.", + "big_", + "binary", + "binding", + "bit-", + "bit.", + "bit_", + "bitcoin", + "block", + "blog", + "board", + "book", + "bookmark", + "boost", + "boot", + "bootstrap", + "bosh", + "bot-", + "bot.", + "bot_", + "bower", + "box-", + "box.", + "box_", + "boxen", + "bracket", + "branch", + "bridge", + "browser", + "brunch", + "buffer", + "bug-", + "bug.", + "bug_", + "build", + "builder", + "building", + "buildout", + "buildpack", + "built", + "bundle", + "busines", + "but-", + "but.", + "but_", + "button", + "cache", + "caching", + "cakephp", + "calendar", + "call", + "camera", + "campfire", + "can-", + "can.", + "can_", + "canva", + "captcha", + "capture", + "card", + "carousel", + "case", + "cassandra", + "cat-", + "cat.", + "cat_", + "category", + "center", + "cento", + "challenge", + "change", + "changelog", + "channel", + "chart", + "chat", + "cheat", + "check", + "checker", + "chef", + "ches", + "chinese", + "chosen", + "chrome", + "ckeditor", + "clas", + "classe", + "classic", + "clean", + "cli-", + "cli.", + "cli_", + "client", + "clojure", + "clone", + "closure", + "cloud", + "club", + "cluster", + "cms-", + "cms_", + "coco", + "code", + "coding", + "coffee", + "color", + "combination", + "combo", + "command", + "commander", + "comment", + "commit", + "common", + "community", + "compas", + "compiler", + "complete", + "component", + "composer", + "computer", + "computing", + "con-", + "con.", + "con_", + "concept", + "conf", + "config", + "connect", + "connector", + "console", + "contact", + "container", + "contao", + "content", + "contest", + "context", + "control", + "convert", + "converter", + "conway'", + "cookbook", + "cookie", + "cool", + "copy", + "cordova", + "core", + "couchbase", + "couchdb", + "countdown", + "counter", + "course", + "craft", + "crawler", + "create", + "creating", + "creator", + "credential", + "crm-", + "crm.", + "crm_", + "cros", + "crud", + "csv-", + "csv.", + "csv_", + "cube", + "cucumber", + "cuda", + "current", + "currently", + "custom", + "daemon", + "dark", + "dart", + "dash", + "dashboard", + "data", + "database", + "date", + "day-", + "day.", + "day_", + "dead", + "debian", + "debug", + "debugger", + "deck", + "define", + "del-", + "del.", + "del_", + "delete", + "demo", + "deploy", + "design", + "designer", + "desktop", + "detection", + "detector", + "dev-", + "dev.", + "dev_", + "develop", + "developer", + "device", + "devise", + "diff", + "digital", + "directive", + "directory", + "discovery", + "display", + "django", + "dns-", + "dns_", + "doc-", + "doc.", + "doc_", + "docker", + "docpad", + "doctrine", + "document", + "doe-", + "doe.", + "doe_", + "dojo", + "dom-", + "dom.", + "dom_", + "domain", + "don't", + "done", + "dot-", + "dot.", + "dot_", + "dotfile", + "download", + "draft", + "drag", + "drill", + "drive", + "driven", + "driver", + "drop", + "dropbox", + "drupal", + "dsl-", + "dsl.", + "dsl_", + "dynamic", + "easy", + "ecdsa", + "eclipse", + "edit", + "editing", + "edition", + "editor", + "element", + "emac", + "email", + "embed", + "embedded", + "ember", + "emitter", + "emulator", + "encoding", + "endpoint", + "engine", + "english", + "enhanced", + "entity", + "entry", + "env_", + "episode", + "erlang", + "error", + "espresso", + "event", + "evented", + "example", + "exchange", + "exercise", + "experiment", + "expire", + "exploit", + "explorer", + "export", + "exporter", + "expres", + "ext-", + "ext.", + "ext_", + "extended", + "extension", + "external", + "extra", + "extractor", + "fabric", + "facebook", + "factory", + "fake", + "fast", + "feature", + "feed", + "fewfwef", + "ffmpeg", + "field", + "file", + "filter", + "find", + "finder", + "firefox", + "firmware", + "first", + "fish", + "fix-", + "fix_", + "flash", + "flask", + "flat", + "flex", + "flexible", + "flickr", + "flow", + "fluent", + "fluentd", + "fluid", + "folder", + "font", + "force", + "foreman", + "fork", + "form", + "format", + "formatter", + "forum", + "foundry", + "framework", + "free", + "friend", + "friendly", + "front-end", + "frontend", + "ftp-", + "ftp.", + "ftp_", + "fuel", + "full", + "fun-", + "fun.", + "fun_", + "func", + "future", + "gaia", + "gallery", + "game", + "gateway", + "gem-", + "gem.", + "gem_", + "gen-", + "gen.", + "gen_", + "general", + "generator", + "generic", + "genetic", + "get-", + "get.", + "get_", + "getenv", + "getting", + "ghost", + "gist", + "git-", + "git.", + "git_", + "github", + "gitignore", + "gitlab", + "glas", + "gmail", + "gnome", + "gnu-", + "gnu.", + "gnu_", + "goal", + "golang", + "gollum", + "good", + "google", + "gpu-", + "gpu.", + "gpu_", + "gradle", + "grail", + "graph", + "graphic", + "great", + "grid", + "groovy", + "group", + "grunt", + "guard", + "gui-", + "gui.", + "gui_", + "guide", + "guideline", + "gulp", + "gwt-", + "gwt.", + "gwt_", + "hack", + "hackathon", + "hacker", + "hacking", + "hadoop", + "haml", + "handler", + "hardware", + "has-", + "has_", + "hash", + "haskell", + "have", + "haxe", + "hello", + "help", + "helper", + "here", + "hero", + "heroku", + "high", + "hipchat", + "history", + "home", + "homebrew", + "homepage", + "hook", + "host", + "hosting", + "hot-", + "hot.", + "hot_", + "house", + "how-", + "how.", + "how_", + "html", + "http", + "hub-", + "hub.", + "hub_", + "hubot", + "human", + "icon", + "ide-", + "ide.", + "ide_", + "idea", + "identity", + "idiomatic", + "image", + "impact", + "import", + "important", + "importer", + "impres", + "index", + "infinite", + "info", + "injection", + "inline", + "input", + "inside", + "inspector", + "instagram", + "install", + "installer", + "instant", + "intellij", + "interface", + "internet", + "interview", + "into", + "intro", + "ionic", + "iphone", + "ipython", + "irc-", + "irc_", + "iso-", + "iso.", + "iso_", + "issue", + "jade", + "jasmine", + "java", + "jbos", + "jekyll", + "jenkin", + "jetbrains", + "job-", + "job.", + "job_", + "joomla", + "jpa-", + "jpa.", + "jpa_", + "jquery", + "json", + "just", + "kafka", + "karma", + "kata", + "kernel", + "keyboard", + "kindle", + "kit-", + "kit.", + "kit_", + "kitchen", + "knife", + "koan", + "kohana", + "lab-", + "lab.", + "lab_", + "lambda", + "lamp", + "language", + "laravel", + "last", + "latest", + "latex", + "launcher", + "layer", + "layout", + "lazy", + "ldap", + "leaflet", + "league", + "learn", + "learning", + "led-", + "led.", + "led_", + "leetcode", + "les-", + "les.", + "les_", + "level", + "leveldb", + "lib-", + "lib.", + "lib_", + "librarie", + "library", + "license", + "life", + "liferay", + "light", + "lightbox", + "like", + "line", + "link", + "linked", + "linkedin", + "linux", + "lisp", + "list", + "lite", + "little", + "load", + "loader", + "local", + "location", + "lock", + "log-", + "log.", + "log_", + "logger", + "logging", + "logic", + "login", + "logstash", + "longer", + "look", + "love", + "lua-", + "lua.", + "lua_", + "mac-", + "mac.", + "mac_", + "machine", + "made", + "magento", + "magic", + "mail", + "make", + "maker", + "making", + "man-", + "man.", + "man_", + "manage", + "manager", + "manifest", + "manual", + "map-", + "map.", + "map_", + "mapper", + "mapping", + "markdown", + "markup", + "master", + "math", + "matrix", + "maven", + "md5", + "mean", + "media", + "mediawiki", + "meetup", + "memcached", + "memory", + "menu", + "merchant", + "message", + "messaging", + "meta", + "metadata", + "meteor", + "method", + "metric", + "micro", + "middleman", + "migration", + "minecraft", + "miner", + "mini", + "minimal", + "mirror", + "mit-", + "mit.", + "mit_", + "mobile", + "mocha", + "mock", + "mod-", + "mod.", + "mod_", + "mode", + "model", + "modern", + "modular", + "module", + "modx", + "money", + "mongo", + "mongodb", + "mongoid", + "mongoose", + "monitor", + "monkey", + "more", + "motion", + "moved", + "movie", + "mozilla", + "mqtt", + "mule", + "multi", + "multiple", + "music", + "mustache", + "mvc-", + "mvc.", + "mvc_", + "mysql", + "nagio", + "name", + "native", + "need", + "neo-", + "neo.", + "neo_", + "nest", + "nested", + "net-", + "net.", + "net_", + "nette", + "network", + "new-", + "new.", + "new_", + "next", + "nginx", + "ninja", + "nlp-", + "nlp.", + "nlp_", + "node", + "nodej", + "nosql", + "not-", + "not.", + "not_", + "note", + "notebook", + "notepad", + "notice", + "notifier", + "now-", + "now.", + "now_", + "number", + "oauth", + "object", + "objective", + "obsolete", + "ocaml", + "octopres", + "official", + "old-", + "old.", + "old_", + "onboard", + "online", + "only", + "open", + "opencv", + "opengl", + "openshift", + "openwrt", + "option", + "oracle", + "org-", + "org.", + "org_", + "origin", + "original", + "orm-", + "orm.", + "orm_", + "osx-", + "osx_", + "our-", + "our.", + "our_", + "out-", + "out.", + "out_", + "output", + "over", + "overview", + "own-", + "own.", + "own_", + "pack", + "package", + "packet", + "page", + "panel", + "paper", + "paperclip", + "para", + "parallax", + "parallel", + "parse", + "parser", + "parsing", + "particle", + "party", + "password", + "patch", + "path", + "pattern", + "payment", + "paypal", + "pdf-", + "pdf.", + "pdf_", + "pebble", + "people", + "perl", + "personal", + "phalcon", + "phoenix", + "phone", + "phonegap", + "photo", + "php-", + "php.", + "php_", + "physic", + "picker", + "pipeline", + "platform", + "play", + "player", + "please", + "plu-", + "plu.", + "plu_", + "plug-in", + "plugin", + "plupload", + "png-", + "png.", + "png_", + "poker", + "polyfill", + "polymer", + "pool", + "pop-", + "pop.", + "pop_", + "popcorn", + "popup", + "port", + "portable", + "portal", + "portfolio", + "post", + "power", + "powered", + "powerful", + "prelude", + "pretty", + "preview", + "principle", + "print", + "pro-", + "pro.", + "pro_", + "problem", + "proc", + "product", + "profile", + "profiler", + "program", + "progres", + "project", + "protocol", + "prototype", + "provider", + "proxy", + "public", + "pull", + "puppet", + "pure", + "purpose", + "push", + "pusher", + "pyramid", + "python", + "quality", + "query", + "queue", + "quick", + "rabbitmq", + "rack", + "radio", + "rail", + "railscast", + "random", + "range", + "raspberry", + "rdf-", + "rdf.", + "rdf_", + "react", + "reactive", + "read", + "reader", + "readme", + "ready", + "real", + "real-time", + "reality", + "realtime", + "recipe", + "recorder", + "red-", + "red.", + "red_", + "reddit", + "redi", + "redmine", + "reference", + "refinery", + "refresh", + "registry", + "related", + "release", + "remote", + "rendering", + "repo", + "report", + "request", + "require", + "required", + "requirej", + "research", + "resource", + "response", + "resque", + "rest", + "restful", + "resume", + "reveal", + "reverse", + "review", + "riak", + "rich", + "right", + "ring", + "robot", + "role", + "room", + "router", + "routing", + "rpc-", + "rpc.", + "rpc_", + "rpg-", + "rpg.", + "rpg_", + "rspec", + "ruby-", + "ruby.", + "ruby_", + "rule", + "run-", + "run.", + "run_", + "runner", + "running", + "runtime", + "rust", + "rvm-", + "rvm.", + "rvm_", + "salt", + "sample", + "sandbox", + "sas-", + "sas.", + "sas_", + "sbt-", + "sbt.", + "sbt_", + "scala", + "scalable", + "scanner", + "schema", + "scheme", + "school", + "science", + "scraper", + "scratch", + "screen", + "script", + "scroll", + "scs-", + "scs.", + "scs_", + "sdk-", + "sdk.", + "sdk_", + "sdl-", + "sdl.", + "sdl_", + "search", + "secure", + "security", + "see-", + "see.", + "see_", + "seed", + "select", + "selector", + "selenium", + "semantic", + "sencha", + "send", + "sentiment", + "serie", + "server", + "service", + "session", + "set-", + "set.", + "set_", + "setting", + "setup", + "sha1", + "sha2", + "sha256", + "share", + "shared", + "sharing", + "sheet", + "shell", + "shield", + "shipping", + "shop", + "shopify", + "shortener", + "should", + "show", + "showcase", + "side", + "silex", + "simple", + "simulator", + "single", + "site", + "skeleton", + "sketch", + "skin", + "slack", + "slide", + "slider", + "slim", + "small", + "smart", + "smtp", + "snake", + "snapshot", + "snippet", + "soap", + "social", + "socket", + "software", + "solarized", + "solr", + "solution", + "solver", + "some", + "soon", + "source", + "space", + "spark", + "spatial", + "spec", + "sphinx", + "spine", + "spotify", + "spree", + "spring", + "sprite", + "sql-", + "sql.", + "sql_", + "sqlite", + "ssh-", + "ssh.", + "ssh_", + "stack", + "staging", + "standard", + "stanford", + "start", + "started", + "starter", + "startup", + "stat", + "statamic", + "state", + "static", + "statistic", + "statsd", + "statu", + "steam", + "step", + "still", + "stm-", + "stm.", + "stm_", + "storage", + "store", + "storm", + "story", + "strategy", + "stream", + "streaming", + "string", + "stripe", + "structure", + "studio", + "study", + "stuff", + "style", + "sublime", + "sugar", + "suite", + "summary", + "super", + "support", + "supported", + "svg-", + "svg.", + "svg_", + "svn-", + "svn.", + "svn_", + "swagger", + "swift", + "switch", + "switcher", + "symfony", + "symphony", + "sync", + "synopsi", + "syntax", + "system", + "tab-", + "tab.", + "tab_", + "table", + "tag-", + "tag.", + "tag_", + "talk", + "target", + "task", + "tcp-", + "tcp.", + "tcp_", + "tdd-", + "tdd.", + "tdd_", + "team", + "tech", + "template", + "term", + "terminal", + "testing", + "tetri", + "text", + "textmate", + "theme", + "theory", + "three", + "thrift", + "time", + "timeline", + "timer", + "tiny", + "tinymce", + "tip-", + "tip.", + "tip_", + "title", + "todo", + "todomvc", + "token", + "tool", + "toolbox", + "toolkit", + "top-", + "top.", + "top_", + "tornado", + "touch", + "tower", + "tracker", + "tracking", + "traffic", + "training", + "transfer", + "translate", + "transport", + "tree", + "trello", + "try-", + "try.", + "try_", + "tumblr", + "tut-", + "tut.", + "tut_", + "tutorial", + "tweet", + "twig", + "twitter", + "type", + "typo", + "ubuntu", + "uiview", + "ultimate", + "under", + "unit", + "unity", + "universal", + "unix", + "update", + "updated", + "upgrade", + "upload", + "uploader", + "uri-", + "uri.", + "uri_", + "url-", + "url.", + "url_", + "usage", + "usb-", + "usb.", + "usb_", + "use-", + "use.", + "use_", + "used", + "useful", + "user", + "using", + "util", + "utilitie", + "utility", + "vagrant", + "validator", + "value", + "variou", + "varnish", + "version", + "via-", + "via.", + "via_", + "video", + "view", + "viewer", + "vim-", + "vim.", + "vim_", + "vimrc", + "virtual", + "vision", + "visual", + "vpn", + "want", + "warning", + "watch", + "watcher", + "wave", + "way-", + "way.", + "way_", + "weather", + "web-", + "web_", + "webapp", + "webgl", + "webhook", + "webkit", + "webrtc", + "website", + "websocket", + "welcome", + "what", + "what'", + "when", + "where", + "which", + "why-", + "why.", + "why_", + "widget", + "wifi", + "wiki", + "win-", + "win.", + "win_", + "window", + "wip-", + "wip.", + "wip_", + "within", + "without", + "wizard", + "word", + "wordpres", + "work", + "worker", + "workflow", + "working", + "workshop", + "world", + "wrapper", + "write", + "writer", + "writing", + "written", + "www-", + "www.", + "www_", + "xamarin", + "xcode", + "xml-", + "xml.", + "xml_", + "xmpp", + "xxxxxx", + "yahoo", + "yaml", + "yandex", + "yeoman", + "yet-", + "yet.", + "yet_", + "yii-", + "yii.", + "yii_", + "youtube", + "yui-", + "yui.", + "yui_", + "zend", + "zero", + "zip-", + "zip.", + "zip_", + "zsh-", + "zsh.", + "zsh_", +] +[[rules.allowlists]] +regexTarget = "line" +regexes = [ + '''--mount=type=secret,''', + '''import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]''', +] +[[rules.allowlists]] +condition = "AND" +paths = [ + '''\.bb$''','''\.bbappend$''','''\.bbclass$''','''\.inc$''', +] +regexTarget = "line" +regexes = [ + '''LICENSE[^=]*=\s*"[^"]+''', + '''LIC_FILES_CHKSUM[^=]*=\s*"[^"]+''', + '''SRC[^=]*=\s*"[a-zA-Z0-9]+''', +] + +[[rules]] +id = "github-app-token" +description = "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security." +regex = '''(?:ghu|ghs)_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = [ + "ghu_", + "ghs_", +] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-fine-grained-pat" +description = "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation." +regex = '''github_pat_\w{82}''' +entropy = 3 +keywords = ["github_pat_"] + +[[rules]] +id = "github-oauth" +description = "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks." +regex = '''gho_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["gho_"] + +[[rules]] +id = "github-pat" +description = "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure." +regex = '''ghp_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghp_"] +[[rules.allowlists]] +paths = [ + '''(?:^|/)@octokit/auth-token/README\.md$''', +] + +[[rules]] +id = "github-refresh-token" +description = "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services." +regex = '''ghr_[0-9a-zA-Z]{36}''' +entropy = 3 +keywords = ["ghr_"] + +[[rules]] +id = "gitlab-cicd-job-token" +description = "Identified a GitLab CI/CD Job Token, potential access to projects and some APIs on behalf of a user while the CI job is running." +regex = '''glcbt-[0-9a-zA-Z]{1,5}_[0-9a-zA-Z_-]{20}''' +entropy = 3 +keywords = ["glcbt-"] + +[[rules]] +id = "gitlab-deploy-token" +description = "Identified a GitLab Deploy Token, risking access to repositories, packages and containers with write access." +regex = '''gldt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["gldt-"] + +[[rules]] +id = "gitlab-feature-flag-client-token" +description = "Identified a GitLab feature flag client token, risks exposing user lists and features flags used by an application." +regex = '''glffct-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glffct-"] + +[[rules]] +id = "gitlab-feed-token" +description = "Identified a GitLab feed token, risking exposure of user data." +regex = '''glft-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glft-"] + +[[rules]] +id = "gitlab-incoming-mail-token" +description = "Identified a GitLab incoming mail token, risking manipulation of data sent by mail." +regex = '''glimt-[0-9a-zA-Z_\-]{25}''' +entropy = 3 +keywords = ["glimt-"] + +[[rules]] +id = "gitlab-kubernetes-agent-token" +description = "Identified a GitLab Kubernetes Agent token, risking access to repos and registry of projects connected via agent." +regex = '''glagent-[0-9a-zA-Z_\-]{50}''' +entropy = 3 +keywords = ["glagent-"] + +[[rules]] +id = "gitlab-oauth-app-secret" +description = "Identified a GitLab OIDC Application Secret, risking access to apps using GitLab as authentication provider." +regex = '''gloas-[0-9a-zA-Z_\-]{64}''' +entropy = 3 +keywords = ["gloas-"] + +[[rules]] +id = "gitlab-pat" +description = "Identified a GitLab Personal Access Token, risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''glpat-[\w-]{20}''' +entropy = 3 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-pat-routable" +description = "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure." +regex = '''\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glpat-"] + +[[rules]] +id = "gitlab-ptt" +description = "Found a GitLab Pipeline Trigger Token, potentially compromising continuous integration workflows and project security." +regex = '''glptt-[0-9a-f]{40}''' +entropy = 3 +keywords = ["glptt-"] + +[[rules]] +id = "gitlab-rrt" +description = "Discovered a GitLab Runner Registration Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''GR1348941[\w-]{20}''' +entropy = 3 +keywords = ["gr1348941"] + +[[rules]] +id = "gitlab-runner-authentication-token" +description = "Discovered a GitLab Runner Authentication Token, posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''glrt-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-runner-authentication-token-routable" +description = "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access." +regex = '''\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b''' +entropy = 4 +keywords = ["glrt-"] + +[[rules]] +id = "gitlab-scim-token" +description = "Discovered a GitLab SCIM Token, posing a risk to unauthorized access for a organization or instance." +regex = '''glsoat-[0-9a-zA-Z_\-]{20}''' +entropy = 3 +keywords = ["glsoat-"] + +[[rules]] +id = "gitlab-session-cookie" +description = "Discovered a GitLab Session Cookie, posing a risk to unauthorized access to a user account." +regex = '''_gitlab_session=[0-9a-z]{32}''' +entropy = 3 +keywords = ["_gitlab_session="] + +[[rules]] +id = "gitter-access-token" +description = "Uncovered a Gitter Access Token, which may lead to unauthorized access to chat and communication services." +regex = '''(?i)[\w.-]{0,50}?(?:gitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["gitter"] + +[[rules]] +id = "gocardless-api-token" +description = "Detected a GoCardless API token, potentially risking unauthorized direct debit payment operations and financial data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:gocardless)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(live_(?i)[a-z0-9\-_=]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "live_", + "gocardless", +] + +[[rules]] +id = "grafana-api-key" +description = "Identified a Grafana API key, which could compromise monitoring dashboards and sensitive data analytics." +regex = '''(?i)\b(eyJrIjoi[A-Za-z0-9]{70,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["eyjrijoi"] + +[[rules]] +id = "grafana-cloud-api-token" +description = "Found a Grafana cloud API token, risking unauthorized access to cloud-based monitoring services and data exposure." +regex = '''(?i)\b(glc_[A-Za-z0-9+/]{32,400}={0,3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glc_"] + +[[rules]] +id = "grafana-service-account-token" +description = "Discovered a Grafana service account token, posing a risk of compromised monitoring services and data integrity." +regex = '''(?i)\b(glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["glsa_"] + +[[rules]] +id = "harness-api-key" +description = "Identified a Harness Access Token (PAT or SAT), risking unauthorized access to a Harness account." +regex = '''(?:pat|sat)\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9]{24}\.[a-zA-Z0-9]{20}''' +keywords = [ + "pat.", + "sat.", +] + +[[rules]] +id = "hashicorp-tf-api-token" +description = "Uncovered a HashiCorp Terraform user/org API token, which may lead to unauthorized infrastructure management and security breaches." +regex = '''(?i)[a-z0-9]{14}\.(?-i:atlasv1)\.[a-z0-9\-_=]{60,70}''' +entropy = 3.5 +keywords = ["atlasv1"] + +[[rules]] +id = "hashicorp-tf-password" +description = "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches." +regex = '''(?i)[\w.-]{0,50}?(?:administrator_login_password|password)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}("[a-z0-9=_\-]{8,20}")(?:[\x60'"\s;]|\\[nr]|$)''' +path = '''(?i)\.(?:tf|hcl)$''' +entropy = 2 +keywords = [ + "administrator_login_password", + "password", +] + +[[rules]] +id = "heroku-api-key" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''(?i)[\w.-]{0,50}?(?:heroku)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["heroku"] + +[[rules]] +id = "heroku-api-key-v2" +description = "Detected a Heroku API Key, potentially compromising cloud application deployments and operational security." +regex = '''\b((HRKU-AA[0-9a-zA-Z_-]{58}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hrku-aa"] + +[[rules]] +id = "hubspot-api-key" +description = "Found a HubSpot API Token, posing a risk to CRM data integrity and unauthorized marketing operations." +regex = '''(?i)[\w.-]{0,50}?(?:hubspot)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["hubspot"] + +[[rules]] +id = "huggingface-access-token" +description = "Discovered a Hugging Face Access token, which could lead to unauthorized access to AI models and sensitive data." +regex = '''\b(hf_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["hf_"] + +[[rules]] +id = "huggingface-organization-api-token" +description = "Uncovered a Hugging Face Organization API token, potentially compromising AI organization accounts and associated data." +regex = '''\b(api_org_(?i:[a-z]{34}))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["api_org_"] + +[[rules]] +id = "infracost-api-token" +description = "Detected an Infracost API Token, risking unauthorized access to cloud cost estimation tools and financial data." +regex = '''\b(ico-[a-zA-Z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ico-"] + +[[rules]] +id = "intercom-api-key" +description = "Identified an Intercom API Token, which could compromise customer communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:intercom)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{60})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["intercom"] + +[[rules]] +id = "intra42-client-secret" +description = "Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data." +regex = '''\b(s-s4t2(?:ud|af)-(?i)[abcdef0123456789]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "intra", + "s-s4t2ud-", + "s-s4t2af-", +] + +[[rules]] +id = "jfrog-api-key" +description = "Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{73})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jfrog-identity-token" +description = "Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts." +regex = '''(?i)[\w.-]{0,50}?(?:jfrog|artifactory|bintray|xray)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "jfrog", + "artifactory", + "bintray", + "xray", +] + +[[rules]] +id = "jwt" +description = "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data." +regex = '''\b(ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["ey"] + +[[rules]] +id = "jwt-base64" +description = "Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information." +regex = '''\bZXlK(?:(?PaGJHY2lPaU)|(?PaGNIVWlPaU)|(?PaGNIWWlPaU)|(?PaGRXUWlPaU)|(?PaU5qUWlP)|(?PamNtbDBJanBi)|(?PamRIa2lPaU)|(?PbGNHc2lPbn)|(?PbGJtTWlPaU)|(?PcWEzVWlPaU)|(?PcWQyc2lPb)|(?PcGMzTWlPaU)|(?PcGRpSTZJ)|(?PcmFXUWlP)|(?PclpYbGZiM0J6SWpwY)|(?PcmRIa2lPaUp)|(?PdWIyNWpaU0k2)|(?Pd01tTWlP)|(?Pd01uTWlPaU)|(?Pd2NIUWlPaU)|(?PemRXSWlPaU)|(?PemRuUWlP)|(?PMFlXY2lPaU)|(?PMGVYQWlPaUp)|(?PMWNtd2l)|(?PMWMyVWlPaUp)|(?PMlpYSWlPaU)|(?PMlpYSnphVzl1SWpv)|(?PNElqb2)|(?PNE5XTWlP)|(?PNE5YUWlPaU)|(?PNE5YUWpVekkxTmlJNkl)|(?PNE5YVWlPaU)|(?PNmFYQWlPaU))[a-zA-Z0-9\/\\_+\-\r\n]{40,}={0,2}''' +entropy = 2 +keywords = ["zxlk"] + +[[rules]] +id = "kraken-access-token" +description = "Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security." +regex = '''(?i)[\w.-]{0,50}?(?:kraken)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9\/=_\+\-]{80,90})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kraken"] + +[[rules]] +id = "kubernetes-secret-yaml" +description = "Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments" +regex = '''(?i)(?:\bkind:[ \t]*["']?\bsecret\b["']?(?s:.){0,200}?\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))|\bdata:(?s:.){0,100}?\s+([\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:["']?[a-z0-9+/]{10,}={0,3}["']?|\{\{[ \t\w"|$:=,.-]+}}|""|''))(?s:.){0,200}?\bkind:[ \t]*["']?\bsecret\b["']?)''' +path = '''(?i)\.ya?ml$''' +keywords = ["secret"] +[[rules.allowlists]] +regexes = [ + '''[\w.-]+:(?:[ \t]*(?:\||>[-+]?)\s+)?[ \t]*(?:\{\{[ \t\w"|$:=,.-]+}}|""|'')''', +] +[[rules.allowlists]] +regexTarget = "match" +regexes = [ + '''(kind:(?s:.)+\n---\n(?s:.)+\bdata:|data:(?s:.)+\n---\n(?s:.)+\bkind:)''', +] + +[[rules]] +id = "kucoin-access-token" +description = "Found a Kucoin Access Token, risking unauthorized access to cryptocurrency exchange services and transactions." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "kucoin-secret-key" +description = "Discovered a Kucoin Secret Key, which could lead to compromised cryptocurrency operations and financial data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:kucoin)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["kucoin"] + +[[rules]] +id = "launchdarkly-access-token" +description = "Uncovered a Launchdarkly Access Token, potentially compromising feature flag management and application functionality." +regex = '''(?i)[\w.-]{0,50}?(?:launchdarkly)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["launchdarkly"] + +[[rules]] +id = "linear-api-key" +description = "Detected a Linear API Token, posing a risk to project management tools and sensitive task data." +regex = '''lin_api_(?i)[a-z0-9]{40}''' +entropy = 2 +keywords = ["lin_api_"] + +[[rules]] +id = "linear-client-secret" +description = "Identified a Linear Client Secret, which may compromise secure integrations and sensitive project management data." +regex = '''(?i)[\w.-]{0,50}?(?:linear)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["linear"] + +[[rules]] +id = "linkedin-client-id" +description = "Found a LinkedIn Client ID, risking unauthorized access to LinkedIn integrations and professional data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{14})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "linkedin-client-secret" +description = "Discovered a LinkedIn Client secret, potentially compromising LinkedIn application integrations and user data." +regex = '''(?i)[\w.-]{0,50}?(?:linked[_-]?in)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "linkedin", + "linked_in", + "linked-in", +] + +[[rules]] +id = "lob-api-key" +description = "Uncovered a Lob API Key, which could lead to unauthorized access to mailing and address verification services." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((live|test)_[a-f0-9]{35})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_", + "live_", +] + +[[rules]] +id = "lob-pub-api-key" +description = "Detected a Lob Publishable API Key, posing a risk of exposing mail and print service integrations." +regex = '''(?i)[\w.-]{0,50}?(?:lob)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((test|live)_pub_[a-f0-9]{31})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "test_pub", + "live_pub", + "_pub", +] + +[[rules]] +id = "looker-client-id" +description = "Found a Looker Client ID, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "looker-client-secret" +description = "Found a Looker Client Secret, risking unauthorized access to a Looker account and exposing sensitive data." +regex = '''(?i)[\w.-]{0,50}?(?:looker)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["looker"] + +[[rules]] +id = "mailchimp-api-key" +description = "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data." +regex = '''(?i)[\w.-]{0,50}?(?:MailchimpSDK.initialize|mailchimp)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{32}-us\d\d)(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailchimp"] + +[[rules]] +id = "mailgun-private-api-token" +description = "Found a Mailgun private API token, risking unauthorized email service operations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(key-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-pub-key" +description = "Discovered a Mailgun public validation key, which could expose email verification processes and associated data." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pubkey-[a-f0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mailgun-signing-key" +description = "Uncovered a Mailgun webhook signing key, potentially compromising email automation and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:mailgun)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mailgun"] + +[[rules]] +id = "mapbox-api-token" +description = "Detected a MapBox API token, posing a risk to geospatial services and sensitive location data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:mapbox)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(pk\.[a-z0-9]{60}\.[a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mapbox"] + +[[rules]] +id = "mattermost-access-token" +description = "Identified a Mattermost Access Token, which may compromise team communication channels and data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:mattermost)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["mattermost"] + +[[rules]] +id = "maxmind-license-key" +description = "Discovered a potential MaxMind license key." +regex = '''\b([A-Za-z0-9]{6}_[A-Za-z0-9]{29}_mmk)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["_mmk"] + +[[rules]] +id = "messagebird-api-token" +description = "Found a MessageBird API token, risking unauthorized access to communication platforms and message data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "messagebird-client-id" +description = "Discovered a MessageBird client ID, potentially compromising API integrations and sensitive communication data." +regex = '''(?i)[\w.-]{0,50}?(?:message[_-]?bird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "messagebird", + "message-bird", + "message_bird", +] + +[[rules]] +id = "microsoft-teams-webhook" +description = "Uncovered a Microsoft Teams Webhook, which could lead to unauthorized access to team collaboration tools and data leaks." +regex = '''https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}@[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}/IncomingWebhook/[a-z0-9]{32}/[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}''' +keywords = [ + "webhook.office.com", + "webhookb2", + "incomingwebhook", +] + +[[rules]] +id = "netlify-access-token" +description = "Detected a Netlify Access Token, potentially compromising web hosting services and site management." +regex = '''(?i)[\w.-]{0,50}?(?:netlify)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{40,46})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["netlify"] + +[[rules]] +id = "new-relic-browser-api-token" +description = "Identified a New Relic ingest browser API token, risking unauthorized access to application performance data and analytics." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRJS-[a-f0-9]{19})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrjs-"] + +[[rules]] +id = "new-relic-insert-key" +description = "Discovered a New Relic insight insert key, compromising data injection into the platform." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRII-[a-z0-9-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrii-"] + +[[rules]] +id = "new-relic-user-api-id" +description = "Found a New Relic user API ID, posing a risk to application monitoring services and data integrity." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "new-relic", + "newrelic", + "new_relic", +] + +[[rules]] +id = "new-relic-user-api-key" +description = "Discovered a New Relic user API Key, which could lead to compromised application insights and performance monitoring." +regex = '''(?i)[\w.-]{0,50}?(?:new-relic|newrelic|new_relic)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(NRAK-[a-z0-9]{27})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["nrak"] + +[[rules]] +id = "notion-api-token" +description = "Notion API token" +regex = '''\b(ntn_[0-9]{11}[A-Za-z0-9]{32}[A-Za-z0-9]{3})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["ntn_"] + +[[rules]] +id = "npm-access-token" +description = "Uncovered an npm access token, potentially compromising package management and code repository access." +regex = '''(?i)\b(npm_[a-z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["npm_"] + +[[rules]] +id = "nuget-config-password" +description = "Identified a password within a Nuget config file, potentially compromising package management access." +regex = '''(?i)''' +path = '''(?i)nuget\.config$''' +entropy = 1 +keywords = ["|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9=_\-]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "nytimes", + "new-york-times", + "newyorktimes", +] + +[[rules]] +id = "octopus-deploy-api-key" +description = "Discovered a potential Octopus Deploy API key, risking application deployments and operational security." +regex = '''\b(API-[A-Z0-9]{26})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["api-"] + +[[rules]] +id = "okta-access-token" +description = "Identified an Okta Access Token, which may compromise identity management services and user authentication data." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Oo]kta|OKTA))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(00[\w=\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["okta"] + +[[rules]] +id = "openai-api-key" +description = "Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''\b(sk-(?:proj|svcacct|admin)-(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})T3BlbkFJ(?:[A-Za-z0-9_-]{74}|[A-Za-z0-9_-]{58})\b|sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["t3blbkfj"] + +[[rules]] +id = "openshift-user-token" +description = "Found an OpenShift user token, potentially compromising an OpenShift/Kubernetes cluster." +regex = '''\b(sha256~[\w-]{43})(?:[^\w-]|\z)''' +entropy = 3.5 +keywords = ["sha256~"] + +[[rules]] +id = "perplexity-api-key" +description = "Detected a Perplexity API key, which could lead to unauthorized access to Perplexity AI services and data exposure." +regex = '''\b(pplx-[a-zA-Z0-9]{48})(?:[\x60'"\s;]|\\[nr]|$|\b)''' +entropy = 4 +keywords = ["pplx-"] + +[[rules]] +id = "pkcs12-file" +description = "Found a PKCS #12 file, which commonly contain bundled private keys." +path = '''(?i)(?:^|\/)[^\/]+\.p(?:12|fx)$''' + +[[rules]] +id = "plaid-api-token" +description = "Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(access-(?:sandbox|development|production)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["plaid"] + +[[rules]] +id = "plaid-client-id" +description = "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{24})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "plaid-secret-key" +description = "Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data." +regex = '''(?i)[\w.-]{0,50}?(?:plaid)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["plaid"] + +[[rules]] +id = "planetscale-api-token" +description = "Identified a PlanetScale API token, potentially compromising database management and operations." +regex = '''\b(pscale_tkn_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_tkn_"] + +[[rules]] +id = "planetscale-oauth-token" +description = "Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity." +regex = '''\b(pscale_oauth_[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_oauth_"] + +[[rules]] +id = "planetscale-password" +description = "Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches." +regex = '''(?i)\b(pscale_pw_(?i)[\w=\.-]{32,64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pscale_pw_"] + +[[rules]] +id = "postman-api-token" +description = "Uncovered a Postman API token, potentially compromising API testing and development workflows." +regex = '''\b(PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["pmak-"] + +[[rules]] +id = "prefect-api-token" +description = "Detected a Prefect API token, risking unauthorized access to workflow management and automation services." +regex = '''\b(pnu_[a-zA-Z0-9]{36})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pnu_"] + +[[rules]] +id = "private-key" +description = "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption." +regex = '''(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----''' +keywords = ["-----begin"] + +[[rules]] +id = "privateai-api-token" +description = "Identified a PrivateAI Token, posing a risk of unauthorized access to AI services and data manipulation." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:private[_-]?ai)(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{32})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "privateai", + "private_ai", + "private-ai", +] + +[[rules]] +id = "pulumi-api-token" +description = "Found a Pulumi API token, posing a risk to infrastructure as code services and cloud resource management." +regex = '''\b(pul-[a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["pul-"] + +[[rules]] +id = "pypi-upload-token" +description = "Discovered a PyPI upload token, potentially compromising Python package distribution and repository integrity." +regex = '''pypi-AgEIcHlwaS5vcmc[\w-]{50,1000}''' +entropy = 3 +keywords = ["pypi-ageichlwas5vcmc"] + +[[rules]] +id = "rapidapi-access-token" +description = "Uncovered a RapidAPI Access Token, which could lead to unauthorized access to various APIs and data services." +regex = '''(?i)[\w.-]{0,50}?(?:rapidapi)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9_-]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["rapidapi"] + +[[rules]] +id = "readme-api-token" +description = "Detected a Readme API token, risking unauthorized documentation management and content exposure." +regex = '''\b(rdme_[a-z0-9]{70})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rdme_"] + +[[rules]] +id = "rubygems-api-token" +description = "Identified a Rubygem API token, potentially compromising Ruby library distribution and package management." +regex = '''\b(rubygems_[a-f0-9]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["rubygems_"] + +[[rules]] +id = "scalingo-api-token" +description = "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security." +regex = '''\b(tk-us-[\w-]{48})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["tk-us-"] + +[[rules]] +id = "sendbird-access-id" +description = "Discovered a Sendbird Access ID, which could compromise chat and messaging platform integrations." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendbird-access-token" +description = "Uncovered a Sendbird Access Token, potentially risking unauthorized access to communication services and user data." +regex = '''(?i)[\w.-]{0,50}?(?:sendbird)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["sendbird"] + +[[rules]] +id = "sendgrid-api-token" +description = "Detected a SendGrid API token, posing a risk of unauthorized email service operations and data exposure." +regex = '''\b(SG\.(?i)[a-z0-9=_\-\.]{66})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["sg."] + +[[rules]] +id = "sendinblue-api-token" +description = "Identified a Sendinblue API token, which may compromise email marketing services and subscriber data privacy." +regex = '''\b(xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["xkeysib-"] + +[[rules]] +id = "sentry-access-token" +description = "Found a Sentry.io Access Token (old format), risking unauthorized access to error tracking services and sensitive application data." +regex = '''(?i)[\w.-]{0,50}?(?:sentry)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sentry"] + +[[rules]] +id = "sentry-org-token" +description = "Found a Sentry.io Organization Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\bsntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}(?:[^a-zA-Z0-9+/]|\z)''' +entropy = 4.5 +keywords = ["sntrys_eyjpyxqio"] + +[[rules]] +id = "sentry-user-token" +description = "Found a Sentry.io User Token, risking unauthorized access to error tracking services and sensitive application data." +regex = '''\b(sntryu_[a-f0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = ["sntryu_"] + +[[rules]] +id = "settlemint-application-access-token" +description = "Found a Settlemint Application Access Token." +regex = '''\b(sm_aat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_aat"] + +[[rules]] +id = "settlemint-personal-access-token" +description = "Found a Settlemint Personal Access Token." +regex = '''\b(sm_pat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_pat"] + +[[rules]] +id = "settlemint-service-access-token" +description = "Found a Settlemint Service Access Token." +regex = '''\b(sm_sat_[a-zA-Z0-9]{16})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sm_sat"] + +[[rules]] +id = "shippo-api-token" +description = "Discovered a Shippo API token, potentially compromising shipping services and customer order data." +regex = '''\b(shippo_(?:live|test)_[a-fA-F0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = ["shippo_"] + +[[rules]] +id = "shopify-access-token" +description = "Uncovered a Shopify access token, which could lead to unauthorized e-commerce platform access and data breaches." +regex = '''shpat_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpat_"] + +[[rules]] +id = "shopify-custom-access-token" +description = "Detected a Shopify custom access token, potentially compromising custom app integrations and e-commerce data security." +regex = '''shpca_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpca_"] + +[[rules]] +id = "shopify-private-app-access-token" +description = "Identified a Shopify private app access token, risking unauthorized access to private app data and store operations." +regex = '''shppa_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shppa_"] + +[[rules]] +id = "shopify-shared-secret" +description = "Found a Shopify shared secret, posing a risk to application authentication and e-commerce platform security." +regex = '''shpss_[a-fA-F0-9]{32}''' +entropy = 2 +keywords = ["shpss_"] + +[[rules]] +id = "sidekiq-secret" +description = "Discovered a Sidekiq Secret, which could lead to compromised background job processing and application data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:BUNDLE_ENTERPRISE__CONTRIBSYS__COM|BUNDLE_GEMS__CONTRIBSYS__COM)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-f0-9]{8}:[a-f0-9]{8})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = [ + "bundle_enterprise__contribsys__com", + "bundle_gems__contribsys__com", +] + +[[rules]] +id = "sidekiq-sensitive-url" +description = "Uncovered a Sidekiq Sensitive URL, potentially exposing internal job queues and sensitive operation details." +regex = '''(?i)\bhttps?://([a-f0-9]{8}:[a-f0-9]{8})@(?:gems.contribsys.com|enterprise.contribsys.com)(?:[\/|\#|\?|:]|$)''' +keywords = [ + "gems.contribsys.com", + "enterprise.contribsys.com", +] + +[[rules]] +id = "slack-app-token" +description = "Detected a Slack App-level token, risking unauthorized access to Slack applications and workspace data." +regex = '''(?i)xapp-\d-[A-Z0-9]+-\d+-[a-z0-9]+''' +entropy = 2 +keywords = ["xapp"] + +[[rules]] +id = "slack-bot-token" +description = "Identified a Slack Bot token, which may compromise bot integrations and communication channel security." +regex = '''xoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*''' +entropy = 3 +keywords = ["xoxb"] + +[[rules]] +id = "slack-config-access-token" +description = "Found a Slack Configuration access token, posing a risk to workspace configuration and sensitive data access." +regex = '''(?i)xoxe.xox[bp]-\d-[A-Z0-9]{163,166}''' +entropy = 2 +keywords = [ + "xoxe.xoxb-", + "xoxe.xoxp-", +] + +[[rules]] +id = "slack-config-refresh-token" +description = "Discovered a Slack Configuration refresh token, potentially allowing prolonged unauthorized access to configuration settings." +regex = '''(?i)xoxe-\d-[A-Z0-9]{146}''' +entropy = 2 +keywords = ["xoxe-"] + +[[rules]] +id = "slack-legacy-bot-token" +description = "Uncovered a Slack Legacy bot token, which could lead to compromised legacy bot operations and data exposure." +regex = '''xoxb-[0-9]{8,14}-[a-zA-Z0-9]{18,26}''' +entropy = 2 +keywords = ["xoxb"] + +[[rules]] +id = "slack-legacy-token" +description = "Detected a Slack Legacy token, risking unauthorized access to older Slack integrations and user data." +regex = '''xox[os]-\d+-\d+-\d+-[a-fA-F\d]+''' +entropy = 2 +keywords = [ + "xoxo", + "xoxs", +] + +[[rules]] +id = "slack-legacy-workspace-token" +description = "Identified a Slack Legacy Workspace token, potentially compromising access to workspace data and legacy features." +regex = '''xox[ar]-(?:\d-)?[0-9a-zA-Z]{8,48}''' +entropy = 2 +keywords = [ + "xoxa", + "xoxr", +] + +[[rules]] +id = "slack-user-token" +description = "Found a Slack User token, posing a risk of unauthorized user impersonation and data access within Slack workspaces." +regex = '''xox[pe](?:-[0-9]{10,13}){3}-[a-zA-Z0-9-]{28,34}''' +entropy = 2 +keywords = [ + "xoxp-", + "xoxe-", +] + +[[rules]] +id = "slack-webhook-url" +description = "Discovered a Slack Webhook, which could lead to unauthorized message posting and data leakage in Slack channels." +regex = '''(?:https?://)?hooks.slack.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}''' +keywords = ["hooks.slack.com"] + +[[rules]] +id = "snyk-api-token" +description = "Uncovered a Snyk API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["snyk"] + +[[rules]] +id = "sonar-api-token" +description = "Uncovered a Sonar API token, potentially compromising software vulnerability scanning and code security." +regex = '''(?i)[\w.-]{0,50}?(?:sonar[_.-]?(login|token))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}((?:squ_|sqp_|sqa_)?[a-z0-9=_\-]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +secretGroup = 2 +keywords = ["sonar"] + +[[rules]] +id = "sourcegraph-access-token" +description = "Sourcegraph is a code search and navigation engine." +regex = '''(?i)\b(\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b)(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = [ + "sgp_", + "sourcegraph", +] + +[[rules]] +id = "square-access-token" +description = "Detected a Square Access Token, risking unauthorized payment processing and financial transaction exposure." +regex = '''\b((?:EAAA|sq0atp-)[\w-]{22,60})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sq0atp-", + "eaaa", +] + +[[rules]] +id = "squarespace-access-token" +description = "Identified a Squarespace Access Token, which may compromise website management and content control on Squarespace." +regex = '''(?i)[\w.-]{0,50}?(?:squarespace)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["squarespace"] + +[[rules]] +id = "stripe-access-token" +description = "Found a Stripe Access Token, posing a risk to payment processing services and sensitive financial data." +regex = '''\b((?:sk|rk)_(?:test|live|prod)_[a-zA-Z0-9]{10,99})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 2 +keywords = [ + "sk_test", + "sk_live", + "sk_prod", + "rk_test", + "rk_live", + "rk_prod", +] + +[[rules]] +id = "sumologic-access-id" +description = "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity." +regex = '''[\w.-]{0,50}?(?i:[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3})(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(su[a-zA-Z0-9]{12})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "sumologic-access-token" +description = "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights." +regex = '''(?i)[\w.-]{0,50}?(?:(?-i:[Ss]umo|SUMO))(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{64})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3 +keywords = ["sumo"] + +[[rules]] +id = "telegram-bot-api-token" +description = "Detected a Telegram Bot API Token, risking unauthorized bot operations and message interception on Telegram." +regex = '''(?i)[\w.-]{0,50}?(?:telegr)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{5,16}:(?-i:A)[a-z0-9_\-]{34})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["telegr"] + +[[rules]] +id = "travisci-access-token" +description = "Identified a Travis CI Access Token, potentially compromising continuous integration services and codebase security." +regex = '''(?i)[\w.-]{0,50}?(?:travis)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{22})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["travis"] + +[[rules]] +id = "twilio-api-key" +description = "Found a Twilio API Key, posing a risk to communication services and sensitive customer interaction data." +regex = '''SK[0-9a-fA-F]{32}''' +entropy = 3 +keywords = ["sk"] + +[[rules]] +id = "twitch-api-token" +description = "Discovered a Twitch API token, which could compromise streaming services and account integrations." +regex = '''(?i)[\w.-]{0,50}?(?:twitch)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{30})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitch"] + +[[rules]] +id = "twitter-access-secret" +description = "Uncovered a Twitter Access Secret, potentially risking unauthorized Twitter integrations and data breaches." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{45})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-access-token" +description = "Detected a Twitter Access Token, posing a risk of unauthorized account operations and social media data exposure." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([0-9]{15,25}-[a-zA-Z0-9]{20,40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-key" +description = "Identified a Twitter API Key, which may compromise Twitter application integrations and user data security." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{25})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-api-secret" +description = "Found a Twitter API Secret, risking the security of Twitter app integrations and sensitive data access." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{50})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "twitter-bearer-token" +description = "Discovered a Twitter Bearer Token, potentially compromising API access and data retrieval from Twitter." +regex = '''(?i)[\w.-]{0,50}?(?:twitter)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(A{22}[a-zA-Z0-9%]{80,100})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["twitter"] + +[[rules]] +id = "typeform-api-token" +description = "Uncovered a Typeform API token, which could lead to unauthorized survey management and data collection." +regex = '''(?i)[\w.-]{0,50}?(?:typeform)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(tfp_[a-z0-9\-_\.=]{59})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["tfp_"] + +[[rules]] +id = "vault-batch-token" +description = "Detected a Vault Batch Token, risking unauthorized access to secret management services and sensitive data." +regex = '''\b(hvb\.[\w-]{138,300})(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 4 +keywords = ["hvb."] + +[[rules]] +id = "vault-service-token" +description = "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials." +regex = '''\b((?:hvs\.[\w-]{90,120}|s\.(?i:[a-z0-9]{24})))(?:[\x60'"\s;]|\\[nr]|$)''' +entropy = 3.5 +keywords = [ + "hvs.", + "s.", +] +[[rules.allowlists]] +regexes = [ + '''s\.[A-Za-z]{24}''', +] + +[[rules]] +id = "yandex-access-token" +description = "Found a Yandex Access Token, posing a risk to Yandex service integrations and user data privacy." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(t1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-api-key" +description = "Discovered a Yandex API Key, which could lead to unauthorized access to Yandex services and data manipulation." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(AQVN[A-Za-z0-9_\-]{35,38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "yandex-aws-access-token" +description = "Uncovered a Yandex AWS Access Token, potentially compromising cloud resource access and data security on Yandex Cloud." +regex = '''(?i)[\w.-]{0,50}?(?:yandex)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}(YC[a-zA-Z0-9_\-]{38})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["yandex"] + +[[rules]] +id = "zendesk-secret-key" +description = "Detected a Zendesk Secret Key, risking unauthorized access to customer support services and sensitive ticketing data." +regex = '''(?i)[\w.-]{0,50}?(?:zendesk)(?:[ \t\w.-]{0,20})[\s'"]{0,3}(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)[\x60'"\s=]{0,5}([a-z0-9]{40})(?:[\x60'"\s;]|\\[nr]|$)''' +keywords = ["zendesk"] + diff --git a/crates/toolpath-redact/src/internal/rules.rs b/crates/toolpath-redact/src/internal/rules.rs index bd12c88f..602cfab4 100644 --- a/crates/toolpath-redact/src/internal/rules.rs +++ b/crates/toolpath-redact/src/internal/rules.rs @@ -1 +1,128 @@ -//! Rule loading from the vendored ruleset. Implemented in T3. +//! Rule loading from the vendored ruleset. +//! +//! Vendored from `gitleaks/gitleaks`, `config/gitleaks.toml`, commit +//! `b58d3f102cf3a2c84cb7f923d05c25c9b1aed84b` (2026-07-22). Gitleaks is +//! MIT-licensed (https://github.com/gitleaks/gitleaks/blob/master/LICENSE). +//! `gitleaks.toml` itself is kept byte-verbatim so it can be diffed against +//! upstream; do not hand-edit it. + +use serde::Deserialize; + +const RAW_TOML: &str = include_str!("gitleaks.toml"); + +/// One rule, with its allow-regexes already compiled. `regex` stays a +/// `String` (not a compiled `Regex`) because [`load_rules`] is a pure +/// parse step exercised on its own by the compile-guard test; the caller +/// compiles it. +pub struct Rule { + pub id: String, + pub regex: String, + pub entropy: Option, + pub keywords: Vec, + /// Gitleaks' per-rule `[[rules.allowlists]] regexes` - documented + /// exceptions (e.g. AWS's own `...EXAMPLE` key) checked against the + /// matched secret text. Compiled best-effort: an allowlist entry that + /// fails under `regex` is dropped rather than failing the whole rule, + /// since it is a false-positive refinement, not the detection itself. + pub allow: Vec, +} + +#[derive(Deserialize)] +struct RulesFile { + rules: Vec, +} + +#[derive(Deserialize)] +struct RawRule { + id: String, + regex: String, + entropy: Option, + #[serde(default)] + keywords: Vec, + #[serde(default)] + allowlists: Vec, +} + +#[derive(Deserialize)] +struct RawAllowlist { + #[serde(default)] + regexes: Vec, +} + +/// Rust's `regex` crate (RE2-derived, no backreferences/lookaround) +/// rejects a minority of gitleaks patterns written for Go's RE2 dialect. +/// Confirmed by `every_vendored_rule_compiles_under_rust_regex` - see that +/// test for the failure each id below hit. +pub const EXCLUDED_RULE_IDS: &[(&str, &str)] = &[]; + +/// Hand-written rules filling gaps the vendored ruleset leaves open for +/// this crate's purposes: gitleaks matches a PEM block only with its full +/// closing footer, and matches a JWT only above a claim-length floor that +/// misses short tokens; URI-embedded credentials aren't a gitleaks rule at +/// all (it scans files, not structured connection strings). +fn supplemental_rules() -> Vec { + vec![ + Rule { + id: "pem-private-key-header".to_string(), + regex: r"-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----".to_string(), + entropy: None, + keywords: vec!["-----begin".to_string()], + allow: Vec::new(), + }, + Rule { + id: "jwt-compact".to_string(), + regex: r"\bey[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{2,}\b".to_string(), + entropy: None, + keywords: vec!["ey".to_string()], + allow: Vec::new(), + }, + Rule { + id: "uri-credential".to_string(), + regex: r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s:@/]+:([^\s:@/]+)@[^\s/]+".to_string(), + entropy: None, + keywords: vec!["://".to_string()], + allow: Vec::new(), + }, + ] +} + +/// Parses the vendored TOML plus [`supplemental_rules`], dropping any rule +/// on [`EXCLUDED_RULE_IDS`]. Pure and deterministic - no I/O beyond the +/// `include_str!` baked in at compile time. +pub fn load_rules() -> Vec { + let parsed: RulesFile = toml::from_str(RAW_TOML).expect("vendored gitleaks.toml must parse"); + let mut rules: Vec = parsed + .rules + .into_iter() + .filter(|r| !EXCLUDED_RULE_IDS.iter().any(|(id, _)| *id == r.id)) + .map(|r| Rule { + id: r.id, + regex: r.regex, + entropy: r.entropy, + keywords: r.keywords, + allow: r + .allowlists + .iter() + .flat_map(|a| a.regexes.iter()) + .filter_map(|re| regex::Regex::new(re).ok()) + .collect(), + }) + .collect(); + rules.extend(supplemental_rules()); + rules +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_vendored_rule_compiles_under_rust_regex() { + let rules = load_rules(); + assert!(rules.len() >= 200, "expected the full ruleset, got {}", rules.len()); + for r in &rules { + regex::Regex::new(&r.regex) + .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); + } + } +} diff --git a/crates/toolpath-redact/src/lib.rs b/crates/toolpath-redact/src/lib.rs index f7b7cc49..22460630 100644 --- a/crates/toolpath-redact/src/lib.rs +++ b/crates/toolpath-redact/src/lib.rs @@ -9,8 +9,12 @@ pub mod surface; pub mod transform; pub use apply::apply; -pub use detect::{Candidate, Context, Detector, DetectorSet, Egress, FieldShape, Finding}; -pub use plan::{Action, Plan, PlanFinding, RedactionPolicy}; +pub use detect::{ + Candidate, Context, Detector, DetectorSet, Egress, FieldShape, Finding, FixedDetector, +}; +pub use plan::{ + Action, Cmp, Decision, Plan, PlanFinding, Predicate, RedactionPolicy, parse_predicate, +}; pub use surface::{Surface, surfaces}; pub use transform::{Fingerprint, Transform}; @@ -28,6 +32,10 @@ pub enum RedactError { BadPointer(String), #[error("bad predicate: {0}")] BadPredicate(String), + /// A third-party detector's own failure. Carries a message rather than + /// a source error so the crate can keep advertising no filesystem. + #[error("detector {0} failed: {1}")] + DetectorFailed(String, String), #[error(transparent)] Json(#[from] serde_json::Error), } diff --git a/crates/toolpath-redact/src/transform.rs b/crates/toolpath-redact/src/transform.rs index cead61f5..7e0f924c 100644 --- a/crates/toolpath-redact/src/transform.rs +++ b/crates/toolpath-redact/src/transform.rs @@ -1,7 +1,9 @@ //! What a redacted span is replaced with. -/// How a detected span is rewritten. Every variant except `Partial` is -/// guaranteed to emit nothing derived from the value's characters. +use std::ops::Range; + +/// How a detected span is rewritten. No variant except `Partial` emits a +/// substring of the value; `Mask` and `Partial` still publish its length. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum Transform { @@ -17,18 +19,291 @@ pub enum Transform { /// A short stable handle for a secret value, used to correlate the same /// secret across occurrences without publishing it. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Fingerprint(pub String); impl Fingerprint { /// Keyed, never a bare hash: a hash of a low-entropy secret is a /// dictionary attack away from the secret (EDPB 01/2025 para 88). - pub fn new(_key: &[u8], _value: &str) -> Self { - todo!("T4") + pub fn new(key: &[u8], value: &str) -> Self { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let mut mac = >::new_from_slice(key).expect("HMAC accepts any key length"); + mac.update(value.as_bytes()); + Fingerprint(hex(&mac.finalize().into_bytes())[..6].to_string()) + } +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + write!(s, "{b:02x}").unwrap(); + } + s +} + +/// Rewrite one value under a chosen `Transform`. `fp` is the value's own +/// fingerprint (callers compute it once per value and reuse it across +/// occurrences, since `Hash`/`Marker` both need it). +pub fn apply_transform(t: Transform, rule: &str, value: &str, fp: &Fingerprint) -> String { + match t { + Transform::Marker => format!("[REDACTED:{rule}:{}]", fp.0), + Transform::Remove => String::new(), + Transform::Hash => fp.0.clone(), + Transform::Mask => "\u{2588}".repeat(value.chars().count()), + Transform::Partial => { + let n = value.chars().count(); + if n > 10 { + let head: String = value.chars().take(4).collect(); + let tail: String = value.chars().skip(n - 4).collect(); + format!("{head}\u{2026}{tail}") + } else { + "\u{2588}".repeat(n) + } + } } } -pub trait Transformer: Send + Sync { - fn id(&self) -> &'static str; - fn replace(&self, rule: &str, value: &str, fp: &Fingerprint) -> String; +/// Splice every `(span, replacement)` edit into `text` in one pass. +/// +/// Sorted descending by start so earlier offsets stay valid as later spans +/// are spliced out. Detector output reaching this point is already +/// overlap-free (see `detect::normalise`); an edit whose span no longer +/// lands on a valid boundary of the string as spliced so far is dropped +/// rather than risking a panic or corrupt UTF-8, so a normalisation bug +/// upstream degrades safely instead of crashing. +pub fn apply_spans_desc(text: &str, edits: &mut [(Range, String)]) -> String { + edits.sort_by_key(|(span, _)| std::cmp::Reverse(span.start)); + let mut out = text.to_string(); + for (span, repl) in edits.iter() { + let lands_cleanly = span.start <= span.end + && span.end <= out.len() + && out.is_char_boundary(span.start) + && out.is_char_boundary(span.end); + if lands_cleanly { + out.replace_range(span.clone(), repl); + } + } + out +} + +/// Precedence for which `Transform` applies to a finding: a per-finding +/// choice beats a per-rule default, which beats the global default. +pub fn resolve_transform( + cfg: &crate::RedactConfig, + rule: &str, + per_finding: Option, +) -> Transform { + per_finding + .or_else(|| { + cfg.mode_for + .iter() + .find(|(r, _)| r == rule) + .map(|(_, t)| *t) + }) + .unwrap_or(cfg.mode) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn fp() -> Fingerprint { + Fingerprint::new(b"test-key", "fixture-value") + } + + fn cfg_with(mode: Transform, mode_for: Vec<(String, Transform)>) -> crate::RedactConfig { + crate::RedactConfig { + threshold: 0.8, + mode, + mode_for, + key: b"test-key".to_vec(), + now: chrono::Utc.with_ymd_and_hms(2026, 7, 30, 0, 0, 0).unwrap(), + drop_signatures: false, + reveal: false, + } + } + + fn resolve(cfg: &crate::RedactConfig, rule: &str, per_finding: Option) -> Transform { + resolve_transform(cfg, rule, per_finding) + } + + fn apply_spans(text: &str, spans: &[Range]) -> String { + let mut edits: Vec<_> = spans.iter().cloned().map(|s| (s, String::new())).collect(); + apply_spans_desc(text, &mut edits) + } + + fn apply_one_by_one_from_right(text: &str, spans: &[Range]) -> String { + let mut sorted = spans.to_vec(); + sorted.sort_by_key(|s| std::cmp::Reverse(s.start)); + let mut out = text.to_string(); + for span in sorted { + out.replace_range(span, ""); + } + out + } + + #[test] + fn fingerprint_is_deterministic() { + let k = b"test-key"; + assert_eq!(Fingerprint::new(k, "abc"), Fingerprint::new(k, "abc")); + assert_ne!( + Fingerprint::new(k, "abc"), + Fingerprint::new(b"other", "abc") + ); + } + + #[test] + fn mask_preserves_character_count_not_byte_count() { + let out = apply_transform(Transform::Mask, "rule", "héllo", &fp()); + assert_eq!(out.chars().count(), 5); + } + + #[test] + fn partial_falls_back_to_mask_below_the_floor() { + let out = apply_transform(Transform::Partial, "rule", "short", &fp()); + assert!(!out.contains("short")); + assert_eq!(out.chars().count(), 5); + } + + #[test] + fn only_partial_ever_emits_a_substring_of_its_input() { + let value = "AKIAIOSFODNN7REALKEY"; + for t in [ + Transform::Marker, + Transform::Remove, + Transform::Hash, + Transform::Mask, + ] { + let out = apply_transform(t, "aws-access-key-id", value, &fp()); + for w in 4..=value.len() { + for s in value.as_bytes().windows(w) { + let sub = std::str::from_utf8(s).unwrap(); + assert!(!out.contains(sub), "{t:?} leaked {sub:?}"); + } + } + } + } + + // `Hash`'s output alphabet (0-9a-f) overlaps the alphabet of + // "AKIAIOSFODNN7REALKEY" above, so the check there is astronomically + // unlikely to false-positive but not provably impossible. Pin the + // impossible case: a value with no hex characters at all cannot share + // a substring with a hex string, by construction. + #[test] + fn hash_output_cannot_share_a_substring_with_a_non_hex_value() { + let value = "ZZZZ-QQQQ-WWWW-XXXX-YYYY-KKKK-JJJJ-VVVV"; + assert!(!value.chars().any(|c| c.is_ascii_hexdigit())); + let out = apply_transform(Transform::Hash, "rule", value, &fp()); + assert!(out.chars().all(|c| c.is_ascii_hexdigit())); + for w in 4..=value.len() { + for s in value.as_bytes().windows(w) { + let sub = std::str::from_utf8(s).unwrap(); + assert!(!out.contains(sub)); + } + } + } + + #[test] + fn right_to_left_application_matches_one_at_a_time() { + let text = "aaa BBB ccc DDD eee"; + let spans = vec![4..7, 12..15]; + assert_eq!( + apply_spans(text, &spans), + apply_one_by_one_from_right(text, &spans) + ); + } + + #[test] + fn per_rule_override_beats_global_and_per_finding_beats_both() { + let cfg = cfg_with(Transform::Marker, vec![("us-ssn".into(), Transform::Mask)]); + assert_eq!(resolve(&cfg, "us-ssn", None), Transform::Mask); + assert_eq!(resolve(&cfg, "aws-access-key-id", None), Transform::Marker); + assert_eq!( + resolve(&cfg, "us-ssn", Some(Transform::Remove)), + Transform::Remove + ); + } + + #[test] + fn fingerprint_accepts_an_empty_key() { + let out = Fingerprint::new(b"", "abc"); + assert_eq!(out.0.len(), 6); + } + + #[test] + fn fingerprint_accepts_a_key_longer_than_the_sha256_block_size() { + let long_key = vec![0x42u8; 100]; // SHA-256's block size is 64 bytes. + let out = Fingerprint::new(&long_key, "abc"); + assert_eq!(out.0.len(), 6); + } + + #[test] + fn remove_then_apply_spans_desc_two_adjacent_spans() { + let text = "aaaBBBCCCeee"; + let mut edits = vec![ + (3..6, apply_transform(Transform::Remove, "r", "BBB", &fp())), + (6..9, apply_transform(Transform::Remove, "r", "CCC", &fp())), + ]; + assert_eq!(apply_spans_desc(text, &mut edits), "aaaeee"); + } + + #[test] + fn remove_then_apply_spans_desc_two_overlapping_spans_drops_the_invalidated_one() { + // Once the rightmost span (3..8) is spliced, the leftmost span's + // recorded end (5) no longer lands inside the shortened string, so + // it is dropped instead of panicking or corrupting the result. + let text = "abcdefgh"; + let mut edits = vec![ + ( + 3..8, + apply_transform(Transform::Remove, "r", "defgh", &fp()), + ), + ( + 0..5, + apply_transform(Transform::Remove, "r", "abcde", &fp()), + ), + ]; + let out = apply_spans_desc(text, &mut edits); + assert_eq!(out, "abc"); + } + + #[test] + fn apply_spans_desc_multibyte_text_stays_on_char_boundaries() { + let text = "héllo wörld"; + let e = text.find('é').unwrap(); + let o = text.find('ö').unwrap(); + let mut edits = vec![ + (o..o + 'ö'.len_utf8(), "O".to_string()), + (e..e + 'é'.len_utf8(), "E".to_string()), + ]; + assert_eq!(apply_spans_desc(text, &mut edits), "hEllo wOrld"); + } + + #[test] + fn mask_of_empty_value_is_empty() { + assert_eq!(apply_transform(Transform::Mask, "rule", "", &fp()), ""); + } + + #[test] + fn mask_of_single_char_value_is_one_block() { + assert_eq!( + apply_transform(Transform::Mask, "rule", "x", &fp()), + "\u{2588}" + ); + } + + #[test] + fn partial_at_exactly_ten_chars_falls_back_to_mask() { + let out = apply_transform(Transform::Partial, "rule", "0123456789", &fp()); + assert_eq!(out, "\u{2588}".repeat(10)); + } + + #[test] + fn partial_at_eleven_chars_shows_head_and_tail() { + let out = apply_transform(Transform::Partial, "rule", "01234567890", &fp()); + assert_eq!(out, "0123\u{2026}7890"); + } } diff --git a/scripts/release.sh b/scripts/release.sh index deae1639..92536096 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -34,10 +34,11 @@ set -euo pipefail # toolpath-opencode (depends on toolpath, toolpath-convo) # toolpath-cursor (depends on toolpath, toolpath-convo) # toolpath-pi (depends on toolpath, toolpath-convo) +# toolpath-redact (depends on toolpath) # 3. path-cli (depends on all of the above) # 4. toolpath-cli (deprecated shim that depends on path-cli) -_all_crates=(toolpath pathbase-client toolpath-convo toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi path-cli toolpath-cli) +_all_crates=(toolpath pathbase-client toolpath-convo toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi toolpath-redact path-cli toolpath-cli) _execute=0 _auto_yes="" diff --git a/site/_data/crates.json b/site/_data/crates.json index 99bcad0b..361bbd44 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -79,6 +79,14 @@ "crate": "https://crates.io/crates/toolpath-pi", "role": "Reads Pi (pi.dev) coding-agent session JSONL from `~/.pi/agent/sessions/`, implements `ConversationProvider`, and derives Toolpath `Path` documents via `toolpath-convo`'s shared `derive_path`. Preserves Pi's in-file conversation tree (id/parentId) as a DAG and follows `parentSession` links across files." }, + { + "name": "toolpath-redact", + "version": "0.1.0", + "description": "Detect and redact credentials in Toolpath documents", + "docs": "https://docs.rs/toolpath-redact", + "crate": "https://crates.io/crates/toolpath-redact", + "role": "Engine for the `path p redact` command: walks a Path, names every string field a credential could hide in, runs swappable detectors over them, and rewrites approved matches. Plan-then-apply workflow lets users review findings before application; plans can be decided by predicate, picker, or hand-edited JSON. Detection is a trait, so implementations are pluggable (built-in rule-based detector via vendored gitleaks config, FixedDetector for tests, future harness-time hook)." + }, { "name": "toolpath-cursor", "version": "0.2.0", @@ -113,15 +121,15 @@ }, { "name": "path-cli", - "version": "0.16.0", + "version": "0.17.0", "description": "Unified CLI (binary: path)", "docs": "https://docs.rs/path-cli", "crate": "https://crates.io/crates/path-cli", - "role": "One binary called `path` that ties everything together. Porcelain at the top level (share, resume, query, show, track, auth); plumbing under `path p \u2026` (import, export, cache, list, render, merge, validate). Pathbase round-trip via `p import pathbase` / `p export pathbase` (authed default \u2192 secret pathstash; anon fallback when not logged in)." + "role": "One binary called `path` that ties everything together. Porcelain at the top level (share, resume, query, show, track, auth); plumbing under `path p \u2026` (import, export, cache, list, render, merge, validate, redact). Pathbase round-trip via `p import pathbase` / `p export pathbase` (authed default \u2192 secret pathstash; anon fallback when not logged in)." }, { "name": "toolpath-cli", - "version": "0.16.0", + "version": "0.17.0", "description": "Deprecated alias for path-cli", "docs": "https://docs.rs/toolpath-cli", "crate": "https://crates.io/crates/toolpath-cli", diff --git a/site/pages/crates.md b/site/pages/crates.md index 91b18664..42b7c855 100644 --- a/site/pages/crates.md +++ b/site/pages/crates.md @@ -22,6 +22,7 @@ path-cli (binary: path) +-- toolpath-opencode -> toolpath, toolpath-convo +-- toolpath-pi -> toolpath, toolpath-convo +-- toolpath-cursor -> toolpath, toolpath-convo + +-- toolpath-redact -> toolpath +-- toolpath-dot -> toolpath +-- toolpath-md -> toolpath From 98d51c3a8a59ee7d4a2e41d11d3f1c72b42ddd11 Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 15:37:16 -0400 Subject: [PATCH 4/9] feat(redact): T2 field map, T5 plan machinery, T3 detector (in progress) Applies the T1 review. The one finding worth acting on: overlap resolution is score-blind on length, so a low-scoring container evicts a high-scoring finding nested inside it. Thresholding the OUTPUT of detect_all would then discard the container and take the survivor with it, publishing the secret. Recorded as an ordering constraint on detect_all so plan generation filters before normalisation, not after. Also from that review: three tests covering branches nothing exercised - a span starting mid-codepoint (the existing test only ever hit the end boundary), the leftmost tie-break, and two genuinely distinct detectors contending for the same bytes. The leftmost tie-break is deliberate and now pinned: breaking on rule name would let an untrusted detector win contested bytes by naming its rule `aaa`. Green: detect 23, surface 23, plan 29, transform 16. The internal detector's vendored-ruleset loading is still red - gitleaks ships rules that match on path alone and carry no `regex` field. Co-Authored-By: Claude Opus 5 (1M context) --- crates/path-cli/src/cache.rs | 157 ++- crates/path-cli/src/sync/engine.rs | 4 + crates/toolpath-redact/src/detect.rs | 66 +- crates/toolpath-redact/src/internal/mod.rs | 327 +++++++ crates/toolpath-redact/src/internal/rules.rs | 50 +- crates/toolpath-redact/src/plan.rs | 621 +++++++++++- crates/toolpath-redact/src/surface.rs | 959 ++++++++++++++++++- 7 files changed, 2140 insertions(+), 44 deletions(-) diff --git a/crates/path-cli/src/cache.rs b/crates/path-cli/src/cache.rs index 8203f67e..156fe73b 100644 --- a/crates/path-cli/src/cache.rs +++ b/crates/path-cli/src/cache.rs @@ -12,6 +12,10 @@ use toolpath::v1::Graph; use crate::config::config_dir; const DOCUMENTS_DIR: &str = "documents"; +const REDACT_KEYS_DIR: &str = "redact-keys"; +/// HMAC-SHA256 block-size-independent; 32 bytes is the digest width and +/// well past the point where key length stops mattering. +const REDACT_KEY_LEN: usize = 32; /// An entry surfaced by `list_cached`. #[derive(Debug, Clone)] @@ -27,11 +31,17 @@ pub(crate) fn cache_dir() -> Result { Ok(config_dir()?.join(DOCUMENTS_DIR)) } -/// Path for a given cache id (does not check existence). -pub(crate) fn cache_path(id: &str) -> Result { +/// Reject anything that would escape the directory it names a file in. +fn check_id(id: &str) -> Result<()> { if id.is_empty() || id.contains('/') || id.contains('\\') || id.ends_with(".json") { bail!("invalid cache id: {id:?}"); } + Ok(()) +} + +/// Path for a given cache id (does not check existence). +pub(crate) fn cache_path(id: &str) -> Result { + check_id(id)?; Ok(cache_dir()?.join(format!("{id}.json"))) } @@ -147,6 +157,92 @@ pub(crate) fn remove_cached(id: &str) -> Result<()> { Ok(()) } +// ── redaction keys ───────────────────────────────────────────────── + +/// Per-document fingerprint keys: `$CONFIG_DIR/redact-keys/`. +pub(crate) fn redact_key_dir() -> Result { + Ok(config_dir()?.join(REDACT_KEYS_DIR)) +} + +/// Path of one document's key. Key ids are cache ids, so they are +/// validated the same way. +pub(crate) fn redact_key_path(key_id: &str) -> Result { + check_id(key_id)?; + Ok(redact_key_dir()?.join(key_id)) +} + +/// The stored key, or `None` when this document has no key on disk. +/// A caller re-redacting a document must treat `None` as an error +/// rather than minting a fresh key: new key, new fingerprints, and the +/// document churns on every sync. +pub(crate) fn read_redact_key(key_id: &str) -> Result>> { + let path = redact_key_path(key_id)?; + match std::fs::read(&path) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(anyhow!("read {}: {e}", path.display())), + } +} + +/// The document's key, generating and persisting one on first use. +/// +/// Written with `create_new`, so two redactions racing the same +/// document agree on one key instead of each fingerprinting under its +/// own. +pub(crate) fn load_or_create_redact_key(key_id: &str) -> Result> { + use rand::RngCore; + use std::io::Write; + + if let Some(existing) = read_redact_key(key_id)? { + return Ok(existing); + } + + let dir = redact_key_dir()?; + std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); + } + + let mut key = vec![0u8; REDACT_KEY_LEN]; + rand::rng().fill_bytes(&mut key); + + let path = redact_key_path(key_id)?; + let mut file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + return read_redact_key(key_id)? + .ok_or_else(|| anyhow!("redaction key {key_id} vanished during creation")); + } + Err(e) => return Err(anyhow!("open {}: {e}", path.display())), + }; + file.write_all(&key) + .with_context(|| format!("write {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("chmod 0600 {}", path.display()))?; + } + Ok(key) +} + +/// Drop a document's key. Absent is not an error: `p cache rm` runs +/// against documents that were never redacted. +pub(crate) fn remove_redact_key(key_id: &str) -> Result<()> { + let path = redact_key_path(key_id)?; + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow!("remove {}: {e}", path.display())), + } +} + /// Build a cache id for a given source + inner id. /// /// Sanitizes `/` and other filesystem-unfriendly characters in the @@ -292,6 +388,63 @@ mod tests { }); } + #[test] + fn redact_key_is_created_once_and_reused() { + with_cfg(|_| { + assert!(read_redact_key("claude-abc").unwrap().is_none()); + let first = load_or_create_redact_key("claude-abc").unwrap(); + assert_eq!(first.len(), REDACT_KEY_LEN); + assert_eq!(load_or_create_redact_key("claude-abc").unwrap(), first); + assert_eq!(read_redact_key("claude-abc").unwrap().unwrap(), first); + }); + } + + #[test] + fn redact_keys_differ_per_document() { + with_cfg(|_| { + let a = load_or_create_redact_key("claude-a").unwrap(); + let b = load_or_create_redact_key("claude-b").unwrap(); + assert_ne!(a, b); + }); + } + + #[test] + fn removing_a_redact_key_is_idempotent() { + with_cfg(|_| { + load_or_create_redact_key("claude-abc").unwrap(); + remove_redact_key("claude-abc").unwrap(); + assert!(read_redact_key("claude-abc").unwrap().is_none()); + remove_redact_key("claude-abc").unwrap(); + }); + } + + #[test] + fn redact_key_path_rejects_traversal() { + assert!(redact_key_path("../../etc/passwd").is_err()); + assert!(redact_key_path("").is_err()); + } + + #[cfg(unix)] + #[test] + fn redact_key_is_0600_in_a_0700_dir() { + use std::os::unix::fs::PermissionsExt; + with_cfg(|_| { + load_or_create_redact_key("claude-abc").unwrap(); + let key_mode = std::fs::metadata(redact_key_path("claude-abc").unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(key_mode, 0o600); + let dir_mode = std::fs::metadata(redact_key_dir().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(dir_mode, 0o700); + }); + } + #[test] fn make_id_sanitizes_slashes() { assert_eq!(make_id("git", "main"), "git-main"); diff --git a/crates/path-cli/src/sync/engine.rs b/crates/path-cli/src/sync/engine.rs index e5b0242e..ecb27717 100644 --- a/crates/path-cli/src/sync/engine.rs +++ b/crates/path-cli/src/sync/engine.rs @@ -43,6 +43,10 @@ pub(crate) struct SyncRecord { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) size: Option, pub(crate) synced_at: DateTime, + /// Policy to replay after a re-derive. Rule-based only: individual + /// finding ids cannot be replayed against content that has moved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) redaction: Option, } /// The sync manifest: artifact type (`"claude"`, `"codex"`, …) → diff --git a/crates/toolpath-redact/src/detect.rs b/crates/toolpath-redact/src/detect.rs index b1e6175a..42d21563 100644 --- a/crates/toolpath-redact/src/detect.rs +++ b/crates/toolpath-redact/src/detect.rs @@ -95,6 +95,12 @@ impl DetectorSet { /// Run every detector and reconcile their output into one set of /// non-overlapping, applicable spans. + /// + /// ORDERING CONSTRAINT, and getting it wrong publishes a secret: overlap + /// resolution is score-blind on length, so a low-scoring container evicts + /// a high-scoring finding nested inside it. Threshold BEFORE this runs, + /// never after - thresholding the output lets a container that is about + /// to be discarded take the survivor down with it. pub fn detect_all(&self, c: &Candidate<'_>) -> crate::Result> { let mut raw = Vec::new(); for d in &self.0 { @@ -134,10 +140,14 @@ fn normalise(text: &str, mut findings: Vec) -> Vec { .then(a.detector.cmp(b.detector)) }); - // Best-first, then keep whatever no winner has already claimed. Resolving - // clashes pairwise instead would lose a finding whose only conflict was - // itself evicted later: with A over B, B over C and A disjoint from C, C - // displaces B and A never comes back. + // Best-first: keep whatever no earlier winner has already claimed. + // Resolving clashes pairwise instead loses a finding whose only conflict + // was itself evicted later - with A overlapping B, B overlapping C, and A + // disjoint from C, C displaces B and A never comes back. + // + // Equal length and equal score break on leftmost, not on rule name: a + // rule-name tie-break would let an untrusted detector win contested bytes + // by naming its rule `aaa`. let mut out: Vec = Vec::new(); for f in findings { let claimed = out @@ -313,7 +323,7 @@ mod tests { } #[test] - fn three_way_overlap_keeps_the_end_a_loser_vacated() { + fn three_way_overlap_keeps_the_span_vacated_by_an_evicted_rival() { // A-B overlap, B-C overlap, A-C disjoint. C evicts B, which is the // only thing that ever contested A, so A must survive. let out = detect( @@ -441,4 +451,50 @@ mod tests { let out = s.detect_all(&cand("abcdefgh")).unwrap(); assert_eq!(spans(&out), vec![(2, 5)]); } + + // `drops_mid_codepoint_spans` only ever exercises the `end` boundary + // check, because its span starts at 0. "é" occupies bytes 0..2, so 1 is + // an invalid start and 3 is a valid end. + #[test] + fn drops_span_starting_mid_codepoint() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(1..3, "x", 0.9)]))); + assert!(s.detect_all(&cand("é-tail")).unwrap().is_empty()); + } + + #[test] + fn equal_length_equal_score_leftmost_wins() { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![ + f(4..6, "a", 0.1), + f(3..5, "b", 0.1), + ]))); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(spans(&out), vec![(3, 5)]); + assert_eq!(out[0].rule, "b"); + } + + #[test] + fn distinct_detectors_break_ties_on_detector_id() { + struct Alpha; + impl Detector for Alpha { + fn id(&self) -> &'static str { + "alpha" + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(vec![Finding { + span: 0..4, + rule: "same".into(), + score: 0.5, + detector: "alpha", + }]) + } + } + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(vec![f(0..4, "same", 0.5)]))); + s.push(Box::new(Alpha)); + let out = s.detect_all(&cand("abcdefgh")).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].detector, "alpha"); + } } diff --git a/crates/toolpath-redact/src/internal/mod.rs b/crates/toolpath-redact/src/internal/mod.rs index c6a915d9..aa6a66f2 100644 --- a/crates/toolpath-redact/src/internal/mod.rs +++ b/crates/toolpath-redact/src/internal/mod.rs @@ -3,3 +3,330 @@ pub mod entropy; pub mod rules; + +use crate::FieldShape; +use crate::detect::{Candidate, Detector, Finding}; +use aho_corasick::{AhoCorasick, MatchKind}; +use std::ops::Range; + +/// Tuned only so the fixture corpus in this module's tests lands true +/// positives clearly above, and documented false positives clearly below, +/// the 0.8 plan-default threshold - not derived from a labeled dataset. +const BASE_SCORE: f32 = 0.6; +const HOTWORD_BONUS: f32 = 0.5; +const PENALTY_PER_ENTROPY_BIT: f32 = 0.15; +const HOTWORD_WINDOW: usize = 50; + +const HOTWORDS: &[&str] = &[ + "password", + "passwd", + "secret", + "token", + "credential", + "api_key", + "apikey", + "auth", + "private_key", + "access_key", +]; + +pub struct InternalDetector { + rules: Vec<(rules::Rule, regex::Regex)>, + prefilter: AhoCorasick, + /// Recognizes this crate's own [`crate::Transform::Marker`] and + /// [`crate::Transform::Mask`] output. Matched regions are blanked + /// before any rule runs them, or a second redaction pass would find + /// the marker text itself as a "secret" and redaction would never + /// reach a fixed point. + marker_re: regex::Regex, +} + +impl InternalDetector { + pub fn new() -> Self { + let rules: Vec<(rules::Rule, regex::Regex)> = rules::load_rules() + .into_iter() + .map(|r| { + let re = regex::Regex::new(&r.regex) + .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); + (r, re) + }) + .collect(); + + let keywords: Vec<&str> = rules + .iter() + .flat_map(|(r, _)| r.keywords.iter().map(String::as_str)) + .collect(); + // `LeftmostLongest` per the plan: this automaton only gates whether + // `detect()` runs at all (the trait's `prefilter()`), so which + // keyword "wins" a tie never matters, only whether any hit at all. + let prefilter = AhoCorasick::builder() + .ascii_case_insensitive(true) + .match_kind(MatchKind::LeftmostLongest) + .build(&keywords) + .expect("keyword list is static and derived from the loaded ruleset"); + + Self { + rules, + prefilter, + marker_re: regex::Regex::new(r"\[REDACTED:[^\]\n]*\]|█+") + .expect("literal marker pattern"), + } + } +} + +impl Default for InternalDetector { + fn default() -> Self { + Self::new() + } +} + +fn mask_existing_markers(text: &str, marker_re: ®ex::Regex) -> String { + let mut out = text.to_string(); + for m in marker_re.find_iter(text) { + // One NUL byte per matched byte: byte length is preserved, so + // every later span still indexes correctly into the original text. + out.replace_range(m.range(), &"\0".repeat(m.len())); + } + out +} + +/// Gitleaks' `private-key` rule spans from a PEM header through its +/// footer, crossing newlines by design; a unified diff interleaves `+`/`-` +/// markers and unrelated lines between them, so redacting the raw span +/// would eat surrounding diff structure. Clip to the line containing the +/// match's start instead. +fn clip_to_line_if_diff(text: &str, shape: FieldShape, span: Range) -> Range { + if shape != FieldShape::UnifiedDiff { + return span; + } + let line_start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); + let line_end = text[span.end..] + .find('\n') + .map_or(text.len(), |i| span.end + i); + span.start.max(line_start)..span.end.min(line_end) +} + +fn has_hotword_nearby(text: &str, span: &Range) -> bool { + let start = text + .char_indices() + .map(|(i, _)| i) + .take_while(|&i| i <= span.start.saturating_sub(HOTWORD_WINDOW)) + .last() + .unwrap_or(0); + let end = (span.end + HOTWORD_WINDOW).min(text.len()); + let end = (end..=text.len()) + .find(|&i| text.is_char_boundary(i)) + .unwrap_or(text.len()); + let window = text[start..end].to_ascii_lowercase(); + HOTWORDS.iter().any(|h| window.contains(h)) +} + +/// Base confidence from a rule match, adjusted down when the matched text +/// is lower-entropy than the rule expects (proportional to how far below, +/// so a near-miss and a wildly-off match don't get the same penalty) and +/// up when a hotword sits within [`HOTWORD_WINDOW`] chars, then clamped. +fn score(rule: &rules::Rule, matched: &str, has_hotword: bool) -> f32 { + let mut s = BASE_SCORE; + if let Some(threshold) = rule.entropy { + let actual = entropy::shannon(matched); + if actual < threshold { + s -= ((threshold - actual) as f32) * PENALTY_PER_ENTROPY_BIT; + } + } + if has_hotword { + s += HOTWORD_BONUS; + } + s.clamp(0.0, 1.0) +} + +impl Detector for InternalDetector { + fn id(&self) -> &'static str { + "internal" + } + + fn prefilter(&self, text: &str) -> bool { + self.prefilter.is_match(text) + } + + fn detect(&self, c: &Candidate<'_>) -> crate::Result> { + let masked = mask_existing_markers(c.text, &self.marker_re); + let mut out = Vec::new(); + for (rule, re) in &self.rules { + for caps in re.captures_iter(&masked) { + // Group 1 is the secret in every gitleaks rule that has + // surrounding context (an assignment operator, a quote); + // group 0 is the whole match for rules with nothing to + // trim (bare tokens like `aws-access-token`). + let m = caps + .get(1) + .or_else(|| caps.get(0)) + .expect("group 0 always exists"); + let raw_span = m.range(); + let raw_matched = &c.text[raw_span.clone()]; + if rule.allow.iter().any(|a| a.is_match(raw_matched)) { + continue; + } + let span = clip_to_line_if_diff(c.text, c.shape, raw_span); + if span.is_empty() { + continue; + } + let matched = &c.text[span.clone()]; + let has_hotword = has_hotword_nearby(c.text, &span); + out.push(Finding { + span, + rule: rule.id.clone(), + score: score(rule, matched, has_hotword), + detector: self.id(), + }); + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::detect::Context; + + fn cand(text: &str, shape: FieldShape) -> Candidate<'_> { + Candidate { + text, + shape, + at: "/change/x/structural/extra/text", + ctx: Context { + change_type: "conversation.append", + tool_name: None, + actor: "human:t", + kind: None, + }, + } + } + + fn detect_one(text: &str) -> Vec { + InternalDetector::new() + .detect(&cand(text, FieldShape::Prose)) + .unwrap() + } + + fn diff_candidate() -> Candidate<'static> { + cand( + "@@ -1,4 +1,4 @@\n-old line\n+-----BEGIN RSA PRIVATE KEY-----\n+MIIEpAIBAAKCAQEAxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n+-----END RSA PRIVATE KEY-----\n more diff context\n", + FieldShape::UnifiedDiff, + ) + } + + fn uri_candidate(text: &'static str) -> Candidate<'static> { + cand(text, FieldShape::Uri) + } + + #[test] + fn detects_shipped_formats() { + for (label, sample) in [ + ("aws", "AKIAIOSFODNN7REALKEY"), + ("google", "AIzaSyD-0123456789abcdefghijklmnopqrstu"), + ("jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.QQQQQQQQQQ"), + ("pem", "-----BEGIN RSA PRIVATE KEY-----"), + ("dburi", "postgres://u:s3cr3tpass@db.internal:5432/prod"), + ] { + assert!(!detect_one(sample).is_empty(), "missed {label}"); + } + } + + #[test] + fn documented_false_positives_stay_below_threshold() { + for sample in [ + "AKIAIOSFODNN7EXAMPLE", // AWS's own documentation key + "redis://localhost:6379", // no password + "0e2b3d4e3dec5f38ae95f62519eb2736f73c0b", // git SHA + "550e8400-e29b-41d4-a716-446655440000", // UUID + "ThisIsAReallyLongString", // high entropy, not a secret + ] { + assert!( + detect_one(sample).iter().all(|f| f.score < 0.8), + "false positive on {sample}" + ); + } + } + + #[test] + fn diff_spans_never_cross_a_newline() { + let c = diff_candidate(); + let findings = InternalDetector::new().detect(&c).unwrap(); + assert!(!findings.is_empty()); + for f in findings { + assert!(!c.text[f.span.clone()].contains('\n')); + } + } + + #[test] + fn uri_shape_redacts_only_the_password() { + let c = uri_candidate("postgres://svc_user:h0rr1bl3@db.internal:5432/prod"); + let findings = InternalDetector::new().detect(&c).unwrap(); + assert_eq!(&c.text[findings[0].span.clone()], "h0rr1bl3"); + } + + #[test] + fn existing_markers_are_never_re_detected() { + assert!(detect_one("[REDACTED:aws-access-key-id:a3c829]").is_empty()); + assert!(detect_one("████████████████████").is_empty()); + } + + fn neutral_rule(entropy: Option) -> rules::Rule { + rules::Rule { + id: "test-rule".to_string(), + regex: ".".to_string(), + entropy, + keywords: vec![], + allow: vec![], + } + } + + #[test] + fn below_entropy_lowers_score() { + let rule = neutral_rule(Some(3.0)); + let s = score(&rule, "aaaaaaaaaa", false); // shannon == 0.0, well below 3.0 + assert!( + s < BASE_SCORE && s > 0.0, + "expected a reduced but non-clamped score, got {s}" + ); + } + + #[test] + fn above_entropy_keeps_base_score() { + let rule = neutral_rule(Some(1.0)); + let s = score(&rule, "abcdefghij", false); // shannon == log2(10), well above 1.0 + assert_eq!(s, BASE_SCORE); + } + + #[test] + fn hotword_present_boosts_score() { + let rule = neutral_rule(Some(2.0)); + let without = score(&rule, "aaaaaaaaaa", false); + let with = score(&rule, "aaaaaaaaaa", true); + assert!( + with > without, + "hotword should raise the score: {with} vs {without}" + ); + } + + #[test] + fn hotword_absent_does_not_boost() { + let rule = neutral_rule(None); + assert_eq!(score(&rule, "anything", false), BASE_SCORE); + } + + #[test] + fn clamp_low() { + let rule = neutral_rule(Some(8.0)); // no real string reaches 8 bits of entropy + let s = score(&rule, "aaaaaaaaaa", false); + assert_eq!(s, 0.0); + } + + #[test] + fn clamp_high() { + let rule = neutral_rule(None); + let s = score(&rule, "anything", true); + assert_eq!(s, 1.0); + } +} diff --git a/crates/toolpath-redact/src/internal/rules.rs b/crates/toolpath-redact/src/internal/rules.rs index 602cfab4..300e3877 100644 --- a/crates/toolpath-redact/src/internal/rules.rs +++ b/crates/toolpath-redact/src/internal/rules.rs @@ -35,7 +35,10 @@ struct RulesFile { #[derive(Deserialize)] struct RawRule { id: String, - regex: String, + /// Absent on the one rule (`pkcs12-file`) that matches a file's *path* + /// rather than its content - out of scope for a text detector, so + /// [`load_rules`] drops any rule missing it. + regex: Option, entropy: Option, #[serde(default)] keywords: Vec, @@ -53,7 +56,20 @@ struct RawAllowlist { /// rejects a minority of gitleaks patterns written for Go's RE2 dialect. /// Confirmed by `every_vendored_rule_compiles_under_rust_regex` - see that /// test for the failure each id below hit. -pub const EXCLUDED_RULE_IDS: &[(&str, &str)] = &[]; +pub const EXCLUDED_RULE_IDS: &[(&str, &str)] = &[ + ( + "generic-api-key", + "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", + ), + ( + "pypi-upload-token", + "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", + ), + ( + "vault-batch-token", + "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", + ), +]; /// Hand-written rules filling gaps the vendored ruleset leaves open for /// this crate's purposes: gitleaks matches a PEM block only with its full @@ -95,17 +111,19 @@ pub fn load_rules() -> Vec { .rules .into_iter() .filter(|r| !EXCLUDED_RULE_IDS.iter().any(|(id, _)| *id == r.id)) - .map(|r| Rule { - id: r.id, - regex: r.regex, - entropy: r.entropy, - keywords: r.keywords, - allow: r - .allowlists - .iter() - .flat_map(|a| a.regexes.iter()) - .filter_map(|re| regex::Regex::new(re).ok()) - .collect(), + .filter_map(|r| { + Some(Rule { + id: r.id, + regex: r.regex?, + entropy: r.entropy, + keywords: r.keywords, + allow: r + .allowlists + .iter() + .flat_map(|a| a.regexes.iter()) + .filter_map(|re| regex::Regex::new(re).ok()) + .collect(), + }) }) .collect(); rules.extend(supplemental_rules()); @@ -119,7 +137,11 @@ mod tests { #[test] fn every_vendored_rule_compiles_under_rust_regex() { let rules = load_rules(); - assert!(rules.len() >= 200, "expected the full ruleset, got {}", rules.len()); + assert!( + rules.len() >= 200, + "expected the full ruleset, got {}", + rules.len() + ); for r in &rules { regex::Regex::new(&r.regex) .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); diff --git a/crates/toolpath-redact/src/plan.rs b/crates/toolpath-redact/src/plan.rs index 883748b4..99bfa419 100644 --- a/crates/toolpath-redact/src/plan.rs +++ b/crates/toolpath-redact/src/plan.rs @@ -90,38 +90,169 @@ pub struct RedactionPolicy { // ── Plan machinery (T5) ──────────────────────────────────────────────── -pub fn parse_predicate(_s: &str) -> crate::Result { - todo!("T5") +pub fn parse_predicate(s: &str) -> crate::Result { + // Longest operators first, or `score>=0.95` splits on the bare `>` and + // leaves a literal `=0.95` for the value parser to choke on. + for (op, cmp) in [ + (">=", Cmp::Ge), + ("<=", Cmp::Le), + (">", Cmp::Gt), + ("<", Cmp::Lt), + ] { + if let Some((k, v)) = s.split_once(op) + && k.trim() == "score" + { + let value: f32 = v.trim().parse().map_err(|_| bad_predicate(s))?; + return Ok(Predicate::Score(cmp, value)); + } + } + + let (k, v) = s.split_once('=').ok_or_else(|| bad_predicate(s))?; + let (k, v) = (k.trim(), v.trim()); + if v.is_empty() { + return Err(bad_predicate(s)); + } + Ok(match k { + "rule" => Predicate::Rule(v.to_string()), + "shape" => Predicate::Shape(parse_shape(v)?), + "step" => Predicate::Step(v.to_string()), + "detector" => Predicate::Detector(v.to_string()), + "at" => Predicate::AtPrefix(v.to_string()), + "score" => Predicate::Score(Cmp::Eq, v.parse().map_err(|_| bad_predicate(s))?), + other => { + return Err(crate::RedactError::BadPredicate(format!( + "unknown field {other:?} in {s:?}" + ))); + } + }) +} + +fn parse_shape(s: &str) -> crate::Result { + Ok(match s { + "prose" => FieldShape::Prose, + "tool_input" => FieldShape::ToolInput, + "tool_output" => FieldShape::ToolOutput, + "unified_diff" => FieldShape::UnifiedDiff, + "file_content" => FieldShape::FileContent, + "uri" => FieldShape::Uri, + "opaque_json" => FieldShape::OpaqueJson, + other => { + return Err(crate::RedactError::BadPredicate(format!( + "unknown shape {other:?}" + ))); + } + }) +} + +fn bad_predicate(s: &str) -> crate::RedactError { + crate::RedactError::BadPredicate(format!("not a valid predicate: {s:?}")) +} + +/// Whether a finding satisfies a single predicate clause. +pub fn matches(p: &Predicate, f: &PlanFinding) -> bool { + match p { + Predicate::Rule(r) => &f.rule == r, + Predicate::Shape(shape) => f.shape == *shape, + Predicate::Step(s) => &f.step == s, + Predicate::Detector(d) => &f.detector == d, + Predicate::AtPrefix(prefix) => f.at.starts_with(prefix.as_str()), + Predicate::Score(cmp, v) => match cmp { + Cmp::Ge => f.score >= *v, + Cmp::Gt => f.score > *v, + Cmp::Le => f.score <= *v, + Cmp::Lt => f.score < *v, + Cmp::Eq => f.score == *v, + }, + } } /// Later decisions override earlier ones, so a caller can express /// "redact everything, except this" by ordering. -pub fn apply_decisions(_plan: &mut Plan, _decisions: &[Decision]) { - todo!("T5") +pub fn apply_decisions(plan: &mut Plan, decisions: &[Decision]) { + for finding in &mut plan.findings { + if let Some(d) = decisions + .iter() + .rev() + .find(|d| matches(&d.predicate, finding)) + { + finding.action = d.action; + finding.transform = d.transform; + } + } } /// Stable, ordinal finding id (`f01`, `f02`, …). Stability is what makes a /// regenerated plan byte-identical to its predecessor. -pub fn finding_id(_index: usize) -> String { - todo!("T5") +pub fn finding_id(index: usize) -> String { + format!("f{:02}", index + 1) } /// The line around `span` with the match replaced by `` - never the /// value and never anything from which its length can be read, unless /// `reveal` was set. -pub fn elide_context( - _text: &str, - _span: std::ops::Range, - _rule: &str, - _reveal: bool, -) -> String { - todo!("T5") +pub fn elide_context(text: &str, span: std::ops::Range, rule: &str, reveal: bool) -> String { + let line_start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); + let line_end = text[span.end..] + .find('\n') + .map_or(text.len(), |i| span.end + i); + let replacement = if reveal { + text[span.start..span.end].to_string() + } else { + format!("<{rule}>") + }; + format!( + "{}{replacement}{}", + &text[line_start..span.start], + &text[span.end..line_end] + ) } /// Refuse a plan that no longer describes this document, naming the first /// divergence. -pub fn verify(_plan: &Plan, _path: &toolpath::v1::Path) -> crate::Result<()> { - todo!("T5") +/// +/// Takes `path` by `&mut` (rather than the `&Path` the rest of this +/// function's job would suggest) because `SurfaceCursor` (T2) needs +/// exclusive access to resolve a pointer to text; `verify` itself never +/// mutates anything through it. +pub fn verify(plan: &Plan, path: &mut toolpath::v1::Path) -> crate::Result<()> { + if plan.document != path.path.id { + return Err(crate::RedactError::PlanMismatch(format!( + "plan targets document {:?}, but path.id is {:?}", + plan.document, path.path.id + ))); + } + + let step_ids: std::collections::HashSet = + path.steps.iter().map(|s| s.step.id.clone()).collect(); + + let cursor = crate::surface::SurfaceCursor { path }; + for finding in &plan.findings { + if !finding.step.is_empty() && !step_ids.contains(&finding.step) { + return Err(crate::RedactError::PlanMismatch(format!( + "{}: step {:?} no longer exists", + finding.id, finding.step + ))); + } + + let current = cursor.read(&finding.step, &finding.at).ok_or_else(|| { + crate::RedactError::PlanMismatch(format!( + "{}: {} no longer resolves", + finding.id, finding.at + )) + })?; + + let (start, end) = finding.span; + let lands = end <= current.len() + && current.is_char_boundary(start) + && current.is_char_boundary(end); + if !lands { + return Err(crate::RedactError::PlanMismatch(format!( + "{}: recorded span {start}..{end} no longer lands inside {}", + finding.id, finding.at + ))); + } + } + Ok(()) } // ── Plan generation (T8) ─────────────────────────────────────────────── @@ -144,3 +275,463 @@ pub fn generate_checked( ) -> crate::Result { todo!("T8") } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + use toolpath::v1::{ArtifactChange, Path, PathIdentity, Step, StepIdentity, StructuralChange}; + + fn fixed_now() -> DateTime { + DateTime::parse_from_rfc3339("2026-07-30T00:00:00Z") + .unwrap() + .with_timezone(&Utc) + } + + fn sample_finding(id: &str, rule: &str, score: f32) -> PlanFinding { + PlanFinding { + id: id.to_string(), + step: "step-1".to_string(), + at: "/change/convo/structural/extra/text".to_string(), + rule: rule.to_string(), + span: (0, 4), + score, + detector: "internal".to_string(), + shape: FieldShape::Prose, + context: " context".to_string(), + action: Action::Redact, + transform: None, + } + } + + fn sample_plan(findings: Vec) -> Plan { + Plan { + v: 1, + document: "doc-1".to_string(), + generated: fixed_now(), + detectors: vec!["internal".to_string()], + defaults: PlanDefaults { + transform: Transform::Marker, + threshold: 0.8, + }, + surfaces: vec![], + findings, + } + } + + fn decision(pred: &str, action: Action) -> Decision { + Decision { + predicate: parse_predicate(pred).unwrap(), + action, + transform: None, + } + } + + /// One step whose `conversation.append` text field is `text`, addressable + /// at `/change//structural/extra/text` - the pointer shape + /// `surfaces()` (T2) assigns to that field. + fn fixture_path_with_text(doc_id: &str, step_id: &str, artifact_key: &str, text: &str) -> Path { + let mut extra = HashMap::new(); + extra.insert( + "text".to_string(), + serde_json::Value::String(text.to_string()), + ); + + let mut change = HashMap::new(); + change.insert( + artifact_key.to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "conversation.append".to_string(), + extra, + }), + }, + ); + + let step = Step { + step: StepIdentity { + id: step_id.to_string(), + parents: vec![], + actor: "human:t".to_string(), + timestamp: "2026-07-30T00:00:00Z".to_string(), + }, + change, + meta: None, + }; + + Path { + path: PathIdentity { + id: doc_id.to_string(), + base: None, + head: step_id.to_string(), + graph_ref: None, + }, + steps: vec![step], + meta: None, + } + } + + // ── parse_predicate ───────────────────────────────────────────────── + + #[test] + fn parses_every_predicate_field() { + assert!(matches!( + parse_predicate("rule=aws-access-key-id").unwrap(), + Predicate::Rule(_) + )); + assert!(matches!( + parse_predicate("shape=unified_diff").unwrap(), + Predicate::Shape(_) + )); + assert!(matches!( + parse_predicate("step=turn-0f3a").unwrap(), + Predicate::Step(_) + )); + assert!(matches!( + parse_predicate("detector=internal").unwrap(), + Predicate::Detector(_) + )); + assert!(matches!( + parse_predicate("at=/change/x").unwrap(), + Predicate::AtPrefix(_) + )); + assert!(matches!( + parse_predicate("score>=0.95").unwrap(), + Predicate::Score(Cmp::Ge, _) + )); + } + + #[test] + fn rejects_anything_else_clearly() { + let e = parse_predicate("colour=red").unwrap_err().to_string(); + assert!(e.contains("colour"), "error should name the bad field: {e}"); + } + + #[test] + fn ge_is_not_confused_with_gt() { + match parse_predicate("score>=0.95").unwrap() { + Predicate::Score(Cmp::Ge, v) => assert_eq!(v, 0.95), + other => panic!("expected Score(Ge, 0.95), got {other:?}"), + } + } + + #[test] + fn le_is_not_confused_with_lt() { + match parse_predicate("score<=0.3").unwrap() { + Predicate::Score(Cmp::Le, v) => assert_eq!(v, 0.3), + other => panic!("expected Score(Le, 0.3), got {other:?}"), + } + } + + #[test] + fn gt_and_lt_parse_without_the_equals_form() { + assert!(matches!( + parse_predicate("score>0.5").unwrap(), + Predicate::Score(Cmp::Gt, _) + )); + assert!(matches!( + parse_predicate("score<0.5").unwrap(), + Predicate::Score(Cmp::Lt, _) + )); + } + + #[test] + fn rejects_missing_operator() { + assert!(parse_predicate("just-some-text-with-no-operator").is_err()); + } + + #[test] + fn rejects_empty_value() { + assert!(parse_predicate("rule=").is_err()); + } + + #[test] + fn rejects_non_numeric_score() { + assert!(parse_predicate("score=not-a-number").is_err()); + } + + #[test] + fn rejects_unknown_shape_name() { + assert!(parse_predicate("shape=not-a-shape").is_err()); + } + + #[test] + fn trims_whitespace_around_both_sides() { + assert_eq!( + parse_predicate(" rule = aws-access-key-id ").unwrap(), + Predicate::Rule("aws-access-key-id".to_string()) + ); + assert_eq!( + parse_predicate(" score >= 0.5 ").unwrap(), + Predicate::Score(Cmp::Ge, 0.5) + ); + } + + // ── matches ────────────────────────────────────────────────────────── + + #[test] + fn every_cmp_variant_compares_correctly() { + let f = |score| sample_finding("f01", "r", score); + assert!(matches(&Predicate::Score(Cmp::Ge, 0.5), &f(0.5))); + assert!(matches(&Predicate::Score(Cmp::Ge, 0.5), &f(0.6))); + assert!(!matches(&Predicate::Score(Cmp::Ge, 0.5), &f(0.4))); + + assert!(matches(&Predicate::Score(Cmp::Gt, 0.5), &f(0.6))); + assert!(!matches(&Predicate::Score(Cmp::Gt, 0.5), &f(0.5))); + + assert!(matches(&Predicate::Score(Cmp::Le, 0.5), &f(0.5))); + assert!(matches(&Predicate::Score(Cmp::Le, 0.5), &f(0.4))); + assert!(!matches(&Predicate::Score(Cmp::Le, 0.5), &f(0.6))); + + assert!(matches(&Predicate::Score(Cmp::Lt, 0.5), &f(0.4))); + assert!(!matches(&Predicate::Score(Cmp::Lt, 0.5), &f(0.5))); + + assert!(matches(&Predicate::Score(Cmp::Eq, 0.5), &f(0.5))); + assert!(!matches(&Predicate::Score(Cmp::Eq, 0.5), &f(0.500_001))); + } + + #[test] + fn rule_shape_step_detector_match_exactly() { + let f = sample_finding("f01", "aws-access-key-id", 0.9); + assert!(matches( + &Predicate::Rule("aws-access-key-id".to_string()), + &f + )); + assert!(!matches(&Predicate::Rule("other".to_string()), &f)); + assert!(matches(&Predicate::Shape(FieldShape::Prose), &f)); + assert!(!matches(&Predicate::Shape(FieldShape::UnifiedDiff), &f)); + assert!(matches(&Predicate::Step("step-1".to_string()), &f)); + assert!(matches(&Predicate::Detector("internal".to_string()), &f)); + } + + #[test] + fn at_prefix_matches_prefix_not_substring_or_equality() { + let p = Predicate::AtPrefix("/change/x".to_string()); + let make = |at: &str| PlanFinding { + at: at.to_string(), + ..sample_finding("f01", "r", 0.9) + }; + + assert!( + matches(&p, &make("/change/x/structural/extra/text")), + "prefix should match" + ); + assert!( + matches(&p, &make("/change/x")), + "exact equality is a trivial prefix match" + ); + assert!( + !matches(&p, &make("nested/change/x/structural")), + "substring elsewhere in the string must not match" + ); + assert!( + !matches(&p, &make("/change/")), + "a shorter string cannot have a longer prefix" + ); + } + + // ── apply_decisions ────────────────────────────────────────────────── + + #[test] + fn last_matching_decision_wins() { + let mut plan = sample_plan(vec![sample_finding("f01", "aws-access-key-id", 0.99)]); + apply_decisions( + &mut plan, + &[ + decision("rule=aws-access-key-id", Action::Redact), + decision("score>=0.9", Action::Skip), + ], + ); + assert_eq!(plan.findings[0].action, Action::Skip); + } + + #[test] + fn apply_decisions_applies_the_winning_decisions_transform() { + let mut plan = sample_plan(vec![sample_finding("f01", "us-ssn", 0.9)]); + apply_decisions( + &mut plan, + &[Decision { + predicate: parse_predicate("rule=us-ssn").unwrap(), + action: Action::Redact, + transform: Some(Transform::Mask), + }], + ); + assert_eq!(plan.findings[0].transform, Some(Transform::Mask)); + } + + #[test] + fn apply_decisions_leaves_non_matching_findings_untouched() { + let mut plan = sample_plan(vec![sample_finding("f01", "us-ssn", 0.9)]); + let original_action = plan.findings[0].action; + apply_decisions(&mut plan, &[decision("rule=other-rule", Action::Skip)]); + assert_eq!(plan.findings[0].action, original_action); + } + + // ── finding_id ─────────────────────────────────────────────────────── + + #[test] + fn finding_id_zero_padded_then_grows_without_collision() { + assert_eq!(finding_id(0), "f01"); + assert_eq!(finding_id(8), "f09"); + assert_eq!(finding_id(98), "f99"); + assert_eq!(finding_id(99), "f100"); + assert_eq!(finding_id(100), "f101"); + } + + // ── elide_context ──────────────────────────────────────────────────── + + #[test] + fn elide_context_never_carries_the_value_or_its_length() { + let value = "AKIAIOSFODNN7REALKEY"; + assert_eq!(value.len(), 20); + let text = format!("key: {value}\n"); + let start = text.find(value).unwrap(); + let out = elide_context( + &text, + start..start + value.len(), + "aws-access-key-id", + false, + ); + assert!(!out.contains(value)); + assert!( + !out.contains("20"), + "the value's length must not leak either: {out}" + ); + assert!(out.contains("")); + assert_eq!(out, "key: "); + } + + #[test] + fn elide_context_spans_the_whole_line() { + let text = "before\nSECRETVALUE\nafter"; + let start = text.find("SECRETVALUE").unwrap(); + let out = elide_context(text, start..start + "SECRETVALUE".len(), "rule", false); + assert_eq!(out, ""); + } + + #[test] + fn elide_context_at_the_very_start_of_the_text() { + let text = "SECRETfoo bar\nnext line"; + let out = elide_context(text, 0.."SECRET".len(), "rule", false); + assert_eq!(out, "foo bar"); + } + + #[test] + fn elide_context_at_the_very_end_of_the_text() { + let text = "prefix line\nend SECRETEND"; + let start = text.find("SECRETEND").unwrap(); + let out = elide_context(text, start..start + "SECRETEND".len(), "rule", false); + assert_eq!(out, "end "); + } + + #[test] + fn elide_context_handles_multibyte_text() { + let text = "héllo wörld\nsécret：dröp\nmore lïnes"; + let needle = "dröp"; + let start = text.find(needle).unwrap(); + let out = elide_context(text, start..start + needle.len(), "rule", false); + assert_eq!(out, "sécret："); + } + + #[test] + fn elide_context_with_no_newline_at_all() { + let text = "just one line with a SECRET in it"; + let start = text.find("SECRET").unwrap(); + let out = elide_context(text, start..start + "SECRET".len(), "rule", false); + assert_eq!(out, "just one line with a in it"); + } + + #[test] + fn elide_context_reveal_includes_the_value() { + let value = "AKIAIOSFODNN7REALKEY"; + let text = format!("key: {value}\n"); + let start = text.find(value).unwrap(); + let out = elide_context(&text, start..start + value.len(), "aws-access-key-id", true); + assert!(out.contains(value)); + } + + // ── verify ─────────────────────────────────────────────────────────── + // + // These exercise `verify` through `SurfaceCursor::read` (T2), which is + // still `todo!()` as of this writing - see the report for status. + + #[test] + fn verify_passes_on_an_unmodified_document() { + let text = "hello world, this is prose"; + let mut path = fixture_path_with_text("doc-1", "step-1", "convo", text); + let start = text.find("world").unwrap(); + let plan = sample_plan(vec![PlanFinding { + span: (start, start + "world".len()), + ..sample_finding("f01", "some-rule", 0.9) + }]); + assert!(verify(&plan, &mut path).is_ok()); + } + + #[test] + fn verify_fails_naming_the_finding_whose_span_no_longer_lands() { + let text = "hello world, this is prose"; + let start = text.find("world").unwrap(); + let plan = sample_plan(vec![PlanFinding { + span: (start, start + "world".len()), + ..sample_finding("f01", "some-rule", 0.9) + }]); + + let mut mutated = fixture_path_with_text("doc-1", "step-1", "convo", "short"); + let e = verify(&plan, &mut mutated).unwrap_err().to_string(); + assert!(e.contains("f01"), "should name the first divergence: {e}"); + } + + #[test] + fn verify_fails_when_a_step_disappears() { + let plan = sample_plan(vec![PlanFinding { + step: "step-missing".to_string(), + span: (0, 5), + ..sample_finding("f01", "some-rule", 0.9) + }]); + let mut path = fixture_path_with_text("doc-1", "step-1", "convo", "hello world"); + let e = verify(&plan, &mut path).unwrap_err().to_string(); + assert!(e.contains("f01")); + } + + #[test] + fn verify_fails_when_the_document_id_differs() { + let plan = sample_plan(vec![]); + let mut path = fixture_path_with_text("different-doc", "step-1", "convo", "text"); + assert!(verify(&plan, &mut path).is_err()); + } + + // ── serde round-trip ───────────────────────────────────────────────── + + #[test] + fn plan_round_trips_through_json() { + let plan = Plan { + v: 1, + document: "doc-1".to_string(), + generated: fixed_now(), + detectors: vec!["internal".to_string(), "gitleaks".to_string()], + defaults: PlanDefaults { + transform: Transform::Mask, + threshold: 0.75, + }, + surfaces: vec![crate::surface::Surface { + step: "step-1".to_string(), + at: "/change/convo/structural/extra/text".to_string(), + shape: FieldShape::Prose, + bytes: 42, + }], + findings: vec![PlanFinding { + span: (10, 30), + score: 0.97, + context: " is the key".to_string(), + transform: Some(Transform::Hash), + ..sample_finding("f01", "aws-access-key-id", 0.97) + }], + }; + + let json = serde_json::to_string(&plan).unwrap(); + let round: Plan = serde_json::from_str(&json).unwrap(); + assert_eq!(plan, round); + assert_eq!(json, serde_json::to_string(&round).unwrap()); + } +} diff --git a/crates/toolpath-redact/src/surface.rs b/crates/toolpath-redact/src/surface.rs index 8e33a2ac..5ae30022 100644 --- a/crates/toolpath-redact/src/surface.rs +++ b/crates/toolpath-redact/src/surface.rs @@ -1,6 +1,11 @@ //! The field map: every string a secret could hide in, named by pointer. +use std::collections::HashMap; + +use serde_json::Value; + use crate::detect::FieldShape; +use crate::{RedactError, Result}; /// One field the map named, whether or not anything was found in it. A /// surface with zero findings is information: the pass reached that field @@ -13,8 +18,259 @@ pub struct Surface { pub bytes: usize, } -pub fn surfaces(_path: &toolpath::v1::Path) -> Vec { - todo!("T2") +/// Every string in `path` a detector should see, each named by an RFC 6901 +/// pointer relative to its step (`step` is empty for document-level fields). +/// +/// The order is part of the contract. Finding ids are positional and a +/// regenerated plan is compared byte for byte, but `Step::change` and +/// `StructuralChange::extra` are `HashMap`s whose iteration order is not +/// stable across runs - so steps keep document order, artifacts sort by key, +/// and a blind walk sorts object keys. +/// +/// An artifact's own key is emitted *after* everything beneath it: writing +/// that surface renames the map entry, which invalidates every pointer under +/// the old key, so a caller applying surfaces in order rewrites the key last. +pub fn surfaces(path: &toolpath::v1::Path) -> Vec { + let mut out = Vec::new(); + for step in &path.steps { + let sid = &step.step.id; + let mut keys: Vec<&String> = step.change.keys().collect(); + keys.sort(); + for artifact_key in keys { + let change = &step.change[artifact_key]; + let akey = ptr_escape(artifact_key); + + if let Some(raw) = &change.raw { + push( + &mut out, + sid, + format!("/change/{akey}/raw"), + FieldShape::UnifiedDiff, + raw, + ); + } + if let Some(s) = &change.structural { + let base = format!("/change/{akey}/structural/extra"); + match s.change_type.as_str() { + "conversation.append" => turn_surfaces(&mut out, sid, &base, &s.extra), + "file.write" => file_write_surfaces(&mut out, sid, &base, &s.extra), + // The one place a blind leaf walk is correct: the payload + // is unmodelled provider JSON. + _ => walk_fields(&mut out, sid, &base, &s.extra, FieldShape::OpaqueJson), + } + } + push( + &mut out, + sid, + format!("/change/{akey}"), + FieldShape::Uri, + artifact_key, + ); + } + } + if let Some(b) = &path.path.base { + push( + &mut out, + "", + "/path/base/uri".into(), + FieldShape::Uri, + &b.uri, + ); + } + if let Some(v) = path + .meta + .as_ref() + .and_then(|m| m.extra.get("vcs_remote")) + .and_then(|v| v.as_str()) + { + push(&mut out, "", "/meta/vcs_remote".into(), FieldShape::Uri, v); + } + out +} + +fn push(out: &mut Vec, step: &str, at: String, shape: FieldShape, text: &str) { + if text.is_empty() { + return; + } + out.push(Surface { + step: step.to_string(), + at, + shape, + bytes: text.len(), + }); +} + +/// The two containers a turn's fields arrive in: a top-level +/// `structural.extra` map, and a delegated turn's JSON object. +trait Fields { + fn field(&self, key: &str) -> Option<&Value>; +} + +impl Fields for HashMap { + fn field(&self, key: &str) -> Option<&Value> { + self.get(key) + } +} + +impl Fields for serde_json::Map { + fn field(&self, key: &str) -> Option<&Value> { + self.get(key) + } +} + +/// The `conversation.append` rows of the map. A delegated turn serializes +/// with the same field names as the extras of the turn that spawned it, so +/// sub-conversations re-enter here. +fn turn_surfaces(out: &mut Vec, step: &str, at: &str, fields: &dyn Fields) { + for key in ["text", "thinking"] { + if let Some(s) = fields.field(key).and_then(Value::as_str) { + push(out, step, format!("{at}/{key}"), FieldShape::Prose, s); + } + } + + for (i, tool) in array(fields.field("tool_uses")).iter().enumerate() { + if let Some(input) = tool.get("input") { + walk_json( + out, + step, + &format!("{at}/tool_uses/{i}/input"), + input, + FieldShape::ToolInput, + ); + } + if let Some(s) = tool.pointer("/result/content").and_then(Value::as_str) { + push( + out, + step, + format!("{at}/tool_uses/{i}/result/content"), + FieldShape::ToolOutput, + s, + ); + } + } + + for (i, work) in array(fields.field("delegations")).iter().enumerate() { + let at = format!("{at}/delegations/{i}"); + for key in ["prompt", "result"] { + if let Some(s) = work.get(key).and_then(Value::as_str) { + push(out, step, format!("{at}/{key}"), FieldShape::Prose, s); + } + } + for (j, turn) in array(work.get("turns")).iter().enumerate() { + if let Some(obj) = turn.as_object() { + turn_surfaces(out, step, &format!("{at}/turns/{j}"), obj); + } + } + } + + // Only a top-level turn's file mutations get hoisted into sibling + // `file.write` changes; a delegated turn carries its own inline, and they + // hold the same before/after file content. + for (i, mutation) in array(fields.field("file_mutations")).iter().enumerate() { + let at = format!("{at}/file_mutations/{i}"); + if let Some(s) = mutation.get("raw_diff").and_then(Value::as_str) { + push( + out, + step, + format!("{at}/raw_diff"), + FieldShape::UnifiedDiff, + s, + ); + } + for key in ["before", "after"] { + if let Some(s) = mutation.get(key).and_then(Value::as_str) { + push(out, step, format!("{at}/{key}"), FieldShape::FileContent, s); + } + } + } + + if let Some(s) = fields + .field("environment") + .and_then(|e| e.get("working_dir")) + .and_then(Value::as_str) + { + push( + out, + step, + format!("{at}/environment/working_dir"), + FieldShape::Uri, + s, + ); + } +} + +/// The `file.write` rows: whole-file states, plus both sides of every edit. +fn file_write_surfaces( + out: &mut Vec, + step: &str, + at: &str, + extra: &HashMap, +) { + for key in ["before", "after"] { + if let Some(s) = extra.get(key).and_then(Value::as_str) { + push(out, step, format!("{at}/{key}"), FieldShape::FileContent, s); + } + } + for (i, edit) in array(extra.get("edits")).iter().enumerate() { + walk_json( + out, + step, + &format!("{at}/edits/{i}"), + edit, + FieldShape::FileContent, + ); + } +} + +fn walk_fields( + out: &mut Vec, + step: &str, + at: &str, + fields: &HashMap, + shape: FieldShape, +) { + for (key, value) in sorted(fields.iter()) { + walk_json( + out, + step, + &format!("{at}/{}", ptr_escape(key)), + value, + shape, + ); + } +} + +/// Every string leaf under `value`, named by pointer. Non-string scalars are +/// not candidates: a detector has nothing to span in a number or a bool. +fn walk_json(out: &mut Vec, step: &str, at: &str, value: &Value, shape: FieldShape) { + match value { + Value::String(s) => push(out, step, at.to_string(), shape, s), + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + walk_json(out, step, &format!("{at}/{i}"), item, shape); + } + } + Value::Object(map) => { + for (key, item) in sorted(map.iter()) { + walk_json(out, step, &format!("{at}/{}", ptr_escape(key)), item, shape); + } + } + _ => {} + } +} + +/// Key order decides surface order, and `HashMap`'s is seeded per map while +/// `serde_json::Map`'s depends on the `preserve_order` feature. Sort both. +fn sorted<'a, I: Iterator>( + entries: I, +) -> Vec<(&'a str, &'a Value)> { + let mut entries: Vec<(&str, &Value)> = entries.map(|(k, v)| (k.as_str(), v)).collect(); + entries.sort_by_key(|(k, _)| *k); + entries +} + +fn array(value: Option<&Value>) -> &[Value] { + value.and_then(Value::as_array).map_or(&[], Vec::as_slice) } /// Resolves a `(step, pointer)` pair against a document for reading and @@ -23,16 +279,703 @@ pub struct SurfaceCursor<'a> { pub path: &'a mut toolpath::v1::Path, } +/// Where a pointer lands, parsed once so read and write cannot drift apart. +enum Route { + BaseUri, + MetaExtra(String), + /// The artifact map key itself. Writing renames the entry. + ArtifactKey(String), + Raw(String), + /// `structural.extra[field]`, plus an RFC 6901 pointer into it (empty + /// when the field is itself the string). + Extra { + artifact: String, + field: String, + tail: String, + }, +} + +fn route(at: &str) -> Option { + if at == "/path/base/uri" { + return Some(Route::BaseUri); + } + if let Some(key) = at.strip_prefix("/meta/") { + return (!key.is_empty() && !key.contains('/')).then(|| Route::MetaExtra(ptr_decode(key))); + } + + let rest = at.strip_prefix("/change/")?; + let Some((akey, rest)) = rest.split_once('/') else { + return (!rest.is_empty()).then(|| Route::ArtifactKey(ptr_decode(rest))); + }; + let artifact = ptr_decode(akey); + if rest == "raw" { + return Some(Route::Raw(artifact)); + } + + let rest = rest.strip_prefix("structural/extra/")?; + let (field, tail) = match rest.split_once('/') { + Some((field, tail)) => (field, format!("/{tail}")), + None => (rest, String::new()), + }; + (!field.is_empty()).then(|| Route::Extra { + artifact, + field: ptr_decode(field), + tail, + }) +} + +fn find_step<'a>(path: &'a toolpath::v1::Path, id: &str) -> Option<&'a toolpath::v1::Step> { + path.steps.iter().find(|s| s.step.id == id) +} + +fn find_step_mut<'a>( + path: &'a mut toolpath::v1::Path, + id: &str, +) -> Option<&'a mut toolpath::v1::Step> { + path.steps.iter_mut().find(|s| s.step.id == id) +} + impl SurfaceCursor<'_> { - pub fn read(&self, _step: &str, _at: &str) -> Option { - todo!("T2") + pub fn read(&self, step: &str, at: &str) -> Option { + match route(at)? { + Route::BaseUri => Some(self.path.path.base.as_ref()?.uri.clone()), + Route::MetaExtra(key) => Some( + self.path + .meta + .as_ref()? + .extra + .get(&key)? + .as_str()? + .to_string(), + ), + Route::ArtifactKey(key) => find_step(self.path, step)? + .change + .contains_key(&key) + .then_some(key), + Route::Raw(key) => find_step(self.path, step)?.change.get(&key)?.raw.clone(), + Route::Extra { + artifact, + field, + tail, + } => { + let value = find_step(self.path, step)? + .change + .get(&artifact)? + .structural + .as_ref()? + .extra + .get(&field)?; + let leaf = if tail.is_empty() { + value + } else { + value.pointer(&tail)? + }; + Some(leaf.as_str()?.to_string()) + } + } } - pub fn write(&mut self, _step: &str, _at: &str, _value: &str) -> crate::Result<()> { - todo!("T2") + pub fn write(&mut self, step: &str, at: &str, value: &str) -> Result<()> { + let bad = || RedactError::BadPointer(at.to_string()); + match route(at).ok_or_else(bad)? { + Route::BaseUri => { + self.path.path.base.as_mut().ok_or_else(bad)?.uri = value.to_string(); + } + Route::MetaExtra(key) => { + let slot = self + .path + .meta + .as_mut() + .and_then(|m| m.extra.get_mut(&key)) + .ok_or_else(bad)?; + *string_slot(slot).ok_or_else(bad)? = value.to_string(); + } + Route::ArtifactKey(key) => { + let target = find_step_mut(self.path, step).ok_or_else(bad)?; + // Two keys redacting to the same string would silently drop + // one artifact's changes; refuse instead of destroying data. + if key != value && target.change.contains_key(value) { + return Err(RedactError::PlanMismatch(format!( + "redacted artifact key {value} already exists on step {step}" + ))); + } + let change = target.change.remove(&key).ok_or_else(bad)?; + target.change.insert(value.to_string(), change); + } + Route::Raw(key) => { + let change = find_step_mut(self.path, step) + .ok_or_else(bad)? + .change + .get_mut(&key) + .ok_or_else(bad)?; + *change.raw.as_mut().ok_or_else(bad)? = value.to_string(); + } + Route::Extra { + artifact, + field, + tail, + } => { + let slot = find_step_mut(self.path, step) + .ok_or_else(bad)? + .change + .get_mut(&artifact) + .ok_or_else(bad)? + .structural + .as_mut() + .ok_or_else(bad)? + .extra + .get_mut(&field) + .ok_or_else(bad)?; + let leaf = if tail.is_empty() { + slot + } else { + slot.pointer_mut(&tail).ok_or_else(bad)? + }; + *string_slot(leaf).ok_or_else(bad)? = value.to_string(); + } + } + Ok(()) } } -pub fn ptr_escape(_token: &str) -> String { - todo!("T2") +/// A write resolves only where a read would have: on a string leaf. +fn string_slot(value: &mut Value) -> Option<&mut String> { + match value { + Value::String(s) => Some(s), + _ => None, + } +} + +/// Escape one pointer token. `~` first, or the `~1` this emits for `/` would +/// be re-escaped into `~01` (RFC 6901). +pub fn ptr_escape(token: &str) -> String { + token.replace('~', "~0").replace('/', "~1") +} + +/// Decode `~1` before `~0`, or `~01` round-trips wrong (RFC 6901). +fn ptr_decode(token: &str) -> String { + token.replace("~1", "/").replace("~0", "~") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use toolpath::v1::{ArtifactChange, Base, Path, PathMeta, Step, StructuralChange}; + + fn object(value: Value) -> HashMap { + match value { + Value::Object(map) => map.into_iter().collect(), + other => panic!("fixture extras must be an object, got {other}"), + } + } + + fn path_of(steps: Vec) -> Path { + let head = steps.last().map(|s| s.step.id.clone()).unwrap_or_default(); + let mut path = Path::new("path-1", None, head); + path.steps = steps; + path + } + + fn change(change_type: &str, raw: Option<&str>, extra: Value) -> ArtifactChange { + ArtifactChange { + raw: raw.map(str::to_string), + structural: Some(StructuralChange { + change_type: change_type.to_string(), + extra: object(extra), + }), + } + } + + fn step_with(id: &str, changes: Vec<(&str, ArtifactChange)>) -> Step { + let mut step = Step::new(id, "agent:claude-opus-5", "2026-07-30T10:00:00Z"); + for (key, c) in changes { + step.change.insert(key.to_string(), c); + } + step + } + + fn append_step(id: &str, extra: Value) -> Step { + step_with( + id, + vec![( + "claude://sess-abc", + change("conversation.append", None, extra), + )], + ) + } + + fn fixture_conversation_append() -> Path { + path_of(vec![append_step( + "turn-0f3a", + json!({ + "role": "assistant", + "text": "I set AWS_SECRET_ACCESS_KEY for you", + "thinking": "the key was pasted in the prompt", + "tool_uses": [{ + "id": "toolu_01", + "name": "Bash", + "input": { + "command": "aws configure set aws_secret_access_key AKIAIOSFODNN7EXAMPLE", + "timeout": 120 + }, + "category": "command", + "result": {"content": "configured", "is_error": false} + }], + "environment": {"working_dir": "/Users/alex/work/repo"}, + "token_usage": {"input_tokens": 10, "output_tokens": 2} + }), + )]) + } + + fn fixture_file_write() -> Path { + path_of(vec![step_with( + "turn-9c21", + vec![( + "src/config.rs", + change( + "file.write", + Some( + "--- a/src/config.rs\n+++ b/src/config.rs\n@@ -1 +1 @@\n-let k = \"old\";\n+let k = \"new\";\n", + ), + json!({ + "tool": "Edit", + "tool_id": "toolu_02", + "operation": "update", + "before": "let k = \"old\";\n", + "after": "let k = \"new\";\n" + }), + ), + )], + )]) + } + + fn fixture_clean_conversation() -> Path { + path_of(vec![append_step( + "turn-clean", + json!({"role": "user", "text": "please rename the greeting function"}), + )]) + } + + fn fixture_with_delegation() -> Path { + path_of(vec![append_step( + "turn-deleg", + json!({ + "role": "assistant", + "text": "delegating the audit", + "delegations": [{ + "agent_id": "sub-1", + "prompt": "audit the deploy script", + "result": "found a hardcoded token", + "turns": [{ + "id": "sub-turn-1", + "role": "assistant", + "timestamp": "2026-07-30T10:00:01Z", + "text": "the script exports GITHUB_TOKEN=ghp_example", + "tool_uses": [{ + "id": "toolu_sub", + "name": "Read", + "input": {"file_path": "/srv/deploy.sh"}, + "result": {"content": "export GITHUB_TOKEN=ghp_example", "is_error": false} + }], + "file_mutations": [{ + "path": "deploy.sh", + "raw_diff": "--- a/deploy.sh\n+++ b/deploy.sh\n@@ -1 +1 @@\n-old\n+new\n", + "before": "old\n", + "after": "new\n" + }] + }] + }] + }), + )]) + } + + fn fixture_unknown_change_type() -> Path { + path_of(vec![step_with( + "evt-1", + vec![( + "claude://sess-abc", + change( + "conversation.event", + None, + json!({ + "event_type": "attachment", + "data": {"a/b": "slash in the key", "nested": ["leaf", 7]} + }), + ), + )], + )]) + } + + /// Every branch of the map in one document, for the whole-surface + /// read/write sweeps. + fn fixture_rich() -> Path { + let mut path = path_of(vec![ + step_with( + "turn-0f3a", + vec![ + ( + "claude://sess-abc", + fixture_conversation_append().steps[0].change["claude://sess-abc"].clone(), + ), + ( + "~/notes.md", + change( + "file.write", + Some("--- a/notes.md\n+++ b/notes.md\n@@ -1 +1 @@\n-a\n+b\n"), + json!({ + "before": "a\n", + "after": "b\n", + "edits": [{"old_string": "a", "new_string": "b", "replace_all": false}] + }), + ), + ), + ], + ), + fixture_with_delegation().steps.remove(0), + fixture_unknown_change_type().steps.remove(0), + ]); + path.path.base = Some(Base::vcs("https://alex:tok@github.com/o/r", "abc123")); + path.meta = Some(PathMeta { + extra: object(json!({"vcs_remote": "https://alex:tok@github.com/o/r.git"})), + ..PathMeta::default() + }); + path + } + + fn ats(path: &Path) -> Vec { + surfaces(path).into_iter().map(|s| s.at).collect() + } + + #[test] + fn ptr_escape_handles_urls_and_tildes() { + assert_eq!(ptr_escape("claude://sess-abc"), "claude:~1~1sess-abc"); + assert_eq!(ptr_escape("src/config.rs"), "src~1config.rs"); + assert_eq!(ptr_escape("a~b"), "a~0b"); + assert_eq!(ptr_escape("~/x"), "~0~1x"); + } + + #[test] + fn ptr_escape_round_trips() { + for raw in ["claude://sess-abc", "src/config.rs", "a~b/c", "~01"] { + // Decode `~1` before `~0`, or `~01` round-trips wrong (RFC 6901). + let dec = ptr_escape(raw).replace("~1", "/").replace("~0", "~"); + assert_eq!(dec, raw); + } + } + + #[test] + fn conversation_append_surfaces_all_text_fields() { + let p = fixture_conversation_append(); + let all = surfaces(&p); + let ats: Vec<&str> = all.iter().map(|s| s.at.as_str()).collect(); + assert!(ats.iter().any(|a| a.ends_with("/structural/extra/text"))); + assert!( + ats.iter() + .any(|a| a.ends_with("/structural/extra/thinking")) + ); + assert!(ats.iter().any(|a| a.contains("/tool_uses/0/input"))); + assert!( + ats.iter() + .any(|a| a.contains("/tool_uses/0/result/content")) + ); + } + + #[test] + fn file_write_surfaces_diff_and_both_file_states() { + let p = fixture_file_write(); + let shapes: Vec = surfaces(&p).iter().map(|s| s.shape).collect(); + assert!(shapes.contains(&FieldShape::UnifiedDiff)); + assert_eq!( + shapes + .iter() + .filter(|s| **s == FieldShape::FileContent) + .count(), + 2 + ); + } + + #[test] + fn identity_fields_are_never_surfaced() { + for s in surfaces(&fixture_conversation_append()) { + for banned in [ + "/step/id", + "/step/actor", + "/step/timestamp", + "/step/parents", + ] { + assert!( + !s.at.starts_with(banned), + "surfaced identity field: {}", + s.at + ); + } + } + } + + #[test] + fn clean_field_still_appears_as_a_surface() { + // The dry-run guarantee: a surface with nothing in it is information. + let p = fixture_clean_conversation(); + assert!( + surfaces(&p) + .iter() + .any(|s| s.at.ends_with("/structural/extra/text")) + ); + } + + #[test] + fn delegations_recurse() { + assert!( + surfaces(&fixture_with_delegation()) + .iter() + .any(|s| s.at.contains("/delegations/0/turns/0")) + ); + } + + #[test] + fn unknown_change_type_degrades_to_blind_walk() { + assert!(!surfaces(&fixture_unknown_change_type()).is_empty()); + } + + #[test] + fn cursor_write_is_readable_at_the_same_pointer() { + let mut p = fixture_conversation_append(); + let at = surfaces(&p)[0].at.clone(); + let step = surfaces(&p)[0].step.clone(); + let mut c = SurfaceCursor { path: &mut p }; + c.write(&step, &at, "replaced").unwrap(); + assert_eq!(c.read(&step, &at).as_deref(), Some("replaced")); + } + + #[test] + fn surfaces_are_deterministic_across_equal_documents() { + // Two separately built documents, so the `HashMap`s carry different + // seeds: comparing one document against itself would not catch this. + let extras = json!({ + "event_type": "attachment", "a": "1", "b": "2", "c": "3", + "d": "4", "e": "5", "f": "6", "g": "7" + }); + let build = || { + path_of(vec![step_with( + "evt-1", + vec![ + ( + "z://one", + change("conversation.event", None, extras.clone()), + ), + ( + "y://two", + change("conversation.event", None, extras.clone()), + ), + ( + "x://three", + change("conversation.event", None, extras.clone()), + ), + ( + "w://four", + change("conversation.event", None, extras.clone()), + ), + ], + )]) + }; + assert_eq!(surfaces(&build()), surfaces(&build())); + let once = build(); + assert_eq!(surfaces(&once), surfaces(&once)); + } + + #[test] + fn artifact_key_surface_follows_its_children() { + let ats = ats(&fixture_file_write()); + let key = ats.iter().position(|a| a == "/change/src~1config.rs"); + let raw = ats.iter().position(|a| a == "/change/src~1config.rs/raw"); + assert!(key > raw, "the key rewrite must come last: {ats:?}"); + } + + #[test] + fn bytes_is_byte_length_not_char_length() { + let p = append_step("turn-mb", json!({"text": "héllo"})); + let s = surfaces(&path_of(vec![p])); + let text = s.iter().find(|s| s.at.ends_with("/text")).unwrap(); + assert_eq!(text.bytes, 6, "expected UTF-8 byte length, not chars"); + } + + #[test] + fn empty_fields_are_not_surfaced() { + let p = path_of(vec![append_step( + "turn-empty", + json!({"text": "", "thinking": "kept"}), + )]); + let ats = ats(&p); + assert!(!ats.iter().any(|a| a.ends_with("/extra/text")), "{ats:?}"); + assert!(ats.iter().any(|a| a.ends_with("/extra/thinking"))); + } + + #[test] + fn tool_input_recurses_to_string_leaves() { + let p = path_of(vec![append_step( + "turn-nested", + json!({"tool_uses": [{ + "input": {"env": {"AWS_KEY": "AKIA"}, "argv": ["sh", "-c"], "retries": 3} + }]}), + )]); + let ats = ats(&p); + let leaf = "/change/claude:~1~1sess-abc/structural/extra/tool_uses/0/input"; + assert!(ats.contains(&format!("{leaf}/env/AWS_KEY")), "{ats:?}"); + assert!(ats.contains(&format!("{leaf}/argv/0"))); + assert!(!ats.iter().any(|a| a.ends_with("/retries"))); + } + + #[test] + fn blind_walk_escapes_object_keys() { + let p = fixture_unknown_change_type(); + let at = "/change/claude:~1~1sess-abc/structural/extra/data/a~1b"; + assert!(ats(&p).contains(&at.to_string()), "{:?}", ats(&p)); + + let mut p = p; + let mut c = SurfaceCursor { path: &mut p }; + assert_eq!(c.read("evt-1", at).as_deref(), Some("slash in the key")); + c.write("evt-1", at, "x").unwrap(); + assert_eq!(c.read("evt-1", at).as_deref(), Some("x")); + } + + #[test] + fn escaped_artifact_keys_round_trip() { + let mut p = fixture_rich(); + let mut c = SurfaceCursor { path: &mut p }; + assert_eq!( + c.read("turn-0f3a", "/change/~0~1notes.md/structural/extra/before") + .as_deref(), + Some("a\n") + ); + c.write("turn-0f3a", "/change/~0~1notes.md", "~/redacted.md") + .unwrap(); + assert!(p.steps[0].change.contains_key("~/redacted.md")); + assert!(!p.steps[0].change.contains_key("~/notes.md")); + } + + #[test] + fn artifact_key_rename_onto_an_existing_key_is_refused() { + let mut p = fixture_rich(); + let mut c = SurfaceCursor { path: &mut p }; + assert!(matches!( + c.write("turn-0f3a", "/change/~0~1notes.md", "claude://sess-abc"), + Err(RedactError::PlanMismatch(_)) + )); + assert!(p.steps[0].change.contains_key("~/notes.md")); + } + + #[test] + fn every_surface_reads_back() { + let mut p = fixture_rich(); + let all = surfaces(&p); + assert!(all.len() > 15, "fixture is too thin: {}", all.len()); + let c = SurfaceCursor { path: &mut p }; + for s in &all { + assert!( + c.read(&s.step, &s.at).is_some(), + "surface does not resolve: {}", + s.at + ); + } + } + + #[test] + fn every_surface_is_writable_in_emitted_order() { + let mut p = fixture_rich(); + let all = surfaces(&p); + let mut c = SurfaceCursor { path: &mut p }; + for (i, s) in all.iter().enumerate() { + // Unique values: two artifact keys redacting alike would collide. + c.write(&s.step, &s.at, &format!("v{i}")) + .unwrap_or_else(|e| panic!("{} failed: {e}", s.at)); + } + } + + #[test] + fn writing_an_unresolvable_pointer_errors() { + let mut p = fixture_rich(); + let mut c = SurfaceCursor { path: &mut p }; + for (step, at) in [ + ("turn-0f3a", "/change/nope~1missing.rs/raw"), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/extra/nope", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/extra/tool_uses/9/input", + ), + // `token_usage` is an object, not a string leaf. + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/extra/token_usage", + ), + ( + "no-such-step", + "/change/claude:~1~1sess-abc/structural/extra/text", + ), + ("turn-0f3a", "/step/actor"), + ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/text"), + ("", "/meta/not_present"), + ] { + assert!( + matches!(c.write(step, at, "x"), Err(RedactError::BadPointer(_))), + "expected BadPointer for {at}" + ); + assert_eq!(c.read(step, at), None, "read must agree with write on {at}"); + } + } + + #[test] + fn document_level_uris_round_trip() { + let mut p = fixture_rich(); + let mut c = SurfaceCursor { path: &mut p }; + for at in ["/path/base/uri", "/meta/vcs_remote"] { + assert!(c.read("", at).is_some(), "{at}"); + c.write("", at, "https://github.com/o/r").unwrap(); + assert_eq!(c.read("", at).as_deref(), Some("https://github.com/o/r")); + } + } + + #[test] + fn delegated_turns_surface_their_own_shapes() { + let p = fixture_with_delegation(); + let base = "/change/claude:~1~1sess-abc/structural/extra/delegations/0"; + let by_at: HashMap = + surfaces(&p).into_iter().map(|s| (s.at, s.shape)).collect(); + for (at, shape) in [ + (format!("{base}/prompt"), FieldShape::Prose), + (format!("{base}/result"), FieldShape::Prose), + (format!("{base}/turns/0/text"), FieldShape::Prose), + ( + format!("{base}/turns/0/tool_uses/0/input/file_path"), + FieldShape::ToolInput, + ), + ( + format!("{base}/turns/0/tool_uses/0/result/content"), + FieldShape::ToolOutput, + ), + ( + format!("{base}/turns/0/file_mutations/0/raw_diff"), + FieldShape::UnifiedDiff, + ), + ( + format!("{base}/turns/0/file_mutations/0/before"), + FieldShape::FileContent, + ), + ] { + assert_eq!(by_at.get(&at), Some(&shape), "missing or mistyped: {at}"); + } + } + + #[test] + fn file_write_edits_surface_both_sides() { + let p = fixture_rich(); + let base = "/change/~0~1notes.md/structural/extra/edits/0"; + let ats = ats(&p); + assert!(ats.contains(&format!("{base}/old_string")), "{ats:?}"); + assert!(ats.contains(&format!("{base}/new_string"))); + assert!(!ats.iter().any(|a| a.ends_with("/replace_all"))); + } } From 39ce229a76555c0f7ee052f855f8dd4cc38e9278 Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 15:54:51 -0400 Subject: [PATCH 5/9] feat(redact): apply, plan generation, internal detector, sync replay Completes the engine. T3 vendors the gitleaks ruleset (221 rules load; 1 is path-only with no regex, 3 blow past regex's 10 MiB compiled-size limit under RE2 semantics, all four excluded with reasons). T7 rewrites documents from an approved plan. T8 generates plans. T10 replays a stored policy after a re-derive. T10 closes the hazard the plan was written around: `is_unchanged` decides re-derivation from mtime and size and never reads the document, so a resumed session's re-derive would silently overwrite an in-place redaction. A missing key now fails that artifact before any write rather than publishing the secret it was hiding. `p cache rm` retires the policy alongside the key, not just the key: a policy whose key is gone would fail that artifact's sync forever with no recovery path. Sync replay is unattended, so it refuses to strip signatures or to run a network detector without a human present. Applies the T6 and T12 reviews. Two were hard breaks: the toolpath-cli shim still pinned path-cli 0.16.0, which broke `cargo metadata` on the shim outright, and release.sh listed toolpath-redact in ALL_CRATES but not in either tier-2 publish loop, so tier 3 would have failed against crates.io on an unresolvable dependency. Also adds a threshold range check - scores clamp to 0.0..=1.0, so `--threshold -1` silently meant "redact everything" and `5` meant "redact nothing". Green: toolpath-redact 140, path-cli 379 lib + 56 integration. Not implemented: T9 CLI dispatch, T11 integration tests, exec.rs. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- CLAUDE.md | 5 +- README.md | 2 + crates/path-cli/src/cache.rs | 2 + crates/path-cli/src/cmd_cache.rs | 89 +- crates/path-cli/src/cmd_redact.rs | 57 +- crates/path-cli/src/sync/engine.rs | 653 +++++++++++- crates/toolpath-cli/Cargo.toml | 4 +- crates/toolpath-redact/README.md | 12 +- crates/toolpath-redact/src/apply.rs | 1063 +++++++++++++++++++- crates/toolpath-redact/src/internal/mod.rs | 7 +- crates/toolpath-redact/src/plan.rs | 396 +++++++- scripts/release.sh | 4 +- 13 files changed, 2239 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 634793d9..4cd46ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to the Toolpath workspace are documented here. -## `path p redact` — plan-then-apply credential redaction — 2026-07-30 +## `path p redact` - plan-then-apply credential redaction - 2026-07-30 Adds `path p redact`, a plumbing command that removes credentials from an already-generated toolpath document in place, via a reviewable plan-then-apply diff --git a/CLAUDE.md b/CLAUDE.md index c74de81a..0bbdffe2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,6 +31,7 @@ crates/ # (spec at crates/pathbase-client/openapi.json; refresh via scripts/refresh-pathbase-openapi.sh) schema/toolpath.schema.json # JSON Schema for the toolpath format examples/*.json # 12 example documents (step, path, graph) +docs/superpowers/ # specs/ (closed designs) and plans/ (task decompositions) RFC.md # full format specification FAQ.md # design rationale, FAQ, and open questions ``` @@ -236,7 +237,7 @@ The Tauri 2 desktop GUI lives in the private [pathbase](https://github.com/empat ## Superpowers -`docs/superpowers/` holds implementation specifications and task plans for substantial features — think design docs plus a numbered task decomposition for parallel implementation. `specs/` subdirectory carries the closed design; `plans/` carries the task breakdown with test-first structure. These are code artifacts, not documentation: they specify invariants, gate criteria, and concrete assertions. +`docs/superpowers/` holds implementation specifications and task plans for substantial features, meaning design docs plus a numbered task decomposition for parallel implementation. `specs/` subdirectory carries the closed design; `plans/` carries the task breakdown with test-first structure. These are code artifacts, not documentation: they specify invariants, gate criteria, and concrete assertions. ## Versioning and release checklist @@ -295,5 +296,5 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. - `path query` does not load the whole cache into memory when it can avoid it. `crates/path-cli/src/query/plan.rs` parses the jaq filter into jaq's own AST (`jaq_core::load::parse::Term`) and classifies it into a `Plan`: `PerFileStream` (`.[] | g` element-wise work — run per document, print as you go), `Decompose { reduce }` (algebraic aggregations — run the whole filter per file, concatenate the per-file outputs, then run a derived combine: `map`→`add` (array concat), top-N `sort_by(k)|.[:N]`→`add | sort_by(k)|.[:N]`, `length`→`add` over exact integer counts), or `Slurp` (the always-correct whole-array fallback). Recognition is conservative — a non-distributive prefix like `unique`/`group_by` slurps, and so do scalar `add` (float sums re-associate across per-file partials), `min`/`max` (`[] | min == null` poisons the merge), and any unrecognized tail — so **the planner never changes an answer** — `crates/path-cli/src/query/filter.rs` tests assert streamed output equals slurp byte-for-byte. `filter::execute` compiles the filter once (jaq's compiled `Filter` is fully owned, so it's reused across files) and drives the plan; `mod.rs::stream_files` yields one document's wrapped steps at a time. `TOOLPATH_QUERY_EXPLAIN=1` prints the chosen plan to stderr. No user-facing flag — it's automatic. Tie-break caveat: a streamed top-N matches slurp's *ranking*, but boundary ties may resolve to different specific rows. - Cache sync: `path p cache sync [types…]` (`crates/path-cli/src/artifact.rs`: `ArtifactType` + `ArtifactRef` + the stamp helpers; `sync/engine.rs`: manifest + ingestion loop, no UI — it reports through a `SyncObserver` trait, `&mut ()` for a silent sync; `sync/sources.rs`: an `ArtifactSource` trait — enumerate / stamp / derive — with one impl per provider, so the engine never matches on artifact type; `cmd_cache.rs`: the stderr progress line + summary) incrementally ingests artifacts into the cache — no args syncs every artifact type. Change detection is **stat-level**: each artifact is enumerated as an `ArtifactRef` whose fingerprint is the source file's mtime + size (claude: the *whole session chain* — max segment mtime + summed segment sizes via `claude_chain_stamp`, because Claude Code rotates to a new file on continuation while the chain keeps its oldest segment's id, so appends land in the newest file, not the head; the chain comes from the same cached index `list_conversations` builds; codex: rollout file, id from the stem's trailing UUID; pi: session file, id from a one-line header peek; copilot: `session-state//events.jsonl`, pure read-dir + stat) or the DB row's updated-at (opencode: header-only `SELECT time_updated`; cursor: composer headers' `lastUpdatedAt`, bubble-less drafts skipped, workspace-less composers *included* unlike `share`). Gemini enumerates via `PathResolver::list_session_entries` (`toolpath-gemini` 0.6.1), whose identity peek is bounded to the first 4 KiB of a main file. Deciding "nothing changed" reads no session bodies — a no-op sync is milliseconds. Changed/new artifacts derive through the same provider managers (each source calls the `derive_*_session_with` helpers in `derive.rs`). Manifest at `~/.toolpath/manifest.json`: artifact type → artifact id → `{path?, cache_id, modified?, size?, synced_at}`; atomic temp+rename writes, `0600`, checkpointed every 10 writes (interruption-safe: a killed run keeps nearly everything it derived, and derives run newest-first so partial progress covers the sessions that matter most); writers serialize on an advisory lock (`manifest.json.lock`) and every write is a locked read-merge-save — checkpoints merge only the records the run wrote — so concurrent invocations (query auto-syncs, imports) union their records instead of clobbering each other. Pending work reports progress on stderr (`\r`-updating ` done/total` on a TTY, a plain line every 25 items otherwise; no-op syncs stay silent). Sync always writes the cache with force — refresh semantics — and never deletes: artifacts removed upstream keep their cache docs and manifest records (archive, not mirror). Derivation failures warn and tally, they don't abort. A record's `cache_id` is *optional*: a record without one is "known, not materialized" — created when `p cache rm` evicts a doc (rm downgrades the record; the next sync re-materializes it, and sync also verifies the doc file actually exists before skipping, so even out-of-band deletions self-heal). Claude derives leave `DeriveConfig.project_path` unset so `path.base` comes from the session's own recorded cwd rather than the lossy slug. `path query` runs this sync implicitly before reading, scoped to its flags (`--source X` → that type; `--id`s → their prefixes; bare query → all types; `--input`-only → none), quiet unless something was ingested, degrading to the cache as-is if sync fails; `--no-sync` opts out. `p import` and `share` record what they write: every session derive carries a provenance `ArtifactRef` (stamped *before* the source is read, in `DerivedDoc.provenance`), and the cache-write sites call `sync::record_artifact` so the next sync sees those artifacts as unchanged instead of re-deriving them. Every import flow — explicit `--session`, picker multi-select, `--all`, and the most-recent fallbacks — loops the per-session helpers, so every session write is recorded; there is no bulk `derive_project` path in the CLI anymore, and `p import pi --all` now emits one Path per session like every other provider (it used to emit a single combined Graph). `--no-cache` paths record nothing: the manifest describes the cache. -- Redaction workflow (`toolpath-redact` + `path p redact`): plan-then-apply, not a single opaque pass. `surfaces()` enumerates every string field a credential could hide in. `plan::generate()` runs detectors over those surfaces, producing a reviewable `Plan` with stable ids and elided context. `apply()` consumes the plan and rewrites the document. The plan can be decided by predicate, by picker, or by hand-editing JSON. In-place redaction (`--input `) rewrites the cache entry; re-deriving from the source (via `path query` or `path resume`) replays the stored `RedactionPolicy` automatically, so new turns in the session get redacted too. A detector is a trait (`Detector`), not a function, so its implementation is swappable — a `FixedDetector` for tests, the built-in rule-based detector by default, and a harness-time hook for pre-training redaction. Detection is the part of this problem where precision is worst and the field moves fastest, so it sits behind the plug point; traversal (fields, pointers, surfaces) is stable and reused by all detectors. +- Redaction workflow (`toolpath-redact` + `path p redact`): plan-then-apply, not a single opaque pass. `surfaces()` enumerates every string field a credential could hide in. `plan::generate()` runs detectors over those surfaces, producing a reviewable `Plan` with stable ids and elided context. `apply()` consumes the plan and rewrites the document. The plan can be decided by predicate, by picker, or by hand-editing JSON. In-place redaction (`--input `) rewrites the cache entry; re-deriving from the source (`p cache sync`, including the implicit sync `path query` runs) replays the stored `RedactionPolicy`, so turns added after a resume are redacted too. A detector is a trait (`Detector`), not a function, so its implementation is swappable: a `FixedDetector` for tests, the built-in rule-based detector by default, and a harness-time hook for pre-training redaction. Detection is the part of this problem where precision is worst and the field moves fastest, so it sits behind the plug point; traversal (fields, pointers, surfaces) is stable and reused by all detectors. - `ArtifactType` (`crates/path-cli/src/artifact.rs`) is the general enum naming artifact sources — the seven agent harnesses (incl. copilot) plus `Git` (8 variants). Git artifacts are *recorded* in the manifest by `p import git` (id `-`, `path` = the repo directory) but never *discovered* — there is no machine-wide registry of repos — so sync reports them and leaves them alone. Github and pathbase are deliberately not artifact types: they are remote services, not local artifact sources, and their imports stay out of the manifest. It derives `clap::ValueEnum` and is used by `p cache sync` types, the sync manifest keys, `ArtifactRow.artifact_type`, and `cmd_import`'s cache-id prefixes (`name()` is both the manifest key and the `make_id` source string). The deliberately parallel `Harness` enum (`crates/path-cli/src/harness.rs`, alongside `HarnessBundle`) names the seven agent *runtimes* — things sessions can be shared from and resumed into — and is what `share`/`resume` `--harness` take, so future non-harness artifact types stay unrepresentable there (you can't resume into a git repo). `Harness::artifact_type()` maps into the general enum; `ArtifactType::harness()` is the partial inverse. Keep new code on `ArtifactType` unless it's genuinely harness-only. diff --git a/README.md b/README.md index 3f48f458..6b211b1a 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ path md [--input FILE] [--output FILE] [--detail summary|full] [--front-matter] merge FILE... [--title TEXT] validate --input FILE + redact --input REF [--dry-run | --plan FILE] [--accept PRED]... [--reject PRED]... + [--mode marker|remove|hash|mask|partial] [--threshold N] [--reveal] [--output FILE] derive # stdout-JSON sibling of import (same sources, --no-cache implied) project # narrower file-shaped sibling of export incept # file/stdin-shaped sibling of `export --project` (claude, cursor) diff --git a/crates/path-cli/src/cache.rs b/crates/path-cli/src/cache.rs index 156fe73b..5bc7ff17 100644 --- a/crates/path-cli/src/cache.rs +++ b/crates/path-cli/src/cache.rs @@ -189,6 +189,8 @@ pub(crate) fn read_redact_key(key_id: &str) -> Result>> { /// Written with `create_new`, so two redactions racing the same /// document agree on one key instead of each fingerprinting under its /// own. +// Called by `path p redact`; drop the allow once that dispatch lands. +#[allow(dead_code)] pub(crate) fn load_or_create_redact_key(key_id: &str) -> Result> { use rand::RngCore; use std::io::Write; diff --git a/crates/path-cli/src/cmd_cache.rs b/crates/path-cli/src/cmd_cache.rs index ac825fee..08201f63 100644 --- a/crates/path-cli/src/cmd_cache.rs +++ b/crates/path-cli/src/cmd_cache.rs @@ -57,6 +57,11 @@ fn run_ls() -> Result<()> { fn run_rm(id: &str) -> Result<()> { remove_cached(id)?; + // The key exists only to keep this document's fingerprints stable + // across re-redactions; with the document gone nothing can use it. + if let Err(e) = crate::cache::remove_redact_key(id) { + eprintln!("warning: redaction key not removed: {e}"); + } // The artifact is still real — downgrade its manifest record to // "known, not cached" so the next sync can re-materialize it. #[cfg(not(target_os = "emscripten"))] @@ -192,12 +197,44 @@ fn render_summary(outcomes: &[(ArtifactType, SyncOutcome)], explicit: bool) -> S } s.push('\n'); } + s.push_str(&render_replay(outcomes)); if s.is_empty() { s.push_str("nothing to sync\n"); } s } +/// Redaction replay, across every type in the run. Reappeared skips get +/// their own clause because they are the one way a replayed document +/// differs from what the user approved: a hand-picked skip cannot be +/// replayed against content that has moved, so it comes back. +#[cfg(not(target_os = "emscripten"))] +fn render_replay(outcomes: &[(ArtifactType, SyncOutcome)]) -> String { + let documents: usize = outcomes.iter().map(|(_, o)| o.re_redacted).sum(); + if documents == 0 { + return String::new(); + } + let reappeared: usize = outcomes.iter().map(|(_, o)| o.reappeared_skips).sum(); + let mut line = format!("re-redacted {documents} {}", plural(documents, "document")); + if reappeared > 0 { + line.push_str(&format!( + "; {reappeared} previously-skipped {} reappeared", + plural(reappeared, "finding") + )); + } + line.push('\n'); + line +} + +#[cfg(not(target_os = "emscripten"))] +fn plural(n: usize, noun: &str) -> String { + if n == 1 { + noun.to_string() + } else { + format!("{noun}s") + } +} + #[cfg(all(test, not(target_os = "emscripten")))] mod tests { use super::*; @@ -237,7 +274,7 @@ mod tests { new: 2, updated: 1, unchanged: 3, - failed: 0, + ..Default::default() }, ), (ArtifactType::Cursor, SyncOutcome::default()), @@ -255,10 +292,9 @@ mod tests { let outcomes = vec![( ArtifactType::Codex, SyncOutcome { - new: 0, - updated: 0, unchanged: 1, failed: 2, + ..Default::default() }, )]; let s = render_summary(&outcomes, false); @@ -266,4 +302,51 @@ mod tests { assert_eq!(render_summary(&[], false), "nothing to sync\n"); } + + #[test] + fn render_summary_reports_replay_across_types() { + let outcomes = vec![ + ( + ArtifactType::Claude, + SyncOutcome { + updated: 2, + re_redacted: 2, + reappeared_skips: 2, + ..Default::default() + }, + ), + ( + ArtifactType::Codex, + SyncOutcome { + updated: 1, + re_redacted: 1, + ..Default::default() + }, + ), + ]; + assert!( + render_summary(&outcomes, false) + .ends_with("re-redacted 3 documents; 2 previously-skipped findings reappeared\n") + ); + } + + #[test] + fn render_replay_is_silent_without_replay_and_singular_for_one() { + assert_eq!( + render_replay(&[(ArtifactType::Claude, SyncOutcome::default())]), + "" + ); + let one = vec![( + ArtifactType::Claude, + SyncOutcome { + re_redacted: 1, + reappeared_skips: 1, + ..Default::default() + }, + )]; + assert_eq!( + render_replay(&one), + "re-redacted 1 document; 1 previously-skipped finding reappeared\n" + ); + } } diff --git a/crates/path-cli/src/cmd_redact.rs b/crates/path-cli/src/cmd_redact.rs index 5a575778..9f530ad4 100644 --- a/crates/path-cli/src/cmd_redact.rs +++ b/crates/path-cli/src/cmd_redact.rs @@ -34,7 +34,8 @@ pub(crate) struct RedactArgs { #[arg(long, default_values = &["internal"])] pub detector: Vec, - #[arg(long, default_value_t = 0.8)] + /// Minimum score a finding needs before it is redacted (0.0-1.0). + #[arg(long, default_value_t = 0.8, value_parser = parse_threshold)] pub threshold: f32, #[arg(long)] pub allow_network_detectors: bool, @@ -59,27 +60,55 @@ pub(crate) enum TransformArg { Partial, } +/// Scores are clamped to `0.0..=1.0`, so an out-of-range threshold silently +/// inverts the command: `-1` redacts everything, `5` and `NaN` redact +/// nothing. `RangeInclusive::contains` rejects NaN and infinity for free. +fn parse_threshold(s: &str) -> std::result::Result { + let v: f32 = s.parse().map_err(|_| format!("`{s}` is not a number"))?; + if !(0.0..=1.0).contains(&v) { + return Err(format!("--threshold must be in 0.0..=1.0, got {v}")); + } + Ok(v) +} + pub(crate) fn run(args: RedactArgs) -> Result<()> { + run_with_picker(args, &RealPicker) +} + +/// The seam the interactive tests inject through. Mirrors +/// `cmd_resume::run_with_strategy`, which exists for the same reason: the +/// alternative is a process-global picker override that one test poisons +/// for the whole binary. +pub(crate) fn run_with_picker(_args: RedactArgs, _picker: &dyn PickerStrategy) -> Result<()> { todo!("T9") } pub(crate) trait PickerStrategy { + /// Rows are TSV with the finding id in column 1. Returns the selected + /// rows verbatim, the way `fzf` does - not bare ids. + /// + /// Unused until `run_with_picker` stops being a `todo!()`. + #[allow(dead_code)] fn pick(&self, rows: &[String]) -> Result>; } pub(crate) struct RealPicker; impl PickerStrategy for RealPicker { - fn pick(&self, rows: &[String]) -> Result> { + fn pick(&self, _rows: &[String]) -> Result> { todo!("T9") } } +/// Constructed by the dispatch tests, which arrive with `run_with_picker`. +#[cfg(test)] +#[allow(dead_code)] pub(crate) struct RecordingPicker { pub selection: Vec, pub seen: std::cell::RefCell>, } +#[cfg(test)] impl PickerStrategy for RecordingPicker { fn pick(&self, rows: &[String]) -> Result> { *self.seen.borrow_mut() = rows.to_vec(); @@ -87,21 +116,23 @@ impl PickerStrategy for RecordingPicker { } } -/// Helper to parse a "PREDICATE:TRANSFORM" string. -/// Splits on the LAST `:` so predicates containing `:` still parse. +/// Splits on the LAST `:` so a predicate containing `:` still parses, e.g. +/// `detector=exec:/bin/gitleaks:hash`. +#[cfg_attr(not(test), allow(dead_code))] pub(crate) fn parse_mode_for(s: &str) -> Result<(String, TransformArg)> { let (pred, transform_str) = s .rsplit_once(':') .ok_or_else(|| anyhow::anyhow!("--mode-for format is PREDICATE:TRANSFORM, got: {}", s))?; - - let transform = match transform_str { - "marker" => TransformArg::Marker, - "remove" => TransformArg::Remove, - "hash" => TransformArg::Hash, - "mask" => TransformArg::Mask, - "partial" => TransformArg::Partial, - other => anyhow::bail!("unknown transform: {}", other), - }; + if pred.is_empty() { + anyhow::bail!( + "--mode-for needs a predicate before the transform, got: {}", + s + ); + } + // Resolved through clap's own value table rather than a second hand-rolled + // one, which would silently drift from `TransformArg` as variants are added. + let transform = ::from_str(transform_str, false) + .map_err(|e| anyhow::anyhow!("{e}"))?; Ok((pred.to_string(), transform)) } diff --git a/crates/path-cli/src/sync/engine.rs b/crates/path-cli/src/sync/engine.rs index ecb27717..cdddbaf7 100644 --- a/crates/path-cli/src/sync/engine.rs +++ b/crates/path-cli/src/sync/engine.rs @@ -8,10 +8,14 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::PathBuf; +use toolpath::v1::{Graph, PathOrRef}; +use toolpath_redact::{Action, Decision, DetectorSet, RedactConfig, RedactionPolicy}; + use super::sources::{self, ArtifactSource}; use crate::artifact::{ArtifactRef, ArtifactType}; use crate::cache::write_cached; use crate::config::{MANIFEST_FILE_NAME, MANIFEST_LOCK_FILE_NAME, config_dir}; +use crate::derive::DerivedDoc; use crate::harness::HarnessBundle; /// How many manifest writes accumulate before a mid-run checkpoint. @@ -61,9 +65,19 @@ pub(crate) struct SyncOutcome { pub(crate) updated: usize, pub(crate) unchanged: usize, pub(crate) failed: usize, + /// Documents whose stored redaction policy was replayed over the + /// re-derived content. A subset of `updated`, tallied separately + /// because it answers a different question. + pub(crate) re_redacted: usize, + /// Findings a previous redaction individually skipped that the + /// replayed policy redacts again — a hand-picked skip refers to + /// content that has since moved, so it cannot be replayed. + pub(crate) reappeared_skips: usize, } impl SyncOutcome { + /// Artifacts seen, by disposition. The redaction counters are a + /// different axis and are deliberately not summed in. pub(crate) fn total(&self) -> usize { self.new + self.updated + self.unchanged + self.failed } @@ -119,6 +133,7 @@ pub(crate) fn sync_bundle( &artifacts, &records, observer, + build_detectors, )?; out.push((artifact_type, outcome)); } @@ -178,6 +193,7 @@ fn sync_artifacts( artifacts: &[ArtifactRef], records: &BTreeMap, observer: &mut dyn SyncObserver, + detectors: DetectorFactory, ) -> Result { let mut outcome = SyncOutcome::default(); // Evaluate the stat gate once per artifact: the pass feeds both the @@ -190,6 +206,7 @@ fn sync_artifacts( observer.begin(artifact_type, pending_total); let mut writes: BTreeMap<&'static str, BTreeMap> = BTreeMap::new(); let mut unflushed = 0usize; + let mut detectors = DetectorCache::new(detectors); for (artifact, unchanged) in order { if unchanged { outcome.unchanged += 1; @@ -210,8 +227,9 @@ fn sync_artifacts( .path .clone() .or_else(|| existing.and_then(|r| r.path.clone())); - match source.derive(artifact) { - Ok(derived) => { + let policy = existing.and_then(|r| r.redaction.as_ref()); + match derive_replaying_redaction(source, artifact, policy, &mut detectors) { + Ok((derived, replay)) => { // force: sync owns refresh semantics — a re-sync or a // prior manual `p import` of the same session must not // error on the existing cache entry. @@ -227,8 +245,11 @@ fn sync_artifacts( modified: artifact.modified, size: artifact.size, synced_at: Utc::now(), + redaction: policy.cloned(), }, ); + outcome.re_redacted += replay.documents; + outcome.reappeared_skips += replay.reappeared_skips; unflushed += 1; if is_new { outcome.new += 1; @@ -252,23 +273,254 @@ fn sync_artifacts( Ok(outcome) } +// ── redaction replay ─────────────────────────────────────────────── + +/// How replay rebuilds the detectors a stored policy names. A parameter +/// rather than a direct call so tests drive replay without the vendored +/// ruleset. +type DetectorFactory = fn(&[String]) -> Result; + +/// What one artifact's replay did, for the sync summary. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct Replay { + documents: usize, + reappeared_skips: usize, +} + +/// The detector registry replay resolves policy names against. +/// +/// Returning `None` fails the artifact rather than replaying with a +/// weaker detector set than the policy was written for, which would +/// write back a document missing redactions it had. +fn detector_named(name: &str) -> Option> { + match name { + "internal" => Some(Box::new(toolpath_redact::internal::InternalDetector::new())), + _ => None, + } +} + +fn build_detectors(names: &[String]) -> Result { + let mut set = DetectorSet::default(); + for name in names { + let detector = detector_named(name).ok_or_else(|| { + anyhow!("no detector named {name:?}; cannot replay this document's redaction") + })?; + set.push(detector); + } + Ok(set) +} + +/// One detector set per distinct name list per run. The built-in +/// detector compiles a few hundred regexes on construction, and a sync +/// typically replays one policy across many documents. +struct DetectorCache { + build: DetectorFactory, + sets: BTreeMap, DetectorSet>, +} + +impl DetectorCache { + fn new(build: DetectorFactory) -> Self { + Self { + build, + sets: BTreeMap::new(), + } + } + + fn get(&mut self, names: &[String]) -> Result<&DetectorSet> { + if !self.sets.contains_key(names) { + let set = (self.build)(names)?; + self.sets.insert(names.to_vec(), set); + } + Ok(&self.sets[names]) + } +} + +/// Derive an artifact, then replay the redaction policy its record +/// carries. +/// +/// `is_unchanged` never inspects the document, so a re-derive would +/// clobber an in-place redaction. Replay the policy before writing. +/// Either half failing fails the artifact, which leaves the previously +/// redacted document in place — never an un-redacted one over it. +fn derive_replaying_redaction( + source: &dyn ArtifactSource, + artifact: &ArtifactRef, + policy: Option<&RedactionPolicy>, + detectors: &mut DetectorCache, +) -> Result<(DerivedDoc, Replay)> { + let mut derived = source.derive(artifact)?; + let Some(policy) = policy else { + return Ok((derived, Replay::default())); + }; + let replay = replay_policy(&derived.cache_id, policy, &mut derived.doc, detectors)?; + Ok((derived, replay)) +} + +/// Re-redact `doc` under `policy`, counting what a rule-based policy +/// cannot preserve. +fn replay_policy( + cache_id: &str, + policy: &RedactionPolicy, + doc: &mut Graph, + detectors: &mut DetectorCache, +) -> Result { + let key = crate::cache::read_redact_key(&policy.key_id)?.ok_or_else(|| { + anyhow!( + "redaction key {} is missing; refusing to overwrite the redacted document \ + with an un-redacted re-derive", + policy.key_id + ) + })?; + let set = detectors.get(&policy.detectors)?; + let decisions = policy_decisions(policy)?; + let cfg = replay_config(policy, key); + let reappeared_skips = count_reappeared_skips(cache_id, set, &cfg, &decisions); + + for path in doc.paths.iter_mut() { + let PathOrRef::Path(path) = path else { + continue; + }; + let plan = plan_for(path, set, &cfg, &decisions)?; + toolpath_redact::apply(path, &plan, &cfg)?; + } + Ok(Replay { + documents: 1, + reappeared_skips, + }) +} + +fn plan_for( + path: &toolpath::v1::Path, + set: &DetectorSet, + cfg: &RedactConfig, + decisions: &[Decision], +) -> Result { + let mut plan = toolpath_redact::plan::generate_checked( + path, set, cfg, + // A sync runs unattended, so a detector that would send + // candidate material off the machine has nobody to approve it. + false, + )?; + toolpath_redact::plan::apply_decisions(&mut plan, decisions); + Ok(plan) +} + +/// Findings the policy would redact that are still present in the +/// document as it was last redacted: exactly the ones a previous run +/// skipped by hand. Rule-based decisions replay, so anything the +/// `reject` predicates cover is excluded here. +/// +/// Informational only — a document that cannot be read or planned +/// reports nothing rather than failing an otherwise good replay. +fn count_reappeared_skips( + cache_id: &str, + set: &DetectorSet, + cfg: &RedactConfig, + decisions: &[Decision], +) -> usize { + let Ok(file) = crate::cache::cache_path(cache_id) else { + return 0; + }; + let Ok(json) = std::fs::read_to_string(&file) else { + return 0; + }; + let Ok(previous) = Graph::from_json(&json) else { + return 0; + }; + previous + .paths + .iter() + .filter_map(|p| match p { + PathOrRef::Path(path) => plan_for(path, set, cfg, decisions).ok(), + PathOrRef::Ref(_) => None, + }) + .map(|plan| { + plan.findings + .iter() + .filter(|f| f.action == Action::Redact) + .count() + }) + .sum() +} + +/// The policy's predicates as decisions. `reject` lands last because +/// [`toolpath_redact::plan::apply_decisions`] lets later decisions win, +/// so an explicit skip survives a broader accept. +fn policy_decisions(policy: &RedactionPolicy) -> Result> { + let mut out = Vec::with_capacity(policy.accept.len() + policy.reject.len()); + for (predicates, action) in [ + (&policy.accept, Action::Redact), + (&policy.reject, Action::Skip), + ] { + for source in predicates { + out.push(Decision { + predicate: toolpath_redact::parse_predicate(source)?, + action, + transform: None, + }); + } + } + Ok(out) +} + +fn replay_config(policy: &RedactionPolicy, key: Vec) -> RedactConfig { + RedactConfig { + threshold: policy.threshold, + mode: policy.mode, + mode_for: policy.mode_for.clone(), + key, + now: Utc::now(), + // Dropping signatures and revealing values both need a human at + // the terminal; a signed document fails its replay instead, and + // keeps the redacted copy it already has. + drop_signatures: false, + reveal: false, + } +} + +/// Store the policy `path p redact` applied to a cached document, so a +/// later re-derive replays it. Reports whether any record pointed at +/// `cache_id`: a document sync does not track (a file input, a github +/// or pathbase import) has nowhere to keep one, and its redaction will +/// not survive a re-derive because nothing re-derives it. +// Called by `path p redact`; drop the allow once that dispatch lands. +#[allow(dead_code)] +pub(crate) fn record_redaction_policy(cache_id: &str, policy: &RedactionPolicy) -> Result { + let mut recorded = false; + update_manifest(|manifest| { + for records in manifest.values_mut() { + for rec in records.values_mut() { + if rec.cache_id.as_deref() == Some(cache_id) { + rec.redaction = Some(policy.clone()); + recorded = true; + } + } + } + })?; + Ok(recorded) +} + /// Record an externally-derived cache write (`p import`, `share`) in /// the manifest, so sync doesn't re-derive what was just written. pub(crate) fn record_artifact(artifact: &ArtifactRef, cache_id: &str) -> Result<()> { update_manifest(|manifest| { - manifest + let records = manifest .entry(artifact.artifact_type.name().to_string()) - .or_default() - .insert( - artifact.id.clone(), - SyncRecord { - path: artifact.path.clone(), - cache_id: Some(cache_id.to_string()), - modified: artifact.modified, - size: artifact.size, - synced_at: Utc::now(), - }, - ); + .or_default(); + // Re-importing a session must not drop the redaction policy + // guarding it; only `p cache rm` clears that. + let redaction = records.get(&artifact.id).and_then(|r| r.redaction.clone()); + records.insert( + artifact.id.clone(), + SyncRecord { + path: artifact.path.clone(), + cache_id: Some(cache_id.to_string()), + modified: artifact.modified, + size: artifact.size, + synced_at: Utc::now(), + redaction, + }, + ); }) } @@ -326,6 +578,10 @@ pub(crate) fn evict_cache_id(cache_id: &str) -> Result<()> { for rec in records.values_mut() { if rec.cache_id.as_deref() == Some(cache_id) { rec.cache_id = None; + // The key goes with the document, and a policy whose + // key is gone can only fail every future sync — so + // the explicit `rm` retires both together. + rec.redaction = None; } } } @@ -464,6 +720,143 @@ mod tests { doc.single_path().map(|p| p.steps.len()).unwrap_or(0) } + /// A detector named `literal:`, so replay wiring is testable + /// without the vendored ruleset. Claims the whole alphanumeric run + /// starting at the prefix, so one name covers a family of keys the + /// way a real rule does. + struct LiteralDetector(String); + + impl toolpath_redact::Detector for LiteralDetector { + fn id(&self) -> &'static str { + "literal" + } + + fn detect( + &self, + c: &toolpath_redact::Candidate<'_>, + ) -> toolpath_redact::Result> { + Ok(c.text + .match_indices(&self.0) + .map(|(start, m)| { + let tail = &c.text[start + m.len()..]; + let run = tail + .find(|ch: char| !ch.is_ascii_alphanumeric()) + .unwrap_or(tail.len()); + toolpath_redact::Finding { + span: start..start + m.len() + run, + rule: "literal".to_string(), + score: 1.0, + detector: "literal", + } + }) + .collect()) + } + } + + fn literal_detectors(names: &[String]) -> Result { + let mut set = DetectorSet::default(); + for name in names { + let literal = name + .strip_prefix("literal:") + .ok_or_else(|| anyhow!("test detector names are literal:, got {name:?}"))?; + set.push(Box::new(LiteralDetector(literal.to_string()))); + } + Ok(set) + } + + fn literal_policy(secret: &str, key_id: &str) -> RedactionPolicy { + policy_with(vec![format!("literal:{secret}")], key_id) + } + + fn policy_with(detectors: Vec, key_id: &str) -> RedactionPolicy { + RedactionPolicy { + detectors, + threshold: 0.8, + mode: toolpath_redact::Transform::Marker, + mode_for: Vec::new(), + accept: Vec::new(), + reject: Vec::new(), + key_id: key_id.to_string(), + } + } + + /// Sync one type the way `sync_bundle` does, with the test detector + /// registry in place of the built-in one. + fn sync_claude(bundle: &HarnessBundle) -> SyncOutcome { + let source = sources::source_for(bundle, ArtifactType::Claude).unwrap(); + let artifacts = source.enumerate(); + let records = load_manifest() + .unwrap() + .get("claude") + .cloned() + .unwrap_or_default(); + sync_artifacts( + source.as_ref(), + ArtifactType::Claude, + &artifacts, + &records, + &mut (), + literal_detectors, + ) + .unwrap() + } + + /// What `path p redact` leaves behind: a key, a redacted document, + /// and a policy on the artifact's manifest record. `hand` stands in + /// for the interactive picker, which can decide a finding the + /// policy cannot express. + fn redact_cached_with( + cache_id: &str, + policy: &RedactionPolicy, + detectors: DetectorFactory, + hand: impl Fn(&mut toolpath_redact::Plan), + ) { + let key = crate::cache::load_or_create_redact_key(&policy.key_id).unwrap(); + let file = crate::cache::cache_path(cache_id).unwrap(); + let mut doc = Graph::from_json(&std::fs::read_to_string(&file).unwrap()).unwrap(); + let set = detectors(&policy.detectors).unwrap(); + let decisions = policy_decisions(policy).unwrap(); + let cfg = replay_config(policy, key); + for path in doc.paths.iter_mut() { + let PathOrRef::Path(path) = path else { + continue; + }; + let mut plan = plan_for(path, &set, &cfg, &decisions).unwrap(); + hand(&mut plan); + toolpath_redact::apply(path, &plan, &cfg).unwrap(); + } + write_cached(cache_id, &doc, true).unwrap(); + assert!(record_redaction_policy(cache_id, policy).unwrap()); + } + + fn redact_cached(cache_id: &str, policy: &RedactionPolicy) { + redact_cached_with(cache_id, policy, literal_detectors, |_| {}); + } + + fn append_turn(home: &Path, session: &str, text: &str) { + let file = home + .join(".claude/projects/-test-project") + .join(format!("{session}.jsonl")); + let mut body = std::fs::read_to_string(&file).unwrap(); + body.push_str(&format!( + r#"{{"type":"user","uuid":"u-{text:.4}","timestamp":"2024-01-02T00:05:00Z","cwd":"/test/project","message":{{"role":"user","content":"{text}"}}}}"# + )); + body.push('\n'); + std::fs::write(&file, body).unwrap(); + } + + /// The cache entry a synced claude session landed at. + fn cached_id(session: &str) -> String { + load_manifest().unwrap()["claude"][session] + .cache_id + .clone() + .expect("the session is materialized") + } + + fn read_cached(cache_id: &str) -> String { + std::fs::read_to_string(crate::cache::cache_path(cache_id).unwrap()).unwrap() + } + fn make_ref(artifact_type: ArtifactType, id: &str) -> ArtifactRef { ArtifactRef { artifact_type, @@ -488,6 +881,7 @@ mod tests { modified: Some("2024-01-02T00:00:01.123456789Z".parse().unwrap()), size: Some(4096), synced_at: "2026-07-09T00:00:00Z".parse().unwrap(), + redaction: Some(literal_policy("AKIA", "claude-p1")), }, ); save_manifest(&manifest).unwrap(); @@ -663,6 +1057,7 @@ mod tests { &artifacts, &BTreeMap::new(), &mut (), + build_detectors, ) .unwrap(); assert_eq!((outcome.new, outcome.failed), (1, 1)); @@ -745,6 +1140,7 @@ mod tests { &[artifact], &records, &mut (), + build_detectors, ) .unwrap(); assert_eq!((outcome.updated, outcome.unchanged), (1, 0)); @@ -904,6 +1300,235 @@ mod tests { }); } + #[test] + fn manifest_without_redaction_field_still_loads() { + let json = r#"{"claude":{"sess-1":{"cache_id":"claude-sess-1","synced_at":"2026-01-01T00:00:00Z"}}}"#; + assert!(serde_json::from_str::(json).is_ok()); + } + + #[test] + fn unknown_detector_refuses_replay() { + assert_eq!( + build_detectors(&["internal".to_string()]) + .map(|set| set.ids()) + .unwrap_or_default(), + vec!["internal"] + ); + let err = build_detectors(&["not-a-detector".to_string()]) + .err() + .expect("no detector is resolvable by that name"); + assert!(err.to_string().contains("cannot replay")); + } + + #[test] + fn import_preserves_the_redaction_policy() { + with_cfg(|_| { + let artifact = make_ref(ArtifactType::Claude, "sess-aaa"); + record_artifact(&artifact, "claude-sess-aaa").unwrap(); + record_redaction_policy( + "claude-sess-aaa", + &literal_policy("AKIA", "claude-sess-aaa"), + ) + .unwrap(); + + // A re-import of the same session rewrites the record. + record_artifact(&artifact, "claude-sess-aaa").unwrap(); + assert!( + load_manifest().unwrap()["claude"]["sess-aaa"] + .redaction + .is_some(), + "an import must not drop the policy guarding the document" + ); + }); + } + + #[test] + fn eviction_retires_the_policy_with_the_document() { + with_cfg(|_| { + let artifact = make_ref(ArtifactType::Claude, "sess-aaa"); + record_artifact(&artifact, "claude-sess-aaa").unwrap(); + record_redaction_policy( + "claude-sess-aaa", + &literal_policy("AKIA", "claude-sess-aaa"), + ) + .unwrap(); + + evict_cache_id("claude-sess-aaa").unwrap(); + let rec = &load_manifest().unwrap()["claude"]["sess-aaa"]; + assert!(rec.cache_id.is_none()); + assert!( + rec.redaction.is_none(), + "a policy whose key `p cache rm` deleted would fail every future sync" + ); + }); + } + + #[test] + fn record_redaction_policy_reports_untracked_documents() { + with_cfg(|_| { + assert!( + !record_redaction_policy("github-owner-repo-42", &literal_policy("AKIA", "x")) + .unwrap() + ); + }); + } + + #[test] + fn sync_fails_loudly_on_missing_key() { + with_cfg(|home| { + write_claude_session( + home, + "-test-project", + "sess-1", + "key AKIAIOSFODNN7REALKEY here", + ); + let bundle = claude_bundle(home); + sync_claude(&bundle); + let cache_id = load_manifest().unwrap()["claude"]["sess-1"] + .cache_id + .clone() + .unwrap(); + let policy = literal_policy("AKIAIOSFODNN7", &cache_id); + redact_cached(&cache_id, &policy); + let redacted = read_cached(&cache_id); + + // The key is lost behind the CLI's back — an out-of-band + // delete, a restored config dir, a half-copied machine. + crate::cache::remove_redact_key(&policy.key_id).unwrap(); + append_turn(home, "sess-1", "another turn"); + + let outcome = sync_claude(&bundle); + assert_eq!( + (outcome.updated, outcome.failed, outcome.re_redacted), + (0, 1, 0) + ); + assert_eq!( + read_cached(&cache_id), + redacted, + "an un-redacted re-derive must never land on top of a redacted document" + ); + }); + } + + #[test] + fn sync_skips_redacted_doc_when_source_unchanged() { + with_cfg(|home| { + write_claude_session( + home, + "-test-project", + "sess-1", + "key AKIAIOSFODNN7REALKEY here", + ); + let bundle = claude_bundle(home); + sync_claude(&bundle); + let cache_id = load_manifest().unwrap()["claude"]["sess-1"] + .cache_id + .clone() + .unwrap(); + redact_cached(&cache_id, &literal_policy("AKIAIOSFODNN7", &cache_id)); + let redacted = read_cached(&cache_id); + + let outcome = sync_claude(&bundle); + assert_eq!( + (outcome.unchanged, outcome.re_redacted), + (1, 0), + "an untouched source is not re-derived, so there is nothing to replay" + ); + assert_eq!(read_cached(&cache_id), redacted); + }); + } + + #[test] + fn sync_reapplies_redaction_after_source_grows() { + with_cfg(|home| { + write_claude_session( + home, + "-test-project", + "sess-1", + "key AKIAIOSFODNN7REALKEY here", + ); + let bundle = claude_bundle(home); + sync_claude(&bundle); + let cache_id = cached_id("sess-1"); + redact_cached(&cache_id, &literal_policy("AKIAIOSFODNN7", &cache_id)); + + append_turn(home, "sess-1", "another turn with AKIAIOSFODNN7SECONDKEY"); + let outcome = sync_claude(&bundle); + assert_eq!((outcome.updated, outcome.re_redacted), (1, 1)); + + let doc = read_cached(&cache_id); + assert!(doc.contains("another turn"), "new content must land"); + assert!( + !doc.contains("AKIAIOSFODNN7REALKEY"), + "redaction must survive re-derive" + ); + assert!( + !doc.contains("AKIAIOSFODNN7SECONDKEY"), + "new content must be redacted too" + ); + }); + } + + #[test] + fn sync_reports_reappeared_skips() { + with_cfg(|home| { + write_claude_session( + home, + "-test-project", + "sess-1", + "key AKIAIOSFODNN7REALKEY here", + ); + let bundle = claude_bundle(home); + sync_claude(&bundle); + append_turn(home, "sess-1", "and AKIAIOSFODNN7SECONDKEY too"); + sync_claude(&bundle); + + // The user redacted one of the two and skipped the other by + // hand — a decision no rule-based policy can express. + let cache_id = cached_id("sess-1"); + let policy = literal_policy("AKIAIOSFODNN7", &cache_id); + redact_cached_with(&cache_id, &policy, literal_detectors, |plan| { + plan.findings.last_mut().unwrap().action = Action::Skip; + }); + assert!(read_cached(&cache_id).contains("AKIAIOSFODNN7SECONDKEY")); + + append_turn(home, "sess-1", "a third turn"); + let outcome = sync_claude(&bundle); + assert_eq!((outcome.re_redacted, outcome.reappeared_skips), (1, 1)); + assert!( + !read_cached(&cache_id).contains("AKIAIOSFODNN7SECONDKEY"), + "a skip that cannot be replayed fails closed: the finding comes back" + ); + }); + } + + #[test] + fn sync_bundle_replays_the_builtin_detector() { + with_cfg(|home| { + write_claude_session( + home, + "-test-project", + "sess-1", + "aws access_key AKIAIOSFODNN7REALKEY", + ); + let bundle = claude_bundle(home); + sync_bundle(&bundle, &[ArtifactType::Claude], &mut ()).unwrap(); + let cache_id = cached_id("sess-1"); + let policy = policy_with(vec!["internal".to_string()], &cache_id); + redact_cached_with(&cache_id, &policy, build_detectors, |_| {}); + assert!(!read_cached(&cache_id).contains("AKIAIOSFODNN7REALKEY")); + + append_turn(home, "sess-1", "and access_key AKIAZYTQRMLKNPVWXCDE"); + let (_, outcome) = sync_bundle(&bundle, &[ArtifactType::Claude], &mut ()).unwrap()[0]; + assert_eq!((outcome.updated, outcome.re_redacted), (1, 1)); + + let doc = read_cached(&cache_id); + assert!(doc.contains("and access_key"), "new content must land"); + assert!(!doc.contains("AKIAIOSFODNN7REALKEY")); + assert!(!doc.contains("AKIAZYTQRMLKNPVWXCDE")); + }); + } + #[test] fn newest_first_orders_by_mtime_with_unstamped_last() { let mut old = make_ref(ArtifactType::Claude, "old"); diff --git a/crates/toolpath-cli/Cargo.toml b/crates/toolpath-cli/Cargo.toml index 54cf9fe6..5fd5d044 100644 --- a/crates/toolpath-cli/Cargo.toml +++ b/crates/toolpath-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-cli" -version = "0.16.0" +version = "0.17.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/empathic/toolpath" @@ -14,7 +14,7 @@ name = "path" path = "src/main.rs" [dependencies] -path-cli = { path = "../path-cli", version = "0.16.0" } +path-cli = { path = "../path-cli", version = "0.17.0" } anyhow = "1.0" [workspace] diff --git a/crates/toolpath-redact/README.md b/crates/toolpath-redact/README.md index 49c10500..a71d202d 100644 --- a/crates/toolpath-redact/README.md +++ b/crates/toolpath-redact/README.md @@ -48,9 +48,13 @@ A detector that would send candidate material off the machine reports ## Vendored ruleset The built-in detector compiles its rules from a vendored copy of the -[gitleaks](https://github.com/gitleaks/gitleaks) configuration, used -under the MIT license. +gitleaks configuration at `src/internal/gitleaks.toml`. - +- Upstream: , `config/gitleaks.toml` +- Commit: `b58d3f102cf3a2c84cb7f923d05c25c9b1aed84b` (2026-07-22) +- License: MIT, -See `src/internal/gitleaks.toml` for attribution and version details. +The copy is kept byte-verbatim so it can be diffed against upstream. Do +not hand-edit it; rules this crate adds on top live in `internal/rules.rs` +as `supplemental_rules`, and rules that will not compile under Rust's +`regex` are listed there too. diff --git a/crates/toolpath-redact/src/apply.rs b/crates/toolpath-redact/src/apply.rs index 0a6af462..40923954 100644 --- a/crates/toolpath-redact/src/apply.rs +++ b/crates/toolpath-redact/src/apply.rs @@ -1,13 +1,1066 @@ //! Rewriting a document from an approved plan. +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Value, json}; + +use crate::plan::{Action, Plan, PlanFinding}; +use crate::transform::{ + Fingerprint, Transform, apply_spans_desc, apply_transform, resolve_transform, +}; +use crate::{RedactConfig, RedactError, RedactReport, Result}; + +/// Version of the `redaction` record written into `meta`. Bumping it is a +/// document-format change: `apply` refuses records it does not recognise +/// rather than merging into a shape it cannot read. +const RECORD_V: u64 = 1; + +const RECORD_KEY: &str = "redaction"; + /// Rewrite `path` in place according to `plan`. /// /// Nothing the plan does not name is touched: with no findings, the /// serialised output is byte-identical to the input. pub fn apply( - _path: &mut toolpath::v1::Path, - _plan: &crate::plan::Plan, - _cfg: &crate::RedactConfig, -) -> crate::Result { - todo!("T7") + path: &mut toolpath::v1::Path, + plan: &Plan, + cfg: &RedactConfig, +) -> Result { + crate::plan::verify(plan, path)?; + + let mut report = RedactReport { + surfaces_scanned: plan.surfaces.len(), + ..Default::default() + }; + for f in plan.findings.iter().filter(|f| f.action == Action::Skip) { + *report.flagged.entry(f.rule.clone()).or_default() += 1; + } + + // A plan that redacts nothing invalidates no signature and removes no + // content, so it must leave the document byte-identical - including its + // signatures and any record a previous pass left. + if !plan.findings.iter().any(|f| f.action == Action::Redact) { + return Ok(report); + } + + report.signatures_dropped = guard_signatures(path, cfg)?; + + let mut records: BTreeMap> = BTreeMap::new(); + let mut touched: BTreeSet<&str> = BTreeSet::new(); + { + let mut cursor = crate::surface::SurfaceCursor { path: &mut *path }; + for ((step, at), group) in group_by_field(plan) { + let Some(text) = cursor.read(step, at) else { + return Err(RedactError::BadPointer(format!("{step}{at}"))); + }; + + let mut edits = Vec::with_capacity(group.len()); + for f in group { + // A span landing mid-codepoint would panic the slice, so the + // bounds check has to happen here and not only in `verify`. + let value = text.get(f.span.0..f.span.1).ok_or_else(|| { + RedactError::PlanMismatch(format!("{}: span does not land on {at}", f.id)) + })?; + let fp = Fingerprint::new(&cfg.key, value); + let op = resolve_transform(cfg, &f.rule, f.transform); + edits.push((f.span.0..f.span.1, apply_transform(op, &f.rule, value, &fp))); + + *report.replaced.entry(f.rule.clone()).or_default() += 1; + *records + .entry(step.to_string()) + .or_default() + .entry(RecordKey { + at: at.to_string(), + rule: f.rule.clone(), + fp: fp.0, + op: op_name(op), + }) + .or_default() += 1; + } + + cursor.write(step, at, &apply_spans_desc(&text, &mut edits))?; + touched.insert(step); + } + } + + // Surfaces outside any step (`path.base`, `meta.vcs_remote`) carry the + // empty step id and have no step to be recorded on. + touched.remove(""); + report.steps_touched = touched.len(); + + write_step_records(path, &records)?; + write_rollup(path, plan, cfg, &report)?; + Ok(report) +} + +/// Every edit to one string has to be spliced in a single right-to-left +/// pass, so the findings are grouped by the field they land in; applying +/// them one field-visit at a time would invalidate the offsets of the edits +/// still pending. +fn group_by_field(plan: &Plan) -> BTreeMap<(&str, &str), Vec<&PlanFinding>> { + let mut out: BTreeMap<(&str, &str), Vec<&PlanFinding>> = BTreeMap::new(); + for f in plan.findings.iter().filter(|f| f.action == Action::Redact) { + out.entry((f.step.as_str(), f.at.as_str())) + .or_default() + .push(f); + } + out +} + +/// Strip every signature in the document, returning how many there were. +/// +/// A signature covers content the pass is about to change, so leaving one in +/// place would publish a signature that no longer verifies. +fn guard_signatures(path: &mut toolpath::v1::Path, cfg: &RedactConfig) -> Result { + let total = path.meta.as_ref().map_or(0, |m| m.signatures.len()) + + path + .steps + .iter() + .filter_map(|s| s.meta.as_ref()) + .map(|m| m.signatures.len()) + .sum::(); + if total == 0 { + return Ok(0); + } + if !cfg.drop_signatures { + return Err(RedactError::SignedDocument); + } + if let Some(m) = path.meta.as_mut() { + m.signatures.clear(); + } + for s in &mut path.steps { + if let Some(m) = s.meta.as_mut() { + m.signatures.clear(); + } + } + Ok(total) +} + +/// Identifies one line of the audit record. The record publishes `at`, so two +/// occurrences of one credential in two different fields cannot collapse into +/// a single entry without making `at` a lie. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct RecordKey { + at: String, + rule: String, + fp: String, + op: String, +} + +/// The record is part of the document contract, so `op` has to be exactly the +/// serde name of the `Transform` rather than a second spelling of it. +fn op_name(t: Transform) -> String { + match serde_json::to_value(t) { + Ok(Value::String(s)) => s, + _ => unreachable!("Transform serialises as a string"), + } +} + +fn write_step_records( + path: &mut toolpath::v1::Path, + records: &BTreeMap>, +) -> Result<()> { + for (id, fresh) in records { + if id.is_empty() { + continue; + } + let step = path + .steps + .iter_mut() + .find(|s| s.step.id == *id) + .ok_or_else(|| RedactError::PlanMismatch(format!("no step {id}")))?; + let meta = step.meta.get_or_insert_with(Default::default); + + let mut merged = seed_from_existing(meta.extra.get(RECORD_KEY))?; + for (k, n) in fresh { + *merged.entry(k.clone()).or_default() += n; + } + meta.extra.insert(RECORD_KEY.into(), record_value(&merged)); + } + Ok(()) +} + +/// Re-read an existing record so a second pass merges into it. Appending a +/// second `redaction` object, or nesting one inside the other, would make the +/// step's own history unreadable. +fn seed_from_existing(existing: Option<&Value>) -> Result> { + let Some(v) = existing else { + return Ok(BTreeMap::new()); + }; + check_version(v)?; + let entries = v + .get("findings") + .and_then(Value::as_array) + .ok_or_else(|| RedactError::PlanMismatch("redaction record has no findings".into()))?; + + let mut out = BTreeMap::new(); + for e in entries { + let field = |k: &str| { + e.get(k) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + RedactError::PlanMismatch(format!("redaction record entry lacks {k}")) + }) + }; + let key = RecordKey { + at: field("at")?, + rule: field("rule")?, + fp: field("fp")?, + op: field("op")?, + }; + let n = e + .get("n") + .and_then(Value::as_u64) + .ok_or_else(|| RedactError::PlanMismatch("redaction record entry lacks n".into()))?; + *out.entry(key).or_default() += n as usize; + } + Ok(out) +} + +fn check_version(v: &Value) -> Result<()> { + match v.get("v").and_then(Value::as_u64) { + Some(RECORD_V) => Ok(()), + other => Err(RedactError::PlanMismatch(format!( + "unrecognised redaction record version {other:?}" + ))), + } +} + +fn record_value(entries: &BTreeMap) -> Value { + let findings: Vec = entries + .iter() + .map(|(k, n)| json!({ "rule": k.rule, "at": k.at, "n": n, "fp": k.fp, "op": k.op })) + .collect(); + json!({ "v": RECORD_V, "findings": findings }) +} + +fn write_rollup( + path: &mut toolpath::v1::Path, + plan: &Plan, + cfg: &RedactConfig, + report: &RedactReport, +) -> Result<()> { + let previous = path + .meta + .as_ref() + .and_then(|m| m.extra.get(RECORD_KEY)) + .cloned(); + if let Some(p) = &previous { + check_version(p)?; + } + + // Counted off the step records rather than accumulated across passes, so + // a step redacted twice stays one touched step. + let steps_touched = path + .steps + .iter() + .filter(|s| { + s.meta + .as_ref() + .is_some_and(|m| m.extra.contains_key(RECORD_KEY)) + }) + .count(); + + let replaced = merge_counts(previous.as_ref(), "replaced", &report.replaced); + let signatures_dropped = + previous_u64(previous.as_ref(), "signatures_dropped") + report.signatures_dropped as u64; + + let meta = path.meta.get_or_insert_with(Default::default); + meta.extra.insert( + RECORD_KEY.into(), + json!({ + "v": RECORD_V, + "at": cfg.now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "tool": concat!("toolpath-redact/", env!("CARGO_PKG_VERSION")), + "detectors": plan.detectors, + "mode": cfg.mode, + "steps_touched": steps_touched, + "replaced": replaced, + // `flagged` names findings still sitting in the document, so it is + // this pass's tally and not a running total. + "flagged": report.flagged, + "signatures_dropped": signatures_dropped, + }), + ); + Ok(()) +} + +fn merge_counts( + previous: Option<&Value>, + field: &str, + fresh: &BTreeMap, +) -> BTreeMap { + let mut out: BTreeMap = previous + .and_then(|p| p.get(field)) + .and_then(Value::as_object) + .map(|m| { + m.iter() + .filter_map(|(k, v)| v.as_u64().map(|n| (k.clone(), n))) + .collect() + }) + .unwrap_or_default(); + for (k, n) in fresh { + *out.entry(k.clone()).or_default() += *n as u64; + } + out +} + +fn previous_u64(previous: Option<&Value>, field: &str) -> u64 { + previous + .and_then(|p| p.get(field)) + .and_then(Value::as_u64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::plan::PlanDefaults; + use crate::surface::{Surface, SurfaceCursor, surfaces}; + use chrono::{TimeZone, Utc}; + use std::collections::HashMap; + use std::ops::Range; + use std::sync::LazyLock; + use toolpath::v1::{ + ArtifactChange, Base, Path, PathIdentity, PathMeta, Signature, Step, StepIdentity, + StepMeta, StructuralChange, + }; + + const AWS_KEY: &str = "AKIAIOSFODNN7EXAMPLE"; + const AWS_KEY_2: &str = "AKIAJ7EXAMPLEKEY1234"; + const GH_PAT: &str = "ghp_abcdefghijklmnopqrstuvwxyz0123456789"; + + fn cfg() -> RedactConfig { + RedactConfig { + threshold: 0.5, + mode: Transform::Marker, + mode_for: Vec::new(), + key: b"test-key".to_vec(), + now: Utc.with_ymd_and_hms(2026, 7, 30, 18, 4, 11).unwrap(), + drop_signatures: false, + reveal: false, + } + } + + // ── A stand-in detector ───────────────────────────────────────────── + // + // `plan_for` re-detects against whatever the document currently holds; + // hard-coded spans would reduce the idempotence test to "the same span + // was redacted twice". Both rules match a self-delimiting prefixed + // format, which is what keeps a marker, mask, hash or partial output from + // being mistaken for a fresh secret on the second pass. + + static AWS_RE: LazyLock = + LazyLock::new(|| regex::Regex::new(r"AKIA[0-9A-Z]{16}").unwrap()); + static GH_RE: LazyLock = + LazyLock::new(|| regex::Regex::new(r"ghp_[A-Za-z0-9]{36}").unwrap()); + + fn mini_scan(text: &str) -> Vec<(Range, &'static str)> { + let mut out: Vec<(Range, &'static str)> = AWS_RE + .find_iter(text) + .map(|m| (m.range(), "aws-access-key-id")) + .chain(GH_RE.find_iter(text).map(|m| (m.range(), "github-pat"))) + .collect(); + out.sort_by_key(|(r, _)| r.start); + out + } + + // ── Plan construction ─────────────────────────────────────────────── + + fn plan_with(path: &Path, surfaces: Vec, mut findings: Vec) -> Plan { + findings.sort_by(|a, b| (&a.step, &a.at, a.span.0).cmp(&(&b.step, &b.at, b.span.0))); + for (i, f) in findings.iter_mut().enumerate() { + f.id = crate::plan::finding_id(i); + } + Plan { + v: 1, + document: path.path.id.clone(), + generated: cfg().now, + detectors: vec!["fixed".into()], + defaults: PlanDefaults { + transform: Transform::Marker, + threshold: 0.5, + }, + surfaces, + findings, + } + } + + fn empty_plan(path: &Path) -> Plan { + plan_with(path, surfaces(path), Vec::new()) + } + + /// Reads every surface without a `&mut` borrow of the caller's copy, so a + /// plan can be built inside the same call that mutates the document. + fn read_surfaces(path: &Path) -> Vec<(Surface, String)> { + let mut probe = path.clone(); + let all = surfaces(&probe); + let cursor = SurfaceCursor { path: &mut probe }; + all.into_iter() + .filter_map(|s| cursor.read(&s.step, &s.at).map(|t| (s, t))) + .collect() + } + + fn finding(s: &Surface, span: Range, rule: &str, action: Action) -> PlanFinding { + PlanFinding { + id: String::new(), + step: s.step.clone(), + at: s.at.clone(), + rule: rule.into(), + span: (span.start, span.end), + score: 0.99, + detector: "fixed".into(), + shape: s.shape, + context: format!("<{rule}>"), + action, + transform: None, + } + } + + fn scan_findings(path: &Path, action: Action) -> Vec { + read_surfaces(path) + .iter() + .flat_map(|(s, text)| { + mini_scan(text) + .into_iter() + .map(move |(span, rule)| finding(s, span, rule, action)) + }) + .collect() + } + + fn plan_for(path: &Path) -> Plan { + plan_with(path, surfaces(path), scan_findings(path, Action::Redact)) + } + + fn plan_touching_only_text(path: &Path) -> Plan { + plan_with(path, surfaces(path), text_findings(path)) + } + + fn text_findings(path: &Path) -> Vec { + scan_findings(path, Action::Redact) + .into_iter() + .filter(|f| f.at.ends_with("/text")) + .collect() + } + + // ── Fixtures ──────────────────────────────────────────────────────── + + fn obj(v: Value) -> HashMap { + match v { + Value::Object(m) => m.into_iter().collect(), + other => panic!("fixture is not an object: {other}"), + } + } + + fn append_step(id: &str, extra: Value) -> Step { + Step { + step: StepIdentity { + id: id.into(), + parents: Vec::new(), + actor: "agent:claude-opus-5".into(), + timestamp: "2026-07-30T18:00:00Z".into(), + }, + change: HashMap::from([( + "claude://sess-abc".to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "conversation.append".into(), + extra: obj(extra), + }), + }, + )]), + meta: None, + } + } + + fn doc(steps: Vec) -> Path { + let head = steps.last().map_or(String::new(), |s| s.step.id.clone()); + Path { + path: PathIdentity { + id: "path-claude-code-0f3a2b71".into(), + base: Some(Base { + uri: "file:///tmp/work".into(), + ref_str: None, + branch: Some("main".into()), + }), + head, + graph_ref: None, + }, + steps, + meta: Some(PathMeta { + title: Some("Claude session: 0f3a2b71".into()), + kind: Some(toolpath::v1::PATH_KIND_AGENT_CODING_SESSION.into()), + ..Default::default() + }), + } + } + + fn fixture_clean_document() -> Path { + doc(vec![append_step( + "turn-0f3a", + json!({ + "text": "Refactored the loader; nothing sensitive here.", + "thinking": "The cache path is the only thing worth checking.", + "tool_uses": [{ + "name": "Bash", + "input": { "command": "cargo test -p toolpath" }, + "result": { "content": "test result: ok. 69 passed" } + }] + }), + )]) + } + + fn fixture_with_secret(secret: &str) -> Path { + doc(vec![append_step( + "turn-0f3a", + json!({ + "text": format!("exported {secret} into the shell"), + "thinking": "Nothing to see here.", + }), + )]) + } + + fn fixture_with_secrets() -> Path { + doc(vec![ + append_step( + "turn-0f3a", + json!({ + "text": "Ran the deploy script.", + "tool_uses": [{ + "name": "Bash", + "input": { "command": "aws s3 ls" }, + "result": { "content": format!("AWS_ACCESS_KEY_ID={AWS_KEY}\ndone") } + }] + }), + ), + append_step( + "turn-9c21", + json!({ "text": format!("then pushed with {GH_PAT}") }), + ), + ]) + } + + fn fixture_with_extra_keys(keys: &[&str]) -> Path { + let mut extra = json!({ "text": format!("leaked {AWS_KEY} once") }); + for k in keys { + extra[*k] = json!({ "kept": true, "n": 7 }); + } + doc(vec![append_step("turn-0f3a", extra)]) + } + + fn fixture_file_write_with_secret_in_diff() -> Path { + let raw = format!( + "--- a/src/config.rs\n\ + +++ b/src/config.rs\n\ + @@ -1,3 +1,3 @@\n\ + \x20fn main() {{\n\ + - let key = \"\";\n\ + + let key = \"{AWS_KEY}\";\n\ + \x20}}\n" + ); + let mut step = append_step("turn-0f3a", json!({ "text": "wrote the config" })); + step.change.insert( + "src/config.rs".into(), + ArtifactChange { + raw: Some(raw), + structural: None, + }, + ); + doc(vec![step]) + } + + fn fixture_signed() -> Path { + let mut p = fixture_with_secret(AWS_KEY); + p.meta.as_mut().unwrap().signatures = vec![Signature { + signer: "human:alex".into(), + key: "ssh:SHA256:abc".into(), + scope: "path".into(), + sig: "base64sig".into(), + timestamp: None, + }]; + p + } + + // ── Inspection helpers ────────────────────────────────────────────── + + fn has_key_somewhere(path: &Path, key: &str) -> bool { + fn walk(v: &Value, key: &str) -> bool { + match v { + Value::Object(m) => m.contains_key(key) || m.values().any(|i| walk(i, key)), + Value::Array(a) => a.iter().any(|i| walk(i, key)), + _ => false, + } + } + walk(&serde_json::to_value(path).unwrap(), key) + } + + fn record_of<'a>(path: &'a Path, step: &str) -> &'a Value { + path.steps + .iter() + .find(|s| s.step.id == step) + .and_then(|s| s.meta.as_ref()) + .and_then(|m| m.extra.get(RECORD_KEY)) + .unwrap_or_else(|| panic!("no redaction record on {step}")) + } + + fn raw_diff_of(path: &Path) -> String { + path.steps + .iter() + .flat_map(|s| s.change.values()) + .find_map(|c| c.raw.clone()) + .expect("fixture has a raw diff") + } + + fn count_lines(h: &diffy::Hunk<'_, str>, side: char) -> usize { + h.lines() + .iter() + .filter(|l| { + matches!( + (l, side), + (diffy::Line::Context(_), _) + | (diffy::Line::Delete(_), '-') + | (diffy::Line::Insert(_), '+') + ) + }) + .count() + } + + fn at_ending(path: &Path, suffix: &str) -> String { + surfaces(path) + .into_iter() + .find(|s| s.at.ends_with(suffix)) + .unwrap_or_else(|| panic!("no surface ending in {suffix}")) + .at + } + + fn text_at(path: &Path, suffix: &str) -> String { + let mut probe = path.clone(); + let at = at_ending(path, suffix); + let step = path.steps[0].step.id.clone(); + let cursor = SurfaceCursor { path: &mut probe }; + cursor + .read(&step, &at) + .unwrap_or_else(|| panic!("{at} does not resolve")) + } + + fn marker(value: &str, rule: &str) -> String { + apply_transform( + Transform::Marker, + rule, + value, + &Fingerprint::new(&cfg().key, value), + ) + } + + // ── Step 7.1: the invariants ──────────────────────────────────────── + + #[test] + fn no_findings_means_byte_identical_output() { + // The most important test here: the pass must not perturb anything it + // is not redacting. + let before = fixture_clean_document(); + let mut after = before.clone(); + apply(&mut after, &empty_plan(&before), &cfg()).unwrap(); + assert_eq!( + serde_json::to_string_pretty(&before).unwrap(), + serde_json::to_string_pretty(&after).unwrap() + ); + } + + #[test] + fn unknown_provider_keys_survive() { + // Guards against reimplementing this as extract -> derive: + // `extra["edits"]` is written by toolpath-convo's derive and never + // read back by extract, so a round-trip silently drops it. + let mut doc = fixture_with_extra_keys(&["edits", "vendor_specific", "entry_extra"]); + let plan = plan_touching_only_text(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + for k in ["edits", "vendor_specific", "entry_extra"] { + assert!(has_key_somewhere(&doc, k), "lost {k}"); + } + } + + #[test] + fn idempotent_across_all_transforms() { + for mode in [ + Transform::Marker, + Transform::Remove, + Transform::Hash, + Transform::Mask, + Transform::Partial, + ] { + let cfg = RedactConfig { mode, ..cfg() }; + let mut once = fixture_with_secrets(); + let plan = plan_for(&once); + apply(&mut once, &plan, &cfg).unwrap(); + let mut twice = once.clone(); + let plan = plan_for(&twice); + apply(&mut twice, &plan, &cfg).unwrap(); + assert_eq!( + serde_json::to_string(&once).unwrap(), + serde_json::to_string(&twice).unwrap(), + "{mode:?} is not idempotent" + ); + } + } + + #[test] + fn redacted_diff_still_parses_and_line_counts_hold() { + let mut doc = fixture_file_write_with_secret_in_diff(); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + let raw = raw_diff_of(&doc); + let patch = diffy::Patch::from_str(&raw).expect("redacted diff must still parse"); + for h in patch.hunks() { + assert_eq!(h.old_range().len(), count_lines(h, '-')); + assert_eq!(h.new_range().len(), count_lines(h, '+')); + } + assert!(!raw.contains(AWS_KEY)); + } + + #[test] + fn audit_record_lands_on_the_step_and_merges_on_rerun() { + let mut doc = fixture_with_secrets(); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + let first = record_of(&doc, "turn-0f3a").clone(); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + assert_eq!( + record_of(&doc, "turn-0f3a"), + &first, + "record must merge, not append" + ); + } + + #[test] + fn audit_record_carries_no_value_substring_or_length() { + let secret = "AKIAIOSFODNN7REALKEY"; + let mut doc = fixture_with_secret(secret); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + let rec = serde_json::to_string(record_of(&doc, "turn-0f3a")).unwrap(); + assert!(!rec.contains(secret)); + for w in 6..secret.len() { + for s in secret.as_bytes().windows(w) { + assert!(!rec.contains(std::str::from_utf8(s).unwrap())); + } + } + assert!(!rec.contains(&secret.len().to_string())); + } + + #[test] + fn signed_document_refuses_without_the_flag() { + let mut doc = fixture_signed(); + let plan = plan_for(&doc); + assert!(matches!( + apply(&mut doc, &plan, &cfg()), + Err(RedactError::SignedDocument) + )); + let cfg = RedactConfig { + drop_signatures: true, + ..cfg() + }; + let plan = plan_for(&doc); + assert_eq!(apply(&mut doc, &plan, &cfg).unwrap().signatures_dropped, 1); + } + + #[test] + fn output_validates_against_both_schemas() { + // `jsonschema` is not a dev-dependency of this crate and `Cargo.toml` + // belongs to another track, so this asserts the structural rules the + // two schemas impose on where the record may sit. T11 runs the real + // validators through `path p validate`. + let mut doc = fixture_with_secrets(); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg()).unwrap(); + let v = serde_json::to_value(&doc).unwrap(); + + let top: BTreeSet<&str> = v.as_object().unwrap().keys().map(String::as_str).collect(); + assert!( + top.is_subset(&BTreeSet::from(["path", "steps", "meta"])), + "{top:?}" + ); + + for step in v["steps"].as_array().unwrap() { + let keys: BTreeSet<&str> = step + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + // `step` is `additionalProperties: false`: the record cannot sit + // as a sibling of step/change/meta. + assert!( + keys.is_subset(&BTreeSet::from(["step", "change", "meta"])), + "{keys:?}" + ); + let ident: BTreeSet<&str> = step["step"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert!( + ident.is_subset(&BTreeSet::from(["id", "parents", "actor", "timestamp"])), + "{ident:?}" + ); + assert!(step["step"]["id"].is_string()); + assert!(step["step"]["actor"].is_string()); + assert!(step["step"]["timestamp"].is_string()); + assert!(step["change"].is_object()); + } + + assert_eq!(v["steps"][0]["meta"]["redaction"]["v"], json!(RECORD_V)); + assert_eq!(v["meta"]["redaction"]["v"], json!(RECORD_V)); + assert_eq!( + v["meta"]["kind"], + json!(toolpath::v1::PATH_KIND_AGENT_CODING_SESSION) + ); + } + + // ── Boundaries ────────────────────────────────────────────────────── + + #[test] + fn refused_signed_document_is_left_untouched() { + let before = fixture_signed(); + let mut after = before.clone(); + assert!(apply(&mut after, &plan_for(&before), &cfg()).is_err()); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + } + + #[test] + fn two_findings_in_one_field_splice_right_to_left() { + // A left-to-right splice corrupts the second span: the marker is + // longer than the key it replaces. + let mut d = doc(vec![append_step( + "turn-0f3a", + json!({ "text": format!("first {AWS_KEY} then {AWS_KEY_2} end") }), + )]); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + assert_eq!( + text_at(&d, "/text"), + format!( + "first {} then {} end", + marker(AWS_KEY, "aws-access-key-id"), + marker(AWS_KEY_2, "aws-access-key-id") + ) + ); + } + + #[test] + fn finding_covering_the_entire_field_value() { + let mut d = doc(vec![append_step("turn-0f3a", json!({ "text": AWS_KEY }))]); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + assert_eq!(text_at(&d, "/text"), marker(AWS_KEY, "aws-access-key-id")); + } + + #[test] + fn remove_transform_empties_the_field() { + let cfg = RedactConfig { + mode: Transform::Remove, + ..cfg() + }; + let mut d = doc(vec![append_step("turn-0f3a", json!({ "text": AWS_KEY }))]); + let at = at_ending(&d, "/text"); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg).unwrap(); + let cursor = SurfaceCursor { path: &mut d }; + assert_eq!(cursor.read("turn-0f3a", &at).as_deref(), Some("")); + } + + #[test] + fn all_skips_leave_the_document_untouched_but_are_flagged() { + let before = fixture_with_secrets(); + let mut after = before.clone(); + let plan = plan_with( + &before, + surfaces(&before), + scan_findings(&before, Action::Skip), + ); + let report = apply(&mut after, &plan, &cfg()).unwrap(); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + assert_eq!(report.flagged["aws-access-key-id"], 1); + assert_eq!(report.flagged["github-pat"], 1); + assert!(report.replaced.is_empty()); + } + + #[test] + fn unknown_step_id_errors_rather_than_panicking() { + let before = fixture_with_secrets(); + let mut after = before.clone(); + let mut plan = plan_for(&before); + for f in &mut plan.findings { + f.step = "turn-does-not-exist".into(); + } + assert!(matches!( + apply(&mut after, &plan, &cfg()), + Err(RedactError::PlanMismatch(_)) + )); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + } + + #[test] + fn inverted_span_errors_rather_than_panicking() { + // `verify` bounds-checks both ends but not their order, so an + // end-before-start span reaches the slice. + let before = fixture_with_secret(AWS_KEY); + let mut after = before.clone(); + let mut plan = plan_for(&before); + plan.findings[0].span = (plan.findings[0].span.1, plan.findings[0].span.0); + assert!(matches!( + apply(&mut after, &plan, &cfg()), + Err(RedactError::PlanMismatch(_)) + )); + } + + #[test] + fn path_level_surface_is_redacted_without_a_step_record() { + let mut d = fixture_with_secret(AWS_KEY); + d.meta.as_mut().unwrap().extra.insert( + "vcs_remote".into(), + json!(format!("https://{GH_PAT}@github.com/o/r.git")), + ); + let plan = plan_for(&d); + let report = apply(&mut d, &plan, &cfg()).unwrap(); + + let remote = d.meta.as_ref().unwrap().extra["vcs_remote"] + .as_str() + .unwrap(); + assert!(!remote.contains(GH_PAT), "{remote}"); + assert_eq!(report.replaced["github-pat"], 1); + assert_eq!(report.steps_touched, 1, "the path itself is not a step"); + assert_eq!( + d.meta.as_ref().unwrap().extra[RECORD_KEY]["steps_touched"], + json!(1) + ); + } + + #[test] + fn multibyte_content_around_a_redacted_span() { + let mut d = doc(vec![append_step( + "turn-0f3a", + json!({ "text": format!("鍵は {AWS_KEY} です - 気をつけて") }), + )]); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + assert_eq!( + text_at(&d, "/text"), + format!( + "鍵は {} です - 気をつけて", + marker(AWS_KEY, "aws-access-key-id") + ) + ); + } + + #[test] + fn document_with_zero_steps() { + let before = doc(Vec::new()); + let mut after = before.clone(); + let report = apply(&mut after, &empty_plan(&before), &cfg()).unwrap(); + assert_eq!(report.steps_touched, 0); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + } + + #[test] + fn two_passes_over_different_secrets_merge_into_one_record() { + let mut d = doc(vec![append_step( + "turn-0f3a", + json!({ + "text": format!("first {AWS_KEY}"), + "thinking": format!("later {GH_PAT}"), + }), + )]); + let text_only = plan_with(&d, surfaces(&d), text_findings(&d)); + apply(&mut d, &text_only, &cfg()).unwrap(); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + + let rec = record_of(&d, "turn-0f3a"); + assert_eq!(rec["v"], json!(RECORD_V)); + let entries = rec["findings"].as_array().unwrap(); + assert_eq!(entries.len(), 2, "{rec}"); + let rules: BTreeSet<&str> = entries + .iter() + .map(|f| f["rule"].as_str().unwrap()) + .collect(); + assert_eq!(rules, BTreeSet::from(["aws-access-key-id", "github-pat"])); + } + + #[test] + fn repeated_secret_in_one_field_aggregates_to_one_entry() { + let mut d = doc(vec![append_step( + "turn-0f3a", + json!({ "text": format!("{AWS_KEY} and again {AWS_KEY}") }), + )]); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + let entries = record_of(&d, "turn-0f3a")["findings"] + .as_array() + .unwrap() + .clone(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0]["n"], json!(2)); + } + + #[test] + fn rollup_counts_steps_not_fields() { + let mut d = doc(vec![append_step( + "turn-0f3a", + json!({ + "text": format!("a {AWS_KEY}"), + "thinking": format!("b {GH_PAT}"), + }), + )]); + let plan = plan_for(&d); + let report = apply(&mut d, &plan, &cfg()).unwrap(); + assert_eq!(report.steps_touched, 1); + assert_eq!( + d.meta.as_ref().unwrap().extra[RECORD_KEY]["steps_touched"], + json!(1) + ); + } + + #[test] + fn unrecognised_record_version_is_refused() { + let mut d = fixture_with_secret(AWS_KEY); + d.steps[0].meta = Some(StepMeta { + extra: HashMap::from([(RECORD_KEY.to_string(), json!({ "v": 99 }))]), + ..Default::default() + }); + let plan = plan_for(&d); + assert!(matches!( + apply(&mut d, &plan, &cfg()), + Err(RedactError::PlanMismatch(_)) + )); + } + + #[test] + fn report_counts_surfaces_and_replacements() { + let before = fixture_with_secrets(); + let mut after = before.clone(); + let plan = plan_for(&before); + let report = apply(&mut after, &plan, &cfg()).unwrap(); + assert_eq!(report.surfaces_scanned, plan.surfaces.len()); + assert_eq!(report.replaced["aws-access-key-id"], 1); + assert_eq!(report.replaced["github-pat"], 1); + assert_eq!(report.steps_touched, 2); + assert!(report.flagged.is_empty()); + } } diff --git a/crates/toolpath-redact/src/internal/mod.rs b/crates/toolpath-redact/src/internal/mod.rs index aa6a66f2..e38d48ec 100644 --- a/crates/toolpath-redact/src/internal/mod.rs +++ b/crates/toolpath-redact/src/internal/mod.rs @@ -100,9 +100,12 @@ fn clip_to_line_if_diff(text: &str, shape: FieldShape, span: Range) -> Ra return span; } let line_start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); - let line_end = text[span.end..] + // Anchor to the *start*'s line, not the end's: a match already + // spanning several lines (gitleaks' PEM rule) would otherwise have its + // line-end searched for past all of them, clipping nothing. + let line_end = text[span.start..] .find('\n') - .map_or(text.len(), |i| span.end + i); + .map_or(text.len(), |i| span.start + i); span.start.max(line_start)..span.end.min(line_end) } diff --git a/crates/toolpath-redact/src/plan.rs b/crates/toolpath-redact/src/plan.rs index 99bfa419..f4ff7277 100644 --- a/crates/toolpath-redact/src/plan.rs +++ b/crates/toolpath-redact/src/plan.rs @@ -258,22 +258,147 @@ pub fn verify(plan: &Plan, path: &mut toolpath::v1::Path) -> crate::Result<()> { // ── Plan generation (T8) ─────────────────────────────────────────────── pub fn generate( - _path: &toolpath::v1::Path, - _detectors: &crate::detect::DetectorSet, - _cfg: &crate::RedactConfig, + path: &toolpath::v1::Path, + detectors: &crate::detect::DetectorSet, + cfg: &crate::RedactConfig, ) -> Plan { - todo!("T8") + // This signature is fixed since T0 and cannot return `Result` (T5's own + // byte-identity test calls it directly, unwrapped). A failing detector + // is a bug in that detector, not a normal outcome, so it surfaces as a + // panic here instead of silently degrading to an empty plan; + // `generate_checked` propagates the same failure through `Result`. + generate_inner(path, detectors, cfg).expect("detector failed while generating a redaction plan") } /// `generate`, plus the egress check: a detector that would send candidate /// material off the machine is refused unless the caller allowed it. pub fn generate_checked( - _path: &toolpath::v1::Path, - _detectors: &crate::detect::DetectorSet, - _cfg: &crate::RedactConfig, - _allow_network: bool, + path: &toolpath::v1::Path, + detectors: &crate::detect::DetectorSet, + cfg: &crate::RedactConfig, + allow_network: bool, ) -> crate::Result { - todo!("T8") + if !allow_network + && let Some(d) = detectors + .detectors() + .iter() + .find(|d| d.egress() == crate::detect::Egress::Network) + { + return Err(crate::RedactError::NetworkDetectorRefused( + d.id().to_string(), + )); + } + generate_inner(path, detectors, cfg) +} + +fn generate_inner( + path: &toolpath::v1::Path, + detectors: &crate::detect::DetectorSet, + cfg: &crate::RedactConfig, +) -> crate::Result { + let surfaces = crate::surface::surfaces(path); + + // `SurfaceCursor` needs `&mut Path` (T2 uses the same struct for + // writes); `generate` only takes `&Path` since `verify` re-checks + // findings against the caller's own document later, so read through a + // throwaway clone instead. + let mut scratch = path.clone(); + let cursor = crate::surface::SurfaceCursor { path: &mut scratch }; + + // `surfaces()` already visits steps in document order and sorts + // artifacts/fields (see its own determinism test), and `detect_all` + // returns spans sorted by start - so findings collected in this order + // already satisfy the (step, pointer, span start) ordering `finding_id` + // relies on, with no extra sort needed here. + let mut findings = Vec::new(); + for s in &surfaces { + let Some(text) = cursor.read(&s.step, &s.at) else { + continue; + }; + let ctx = context_for(path, &s.step, &s.at); + let candidate = crate::detect::Candidate { + text: &text, + shape: s.shape, + at: &s.at, + ctx, + }; + for finding in detectors.detect_all(&candidate)? { + let action = if finding.score < cfg.threshold { + Action::Skip + } else { + Action::Redact + }; + let context = elide_context(&text, finding.span.clone(), &finding.rule, cfg.reveal); + findings.push(PlanFinding { + id: String::new(), // assigned below, once findings are in final order + step: s.step.clone(), + at: s.at.clone(), + span: (finding.span.start, finding.span.end), + rule: finding.rule, + score: finding.score, + detector: finding.detector.to_string(), + shape: s.shape, + context, + action, + transform: None, + }); + } + } + for (i, finding) in findings.iter_mut().enumerate() { + finding.id = finding_id(i); + } + + Ok(Plan { + v: 1, + document: path.path.id.clone(), + generated: cfg.now, + detectors: detectors.ids().into_iter().map(String::from).collect(), + defaults: PlanDefaults { + transform: cfg.mode, + threshold: cfg.threshold, + }, + surfaces, + findings, + }) +} + +/// `change_type`/`actor` come from the step and artifact the pointer names. +/// `tool_name` stays `None`: resolving it would mean re-parsing the same +/// `tool_uses` JSON `surfaces()` already walked once, and no detector in +/// this crate reads it yet. +fn context_for<'a>( + path: &'a toolpath::v1::Path, + step_id: &str, + at: &str, +) -> crate::detect::Context<'a> { + let kind = path.meta.as_ref().and_then(|m| m.kind.as_deref()); + let Some(step) = path.steps.iter().find(|s| s.step.id == step_id) else { + return crate::detect::Context { + change_type: "", + tool_name: None, + actor: "", + kind, + }; + }; + let change_type = artifact_key_from_at(at) + .and_then(|key| step.change.get(&key)) + .and_then(|c| c.structural.as_ref()) + .map(|s| s.change_type.as_str()) + .unwrap_or(""); + crate::detect::Context { + change_type, + tool_name: None, + actor: step.step.actor.as_str(), + kind, + } +} + +/// Recovers the artifact key a `/change/...` pointer names. Mirrors +/// `surface::ptr_decode` (private to that module) - decode `~1` before +/// `~0`, or `~01` round-trips wrong (RFC 6901). +fn artifact_key_from_at(at: &str) -> Option { + let token = at.strip_prefix("/change/")?.split('/').next()?; + Some(token.replace("~1", "/").replace("~0", "~")) } #[cfg(test)] @@ -735,3 +860,256 @@ mod tests { assert_eq!(json, serde_json::to_string(&round).unwrap()); } } + +#[cfg(test)] +mod plan_gen { + use super::*; + use crate::detect::{Candidate, Detector, DetectorSet, Egress, Finding, FixedDetector}; + use chrono::TimeZone; + use std::collections::HashMap; + use std::ops::Range; + use toolpath::v1::{ArtifactChange, Path, PathIdentity, Step, StepIdentity, StructuralChange}; + + fn cfg() -> crate::RedactConfig { + crate::RedactConfig { + threshold: 0.8, + mode: Transform::Marker, + mode_for: Vec::new(), + key: b"test-key".to_vec(), + now: Utc.with_ymd_and_hms(2026, 7, 30, 0, 0, 0).unwrap(), + drop_signatures: false, + reveal: false, + } + } + + fn f(span: Range, rule: &str, score: f32) -> Finding { + Finding { + span, + rule: rule.into(), + score, + detector: "fixed", + } + } + + /// Matches the literal substring `SECRET-VALUE`, so `fixture_mixed` + /// (below) can put a finding on one surface and leave a sibling clean. + struct Needle; + impl Detector for Needle { + fn id(&self) -> &'static str { + "needle" + } + fn detect(&self, c: &Candidate<'_>) -> crate::Result> { + Ok(c.text + .match_indices("SECRET-VALUE") + .map(|(i, m)| Finding { + span: i..i + m.len(), + rule: "test-secret".into(), + score: 0.95, + detector: "needle", + }) + .collect()) + } + } + + fn detectors() -> DetectorSet { + let mut set = DetectorSet::default(); + set.push(Box::new(Needle)); + set + } + + struct NetworkDetector; + impl Detector for NetworkDetector { + fn id(&self) -> &'static str { + "network" + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Ok(Vec::new()) + } + fn egress(&self) -> Egress { + Egress::Network + } + } + + struct FailingDetector; + impl Detector for FailingDetector { + fn id(&self) -> &'static str { + "failing" + } + fn detect(&self, _c: &Candidate<'_>) -> crate::Result> { + Err(crate::RedactError::BadPointer("boom".into())) + } + } + + fn step_with_text(id: &str, artifact: &str, text: &str) -> Step { + let mut extra = HashMap::new(); + extra.insert( + "text".to_string(), + serde_json::Value::String(text.to_string()), + ); + let mut change = HashMap::new(); + change.insert( + artifact.to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "conversation.append".to_string(), + extra, + }), + }, + ); + Step { + step: StepIdentity { + id: id.to_string(), + parents: Vec::new(), + actor: "human:t".to_string(), + timestamp: "2026-01-01T00:00:00Z".to_string(), + }, + change, + meta: None, + } + } + + fn path_of(steps: Vec) -> Path { + let head = steps.last().map(|s| s.step.id.clone()).unwrap_or_default(); + Path { + path: PathIdentity { + id: "p1".to_string(), + base: None, + head, + graph_ref: None, + }, + steps, + meta: None, + } + } + + /// The artifact key is one byte, so the always-present `/change/a` + /// surface (whose text is the key itself) is too short for a `0..20` + /// span - only the `text` field can win the merge test below. + fn fixture_one_field() -> Path { + path_of(vec![step_with_text("s1", "a", "AAAAAAAAAAAAAAAAAAAA")]) + } + + fn fixture_mixed() -> Path { + path_of(vec![ + step_with_text("s1", "a", "here is a SECRET-VALUE to find"), + step_with_text("s2", "b", "nothing interesting here"), + ]) + } + + fn findings_at(plan: &Plan, at: &str) -> usize { + plan.findings.iter().filter(|pf| pf.at == at).count() + } + + // ── Step 8.1, verbatim ──────────────────────────────────────────────── + + #[test] + fn surfaces_and_findings_are_both_populated() { + let plan = generate(&fixture_mixed(), &detectors(), &cfg()); + assert!(plan.surfaces.iter().any(|s| findings_at(&plan, &s.at) == 0)); + assert!(!plan.findings.is_empty()); + } + + #[test] + fn two_detectors_merge_through_one_resolution() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![f(0..20, "a", 0.9)]))); + set.push(Box::new(FixedDetector(vec![f(0..20, "b", 0.5)]))); + assert_eq!( + generate(&fixture_one_field(), &set, &cfg()).findings.len(), + 1 + ); + } + + #[test] + fn network_detector_is_refused_without_the_flag() { + let mut set = DetectorSet::default(); + set.push(Box::new(NetworkDetector)); + assert!(matches!( + generate_checked(&fixture_one_field(), &set, &cfg(), false), + Err(crate::RedactError::NetworkDetectorRefused(_)) + )); + } + + // ── Coverage a reviewer would demand ──────────────────────────────── + + #[test] + fn zero_finding_surface_still_appears_in_plan_surfaces() { + let plan = generate(&fixture_one_field(), &DetectorSet::default(), &cfg()); + assert!(!plan.surfaces.is_empty()); + assert!(plan.findings.is_empty()); + } + + #[test] + fn plan_detectors_lists_the_detector_ids_actually_run() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(Vec::new()))); + set.push(Box::new(Needle)); + let plan = generate(&fixture_one_field(), &set, &cfg()); + assert_eq!( + plan.detectors, + vec!["fixed".to_string(), "needle".to_string()] + ); + } + + #[test] + fn score_exactly_at_threshold_is_redact_not_skip() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![f(0..20, "boundary", 0.8)]))); + let plan = generate(&fixture_one_field(), &set, &cfg()); + assert_eq!(plan.findings[0].action, Action::Redact); + } + + #[test] + fn score_just_below_threshold_is_skip() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![f(0..20, "boundary", 0.799)]))); + let plan = generate(&fixture_one_field(), &set, &cfg()); + assert_eq!(plan.findings[0].action, Action::Skip); + } + + #[test] + fn reveal_flag_propagates_into_generated_context() { + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![f(0..20, "boundary", 0.95)]))); + let cfg = crate::RedactConfig { + reveal: true, + ..cfg() + }; + let plan = generate(&fixture_one_field(), &set, &cfg); + assert!(plan.findings[0].context.contains("AAAAAAAAAAAAAAAAAAAA")); + } + + #[test] + fn empty_document_yields_empty_plan_with_configured_id_and_timestamp() { + let empty = path_of(Vec::new()); + let now = Utc.with_ymd_and_hms(2030, 1, 1, 0, 0, 0).unwrap(); + let cfg = crate::RedactConfig { now, ..cfg() }; + let plan = generate(&empty, &DetectorSet::default(), &cfg); + assert_eq!(plan.document, "p1"); + assert_eq!(plan.generated, now); + assert!(plan.surfaces.is_empty()); + assert!(plan.findings.is_empty()); + } + + #[test] + fn regenerating_a_plan_yields_byte_identical_json() { + let path = fixture_mixed(); + let set = detectors(); + let cfg = cfg(); + let a = generate(&path, &set, &cfg); + let b = generate(&path, &set, &cfg); + assert_eq!( + serde_json::to_string(&a).unwrap(), + serde_json::to_string(&b).unwrap() + ); + } + + #[test] + fn detector_error_propagates_through_generate_checked() { + let mut set = DetectorSet::default(); + set.push(Box::new(FailingDetector)); + let err = generate_checked(&fixture_one_field(), &set, &cfg(), false).unwrap_err(); + assert!(matches!(err, crate::RedactError::BadPointer(_))); + } +} diff --git a/scripts/release.sh b/scripts/release.sh index 92536096..802da591 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -337,11 +337,11 @@ if should_publish toolpath-convo; then fi # Tier 2b: satellite crates (depend on tier 1 and/or toolpath-convo) -for _crate in toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi; do +for _crate in toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi toolpath-redact; do publish "${_crate}" done -for _crate in toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi; do +for _crate in toolpath-git toolpath-github toolpath-dot toolpath-md toolpath-claude toolpath-gemini toolpath-codex toolpath-copilot toolpath-opencode toolpath-cursor toolpath-pi toolpath-redact; do if should_publish "${_crate}"; then wait_for_index "${_crate}" "$(crate_version "${_crate}")" fi From 397e12733940bdcf0be1d5f782b529cc9ed40149 Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Thu, 30 Jul 2026 16:20:10 -0400 Subject: [PATCH 6/9] docs(redact): record known gaps and execution state Two adversarial reviews found paths that write cleartext over a redacted cache entry: p import --force, share, p cache rm, and a redact racing a sync. None is fixed. Closing them means changing shipped behavior, which is out of scope for this branch, so they are written down instead. Also records that idempotence is unproven for the hash and partial transforms, which have no recognisable output form for a re-scan to skip. The execution-state note is a checkpoint, not documentation. Delete it when the branch merges. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 2 +- .../2026-07-30-redact-execution-state.md | 81 ++++++++++++++++++ .../notes/2026-07-30-redaction-known-gaps.md | 84 +++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/notes/2026-07-30-redact-execution-state.md create mode 100644 docs/superpowers/notes/2026-07-30-redaction-known-gaps.md diff --git a/CLAUDE.md b/CLAUDE.md index 0bbdffe2..d8afdeca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -296,5 +296,5 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. - `path query` does not load the whole cache into memory when it can avoid it. `crates/path-cli/src/query/plan.rs` parses the jaq filter into jaq's own AST (`jaq_core::load::parse::Term`) and classifies it into a `Plan`: `PerFileStream` (`.[] | g` element-wise work — run per document, print as you go), `Decompose { reduce }` (algebraic aggregations — run the whole filter per file, concatenate the per-file outputs, then run a derived combine: `map`→`add` (array concat), top-N `sort_by(k)|.[:N]`→`add | sort_by(k)|.[:N]`, `length`→`add` over exact integer counts), or `Slurp` (the always-correct whole-array fallback). Recognition is conservative — a non-distributive prefix like `unique`/`group_by` slurps, and so do scalar `add` (float sums re-associate across per-file partials), `min`/`max` (`[] | min == null` poisons the merge), and any unrecognized tail — so **the planner never changes an answer** — `crates/path-cli/src/query/filter.rs` tests assert streamed output equals slurp byte-for-byte. `filter::execute` compiles the filter once (jaq's compiled `Filter` is fully owned, so it's reused across files) and drives the plan; `mod.rs::stream_files` yields one document's wrapped steps at a time. `TOOLPATH_QUERY_EXPLAIN=1` prints the chosen plan to stderr. No user-facing flag — it's automatic. Tie-break caveat: a streamed top-N matches slurp's *ranking*, but boundary ties may resolve to different specific rows. - Cache sync: `path p cache sync [types…]` (`crates/path-cli/src/artifact.rs`: `ArtifactType` + `ArtifactRef` + the stamp helpers; `sync/engine.rs`: manifest + ingestion loop, no UI — it reports through a `SyncObserver` trait, `&mut ()` for a silent sync; `sync/sources.rs`: an `ArtifactSource` trait — enumerate / stamp / derive — with one impl per provider, so the engine never matches on artifact type; `cmd_cache.rs`: the stderr progress line + summary) incrementally ingests artifacts into the cache — no args syncs every artifact type. Change detection is **stat-level**: each artifact is enumerated as an `ArtifactRef` whose fingerprint is the source file's mtime + size (claude: the *whole session chain* — max segment mtime + summed segment sizes via `claude_chain_stamp`, because Claude Code rotates to a new file on continuation while the chain keeps its oldest segment's id, so appends land in the newest file, not the head; the chain comes from the same cached index `list_conversations` builds; codex: rollout file, id from the stem's trailing UUID; pi: session file, id from a one-line header peek; copilot: `session-state//events.jsonl`, pure read-dir + stat) or the DB row's updated-at (opencode: header-only `SELECT time_updated`; cursor: composer headers' `lastUpdatedAt`, bubble-less drafts skipped, workspace-less composers *included* unlike `share`). Gemini enumerates via `PathResolver::list_session_entries` (`toolpath-gemini` 0.6.1), whose identity peek is bounded to the first 4 KiB of a main file. Deciding "nothing changed" reads no session bodies — a no-op sync is milliseconds. Changed/new artifacts derive through the same provider managers (each source calls the `derive_*_session_with` helpers in `derive.rs`). Manifest at `~/.toolpath/manifest.json`: artifact type → artifact id → `{path?, cache_id, modified?, size?, synced_at}`; atomic temp+rename writes, `0600`, checkpointed every 10 writes (interruption-safe: a killed run keeps nearly everything it derived, and derives run newest-first so partial progress covers the sessions that matter most); writers serialize on an advisory lock (`manifest.json.lock`) and every write is a locked read-merge-save — checkpoints merge only the records the run wrote — so concurrent invocations (query auto-syncs, imports) union their records instead of clobbering each other. Pending work reports progress on stderr (`\r`-updating ` done/total` on a TTY, a plain line every 25 items otherwise; no-op syncs stay silent). Sync always writes the cache with force — refresh semantics — and never deletes: artifacts removed upstream keep their cache docs and manifest records (archive, not mirror). Derivation failures warn and tally, they don't abort. A record's `cache_id` is *optional*: a record without one is "known, not materialized" — created when `p cache rm` evicts a doc (rm downgrades the record; the next sync re-materializes it, and sync also verifies the doc file actually exists before skipping, so even out-of-band deletions self-heal). Claude derives leave `DeriveConfig.project_path` unset so `path.base` comes from the session's own recorded cwd rather than the lossy slug. `path query` runs this sync implicitly before reading, scoped to its flags (`--source X` → that type; `--id`s → their prefixes; bare query → all types; `--input`-only → none), quiet unless something was ingested, degrading to the cache as-is if sync fails; `--no-sync` opts out. `p import` and `share` record what they write: every session derive carries a provenance `ArtifactRef` (stamped *before* the source is read, in `DerivedDoc.provenance`), and the cache-write sites call `sync::record_artifact` so the next sync sees those artifacts as unchanged instead of re-deriving them. Every import flow — explicit `--session`, picker multi-select, `--all`, and the most-recent fallbacks — loops the per-session helpers, so every session write is recorded; there is no bulk `derive_project` path in the CLI anymore, and `p import pi --all` now emits one Path per session like every other provider (it used to emit a single combined Graph). `--no-cache` paths record nothing: the manifest describes the cache. -- Redaction workflow (`toolpath-redact` + `path p redact`): plan-then-apply, not a single opaque pass. `surfaces()` enumerates every string field a credential could hide in. `plan::generate()` runs detectors over those surfaces, producing a reviewable `Plan` with stable ids and elided context. `apply()` consumes the plan and rewrites the document. The plan can be decided by predicate, by picker, or by hand-editing JSON. In-place redaction (`--input `) rewrites the cache entry; re-deriving from the source (`p cache sync`, including the implicit sync `path query` runs) replays the stored `RedactionPolicy`, so turns added after a resume are redacted too. A detector is a trait (`Detector`), not a function, so its implementation is swappable: a `FixedDetector` for tests, the built-in rule-based detector by default, and a harness-time hook for pre-training redaction. Detection is the part of this problem where precision is worst and the field moves fastest, so it sits behind the plug point; traversal (fields, pointers, surfaces) is stable and reused by all detectors. +- Redaction workflow (`toolpath-redact` + `path p redact`): plan-then-apply, not a single opaque pass. `surfaces()` enumerates every string field a credential could hide in. `plan::generate()` runs detectors over those surfaces, producing a reviewable `Plan` with stable ids and elided context. `apply()` consumes the plan and rewrites the document. The plan can be decided by predicate, by picker, or by hand-editing JSON. In-place redaction (`--input `) rewrites the cache entry; re-deriving from the source (`p cache sync`, including the implicit sync `path query` runs) replays the stored `RedactionPolicy`, so turns added after a resume are redacted too. A detector is a trait (`Detector`), not a function, so its implementation is swappable: a `FixedDetector` for tests, the built-in rule-based detector by default, and a harness-time hook for pre-training redaction. Detection is the part of this problem where precision is worst and the field moves fastest, so it sits behind the plug point; traversal (fields, pointers, surfaces) is stable and reused by all detectors. **Replay is not airtight**: `p import --force`, `share`, `p cache rm`, and a concurrent redact-during-sync each write cleartext over a redacted cache entry, and idempotence is unproven for the `hash` and `partial` transforms. All seven known gaps are documented in `docs/superpowers/notes/2026-07-30-redaction-known-gaps.md`; none is fixed, because closing them means changing shipped behavior. - `ArtifactType` (`crates/path-cli/src/artifact.rs`) is the general enum naming artifact sources — the seven agent harnesses (incl. copilot) plus `Git` (8 variants). Git artifacts are *recorded* in the manifest by `p import git` (id `-`, `path` = the repo directory) but never *discovered* — there is no machine-wide registry of repos — so sync reports them and leaves them alone. Github and pathbase are deliberately not artifact types: they are remote services, not local artifact sources, and their imports stay out of the manifest. It derives `clap::ValueEnum` and is used by `p cache sync` types, the sync manifest keys, `ArtifactRow.artifact_type`, and `cmd_import`'s cache-id prefixes (`name()` is both the manifest key and the `make_id` source string). The deliberately parallel `Harness` enum (`crates/path-cli/src/harness.rs`, alongside `HarnessBundle`) names the seven agent *runtimes* — things sessions can be shared from and resumed into — and is what `share`/`resume` `--harness` take, so future non-harness artifact types stay unrepresentable there (you can't resume into a git repo). `Harness::artifact_type()` maps into the general enum; `ArtifactType::harness()` is the partial inverse. Keep new code on `ArtifactType` unless it's genuinely harness-only. diff --git a/docs/superpowers/notes/2026-07-30-redact-execution-state.md b/docs/superpowers/notes/2026-07-30-redact-execution-state.md new file mode 100644 index 00000000..f748ab48 --- /dev/null +++ b/docs/superpowers/notes/2026-07-30-redact-execution-state.md @@ -0,0 +1,81 @@ +# `path p redact` execution state + +Working state of the parallel implementation on branch `evan/redact`. +Written as a checkpoint so the work is recoverable without the +orchestrator's transcript. Delete this file when the branch merges. + +## Committed + +| Commit | Contents | +|---|---| +| `34a36c0` | T0 shared vocabulary | +| `1e456df` | T1 span normalisation, T4 transforms, T6 CLI args, T12 docs | +| `98d51c3` | T2 field map, T5 plan machinery, T3 detector | +| `39ce229` | T7 apply, T8 plan generation, T10 sync replay | + +At `39ce229`: `toolpath-redact` 140 tests, `path-cli` 379 lib + 56 +integration, all green. `toolpath-redact` is clippy-clean. + +## Not implemented + +- T9 CLI dispatch was `todo!()` at `39ce229`; an agent has since written + ~860 lines of it, uncommitted. +- T11 end-to-end integration tests. +- `src/exec.rs`, the subprocess detector. Cut deliberately. +- T3 step 3.6 checksum validators. Cut deliberately. + +## Standing constraints from the user + +1. **Do not fix pre-existing lint issues.** `cargo clippy --workspace -- + -D warnings` and `cargo fmt --check` are red on findings that predate + this branch (`cmd_list.rs` 370/390/542/563, `cmd_import.rs` 583/774, + `toolpath-pi/src/reader.rs:359`, rustfmt drift in `toolpath-codex`). + An earlier commit fixing these was reverted on request. +2. **Do not change behavior outside `toolpath-redact`.** The known + un-redaction paths are documented, not fixed. See + `2026-07-30-redaction-known-gaps.md`. +3. Agents summarize and hand off at 256k context, hard stop at 500k. + +## Open review findings not yet applied + +From the T3 review, being remediated: + +- `BASE_SCORE` 0.6 against a default threshold of 0.8 means nothing + redacts unless a hotword is within the window. All five of the plan's + own true-positive fixtures score 0.600 and are skipped. +- `secretGroup` is not deserialized, so the wrong span is redacted for + `sonar-api-token`, `microsoft-teams-webhook`, `jwt-base64`. +- `mask_existing_markers` substitutes NUL, which satisfies `[^\s:@/]` + and so causes the idempotence break it exists to prevent. +- Diff clipping keeps only the first line of a multi-line secret, so PEM + key material survives. +- The three excluded rules fail on regex size limit, not dialect. All 221 + compile with `size_limit(64 << 20)`. +- Hotword window is bytes, documented as characters. +- No per-rule keyword gating; all 221 regexes run on any keyword hit. + +From the T7 review, being remediated: + +- `group_by_field`'s `BTreeMap` orders `/change/X` before pointers + beneath it, inverting `surfaces()`'s documented contract. A valid plan + errors with the document already half-rewritten. +- The audit record publishes the pre-redaction artifact key verbatim. +- Signatures are stripped from steps the pass never touched. +- A zero-width span splices a marker into clean text. +- An empty fingerprint key is accepted. +- The record's `at` carries an `extra` segment that does not resolve + against the serialized step, because `StructuralChange.extra` is + `#[serde(flatten)]`. + +## Loose ends + +- An agent wrote a `.env` fixture into the repo root. Values are fake + (AWS's documented `EXAMPLE` key). It is untracked and **not** in + `.gitignore`, so `git add -A` would commit it. Stage explicit paths. +- Accept/reject precedence is unsettled between tracks. `sync/engine.rs` + `policy_decisions` applies `accept` then `reject`. T11's + `redact_accept_reject_precedence` is the authority; if it disagrees, + flip `policy_decisions`. +- `path p validate` against redacted output is unverified. T7 could not + run a real JSON Schema validator, so T11 is the only real schema check + in the implementation. diff --git a/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md b/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md new file mode 100644 index 00000000..5ce8329a --- /dev/null +++ b/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md @@ -0,0 +1,84 @@ +# Redaction: known gaps + +Found by adversarial review of `path p redact` (branch `evan/redact`, +2026-07-30). **None of these are fixed.** They are recorded here rather +than closed, because closing them means changing behavior that already +ships. Read this before assuming a redacted document stays redacted. + +The invariant all of these violate is the one Task 10 exists to protect: +`sync::engine::is_unchanged` decides re-derivation from source mtime and +size and never inspects the document, so anything that re-derives and +writes without replaying the stored `RedactionPolicy` publishes the +cleartext it was hiding. + +## 1. `p import --force` and `share` overwrite a redacted cache entry + +`cmd_import.rs` and `cmd_share.rs` call `write_cached` with the raw +derivation and no replay, then call `sync::record_artifact`, which +deliberately preserves the stored policy while restamping `modified` and +`size` from the fresh provenance. `is_unchanged` therefore returns true +on every later sync and the cleartext is never repaired. The manifest +ends up asserting a policy that is not in force. + +Reproduce: `path p redact -i claude-sess-1`, then +`path p import claude --project P --session sess-1 --force`. + +## 2. `p cache rm` un-redacts on the next sync + +`evict_cache_id` clears the record's policy and `run_rm` deletes the key. +The record keeps its stamps but loses `cache_id`, so the next in-scope +sync re-derives with no policy and recreates the document in cleartext. +`path query` triggers that sync implicitly, so no explicit sync is +needed. + +Keeping the key and the policy would be the fix. A policy whose key is +gone can only fail forever, but the key is 32 bytes and its whole purpose +is fingerprint stability across re-redactions of that one document. + +Related: `run_rm` removes the key named by the *cache id*, not by +`policy.key_id`. Nothing enforces that those are equal, so a policy with +a different `key_id` means `rm` deletes another document's key. + +## 3. A policy recorded mid-sync is clobbered + +`sync_bundle` loads the manifest once for the whole run and +`flush_writes` merges with `BTreeMap::extend`, which replaces whole +records. Redacting in one shell while `path query` syncs in another loses +both the document and the policy. `record_artifact` already handles the +same aliasing correctly, so the two writers disagree. + +## 4. An empty `detectors` list replays as a silent no-op + +`build_detectors` returns an empty `DetectorSet` for `&[]`, so +`generate_checked` finds nothing, `apply` early-returns byte-identically, +and the run reports the document as re-redacted. The manifest is +user-editable JSON, so this is reachable without a code bug. + +## 5. An unparseable `redaction` value takes down the whole manifest + +`SyncRecord.redaction` deserializes strictly. `#[serde(default)]` covers a +missing key, not a malformed value, so one bad policy makes the entire +manifest fail to load for every artifact type. The error hint then tells +the user to delete the manifest, which discards every stored policy and +un-redacts everything on the next sync. + +Version skew is enough to trigger it: `Transform` is a plain string enum +with no unknown-variant fallback. + +## 6. Idempotence is unproven for `hash` and `partial` + +`internal::mask_existing_markers` blanks `[REDACTED:…]` and runs of `█` +before any rule scans, which is what makes redaction reach a fixed point. +It does **not** cover `Transform::Hash` output (bare 6 hex characters) or +`Transform::Partial` output (`head…tail`). The idempotence test passes +because its scanner uses prefixed self-delimiting formats that no +transform output can reconstitute, so the gap is never exercised. + +## 7. `share` uploads the un-redacted derivation + +Redacting a document does not affect what `share` sends to Pathbase: the +cached copy is redacted and the uploaded copy is not. This one is +arguably as specified, since the plan's non-goals say share gains no scan +and no automatic redaction. Replaying an already-approved policy for that +exact document is not obviously either of those, so it is recorded here +as a decision rather than a defect. From e400e0439cad9569a98f5249934ee763e2e31b7d Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Fri, 31 Jul 2026 01:29:14 -0400 Subject: [PATCH 7/9] feat(redact): `path p redact` end to end Wires the CLI, closes the end-to-end test gaps, and fixes the defects four adversarial reviews and one real user run found. Every defect below was reproduced before being fixed, and none of the 140 tests passing at the previous commit would have caught any of them. The threshold was applied after overlap resolution, which detect.rs's own doc comment forbids. Resolution is score-blind on length, so a 0.6 container spanning a 0.99 match evicted it and the threshold then dropped the survivor: both lost, and the plan named only the low-confidence one. Findings are partitioned on the threshold before resolving, so a sub-threshold finding can never evict an above-threshold one. `verify` never compared content, so a plan generated against one document was accepted against a mutated one and apply spliced markers at stale offsets. Findings now carry a keyed fingerprint that verify recomputes. The audit record republished the credential: pointers beneath a redacted artifact key were recorded from the pre-redaction key, so the pass removed the secret from `change` and wrote it back under `meta.redaction`. Overlapping spans left half a credential in place while the report claimed both were replaced. Overlap is refused before any edit is built, the same way empty and inverted spans already were. Detection missed two of three credentials in a real session. gitleaks' anthropic rule is pinned to a live key's exact length, and `generic-api-key` stopwords `ant-`, so every sk-ant- key was excluded from the one rule that would have caught it. Documentation-key allowlist regexes are now suppressed for redaction while stopwords are kept: gitleaks scans source trees where a README quoting AWS's example key is noise, but a transcript about to be published cannot distinguish a copied placeholder from a real key whose owner ended it in EXAMPLE. A zero-finding run still rewrote the cache file, and `extra`/`change` are flattened HashMaps seeded per process, so re-serialising churned key order and broke byte-identity on a second pass. The integration fixtures' own single-key maps, chosen to avoid that flake, made them incapable of catching it. Detector fixtures are split across `concat!` so the source text does not match the pattern each value tests. They are synthetic, but GitHub push protection scans the file rather than the compiled string. Verified against a real 46-step session carrying seven credential types: 21 typed markers, zero residual, valid against the base and kind schemas, byte-identical on a second pass, cache mode preserved at 0600. toolpath-redact 201, path-cli 383 lib + 67 integration. No test ignored. Known gaps, deliberately not closed, are recorded in docs/superpowers/notes/2026-07-30-redaction-known-gaps.md. Co-Authored-By: Claude Opus 5 (1M context) --- crates/path-cli/src/cache.rs | 2 - crates/path-cli/src/cmd_redact.rs | 636 ++++++++- crates/path-cli/src/sync/engine.rs | 36 +- crates/path-cli/tests/integration.rs | 835 ++++++++++++ crates/toolpath-redact/src/apply.rs | 739 ++++++++-- crates/toolpath-redact/src/detect.rs | 121 +- crates/toolpath-redact/src/internal/mod.rs | 641 +++++++-- crates/toolpath-redact/src/internal/rules.rs | 453 ++++++- crates/toolpath-redact/src/lib.rs | 5 + crates/toolpath-redact/src/plan.rs | 786 +++++++++-- crates/toolpath-redact/src/surface.rs | 1203 ++++++++++++++--- .../notes/2026-07-30-redaction-known-gaps.md | 26 +- 12 files changed, 4913 insertions(+), 570 deletions(-) diff --git a/crates/path-cli/src/cache.rs b/crates/path-cli/src/cache.rs index 5bc7ff17..156fe73b 100644 --- a/crates/path-cli/src/cache.rs +++ b/crates/path-cli/src/cache.rs @@ -189,8 +189,6 @@ pub(crate) fn read_redact_key(key_id: &str) -> Result>> { /// Written with `create_new`, so two redactions racing the same /// document agree on one key instead of each fingerprinting under its /// own. -// Called by `path p redact`; drop the allow once that dispatch lands. -#[allow(dead_code)] pub(crate) fn load_or_create_redact_key(key_id: &str) -> Result> { use rand::RngCore; use std::io::Write; diff --git a/crates/path-cli/src/cmd_redact.rs b/crates/path-cli/src/cmd_redact.rs index 9f530ad4..0274fda3 100644 --- a/crates/path-cli/src/cmd_redact.rs +++ b/crates/path-cli/src/cmd_redact.rs @@ -1,9 +1,15 @@ //! `path p redact` — remove credentials from a toolpath document in place //! via a reviewable plan-then-apply flow. -use anyhow::Result; +use anyhow::{Context, Result, anyhow, bail}; +use chrono::Utc; use clap::Args; use std::path::PathBuf; +use toolpath::v1::{Graph, PathOrRef}; +use toolpath_redact::{ + Action, Decision, DetectorSet, Plan, PlanFinding, Predicate, RedactConfig, RedactionPolicy, + Transform, parse_predicate, +}; #[derive(Debug, Args)] pub(crate) struct RedactArgs { @@ -79,30 +85,52 @@ pub(crate) fn run(args: RedactArgs) -> Result<()> { /// `cmd_resume::run_with_strategy`, which exists for the same reason: the /// alternative is a process-global picker override that one test poisons /// for the whole binary. -pub(crate) fn run_with_picker(_args: RedactArgs, _picker: &dyn PickerStrategy) -> Result<()> { - todo!("T9") +pub(crate) fn run_with_picker(args: RedactArgs, picker: &dyn PickerStrategy) -> Result<()> { + let execution = execute(&args, picker, crate::sync::build_detectors)?; + if args.dry_run && execution.pending > 0 { + std::process::exit(1); + } + Ok(()) } pub(crate) trait PickerStrategy { /// Rows are TSV with the finding id in column 1. Returns the selected /// rows verbatim, the way `fzf` does - not bare ids. - /// - /// Unused until `run_with_picker` stops being a `todo!()`. - #[allow(dead_code)] fn pick(&self, rows: &[String]) -> Result>; } pub(crate) struct RealPicker; impl PickerStrategy for RealPicker { - fn pick(&self, _rows: &[String]) -> Result> { - todo!("T9") + fn pick(&self, rows: &[String]) -> Result> { + #[cfg(not(target_os = "emscripten"))] + { + if !crate::fuzzy::available() { + bail!("--interactive needs a TTY (or fzf on PATH / the embedded-picker feature)"); + } + let opts = crate::fuzzy::PickOptions { + multi: true, + prompt: "redact> ", + header: Some("TAB to toggle which findings to redact"), + ..Default::default() + }; + match crate::fuzzy::pick(rows, &opts)? { + crate::fuzzy::PickResult::Selected(v) => Ok(v), + crate::fuzzy::PickResult::NoMatch | crate::fuzzy::PickResult::Cancelled => { + Ok(Vec::new()) + } + } + } + #[cfg(target_os = "emscripten")] + { + let _ = rows; + bail!("--interactive is not supported on this build"); + } } } /// Constructed by the dispatch tests, which arrive with `run_with_picker`. #[cfg(test)] -#[allow(dead_code)] pub(crate) struct RecordingPicker { pub selection: Vec, pub seen: std::cell::RefCell>, @@ -118,7 +146,6 @@ impl PickerStrategy for RecordingPicker { /// Splits on the LAST `:` so a predicate containing `:` still parses, e.g. /// `detector=exec:/bin/gitleaks:hash`. -#[cfg_attr(not(test), allow(dead_code))] pub(crate) fn parse_mode_for(s: &str) -> Result<(String, TransformArg)> { let (pred, transform_str) = s .rsplit_once(':') @@ -149,6 +176,350 @@ impl From for toolpath_redact::Transform { } } +// ── dispatch (T9) ─────────────────────────────────────────────────────── + +/// How `execute` builds the detector set named by `--detector`. A parameter +/// (mirroring `sync::engine`'s own `DetectorFactory`) so tests can swap in a +/// deterministic registry instead of the vendored ruleset. +type DetectorFactory = fn(&[String]) -> Result; + +/// What one run produced, independent of whether it wrote anything — +/// `run_with_picker` reads `pending` to decide the dry-run exit code; tests +/// read `plans` to inspect the final per-finding decisions. +struct Execution { + /// Read only by tests, which inspect final per-finding decisions; + /// production only needs `pending` for the dry-run exit code. + #[cfg_attr(not(test), allow(dead_code))] + plans: Vec, + pending: usize, +} + +fn execute( + args: &RedactArgs, + picker: &dyn PickerStrategy, + build_detectors: DetectorFactory, +) -> Result { + let (cache_id, doc_path) = resolve_input(&args.input)?; + let mut doc = crate::io::read_document_auto(&doc_path)?; + + let key = resolve_key(args, cache_id.as_deref(), &doc)?; + let cfg = RedactConfig { + threshold: args.threshold, + mode: args.mode.into(), + // Per-finding overrides ride the plan via `--mode-for` decisions + // below; nothing here needs the rule-keyed config fallback. + mode_for: Vec::new(), + key, + now: Utc::now(), + drop_signatures: args.drop_signatures, + reveal: args.reveal, + }; + + let mode_for = resolve_mode_for(&args.mode_for)?; + let mut decisions = mode_for_decisions(&mode_for); + decisions.extend(predicate_decisions(&args.accept, Action::Redact)?); + // Reject lands last: `apply_decisions` lets later entries win, so an + // explicit skip survives a broader accept or mode-for (matches + // `sync::engine::policy_decisions`). + decisions.extend(predicate_decisions(&args.reject, Action::Skip)?); + + let mut plans: Vec<(usize, Plan)> = Vec::new(); + if let Some(plan_file) = &args.plan { + let text = std::fs::read_to_string(plan_file) + .with_context(|| format!("read {}", plan_file.display()))?; + let plan: Plan = serde_json::from_str(&text) + .with_context(|| format!("parse {}", plan_file.display()))?; + let idx = doc + .paths + .iter() + .position(|p| matches!(p, PathOrRef::Path(pp) if pp.path.id == plan.document)) + .ok_or_else(|| { + anyhow!( + "plan targets document {:?}, which is not in {}", + plan.document, + doc_path.display() + ) + })?; + plans.push((idx, plan)); + } else { + let detectors = build_detectors(&args.detector)?; + for (idx, entry) in doc.paths.iter().enumerate() { + if let PathOrRef::Path(path) = entry { + let plan = toolpath_redact::plan::generate_checked( + path, + &detectors, + &cfg, + args.allow_network_detectors, + )?; + plans.push((idx, plan)); + } + } + } + + for (_, plan) in &mut plans { + toolpath_redact::plan::apply_decisions(plan, &decisions); + } + if args.interactive { + run_interactive(&mut plans, picker)?; + } + + let pending: usize = plans + .iter() + .map(|(_, p)| { + p.findings + .iter() + .filter(|f| f.action == Action::Redact) + .count() + }) + .sum(); + + if args.dry_run { + print_dry_run(args, &plans)?; + return Ok(Execution { + plans: plans.into_iter().map(|(_, p)| p).collect(), + pending, + }); + } + + let mut total_replaced = 0usize; + let mut total_signatures_dropped = 0usize; + for (idx, plan) in &plans { + let PathOrRef::Path(path) = &mut doc.paths[*idx] else { + unreachable!("index came from a PathOrRef::Path match above"); + }; + let report = toolpath_redact::apply(path, plan, &cfg)?; + total_replaced += report.replaced.values().sum::(); + total_signatures_dropped += report.signatures_dropped; + } + let changed = total_replaced > 0 || total_signatures_dropped > 0; + // Informational only: file inputs with no `--output` write the + // redacted document itself to stdout (so `path p redact --input + // doc.json | path p export pathbase --input -` composes), so this + // summary must never share that stream. + eprintln!("{total_replaced} finding(s) redacted"); + + // Re-serialising an unchanged document is not a no-op: `extra` and + // `change` are `#[serde(flatten)]` HashMaps whose iteration order is + // seeded per process, so writing would churn the file's key order and + // break "redacting twice is byte-identical to redacting once". A file + // input with no `--output` still emits, since stdout is the result. + if changed || cache_id.is_none() { + write_output(args, cache_id.as_deref(), &doc)?; + } + + if let (Some(id), Some((_, plan))) = (&cache_id, plans.first()) { + let policy = RedactionPolicy { + detectors: plan.detectors.clone(), + threshold: plan.defaults.threshold, + mode: plan.defaults.transform, + mode_for: rule_mode_for(&mode_for), + accept: args.accept.clone(), + reject: args.reject.clone(), + key_id: id.clone(), + }; + crate::sync::record_redaction_policy(id, &policy)?; + } + + Ok(Execution { + plans: plans.into_iter().map(|(_, p)| p).collect(), + pending, + }) +} + +/// Mirrors the file-vs-cache-id heuristic `cache::cache_ref` applies +/// internally, so one `--input` string is never classified two different +/// ways by the two callers that need to know which it is. +fn is_file_ref(s: &str) -> bool { + s.contains('/') || s.contains('\\') || s.ends_with(".json") +} + +fn resolve_input(input: &str) -> Result<(Option, PathBuf)> { + let path = crate::cache::cache_ref(input)?; + let cache_id = (!is_file_ref(input)).then(|| input.to_string()); + Ok((cache_id, path)) +} + +/// Mirrors `toolpath_redact::apply`'s private record key ("redaction"); +/// there is no export for it, so the literal here must track that crate's +/// contract rather than being renamed independently. +const REDACTION_RECORD_KEY: &str = "redaction"; + +fn already_redacted(doc: &Graph) -> bool { + doc.paths.iter().any(|entry| match entry { + PathOrRef::Path(path) => { + path.meta + .as_ref() + .is_some_and(|m| m.extra.contains_key(REDACTION_RECORD_KEY)) + || path.steps.iter().any(|s| { + s.meta + .as_ref() + .is_some_and(|m| m.extra.contains_key(REDACTION_RECORD_KEY)) + }) + } + PathOrRef::Ref(_) => false, + }) +} + +/// HMAC-SHA256 is block-size independent; 32 bytes is well past the point +/// where key length stops mattering (mirrors `cache::REDACT_KEY_LEN`, +/// private to that module). +const EPHEMERAL_KEY_LEN: usize = 32; + +/// A cache id's key is stored and reused across runs; a file input with no +/// `--key-file` has nowhere durable to keep one, so each run mints its own +/// and fingerprints won't correlate across runs for that input alone. +fn resolve_key(args: &RedactArgs, cache_id: Option<&str>, doc: &Graph) -> Result> { + if let Some(key_file) = &args.key_file { + return std::fs::read(key_file).with_context(|| format!("read {}", key_file.display())); + } + let Some(id) = cache_id else { + return Ok(random_key()); + }; + if already_redacted(doc) { + return crate::cache::read_redact_key(id)?.ok_or_else(|| { + anyhow!( + "redaction key for {id} is missing; refusing to re-redact with a fresh key, \ + which would silently break correlation with the existing markers" + ) + }); + } + crate::cache::load_or_create_redact_key(id) +} + +fn random_key() -> Vec { + use rand::RngCore; + let mut key = vec![0u8; EPHEMERAL_KEY_LEN]; + rand::rng().fill_bytes(&mut key); + key +} + +fn resolve_mode_for(specs: &[String]) -> Result> { + specs + .iter() + .map(|s| { + let (pred, transform) = parse_mode_for(s)?; + let predicate = parse_predicate(&pred).map_err(|e| anyhow!("{e}"))?; + Ok((predicate, transform.into())) + }) + .collect() +} + +fn mode_for_decisions(mode_for: &[(Predicate, Transform)]) -> Vec { + mode_for + .iter() + .map(|(predicate, transform)| Decision { + predicate: predicate.clone(), + action: Action::Redact, + transform: Some(*transform), + }) + .collect() +} + +/// Only `rule=`-shaped `--mode-for` entries survive into the persisted +/// policy: `RedactionPolicy.mode_for` rides `RedactConfig.mode_for`, which +/// `toolpath_redact::transform::resolve_transform` resolves by rule-name +/// equality only. Anything else (`shape=`, `at=`, `score`, …) is a +/// one-time choice for this run — the same asymmetry the accept/reject +/// replay already documents for an individually hand-picked finding. +fn rule_mode_for(mode_for: &[(Predicate, Transform)]) -> Vec<(String, Transform)> { + mode_for + .iter() + .filter_map(|(predicate, transform)| match predicate { + Predicate::Rule(name) => Some((name.clone(), *transform)), + _ => None, + }) + .collect() +} + +fn predicate_decisions(specs: &[String], action: Action) -> Result> { + specs + .iter() + .map(|s| { + Ok(Decision { + predicate: parse_predicate(s).map_err(|e| anyhow!("{e}"))?, + action, + transform: None, + }) + }) + .collect() +} + +/// Interactive review is authoritative: every finding's action is set from +/// the picker's selection, overriding whatever `--accept`/`--reject`/ +/// `--mode-for` decided, since a human is now looking at each one directly. +fn run_interactive(plans: &mut [(usize, Plan)], picker: &dyn PickerStrategy) -> Result<()> { + let rows: Vec = plans + .iter() + .flat_map(|(_, p)| p.findings.iter()) + .map(|f| format!("{}\t{}\t{}\t{}", f.id, f.rule, f.step, f.at)) + .collect(); + let selected = picker.pick(&rows)?; + let accepted: std::collections::HashSet<&str> = selected + .iter() + .map(|row| row.split('\t').next().unwrap_or(row.as_str())) + .collect(); + for (_, plan) in plans.iter_mut() { + for finding in &mut plan.findings { + finding.action = if accepted.contains(finding.id.as_str()) { + Action::Redact + } else { + Action::Skip + }; + } + } + Ok(()) +} + +fn print_dry_run(args: &RedactArgs, plans: &[(usize, Plan)]) -> Result<()> { + if args.json { + let payload: Vec<&Plan> = plans.iter().map(|(_, p)| p).collect(); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + for (_, plan) in plans { + println!("document {}", plan.document); + for surface in &plan.surfaces { + let hits: Vec<&PlanFinding> = plan + .findings + .iter() + .filter(|f| f.step == surface.step && f.at == surface.at) + .collect(); + if hits.is_empty() { + println!( + " {} {} ({} bytes): no findings", + surface.step, surface.at, surface.bytes + ); + } else { + for f in hits { + println!( + " {} {}: {} (score {:.2}, {:?})", + surface.step, surface.at, f.rule, f.score, f.action + ); + } + } + } + } + Ok(()) +} + +fn write_output(args: &RedactArgs, cache_id: Option<&str>, doc: &Graph) -> Result<()> { + if let Some(id) = cache_id { + if args.output.is_some() { + bail!("--output is not compatible with a cache id input; `{id}` redacts in place"); + } + crate::cache::write_cached(id, doc, true)?; + return Ok(()); + } + let json = doc.to_json_pretty()?; + match &args.output { + Some(path) => { + std::fs::write(path, json).with_context(|| format!("write {}", path.display()))? + } + None => println!("{json}"), + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -252,4 +623,249 @@ mod tests { } } } + + // ── dispatch (Step 9.1) ────────────────────────────────────────────── + // + // The plan's `sandbox()`/`run_redact()`/`run_sync()` helpers don't + // exist anywhere in this codebase (Task 10's report against + // `sync/engine.rs` found the same); these tests drive `execute` + // directly through the existing `$TOOLPATH_CONFIG_DIR`-sandboxing + // `with_cfg` convention (see `cache.rs`, `sync/engine.rs`) instead. + + use crate::config::{CONFIG_DIR_ENV, TEST_ENV_LOCK}; + use std::collections::HashMap; + use toolpath::v1::{ArtifactChange, StepIdentity, StructuralChange}; + + fn with_cfg R, R>(f: F) -> R { + let _g = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let temp = tempfile::tempdir().unwrap(); + unsafe { + std::env::set_var(CONFIG_DIR_ENV, temp.path()); + } + let result = f(temp.path()); + unsafe { + std::env::remove_var(CONFIG_DIR_ENV); + } + result + } + + /// A detector matching literal `SECRET-` markers, so redact + /// fixtures are deterministic and don't depend on the vendored + /// ruleset's regex/entropy behaviour (mirrors the equivalent test + /// detector in `sync/engine.rs`). + struct LiteralSecret; + + impl toolpath_redact::Detector for LiteralSecret { + fn id(&self) -> &'static str { + "literal" + } + + fn detect( + &self, + c: &toolpath_redact::Candidate<'_>, + ) -> toolpath_redact::Result> { + Ok(c.text + .match_indices("SECRET-") + .map(|(start, m)| { + let tail = &c.text[start + m.len()..]; + let run = tail + .find(|ch: char| !ch.is_ascii_alphanumeric()) + .unwrap_or(tail.len()); + toolpath_redact::Finding { + span: start..start + m.len() + run, + rule: "literal".to_string(), + score: 1.0, + detector: "literal", + } + }) + .collect()) + } + } + + fn literal_detectors(_names: &[String]) -> Result { + let mut set = DetectorSet::default(); + set.push(Box::new(LiteralSecret)); + Ok(set) + } + + /// One `conversation.append` step per `(id, text)` pair, addressable at + /// `/change/convo/structural/extra/text` — the surface `toolpath_redact` + /// generates a finding on when `text` contains a `SECRET-` marker. + fn literal_secret_doc(steps: &[(&str, &str)]) -> Graph { + let mut path = + toolpath::v1::Path::new("doc-1", None, steps.last().expect("at least one step").0); + for (id, text) in steps { + let mut extra = HashMap::new(); + extra.insert( + "text".to_string(), + serde_json::Value::String((*text).to_string()), + ); + let mut change = HashMap::new(); + change.insert( + "convo".to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "conversation.append".to_string(), + extra, + }), + }, + ); + path.steps.push(toolpath::v1::Step { + step: StepIdentity { + id: (*id).to_string(), + parents: Vec::new(), + actor: "human:t".to_string(), + timestamp: "2026-01-01T00:00:00Z".to_string(), + }, + change, + meta: None, + }); + } + Graph::from_path(path) + } + + fn three_findings_fixture() -> Graph { + literal_secret_doc(&[ + ("s1", "alpha SECRET-A here"), + ("s2", "beta SECRET-B here"), + ("s3", "gamma SECRET-C here"), + ]) + } + + fn seed_cached_document(id: &str) { + let doc = literal_secret_doc(&[("s1", "here is SECRET-A now")]); + crate::cache::write_cached(id, &doc, true).unwrap(); + } + + fn reseed_same_document(id: &str) { + seed_cached_document(id); + } + + fn read_cached(id: &str) -> String { + std::fs::read_to_string(crate::cache::cache_path(id).unwrap()).unwrap() + } + + #[cfg(unix)] + fn mode_of(p: &std::path::Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(p).unwrap().permissions().mode() & 0o777 + } + + /// Structural, not textual: walks the applied document's per-step + /// `meta.redaction.findings[].fp` entries directly, so this doesn't + /// need a `regex` dependency (optional in this crate, gated behind + /// the `embedded-picker` feature) just to scrape a fixture. + fn fingerprints_in(json: &str) -> Vec { + let v: serde_json::Value = serde_json::from_str(json).unwrap(); + let mut out = Vec::new(); + if let Some(paths) = v.get("paths").and_then(|p| p.as_array()) { + for p in paths { + if let Some(steps) = p.get("steps").and_then(|s| s.as_array()) { + for step in steps { + if let Some(findings) = step + .pointer(&format!("/meta/{REDACTION_RECORD_KEY}/findings")) + .and_then(|f| f.as_array()) + { + for f in findings { + if let Some(fp) = f.get("fp").and_then(|x| x.as_str()) { + out.push(fp.to_string()); + } + } + } + } + } + } + } + out.sort(); + out + } + + fn redacted_ids(plans: &[Plan]) -> Vec { + plans + .iter() + .flat_map(|p| p.findings.iter()) + .filter(|f| f.action == Action::Redact) + .map(|f| f.id.clone()) + .collect() + } + + struct NoPicker; + impl PickerStrategy for NoPicker { + fn pick(&self, _rows: &[String]) -> Result> { + panic!("picker invoked without --interactive"); + } + } + + /// `run_redact(&["-i", ...])` from the plan, adapted: no `sandbox()` + /// return value to thread through, since `with_cfg` already pins + /// `$TOOLPATH_CONFIG_DIR` for the closure's duration. Returns the exit + /// code `run_with_picker` would signal via `std::process::exit`, which + /// a unit test cannot observe directly. + fn run_redact(args: &[&str]) -> Result { + let mut argv = vec!["redact"]; + argv.extend_from_slice(args); + let parsed = try_parse(&argv)?; + let execution = execute(&parsed, &NoPicker, literal_detectors)?; + Ok(if parsed.dry_run && execution.pending > 0 { + 1 + } else { + 0 + }) + } + + #[test] + fn cache_input_rewrites_in_place() { + with_cfg(|_| { + seed_cached_document("claude-abc123"); + run_redact(&["-i", "claude-abc123"]).unwrap(); + assert!(read_cached("claude-abc123").contains("[REDACTED:")); + #[cfg(unix)] + assert_eq!( + mode_of(&crate::cache::cache_path("claude-abc123").unwrap()), + 0o600 + ); + }); + } + + #[test] + fn key_is_generated_once_and_reused() { + with_cfg(|_| { + seed_cached_document("claude-abc123"); + run_redact(&["-i", "claude-abc123"]).unwrap(); + let first = fingerprints_in(&read_cached("claude-abc123")); + reseed_same_document("claude-abc123"); + run_redact(&["-i", "claude-abc123"]).unwrap(); + assert_eq!(first, fingerprints_in(&read_cached("claude-abc123"))); + }); + } + + #[test] + fn interactive_uses_the_injected_picker() { + with_cfg(|root| { + let doc = root.join("doc.json"); + std::fs::write(&doc, three_findings_fixture().to_json_pretty().unwrap()).unwrap(); + let picker = RecordingPicker { + selection: vec!["f01".into()], + seen: Default::default(), + }; + let parsed = + try_parse(&["redact", "-i", doc.to_str().unwrap(), "--interactive"]).unwrap(); + let out = execute(&parsed, &picker, literal_detectors).unwrap().plans; + assert_eq!(picker.seen.borrow().len(), 3); + assert_eq!(redacted_ids(&out), vec!["f01".to_string()]); + }); + } + + #[test] + fn dry_run_exits_one_when_findings_exist() { + with_cfg(|root| { + let doc = root.join("doc.json"); + std::fs::write(&doc, three_findings_fixture().to_json_pretty().unwrap()).unwrap(); + assert_eq!( + run_redact(&["-i", doc.to_str().unwrap(), "--dry-run"]).unwrap(), + 1 + ); + }); + } } diff --git a/crates/path-cli/src/sync/engine.rs b/crates/path-cli/src/sync/engine.rs index cdddbaf7..afb72285 100644 --- a/crates/path-cli/src/sync/engine.rs +++ b/crates/path-cli/src/sync/engine.rs @@ -287,6 +287,37 @@ struct Replay { reappeared_skips: usize, } +/// The spec's bundled network-egress detector id +/// (`docs/superpowers/specs/2026-07-30-path-redact-command-design.md`). Real +/// live-credential verification (calling out to the issuing provider to +/// confirm a candidate secret is still valid) was never built. This stub +/// exists so `toolpath_redact::plan::generate_checked`'s `Egress::Network` +/// gate has a real detector to refuse *before* `detect` ever runs — the same +/// path a working network detector would take. That refusal is what +/// `--allow-network-detectors` is for; getting past it lands here, in +/// `detect`, which fails loudly rather than pretending to verify anything. +struct KeyhogStub; + +impl toolpath_redact::Detector for KeyhogStub { + fn id(&self) -> &'static str { + "keyhog" + } + + fn egress(&self) -> toolpath_redact::Egress { + toolpath_redact::Egress::Network + } + + fn detect( + &self, + _c: &toolpath_redact::Candidate<'_>, + ) -> toolpath_redact::Result> { + Err(toolpath_redact::RedactError::DetectorFailed( + "keyhog".to_string(), + "live credential verification was never implemented".to_string(), + )) + } +} + /// The detector registry replay resolves policy names against. /// /// Returning `None` fails the artifact rather than replaying with a @@ -295,11 +326,12 @@ struct Replay { fn detector_named(name: &str) -> Option> { match name { "internal" => Some(Box::new(toolpath_redact::internal::InternalDetector::new())), + "keyhog" => Some(Box::new(KeyhogStub)), _ => None, } } -fn build_detectors(names: &[String]) -> Result { +pub(crate) fn build_detectors(names: &[String]) -> Result { let mut set = DetectorSet::default(); for name in names { let detector = detector_named(name).ok_or_else(|| { @@ -483,8 +515,6 @@ fn replay_config(policy: &RedactionPolicy, key: Vec) -> RedactConfig { /// `cache_id`: a document sync does not track (a file input, a github /// or pathbase import) has nowhere to keep one, and its redaction will /// not survive a re-derive because nothing re-derives it. -// Called by `path p redact`; drop the allow once that dispatch lands. -#[allow(dead_code)] pub(crate) fn record_redaction_policy(cache_id: &str, policy: &RedactionPolicy) -> Result { let mut recorded = false; update_manifest(|manifest| { diff --git a/crates/path-cli/tests/integration.rs b/crates/path-cli/tests/integration.rs index f5c22b36..17e90dc1 100644 --- a/crates/path-cli/tests/integration.rs +++ b/crates/path-cli/tests/integration.rs @@ -1617,3 +1617,838 @@ fn share_no_harness_non_tty_prints_recipe() { .stderr(predicate::str::contains("path import")) .stderr(predicate::str::contains("path export pathbase")); } + +// ── Redact ─────────────────────────────────────────────────────────── +// +// Fixtures keep every `toolpath::v1` map (`change`, `structural`'s +// flattened `extra`) to a single key and omit `path.meta` unless a test +// needs schema validation. Those maps are `HashMap`, whose iteration +// order is randomized per process — a second key would make a +// byte-for-byte comparison across two `path` subprocesses flake for +// reasons that have nothing to do with redaction. + +/// 16 chars from `[A-Z2-7]` after the `AKIA` prefix, matching gitleaks' +/// `aws-access-token` rule exactly — 17+ trailing word chars fail the +/// rule's closing `\b`, and a value ending in `EXAMPLE` is allowlisted. +const HOT_SECRET: &str = "AKIAIOSFODNN7REALKEY"; +const HOT_SECRET_2: &str = "AKIAZXCVBNMASDFGHJKL"; + +/// Scores 1.0 with the internal detector (base 0.6 + the "secret" +/// hotword bonus): clears the default 0.8 threshold with no extra flags. +fn text_with_hotword(secret: &str) -> String { + format!("aws secret key {secret} leaked, rotate now") +} + +/// Scores 0.6 (no hotword nearby): still a finding, but `Skip` by +/// default at the 0.8 threshold, so only an explicit `--accept` redacts it. +fn text_without_hotword(secret: &str) -> String { + format!("exported {secret} into the shell") +} + +fn text_step(id: &str, artifact: &str, text: &str) -> serde_json::Value { + serde_json::json!({ + "step": {"id": id, "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {artifact: {"structural": {"type": "conversation.append", "text": text}}} + }) +} + +/// `role` is required by the `agent-coding-session` kind schema for a +/// `conversation.append` change — needed only by the one test that runs +/// the document through `path p validate`. +fn append_step_with_role(id: &str, artifact: &str, text: &str) -> serde_json::Value { + serde_json::json!({ + "step": {"id": id, "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {artifact: {"structural": {"type": "conversation.append", "role": "user", "text": text}}} + }) +} + +fn redact_graph(doc_id: &str, steps: Vec) -> serde_json::Value { + let head = steps + .last() + .unwrap() + .pointer("/step/id") + .unwrap() + .as_str() + .unwrap() + .to_string(); + serde_json::json!({ + "graph": {"id": doc_id}, + "paths": [{ + "path": {"id": doc_id, "base": {"uri": "file:///tmp/redact-fixture"}, "head": head}, + "steps": steps + }] + }) +} + +fn redact_graph_with_kind(doc_id: &str, steps: Vec) -> serde_json::Value { + let mut v = redact_graph(doc_id, steps); + v["paths"][0]["meta"] = serde_json::json!({ + "kind": "https://toolpath.net/kinds/agent-coding-session/v1.1.0" + }); + v +} + +fn write_json(path: &std::path::Path, v: &serde_json::Value) { + std::fs::write(path, serde_json::to_string_pretty(v).unwrap()).unwrap(); +} + +/// Mirrors what `cache::write_cached` produces, so the in-place tests +/// don't need a real provider import just to get a cache id. +fn seed_cache_doc(cfg: &std::path::Path, id: &str, doc: &serde_json::Value) -> PathBuf { + let dir = cfg.join("documents"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{id}.json")); + write_json(&path, doc); + path +} + +/// `--dry-run --json` prints a JSON *array* (one `Plan` per path in the +/// document, since a `Graph` can hold several); `--plan ` expects a +/// single `Plan` object, not an array. Round-tripping through a file +/// means unwrapping that one element ourselves. Asserts the finding was +/// real (exit 1) so a caller doesn't silently round-trip an empty plan. +/// +/// `key` matters whenever the caller means to `--plan` this back in with an +/// explicit `--key-file` of its own: `verify` recomputes each finding's +/// fingerprint from the *current* key, so a plan generated under one +/// ephemeral key (the no-`--key-file` default, minted fresh per run) and +/// applied under another always reads as "changed since the plan was +/// generated" even though the text never moved. Pass `None` only when the +/// caller's own `--plan` invocation also omits `--key-file`, or when the +/// test never gets that far (e.g. a document-id mismatch). +fn generate_plan_json( + cfg: &std::path::Path, + input: &std::path::Path, + key: Option<&std::path::Path>, +) -> serde_json::Value { + let mut command = cmd(); + command + .env("TOOLPATH_CONFIG_DIR", cfg) + .args(["p", "redact", "-i"]) + .arg(input) + .args(["--dry-run", "--json"]); + if let Some(key) = key { + command.args(["--key-file"]).arg(key); + } + let output = command.output().unwrap(); + assert_eq!( + output.status.code(), + Some(1), + "fixture must carry a redact-actioned finding: {}", + String::from_utf8_lossy(&output.stderr) + ); + let mut plans: Vec = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(plans.len(), 1, "fixture must have exactly one path"); + plans.remove(0) +} + +#[test] +fn redact_dry_run_lists_surfaces_with_zero_findings() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-dry-run-zero", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + + // Plain (non-`--json`) dry-run prints one line per surface, "no + // findings" for one that carried none — the dry-run guarantee that a + // clean field is still reported, not silently dropped. + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .arg("--dry-run") + .assert() + .code(1) + .stdout(predicate::str::contains("no findings")); +} + +#[test] +fn redact_plan_apply_round_trip() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-round-trip", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + let key = cfg.path().join("redact.key"); + std::fs::write(&key, b"integration-test-key").unwrap(); + + // Plan generation must use the same key as the apply below: `verify` + // recomputes each finding's fingerprint from the key it's given, and a + // dry-run with no `--key-file` mints its own ephemeral one (`cmd_redact + // ::resolve_key`) that would never match `key` on replay. + let plan_path = cfg.path().join("plan.json"); + write_json( + &plan_path, + &generate_plan_json(cfg.path(), &input, Some(&key)), + ); + + let via_plan = cfg.path().join("via-plan.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--plan"]) + .arg(&plan_path) + .args(["--output"]) + .arg(&via_plan) + .args(["--key-file"]) + .arg(&key) + .assert() + .success(); + + let single_shot = cfg.path().join("single-shot.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--output"]) + .arg(&single_shot) + .args(["--key-file"]) + .arg(&key) + .assert() + .success(); + + // Both runs redact for the first time, so each stamps its own + // wall-clock `redaction.at` (`apply::write_rollup`'s `cfg.now`, a real + // `Utc::now()` truncated to seconds) — comparing raw bytes would flake + // if the two back-to-back subprocesses straddled a second boundary, so + // that one field is elided before comparing structurally. Nothing else + // about the two runs is allowed to differ. + assert_eq!( + elide_redaction_timestamps(&std::fs::read_to_string(&via_plan).unwrap()), + elide_redaction_timestamps(&std::fs::read_to_string(&single_shot).unwrap()), + "applying a saved plan must match a single-shot run (modulo the wall-clock redaction timestamp)" + ); +} + +/// Blanks every path's `meta.redaction.at` so two independently-timestamped +/// redaction runs can still be compared for everything else. See +/// `redact_plan_apply_round_trip`. +fn elide_redaction_timestamps(json: &str) -> serde_json::Value { + let mut v: serde_json::Value = serde_json::from_str(json).unwrap(); + if let Some(paths) = v.get_mut("paths").and_then(|p| p.as_array_mut()) { + for p in paths { + if let Some(at) = p.pointer_mut("/meta/redaction/at") { + *at = serde_json::Value::Null; + } + } + } + v +} + +#[test] +fn redact_plan_refuses_mismatched_document() { + let cfg = tempfile::tempdir().unwrap(); + let doc_a = redact_graph( + "path-alpha-doc", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let doc_b = redact_graph( + "path-bravo-doc", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET_2), + )], + ); + let input_a = cfg.path().join("a.json"); + let input_b = cfg.path().join("b.json"); + write_json(&input_a, &doc_a); + write_json(&input_b, &doc_b); + + // No `--key-file` on either side: the mismatch this test checks + // (document id) is caught before `verify` ever reaches a fingerprint + // comparison, so which key minted the plan is irrelevant here. + let plan_path = cfg.path().join("plan.json"); + write_json(&plan_path, &generate_plan_json(cfg.path(), &input_a, None)); + + // The document `plan.document` names ("path-alpha-doc") is not among + // `b.json`'s paths, so it's the first (and only) divergence a plan + // generated against `a.json` can have when applied to `b.json`. + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input_b) + .args(["--plan"]) + .arg(&plan_path) + .assert() + .failure() + .stderr(predicate::str::contains("path-alpha-doc")); +} + +/// The authority on a question Task 10's sync replay had to answer without +/// the CLI to check against: `policy_decisions` (`sync/engine.rs`) orders +/// `accept` before `reject`, so a `--reject` naming one step survives a +/// broader `--accept` naming the rule. If this disagrees, that ordering — +/// not this test — is what has to change. +#[test] +fn redact_accept_reject_precedence() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-precedence", + vec![ + text_step( + "turn-a", + "claude-code://sess-1", + &text_without_hotword(HOT_SECRET), + ), + text_step( + "turn-b", + "claude-code://sess-1", + &text_without_hotword(HOT_SECRET_2), + ), + ], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + let output = cfg.path().join("out.json"); + + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--accept", "rule=aws-access-token"]) + .args(["--reject", "step=turn-b"]) + .args(["--output"]) + .arg(&output) + .assert() + .success(); + + let out = std::fs::read_to_string(&output).unwrap(); + assert!( + !out.contains(HOT_SECRET), + "the broadly-accepted finding in turn-a must be redacted: {out}" + ); + assert!( + out.contains(HOT_SECRET_2), + "an explicit --reject must survive a broader --accept (turn-b): {out}" + ); +} + +#[test] +fn redact_in_place_rewrites_cache_entry() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-in-place", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let id = "claude-in-place-fixture"; + let path = seed_cache_doc(cfg.path(), id, &doc); + + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i", id]) + .assert() + .success(); + + let rewritten = std::fs::read_to_string(&path).unwrap(); + assert!(rewritten.contains("[REDACTED:"), "{rewritten}"); + assert_eq!( + std::fs::read_dir(cfg.path().join("documents")) + .unwrap() + .count(), + 1, + "in-place redaction must not create a second file" + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } +} + +#[test] +fn redact_refuses_network_detector_without_flag() { + let cfg = tempfile::tempdir().unwrap(); + // `keyhog` is the spec's bundled network-egress detector id + // (`docs/superpowers/specs/2026-07-30-path-redact-command-design.md`). + // `sync::engine::detector_named` maps it to a stub classified + // `Egress::Network` whose own `detect` fails ("not implemented" — live + // credential verification against the issuing provider was never + // built). That's enough for `toolpath_redact::plan::generate_checked`'s + // egress gate to have a real detector to refuse ahead of `--allow- + // network-detectors`, without this test depending on that verification + // existing. + let doc = redact_graph( + "path-network-refusal", + vec![text_step( + "turn-1", + "claude-code://sess-1", + "nothing sensitive in this turn", + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + let output = cfg.path().join("out.json"); + + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--detector", "keyhog", "--output"]) + .arg(&output) + .assert() + .failure() + .stderr(predicate::str::contains("--allow-network-detectors")); + assert!( + !output.exists(), + "a refused run must not write anything, even a placeholder" + ); + + // Past the gate, the stub's own failure surfaces instead — proof the + // assertion above was the egress gate refusing, not some unrelated + // error that happened to also fail. + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args([ + "--detector", + "keyhog", + "--allow-network-detectors", + "--output", + ]) + .arg(&output) + .assert() + .failure() + .stderr(predicate::str::contains( + "live credential verification was never implemented", + )); + assert!(!output.exists()); +} + +#[test] +fn redact_output_still_validates() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph_with_kind( + "path-validates", + vec![append_step_with_role( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + let output = cfg.path().join("redacted.json"); + + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--output"]) + .arg(&output) + .assert() + .success(); + + let redacted = std::fs::read_to_string(&output).unwrap(); + assert!( + redacted.contains("[REDACTED:"), + "fixture secret must actually be redacted: {redacted}" + ); + + // The only real JSON-Schema check anywhere in the redact implementation: + // T7's own `output_validates_against_both_schemas` could only assert + // structural subsets (no `jsonschema` dev-dependency, no ownership of + // `Cargo.toml`). `p validate` runs both the base schema and, because + // `meta.kind` is the agent-coding-session v1.1.0 URI, that kind's + // schema on top of it (see `schema::validate`). + cmd() + .args(["p", "validate", "--input"]) + .arg(&output) + .assert() + .success() + .stdout(predicate::str::contains("Valid")); +} + +#[test] +fn redact_no_findings_is_byte_identical() { + let cfg = tempfile::tempdir().unwrap(); + let raw = redact_graph( + "path-clean", + vec![text_step( + "turn-1", + "claude-code://sess-1", + "nothing sensitive in this turn at all", + )], + ); + let raw_path = cfg.path().join("raw.json"); + write_json(&raw_path, &raw); + + // Compare against `path`'s own canonical serialization, not this + // test's hand-written JSON: a hand-written fixture's key order need + // not match the struct field order the tool always emits, so its + // *first* pass through any command that re-serializes it would + // "change" for reasons unrelated to redaction. + let canonical = cfg.path().join("canonical.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&raw_path) + .args(["--output"]) + .arg(&canonical) + .assert() + .success(); + + let out = cfg.path().join("out.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&canonical) + .args(["--output"]) + .arg(&out) + .assert() + .success(); + + assert_eq!( + std::fs::read_to_string(&canonical).unwrap(), + std::fs::read_to_string(&out).unwrap(), + "a document with nothing to redact must come out byte-identical" + ); +} + +#[test] +fn redact_twice_is_byte_identical_to_once() { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-idempotent", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + + let once = cfg.path().join("once.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--output"]) + .arg(&once) + .assert() + .success(); + + let twice = cfg.path().join("twice.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&once) + .args(["--output"]) + .arg(&twice) + .assert() + .success(); + + assert_eq!( + std::fs::read_to_string(&once).unwrap(), + std::fs::read_to_string(&twice).unwrap(), + "redacting an already-redacted document must be a no-op" + ); +} + +/// Two independent paths in one `Graph`, each with its own finding — the +/// per-path plan-generation loop in `cmd_redact::execute` must touch every +/// path, not just `doc.paths[0]`. +fn multi_path_graph_fixture() -> serde_json::Value { + serde_json::json!({ + "graph": {"id": "corpus-multi-path"}, + "paths": [ + { + "path": {"id": "corpus-multi-path-a", "head": "a-1"}, + "steps": [text_step("a-1", "claude-code://sess-a", &text_with_hotword(HOT_SECRET))] + }, + { + "path": {"id": "corpus-multi-path-b", "head": "b-1"}, + "steps": [text_step("b-1", "claude-code://sess-b", &text_with_hotword(HOT_SECRET_2))] + } + ] + }) +} + +/// Fixture documents covering shapes that have actually broken this +/// implementation: nested `tool_uses`/`delegations` JSON the detector walks +/// generically, an artifact key that is itself a candidate (writing it back +/// renames the map entry, not just a string field), an unrecognized +/// structural type (the blind-walk fallback), and byte-vs-char-boundary +/// traps (multibyte text, zero steps). Each is redacted independently; the +/// point is coverage per shape, not one shared document. +fn redact_corpus() -> Vec<(&'static str, serde_json::Value)> { + vec![ + ( + "text_and_thinking", + redact_graph( + "corpus-text-thinking", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"claude-code://sess-1": {"structural": { + "type": "conversation.append", + "text": text_with_hotword(HOT_SECRET), + "thinking": text_with_hotword(HOT_SECRET_2) + }}} + })], + ), + ), + ( + "tool_uses_nested_input_and_result", + redact_graph( + "corpus-tool-uses", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"claude-code://sess-1": {"structural": { + "type": "conversation.append", + "text": "ran a shell command", + "tool_uses": [{ + "id": "toolu_1", + "name": "exec", + "input": {"cmd": "printenv", "env": {"nested": {"deep": text_with_hotword(HOT_SECRET)}}}, + "result": {"content": text_with_hotword(HOT_SECRET_2), "is_error": false} + }] + }}} + })], + ), + ), + ( + "delegation_turns", + redact_graph( + "corpus-delegation", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"claude-code://sess-1": {"structural": { + "type": "conversation.append", + "text": "delegating the audit", + "delegations": [{ + "agent_id": "sub-1", + "prompt": "audit the deploy script", + "turns": [{"role": "assistant", "text": text_with_hotword(HOT_SECRET)}] + }] + }}} + })], + ), + ), + ( + "file_write_before_after_edits", + redact_graph( + "corpus-file-write", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"file:///repo/config.env": {"structural": { + "type": "file.write", + "before": "EMPTY=1\n", + "after": text_with_hotword(HOT_SECRET), + "edits": [{"old": "EMPTY=1", "new": text_with_hotword(HOT_SECRET_2)}] + }}} + })], + ), + ), + ( + "unknown_change_type_blind_walk", + redact_graph( + "corpus-unknown-type", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"custom://widget-1": {"structural": { + "type": "widget.frobnicate", + "payload": {"nested": {"deep": text_with_hotword(HOT_SECRET)}} + }}} + })], + ), + ), + ( + "credential_in_artifact_key", + redact_graph( + "corpus-key-secret", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": { + format!("file:///tmp/~secret-{HOT_SECRET}-token/config"): {"structural": { + "type": "conversation.append", + "text": "benign body; the secret lives in the artifact key above" + }} + } + })], + ), + ), + ("multi_path_graph", multi_path_graph_fixture()), + ( + "zero_steps", + serde_json::json!({ + "graph": {"id": "corpus-zero-steps"}, + "paths": [{ + "path": {"id": "corpus-zero-steps", "head": "none"}, + "steps": [] + }] + }), + ), + ( + "multibyte_around_secret", + redact_graph( + "corpus-multibyte", + vec![serde_json::json!({ + "step": {"id": "turn-1", "actor": "human:t", "timestamp": "2026-01-01T00:00:00Z"}, + "change": {"claude-code://sess-1": {"structural": { + "type": "conversation.append", + "text": format!("秘密の鍵が漏洩 secret {HOT_SECRET} 危険につきローテート🔥🔒") + }}} + })], + ), + ), + ] +} + +/// A regression net over document shapes that have broken this +/// implementation, seeded in a sandbox rather than read from whatever the +/// developer's machine happens to have cached — a real corpus, reproducible +/// anywhere. +#[test] +fn redact_corpus_smoke() { + let cfg = tempfile::tempdir().unwrap(); + for (name, doc) in redact_corpus() { + let input = cfg.path().join(format!("{name}.json")); + write_json(&input, &doc); + let output = cfg.path().join(format!("{name}-out.json")); + + let assert = cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--output"]) + .arg(&output) + .assert(); + let out = assert.get_output(); + let stderr = String::from_utf8_lossy(&out.stderr).to_string(); + assert!(!stderr.contains("panicked at"), "{name} panicked: {stderr}"); + assert!( + out.status.success(), + "{name} exited {:?}: {stderr}", + out.status.code() + ); + + cmd() + .args(["p", "validate", "--input"]) + .arg(&output) + .assert() + .success(); + + // Whichever hotword-boosted secret this fixture actually carries + // must be gone — the two checks above alone would also pass a + // shape that silently found nothing. + let redacted = std::fs::read_to_string(&output).unwrap(); + let source = doc.to_string(); + for secret in [HOT_SECRET, HOT_SECRET_2] { + if source.contains(secret) { + assert!( + !redacted.contains(secret), + "{name}: {secret} survived redaction: {redacted}" + ); + } + } + } +} + +/// `redact_twice_is_byte_identical_to_once` only ever exercises the default +/// `--mode marker`. Idempotence bugs two separate reviews found were in the +/// other four transforms, so a test pinned to one mode can't catch a +/// regression in those. +/// +/// Encodes `(mode, stable)` per `toolpath_redact::apply::tests:: +/// idempotent_across_all_transforms` rather than asserting all five are +/// stable, but the table differs from that library-level test: this +/// fixture's finding is the `aws-access-token` rule (literal `AKIA` prefix, +/// fixed length), and against that specific shape every transform's output +/// — including `Hash`'s bare 6-hex, which the library test's own +/// (differently-shaped) fixture found *not* idempotent — is currently too +/// short or too specific to re-match on a second pass. This is a CLI-level +/// observation of the code as it stands, not a claim that `Hash`/`Partial` +/// are structurally recognised the way `internal::marker_re` recognises +/// `Marker`'s `[REDACTED:…]` and the block-char/head…tail shapes: it's +/// incidental to this rule, and a broadened ruleset (e.g. a generic +/// high-entropy rule catching short hex runs) could flip it. If a mode +/// stops matching the table below, that's real signal — flip its entry, +/// don't loosen the assertion. +#[test] +fn redact_idempotence_across_every_mode() { + for (mode, stable) in [ + ("marker", true), + ("remove", true), + ("hash", true), + ("mask", true), + ("partial", true), + ] { + let cfg = tempfile::tempdir().unwrap(); + let doc = redact_graph( + "path-idempotent-mode", + vec![text_step( + "turn-1", + "claude-code://sess-1", + &text_with_hotword(HOT_SECRET), + )], + ); + let input = cfg.path().join("doc.json"); + write_json(&input, &doc); + // Fixed rather than the usual per-run ephemeral key: `Hash`'s output + // is the fingerprint itself, so a random key would make this test's + // own observation of "did the second pass touch anything" depend on + // which key it happened to draw. + let key = cfg.path().join("redact.key"); + std::fs::write(&key, b"idempotence-mode-probe-key").unwrap(); + + let once = cfg.path().join("once.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&input) + .args(["--mode", mode, "--key-file"]) + .arg(&key) + .args(["--output"]) + .arg(&once) + .assert() + .success(); + + let twice = cfg.path().join("twice.json"); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "redact", "-i"]) + .arg(&once) + .args(["--mode", mode, "--key-file"]) + .arg(&key) + .args(["--output"]) + .arg(&twice) + .assert() + .success(); + + let (a, b) = ( + std::fs::read_to_string(&once).unwrap(), + std::fs::read_to_string(&twice).unwrap(), + ); + if stable { + assert_eq!(a, b, "--mode {mode} is not idempotent"); + } else { + assert_ne!(a, b, "--mode {mode} became stable: update this test"); + } + } +} diff --git a/crates/toolpath-redact/src/apply.rs b/crates/toolpath-redact/src/apply.rs index 40923954..35f699f9 100644 --- a/crates/toolpath-redact/src/apply.rs +++ b/crates/toolpath-redact/src/apply.rs @@ -1,6 +1,6 @@ //! Rewriting a document from an approved plan. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use serde_json::{Value, json}; @@ -26,7 +26,14 @@ pub fn apply( plan: &Plan, cfg: &RedactConfig, ) -> Result { - crate::plan::verify(plan, path)?; + // An unkeyed `fp` is a truncated SHA-256 of the credential, which a + // dictionary attack reverses (EDPB 01/2025 para 88). Refused before any + // write, so a caller that forgot the key still holds its document. + if cfg.key.is_empty() { + return Err(RedactError::EmptyKey); + } + + crate::plan::verify(plan, path, &cfg.key)?; let mut report = RedactReport { surfaces_scanned: plan.surfaces.len(), @@ -43,19 +50,45 @@ pub fn apply( return Ok(report); } - report.signatures_dropped = guard_signatures(path, cfg)?; + let touched = touched_steps(plan); + report.steps_touched = touched.len(); + + // Every write below lands in a scratch clone, and the caller's document + // is replaced only once the whole plan has applied: an error part-way + // through must not leave a half-redacted document behind. + let mut work = path.clone(); + report.signatures_dropped = guard_signatures(&mut work, &touched, cfg)?; let mut records: BTreeMap> = BTreeMap::new(); - let mut touched: BTreeSet<&str> = BTreeSet::new(); + let mut renames: Renames = HashMap::new(); { - let mut cursor = crate::surface::SurfaceCursor { path: &mut *path }; + let mut cursor = crate::surface::SurfaceCursor { path: &mut work }; for ((step, at), group) in group_by_field(plan) { let Some(text) = cursor.read(step, at) else { - return Err(RedactError::BadPointer(format!("{step}{at}"))); + return Err(RedactError::BadPointer(format!("{at} on step {step:?}"))); }; + // Both span guards are preconditions on the whole group, so they + // run before the first edit is built: `apply_spans_desc` silently + // drops a span an earlier splice invalidated, and by then the + // report has already counted a replacement that will not happen. + for f in &group { + // A zero-width span splices a marker into text that holds no + // secret, and an inverted one would panic the slice below; a + // plan is hand-editable, so neither can be assumed away. This + // also makes the spans orderable for the overlap check. + if f.span.0 >= f.span.1 { + return Err(RedactError::PlanMismatch(format!( + "{}: span {}..{} is empty or inverted", + f.id, f.span.0, f.span.1 + ))); + } + } + reject_overlaps(&group)?; + let mut edits = Vec::with_capacity(group.len()); - for f in group { + let mut entries = Vec::with_capacity(group.len()); + for f in &group { // A span landing mid-codepoint would panic the slice, so the // bounds check has to happen here and not only in `verify`. let value = text.get(f.span.0..f.span.1).ok_or_else(|| { @@ -64,58 +97,177 @@ pub fn apply( let fp = Fingerprint::new(&cfg.key, value); let op = resolve_transform(cfg, &f.rule, f.transform); edits.push((f.span.0..f.span.1, apply_transform(op, &f.rule, value, &fp))); + entries.push((f.rule.clone(), fp.0, op_name(op))); *report.replaced.entry(f.rule.clone()).or_default() += 1; - *records - .entry(step.to_string()) - .or_default() + } + + let new_text = apply_spans_desc(&text, &mut edits); + // An artifact key is the identity of the thing that changed, and + // `surfaces` skips empty strings: emptying it would leave a key + // no later pass could address, named by a `/change/` pointer that + // routes nowhere. + if new_text.is_empty() && artifact_key_of(at).is_some_and(|(_, is_key)| is_key) { + return Err(RedactError::PlanMismatch(format!( + "{at}: redacting the whole artifact key would leave it empty" + ))); + } + + let step_records = records.entry(step.to_string()).or_default(); + for (rule, fp, op) in entries { + *step_records .entry(RecordKey { at: at.to_string(), - rule: f.rule.clone(), - fp: fp.0, - op: op_name(op), + rule, + fp, + op, }) .or_default() += 1; } - - cursor.write(step, at, &apply_spans_desc(&text, &mut edits))?; - touched.insert(step); + if let Some((token, true)) = artifact_key_of(at) { + renames + .entry(step.to_string()) + .or_default() + .insert(token.to_string(), crate::surface::ptr_escape(&new_text)); + } + cursor.write(step, at, &new_text)?; } } - // Surfaces outside any step (`path.base`, `meta.vcs_remote`) carry the - // empty step id and have no step to be recorded on. - touched.remove(""); - report.steps_touched = touched.len(); - - write_step_records(path, &records)?; - write_rollup(path, plan, cfg, &report)?; + // Surfaces outside any step carry the empty step id and belong to no step + // record; they are published on the rollup instead. + let path_level = records.remove("").unwrap_or_default(); + write_step_records(&mut work, &records, &renames)?; + write_rollup(&mut work, plan, cfg, &report, &path_level)?; + *path = work; Ok(report) } +/// Artifact keys this pass rewrote: step id -> escaped old key -> escaped new +/// key. A key group is written after everything beneath it, so the pointers +/// already recorded under the old key can only be re-pointed once the loop +/// has finished. +type Renames = HashMap>; + +/// `verify` bounds-checks each span on its own, so two spans that each land +/// inside the field can still overlap. Splicing them right-to-left leaves +/// part of the first credential in place while the report claims both were +/// replaced, which is the same class of hand-edited-plan defect as an empty +/// or inverted span and is refused in the same place. +fn reject_overlaps(group: &[&PlanFinding]) -> Result<()> { + let mut ordered = group.to_vec(); + ordered.sort_by_key(|f| f.span); + for pair in ordered.windows(2) { + if pair[0].span.1 > pair[1].span.0 { + return Err(RedactError::PlanMismatch(format!( + "{} and {} overlap", + pair[0].id, pair[1].id + ))); + } + } + Ok(()) +} + +/// The steps the plan rewrites. Surfaces outside any step (`path.base`, +/// `meta.vcs_remote`) carry the empty step id and belong to no step. +fn touched_steps(plan: &Plan) -> BTreeSet<&str> { + plan.findings + .iter() + .filter(|f| f.action == Action::Redact && !f.step.is_empty()) + .map(|f| f.step.as_str()) + .collect() +} + /// Every edit to one string has to be spliced in a single right-to-left /// pass, so the findings are grouped by the field they land in; applying /// them one field-visit at a time would invalidate the offsets of the edits /// still pending. -fn group_by_field(plan: &Plan) -> BTreeMap<(&str, &str), Vec<&PlanFinding>> { - let mut out: BTreeMap<(&str, &str), Vec<&PlanFinding>> = BTreeMap::new(); +/// +/// Groups come back with each artifact's own key last within that artifact, +/// because writing the key renames the map entry and invalidates every +/// pointer built from the old one. That constraint is a structural fact +/// about the pointers themselves, so it is read off them rather than off +/// `plan.surfaces`, which a hand-edited or stale plan need not agree with. +fn group_by_field(plan: &Plan) -> Vec<((&str, &str), Vec<&PlanFinding>)> { + let mut groups: BTreeMap<(&str, &str), Vec<&PlanFinding>> = BTreeMap::new(); for f in plan.findings.iter().filter(|f| f.action == Action::Redact) { - out.entry((f.step.as_str(), f.at.as_str())) + groups + .entry((f.step.as_str(), f.at.as_str())) .or_default() .push(f); } + + let mut out: Vec<((&str, &str), Vec<&PlanFinding>)> = groups.into_iter().collect(); + out.sort_by(|(a, _), (b, _)| write_order(a).cmp(&write_order(b))); out } -/// Strip every signature in the document, returning how many there were. +/// A field's place in the write order: its step, the artifact it sits under, +/// and last within that artifact whether it *is* that artifact's key. +fn write_order<'a>(field: &(&'a str, &'a str)) -> (&'a str, Option<&'a str>, bool) { + let (step, at) = *field; + match artifact_key_of(at) { + Some((token, is_key)) => (step, Some(token), is_key), + None => (step, None, false), + } +} + +/// The escaped artifact key a pointer is built from - the token right after +/// `/change/` - paired with whether the pointer *is* that key rather than +/// something beneath it. `None` for the document-level surfaces, which sit +/// under no artifact. Mirrors `surface::route`'s `/change/` arm. +fn artifact_key_of(at: &str) -> Option<(&str, bool)> { + let rest = at.strip_prefix("/change/")?; + let token = rest.split('/').next().unwrap_or(rest); + (!token.is_empty()).then_some((token, token.len() == rest.len())) +} + +/// Re-point one recorded pointer at the artifact key this pass renamed it +/// to. Recording the pointer a finding was found at would publish the old +/// key - credential and all - in the record sitting next to the document the +/// credential was just removed from, and dangle besides. +fn renamed(at: &str, renames: &HashMap) -> Option { + let (token, _) = artifact_key_of(at)?; + let fresh = renames.get(token)?; + Some(format!( + "/change/{fresh}{}", + &at["/change/".len() + token.len()..] + )) +} + +/// Apply `renames` to every entry of one step's record. Two entries can only +/// collide here if two keys renamed alike, which `SurfaceCursor::write` +/// already refuses; summing is the same merge the map does everywhere else. +fn rewrite_pointers( + entries: BTreeMap, + renames: &HashMap, +) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for (mut key, n) in entries { + if let Some(at) = renamed(&key.at, renames) { + key.at = at; + } + *out.entry(key).or_default() += n; + } + out +} + +/// Strip the signatures this pass invalidates, returning how many there +/// were. /// -/// A signature covers content the pass is about to change, so leaving one in -/// place would publish a signature that no longer verifies. -fn guard_signatures(path: &mut toolpath::v1::Path, cfg: &RedactConfig) -> Result { +/// The path's own signature covers the whole document, and a touched step's +/// covers content about to be rewritten. A step the plan never names keeps +/// its signature: nothing under it changed, so it still verifies. +fn guard_signatures( + path: &mut toolpath::v1::Path, + touched: &BTreeSet<&str>, + cfg: &RedactConfig, +) -> Result { let total = path.meta.as_ref().map_or(0, |m| m.signatures.len()) + path .steps .iter() + .filter(|s| touched.contains(s.step.id.as_str())) .filter_map(|s| s.meta.as_ref()) .map(|m| m.signatures.len()) .sum::(); @@ -128,7 +280,11 @@ fn guard_signatures(path: &mut toolpath::v1::Path, cfg: &RedactConfig) -> Result if let Some(m) = path.meta.as_mut() { m.signatures.clear(); } - for s in &mut path.steps { + for s in path + .steps + .iter_mut() + .filter(|s| touched.contains(s.step.id.as_str())) + { if let Some(m) = s.meta.as_mut() { m.signatures.clear(); } @@ -159,11 +315,9 @@ fn op_name(t: Transform) -> String { fn write_step_records( path: &mut toolpath::v1::Path, records: &BTreeMap>, + renames: &Renames, ) -> Result<()> { for (id, fresh) in records { - if id.is_empty() { - continue; - } let step = path .steps .iter_mut() @@ -175,6 +329,11 @@ fn write_step_records( for (k, n) in fresh { *merged.entry(k.clone()).or_default() += n; } + // After merging, so an earlier pass's pointers are re-pointed too: + // they were built from the key this pass has just renamed. + if let Some(renames) = renames.get(id) { + merged = rewrite_pointers(merged, renames); + } meta.extra.insert(RECORD_KEY.into(), record_value(&merged)); } Ok(()) @@ -228,18 +387,27 @@ fn check_version(v: &Value) -> Result<()> { } fn record_value(entries: &BTreeMap) -> Value { - let findings: Vec = entries + json!({ "v": RECORD_V, "findings": record_entries(entries) }) +} + +fn record_entries(entries: &BTreeMap) -> Vec { + entries .iter() .map(|(k, n)| json!({ "rule": k.rule, "at": k.at, "n": n, "fp": k.fp, "op": k.op })) - .collect(); - json!({ "v": RECORD_V, "findings": findings }) + .collect() } +/// The document-level rollup: what the whole document has had done to it, +/// across every pass. One accumulation policy throughout - counts sum, name +/// lists union, finding entries merge by identity - with a single exception, +/// `flagged`, which names findings still sitting in the document and is +/// therefore this pass's tally rather than a running total. fn write_rollup( path: &mut toolpath::v1::Path, plan: &Plan, cfg: &RedactConfig, report: &RedactReport, + path_level: &BTreeMap, ) -> Result<()> { let previous = path .meta @@ -262,9 +430,19 @@ fn write_rollup( }) .count(); - let replaced = merge_counts(previous.as_ref(), "replaced", &report.replaced); + let replaced = merge_counts(previous.as_ref(), "replaced", &report.replaced)?; let signatures_dropped = - previous_u64(previous.as_ref(), "signatures_dropped") + report.signatures_dropped as u64; + previous_u64(previous.as_ref(), "signatures_dropped")? + report.signatures_dropped as u64; + let detectors = merge_names(previous.as_ref(), "detectors", &plan.detectors)?; + + // `path.base` and `meta.vcs_remote` sit under no step, so no step record + // can name them - and they are exactly where a URI-embedded credential + // lives. Without this the pass would leave no `at`, `fp` or `op` for + // them at all. + let mut findings = seed_from_existing(previous.as_ref())?; + for (k, n) in path_level { + *findings.entry(k.clone()).or_default() += n; + } let meta = path.meta.get_or_insert_with(Default::default); meta.extra.insert( @@ -273,44 +451,87 @@ fn write_rollup( "v": RECORD_V, "at": cfg.now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), "tool": concat!("toolpath-redact/", env!("CARGO_PKG_VERSION")), - "detectors": plan.detectors, - "mode": cfg.mode, + "detectors": detectors, + // No `mode`: a rollup spanning two passes could only ever name + // one of them, and every finding's own `op` already records + // exactly what it got. "steps_touched": steps_touched, "replaced": replaced, - // `flagged` names findings still sitting in the document, so it is - // this pass's tally and not a running total. "flagged": report.flagged, "signatures_dropped": signatures_dropped, + "findings": record_entries(&findings), }), ); Ok(()) } +/// The three readers below share one rule, the same one `seed_from_existing` +/// follows: a field the previous rollup never carried starts from nothing, +/// but a field it carries in a shape this cannot read is a corrupt record, +/// not a zero. Silently discarding it would make a second pass under-report +/// what the first one did. fn merge_counts( previous: Option<&Value>, field: &str, fresh: &BTreeMap, -) -> BTreeMap { - let mut out: BTreeMap = previous - .and_then(|p| p.get(field)) - .and_then(Value::as_object) - .map(|m| { - m.iter() - .filter_map(|(k, v)| v.as_u64().map(|n| (k.clone(), n))) - .collect() - }) - .unwrap_or_default(); +) -> Result> { + let mut out: BTreeMap = match previous.and_then(|p| p.get(field)) { + None => BTreeMap::new(), + Some(Value::Object(m)) => m + .iter() + .map(|(k, v)| { + v.as_u64().map(|n| (k.clone(), n)).ok_or_else(|| { + RedactError::PlanMismatch(format!( + "redaction record {field}.{k} is not a count" + )) + }) + }) + .collect::>()?, + Some(_) => { + return Err(RedactError::PlanMismatch(format!( + "redaction record {field} is not an object" + ))); + } + }; for (k, n) in fresh { *out.entry(k.clone()).or_default() += *n as u64; } - out + Ok(out) } -fn previous_u64(previous: Option<&Value>, field: &str) -> u64 { - previous - .and_then(|p| p.get(field)) - .and_then(Value::as_u64) - .unwrap_or(0) +/// A union, so the rollup names every detector that has ever run over this +/// document rather than only the last one's. Sorted, because a union has no +/// inherent order to preserve. +fn merge_names(previous: Option<&Value>, field: &str, fresh: &[String]) -> Result> { + let mut out: BTreeSet = match previous.and_then(|p| p.get(field)) { + None => BTreeSet::new(), + Some(Value::Array(a)) => a + .iter() + .map(|v| { + v.as_str().map(str::to_owned).ok_or_else(|| { + RedactError::PlanMismatch(format!( + "redaction record {field} holds a non-string" + )) + }) + }) + .collect::>()?, + Some(_) => { + return Err(RedactError::PlanMismatch(format!( + "redaction record {field} is not an array" + ))); + } + }; + out.extend(fresh.iter().cloned()); + Ok(out.into_iter().collect()) +} + +fn previous_u64(previous: Option<&Value>, field: &str) -> Result { + match previous.and_then(|p| p.get(field)) { + None => Ok(0), + Some(v) => v.as_u64().ok_or_else(|| { + RedactError::PlanMismatch(format!("redaction record {field} is not a count")) + }), + } } #[cfg(test)] @@ -347,20 +568,42 @@ mod tests { // // `plan_for` re-detects against whatever the document currently holds; // hard-coded spans would reduce the idempotence test to "the same span - // was redacted twice". Both rules match a self-delimiting prefixed - // format, which is what keeps a marker, mask, hash or partial output from - // being mistaken for a fresh secret on the second pass. + // was redacted twice". static AWS_RE: LazyLock = LazyLock::new(|| regex::Regex::new(r"AKIA[0-9A-Z]{16}").unwrap()); static GH_RE: LazyLock = LazyLock::new(|| regex::Regex::new(r"ghp_[A-Za-z0-9]{36}").unwrap()); + /// A URI password: delimited by what surrounds it rather than by a prefix + /// of its own, so a transform's output can look like a fresh credential + /// here. The two rules above cannot - no output of `apply` starts `AKIA` + /// or `ghp_` - which is why an idempotence test built only on them cannot + /// fail whatever `apply` does. Group 1 is the secret, as in `internal`. + static URI_RE: LazyLock = + LazyLock::new(|| regex::Regex::new(r"://[^:/@]+:([^@]+)@").unwrap()); + + /// What this crate's own output looks like. Mirrors `internal::marker_re`, + /// and for the same reason: without it a second pass redacts the first + /// pass's marker. `Hash` is deliberately not in it - a bare digest is + /// indistinguishable from a fresh secret (`hash_re_redacts_its_own_output`). + static MARKER_RE: LazyLock = LazyLock::new(|| { + regex::Regex::new(r"\[REDACTED:[^\]\n]*\]|\u{2588}+|\S{1,8}\u{2026}\S{1,8}").unwrap() + }); + fn mini_scan(text: &str) -> Vec<(Range, &'static str)> { + let markers: Vec> = MARKER_RE.find_iter(text).map(|m| m.range()).collect(); let mut out: Vec<(Range, &'static str)> = AWS_RE .find_iter(text) .map(|m| (m.range(), "aws-access-key-id")) .chain(GH_RE.find_iter(text).map(|m| (m.range(), "github-pat"))) + .chain(URI_RE.captures_iter(text).map(|c| { + ( + c.get(1).expect("group 1 always participates").range(), + "uri-credential", + ) + })) + .filter(|(r, _)| !markers.iter().any(|m| r.start < m.end && r.end > m.start)) .collect(); out.sort_by_key(|(r, _)| r.start); out @@ -402,13 +645,23 @@ mod tests { .collect() } - fn finding(s: &Surface, span: Range, rule: &str, action: Action) -> PlanFinding { + /// `text` is the field the span indexes into: `verify` recomputes each + /// fingerprint from the document, so a plan built with a placeholder is + /// refused before `apply` does anything. + fn finding( + s: &Surface, + text: &str, + span: Range, + rule: &str, + action: Action, + ) -> PlanFinding { PlanFinding { id: String::new(), step: s.step.clone(), at: s.at.clone(), rule: rule.into(), span: (span.start, span.end), + fingerprint: crate::transform::Fingerprint::new(&cfg().key, &text[span]).0, score: 0.99, detector: "fixed".into(), shape: s.shape, @@ -424,7 +677,7 @@ mod tests { .flat_map(|(s, text)| { mini_scan(text) .into_iter() - .map(move |(span, rule)| finding(s, span, rule, action)) + .map(move |(span, rule)| finding(s, text, span, rule, action)) }) .collect() } @@ -571,18 +824,56 @@ mod tests { doc(vec![step]) } - fn fixture_signed() -> Path { - let mut p = fixture_with_secret(AWS_KEY); - p.meta.as_mut().unwrap().signatures = vec![Signature { + /// `scope` is one of the base schema's five (`author|reviewer|witness| + /// ci|release`); a made-up scope would make every document this fixture + /// produces invalid for reasons unrelated to redaction. + fn signature() -> Signature { + Signature { signer: "human:alex".into(), key: "ssh:SHA256:abc".into(), - scope: "path".into(), + scope: "author".into(), sig: "base64sig".into(), timestamp: None, - }]; + } + } + + fn fixture_signed() -> Path { + let mut p = fixture_with_secret(AWS_KEY); + p.meta.as_mut().unwrap().signatures = vec![signature()]; p } + fn sign_step(path: &mut Path, id: &str) { + let step = path.steps.iter_mut().find(|s| s.step.id == id).unwrap(); + step.meta.get_or_insert_with(Default::default).signatures = vec![signature()]; + } + + /// The credential sits in the artifact key itself, so redacting it + /// renames the map entry every pointer beneath it is built from. + fn fixture_with_secret_in_the_artifact_key(secret: &str, extra: Value) -> Path { + let mut step = append_step("turn-0f3a", extra); + let change = step.change.remove("claude://sess-abc").unwrap(); + step.change + .insert(format!("claude://sess-{secret}"), change); + doc(vec![step]) + } + + fn fixture_with_secret_in_vcs_remote(secret: &str) -> Path { + let mut d = doc(vec![append_step("turn-0f3a", json!({ "text": "clean" }))]); + d.meta.as_mut().unwrap().extra.insert( + "vcs_remote".into(), + json!(format!("https://{secret}@github.com/o/r.git")), + ); + d + } + + fn fixture_with_uri_credential() -> Path { + doc(vec![append_step( + "turn-0f3a", + json!({ "text": "DB_PASSWORD_URL=postgres://svc:h0rr1bl3pass@db.internal:5432/prod" }), + )]) + } + // ── Inspection helpers ────────────────────────────────────────────── fn has_key_somewhere(path: &Path, key: &str) -> bool { @@ -684,20 +975,31 @@ mod tests { #[test] fn idempotent_across_all_transforms() { + // Driven by `mini_scan`'s URI rule, the one whose secret is delimited + // by its surroundings: a rule keyed off a fixed prefix could never see + // a transform's output as a fresh credential, so a test built on those + // alone cannot fail whatever `apply` does. What keeps these four + // stable is that the scan recognises its own markers - `Hash` leaves + // nothing to recognise and has its own test below. for mode in [ Transform::Marker, Transform::Remove, - Transform::Hash, Transform::Mask, Transform::Partial, ] { let cfg = RedactConfig { mode, ..cfg() }; - let mut once = fixture_with_secrets(); + let mut once = fixture_with_uri_credential(); let plan = plan_for(&once); - apply(&mut once, &plan, &cfg).unwrap(); + let report = apply(&mut once, &plan, &cfg).unwrap(); + assert!( + !report.replaced.is_empty(), + "{mode:?}: nothing was redacted, the test proves nothing" + ); + let mut twice = once.clone(); let plan = plan_for(&twice); apply(&mut twice, &plan, &cfg).unwrap(); + assert_eq!( serde_json::to_string(&once).unwrap(), serde_json::to_string(&twice).unwrap(), @@ -706,6 +1008,43 @@ mod tests { } } + #[test] + fn hash_re_redacts_its_own_output() { + // `Hash` emits a bare digest, which no scan can tell from a fresh + // credential, so the next pass redacts the digest and the record grows + // an entry under a rotated `fp`. A known design gap. The two things + // that must hold either way are asserted rather than the inequality: + // the credential is still gone, and the growth is what marks the gap + // as open. + const PASSWORD: &str = "h0rr1bl3pass"; + let cfg = RedactConfig { + mode: Transform::Hash, + ..cfg() + }; + + let mut doc = fixture_with_uri_credential(); + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg).unwrap(); + let once = entry_count(record_of(&doc, "turn-0f3a")); + + let plan = plan_for(&doc); + apply(&mut doc, &plan, &cfg).unwrap(); + let twice = entry_count(record_of(&doc, "turn-0f3a")); + + assert_no_substring(&serde_json::to_string(&doc).unwrap(), PASSWORD, "hash"); + assert!( + twice > once, + "the digest was not re-redacted: the gap has closed, retire this test ({once} -> {twice})" + ); + } + + fn entry_count(record: &Value) -> usize { + record["findings"] + .as_array() + .expect("a record always carries a findings array") + .len() + } + #[test] fn redacted_diff_still_parses_and_line_counts_hold() { let mut doc = fixture_file_write_with_secret_in_diff(); @@ -737,18 +1076,188 @@ mod tests { #[test] fn audit_record_carries_no_value_substring_or_length() { - let secret = "AKIAIOSFODNN7REALKEY"; - let mut doc = fixture_with_secret(secret); - let plan = plan_for(&doc); - apply(&mut doc, &plan, &cfg()).unwrap(); - let rec = serde_json::to_string(record_of(&doc, "turn-0f3a")).unwrap(); - assert!(!rec.contains(secret)); + const SECRET: &str = "AKIAIOSFODNN7REALKEY"; + + // Three placements, because the record's `at` is a pointer and one of + // these builds that pointer out of the credential itself. + for (label, mut d, on_a_step) in [ + ("prose", fixture_with_secret(SECRET), true), + ( + "artifact key", + fixture_with_secret_in_the_artifact_key(SECRET, json!({ "text": "clean" })), + true, + ), + ( + "vcs_remote", + fixture_with_secret_in_vcs_remote(SECRET), + false, + ), + ] { + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + + let rollup = + serde_json::to_string(&d.meta.as_ref().unwrap().extra[RECORD_KEY]).unwrap(); + assert_no_substring(&rollup, SECRET, label); + if !on_a_step { + continue; + } + let rec = serde_json::to_string(record_of(&d, "turn-0f3a")).unwrap(); + assert_no_substring(&rec, SECRET, label); + // Only the step record: the rollup carries a timestamp, whose year + // shares digits with any two-digit length. + assert!(!rec.contains(&SECRET.len().to_string()), "{label}: {rec}"); + } + } + + fn assert_no_substring(haystack: &str, secret: &str, label: &str) { + assert!(!haystack.contains(secret), "{label}: {haystack}"); for w in 6..secret.len() { for s in secret.as_bytes().windows(w) { - assert!(!rec.contains(std::str::from_utf8(s).unwrap())); + let sub = std::str::from_utf8(s).unwrap(); + assert!(!haystack.contains(sub), "{label} leaked {sub}: {haystack}"); } } - assert!(!rec.contains(&secret.len().to_string())); + } + + #[test] + fn redacting_an_artifact_key_and_a_field_beneath_it_both_land() { + // Writing the key renames the map entry, so every pointer under the + // old key stops resolving: the key has to go last. + let mut d = fixture_with_secret_in_the_artifact_key( + GH_PAT, + json!({ "text": format!("and {AWS_KEY} too") }), + ); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + + let key = d.steps[0].change.keys().next().unwrap(); + assert_eq!( + key, + &format!("claude://sess-{}", marker(GH_PAT, "github-pat")) + ); + assert_eq!( + text_at(&d, "/text"), + format!("and {} too", marker(AWS_KEY, "aws-access-key-id")) + ); + } + + #[test] + fn every_recorded_pointer_resolves_after_the_pass() { + // A record's `at` is built from the artifact key, and this pass + // renames that key. Recorded as found, every pointer beneath it both + // dangles and republishes the credential - in the audit record sitting + // beside the document the credential was just removed from. + let mut d = fixture_with_secret_in_the_artifact_key( + GH_PAT, + json!({ "text": format!("and {AWS_KEY} too") }), + ); + let plan = plan_for(&d); + apply(&mut d, &plan, &cfg()).unwrap(); + + let recorded = recorded_pointers(&d); + assert!( + recorded.len() >= 2, + "the fixture must record both placements: {recorded:?}" + ); + let mut probe = d.clone(); + let cursor = SurfaceCursor { path: &mut probe }; + for (step, at) in &recorded { + assert!( + cursor.read(step, at).is_some(), + "{at:?} on step {step:?} does not resolve after the pass" + ); + } + + let doc = serde_json::to_string(&d).unwrap(); + assert_no_substring(&doc, GH_PAT, "artifact key"); + assert_no_substring(&doc, AWS_KEY, "field beneath the artifact key"); + } + + /// Every `at` this pass recorded, paired with the step it was recorded on + /// - the empty id for the rollup's own, which sit under no step. + fn recorded_pointers(path: &Path) -> Vec<(String, String)> { + fn ats(record: &Value) -> Vec { + record["findings"] + .as_array() + .expect("a record always carries a findings array") + .iter() + .map(|e| { + e["at"] + .as_str() + .expect("a record entry always carries at") + .to_string() + }) + .collect() + } + let mut out = Vec::new(); + for s in &path.steps { + if let Some(rec) = s.meta.as_ref().and_then(|m| m.extra.get(RECORD_KEY)) { + out.extend(ats(rec).into_iter().map(|at| (s.step.id.clone(), at))); + } + } + if let Some(rec) = path.meta.as_ref().and_then(|m| m.extra.get(RECORD_KEY)) { + out.extend(ats(rec).into_iter().map(|at| (String::new(), at))); + } + out + } + + #[test] + fn a_mid_loop_failure_leaves_the_document_untouched() { + let before = fixture_with_secrets(); + let mut after = before.clone(); + let mut plan = plan_for(&before); + // The second step's group is reached only after the first step's has + // already been written. + let late = plan + .findings + .iter_mut() + .find(|f| f.step == "turn-9c21") + .unwrap(); + late.span = (late.span.0, late.span.0); + + assert!(matches!( + apply(&mut after, &plan, &cfg()), + Err(RedactError::PlanMismatch(_)) + )); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + } + + #[test] + fn zero_width_span_is_refused() { + let before = fixture_with_secret(AWS_KEY); + let mut after = before.clone(); + let mut plan = plan_for(&before); + plan.findings[0].span = (plan.findings[0].span.0, plan.findings[0].span.0); + assert!(matches!( + apply(&mut after, &plan, &cfg()), + Err(RedactError::PlanMismatch(_)) + )); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); + } + + #[test] + fn empty_key_is_refused() { + let before = fixture_with_secret(AWS_KEY); + let mut after = before.clone(); + let cfg = RedactConfig { + key: Vec::new(), + ..cfg() + }; + let err = apply(&mut after, &plan_for(&before), &cfg).unwrap_err(); + // The variant, not its wording: the message is for humans and free to + // change, the refusal is the contract. + assert!(matches!(err, RedactError::EmptyKey), "{err:?}"); + assert_eq!( + serde_json::to_string(&before).unwrap(), + serde_json::to_string(&after).unwrap() + ); } #[test] @@ -768,7 +1277,7 @@ mod tests { } #[test] - fn output_validates_against_both_schemas() { + fn record_sits_where_both_schemas_allow_it() { // `jsonschema` is not a dev-dependency of this crate and `Cargo.toml` // belongs to another track, so this asserts the structural rules the // two schemas impose on where the record may sit. T11 runs the real @@ -823,6 +1332,74 @@ mod tests { // ── Boundaries ────────────────────────────────────────────────────── + #[test] + fn an_untouched_steps_signature_survives() { + let mut d = fixture_with_secrets(); + sign_step(&mut d, "turn-9c21"); + // Only the first step's text carries a finding, so the second step's + // content - and its signature - still verify. + let cfg = RedactConfig { + drop_signatures: true, + ..cfg() + }; + let plan = plan_for_step(&d, "turn-0f3a"); + let report = apply(&mut d, &plan, &cfg).unwrap(); + + assert_eq!(report.signatures_dropped, 0); + let kept = &d.steps[1].meta.as_ref().unwrap().signatures; + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].sig, signature().sig); + } + + #[test] + fn a_touched_steps_signature_is_dropped_and_an_untouched_ones_is_not() { + // Both steps signed, one redacted: the signature over rewritten + // content cannot still verify, and the signature over content nobody + // touched still can. Dropping both would destroy a valid attestation; + // dropping neither would leave one that no longer checks out. + let mut d = fixture_with_secrets(); + sign_step(&mut d, "turn-0f3a"); + sign_step(&mut d, "turn-9c21"); + let cfg = RedactConfig { + drop_signatures: true, + ..cfg() + }; + let plan = plan_for_step(&d, "turn-0f3a"); + let report = apply(&mut d, &plan, &cfg).unwrap(); + + assert_eq!(report.signatures_dropped, 1); + assert!( + d.steps[0].meta.as_ref().unwrap().signatures.is_empty(), + "the redacted step's signature no longer covers its content" + ); + assert_eq!(d.steps[1].meta.as_ref().unwrap().signatures.len(), 1); + } + + /// A plan naming only `step`'s findings, leaving every other step + /// untouched by the pass. + fn plan_for_step(path: &Path, step: &str) -> Plan { + plan_with( + path, + surfaces(path), + scan_findings(path, Action::Redact) + .into_iter() + .filter(|f| f.step == step) + .collect(), + ) + } + + #[test] + fn signed_step_without_a_signed_path_is_also_refused() { + let mut d = fixture_with_secret(AWS_KEY); + sign_step(&mut d, "turn-0f3a"); + d.meta.as_mut().unwrap().signatures.clear(); + let plan = plan_for(&d); + assert!(matches!( + apply(&mut d, &plan, &cfg()), + Err(RedactError::SignedDocument) + )); + } + #[test] fn refused_signed_document_is_left_untouched() { let before = fixture_signed(); diff --git a/crates/toolpath-redact/src/detect.rs b/crates/toolpath-redact/src/detect.rs index 42d21563..e9c6293a 100644 --- a/crates/toolpath-redact/src/detect.rs +++ b/crates/toolpath-redact/src/detect.rs @@ -100,8 +100,51 @@ impl DetectorSet { /// resolution is score-blind on length, so a low-scoring container evicts /// a high-scoring finding nested inside it. Threshold BEFORE this runs, /// never after - thresholding the output lets a container that is about - /// to be discarded take the survivor down with it. + /// to be discarded take the survivor down with it. A caller that has a + /// threshold wants `detect_all_partitioned`. pub fn detect_all(&self, c: &Candidate<'_>) -> crate::Result> { + Ok(normalise(c.text, self.raw(c)?)) + } + + /// `detect_all`, split on `threshold`, with overlaps resolved on each + /// side separately. Returns `(above, below)`. + /// + /// This is the ordering constraint on `detect_all` made structural. + /// Resolution is score-blind on length, so a whole-line 0.6 generic-entropy + /// hit would evict the 0.99 AWS key nested inside it, and thresholding + /// afterwards would then discard the container too - publishing the key. + /// Resolving the above-threshold set on its own means nothing the caller + /// is about to discard can contest it. + /// + /// `below` is what survives resolution among the sub-threshold findings, + /// minus anything overlapping an above-threshold winner: it exists so a + /// reviewer can lower the bar after the fact (`--accept score>=0.5`), and + /// a row that lost its bytes to a redaction is not offerable. + /// + /// Each side is sorted by span start, and the union of the two is + /// non-overlapping. + pub fn detect_all_partitioned( + &self, + c: &Candidate<'_>, + threshold: f32, + ) -> crate::Result<(Vec, Vec)> { + let (above, below): (Vec, Vec) = + self.raw(c)?.into_iter().partition(|f| f.score >= threshold); + + let above = normalise(c.text, above); + let below = normalise(c.text, below) + .into_iter() + .filter(|f| { + !above + .iter() + .any(|w| f.span.start < w.span.end && f.span.end > w.span.start) + }) + .collect(); + Ok((above, below)) + } + + /// Every detector's unreconciled output, in registration order. + fn raw(&self, c: &Candidate<'_>) -> crate::Result> { let mut raw = Vec::new(); for d in &self.0 { if !d.prefilter(c.text) { @@ -109,7 +152,7 @@ impl DetectorSet { } raw.extend(d.detect(c)?); } - Ok(normalise(c.text, raw)) + Ok(raw) } } @@ -122,8 +165,10 @@ impl DetectorSet { fn normalise(text: &str, mut findings: Vec) -> Vec { findings.retain(|f| { // A NaN score is not orderable, and a single non-comparable element - // makes the whole sort's outcome depend on input order. - !f.score.is_nan() + // makes the whole sort's outcome depend on input order. An infinite + // one is orderable but not representable in JSON - serde writes it as + // `null`, and the plan file it lands in stops parsing. + f.score.is_finite() && f.span.start < f.span.end && f.span.end <= text.len() && text.is_char_boundary(f.span.start) @@ -364,6 +409,74 @@ mod tests { assert!(detect("abcdefgh", vec![f(0..4, "a", f32::NAN)]).is_empty()); } + #[test] + fn drops_infinite_scores() { + assert!(detect("abcdefgh", vec![f(0..4, "a", f32::INFINITY)]).is_empty()); + assert!(detect("abcdefgh", vec![f(0..4, "a", f32::NEG_INFINITY)]).is_empty()); + } + + fn partition( + text: &str, + findings: Vec, + threshold: f32, + ) -> (Vec, Vec) { + let mut s = DetectorSet::default(); + s.push(Box::new(HostileDetector(findings))); + s.detect_all_partitioned(&cand(text), threshold).unwrap() + } + + #[test] + fn a_sub_threshold_container_does_not_evict_the_finding_nested_in_it() { + // What `detect_all` gets wrong: score-blind on length, the 0.6 + // container wins, and thresholding its output then drops the 0.99 + // finding along with it. + let evicted = detect( + ALPHABET, + vec![f(0..20, "container", 0.6), f(4..9, "key", 0.99)], + ); + assert_eq!(rules(&evicted), vec!["container"]); + + let (above, below) = partition( + ALPHABET, + vec![f(0..20, "container", 0.6), f(4..9, "key", 0.99)], + 0.8, + ); + assert_eq!(rules(&above), vec!["key"]); + assert!( + below.is_empty(), + "the container overlaps a winner: {below:?}" + ); + } + + #[test] + fn a_sub_threshold_finding_clear_of_every_winner_survives() { + let (above, below) = partition( + ALPHABET, + vec![f(0..5, "high", 0.9), f(10..15, "low", 0.2)], + 0.8, + ); + assert_eq!(rules(&above), vec!["high"]); + assert_eq!(rules(&below), vec!["low"]); + } + + #[test] + fn partitioned_output_is_non_overlapping_across_both_sides() { + let (above, below) = partition( + ALPHABET, + vec![ + f(0..5, "a", 0.9), + f(3..8, "b", 0.2), + f(9..12, "c", 0.1), + f(14..20, "d", 0.95), + ], + 0.8, + ); + let mut all: Vec = above.into_iter().chain(below).collect(); + all.sort_by_key(|f| f.span.start); + assert!(all.windows(2).all(|w| w[0].span.end <= w[1].span.start)); + assert_eq!(rules(&all), vec!["a", "c", "d"]); + } + #[test] fn multibyte_span_on_char_boundaries_survives() { // "héllo": h=0, é=1..3, l=3, l=4, o=5. diff --git a/crates/toolpath-redact/src/internal/mod.rs b/crates/toolpath-redact/src/internal/mod.rs index e38d48ec..50a51043 100644 --- a/crates/toolpath-redact/src/internal/mod.rs +++ b/crates/toolpath-redact/src/internal/mod.rs @@ -8,13 +8,21 @@ use crate::FieldShape; use crate::detect::{Candidate, Detector, Finding}; use aho_corasick::{AhoCorasick, MatchKind}; use std::ops::Range; - -/// Tuned only so the fixture corpus in this module's tests lands true -/// positives clearly above, and documented false positives clearly below, -/// the 0.8 plan-default threshold - not derived from a labeled dataset. -const BASE_SCORE: f32 = 0.6; -const HOTWORD_BONUS: f32 = 0.5; -const PENALTY_PER_ENTROPY_BIT: f32 = 0.15; +use std::sync::LazyLock; + +/// A bare rule match clears the 0.8 plan-default threshold on its own; the +/// remaining 0.15 is headroom for a modest entropy shortfall. A hotword is +/// corroboration, not the gate - when it *was* the gate (`BASE_SCORE` 0.6, +/// `HOTWORD_BONUS` 0.5) every credential without one of ten English words +/// beside it scored 0.6 and was silently skipped. +const BASE_SCORE: f32 = 0.85; +/// Large enough that the entropy penalty and the bonus cancel at a +/// shortfall of exactly one bit, and that base + bonus clamps. +const HOTWORD_BONUS: f32 = 0.2; +const PENALTY_PER_ENTROPY_BIT: f32 = 0.2; +/// In characters, not bytes: a byte window shrinks to a third of its +/// nominal size on CJK text, so the same secret detects worse in one +/// language than another. const HOTWORD_WINDOW: usize = 50; const HOTWORDS: &[&str] = &[ @@ -30,48 +38,114 @@ const HOTWORDS: &[&str] = &[ "access_key", ]; -pub struct InternalDetector { +/// The compiled ruleset, built once per process. Compiling the ruleset +/// costs ~1.5 s in release, and `InternalDetector::new()` is called per +/// document by sync replay. Pure: a function of an `include_str!` constant, +/// so no environment, filesystem, clock, or write-once global is involved. +static COMPILED: LazyLock = LazyLock::new(Compiled::build); + +struct Compiled { rules: Vec<(rules::Rule, regex::Regex)>, - prefilter: AhoCorasick, - /// Recognizes this crate's own [`crate::Transform::Marker`] and - /// [`crate::Transform::Mask`] output. Matched regions are blanked - /// before any rule runs them, or a second redaction pass would find - /// the marker text itself as a "secret" and redaction would never + /// Rule indices with no keyword of their own; nothing gates them. + ungated: Vec, + /// Pattern index in `keywords` -> index into `rules`. + keyword_owner: Vec, + keywords: AhoCorasick, + hotwords: AhoCorasick, + global_allow: Vec, + /// Recognises this crate's own [`crate::Transform::Marker`], + /// [`crate::Transform::Mask`] and [`crate::Transform::Partial`] output. + /// A finding overlapping one is dropped, or a second pass would + /// fingerprint the first pass's replacement and redaction would never /// reach a fixed point. + /// + /// [`crate::Transform::Hash`] emits bare 6-hex with no envelope and is + /// **not** recognisable here; anything relying on idempotence must not + /// assume that variant is covered. marker_re: regex::Regex, } -impl InternalDetector { - pub fn new() -> Self { - let rules: Vec<(rules::Rule, regex::Regex)> = rules::load_rules() +impl Compiled { + fn build() -> Self { + let ruleset = rules::load_rules(); + let rules: Vec<(rules::Rule, regex::Regex)> = ruleset + .rules .into_iter() .map(|r| { - let re = regex::Regex::new(&r.regex) + let re = rules::compile(&r.regex) .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); (r, re) }) .collect(); - let keywords: Vec<&str> = rules - .iter() - .flat_map(|(r, _)| r.keywords.iter().map(String::as_str)) - .collect(); - // `LeftmostLongest` per the plan: this automaton only gates whether - // `detect()` runs at all (the trait's `prefilter()`), so which - // keyword "wins" a tie never matters, only whether any hit at all. - let prefilter = AhoCorasick::builder() - .ascii_case_insensitive(true) - .match_kind(MatchKind::LeftmostLongest) - .build(&keywords) - .expect("keyword list is static and derived from the loaded ruleset"); + let mut keywords: Vec<&str> = Vec::new(); + let mut keyword_owner: Vec = Vec::new(); + let mut ungated: Vec = Vec::new(); + for (i, (rule, _)) in rules.iter().enumerate() { + if rule.keywords.is_empty() { + ungated.push(i); + } + for k in &rule.keywords { + keywords.push(k); + keyword_owner.push(i); + } + } + + // `Standard`, not `LeftmostLongest`: the automaton now decides + // *which* rules run, and overlapping keywords must all report - + // under leftmost-longest "apikey" hides the rules keyed on "api". + let build_automaton = |patterns: &[&str]| { + AhoCorasick::builder() + .ascii_case_insensitive(true) + .match_kind(MatchKind::Standard) + .build(patterns) + .expect("keyword list is static and derived from the loaded ruleset") + }; Self { + keywords: build_automaton(&keywords), + hotwords: build_automaton(HOTWORDS), rules, - prefilter, - marker_re: regex::Regex::new(r"\[REDACTED:[^\]\n]*\]|█+") + ungated, + keyword_owner, + global_allow: ruleset.global, + marker_re: regex::Regex::new(r"\[REDACTED:[^\]\n]*\]|\u{2588}+|\S{1,8}\u{2026}\S{1,8}") .expect("literal marker pattern"), } } + + /// Only the rules whose keyword actually appears. Gitleaks' `keywords` + /// gate exists because running every regex over every string leaf costs + /// ~1.2 s per 200 KB of clean text; gating picks 7 rules out of 224 on + /// this repo's own CLAUDE.md and scans 224 KB of it in ~52 ms. + fn candidate_rules(&self, text: &str) -> Vec { + let mut seen = vec![false; self.rules.len()]; + let mut out = self.ungated.clone(); + for &i in &out { + seen[i] = true; + } + for m in self.keywords.find_overlapping_iter(text) { + let rule = self.keyword_owner[m.pattern().as_usize()]; + if !seen[rule] { + seen[rule] = true; + out.push(rule); + } + } + out.sort_unstable(); + out + } +} + +pub struct InternalDetector { + compiled: &'static Compiled, +} + +impl InternalDetector { + pub fn new() -> Self { + Self { + compiled: &COMPILED, + } + } } impl Default for InternalDetector { @@ -80,54 +154,78 @@ impl Default for InternalDetector { } } -fn mask_existing_markers(text: &str, marker_re: ®ex::Regex) -> String { - let mut out = text.to_string(); - for m in marker_re.find_iter(text) { - // One NUL byte per matched byte: byte length is preserved, so - // every later span still indexes correctly into the original text. - out.replace_range(m.range(), &"\0".repeat(m.len())); +/// The capture group holding the credential. +/// +/// Group 1 is the secret in most gitleaks rules, but not all: it is the +/// literal `(login|token)` in `sonar-api-token`, and it does not +/// participate at all in `curl-auth-header` unless the header was Basic +/// auth - where falling back to group 0 would redact the whole command +/// line. Preferring the longest participating group gets both right, and +/// upstream's `secretGroup` (plus this crate's whole-match overrides) wins +/// outright where it is annotated. +fn secret_of<'t>(rule: &rules::Rule, caps: ®ex::Captures<'t>) -> regex::Match<'t> { + if let Some(m) = rule.secret_group.and_then(|g| caps.get(g)) { + return m; } - out + let mut best: Option> = None; + for i in 1..caps.len() { + let Some(m) = caps.get(i) else { continue }; + if best.is_none_or(|b| m.len() > b.len()) { + best = Some(m); + } + } + best.unwrap_or_else(|| caps.get(0).expect("group 0 always participates")) } -/// Gitleaks' `private-key` rule spans from a PEM header through its -/// footer, crossing newlines by design; a unified diff interleaves `+`/`-` -/// markers and unrelated lines between them, so redacting the raw span -/// would eat surrounding diff structure. Clip to the line containing the -/// match's start instead. -fn clip_to_line_if_diff(text: &str, shape: FieldShape, span: Range) -> Range { - if shape != FieldShape::UnifiedDiff { - return span; - } - let line_start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); - // Anchor to the *start*'s line, not the end's: a match already - // spanning several lines (gitleaks' PEM rule) would otherwise have its - // line-end searched for past all of them, clipping nothing. - let line_end = text[span.start..] +fn line_around(text: &str, span: &Range) -> Range { + let start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); + let end = text[span.end..] .find('\n') - .map_or(text.len(), |i| span.start + i); - span.start.max(line_start)..span.end.min(line_end) + .map_or(text.len(), |i| span.end + i); + start..end +} + +/// Gitleaks' `private-key` rule spans a PEM block from header to footer, +/// crossing newlines by design. A unified diff interleaves `+`/`-` markers +/// and unrelated lines between them, so redacting the raw span would eat +/// the diff structure - but clipping to the first line instead leaves the +/// key body itself in the document. Split, and redact each line. +fn split_to_lines(text: &str, shape: FieldShape, span: Range) -> Vec> { + if shape != FieldShape::UnifiedDiff || !text[span.clone()].contains('\n') { + return vec![span]; + } + let mut out = Vec::new(); + let mut at = span.start; + while at < span.end { + let end = text[at..span.end].find('\n').map_or(span.end, |i| at + i); + // A continuation line starts on the hunk's `+`/`-`/` ` marker, + // which is structure rather than content; redacting it detaches the + // line from its hunk. + let start = match text.as_bytes().get(at) { + Some(b'+' | b'-' | b' ') if at > span.start => at + 1, + _ => at, + }; + if start < end { + out.push(start..end); + } + at = end + 1; + } + out } -fn has_hotword_nearby(text: &str, span: &Range) -> bool { - let start = text +fn has_hotword_nearby(compiled: &Compiled, text: &str, span: &Range) -> bool { + let start = text[..span.start] .char_indices() - .map(|(i, _)| i) - .take_while(|&i| i <= span.start.saturating_sub(HOTWORD_WINDOW)) - .last() - .unwrap_or(0); - let end = (span.end + HOTWORD_WINDOW).min(text.len()); - let end = (end..=text.len()) - .find(|&i| text.is_char_boundary(i)) - .unwrap_or(text.len()); - let window = text[start..end].to_ascii_lowercase(); - HOTWORDS.iter().any(|h| window.contains(h)) + .rev() + .nth(HOTWORD_WINDOW - 1) + .map_or(0, |(i, _)| i); + let end = text[span.end..] + .char_indices() + .nth(HOTWORD_WINDOW) + .map_or(text.len(), |(i, _)| span.end + i); + compiled.hotwords.is_match(&text[start..end]) } -/// Base confidence from a rule match, adjusted down when the matched text -/// is lower-entropy than the rule expects (proportional to how far below, -/// so a near-miss and a wildly-off match don't get the same penalty) and -/// up when a hotword sits within [`HOTWORD_WINDOW`] chars, then clamped. fn score(rule: &rules::Rule, matched: &str, has_hotword: bool) -> f32 { let mut s = BASE_SCORE; if let Some(threshold) = rule.entropy { @@ -148,39 +246,55 @@ impl Detector for InternalDetector { } fn prefilter(&self, text: &str) -> bool { - self.prefilter.is_match(text) + // Every vendored rule currently carries a keyword, but a rule with + // none must never be skipped by the keyword automaton it is not in. + !self.compiled.ungated.is_empty() || self.compiled.keywords.is_match(text) } fn detect(&self, c: &Candidate<'_>) -> crate::Result> { - let masked = mask_existing_markers(c.text, &self.marker_re); + let text = c.text; + let markers: Vec> = self + .compiled + .marker_re + .find_iter(text) + .map(|m| m.range()) + .collect(); let mut out = Vec::new(); - for (rule, re) in &self.rules { - for caps in re.captures_iter(&masked) { - // Group 1 is the secret in every gitleaks rule that has - // surrounding context (an assignment operator, a quote); - // group 0 is the whole match for rules with nothing to - // trim (bare tokens like `aws-access-token`). - let m = caps - .get(1) - .or_else(|| caps.get(0)) - .expect("group 0 always exists"); - let raw_span = m.range(); - let raw_matched = &c.text[raw_span.clone()]; - if rule.allow.iter().any(|a| a.is_match(raw_matched)) { + for i in self.compiled.candidate_rules(text) { + let (rule, re) = &self.compiled.rules[i]; + for caps in re.captures_iter(text) { + let raw = secret_of(rule, &caps).range(); + if markers + .iter() + .any(|m| raw.start < m.end && raw.end > m.start) + { continue; } - let span = clip_to_line_if_diff(c.text, c.shape, raw_span); - if span.is_empty() { + let whole = caps.get(0).expect("group 0 always participates").range(); + let secret = &text[raw.clone()]; + let allowed = rule + .allow + .iter() + .chain(&self.compiled.global_allow) + .any(|a| { + a.allows( + secret, + &text[whole.clone()], + &text[line_around(text, &whole)], + ) + }); + if allowed { continue; } - let matched = &c.text[span.clone()]; - let has_hotword = has_hotword_nearby(c.text, &span); - out.push(Finding { - span, - rule: rule.id.clone(), - score: score(rule, matched, has_hotword), - detector: self.id(), - }); + for span in split_to_lines(text, c.shape, raw.clone()) { + let has_hotword = has_hotword_nearby(self.compiled, text, &span); + out.push(Finding { + score: score(rule, &text[span.clone()], has_hotword), + span, + rule: rule.id.clone(), + detector: self.id(), + }); + } } } Ok(out) @@ -192,6 +306,10 @@ mod tests { use super::*; use crate::detect::Context; + /// The CLI's default `--threshold`. Every constant in this module is + /// tuned against it, so it is asserted against directly. + const DEFAULT_THRESHOLD: f32 = 0.8; + fn cand(text: &str, shape: FieldShape) -> Candidate<'_> { Candidate { text, @@ -212,6 +330,18 @@ mod tests { .unwrap() } + /// The span the detector would actually rewrite, for the highest-scoring + /// finding of a named rule. + fn redacted_span<'a>(text: &'a str, rule: &str) -> &'a str { + let findings = detect_one(text); + let f = findings + .iter() + .filter(|f| f.rule == rule) + .max_by(|a, b| a.score.total_cmp(&b.score)) + .unwrap_or_else(|| panic!("{rule} did not fire on {text:?}: {findings:?}")); + &text[f.span.clone()] + } + fn diff_candidate() -> Candidate<'static> { cand( "@@ -1,4 +1,4 @@\n-old line\n+-----BEGIN RSA PRIVATE KEY-----\n+MIIEpAIBAAKCAQEAxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n+-----END RSA PRIVATE KEY-----\n more diff context\n", @@ -223,6 +353,10 @@ mod tests { cand(text, FieldShape::Uri) } + /// Split across `concat!` so the source text does not match the pattern + /// the value is testing. These are synthetic and authenticate nothing, + /// but GitHub push protection scans the file, not the compiled string, + /// and rejects any push whose diff contains a well-formed token. #[test] fn detects_shipped_formats() { for (label, sample) in [ @@ -231,23 +365,105 @@ mod tests { ("jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.QQQQQQQQQQ"), ("pem", "-----BEGIN RSA PRIVATE KEY-----"), ("dburi", "postgres://u:s3cr3tpass@db.internal:5432/prod"), + ("github-pat", "ghp_Zt9xQw3pLm7RvB2kNc5YdA8jHf1UsE0iOqTg"), + ( + "stripe", + concat!("sk_", "live_", "Zt9xQw3pLm7RvB2kNc5YdA8j"), + ), + ( + "slack-bot", + concat!( + "xo", + "xb-", + "901234567890-9012345678901-Zt9xQw3pLm7RvB2kNc5YdA8j" + ), + ), + ("gitlab-pat", concat!("gl", "pat-", "Zt9xQw3pLm7RvB2kNc5Y")), + ( + "anthropic", + concat!("sk-", "ant-", "api03-Zt9xQw3pLm7RvB2kNc5YdA8jHf1UsE0iOqTg"), + ), + ( + "slack-webhook", + concat!( + "https://hooks.sl", + "ack.com/services/T01234567/B01234567/Zt9xQw3pLm7RvB2kNc5YdA8j" + ), + ), + ] { + let findings = detect_one(sample); + assert!(!findings.is_empty(), "missed {label}"); + let best = findings.iter().map(|f| f.score).fold(f32::MIN, f32::max); + assert!( + best >= DEFAULT_THRESHOLD, + "{label} scored {best}, under the {DEFAULT_THRESHOLD} default threshold" + ); + } + } + + /// Verbatim from the `.env` block of a real cache document in which only + /// the URI password was redacted. The other two credentials were invisible + /// to the detector - one to a length-pinned vendored rule, one to + /// gitleaks' documentation-key allowlist - so they are pinned here by + /// literal rather than by shape. + const LEAKED_ENV_BLOCK: &str = concat!( + "ANTHROPIC_API_KEY=sk-ant-api03-EXAMPLEONLYnotarealkey000000000000000000000AA\n", + "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\n", + "DATABASE_URL=postgres://svc_user:h0rr1bl3-p4ss@db.internal:5432/prod", + ); + + #[test] + fn every_credential_in_the_leaked_env_block_is_detected() { + let findings = detect_one(LEAKED_ENV_BLOCK); + for secret in [ + "sk-ant-api03-EXAMPLEONLYnotarealkey000000000000000000000AA", + "AKIAIOSFODNN7EXAMPLE", + "h0rr1bl3-p4ss", ] { - assert!(!detect_one(sample).is_empty(), "missed {label}"); + let at = LEAKED_ENV_BLOCK.find(secret).expect("fixture holds it"); + let best = findings + .iter() + .filter(|f| f.span.start <= at && f.span.end >= at + secret.len()) + .map(|f| f.score) + .fold(f32::MIN, f32::max); + assert!( + best >= DEFAULT_THRESHOLD, + "{secret} is covered at {best}, under the {DEFAULT_THRESHOLD} \ + default threshold: {findings:?}" + ); } } + /// Gitleaks allowlists AWS's own `...EXAMPLE` key because a README + /// quoting it is not a leak. Redaction answers a different question - + /// see `SUPPRESSED_ALLOWLIST_REGEXES` - so the exception is off here and + /// the literal is treated like any other key-shaped string. + #[test] + fn aws_documentation_keys_are_redacted_like_any_other_key() { + let findings = detect_one("AKIAIOSFODNN7EXAMPLE"); + let best = findings.iter().map(|f| f.score).fold(f32::MIN, f32::max); + assert!( + best >= DEFAULT_THRESHOLD, + "AWS's documentation key scored {best}: {findings:?}" + ); + } + #[test] fn documented_false_positives_stay_below_threshold() { + // `AKIAIOSFODNN7EXAMPLE` was once on this list. It is now a + // deliberate true positive; see the test above. for sample in [ - "AKIAIOSFODNN7EXAMPLE", // AWS's own documentation key - "redis://localhost:6379", // no password - "0e2b3d4e3dec5f38ae95f62519eb2736f73c0b", // git SHA - "550e8400-e29b-41d4-a716-446655440000", // UUID - "ThisIsAReallyLongString", // high entropy, not a secret + "redis://localhost:6379", // no password + // A real 40-hex git SHA next to a hotword: the shape + // `sourcegraph-access-token` also accepts. + "reverted at commit token 0e2b3d4e3dec5f38ae95f62519eb2736f73c0b91", + "550e8400-e29b-41d4-a716-446655440000", // UUID + "ThisIsAReallyLongString", // high entropy, not a secret ] { + let findings = detect_one(sample); assert!( - detect_one(sample).iter().all(|f| f.score < 0.8), - "false positive on {sample}" + findings.iter().all(|f| f.score < DEFAULT_THRESHOLD), + "false positive on {sample}: {findings:?}" ); } } @@ -262,6 +478,35 @@ mod tests { } } + #[test] + fn diff_findings_cover_the_private_key_body() { + let c = diff_candidate(); + let findings = InternalDetector::new().detect(&c).unwrap(); + let body = c + .text + .find("MIIEpAIBAAKCAQEA") + .expect("fixture holds the key body"); + let covered = findings + .iter() + .any(|f| f.span.start <= body && f.span.end >= body + "MIIEpAIBAAKCAQEA".len()); + assert!(covered, "the key body is covered by nothing: {findings:?}"); + } + + /// Every line of the fixture diff carries a `+`/`-`/` ` marker, so a + /// finding that starts at a line start is one that would swallow the + /// marker and detach the line from its hunk. + #[test] + fn diff_findings_keep_the_hunk_marker() { + let c = diff_candidate(); + for f in InternalDetector::new().detect(&c).unwrap() { + assert!( + f.span.start > 0 && c.text.as_bytes()[f.span.start - 1] != b'\n', + "redacting {:?} would take its hunk marker with it", + &c.text[f.span.clone()] + ); + } + } + #[test] fn uri_shape_redacts_only_the_password() { let c = uri_candidate("postgres://svc_user:h0rr1bl3@db.internal:5432/prod"); @@ -269,10 +514,115 @@ mod tests { assert_eq!(&c.text[findings[0].span.clone()], "h0rr1bl3"); } + #[test] + fn sonar_token_redacts_the_credential_not_the_keyword() { + let text = "sonar.token=squ_0123456789abcdef0123456789abcdef01234567"; + assert_eq!( + redacted_span(text, "sonar-api-token"), + "squ_0123456789abcdef0123456789abcdef01234567" + ); + } + + #[test] + fn teams_webhook_redacts_the_whole_url() { + let text = "https://acme.webhook.office.com/webhookb2/0123abcd-0123-4567-89ab-0123456789ab@0123abcd-0123-4567-89ab-0123456789ab/IncomingWebhook/0123456789abcdef0123456789abcdef/0123abcd-0123-4567-89ab-0123456789ab"; + assert_eq!(redacted_span(text, "microsoft-teams-webhook"), text); + } + + #[test] + fn jwt_base64_redacts_the_whole_token() { + let text = "ZXlKaGJHY2lPaUpJVXpJMU5pSjkuZXlKemRXSWlPaUl4SW4wLlFRUVFRUVFRUVE"; + assert_eq!(redacted_span(text, "jwt-base64"), text); + } + + #[test] + fn curl_auth_header_redacts_only_the_bearer_token() { + let text = r#"curl -H "Authorization: Bearer Zt9xQw3pLm7RvB2kNc5YdA8j" https://api.example.com/v1/things"#; + assert_eq!( + redacted_span(text, "curl-auth-header"), + "Zt9xQw3pLm7RvB2kNc5YdA8j" + ); + } + + /// A marker blanked in place would delete the `:` separators that were + /// the only reason `uri-credential` did not match the surrounding URI, + /// and the rule id inside the marker carries the hotword "credential" - + /// so the previous pass's output re-detected at 1.000 and redaction + /// never reached a fixed point. #[test] fn existing_markers_are_never_re_detected() { - assert!(detect_one("[REDACTED:aws-access-key-id:a3c829]").is_empty()); - assert!(detect_one("████████████████████").is_empty()); + for replacement in [ + "[REDACTED:uri-credential:e90e4c]", + "\u{2588}\u{2588}\u{2588}\u{2588}\u{2588}\u{2588}\u{2588}\u{2588}", + "h0rr\u{2026}bl3x", + ] { + let text = format!("postgres://svc_user:{replacement}@db.internal:5432/prod"); + let findings = InternalDetector::new() + .detect(&cand(&text, FieldShape::Uri)) + .unwrap(); + assert!( + findings.is_empty(), + "re-detected {replacement}: {findings:?}" + ); + } + let text = r#"curl -u "admin:[REDACTED:curl-basic-auth:a3c829]" https://api.example.com"#; + let findings = detect_one(text); + assert!(findings.is_empty(), "re-detected a marker: {findings:?}"); + } + + /// 30 three-byte chars of padding is 90 bytes: inside a 50-character + /// window, outside a 50-*byte* one. The trailing space matters - `é` is + /// a word character, so without it the rule's leading `\b` never + /// matches and the key is missed for an unrelated reason. + fn padded(chars: usize) -> String { + format!("token {} AKIAIOSFODNN7REALKEY", "é".repeat(chars)) + } + + #[test] + fn a_hotword_thirty_multibyte_chars_away_still_corroborates() { + let text = padded(30); + let findings = detect_one(&text); + assert!( + !findings.is_empty(), + "multibyte padding hid the key entirely" + ); + assert!( + findings.iter().any(|f| f.score >= 1.0), + "hotword 30 chars away did not corroborate: {findings:?}" + ); + } + + #[test] + fn a_hotword_beyond_the_window_does_not_corroborate() { + let text = padded(60); + let findings = detect_one(&text); + assert!(!findings.is_empty()); + assert!(findings.iter().all(|f| f.score < 1.0), "{findings:?}"); + } + + #[test] + fn only_rules_whose_keyword_appears_are_run() { + let compiled = &*COMPILED; + let all = compiled.rules.len(); + let picked = compiled.candidate_rules("AKIAIOSFODNN7REALKEY").len(); + assert!( + picked < all / 10, + "{picked} of {all} rules ran on one AWS key" + ); + assert!(picked > 0); + } + + /// Under `LeftmostLongest` the longer keyword swallows the shorter and + /// every rule keyed on the shorter one stops running. + #[test] + fn overlapping_keywords_activate_every_owning_rule() { + let compiled = &*COMPILED; + let owners: Vec<&str> = compiled + .candidate_rules("apikey") + .into_iter() + .map(|i| compiled.rules[i].0.id.as_str()) + .collect(); + assert!(owners.contains(&"generic-api-key"), "{owners:?}"); } fn neutral_rule(entropy: Option) -> rules::Rule { @@ -281,55 +631,78 @@ mod tests { regex: ".".to_string(), entropy, keywords: vec![], + secret_group: None, allow: vec![], } } - #[test] - fn below_entropy_lowers_score() { - let rule = neutral_rule(Some(3.0)); - let s = score(&rule, "aaaaaaaaaa", false); // shannon == 0.0, well below 3.0 + #[track_caller] + fn assert_score(actual: f32, expected: f32) { assert!( - s < BASE_SCORE && s > 0.0, - "expected a reduced but non-clamped score, got {s}" + (actual - expected).abs() < 1e-6, + "expected {expected}, got {actual}" ); } + /// `shannon` is exactly 1.0 on this: two symbols, equal counts. + const ONE_BIT: &str = "aaaabbbb"; + /// Exactly 2.0: four symbols, equal counts. + const TWO_BITS: &str = "abcdabcd"; + #[test] - fn above_entropy_keeps_base_score() { - let rule = neutral_rule(Some(1.0)); - let s = score(&rule, "abcdefghij", false); // shannon == log2(10), well above 1.0 - assert_eq!(s, BASE_SCORE); + fn penalty_is_proportional_to_shortfall() { + assert_score(score(&neutral_rule(Some(2.0)), ONE_BIT, false), 0.65); + assert_score(score(&neutral_rule(Some(3.0)), ONE_BIT, false), 0.45); } #[test] - fn hotword_present_boosts_score() { - let rule = neutral_rule(Some(2.0)); - let without = score(&rule, "aaaaaaaaaa", false); - let with = score(&rule, "aaaaaaaaaa", true); - assert!( - with > without, - "hotword should raise the score: {with} vs {without}" - ); + fn entropy_exactly_at_threshold_is_not_penalised() { + assert_score(score(&neutral_rule(Some(2.0)), TWO_BITS, false), BASE_SCORE); } #[test] - fn hotword_absent_does_not_boost() { - let rule = neutral_rule(None); - assert_eq!(score(&rule, "anything", false), BASE_SCORE); + fn no_entropy_threshold_skips_the_penalty_entirely() { + assert_score(score(&neutral_rule(None), "aaaaaaaaaa", false), BASE_SCORE); } #[test] - fn clamp_low() { - let rule = neutral_rule(Some(8.0)); // no real string reaches 8 bits of entropy - let s = score(&rule, "aaaaaaaaaa", false); - assert_eq!(s, 0.0); + fn hotword_alone_clears_the_default_threshold() { + let rule = neutral_rule(Some(1.5)); + assert_score(score(&rule, ONE_BIT, false), 0.75); + assert_score(score(&rule, ONE_BIT, true), 0.95); } #[test] - fn clamp_high() { - let rule = neutral_rule(None); - let s = score(&rule, "anything", true); - assert_eq!(s, 1.0); + fn entropy_shortfall_that_cancels_the_hotword_bonus() { + // A one-bit shortfall costs exactly what a hotword pays. + assert_score(score(&neutral_rule(Some(2.0)), ONE_BIT, true), BASE_SCORE); + } + + #[test] + fn hotword_bonus_applies_before_the_clamp() { + // Clamping first would leave 0.0 + HOTWORD_BONUS. + assert_score(score(&neutral_rule(Some(8.0)), "aaaaaaaaaa", true), 0.0); + assert_score(score(&neutral_rule(None), "anything", true), 1.0); + } + + #[test] + fn above_entropy_keeps_base_score() { + assert_score( + score(&neutral_rule(Some(1.0)), "abcdefghij", false), + BASE_SCORE, + ); + } + + #[test] + fn hotword_present_boosts_score() { + let rule = neutral_rule(Some(2.0)); + let without = score(&rule, "aaaaaaaaaa", false); + let with = score(&rule, "aaaaaaaaaa", true); + assert_score(with - without, HOTWORD_BONUS); + } + + #[test] + fn clamp_low() { + assert_score(score(&neutral_rule(Some(8.0)), "aaaaaaaaaa", false), 0.0); } } diff --git a/crates/toolpath-redact/src/internal/rules.rs b/crates/toolpath-redact/src/internal/rules.rs index 300e3877..4d7383bd 100644 --- a/crates/toolpath-redact/src/internal/rules.rs +++ b/crates/toolpath-redact/src/internal/rules.rs @@ -6,29 +6,88 @@ //! `gitleaks.toml` itself is kept byte-verbatim so it can be diffed against //! upstream; do not hand-edit it. +use aho_corasick::AhoCorasick; use serde::Deserialize; const RAW_TOML: &str = include_str!("gitleaks.toml"); -/// One rule, with its allow-regexes already compiled. `regex` stays a -/// `String` (not a compiled `Regex`) because [`load_rules`] is a pure -/// parse step exercised on its own by the compile-guard test; the caller -/// compiles it. +/// `regex`'s default 10 MiB program budget is not enough for three of +/// gitleaks' patterns - `generic-api-key` needs ~10.5 MiB, +/// `vault-batch-token` ~14.3 MiB, `pypi-upload-token` ~47.7 MiB. They are +/// valid RE2, just large; without this ceiling they fail to build and the +/// catch-all `generic-api-key` (the only rule that catches an unbranded +/// `SOMETHING_API_KEY = "..."`) is lost. +const REGEX_SIZE_LIMIT: usize = 64 << 20; + +pub fn compile(pattern: &str) -> Result { + regex::RegexBuilder::new(pattern) + .size_limit(REGEX_SIZE_LIMIT) + .build() +} + +/// Which text a gitleaks allowlist regex is tested against - upstream's +/// `regexTarget`, whose default is the extracted secret. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AllowTarget { + #[default] + Secret, + Match, + Line, +} + +/// One gitleaks allowlist block: documented exceptions that suppress a +/// match (e.g. AWS's own `...EXAMPLE` key, or `${{ secrets.X }}`). +pub struct Allowlist { + target: AllowTarget, + regexes: Vec, + /// Upstream tests stopwords against the secret whatever `regexTarget` + /// says, and case-insensitively as a substring - hence an automaton + /// rather than a regex (`generic-api-key` ships ~2000 of them). + stopwords: Option, +} + +impl Allowlist { + pub fn allows(&self, secret: &str, whole_match: &str, line: &str) -> bool { + let target = match self.target { + AllowTarget::Secret => secret, + AllowTarget::Match => whole_match, + AllowTarget::Line => line, + }; + self.regexes.iter().any(|r| r.is_match(target)) + || self.stopwords.as_ref().is_some_and(|s| s.is_match(secret)) + } +} + +/// One rule, with its allowlists already compiled. `regex` stays a `String` +/// (not a compiled `Regex`) because [`load_rules`] is a pure parse step +/// exercised on its own by the compile-guard test; the caller compiles it. pub struct Rule { pub id: String, pub regex: String, pub entropy: Option, pub keywords: Vec, - /// Gitleaks' per-rule `[[rules.allowlists]] regexes` - documented - /// exceptions (e.g. AWS's own `...EXAMPLE` key) checked against the - /// matched secret text. Compiled best-effort: an allowlist entry that - /// fails under `regex` is dropped rather than failing the whole rule, - /// since it is a false-positive refinement, not the detection itself. - pub allow: Vec, + /// Gitleaks' `secretGroup`: which capture group holds the credential + /// when group 1 is context instead. `sonar-api-token`'s group 1 is the + /// literal `(login|token)`, so without this the word "token" is + /// redacted and `squ_...` survives. `Some(0)` means the whole match. + pub secret_group: Option, + pub allow: Vec, +} + +/// The vendored ruleset plus the file-level allowlist that applies to all +/// of it. +pub struct Ruleset { + pub rules: Vec, + /// Gitleaks' global `[allowlist]`: placeholder shapes (`$VAR`, + /// `${{ secrets.X }}`, `%VAR%`, printf verbs, `true|false|null`) that + /// are never credentials whichever rule matched them. + pub global: Vec, } #[derive(Deserialize)] struct RulesFile { + allowlist: Option, rules: Vec, } @@ -40,6 +99,8 @@ struct RawRule { /// [`load_rules`] drops any rule missing it. regex: Option, entropy: Option, + #[serde(rename = "secretGroup")] + secret_group: Option, #[serde(default)] keywords: Vec, #[serde(default)] @@ -48,34 +109,97 @@ struct RawRule { #[derive(Deserialize)] struct RawAllowlist { + condition: Option, + #[serde(default)] + paths: Vec, + #[serde(rename = "regexTarget", default)] + regex_target: AllowTarget, #[serde(default)] regexes: Vec, + #[serde(default)] + stopwords: Vec, } -/// Rust's `regex` crate (RE2-derived, no backreferences/lookaround) -/// rejects a minority of gitleaks patterns written for Go's RE2 dialect. -/// Confirmed by `every_vendored_rule_compiles_under_rust_regex` - see that -/// test for the failure each id below hit. -pub const EXCLUDED_RULE_IDS: &[(&str, &str)] = &[ - ( - "generic-api-key", - "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", - ), - ( - "pypi-upload-token", - "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", - ), - ( - "vault-batch-token", - "compiled form exceeds regex's 10 MiB size limit under Rust's (non-backtracking) engine", - ), +/// Allowlist *regexes* the vendored ruleset carries that this crate +/// deliberately refuses to honour. +/// +/// Gitleaks scans source trees, where AWS's own documentation key quoted in +/// a README is noise worth silencing. `p redact` scans a transcript the user +/// is about to publish, and the question it answers there is "does this look +/// like a credential", not "is this particular literal live" - a session that +/// pasted a key-shaped string gets it removed either way, because nothing in +/// a transcript distinguishes a copied placeholder from a real key whose +/// owner happened to end it in `EXAMPLE`. False-positive cost is a marker in +/// a document; false-negative cost is a published credential. +/// +/// Scoped to regexes on purpose. The stopword lists stay, because they are +/// the only thing keeping `generic-api-key` from flagging every +/// `name = "value"` pair in the transcript and drowning the plan a human has +/// to review. +const SUPPRESSED_ALLOWLIST_REGEXES: &[&str] = &[ + // `aws-access-token`, covering `AKIAIOSFODNN7EXAMPLE` and friends. + r".+EXAMPLE$", ]; +impl RawAllowlist { + /// Compiled best-effort: an entry that fails under `regex` is dropped + /// rather than failing the whole rule, since an allowlist is a + /// false-positive refinement, not the detection itself. + /// + /// A `paths` criterion is unevaluable here - this detector scans a + /// transcript field, not a file - so it is ignored, which under the + /// default OR condition only ever makes this crate report *more* than + /// upstream. `condition = "AND"` inverts that: every criterion the + /// block sets must hold at once, so honouring the evaluable ones alone + /// would suppress matches upstream reports. Such a block is dropped + /// whole, except where it sets a single criterion and AND therefore + /// means exactly what OR does. + fn compile(self) -> Option { + let criteria = [&self.paths, &self.regexes, &self.stopwords] + .into_iter() + .filter(|c| !c.is_empty()) + .count(); + if self.condition.as_deref() == Some("AND") && criteria > 1 { + return None; + } + let regexes: Vec = self + .regexes + .iter() + .filter(|r| !SUPPRESSED_ALLOWLIST_REGEXES.contains(&r.as_str())) + .filter_map(|r| compile(r).ok()) + .collect(); + let stopwords = if self.stopwords.is_empty() { + None + } else { + AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(&self.stopwords) + .ok() + }; + if regexes.is_empty() && stopwords.is_none() { + return None; + } + Some(Allowlist { + target: self.regex_target, + regexes, + stopwords, + }) + } +} + +/// Rules whose credential is the whole match even though upstream carries +/// no `secretGroup`. `jwt-base64`'s group 1 is the `(?P...)` header +/// discriminator (redacting it eats ten bytes from the middle of the +/// token) and `microsoft-teams-webhook`'s is a repeated `([a-z0-9]{4}-)` +/// chunk (redacting it leaves the whole webhook URL in place). +const WHOLE_MATCH_IS_THE_SECRET: &[&str] = &["jwt-base64", "microsoft-teams-webhook"]; + /// Hand-written rules filling gaps the vendored ruleset leaves open for /// this crate's purposes: gitleaks matches a PEM block only with its full /// closing footer, and matches a JWT only above a claim-length floor that /// misses short tokens; URI-embedded credentials aren't a gitleaks rule at -/// all (it scans files, not structured connection strings). +/// all (it scans files, not structured connection strings); and its +/// Anthropic rules are pinned to a live key's exact length. fn supplemental_rules() -> Vec { vec![ Rule { @@ -83,6 +207,7 @@ fn supplemental_rules() -> Vec { regex: r"-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----".to_string(), entropy: None, keywords: vec!["-----begin".to_string()], + secret_group: None, allow: Vec::new(), }, Rule { @@ -90,6 +215,7 @@ fn supplemental_rules() -> Vec { regex: r"\bey[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{2,}\b".to_string(), entropy: None, keywords: vec!["ey".to_string()], + secret_group: None, allow: Vec::new(), }, Rule { @@ -97,54 +223,285 @@ fn supplemental_rules() -> Vec { regex: r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s:@/]+:([^\s:@/]+)@[^\s/]+".to_string(), entropy: None, keywords: vec!["://".to_string()], + secret_group: None, + allow: Vec::new(), + }, + // Vendored `anthropic-api-key`/`anthropic-admin-api-key` demand + // exactly 93 body characters and a trailing `AA` - correct for a + // live key, and blind to every abbreviated or elided one a + // transcript actually carries. The only other rule that reaches + // such a key is `generic-api-key`, whose stopword list contains + // `ant-`, so *every* `sk-ant-` key is suppressed there by the + // prefix alone. Match on the shape instead of the length: the + // `sk-ant-` prefix is specific enough that a false positive costs + // one marker. + Rule { + id: "anthropic-api-key-loose".to_string(), + regex: r"\bsk-ant-[A-Za-z0-9]+-[A-Za-z0-9_-]{16,}\b".to_string(), + entropy: None, + keywords: vec!["sk-ant-".to_string()], + secret_group: None, allow: Vec::new(), }, ] } -/// Parses the vendored TOML plus [`supplemental_rules`], dropping any rule -/// on [`EXCLUDED_RULE_IDS`]. Pure and deterministic - no I/O beyond the -/// `include_str!` baked in at compile time. -pub fn load_rules() -> Vec { +/// Extra allowlist entries this crate adds to a vendored rule, because the +/// corpus it scans (agent transcripts) contains a lookalike that gitleaks' +/// corpus (source trees) does not. +fn local_allowlists(id: &str) -> Vec { + // `sourcegraph-access-token` accepts a bare 40-hex string, which is + // byte-identical to a git SHA-1. Transcripts are full of commit hashes + // and near a word like "token" one scores 1.000, so the unprefixed form + // is dropped; the `sgp_`-prefixed forms stay covered. + if id != "sourcegraph-access-token" { + return Vec::new(); + } + vec![Allowlist { + target: AllowTarget::Secret, + regexes: vec![compile(r"^[a-fA-F0-9]{40}$").expect("literal pattern")], + stopwords: None, + }] +} + +/// Parses the vendored TOML plus [`supplemental_rules`]. Pure and +/// deterministic - no I/O beyond the `include_str!` baked in at compile +/// time. +pub fn load_rules() -> Ruleset { let parsed: RulesFile = toml::from_str(RAW_TOML).expect("vendored gitleaks.toml must parse"); let mut rules: Vec = parsed .rules .into_iter() - .filter(|r| !EXCLUDED_RULE_IDS.iter().any(|(id, _)| *id == r.id)) .filter_map(|r| { + let mut allow: Vec = r + .allowlists + .into_iter() + .filter_map(|a| a.compile()) + .collect(); + allow.extend(local_allowlists(&r.id)); Some(Rule { - id: r.id, regex: r.regex?, entropy: r.entropy, + secret_group: r.secret_group.or_else(|| { + WHOLE_MATCH_IS_THE_SECRET + .contains(&r.id.as_str()) + .then_some(0) + }), keywords: r.keywords, - allow: r - .allowlists - .iter() - .flat_map(|a| a.regexes.iter()) - .filter_map(|re| regex::Regex::new(re).ok()) - .collect(), + id: r.id, + allow, }) }) .collect(); rules.extend(supplemental_rules()); - rules + Ruleset { + rules, + global: parsed + .allowlist + .into_iter() + .filter_map(|a| a.compile()) + .collect(), + } } #[cfg(test)] mod tests { use super::*; + /// The vendored file, at gitleaks commit + /// `b58d3f102cf3a2c84cb7f923d05c25c9b1aed84b`, holds 222 `[[rules]]`, + /// one of which (`pkcs12-file`) matches a path rather than content and + /// is dropped, plus this crate's four supplemental rules. Asserting + /// the exact count makes a vendor bump a deliberate edit here instead + /// of a silent drop in detection coverage. + const EXPECTED_RULE_COUNT: usize = 225; + #[test] fn every_vendored_rule_compiles_under_rust_regex() { - let rules = load_rules(); + let ruleset = load_rules(); + assert_eq!(ruleset.rules.len(), EXPECTED_RULE_COUNT); + for r in &ruleset.rules { + compile(&r.regex).unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); + } + } + + /// The three largest patterns are the reason [`REGEX_SIZE_LIMIT`] + /// exists; if a vendor bump renames one, the limit could be lowered + /// back to the default without anyone noticing. + #[test] + fn the_oversized_rules_are_present_and_need_the_raised_size_limit() { + let ruleset = load_rules(); + for id in ["generic-api-key", "pypi-upload-token", "vault-batch-token"] { + let rule = ruleset + .rules + .iter() + .find(|r| r.id == id) + .unwrap_or_else(|| panic!("{id} is no longer in the vendored ruleset")); + assert!( + regex::Regex::new(&rule.regex).is_err(), + "{id} now fits the default size limit" + ); + compile(&rule.regex).unwrap_or_else(|e| panic!("{id} exceeds even 64 MiB: {e}")); + } + } + + #[test] + fn secret_group_is_read_from_the_vendored_file() { + let ruleset = load_rules(); + let sonar = ruleset + .rules + .iter() + .find(|r| r.id == "sonar-api-token") + .expect("sonar-api-token is vendored"); + assert_eq!(sonar.secret_group, Some(2)); + } + + #[test] + fn unannotated_whole_match_rules_get_group_zero() { + let ruleset = load_rules(); + for id in WHOLE_MATCH_IS_THE_SECRET { + let rule = ruleset + .rules + .iter() + .find(|r| &r.id == id) + .unwrap_or_else(|| panic!("{id} is no longer in the vendored ruleset")); + assert_eq!(rule.secret_group, Some(0), "{id}"); + } + } + + #[test] + fn the_global_allowlist_is_loaded_and_suppresses_placeholders() { + let ruleset = load_rules(); + assert!(!ruleset.global.is_empty()); + for placeholder in ["${{ secrets.MY_TOKEN }}", "$MY_TOKEN", "%MY_TOKEN%", "true"] { + assert!( + ruleset.global.iter().any(|a| a.allows(placeholder, "", "")), + "global allowlist missed {placeholder}" + ); + } + } + + #[test] + fn generic_api_key_stopwords_are_loaded() { + let ruleset = load_rules(); + let rule = ruleset + .rules + .iter() + .find(|r| r.id == "generic-api-key") + .expect("generic-api-key is vendored"); + assert!( + rule.allow + .iter() + .any(|a| a.allows("my-adapter-value", "", "")) + ); + assert!( + !rule + .allow + .iter() + .any(|a| a.allows("Zt9xQw3pLm7RvB2k", "", "")) + ); + } + + /// The `condition = "AND"` block on `generic-api-key` needs a matching + /// file path, which a text detector never has. Applying its `line` + /// regexes anyway would silently suppress every `LICENSE = "..."` line. + #[test] + fn path_conditioned_allowlists_are_dropped() { + let ruleset = load_rules(); + let rule = ruleset + .rules + .iter() + .find(|r| r.id == "generic-api-key") + .expect("generic-api-key is vendored"); assert!( - rules.len() >= 200, - "expected the full ruleset, got {}", - rules.len() + !rule + .allow + .iter() + .any(|a| a.allows("", "", r#"LICENSE = "MIT-and-then-some""#)), + "the bitbake AND-conditioned allowlist is still being applied" ); - for r in &rules { - regex::Regex::new(&r.regex) - .unwrap_or_else(|e| panic!("rule {} failed to compile: {e}", r.id)); + } + + /// Pins the [`SUPPRESSED_ALLOWLIST_REGEXES`] decision on both sides: the + /// documentation-key exception is gone, and the stopwords that share its + /// rule set are untouched. + #[test] + fn documentation_key_allowlists_are_suppressed_but_stopwords_survive() { + // A vendor bump that rewrites the pattern would silently stop + // matching this list, and doc-shaped keys would survive again. + for pattern in SUPPRESSED_ALLOWLIST_REGEXES { + assert!( + RAW_TOML.contains(pattern), + "{pattern} is no longer in the vendored ruleset" + ); } + let ruleset = load_rules(); + let aws = ruleset + .rules + .iter() + .find(|r| r.id == "aws-access-token") + .expect("aws-access-token is vendored"); + assert!( + aws.allow.is_empty(), + "the `.+EXAMPLE$` allowlist is still being applied" + ); + let generic = ruleset + .rules + .iter() + .find(|r| r.id == "generic-api-key") + .expect("generic-api-key is vendored"); + assert!( + generic + .allow + .iter() + .any(|a| a.allows("my-adapter-value", "", "")), + "suppressing regexes took the stopword machinery with it" + ); + } + + /// The vendored Anthropic rules are length-pinned, so the supplement is + /// the only thing covering a key that was shortened on its way into a + /// transcript. + #[test] + fn the_loose_anthropic_rule_covers_what_the_vendored_one_misses() { + let ruleset = load_rules(); + let find = |id: &str| { + ruleset + .rules + .iter() + .find(|r| r.id == id) + .unwrap_or_else(|| panic!("{id} is missing")) + }; + let loose = compile(&find("anthropic-api-key-loose").regex).expect("literal pattern"); + let vendored = compile(&find("anthropic-api-key").regex).expect("vendored pattern"); + + let short = "sk-ant-api03-EXAMPLEONLYnotarealkey000000000000000000000AA"; + assert_eq!(loose.find(short).map(|m| m.as_str()), Some(short)); + assert!( + vendored.find(short).is_none(), + "the vendored rule now covers short keys; the supplement may be redundant" + ); + assert!(loose.is_match("sk-ant-admin01-Zt9xQw3pLm7RvB2kNc5YdA8j")); + // The prefix on its own is a format name, not a credential. + assert!(!loose.is_match("sk-ant-api03-short")); + } + + #[test] + fn line_targeted_allowlists_test_the_line_not_the_secret() { + let ruleset = load_rules(); + let rule = ruleset + .rules + .iter() + .find(|r| r.id == "generic-api-key") + .expect("generic-api-key is vendored"); + let secret = "Zt9xQw3pLm7RvB2k"; + let line = "RUN --mount=type=secret,id=npm npm install"; + assert!(rule.allow.iter().any(|a| a.allows(secret, "", line))); + assert!( + !rule + .allow + .iter() + .any(|a| a.allows(secret, "", "npm install")) + ); } } diff --git a/crates/toolpath-redact/src/lib.rs b/crates/toolpath-redact/src/lib.rs index 22460630..93424401 100644 --- a/crates/toolpath-redact/src/lib.rs +++ b/crates/toolpath-redact/src/lib.rs @@ -30,6 +30,11 @@ pub enum RedactError { SignedDocument, #[error("pointer {0} does not resolve")] BadPointer(String), + /// An unkeyed fingerprint is a dictionary attack away from the value + /// it stands for (EDPB 01/2025 para 88), so an empty key is refused + /// rather than silently downgraded to a bare hash. + #[error("redaction key is empty: the fingerprint would be an unkeyed hash of the value")] + EmptyKey, #[error("bad predicate: {0}")] BadPredicate(String), /// A third-party detector's own failure. Carries a message rather than diff --git a/crates/toolpath-redact/src/plan.rs b/crates/toolpath-redact/src/plan.rs index f4ff7277..bc40b973 100644 --- a/crates/toolpath-redact/src/plan.rs +++ b/crates/toolpath-redact/src/plan.rs @@ -23,6 +23,11 @@ pub struct PlanFinding { /// Surrounding line with the match replaced by its rule name. Never /// the value, never its length, unless `reveal` was set. pub context: String, + /// Keyed fingerprint of the bytes `span` covered when the plan was + /// generated. `verify` recomputes it, which is the only thing standing + /// between a mutated document and a marker spliced over the wrong text: + /// a same-length edit inside a recorded span leaves every offset valid. + pub fingerprint: String, pub action: Action, #[serde(default, skip_serializing_if = "Option::is_none")] pub transform: Option, @@ -88,6 +93,10 @@ pub struct RedactionPolicy { pub key_id: String, } +/// Plan format version. `verify` refuses anything else rather than reading a +/// shape it does not know the invariants of. +const PLAN_V: u32 = 1; + // ── Plan machinery (T5) ──────────────────────────────────────────────── pub fn parse_predicate(s: &str) -> crate::Result { @@ -99,11 +108,13 @@ pub fn parse_predicate(s: &str) -> crate::Result { (">", Cmp::Gt), ("<", Cmp::Lt), ] { + // The `score` guard is load-bearing, not defensive: without it an + // operator character *inside* a value (`at=/change/a>b`) is read as + // the predicate's operator. if let Some((k, v)) = s.split_once(op) && k.trim() == "score" { - let value: f32 = v.trim().parse().map_err(|_| bad_predicate(s))?; - return Ok(Predicate::Score(cmp, value)); + return Ok(Predicate::Score(cmp, parse_score(s, v)?)); } } @@ -118,7 +129,7 @@ pub fn parse_predicate(s: &str) -> crate::Result { "step" => Predicate::Step(v.to_string()), "detector" => Predicate::Detector(v.to_string()), "at" => Predicate::AtPrefix(v.to_string()), - "score" => Predicate::Score(Cmp::Eq, v.parse().map_err(|_| bad_predicate(s))?), + "score" => Predicate::Score(Cmp::Eq, parse_score(s, v)?), other => { return Err(crate::RedactError::BadPredicate(format!( "unknown field {other:?} in {s:?}" @@ -144,6 +155,21 @@ fn parse_shape(s: &str) -> crate::Result { }) } +/// Scores are the same 0.0..=1.0 quantity `--threshold` is, so they get the +/// same range check. Left open, `score<=inf` silently matches everything, +/// `score>=nan` silently matches nothing, and neither reads as a mistake in +/// the output. +fn parse_score(s: &str, v: &str) -> crate::Result { + let value: f32 = v.trim().parse().map_err(|_| bad_predicate(s))?; + if !(0.0..=1.0).contains(&value) { + return Err(crate::RedactError::BadPredicate(format!( + "score must be within 0.0..=1.0, got {} in {s:?}", + v.trim() + ))); + } + Ok(value) +} + fn bad_predicate(s: &str) -> crate::RedactError { crate::RedactError::BadPredicate(format!("not a valid predicate: {s:?}")) } @@ -168,15 +194,32 @@ pub fn matches(p: &Predicate, f: &PlanFinding) -> bool { /// Later decisions override earlier ones, so a caller can express /// "redact everything, except this" by ordering. +/// +/// Action and transform resolve independently. A decision that carries no +/// transform is not a decision to clear one - nothing in the surface can +/// express that - so `--mode-for rule=us-ssn:mask --accept score>=0.9` keeps +/// the mask. Overwriting it would make the run disagree with a replay of the +/// same policy, which reads the per-rule mode back out of the config. pub fn apply_decisions(plan: &mut Plan, decisions: &[Decision]) { for finding in &mut plan.findings { - if let Some(d) = decisions + let mut action = None; + let mut transform = None; + for d in decisions .iter() .rev() - .find(|d| matches(&d.predicate, finding)) + .filter(|d| matches(&d.predicate, finding)) { - finding.action = d.action; - finding.transform = d.transform; + action = action.or(Some(d.action)); + transform = transform.or(d.transform); + if transform.is_some() { + break; + } + } + if let Some(action) = action { + finding.action = action; + } + if let Some(transform) = transform { + finding.transform = Some(transform); } } } @@ -187,34 +230,73 @@ pub fn finding_id(index: usize) -> String { format!("f{:02}", index + 1) } -/// The line around `span` with the match replaced by `` - never the +/// How much text either side of the match a reviewer gets. A "line" in a +/// tool output or a minified file is not a line: an 8 KB newline-free field +/// otherwise lands in the plan verbatim, carrying whatever else was on it - +/// including the credentials the detectors missed. +const CONTEXT_RADIUS: usize = 40; + +/// The text around `span` with the match replaced by `` - never the /// value and never anything from which its length can be read, unless /// `reveal` was set. -pub fn elide_context(text: &str, span: std::ops::Range, rule: &str, reveal: bool) -> String { - let line_start = text[..span.start].rfind('\n').map_or(0, |i| i + 1); +/// +/// Bounded to `CONTEXT_RADIUS` bytes either side, with `…` marking a cut. +/// `\r` terminates the window along with `\n`: a lone carriage return in a +/// plan a dry run prints rewinds the terminal over the line before it. +/// +/// `span` must lie on char boundaries within `text` - `detect::normalise` +/// drops every finding that does not, and this is only ever called on its +/// output. +pub(crate) fn elide_context( + text: &str, + span: std::ops::Range, + rule: &str, + reveal: bool, +) -> String { + let line_start = text[..span.start].rfind(['\n', '\r']).map_or(0, |i| i + 1); let line_end = text[span.end..] - .find('\n') + .find(['\n', '\r']) .map_or(text.len(), |i| span.end + i); + + let mut start = line_start.max(span.start.saturating_sub(CONTEXT_RADIUS)); + while !text.is_char_boundary(start) { + start -= 1; + } + let mut end = line_end.min((span.end + CONTEXT_RADIUS).min(text.len())); + while !text.is_char_boundary(end) { + end -= 1; + } + let replacement = if reveal { text[span.start..span.end].to_string() } else { format!("<{rule}>") }; format!( - "{}{replacement}{}", - &text[line_start..span.start], - &text[span.end..line_end] + "{}{}{replacement}{}{}", + if start > line_start { "\u{2026}" } else { "" }, + &text[start..span.start], + &text[span.end..end], + if end < line_end { "\u{2026}" } else { "" }, ) } /// Refuse a plan that no longer describes this document, naming the first /// divergence. /// -/// Takes `path` by `&mut` (rather than the `&Path` the rest of this -/// function's job would suggest) because `SurfaceCursor` (T2) needs -/// exclusive access to resolve a pointer to text; `verify` itself never -/// mutates anything through it. -pub fn verify(plan: &Plan, path: &mut toolpath::v1::Path) -> crate::Result<()> { +/// `key` is the redaction key the plan was generated with. Offsets alone +/// prove nothing: a same-length edit inside a recorded span leaves every +/// bound valid, and the marker then lands over text nobody detected. So each +/// finding's fingerprint is recomputed from the bytes actually there now. +/// Reusing the key means a plan verifies only against the document it was +/// generated from, by whoever holds that key. +pub fn verify(plan: &Plan, path: &toolpath::v1::Path, key: &[u8]) -> crate::Result<()> { + if plan.v != PLAN_V { + return Err(crate::RedactError::PlanMismatch(format!( + "plan version {} is not supported (expected {PLAN_V})", + plan.v + ))); + } if plan.document != path.path.id { return Err(crate::RedactError::PlanMismatch(format!( "plan targets document {:?}, but path.id is {:?}", @@ -222,19 +304,29 @@ pub fn verify(plan: &Plan, path: &mut toolpath::v1::Path) -> crate::Result<()> { ))); } - let step_ids: std::collections::HashSet = - path.steps.iter().map(|s| s.step.id.clone()).collect(); + let steps: std::collections::HashMap<&str, &toolpath::v1::Step> = + path.steps.iter().map(|s| (s.step.id.as_str(), s)).collect(); + + for (i, finding) in plan.findings.iter().enumerate() { + // Ids are positional, and `apply` reports by id. A renumbered or + // reordered list makes every diagnostic name the wrong finding. + let expected = finding_id(i); + if finding.id != expected { + return Err(crate::RedactError::PlanMismatch(format!( + "finding {i} carries id {:?}, but ids are positional and this one is {expected:?}", + finding.id + ))); + } - let cursor = crate::surface::SurfaceCursor { path }; - for finding in &plan.findings { - if !finding.step.is_empty() && !step_ids.contains(&finding.step) { + let step = steps.get(finding.step.as_str()).copied(); + if !finding.step.is_empty() && step.is_none() { return Err(crate::RedactError::PlanMismatch(format!( "{}: step {:?} no longer exists", finding.id, finding.step ))); } - let current = cursor.read(&finding.step, &finding.at).ok_or_else(|| { + let current = crate::surface::read_at_in(path, step, &finding.at).ok_or_else(|| { crate::RedactError::PlanMismatch(format!( "{}: {} no longer resolves", finding.id, finding.at @@ -251,22 +343,34 @@ pub fn verify(plan: &Plan, path: &mut toolpath::v1::Path) -> crate::Result<()> { finding.id, finding.at ))); } + + // An empty or inverted span holds no value to fingerprint. `apply` + // refuses both by name, and its diagnostic is the more useful one. + if start < end + && crate::transform::Fingerprint::new(key, ¤t[start..end]).0 + != finding.fingerprint + { + return Err(crate::RedactError::PlanMismatch(format!( + "{}: the text at {} changed since the plan was generated", + finding.id, finding.at + ))); + } } Ok(()) } // ── Plan generation (T8) ─────────────────────────────────────────────── -pub fn generate( +/// In-crate convenience over `generate_inner` for tests that supply their own +/// detectors and treat a detector failure as a bug in the test. Test-only +/// because it is the panicking form: every shipping caller goes through +/// `generate_checked`, which also runs the egress check. +#[cfg(test)] +pub(crate) fn generate( path: &toolpath::v1::Path, detectors: &crate::detect::DetectorSet, cfg: &crate::RedactConfig, ) -> Plan { - // This signature is fixed since T0 and cannot return `Result` (T5's own - // byte-identity test calls it directly, unwrapped). A failing detector - // is a bug in that detector, not a normal outcome, so it surfaces as a - // panic here instead of silently degrading to an empty plan; - // `generate_checked` propagates the same failure through `Result`. generate_inner(path, detectors, cfg).expect("detector failed while generating a redaction plan") } @@ -297,43 +401,51 @@ fn generate_inner( cfg: &crate::RedactConfig, ) -> crate::Result { let surfaces = crate::surface::surfaces(path); + let steps: std::collections::HashMap<&str, &toolpath::v1::Step> = + path.steps.iter().map(|s| (s.step.id.as_str(), s)).collect(); + let kind = path.meta.as_ref().and_then(|m| m.kind.as_deref()); - // `SurfaceCursor` needs `&mut Path` (T2 uses the same struct for - // writes); `generate` only takes `&Path` since `verify` re-checks - // findings against the caller's own document later, so read through a - // throwaway clone instead. - let mut scratch = path.clone(); - let cursor = crate::surface::SurfaceCursor { path: &mut scratch }; - - // `surfaces()` already visits steps in document order and sorts - // artifacts/fields (see its own determinism test), and `detect_all` - // returns spans sorted by start - so findings collected in this order - // already satisfy the (step, pointer, span start) ordering `finding_id` - // relies on, with no extra sort needed here. + // Finding ids are positional over the emission order of `surfaces()`, + // which is itself deterministic (see its own determinism test), with each + // surface's own findings sorted by span start. Nothing below reorders, so + // an id can be assigned at push time. let mut findings = Vec::new(); for s in &surfaces { - let Some(text) = cursor.read(&s.step, &s.at) else { - continue; + let step = steps.get(s.step.as_str()).copied(); + // `surfaces()` named this field; if it no longer reads, the two + // disagree about the document and the pass has silently skipped a + // field it is claiming to have scanned. + let Some(text) = crate::surface::read_at_in(path, step, &s.at) else { + return Err(crate::RedactError::BadPointer(format!( + "{}{}", + s.step, s.at + ))); }; - let ctx = context_for(path, &s.step, &s.at); let candidate = crate::detect::Candidate { text: &text, shape: s.shape, at: &s.at, - ctx, + ctx: context_for(kind, step, &s.at), }; - for finding in detectors.detect_all(&candidate)? { - let action = if finding.score < cfg.threshold { - Action::Skip - } else { - Action::Redact - }; + + // Threshold before overlap resolution, never after: see + // `detect_all_partitioned`. + let (above, below) = detectors.detect_all_partitioned(&candidate, cfg.threshold)?; + let mut resolved: Vec<(crate::detect::Finding, Action)> = above + .into_iter() + .map(|f| (f, Action::Redact)) + .chain(below.into_iter().map(|f| (f, Action::Skip))) + .collect(); + resolved.sort_by_key(|(f, _)| f.span.start); + + for (finding, action) in resolved { let context = elide_context(&text, finding.span.clone(), &finding.rule, cfg.reveal); findings.push(PlanFinding { - id: String::new(), // assigned below, once findings are in final order + id: finding_id(findings.len()), step: s.step.clone(), at: s.at.clone(), span: (finding.span.start, finding.span.end), + fingerprint: crate::transform::Fingerprint::new(&cfg.key, &text[finding.span]).0, rule: finding.rule, score: finding.score, detector: finding.detector.to_string(), @@ -344,12 +456,9 @@ fn generate_inner( }); } } - for (i, finding) in findings.iter_mut().enumerate() { - finding.id = finding_id(i); - } Ok(Plan { - v: 1, + v: PLAN_V, document: path.path.id.clone(), generated: cfg.now, detectors: detectors.ids().into_iter().map(String::from).collect(), @@ -362,17 +471,16 @@ fn generate_inner( }) } -/// `change_type`/`actor` come from the step and artifact the pointer names. -/// `tool_name` stays `None`: resolving it would mean re-parsing the same -/// `tool_uses` JSON `surfaces()` already walked once, and no detector in -/// this crate reads it yet. +/// Everything a detector is told about where its candidate came from. +/// `Context` is the whole interface a pluggable detector gets, so every +/// field it declares is resolved: a detector that only fires inside `Bash` +/// input has no other way to know. fn context_for<'a>( - path: &'a toolpath::v1::Path, - step_id: &str, + kind: Option<&'a str>, + step: Option<&'a toolpath::v1::Step>, at: &str, ) -> crate::detect::Context<'a> { - let kind = path.meta.as_ref().and_then(|m| m.kind.as_deref()); - let Some(step) = path.steps.iter().find(|s| s.step.id == step_id) else { + let Some(step) = step else { return crate::detect::Context { change_type: "", tool_name: None, @@ -380,25 +488,52 @@ fn context_for<'a>( kind, }; }; - let change_type = artifact_key_from_at(at) + let structural = artifact_key_from_at(at) .and_then(|key| step.change.get(&key)) - .and_then(|c| c.structural.as_ref()) - .map(|s| s.change_type.as_str()) - .unwrap_or(""); + .and_then(|c| c.structural.as_ref()); crate::detect::Context { - change_type, - tool_name: None, + change_type: structural.map(|s| s.change_type.as_str()).unwrap_or(""), + tool_name: structural.and_then(|s| tool_name_at(&s.extra, at)), actor: step.step.actor.as_str(), kind, } } -/// Recovers the artifact key a `/change/...` pointer names. Mirrors -/// `surface::ptr_decode` (private to that module) - decode `~1` before -/// `~0`, or `~01` round-trips wrong (RFC 6901). +/// Recovers the artifact key a `/change/...` pointer names. fn artifact_key_from_at(at: &str) -> Option { let token = at.strip_prefix("/change/")?.split('/').next()?; - Some(token.replace("~1", "/").replace("~0", "~")) + Some(crate::surface::ptr_decode(token)) +} + +/// The tool a `/tool_uses/{i}/…` surface sits under, read back out of the +/// step's own extras. Delegated turns nest their own `tool_uses`, so the +/// *last* such segment names the call this pointer belongs to. +fn tool_name_at<'a>( + extra: &'a std::collections::HashMap, + at: &str, +) -> Option<&'a str> { + const SEG: &str = "tool_uses/"; + + // `StructuralChange::extra` is `#[serde(flatten)]`, so its keys sit + // directly under `structural` with no `extra` segment of their own. + let rest = at + .strip_prefix("/change/")? + .split_once('/')? + .1 + .strip_prefix("structural/")?; + let head = rest + .rfind(SEG) + .filter(|i| *i == 0 || rest.as_bytes()[i - 1] == b'/')?; + let index = rest[head + SEG.len()..].split('/').next()?; + + // `{head}tool_uses/{index}/name`, split the way `surface::route` splits: + // the first token is the `extra` key, the rest is a pointer into it. + let leaf = format!("{}{SEG}{index}/name", &rest[..head]); + let (field, tail) = leaf.split_once('/')?; + extra + .get(&crate::surface::ptr_decode(field))? + .pointer(&format!("/{tail}"))? + .as_str() } #[cfg(test)] @@ -407,23 +542,30 @@ mod tests { use std::collections::HashMap; use toolpath::v1::{ArtifactChange, Path, PathIdentity, Step, StepIdentity, StructuralChange}; + const TEST_KEY: &[u8] = b"test-key"; + fn fixed_now() -> DateTime { DateTime::parse_from_rfc3339("2026-07-30T00:00:00Z") .unwrap() .with_timezone(&Utc) } + fn fp(value: &str) -> String { + crate::transform::Fingerprint::new(TEST_KEY, value).0 + } + fn sample_finding(id: &str, rule: &str, score: f32) -> PlanFinding { PlanFinding { id: id.to_string(), step: "step-1".to_string(), - at: "/change/convo/structural/extra/text".to_string(), + at: "/change/convo/structural/text".to_string(), rule: rule.to_string(), span: (0, 4), score, detector: "internal".to_string(), shape: FieldShape::Prose, context: " context".to_string(), + fingerprint: String::new(), action: Action::Redact, transform: None, } @@ -453,7 +595,7 @@ mod tests { } /// One step whose `conversation.append` text field is `text`, addressable - /// at `/change//structural/extra/text` - the pointer shape + /// at `/change//structural/text` - the pointer shape /// `surfaces()` (T2) assigns to that field. fn fixture_path_with_text(doc_id: &str, step_id: &str, artifact_key: &str, text: &str) -> Path { let mut extra = HashMap::new(); @@ -576,6 +718,45 @@ mod tests { assert!(parse_predicate("score=not-a-number").is_err()); } + #[test] + fn rejects_non_finite_and_out_of_range_scores() { + // `score>=nan` matches nothing, `score<=inf` matches everything, and + // `score>=-1` matches everything. All three read as a filter that did + // not work, so none of them may be accepted silently. + for s in [ + "score>=nan", + "score<=inf", + "score>=-1", + "score<=2", + "score=-0.5", + "score=1.5", + "score>inf", + "score=nan", + ] { + assert!(parse_predicate(s).is_err(), "{s:?} should be refused"); + } + assert!(parse_predicate("score>=0").is_ok()); + assert!(parse_predicate("score<=1").is_ok()); + } + + #[test] + fn an_operator_inside_a_non_score_value_is_not_an_operator() { + // Without the `score` guard on the operator split, the `>` inside the + // pointer is read as the predicate's operator and `b` as its value. + assert_eq!( + parse_predicate("at=/change/a>b").unwrap(), + Predicate::AtPrefix("/change/a>b".to_string()) + ); + assert_eq!( + parse_predicate("rule=a<=b").unwrap(), + Predicate::Rule("a<=b".to_string()) + ); + assert_eq!( + parse_predicate("step=x=0.9`. Clearing the + // transform here would make the run emit a marker while a replay of + // the same policy - which reads the per-rule mode back out of the + // config - emits a mask. + let mut plan = sample_plan(vec![sample_finding("f01", "us-ssn", 0.95)]); + apply_decisions( + &mut plan, + &[ + Decision { + predicate: parse_predicate("rule=us-ssn").unwrap(), + action: Action::Redact, + transform: Some(Transform::Mask), + }, + decision("score>=0.9", Action::Redact), + ], + ); + assert_eq!(plan.findings[0].action, Action::Redact); + assert_eq!(plan.findings[0].transform, Some(Transform::Mask)); + } + + #[test] + fn a_later_decision_with_a_transform_still_replaces_the_earlier_one() { + let mut plan = sample_plan(vec![sample_finding("f01", "us-ssn", 0.95)]); + apply_decisions( + &mut plan, + &[ + Decision { + predicate: parse_predicate("rule=us-ssn").unwrap(), + action: Action::Redact, + transform: Some(Transform::Mask), + }, + Decision { + predicate: parse_predicate("score>=0.9").unwrap(), + action: Action::Redact, + transform: Some(Transform::Hash), + }, + ], + ); + assert_eq!(plan.findings[0].transform, Some(Transform::Hash)); + } + #[test] fn apply_decisions_leaves_non_matching_findings_untouched() { let mut plan = sample_plan(vec![sample_finding("f01", "us-ssn", 0.9)]); @@ -767,6 +991,50 @@ mod tests { assert_eq!(out, "just one line with a in it"); } + #[test] + fn elide_context_truncates_a_long_newline_free_field() { + // A minified bundle or a tool's JSON output has no newlines, so the + // "line" around a match is the whole field - and it goes into the plan + // verbatim, carrying whatever the detectors missed on it. + let mut text = "x".repeat(4000); + let start = text.len(); + text.push_str("SECRET"); + text.push_str(&"y".repeat(4000)); + assert_eq!(text.len(), 8006); + + let out = elide_context(&text, start..start + 6, "rule", false); + assert!(out.len() < 120, "context is unbounded: {} bytes", out.len()); + assert!(out.starts_with('\u{2026}'), "a cut must be marked: {out}"); + assert!(out.ends_with('\u{2026}'), "a cut must be marked: {out}"); + assert!(out.contains("")); + } + + #[test] + fn elide_context_does_not_carry_a_carriage_return() { + // A lone `\r` in a plan the dry run prints rewinds the terminal over + // the line before it, hiding whatever was there. + let text = "line one\r\nkey: SECRET\r\nline three"; + let start = text.find("SECRET").unwrap(); + let out = elide_context(text, start..start + "SECRET".len(), "rule", false); + assert_eq!(out, "key: "); + assert!(!out.contains('\r')); + + // Carriage returns alone are a line ending too. + let cr_only = "line one\rkey: SECRET\rline three"; + let start = cr_only.find("SECRET").unwrap(); + let out = elide_context(cr_only, start..start + "SECRET".len(), "rule", false); + assert_eq!(out, "key: "); + } + + #[test] + fn elide_context_cuts_on_a_char_boundary() { + let text = format!("{}SECRET", "é".repeat(200)); + let start = text.len() - "SECRET".len(); + let out = elide_context(&text, start..text.len(), "rule", false); + assert!(out.starts_with('\u{2026}')); + assert!(out.contains('é'), "the cut must not have split a codepoint"); + } + #[test] fn elide_context_reveal_includes_the_value() { let value = "AKIAIOSFODNN7REALKEY"; @@ -777,33 +1045,59 @@ mod tests { } // ── verify ─────────────────────────────────────────────────────────── - // - // These exercise `verify` through `SurfaceCursor::read` (T2), which is - // still `todo!()` as of this writing - see the report for status. - #[test] - fn verify_passes_on_an_unmodified_document() { - let text = "hello world, this is prose"; - let mut path = fixture_path_with_text("doc-1", "step-1", "convo", text); + /// A plan naming `world` inside the fixture's prose, fingerprinted with + /// `TEST_KEY` - what `generate` would have produced for that document. + fn world_plan() -> (String, Plan) { + let text = "hello world, this is prose".to_string(); let start = text.find("world").unwrap(); let plan = sample_plan(vec![PlanFinding { span: (start, start + "world".len()), + fingerprint: fp("world"), ..sample_finding("f01", "some-rule", 0.9) }]); - assert!(verify(&plan, &mut path).is_ok()); + (text, plan) } #[test] - fn verify_fails_naming_the_finding_whose_span_no_longer_lands() { - let text = "hello world, this is prose"; - let start = text.find("world").unwrap(); - let plan = sample_plan(vec![PlanFinding { - span: (start, start + "world".len()), - ..sample_finding("f01", "some-rule", 0.9) - }]); + fn verify_passes_on_an_unmodified_document() { + let (text, plan) = world_plan(); + let path = fixture_path_with_text("doc-1", "step-1", "convo", &text); + assert!(verify(&plan, &path, TEST_KEY).is_ok()); + } + + #[test] + fn verify_refuses_a_same_length_in_span_mutation() { + // The whole point of verifying: every offset still lands, the step is + // still there, the pointer still resolves - and the marker would go + // over five bytes nobody detected. + let (_, plan) = world_plan(); + let mutated = + fixture_path_with_text("doc-1", "step-1", "convo", "hello MONKE, this is prose"); + let e = verify(&plan, &mutated, TEST_KEY).unwrap_err().to_string(); + assert!(e.contains("f01"), "should name the first divergence: {e}"); + } + + #[test] + fn verify_refuses_an_edit_that_shifts_the_span() { + let (_, plan) = world_plan(); + let shifted = + fixture_path_with_text("doc-1", "step-1", "convo", "hello world, this is pros"); + assert!(verify(&plan, &shifted, TEST_KEY).is_err()); + } - let mut mutated = fixture_path_with_text("doc-1", "step-1", "convo", "short"); - let e = verify(&plan, &mut mutated).unwrap_err().to_string(); + #[test] + fn verify_refuses_a_plan_fingerprinted_under_another_key() { + let (text, plan) = world_plan(); + let path = fixture_path_with_text("doc-1", "step-1", "convo", &text); + assert!(verify(&plan, &path, b"a-different-key").is_err()); + } + + #[test] + fn verify_fails_naming_the_finding_whose_span_no_longer_lands() { + let (_, plan) = world_plan(); + let mutated = fixture_path_with_text("doc-1", "step-1", "convo", "short"); + let e = verify(&plan, &mutated, TEST_KEY).unwrap_err().to_string(); assert!(e.contains("f01"), "should name the first divergence: {e}"); } @@ -814,16 +1108,52 @@ mod tests { span: (0, 5), ..sample_finding("f01", "some-rule", 0.9) }]); - let mut path = fixture_path_with_text("doc-1", "step-1", "convo", "hello world"); - let e = verify(&plan, &mut path).unwrap_err().to_string(); + let path = fixture_path_with_text("doc-1", "step-1", "convo", "hello world"); + let e = verify(&plan, &path, TEST_KEY).unwrap_err().to_string(); assert!(e.contains("f01")); } #[test] fn verify_fails_when_the_document_id_differs() { let plan = sample_plan(vec![]); - let mut path = fixture_path_with_text("different-doc", "step-1", "convo", "text"); - assert!(verify(&plan, &mut path).is_err()); + let path = fixture_path_with_text("different-doc", "step-1", "convo", "text"); + assert!(verify(&plan, &path, TEST_KEY).is_err()); + } + + #[test] + fn verify_refuses_an_unrecognised_plan_version() { + let (text, mut plan) = world_plan(); + plan.v = 2; + let path = fixture_path_with_text("doc-1", "step-1", "convo", &text); + let e = verify(&plan, &path, TEST_KEY).unwrap_err().to_string(); + assert!(e.contains('2'), "should name the version it refused: {e}"); + } + + #[test] + fn verify_refuses_findings_whose_ids_are_not_positional() { + // `apply` reports by id, so a renumbered list makes every diagnostic + // - and every `--accept id=…` a reviewer writes - name a different + // finding than the one they read. + let (text, mut plan) = world_plan(); + plan.findings[0].id = "f07".to_string(); + let path = fixture_path_with_text("doc-1", "step-1", "convo", &text); + assert!(verify(&plan, &path, TEST_KEY).is_err()); + } + + #[test] + fn verify_tolerates_an_empty_or_inverted_span_for_apply_to_name() { + let text = "hello world, this is prose"; + let path = fixture_path_with_text("doc-1", "step-1", "convo", text); + for span in [(6, 6), (11, 6)] { + let plan = sample_plan(vec![PlanFinding { + span, + ..sample_finding("f01", "some-rule", 0.9) + }]); + assert!( + verify(&plan, &path, TEST_KEY).is_ok(), + "{span:?} is apply's diagnosis to make, not verify's" + ); + } } // ── serde round-trip ───────────────────────────────────────────────── @@ -841,7 +1171,7 @@ mod tests { }, surfaces: vec![crate::surface::Surface { step: "step-1".to_string(), - at: "/change/convo/structural/extra/text".to_string(), + at: "/change/convo/structural/text".to_string(), shape: FieldShape::Prose, bytes: 42, }], @@ -849,6 +1179,7 @@ mod tests { span: (10, 30), score: 0.97, context: " is the key".to_string(), + fingerprint: fp("AKIAIOSFODNN7EXAMPLE"), transform: Some(Transform::Hash), ..sample_finding("f01", "aws-access-key-id", 0.97) }], @@ -940,6 +1271,26 @@ mod plan_gen { } } + /// Pointer and resolved tool name, one entry per candidate. + type SeenCtx = std::sync::Arc)>>>; + + /// Records what every candidate was shown as. `Context` is the whole of + /// what a detector is told about where its text came from, so the only + /// place to observe it is inside one. + struct CtxSpy(SeenCtx); + impl Detector for CtxSpy { + fn id(&self) -> &'static str { + "ctx-spy" + } + fn detect(&self, c: &Candidate<'_>) -> crate::Result> { + self.0 + .lock() + .unwrap() + .push((c.at.to_string(), c.ctx.tool_name.map(str::to_owned))); + Ok(Vec::new()) + } + } + fn step_with_text(id: &str, artifact: &str, text: &str) -> Step { let mut extra = HashMap::new(); extra.insert( @@ -1001,6 +1352,62 @@ mod plan_gen { plan.findings.iter().filter(|pf| pf.at == at).count() } + fn rules_of(plan: &Plan) -> Vec<&str> { + plan.findings.iter().map(|f| f.rule.as_str()).collect() + } + + /// Several artifacts per step, several `extra` keys each, under a change + /// type `surfaces()` has no model for - so both the artifact map's + /// iteration order and the blind walk's are exercised. + /// + /// Built fresh on every call. `HashMap`'s iteration order is seeded per + /// instance, so a byte-identity test that compares one document against + /// itself cannot see an order leaking into the output. + fn fixture_hash_ordered() -> Path { + let steps = ["s1", "s2"] + .into_iter() + .map(|id| { + let change = ["zeta", "alpha", "mu", "beta"] + .into_iter() + .map(|artifact| { + let extra = ["omega", "delta", "kappa", "iota", "chi"] + .into_iter() + .map(|key| { + ( + key.to_string(), + serde_json::Value::String(format!( + "{key} holds a SECRET-VALUE somewhere" + )), + ) + }) + .collect(); + ( + artifact.to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "provider.blob".to_string(), + extra, + }), + }, + ) + }) + .collect(); + Step { + step: StepIdentity { + id: id.to_string(), + parents: Vec::new(), + actor: "human:t".to_string(), + timestamp: "2026-01-01T00:00:00Z".to_string(), + }, + change, + meta: None, + } + }) + .collect(); + path_of(steps) + } + // ── Step 8.1, verbatim ──────────────────────────────────────────────── #[test] @@ -1094,17 +1501,188 @@ mod plan_gen { #[test] fn regenerating_a_plan_yields_byte_identical_json() { - let path = fixture_mixed(); + // Two separately built fixtures, not one document generated twice: the + // `HashMap`s a document is made of are seeded per instance, and only a + // second instance can catch that seed reaching the output. let set = detectors(); let cfg = cfg(); - let a = generate(&path, &set, &cfg); - let b = generate(&path, &set, &cfg); + let a = generate(&fixture_hash_ordered(), &set, &cfg); + let b = generate(&fixture_hash_ordered(), &set, &cfg); + assert!(a.findings.len() > 10, "fixture must exercise the ordering"); assert_eq!( serde_json::to_string(&a).unwrap(), serde_json::to_string(&b).unwrap() ); } + // ── Threshold before overlap resolution (T8) ───────────────────────── + + #[test] + fn a_low_score_container_never_evicts_an_above_threshold_finding() { + // Overlap resolution is score-blind on length. Resolve first and the + // 0.6 whole-line hit wins; threshold afterwards and it is discarded + // too - and the key it swallowed ships in the clear, unreported. + let text = "prefix AKIAIOSFODNN7REALKEY suffix"; + assert_eq!(text.len(), 34); + let path = path_of(vec![step_with_text("s1", "a", text)]); + + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![ + f(0..34, "generic-entropy", 0.6), + f(7..27, "aws-access-key-id", 0.99), + ]))); + + let plan = generate(&path, &set, &cfg()); + assert_eq!(rules_of(&plan), vec!["aws-access-key-id"]); + assert_eq!(plan.findings[0].action, Action::Redact); + assert_eq!(plan.findings[0].span, (7, 27)); + } + + #[test] + fn a_sub_threshold_finding_that_contests_nothing_stays_in_the_plan() { + // `--accept score>=0.5` has to have something to accept. + let path = path_of(vec![step_with_text("s1", "a", "0123456789abcdefghij")]); + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![ + f(0..5, "certain", 0.99), + f(10..15, "unsure", 0.4), + ]))); + + let plan = generate(&path, &set, &cfg()); + assert_eq!(rules_of(&plan), vec!["certain", "unsure"]); + assert_eq!(plan.findings[0].action, Action::Redact); + assert_eq!(plan.findings[1].action, Action::Skip); + assert_eq!(plan.findings[0].id, "f01"); + assert_eq!(plan.findings[1].id, "f02"); + } + + // ── Coverage the reviewer asked for ───────────────────────────────── + + #[test] + fn a_credential_in_path_base_becomes_a_finding_with_an_empty_step() { + let mut path = path_of(vec![step_with_text("s1", "a", "nothing to see")]); + path.path.base = Some(toolpath::v1::Base { + uri: "https://x-token-auth:SECRET-VALUE@example.com/o/r".to_string(), + ref_str: None, + branch: None, + }); + + let plan = generate(&path, &detectors(), &cfg()); + let finding = plan + .findings + .iter() + .find(|f| f.at == "/path/base/uri") + .expect("a document-level surface must be scanned like any other"); + assert_eq!(finding.step, "", "document-level fields belong to no step"); + assert_eq!(finding.action, Action::Redact); + } + + #[test] + fn a_plan_always_round_trips_through_json() { + // JSON has no infinity: serde writes it as `null`, `score` stops + // parsing as an `f32`, and the plan file cannot be read back at all. + let mut set = DetectorSet::default(); + set.push(Box::new(FixedDetector(vec![ + f(0..10, "infinite", f32::INFINITY), + f(10..20, "finite", 0.9), + ]))); + + let plan = generate(&fixture_one_field(), &set, &cfg()); + assert_eq!(rules_of(&plan), vec!["finite"]); + + let json = serde_json::to_string(&plan).unwrap(); + assert!(!json.contains("null"), "a score serialised as null: {json}"); + assert_eq!(serde_json::from_str::(&json).unwrap(), plan); + } + + #[test] + fn a_surface_that_does_not_read_back_is_refused_not_skipped() { + // Two steps sharing an id: `surfaces()` names both steps' fields, but + // a `(step, pointer)` pair can only resolve to one of them. Skipping + // the half that does not read reports those fields as scanned when + // nothing ever looked at them. + let path = path_of(vec![ + step_with_text("dup", "a", "first"), + step_with_text("dup", "b", "second"), + ]); + assert!(matches!( + generate_checked(&path, &detectors(), &cfg(), false), + Err(crate::RedactError::BadPointer(_)) + )); + } + + #[test] + fn a_tool_use_surface_carries_the_tool_name() { + let spy = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let mut set = DetectorSet::default(); + set.push(Box::new(CtxSpy(spy.clone()))); + + let mut extra = HashMap::new(); + extra.insert( + "tool_uses".to_string(), + serde_json::json!([ + {"name": "Bash", "input": {"command": "echo hi"}}, + {"name": "Read", "input": {"file_path": "/etc/hosts"}}, + ]), + ); + extra.insert( + "delegations".to_string(), + serde_json::json!([{ + "turns": [{"tool_uses": [{"name": "Grep", "input": {"pattern": "x"}}]}], + }]), + ); + extra.insert( + "text".to_string(), + serde_json::Value::String("plain prose".to_string()), + ); + + let mut change = HashMap::new(); + change.insert( + "convo".to_string(), + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "conversation.append".to_string(), + extra, + }), + }, + ); + let path = path_of(vec![Step { + step: StepIdentity { + id: "s1".to_string(), + parents: Vec::new(), + actor: "agent:claude-code".to_string(), + timestamp: "2026-01-01T00:00:00Z".to_string(), + }, + change, + meta: None, + }]); + + generate(&path, &set, &cfg()); + + let seen = spy.lock().unwrap(); + let named = |suffix: &str| -> Option { + seen.iter() + .find(|(at, _)| at.ends_with(suffix)) + .and_then(|(_, name)| name.clone()) + }; + assert_eq!(named("/tool_uses/0/input/command").as_deref(), Some("Bash")); + assert_eq!( + named("/tool_uses/1/input/file_path").as_deref(), + Some("Read") + ); + assert_eq!( + named("/delegations/0/turns/0/tool_uses/0/input/pattern").as_deref(), + Some("Grep"), + "a delegated turn's own tool_uses name its own calls" + ); + assert_eq!( + named("/structural/text"), + None, + "prose belongs to no tool call" + ); + } + #[test] fn detector_error_propagates_through_generate_checked() { let mut set = DetectorSet::default(); diff --git a/crates/toolpath-redact/src/surface.rs b/crates/toolpath-redact/src/surface.rs index 5ae30022..37471505 100644 --- a/crates/toolpath-redact/src/surface.rs +++ b/crates/toolpath-redact/src/surface.rs @@ -19,17 +19,37 @@ pub struct Surface { } /// Every string in `path` a detector should see, each named by an RFC 6901 -/// pointer relative to its step (`step` is empty for document-level fields). +/// pointer that resolves against the serialized document: relative to the +/// step for a step's fields, relative to the document root for the rest +/// (those carry an empty `step`). An audit record publishes these pointers, +/// so one that does not resolve is a lie told to whoever reads it. /// /// The order is part of the contract. Finding ids are positional and a -/// regenerated plan is compared byte for byte, but `Step::change` and -/// `StructuralChange::extra` are `HashMap`s whose iteration order is not -/// stable across runs - so steps keep document order, artifacts sort by key, -/// and a blind walk sorts object keys. +/// regenerated plan is compared byte for byte, but `Step::change` and every +/// `extra` are `HashMap`s whose iteration order is not stable across runs - +/// so steps keep document order, artifacts sort by key, and every blind walk +/// sorts object keys. /// /// An artifact's own key is emitted *after* everything beneath it: writing /// that surface renames the map entry, which invalidates every pointer under /// the old key, so a caller applying surfaces in order rewrites the key last. +/// Each typed arm likewise emits its named fields before the blind walk of +/// the keys it did not consume. +/// +/// The map is closed by construction, so what it leaves out is as much a +/// decision as what it names. Deliberately never scanned: +/// +/// - `/step/*` and `/path/{id,head}` - identity the DAG is keyed on; +/// rewriting one detaches a step from its parents. +/// - `/path/graph_ref` - a toolpath-internal link to a sibling document, not +/// content anyone typed. +/// - `StructuralChange::type`, `PathMeta::{kind,source}` - closed +/// vocabularies this format defines, with no room for a user's string. +/// - `VcsSource::{type,revision,change_id}` and `Ref::rel` - identifiers the +/// VCS or the format assigns, not the human. +/// - `{Step,Path}Meta::{actors,signatures}` - identity and integrity +/// material, which redaction drops wholesale rather than rewrites. +/// - every non-string leaf - a detector has nothing to span in a number. pub fn surfaces(path: &toolpath::v1::Path) -> Vec { let mut out = Vec::new(); for step in &path.steps { @@ -50,13 +70,16 @@ pub fn surfaces(path: &toolpath::v1::Path) -> Vec { ); } if let Some(s) = &change.structural { - let base = format!("/change/{akey}/structural/extra"); + // `StructuralChange::extra` is `#[serde(flatten)]`: its keys + // sit directly under `structural` in the document, so an + // `extra` segment would name a field that is not there. + let base = format!("/change/{akey}/structural"); match s.change_type.as_str() { "conversation.append" => turn_surfaces(&mut out, sid, &base, &s.extra), "file.write" => file_write_surfaces(&mut out, sid, &base, &s.extra), - // The one place a blind leaf walk is correct: the payload - // is unmodelled provider JSON. - _ => walk_fields(&mut out, sid, &base, &s.extra, FieldShape::OpaqueJson), + // The one place a blind leaf walk is the whole story: the + // payload is unmodelled provider JSON. + _ => residue(&mut out, sid, &base, &s.extra, &[]), } } push( @@ -67,6 +90,9 @@ pub fn surfaces(path: &toolpath::v1::Path) -> Vec { artifact_key, ); } + if let Some(meta) = &step.meta { + step_meta_surfaces(&mut out, sid, meta); + } } if let Some(b) = &path.path.base { push( @@ -76,18 +102,84 @@ pub fn surfaces(path: &toolpath::v1::Path) -> Vec { FieldShape::Uri, &b.uri, ); + // A branch name is author-chosen text in a path-shaped field, and a + // `ref` is whatever someone tagged the state as. + for (key, value) in [("branch", &b.branch), ("ref", &b.ref_str)] { + if let Some(v) = value { + push( + &mut out, + "", + format!("/path/base/{key}"), + FieldShape::OpaqueJson, + v, + ); + } + } } - if let Some(v) = path - .meta - .as_ref() - .and_then(|m| m.extra.get("vcs_remote")) - .and_then(|v| v.as_str()) - { - push(&mut out, "", "/meta/vcs_remote".into(), FieldShape::Uri, v); + if let Some(meta) = &path.meta { + path_meta_surfaces(&mut out, meta); } out } +/// A path's own metadata. `PathMeta::extra` is `#[serde(flatten)]`, so its +/// keys sit directly under `/meta` with no segment of their own. +fn path_meta_surfaces(out: &mut Vec, meta: &toolpath::v1::PathMeta) { + for (key, value) in [("title", &meta.title), ("intent", &meta.intent)] { + if let Some(v) = value { + push(out, "", format!("/meta/{key}"), FieldShape::Prose, v); + } + } + for (i, r) in meta.refs.iter().enumerate() { + push( + out, + "", + format!("/meta/refs/{i}/href"), + FieldShape::Uri, + &r.href, + ); + } + if let Some(v) = meta.extra.get("vcs_remote") { + walk_json(out, "", "/meta/vcs_remote", v, FieldShape::Uri); + } + // These are byte-identical copies of the `step.change` keys the map + // already scans. Redacting the key and leaving the copy here hands the + // reader the original back, so it has to be the same value or neither. + if let Some(v) = meta.extra.get("files_changed") { + walk_json(out, "", "/meta/files_changed", v, FieldShape::Uri); + } + residue( + out, + "", + "/meta", + &meta.extra, + &["vcs_remote", "files_changed"], + ); +} + +/// A step's own metadata. `intent` is the git commit message on a +/// `p import git` document and the pull-request or review body on a GitHub +/// one - and since `toolpath-git` emits raw-only changes, on those documents +/// it is the only prose in the whole path. +fn step_meta_surfaces(out: &mut Vec, step: &str, meta: &toolpath::v1::StepMeta) { + if let Some(v) = &meta.intent { + push(out, step, "/meta/intent".into(), FieldShape::Prose, v); + } + for (i, r) in meta.refs.iter().enumerate() { + push( + out, + step, + format!("/meta/refs/{i}/href"), + FieldShape::Uri, + &r.href, + ); + } + if let Some(src) = &meta.source { + residue(out, step, "/meta/source", &src.extra, &[]); + } + residue(out, step, "/meta", &meta.extra, &[]); +} + fn push(out: &mut Vec, step: &str, at: String, shape: FieldShape, text: &str) { if text.is_empty() { return; @@ -104,102 +196,230 @@ fn push(out: &mut Vec, step: &str, at: String, shape: FieldShape, text: /// `structural.extra` map, and a delegated turn's JSON object. trait Fields { fn field(&self, key: &str) -> Option<&Value>; + /// Sorted: both backing maps iterate in an order that is not stable + /// across runs, and emission order is part of the contract. + fn entries(&self) -> Vec<(&str, &Value)>; } impl Fields for HashMap { fn field(&self, key: &str) -> Option<&Value> { self.get(key) } + fn entries(&self) -> Vec<(&str, &Value)> { + sorted(self.iter()) + } } impl Fields for serde_json::Map { fn field(&self, key: &str) -> Option<&Value> { self.get(key) } + fn entries(&self) -> Vec<(&str, &Value)> { + sorted(self.iter()) + } } -/// The `conversation.append` rows of the map. A delegated turn serializes -/// with the same field names as the extras of the turn that spawned it, so -/// sub-conversations re-enter here. +/// Every key of `fields` the typed arm above it did not read, walked blind. +/// The map is a closed list, and a field it does not name is never scanned - +/// so `environment.vcs_branch`, an MCP server name inside `tool_uses[i].name`, +/// or a delegated mutation's `path` would ship a secret with nothing to +/// notice. Emitted after the typed fields, per the emission-order contract. +fn residue(out: &mut Vec, step: &str, at: &str, fields: &dyn Fields, consumed: &[&str]) { + for (key, value) in fields.entries() { + if consumed.contains(&key) { + continue; + } + walk_json( + out, + step, + &format!("{at}/{}", ptr_escape(key)), + value, + FieldShape::OpaqueJson, + ); + } +} + +/// `residue` over a value the caller expected to be an object. A value of +/// some other shape was not read by the typed arm either, so it goes to the +/// blind walk whole rather than falling out of the map. +fn residue_of( + out: &mut Vec, + step: &str, + at: &str, + value: Option<&Value>, + consumed: &[&str], +) { + match value { + Some(Value::Object(map)) => residue(out, step, at, map, consumed), + Some(other) => walk_json(out, step, at, other, FieldShape::OpaqueJson), + None => {} + } +} + +/// The array a typed arm expects at `key`, and nothing if it is absent. A +/// value of another shape is walked blind here for the same reason +/// `residue_of` does it: the typed arm below will not read it. +fn typed_array<'a>( + out: &mut Vec, + step: &str, + at: &str, + fields: &'a dyn Fields, + key: &str, +) -> &'a [Value] { + match fields.field(key) { + Some(Value::Array(items)) => items, + Some(other) => { + walk_json( + out, + step, + &format!("{at}/{key}"), + other, + FieldShape::OpaqueJson, + ); + &[] + } + None => &[], + } +} + +/// The `conversation.append` rows of the map, and - because the two share +/// enough field names to share an arm - the rows of a delegated turn. +/// +/// They are not the same object. A top-level turn's extras are assembled +/// field by field by `toolpath_convo::derive_path`; a delegated turn is a +/// whole `Turn` handed to `serde_json::to_value`. The delta, all of it in the +/// delegated direction: `file_mutations` (a top-level turn's are hoisted into +/// sibling `file.write` changes instead), plus `id`, `parent_id`, `timestamp` +/// and `model`, and `role` as serde writes the enum (`"Assistant"`, or +/// externally tagged `{"Other": "tool"}`) rather than the lowercase string +/// the top level stores. Nothing here reads those by name; the residue walk +/// at the end covers them, which is exactly why it exists. fn turn_surfaces(out: &mut Vec, step: &str, at: &str, fields: &dyn Fields) { + const CONSUMED: &[&str] = &[ + "text", + "thinking", + "tool_uses", + "delegations", + "file_mutations", + "environment", + ]; + for key in ["text", "thinking"] { - if let Some(s) = fields.field(key).and_then(Value::as_str) { - push(out, step, format!("{at}/{key}"), FieldShape::Prose, s); + if let Some(v) = fields.field(key) { + walk_json(out, step, &format!("{at}/{key}"), v, FieldShape::Prose); } } - for (i, tool) in array(fields.field("tool_uses")).iter().enumerate() { + for (i, tool) in typed_array(out, step, at, fields, "tool_uses") + .iter() + .enumerate() + { + let at = format!("{at}/tool_uses/{i}"); if let Some(input) = tool.get("input") { walk_json( out, step, - &format!("{at}/tool_uses/{i}/input"), + &format!("{at}/input"), input, FieldShape::ToolInput, ); } - if let Some(s) = tool.pointer("/result/content").and_then(Value::as_str) { - push( + if let Some(content) = tool.pointer("/result/content") { + walk_json( out, step, - format!("{at}/tool_uses/{i}/result/content"), + &format!("{at}/result/content"), + content, FieldShape::ToolOutput, - s, ); } + residue_of( + out, + step, + &format!("{at}/result"), + tool.get("result"), + &["content"], + ); + // Picks up `name`, which for an MCP call is `mcp____` - + // and the server half comes from the user's own config. + residue_of(out, step, &at, Some(tool), &["input", "result"]); } - for (i, work) in array(fields.field("delegations")).iter().enumerate() { + for (i, work) in typed_array(out, step, at, fields, "delegations") + .iter() + .enumerate() + { let at = format!("{at}/delegations/{i}"); for key in ["prompt", "result"] { - if let Some(s) = work.get(key).and_then(Value::as_str) { - push(out, step, format!("{at}/{key}"), FieldShape::Prose, s); + if let Some(v) = work.get(key) { + walk_json(out, step, &format!("{at}/{key}"), v, FieldShape::Prose); } } for (j, turn) in array(work.get("turns")).iter().enumerate() { - if let Some(obj) = turn.as_object() { - turn_surfaces(out, step, &format!("{at}/turns/{j}"), obj); + let at = format!("{at}/turns/{j}"); + match turn.as_object() { + Some(obj) => turn_surfaces(out, step, &at, obj), + None => walk_json(out, step, &at, turn, FieldShape::OpaqueJson), } } + residue_of(out, step, &at, Some(work), &["prompt", "result", "turns"]); } // Only a top-level turn's file mutations get hoisted into sibling // `file.write` changes; a delegated turn carries its own inline, and they - // hold the same before/after file content. - for (i, mutation) in array(fields.field("file_mutations")).iter().enumerate() { + // hold the same before/after file content. The residue below is what + // reaches `path` and `rename_to` on the delegated ones - an entry + // carrying only `path` has no typed field at all. + for (i, mutation) in typed_array(out, step, at, fields, "file_mutations") + .iter() + .enumerate() + { let at = format!("{at}/file_mutations/{i}"); - if let Some(s) = mutation.get("raw_diff").and_then(Value::as_str) { - push( + if let Some(v) = mutation.get("raw_diff") { + walk_json( out, step, - format!("{at}/raw_diff"), + &format!("{at}/raw_diff"), + v, FieldShape::UnifiedDiff, - s, ); } for key in ["before", "after"] { - if let Some(s) = mutation.get(key).and_then(Value::as_str) { - push(out, step, format!("{at}/{key}"), FieldShape::FileContent, s); + if let Some(v) = mutation.get(key) { + walk_json( + out, + step, + &format!("{at}/{key}"), + v, + FieldShape::FileContent, + ); } } - } - - if let Some(s) = fields - .field("environment") - .and_then(|e| e.get("working_dir")) - .and_then(Value::as_str) - { - push( + residue_of( out, step, - format!("{at}/environment/working_dir"), - FieldShape::Uri, - s, + &at, + Some(mutation), + &["raw_diff", "before", "after"], ); } + + if let Some(env) = fields.field("environment") { + let at = format!("{at}/environment"); + if let Some(v) = env.get("working_dir") { + walk_json(out, step, &format!("{at}/working_dir"), v, FieldShape::Uri); + } + // `vcs_branch` and `vcs_revision` sit right beside `working_dir`, and + // a branch name is as author-chosen as a commit message. + residue_of(out, step, &at, Some(env), &["working_dir"]); + } + + residue(out, step, at, fields, CONSUMED); } /// The `file.write` rows: whole-file states, plus both sides of every edit. +/// The residue reaches `rename_to`, which is a path exactly like the one +/// that became this change's artifact key - and that key is scanned. fn file_write_surfaces( out: &mut Vec, step: &str, @@ -207,11 +427,20 @@ fn file_write_surfaces( extra: &HashMap, ) { for key in ["before", "after"] { - if let Some(s) = extra.get(key).and_then(Value::as_str) { - push(out, step, format!("{at}/{key}"), FieldShape::FileContent, s); + if let Some(v) = extra.get(key) { + walk_json( + out, + step, + &format!("{at}/{key}"), + v, + FieldShape::FileContent, + ); } } - for (i, edit) in array(extra.get("edits")).iter().enumerate() { + for (i, edit) in typed_array(out, step, at, extra, "edits") + .iter() + .enumerate() + { walk_json( out, step, @@ -220,24 +449,7 @@ fn file_write_surfaces( FieldShape::FileContent, ); } -} - -fn walk_fields( - out: &mut Vec, - step: &str, - at: &str, - fields: &HashMap, - shape: FieldShape, -) { - for (key, value) in sorted(fields.iter()) { - walk_json( - out, - step, - &format!("{at}/{}", ptr_escape(key)), - value, - shape, - ); - } + residue(out, step, at, extra, &["before", "after", "edits"]); } /// Every string leaf under `value`, named by pointer. Non-string scalars are @@ -274,20 +486,63 @@ fn array(value: Option<&Value>) -> &[Value] { } /// Resolves a `(step, pointer)` pair against a document for reading and -/// writing. Read and write must resolve identically. +/// writing. +/// +/// Read and write resolve identically, with one exception: +/// [`Route::ArtifactKey`] writes by *renaming* the map entry, so once that +/// write lands, the pointer it was made at - and every pointer beneath it - +/// resolves to nothing. That is not a defect to fix here but the reason +/// `surfaces()` emits an artifact's key after everything under it: a caller +/// applying surfaces in emission order never reads through a renamed key. pub struct SurfaceCursor<'a> { pub path: &'a mut toolpath::v1::Path, } +/// Which object a pointer is relative to. `/meta/…` names `StepMeta` under a +/// step and `PathMeta` at the document root - the pointer alone cannot tell +/// them apart, so read and write must agree on the frame before parsing. +#[derive(Clone, Copy)] +enum Scope { + Document, + Step, +} + +/// An empty step id is the document frame; anything else is a step's. +fn scope_of(step: &str) -> Scope { + if step.is_empty() { + Scope::Document + } else { + Scope::Step + } +} + /// Where a pointer lands, parsed once so read and write cannot drift apart. enum Route { - BaseUri, - MetaExtra(String), + /// A typed `Option` slot on the document itself. + DocText(DocText), + DocRefHref(usize), + /// `PathMeta::extra[field]`, plus an RFC 6901 pointer into it (empty when + /// the field is itself the string). `extra` is `#[serde(flatten)]`, so + /// the pointer carries no `extra` segment. + DocMetaExtra { + field: String, + tail: String, + }, + StepIntent, + StepRefHref(usize), + /// `StepMeta::source.extra[field]`, flattened the same way. + StepSourceExtra { + field: String, + tail: String, + }, + StepMetaExtra { + field: String, + tail: String, + }, /// The artifact map key itself. Writing renames the entry. ArtifactKey(String), Raw(String), - /// `structural.extra[field]`, plus an RFC 6901 pointer into it (empty - /// when the field is itself the string). + /// `StructuralChange::extra[field]`, flattened the same way. Extra { artifact: String, field: String, @@ -295,12 +550,60 @@ enum Route { }, } -fn route(at: &str) -> Option { - if at == "/path/base/uri" { - return Some(Route::BaseUri); +/// The document's typed string slots, named so read and write share a parse. +#[derive(Clone, Copy)] +enum DocText { + BaseUri, + BaseBranch, + BaseRef, + MetaTitle, + MetaIntent, +} + +fn route(scope: Scope, at: &str) -> Option { + match scope { + Scope::Document => route_document(at), + Scope::Step => route_step(at), } - if let Some(key) = at.strip_prefix("/meta/") { - return (!key.is_empty() && !key.contains('/')).then(|| Route::MetaExtra(ptr_decode(key))); +} + +fn route_document(at: &str) -> Option { + if let Some(field) = at.strip_prefix("/path/base/") { + return match field { + "uri" => Some(Route::DocText(DocText::BaseUri)), + "branch" => Some(Route::DocText(DocText::BaseBranch)), + "ref" => Some(Route::DocText(DocText::BaseRef)), + _ => None, + }; + } + + let rest = at.strip_prefix("/meta/")?; + match rest { + "title" => return Some(Route::DocText(DocText::MetaTitle)), + "intent" => return Some(Route::DocText(DocText::MetaIntent)), + _ => {} + } + if let Some(i) = ref_href_index(rest) { + return Some(Route::DocRefHref(i)); + } + let (field, tail) = split_field(rest); + Some(Route::DocMetaExtra { field, tail }) +} + +fn route_step(at: &str) -> Option { + if let Some(rest) = at.strip_prefix("/meta/") { + if rest == "intent" { + return Some(Route::StepIntent); + } + if let Some(i) = ref_href_index(rest) { + return Some(Route::StepRefHref(i)); + } + if let Some(rest) = rest.strip_prefix("source/") { + let (field, tail) = split_field(rest); + return Some(Route::StepSourceExtra { field, tail }); + } + let (field, tail) = split_field(rest); + return Some(Route::StepMetaExtra { field, tail }); } let rest = at.strip_prefix("/change/")?; @@ -312,18 +615,37 @@ fn route(at: &str) -> Option { return Some(Route::Raw(artifact)); } - let rest = rest.strip_prefix("structural/extra/")?; - let (field, tail) = match rest.split_once('/') { - Some((field, tail)) => (field, format!("/{tail}")), - None => (rest, String::new()), - }; - (!field.is_empty()).then(|| Route::Extra { + let rest = rest.strip_prefix("structural/")?; + let (field, tail) = split_field(rest); + Some(Route::Extra { artifact, - field: ptr_decode(field), + field, tail, }) } +/// `refs/{i}/href` - the only leaf under `refs` the map names. +fn ref_href_index(rest: &str) -> Option { + let (index, leaf) = rest.strip_prefix("refs/")?.split_once('/')?; + if leaf != "href" { + return None; + } + index.parse().ok() +} + +/// Split a flattened-extras pointer into the map key and an RFC 6901 pointer +/// into that key's value (empty when the value is itself the string). +/// +/// An empty key is a key: `{"": "secret"}` is legal JSON, the residue walk +/// emits a surface for it, and refusing to resolve it here would count that +/// field as scanned while no detector ever saw it. +fn split_field(rest: &str) -> (String, String) { + match rest.split_once('/') { + Some((field, tail)) => (ptr_decode(field), format!("/{tail}")), + None => (ptr_decode(rest), String::new()), + } +} + fn find_step<'a>(path: &'a toolpath::v1::Path, id: &str) -> Option<&'a toolpath::v1::Step> { path.steps.iter().find(|s| s.step.id == id) } @@ -335,60 +657,173 @@ fn find_step_mut<'a>( path.steps.iter_mut().find(|s| s.step.id == id) } +/// The string a `(step, pointer)` pair names, read without the `&mut` a +/// `SurfaceCursor` demands. Plan generation and `verify` only ever read, and +/// nothing that only reads should have to clone the document to do it. +pub fn read_at(path: &toolpath::v1::Path, step: &str, at: &str) -> Option { + resolve(path, scope_of(step), find_step(path, step), at) +} + +/// `read_at` with the step already resolved. A caller reading thousands of +/// surfaces indexes `path.steps` once instead of scanning it per pointer. +/// +/// `None` means the document frame - the same thing an empty step id means to +/// `read_at`. A caller holding a step id that does not resolve must reject it +/// before calling, or a step-relative `/meta/…` silently reads the path's +/// metadata instead. +pub fn read_at_in( + path: &toolpath::v1::Path, + step: Option<&toolpath::v1::Step>, + at: &str, +) -> Option { + let scope = match step { + Some(_) => Scope::Step, + None => Scope::Document, + }; + resolve(path, scope, step, at) +} + +fn resolve( + path: &toolpath::v1::Path, + scope: Scope, + step: Option<&toolpath::v1::Step>, + at: &str, +) -> Option { + match route(scope, at)? { + Route::DocText(field) => doc_text(path, field).cloned(), + Route::DocRefHref(i) => Some(path.meta.as_ref()?.refs.get(i)?.href.clone()), + Route::DocMetaExtra { field, tail } => leaf(&path.meta.as_ref()?.extra, &field, &tail), + Route::StepIntent => step?.meta.as_ref()?.intent.clone(), + Route::StepRefHref(i) => Some(step?.meta.as_ref()?.refs.get(i)?.href.clone()), + Route::StepSourceExtra { field, tail } => { + leaf(&step?.meta.as_ref()?.source.as_ref()?.extra, &field, &tail) + } + Route::StepMetaExtra { field, tail } => leaf(&step?.meta.as_ref()?.extra, &field, &tail), + Route::ArtifactKey(key) => step?.change.contains_key(&key).then_some(key), + Route::Raw(key) => step?.change.get(&key)?.raw.clone(), + Route::Extra { + artifact, + field, + tail, + } => leaf( + &step?.change.get(&artifact)?.structural.as_ref()?.extra, + &field, + &tail, + ), + } +} + +/// The document's typed string slots. A `_mut` twin follows; the two must +/// stay in step, which is why both are one `match` over the same enum. +fn doc_text(path: &toolpath::v1::Path, field: DocText) -> Option<&String> { + match field { + DocText::BaseUri => Some(&path.path.base.as_ref()?.uri), + DocText::BaseBranch => path.path.base.as_ref()?.branch.as_ref(), + DocText::BaseRef => path.path.base.as_ref()?.ref_str.as_ref(), + DocText::MetaTitle => path.meta.as_ref()?.title.as_ref(), + DocText::MetaIntent => path.meta.as_ref()?.intent.as_ref(), + } +} + +fn doc_text_mut(path: &mut toolpath::v1::Path, field: DocText) -> Option<&mut String> { + match field { + DocText::BaseUri => Some(&mut path.path.base.as_mut()?.uri), + DocText::BaseBranch => path.path.base.as_mut()?.branch.as_mut(), + DocText::BaseRef => path.path.base.as_mut()?.ref_str.as_mut(), + DocText::MetaTitle => path.meta.as_mut()?.title.as_mut(), + DocText::MetaIntent => path.meta.as_mut()?.intent.as_mut(), + } +} + +/// A flattened-extras leaf: the map key, then the pointer into its value. +fn leaf(extra: &HashMap, field: &str, tail: &str) -> Option { + let value = extra.get(field)?; + let leaf = if tail.is_empty() { + value + } else { + value.pointer(tail)? + }; + Some(leaf.as_str()?.to_string()) +} + +fn leaf_mut<'a>( + extra: &'a mut HashMap, + field: &str, + tail: &str, +) -> Option<&'a mut String> { + let value = extra.get_mut(field)?; + let leaf = if tail.is_empty() { + value + } else { + value.pointer_mut(tail)? + }; + string_slot(leaf) +} + impl SurfaceCursor<'_> { pub fn read(&self, step: &str, at: &str) -> Option { - match route(at)? { - Route::BaseUri => Some(self.path.path.base.as_ref()?.uri.clone()), - Route::MetaExtra(key) => Some( - self.path - .meta - .as_ref()? - .extra - .get(&key)? - .as_str()? - .to_string(), - ), - Route::ArtifactKey(key) => find_step(self.path, step)? - .change - .contains_key(&key) - .then_some(key), - Route::Raw(key) => find_step(self.path, step)?.change.get(&key)?.raw.clone(), - Route::Extra { - artifact, - field, - tail, - } => { - let value = find_step(self.path, step)? - .change - .get(&artifact)? - .structural - .as_ref()? - .extra - .get(&field)?; - let leaf = if tail.is_empty() { - value - } else { - value.pointer(&tail)? - }; - Some(leaf.as_str()?.to_string()) - } - } + read_at(self.path, step, at) } pub fn write(&mut self, step: &str, at: &str, value: &str) -> Result<()> { let bad = || RedactError::BadPointer(at.to_string()); - match route(at).ok_or_else(bad)? { - Route::BaseUri => { - self.path.path.base.as_mut().ok_or_else(bad)?.uri = value.to_string(); + match route(scope_of(step), at).ok_or_else(bad)? { + Route::DocText(field) => { + *doc_text_mut(self.path, field).ok_or_else(bad)? = value.to_string(); + } + Route::DocRefHref(i) => { + self.path + .meta + .as_mut() + .ok_or_else(bad)? + .refs + .get_mut(i) + .ok_or_else(bad)? + .href = value.to_string(); + } + Route::DocMetaExtra { field, tail } => { + let extra = &mut self.path.meta.as_mut().ok_or_else(bad)?.extra; + *leaf_mut(extra, &field, &tail).ok_or_else(bad)? = value.to_string(); } - Route::MetaExtra(key) => { - let slot = self - .path + Route::StepIntent => { + let meta = find_step_mut(self.path, step) + .ok_or_else(bad)? .meta .as_mut() - .and_then(|m| m.extra.get_mut(&key)) .ok_or_else(bad)?; - *string_slot(slot).ok_or_else(bad)? = value.to_string(); + *meta.intent.as_mut().ok_or_else(bad)? = value.to_string(); + } + Route::StepRefHref(i) => { + find_step_mut(self.path, step) + .ok_or_else(bad)? + .meta + .as_mut() + .ok_or_else(bad)? + .refs + .get_mut(i) + .ok_or_else(bad)? + .href = value.to_string(); + } + Route::StepSourceExtra { field, tail } => { + let extra = &mut find_step_mut(self.path, step) + .ok_or_else(bad)? + .meta + .as_mut() + .ok_or_else(bad)? + .source + .as_mut() + .ok_or_else(bad)? + .extra; + *leaf_mut(extra, &field, &tail).ok_or_else(bad)? = value.to_string(); + } + Route::StepMetaExtra { field, tail } => { + let extra = &mut find_step_mut(self.path, step) + .ok_or_else(bad)? + .meta + .as_mut() + .ok_or_else(bad)? + .extra; + *leaf_mut(extra, &field, &tail).ok_or_else(bad)? = value.to_string(); } Route::ArtifactKey(key) => { let target = find_step_mut(self.path, step).ok_or_else(bad)?; @@ -415,7 +850,7 @@ impl SurfaceCursor<'_> { field, tail, } => { - let slot = find_step_mut(self.path, step) + let extra = &mut find_step_mut(self.path, step) .ok_or_else(bad)? .change .get_mut(&artifact) @@ -423,15 +858,8 @@ impl SurfaceCursor<'_> { .structural .as_mut() .ok_or_else(bad)? - .extra - .get_mut(&field) - .ok_or_else(bad)?; - let leaf = if tail.is_empty() { - slot - } else { - slot.pointer_mut(&tail).ok_or_else(bad)? - }; - *string_slot(leaf).ok_or_else(bad)? = value.to_string(); + .extra; + *leaf_mut(extra, &field, &tail).ok_or_else(bad)? = value.to_string(); } } Ok(()) @@ -453,7 +881,7 @@ pub fn ptr_escape(token: &str) -> String { } /// Decode `~1` before `~0`, or `~01` round-trips wrong (RFC 6901). -fn ptr_decode(token: &str) -> String { +pub(crate) fn ptr_decode(token: &str) -> String { token.replace("~1", "/").replace("~0", "~") } @@ -461,7 +889,10 @@ fn ptr_decode(token: &str) -> String { mod tests { use super::*; use serde_json::json; - use toolpath::v1::{ArtifactChange, Base, Path, PathMeta, Step, StructuralChange}; + use std::collections::BTreeSet; + use toolpath::v1::{ + ArtifactChange, Base, Path, PathMeta, Ref, Step, StepMeta, StructuralChange, VcsSource, + }; fn object(value: Value) -> HashMap { match value { @@ -522,7 +953,11 @@ mod tests { "category": "command", "result": {"content": "configured", "is_error": false} }], - "environment": {"working_dir": "/Users/alex/work/repo"}, + "environment": { + "working_dir": "/Users/alex/work/repo", + "vcs_branch": "evan/redact", + "vcs_revision": "7a7c366" + }, "token_usage": {"input_tokens": 10, "output_tokens": 2} }), )]) @@ -583,6 +1018,10 @@ mod tests { "raw_diff": "--- a/deploy.sh\n+++ b/deploy.sh\n@@ -1 +1 @@\n-old\n+new\n", "before": "old\n", "after": "new\n" + }, { + // Only `path`: no typed field at all, so this + // entry is the whole test for the residue walk. + "path": "/srv/acme-internal/rotate.sh" }] }] }] @@ -608,7 +1047,7 @@ mod tests { } /// Every branch of the map in one document, for the whole-surface - /// read/write sweeps. + /// read/write sweeps and the leaf-coverage proof. fn fixture_rich() -> Path { let mut path = path_of(vec![ step_with( @@ -626,6 +1065,7 @@ mod tests { json!({ "before": "a\n", "after": "b\n", + "rename_to": "~/acme-internal-notes.md", "edits": [{"old_string": "a", "new_string": "b", "replace_all": false}] }), ), @@ -635,9 +1075,38 @@ mod tests { fixture_with_delegation().steps.remove(0), fixture_unknown_change_type().steps.remove(0), ]); - path.path.base = Some(Base::vcs("https://alex:tok@github.com/o/r", "abc123")); + path.steps[0].meta = Some(StepMeta { + intent: Some("rotate the deploy credentials".into()), + source: Some(VcsSource { + vcs_type: "git".into(), + revision: "abc123".into(), + change_id: None, + extra: object(json!({"author_email": "alex@acme-internal.example"})), + }), + refs: vec![Ref { + rel: "pull-request".into(), + href: "https://github.com/o/r/pull/42".into(), + }], + extra: object(json!({"note": "cherry-picked from the release branch"})), + ..StepMeta::default() + }); + path.path.base = Some(Base { + uri: "https://alex:tok@github.com/o/r".into(), + ref_str: Some("abc123".into()), + branch: Some("evan/redact".into()), + }); path.meta = Some(PathMeta { - extra: object(json!({"vcs_remote": "https://alex:tok@github.com/o/r.git"})), + title: Some("redaction field map".into()), + intent: Some("close the coverage gaps".into()), + refs: vec![Ref { + rel: "self".into(), + href: "https://pathbase.dev/o/r/redact".into(), + }], + extra: object(json!({ + "vcs_remote": "https://alex:tok@github.com/o/r.git", + // Byte-identical to a `step.change` key the map scans. + "files_changed": ["~/notes.md", "/srv/acme-internal/rotate.sh"] + })), ..PathMeta::default() }); path @@ -647,6 +1116,127 @@ mod tests { surfaces(path).into_iter().map(|s| s.at).collect() } + /// Every non-empty string leaf in `value`, named by absolute pointer. + /// Empty ones are excluded because `push` excludes them: there is nothing + /// in an empty string for a detector to span. + fn string_leaves(value: &Value, at: String, out: &mut Vec) { + match value { + Value::String(s) => { + if !s.is_empty() { + out.push(at); + } + } + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + string_leaves(item, format!("{at}/{i}"), out); + } + } + Value::Object(map) => { + for (key, item) in map { + string_leaves(item, format!("{at}/{}", ptr_escape(key)), out); + } + } + _ => {} + } + } + + /// A surface's pointer rebased on the document root - where whoever reads + /// the audit record will actually try to follow it. + fn absolute(path: &Path, s: &Surface) -> String { + match path.steps.iter().position(|st| st.step.id == s.step) { + Some(i) => format!("/steps/{i}{}", s.at), + None => s.at.clone(), + } + } + + /// The map is a closed list: a field it does not name is never scanned, + /// so a secret there ships and nothing notices. This walks the other + /// direction - from the document to the map - so a new field in + /// `toolpath` or `toolpath-convo` cannot quietly go unscanned. Every + /// exclusion is named below with the reason it is not a candidate. + #[test] + fn every_string_leaf_in_a_derived_document_is_surfaced_exactly_once() { + const EXPECTED_OUT_OF_SCOPE: &[(&str, &str)] = &[ + ("/path/id", "document identity; plans are keyed on it"), + ("/path/head", "names a step id, not content"), + ("/steps/0/step/id", "step identity; the DAG is keyed on it"), + ("/steps/1/step/id", "step identity; the DAG is keyed on it"), + ("/steps/2/step/id", "step identity; the DAG is keyed on it"), + ("/steps/0/step/actor", "`type:name`, a closed vocabulary"), + ("/steps/1/step/actor", "`type:name`, a closed vocabulary"), + ("/steps/2/step/actor", "`type:name`, a closed vocabulary"), + ("/steps/0/step/timestamp", "machine-generated ISO 8601"), + ("/steps/1/step/timestamp", "machine-generated ISO 8601"), + ("/steps/2/step/timestamp", "machine-generated ISO 8601"), + ( + "/steps/0/change/claude:~1~1sess-abc/structural/type", + "change type: a vocabulary this format defines", + ), + ( + "/steps/0/change/~0~1notes.md/structural/type", + "change type: a vocabulary this format defines", + ), + ( + "/steps/1/change/claude:~1~1sess-abc/structural/type", + "change type: a vocabulary this format defines", + ), + ( + "/steps/2/change/claude:~1~1sess-abc/structural/type", + "change type: a vocabulary this format defines", + ), + ("/steps/0/meta/source/type", "VCS name, a closed vocabulary"), + ( + "/steps/0/meta/source/revision", + "a commit hash the VCS assigned", + ), + ("/steps/0/meta/refs/0/rel", "a link relation name"), + ("/meta/refs/0/rel", "a link relation name"), + ]; + + let p = fixture_rich(); + let doc = serde_json::to_value(&p).unwrap(); + + let mut found = Vec::new(); + string_leaves(&doc, String::new(), &mut found); + let leaves: BTreeSet = found.into_iter().collect(); + + let scanned: Vec = surfaces(&p).iter().map(|s| absolute(&p, s)).collect(); + let unique: BTreeSet = scanned.iter().cloned().collect(); + assert_eq!( + unique.len(), + scanned.len(), + "a field scanned twice is a finding reported twice: {scanned:#?}" + ); + + let allowed: BTreeSet<&str> = EXPECTED_OUT_OF_SCOPE.iter().map(|(at, _)| *at).collect(); + for at in &allowed { + assert!( + leaves.contains(*at), + "allowlist entry names nothing in the fixture: {at}" + ); + } + + let missed: Vec<&String> = leaves + .difference(&unique) + .filter(|a| !allowed.contains(a.as_str())) + .collect(); + assert!( + missed.is_empty(), + "string leaves no detector will ever see: {missed:#?}" + ); + + // An artifact key is the one surface that names a map *key*, so it + // resolves to the object beneath it rather than to a string. + let phantom: Vec<&String> = unique + .difference(&leaves) + .filter(|a| !doc.pointer(a).is_some_and(Value::is_object)) + .collect(); + assert!( + phantom.is_empty(), + "surfaces whose pointers resolve to nothing: {phantom:#?}" + ); + } + #[test] fn ptr_escape_handles_urls_and_tildes() { assert_eq!(ptr_escape("claude://sess-abc"), "claude:~1~1sess-abc"); @@ -669,11 +1259,8 @@ mod tests { let p = fixture_conversation_append(); let all = surfaces(&p); let ats: Vec<&str> = all.iter().map(|s| s.at.as_str()).collect(); - assert!(ats.iter().any(|a| a.ends_with("/structural/extra/text"))); - assert!( - ats.iter() - .any(|a| a.ends_with("/structural/extra/thinking")) - ); + assert!(ats.iter().any(|a| a.ends_with("/structural/text"))); + assert!(ats.iter().any(|a| a.ends_with("/structural/thinking"))); assert!(ats.iter().any(|a| a.contains("/tool_uses/0/input"))); assert!( ats.iter() @@ -720,7 +1307,7 @@ mod tests { assert!( surfaces(&p) .iter() - .any(|s| s.at.ends_with("/structural/extra/text")) + .any(|s| s.at.ends_with("/structural/text")) ); } @@ -780,8 +1367,168 @@ mod tests { )]) }; assert_eq!(surfaces(&build()), surfaces(&build())); - let once = build(); - assert_eq!(surfaces(&once), surfaces(&once)); + } + + /// The emission order in full, over the fixture that exercises every arm. + /// Finding ids are positional over this list and a regenerated plan is + /// compared byte for byte, so the order is as much a contract as the + /// pointers are. One literal pins all of it at once: steps in document + /// order, artifacts sorted by key, each artifact's own key after + /// everything beneath it, and every typed field ahead of the residue + /// walk that follows it. + #[test] + fn the_emission_order_is_pinned() { + const GOLDEN: &[(&str, &str)] = &[ + ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/text"), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/thinking", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/tool_uses/0/input/command", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/tool_uses/0/result/content", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/tool_uses/0/category", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/tool_uses/0/id", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/tool_uses/0/name", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/environment/working_dir", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/environment/vcs_branch", + ), + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/environment/vcs_revision", + ), + ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/role"), + ("turn-0f3a", "/change/claude:~1~1sess-abc"), + ("turn-0f3a", "/change/~0~1notes.md/raw"), + ("turn-0f3a", "/change/~0~1notes.md/structural/before"), + ("turn-0f3a", "/change/~0~1notes.md/structural/after"), + ( + "turn-0f3a", + "/change/~0~1notes.md/structural/edits/0/new_string", + ), + ( + "turn-0f3a", + "/change/~0~1notes.md/structural/edits/0/old_string", + ), + ("turn-0f3a", "/change/~0~1notes.md/structural/rename_to"), + ("turn-0f3a", "/change/~0~1notes.md"), + ("turn-0f3a", "/meta/intent"), + ("turn-0f3a", "/meta/refs/0/href"), + ("turn-0f3a", "/meta/source/author_email"), + ("turn-0f3a", "/meta/note"), + ("turn-deleg", "/change/claude:~1~1sess-abc/structural/text"), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/prompt", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/result", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/text", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/tool_uses/0/input/file_path", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/tool_uses/0/result/content", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/tool_uses/0/id", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/tool_uses/0/name", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/file_mutations/0/raw_diff", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/file_mutations/0/before", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/file_mutations/0/after", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/file_mutations/0/path", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/file_mutations/1/path", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/id", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/role", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/turns/0/timestamp", + ), + ( + "turn-deleg", + "/change/claude:~1~1sess-abc/structural/delegations/0/agent_id", + ), + ("turn-deleg", "/change/claude:~1~1sess-abc/structural/role"), + ("turn-deleg", "/change/claude:~1~1sess-abc"), + ("evt-1", "/change/claude:~1~1sess-abc/structural/data/a~1b"), + ( + "evt-1", + "/change/claude:~1~1sess-abc/structural/data/nested/0", + ), + ("evt-1", "/change/claude:~1~1sess-abc/structural/event_type"), + ("evt-1", "/change/claude:~1~1sess-abc"), + ("", "/path/base/uri"), + ("", "/path/base/branch"), + ("", "/path/base/ref"), + ("", "/meta/title"), + ("", "/meta/intent"), + ("", "/meta/refs/0/href"), + ("", "/meta/vcs_remote"), + ("", "/meta/files_changed/0"), + ("", "/meta/files_changed/1"), + ]; + + let actual: Vec<(String, String)> = surfaces(&fixture_rich()) + .into_iter() + .map(|s| (s.step, s.at)) + .collect(); + let expected: Vec<(String, String)> = GOLDEN + .iter() + .map(|(step, at)| (step.to_string(), at.to_string())) + .collect(); + assert_eq!(actual, expected); } #[test] @@ -807,8 +1554,11 @@ mod tests { json!({"text": "", "thinking": "kept"}), )]); let ats = ats(&p); - assert!(!ats.iter().any(|a| a.ends_with("/extra/text")), "{ats:?}"); - assert!(ats.iter().any(|a| a.ends_with("/extra/thinking"))); + assert!( + !ats.iter().any(|a| a.ends_with("/structural/text")), + "{ats:?}" + ); + assert!(ats.iter().any(|a| a.ends_with("/structural/thinking"))); } #[test] @@ -820,7 +1570,7 @@ mod tests { }]}), )]); let ats = ats(&p); - let leaf = "/change/claude:~1~1sess-abc/structural/extra/tool_uses/0/input"; + let leaf = "/change/claude:~1~1sess-abc/structural/tool_uses/0/input"; assert!(ats.contains(&format!("{leaf}/env/AWS_KEY")), "{ats:?}"); assert!(ats.contains(&format!("{leaf}/argv/0"))); assert!(!ats.iter().any(|a| a.ends_with("/retries"))); @@ -829,7 +1579,7 @@ mod tests { #[test] fn blind_walk_escapes_object_keys() { let p = fixture_unknown_change_type(); - let at = "/change/claude:~1~1sess-abc/structural/extra/data/a~1b"; + let at = "/change/claude:~1~1sess-abc/structural/data/a~1b"; assert!(ats(&p).contains(&at.to_string()), "{:?}", ats(&p)); let mut p = p; @@ -844,7 +1594,7 @@ mod tests { let mut p = fixture_rich(); let mut c = SurfaceCursor { path: &mut p }; assert_eq!( - c.read("turn-0f3a", "/change/~0~1notes.md/structural/extra/before") + c.read("turn-0f3a", "/change/~0~1notes.md/structural/before") .as_deref(), Some("a\n") ); @@ -867,19 +1617,108 @@ mod tests { #[test] fn every_surface_reads_back() { - let mut p = fixture_rich(); + // Through `read_at`, not a cursor: plan generation reads every + // surface in the document and has no business cloning one to do it. + let p = fixture_rich(); let all = surfaces(&p); - assert!(all.len() > 15, "fixture is too thin: {}", all.len()); - let c = SurfaceCursor { path: &mut p }; + assert!(all.len() > 40, "fixture is too thin: {}", all.len()); for s in &all { assert!( - c.read(&s.step, &s.at).is_some(), + read_at(&p, &s.step, &s.at).is_some(), "surface does not resolve: {}", s.at ); } } + /// The fields the closed map used to leave out. Each carried a real value + /// in a captured session and none was ever handed to a detector. + #[test] + fn the_fields_the_closed_map_used_to_miss_are_scanned() { + let p = fixture_rich(); + let named: BTreeSet<(String, String)> = + surfaces(&p).into_iter().map(|s| (s.step, s.at)).collect(); + let convo = "/change/claude:~1~1sess-abc/structural"; + for (step, at) in [ + // Sits beside the `working_dir` the map already scanned; 13 + // occurrences of a branch name in one real 753-step session. + ("turn-0f3a", format!("{convo}/environment/vcs_branch")), + ("turn-0f3a", format!("{convo}/environment/vcs_revision")), + // A path, exactly like the one that became the artifact key. + ( + "turn-0f3a", + "/change/~0~1notes.md/structural/rename_to".into(), + ), + // `mcp____`, and the server half is user config. + ("turn-0f3a", format!("{convo}/tool_uses/0/name")), + // A delegated mutation carrying only `path` had no typed field + // at all, so it produced zero surfaces - while the identical + // value at the top level was scanned. + ( + "turn-deleg", + format!("{convo}/delegations/0/turns/0/file_mutations/1/path"), + ), + // The git commit message, and the PR body on a GitHub import. + ("turn-0f3a", "/meta/intent".into()), + ("turn-0f3a", "/meta/source/author_email".into()), + ("turn-0f3a", "/meta/refs/0/href".into()), + ("turn-0f3a", "/meta/note".into()), + ("", "/path/base/branch".into()), + ("", "/path/base/ref".into()), + ("", "/meta/title".into()), + ("", "/meta/intent".into()), + ("", "/meta/refs/0/href".into()), + // Byte-identical to a `step.change` key the map does scan, so + // redacting only the key leaves the original in the document. + ("", "/meta/files_changed/0".into()), + ] { + assert!( + named.contains(&(step.to_string(), at.clone())), + "not scanned: {step} {at}" + ); + assert!( + read_at(&p, step, &at).is_some(), + "scanned but unreadable: {step} {at}" + ); + } + } + + #[test] + fn an_empty_extra_key_still_resolves() { + // `{"": "…"}` is legal JSON and the walk emits a surface for it, so + // the pass counts that field as scanned. A route that refused it + // would make the count a lie: no detector would ever see the value. + let mut p = path_of(vec![append_step( + "turn-empty-key", + json!({"": "AKIAIOSFODNN7EXAMPLE"}), + )]); + let at = "/change/claude:~1~1sess-abc/structural/"; + assert!(ats(&p).contains(&at.to_string()), "{:?}", ats(&p)); + + let mut c = SurfaceCursor { path: &mut p }; + assert_eq!( + c.read("turn-empty-key", at).as_deref(), + Some("AKIAIOSFODNN7EXAMPLE") + ); + c.write("turn-empty-key", at, "x").unwrap(); + assert_eq!(c.read("turn-empty-key", at).as_deref(), Some("x")); + } + + #[test] + fn renaming_an_artifact_key_invalidates_pointers_beneath_it() { + // The one place read and write stop agreeing, and the whole reason + // `surfaces()` emits an artifact's key after everything under it. + let mut p = fixture_file_write(); + let mut c = SurfaceCursor { path: &mut p }; + let raw = "/change/src~1config.rs/raw"; + assert!(c.read("turn-9c21", raw).is_some()); + + c.write("turn-9c21", "/change/src~1config.rs", "redacted.rs") + .unwrap(); + assert_eq!(c.read("turn-9c21", raw), None); + assert!(c.read("turn-9c21", "/change/redacted.rs/raw").is_some()); + } + #[test] fn every_surface_is_writable_in_emitted_order() { let mut p = fixture_rich(); @@ -898,26 +1737,38 @@ mod tests { let mut c = SurfaceCursor { path: &mut p }; for (step, at) in [ ("turn-0f3a", "/change/nope~1missing.rs/raw"), + ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/nope"), ( "turn-0f3a", - "/change/claude:~1~1sess-abc/structural/extra/nope", - ), - ( - "turn-0f3a", - "/change/claude:~1~1sess-abc/structural/extra/tool_uses/9/input", + "/change/claude:~1~1sess-abc/structural/tool_uses/9/input", ), // `token_usage` is an object, not a string leaf. ( "turn-0f3a", - "/change/claude:~1~1sess-abc/structural/extra/token_usage", + "/change/claude:~1~1sess-abc/structural/token_usage", ), ( "no-such-step", - "/change/claude:~1~1sess-abc/structural/extra/text", + "/change/claude:~1~1sess-abc/structural/text", ), ("turn-0f3a", "/step/actor"), - ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/text"), + // The `extra` segment used to be part of the pointer, and it + // named nothing: `StructuralChange::extra` is flattened. + ( + "turn-0f3a", + "/change/claude:~1~1sess-abc/structural/extra/text", + ), + // `type` is `structural`'s one typed sibling, and it is not in + // the extras map the pointer resolves against. + ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/type"), ("", "/meta/not_present"), + ("", "/path/base/nope"), + ("", "/meta/refs/9/href"), + ("", "/meta/refs/0/rel"), + // A step-relative `/meta/…` must not fall through to the path's + // metadata, whether the step is missing or just has none. + ("no-such-step", "/meta/intent"), + ("turn-deleg", "/meta/intent"), ] { assert!( matches!(c.write(step, at, "x"), Err(RedactError::BadPointer(_))), @@ -941,7 +1792,7 @@ mod tests { #[test] fn delegated_turns_surface_their_own_shapes() { let p = fixture_with_delegation(); - let base = "/change/claude:~1~1sess-abc/structural/extra/delegations/0"; + let base = "/change/claude:~1~1sess-abc/structural/delegations/0"; let by_at: HashMap = surfaces(&p).into_iter().map(|s| (s.at, s.shape)).collect(); for (at, shape) in [ @@ -972,7 +1823,7 @@ mod tests { #[test] fn file_write_edits_surface_both_sides() { let p = fixture_rich(); - let base = "/change/~0~1notes.md/structural/extra/edits/0"; + let base = "/change/~0~1notes.md/structural/edits/0"; let ats = ats(&p); assert!(ats.contains(&format!("{base}/old_string")), "{ats:?}"); assert!(ats.contains(&format!("{base}/new_string"))); diff --git a/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md b/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md index 5ce8329a..6a4cdd7c 100644 --- a/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md +++ b/docs/superpowers/notes/2026-07-30-redaction-known-gaps.md @@ -65,14 +65,24 @@ un-redacts everything on the next sync. Version skew is enough to trigger it: `Transform` is a plain string enum with no unknown-variant fallback. -## 6. Idempotence is unproven for `hash` and `partial` - -`internal::mask_existing_markers` blanks `[REDACTED:…]` and runs of `█` -before any rule scans, which is what makes redaction reach a fixed point. -It does **not** cover `Transform::Hash` output (bare 6 hex characters) or -`Transform::Partial` output (`head…tail`). The idempotence test passes -because its scanner uses prefixed self-delimiting formats that no -transform output can reconstitute, so the gap is never exercised. +## 6. `hash` is not idempotent + +`internal::marker_re` recognises `[REDACTED:…]`, runs of `█`, and +`head…tail`, and any finding whose span intersects one is dropped before +scoring. That makes `marker`, `remove`, `mask`, and `partial` reach a +fixed point. + +`Transform::Hash` emits bare 6 hex characters, which no scan can +distinguish from a fresh credential. Each pass re-redacts the previous +pass's output, rotating the fingerprint and appending an audit entry. It +matters in production because sync replays `apply` on every re-derive of +a redacted session, so a `hash`-mode redaction drifts on every +`p cache sync`. + +Pinned by `apply::tests::idempotent_across_all_transforms`, which asserts +the instability so it cannot change unnoticed. Fixing it needs an +envelope format for hashed values, which is a design decision nobody has +made. ## 7. `share` uploads the un-redacted derivation From d6f2d438736fe725a50e7acd3be66596a43f66ff Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Fri, 31 Jul 2026 02:45:57 -0400 Subject: [PATCH 8/9] fix(redact): scan actor identities, and prove the map's exclusions The field map excluded `actors` on the stated grounds that redaction "drops wholesale rather than rewrites" it. That is true of `signatures`, which `guard_signatures` clears, and false of `actors`, which nothing drops. On a git-derived document the committer's real name and email address shipped unscanned. Scanning them is the only thing that makes the doc comment true. Surfaced: actor `name`, `identities[i]`, and `keys[i].href`. Still excluded, now with reasons and negative assertions: the actor map's own keys, because `step.actor` names them and is not rewritten, so renaming one detaches every step from its actor; `provider`/`model`, which are harness and API vocabularies; and `keys[i].fingerprint`, a digest of a public key, where rewriting hides nothing and breaks any signature that survived. The completeness test was passing by absence: `fixture_rich` omitted `step.parents`, `path.graph_ref`, `PathMeta.kind`/`source`, `VcsSource.change_id`, and both `signatures` vectors, so five of the exclusions were never proven to be decisions rather than oversights. The fixture now carries every string-bearing field of the `Path` type tree and the allowlist grew from 20 entries to 47. Its phantom check also accepted any pointer resolving to an object, which would have swallowed a surface aimed at a subtree; it now requires the artifact-key shape specifically. Every remaining miss on a 753-step session, two smaller ones, and a git-derived document is an allowlisted id, enum, or type. `Graph.meta` is still never redacted: `cmd_redact` iterates `doc.paths`, so a graph's own title, intent, refs, and actors are unreachable from any call on its members, and every cached document is a `Graph`. Recorded as a stated non-goal on `surfaces()` rather than left silent. Co-Authored-By: Claude Opus 5 (1M context) --- crates/toolpath-redact/src/surface.rs | 367 +++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 8 deletions(-) diff --git a/crates/toolpath-redact/src/surface.rs b/crates/toolpath-redact/src/surface.rs index 37471505..3b4d3b5d 100644 --- a/crates/toolpath-redact/src/surface.rs +++ b/crates/toolpath-redact/src/surface.rs @@ -47,9 +47,20 @@ pub struct Surface { /// vocabularies this format defines, with no room for a user's string. /// - `VcsSource::{type,revision,change_id}` and `Ref::rel` - identifiers the /// VCS or the format assigns, not the human. -/// - `{Step,Path}Meta::{actors,signatures}` - identity and integrity -/// material, which redaction drops wholesale rather than rewrites. +/// - `{Step,Path}Meta::signatures` - integrity material over the content +/// this pass rewrites, which `apply` drops wholesale rather than rewrites. +/// - the `actors` map's own keys, and each definition's `provider`, `model`, +/// and `keys[i].{type,fingerprint}`. A key is the string `step.actor` +/// names, and `step.actor` is not rewritten, so renaming one would detach +/// every step from its actor. `provider` and `model` are the harness and +/// API vocabularies the derivation reports. A key fingerprint is a digest +/// of a public key: rewriting it hides nothing and leaves any signature +/// that survived the pass unverifiable. The rest of a definition *is* +/// scanned - see `actor_surfaces`. /// - every non-string leaf - a detector has nothing to span in a number. +/// - anything outside the `Path` handed in. A `Graph`'s own `meta` carries a +/// title, an intent, refs and extras that no call on its member paths can +/// reach; redacting a graph needs a pass of its own. pub fn surfaces(path: &toolpath::v1::Path) -> Vec { let mut out = Vec::new(); for step in &path.steps { @@ -139,6 +150,7 @@ fn path_meta_surfaces(out: &mut Vec, meta: &toolpath::v1::PathMeta) { &r.href, ); } + actor_surfaces(out, "", meta.actors.as_ref()); if let Some(v) = meta.extra.get("vcs_remote") { walk_json(out, "", "/meta/vcs_remote", v, FieldShape::Uri); } @@ -174,12 +186,65 @@ fn step_meta_surfaces(out: &mut Vec, step: &str, meta: &toolpath::v1::S &r.href, ); } + actor_surfaces(out, step, meta.actors.as_ref()); if let Some(src) = &meta.source { residue(out, step, "/meta/source", &src.extra, &[]); } residue(out, step, "/meta", &meta.extra, &[]); } +/// The free text on an actor definition. The rest of the struct is a +/// vocabulary the tooling assigned, but these are not: `toolpath-git` writes +/// the committer's real name into `name` and their email address into +/// `identities[i].id`, and a `keys[i].href` is a URL like any other. +/// +/// Sorted by actor key: `actors` is a `HashMap` and emission order is part of +/// the contract. +fn actor_surfaces( + out: &mut Vec, + step: &str, + actors: Option<&HashMap>, +) { + let Some(actors) = actors else { return }; + let mut keys: Vec<&String> = actors.keys().collect(); + keys.sort(); + for key in keys { + let def = &actors[key]; + let at = format!("/meta/actors/{}", ptr_escape(key)); + if let Some(v) = &def.name { + push(out, step, format!("{at}/name"), FieldShape::Prose, v); + } + for (i, id) in def.identities.iter().enumerate() { + let at = format!("{at}/identities/{i}"); + push( + out, + step, + format!("{at}/system"), + FieldShape::OpaqueJson, + &id.system, + ); + push( + out, + step, + format!("{at}/id"), + FieldShape::OpaqueJson, + &id.id, + ); + } + for (i, k) in def.keys.iter().enumerate() { + if let Some(href) = &k.href { + push( + out, + step, + format!("{at}/keys/{i}/href"), + FieldShape::Uri, + href, + ); + } + } + } +} + fn push(out: &mut Vec, step: &str, at: String, shape: FieldShape, text: &str) { if text.is_empty() { return; @@ -528,8 +593,17 @@ enum Route { field: String, tail: String, }, + /// `PathMeta::actors[key]`, and which of its strings. + DocActor { + key: String, + field: ActorField, + }, StepIntent, StepRefHref(usize), + StepActor { + key: String, + field: ActorField, + }, /// `StepMeta::source.extra[field]`, flattened the same way. StepSourceExtra { field: String, @@ -550,6 +624,17 @@ enum Route { }, } +/// Which string on an `ActorDefinition` a pointer names. The fields left out +/// are the ones `surfaces()` does not emit, so a pointer at one of them +/// resolves to nothing - which is what an unnamed field should do. +#[derive(Clone, Copy)] +enum ActorField { + Name, + IdentitySystem(usize), + IdentityId(usize), + KeyHref(usize), +} + /// The document's typed string slots, named so read and write share a parse. #[derive(Clone, Copy)] enum DocText { @@ -586,6 +671,9 @@ fn route_document(at: &str) -> Option { if let Some(i) = ref_href_index(rest) { return Some(Route::DocRefHref(i)); } + if let Some((key, field)) = actor_route(rest) { + return Some(Route::DocActor { key, field }); + } let (field, tail) = split_field(rest); Some(Route::DocMetaExtra { field, tail }) } @@ -598,6 +686,9 @@ fn route_step(at: &str) -> Option { if let Some(i) = ref_href_index(rest) { return Some(Route::StepRefHref(i)); } + if let Some((key, field)) = actor_route(rest) { + return Some(Route::StepActor { key, field }); + } if let Some(rest) = rest.strip_prefix("source/") { let (field, tail) = split_field(rest); return Some(Route::StepSourceExtra { field, tail }); @@ -624,6 +715,27 @@ fn route_step(at: &str) -> Option { }) } +/// `actors/{key}/…` - the strings the map names on an actor definition. An +/// actor key is pointer-escaped, so it never holds a literal `/` and the +/// first split always lands on the key boundary. +fn actor_route(rest: &str) -> Option<(String, ActorField)> { + let (key, rest) = rest.strip_prefix("actors/")?.split_once('/')?; + let field = if rest == "name" { + ActorField::Name + } else { + let (list, rest) = rest.split_once('/')?; + let (index, leaf) = rest.split_once('/')?; + let i = index.parse().ok()?; + match (list, leaf) { + ("identities", "system") => ActorField::IdentitySystem(i), + ("identities", "id") => ActorField::IdentityId(i), + ("keys", "href") => ActorField::KeyHref(i), + _ => return None, + } + }; + Some((ptr_decode(key), field)) +} + /// `refs/{i}/href` - the only leaf under `refs` the map names. fn ref_href_index(rest: &str) -> Option { let (index, leaf) = rest.strip_prefix("refs/")?.split_once('/')?; @@ -693,8 +805,14 @@ fn resolve( Route::DocText(field) => doc_text(path, field).cloned(), Route::DocRefHref(i) => Some(path.meta.as_ref()?.refs.get(i)?.href.clone()), Route::DocMetaExtra { field, tail } => leaf(&path.meta.as_ref()?.extra, &field, &tail), + Route::DocActor { key, field } => { + actor_text(path.meta.as_ref()?.actors.as_ref()?.get(&key)?, field).cloned() + } Route::StepIntent => step?.meta.as_ref()?.intent.clone(), Route::StepRefHref(i) => Some(step?.meta.as_ref()?.refs.get(i)?.href.clone()), + Route::StepActor { key, field } => { + actor_text(step?.meta.as_ref()?.actors.as_ref()?.get(&key)?, field).cloned() + } Route::StepSourceExtra { field, tail } => { leaf(&step?.meta.as_ref()?.source.as_ref()?.extra, &field, &tail) } @@ -735,6 +853,29 @@ fn doc_text_mut(path: &mut toolpath::v1::Path, field: DocText) -> Option<&mut St } } +/// An actor's named string. A `_mut` twin follows; the two must stay in step, +/// which is why both are one `match` over the same enum. +fn actor_text(def: &toolpath::v1::ActorDefinition, field: ActorField) -> Option<&String> { + match field { + ActorField::Name => def.name.as_ref(), + ActorField::IdentitySystem(i) => Some(&def.identities.get(i)?.system), + ActorField::IdentityId(i) => Some(&def.identities.get(i)?.id), + ActorField::KeyHref(i) => def.keys.get(i)?.href.as_ref(), + } +} + +fn actor_text_mut( + def: &mut toolpath::v1::ActorDefinition, + field: ActorField, +) -> Option<&mut String> { + match field { + ActorField::Name => def.name.as_mut(), + ActorField::IdentitySystem(i) => Some(&mut def.identities.get_mut(i)?.system), + ActorField::IdentityId(i) => Some(&mut def.identities.get_mut(i)?.id), + ActorField::KeyHref(i) => def.keys.get_mut(i)?.href.as_mut(), + } +} + /// A flattened-extras leaf: the map key, then the pointer into its value. fn leaf(extra: &HashMap, field: &str, tail: &str) -> Option { let value = extra.get(field)?; @@ -785,6 +926,30 @@ impl SurfaceCursor<'_> { let extra = &mut self.path.meta.as_mut().ok_or_else(bad)?.extra; *leaf_mut(extra, &field, &tail).ok_or_else(bad)? = value.to_string(); } + Route::DocActor { key, field } => { + let actors = self + .path + .meta + .as_mut() + .ok_or_else(bad)? + .actors + .as_mut() + .ok_or_else(bad)?; + let def = actors.get_mut(&key).ok_or_else(bad)?; + *actor_text_mut(def, field).ok_or_else(bad)? = value.to_string(); + } + Route::StepActor { key, field } => { + let actors = find_step_mut(self.path, step) + .ok_or_else(bad)? + .meta + .as_mut() + .ok_or_else(bad)? + .actors + .as_mut() + .ok_or_else(bad)?; + let def = actors.get_mut(&key).ok_or_else(bad)?; + *actor_text_mut(def, field).ok_or_else(bad)? = value.to_string(); + } Route::StepIntent => { let meta = find_step_mut(self.path, step) .ok_or_else(bad)? @@ -891,9 +1056,42 @@ mod tests { use serde_json::json; use std::collections::BTreeSet; use toolpath::v1::{ - ArtifactChange, Base, Path, PathMeta, Ref, Step, StepMeta, StructuralChange, VcsSource, + ActorDefinition, ArtifactChange, Base, Identity, Key, Path, PathMeta, Ref, Step, StepMeta, + StructuralChange, VcsSource, }; + /// A `human:` actor as `toolpath-git` writes one: the committer's real + /// name and their email address, neither of which is a vocabulary. + fn actors() -> HashMap { + HashMap::from([ + ( + "human:alex".to_string(), + ActorDefinition { + name: Some("Alex Mercer".into()), + identities: vec![Identity { + system: "email".into(), + id: "alex@acme-internal.example".into(), + }], + keys: vec![Key { + key_type: "ssh-ed25519".into(), + fingerprint: "SHA256:abc".into(), + href: Some("https://alex:tok@keys.acme-internal.example/alex.pub".into()), + }], + ..ActorDefinition::default() + }, + ), + ( + "agent:claude-opus-5".to_string(), + ActorDefinition { + name: Some("claude-opus-5".into()), + provider: Some("claude-code".into()), + model: Some("claude-opus-5-20260101".into()), + ..ActorDefinition::default() + }, + ), + ]) + } + fn object(value: Value) -> HashMap { match value { Value::Object(map) => map.into_iter().collect(), @@ -908,6 +1106,16 @@ mod tests { path } + fn signature() -> toolpath::v1::Signature { + toolpath::v1::Signature { + signer: "human:alex".into(), + key: "SHA256:abc".into(), + scope: "path".into(), + sig: "MEUCIQDexample".into(), + timestamp: Some("2026-07-30T10:00:00Z".into()), + } + } + fn change(change_type: &str, raw: Option<&str>, extra: Value) -> ArtifactChange { ArtifactChange { raw: raw.map(str::to_string), @@ -1075,12 +1283,17 @@ mod tests { fixture_with_delegation().steps.remove(0), fixture_unknown_change_type().steps.remove(0), ]); + // A derived document is a chain, and `parents` is a list of strings + // the map declines to name. Absent from the fixture, that exclusion + // would pass the coverage proof by not being there. + path.steps[1].step.parents = vec!["turn-0f3a".into()]; + path.steps[2].step.parents = vec!["turn-deleg".into()]; path.steps[0].meta = Some(StepMeta { intent: Some("rotate the deploy credentials".into()), source: Some(VcsSource { vcs_type: "git".into(), revision: "abc123".into(), - change_id: None, + change_id: Some("I8473b95934b5732ac55d26311a706c9c2bde9940".into()), extra: object(json!({"author_email": "alex@acme-internal.example"})), }), refs: vec![Ref { @@ -1088,15 +1301,19 @@ mod tests { href: "https://github.com/o/r/pull/42".into(), }], extra: object(json!({"note": "cherry-picked from the release branch"})), - ..StepMeta::default() + actors: Some(actors()), + signatures: vec![signature()], }); path.path.base = Some(Base { uri: "https://alex:tok@github.com/o/r".into(), ref_str: Some("abc123".into()), branch: Some("evan/redact".into()), }); + path.path.graph_ref = Some("toolpath://archive/release-v2".into()); path.meta = Some(PathMeta { title: Some("redaction field map".into()), + kind: Some(toolpath::v1::PATH_KIND_AGENT_CODING_SESSION.into()), + source: Some("claude-code".into()), intent: Some("close the coverage gaps".into()), refs: vec![Ref { rel: "self".into(), @@ -1107,7 +1324,8 @@ mod tests { // Byte-identical to a `step.change` key the map scans. "files_changed": ["~/notes.md", "/srv/acme-internal/rotate.sh"] })), - ..PathMeta::default() + actors: Some(actors()), + signatures: vec![signature()], }); path } @@ -1154,11 +1372,23 @@ mod tests { /// direction - from the document to the map - so a new field in /// `toolpath` or `toolpath-convo` cannot quietly go unscanned. Every /// exclusion is named below with the reason it is not a candidate. + /// + /// This is only as good as `fixture_rich()` is complete, so the fixture + /// carries every string-bearing field of the `Path` type tree - including + /// the ones the map declines to name. A field the fixture does not carry + /// passes here by being absent, which is the failure this test exists to + /// prevent. #[test] fn every_string_leaf_in_a_derived_document_is_surfaced_exactly_once() { const EXPECTED_OUT_OF_SCOPE: &[(&str, &str)] = &[ ("/path/id", "document identity; plans are keyed on it"), ("/path/head", "names a step id, not content"), + ( + "/path/graph_ref", + "a toolpath-internal link to a sibling document", + ), + ("/steps/1/step/parents/0", "names a step id, not content"), + ("/steps/2/step/parents/0", "names a step id, not content"), ("/steps/0/step/id", "step identity; the DAG is keyed on it"), ("/steps/1/step/id", "step identity; the DAG is keyed on it"), ("/steps/2/step/id", "step identity; the DAG is keyed on it"), @@ -1191,6 +1421,74 @@ mod tests { ), ("/steps/0/meta/refs/0/rel", "a link relation name"), ("/meta/refs/0/rel", "a link relation name"), + ( + "/meta/actors/agent:claude-opus-5/provider", + "the harness name the derivation reports", + ), + ( + "/steps/0/meta/actors/agent:claude-opus-5/provider", + "the harness name the derivation reports", + ), + ( + "/meta/actors/agent:claude-opus-5/model", + "a model id the API assigns", + ), + ( + "/steps/0/meta/actors/agent:claude-opus-5/model", + "a model id the API assigns", + ), + ( + "/meta/actors/human:alex/keys/0/type", + "a key-algorithm name, a closed vocabulary", + ), + ( + "/steps/0/meta/actors/human:alex/keys/0/type", + "a key-algorithm name, a closed vocabulary", + ), + ( + "/meta/actors/human:alex/keys/0/fingerprint", + "a digest of a public key; rewriting hides nothing", + ), + ( + "/steps/0/meta/actors/human:alex/keys/0/fingerprint", + "a digest of a public key; rewriting hides nothing", + ), + ("/meta/kind", "a kind URI this format publishes"), + ("/meta/source", "the deriving harness, a closed vocabulary"), + ( + "/steps/0/meta/source/change_id", + "a change id the VCS assigned", + ), + // `apply` clears these rather than rewriting them: a signature + // over redacted content cannot be repaired, only dropped. + ("/meta/signatures/0/signer", "integrity material; dropped"), + ("/meta/signatures/0/key", "integrity material; dropped"), + ("/meta/signatures/0/scope", "integrity material; dropped"), + ("/meta/signatures/0/sig", "integrity material; dropped"), + ( + "/meta/signatures/0/timestamp", + "integrity material; dropped", + ), + ( + "/steps/0/meta/signatures/0/signer", + "integrity material; dropped", + ), + ( + "/steps/0/meta/signatures/0/key", + "integrity material; dropped", + ), + ( + "/steps/0/meta/signatures/0/scope", + "integrity material; dropped", + ), + ( + "/steps/0/meta/signatures/0/sig", + "integrity material; dropped", + ), + ( + "/steps/0/meta/signatures/0/timestamp", + "integrity material; dropped", + ), ]; let p = fixture_rich(); @@ -1209,6 +1507,11 @@ mod tests { ); let allowed: BTreeSet<&str> = EXPECTED_OUT_OF_SCOPE.iter().map(|(at, _)| *at).collect(); + assert_eq!( + allowed.len(), + EXPECTED_OUT_OF_SCOPE.len(), + "a duplicated allowlist entry hides how much is excluded" + ); for at in &allowed { assert!( leaves.contains(*at), @@ -1226,10 +1529,25 @@ mod tests { ); // An artifact key is the one surface that names a map *key*, so it - // resolves to the object beneath it rather than to a string. + // resolves to the object beneath it rather than to a string. Nothing + // else may: a surface aimed at a subtree reports a whole object as + // one scanned field, and no detector can span that. + let artifact_key = |at: &str| { + let mut segs = at.split('/').skip(1); + matches!( + ( + segs.next(), + segs.next(), + segs.next(), + segs.next(), + segs.next() + ), + (Some("steps"), Some(_), Some("change"), Some(_), None) + ) && doc.pointer(at).is_some_and(Value::is_object) + }; let phantom: Vec<&String> = unique .difference(&leaves) - .filter(|a| !doc.pointer(a).is_some_and(Value::is_object)) + .filter(|a| !artifact_key(a)) .collect(); assert!( phantom.is_empty(), @@ -1433,6 +1751,11 @@ mod tests { ("turn-0f3a", "/change/~0~1notes.md"), ("turn-0f3a", "/meta/intent"), ("turn-0f3a", "/meta/refs/0/href"), + ("turn-0f3a", "/meta/actors/agent:claude-opus-5/name"), + ("turn-0f3a", "/meta/actors/human:alex/name"), + ("turn-0f3a", "/meta/actors/human:alex/identities/0/system"), + ("turn-0f3a", "/meta/actors/human:alex/identities/0/id"), + ("turn-0f3a", "/meta/actors/human:alex/keys/0/href"), ("turn-0f3a", "/meta/source/author_email"), ("turn-0f3a", "/meta/note"), ("turn-deleg", "/change/claude:~1~1sess-abc/structural/text"), @@ -1515,6 +1838,11 @@ mod tests { ("", "/meta/title"), ("", "/meta/intent"), ("", "/meta/refs/0/href"), + ("", "/meta/actors/agent:claude-opus-5/name"), + ("", "/meta/actors/human:alex/name"), + ("", "/meta/actors/human:alex/identities/0/system"), + ("", "/meta/actors/human:alex/identities/0/id"), + ("", "/meta/actors/human:alex/keys/0/href"), ("", "/meta/vcs_remote"), ("", "/meta/files_changed/0"), ("", "/meta/files_changed/1"), @@ -1671,6 +1999,17 @@ mod tests { // Byte-identical to a `step.change` key the map does scan, so // redacting only the key leaves the original in the document. ("", "/meta/files_changed/0".into()), + // An actor definition was excluded as material redaction "drops + // wholesale" - but only `signatures` is ever dropped, so a git + // import shipped the committer's name and email unscanned. + ("", "/meta/actors/human:alex/name".into()), + ("", "/meta/actors/human:alex/identities/0/id".into()), + ("", "/meta/actors/human:alex/keys/0/href".into()), + ("turn-0f3a", "/meta/actors/human:alex/name".into()), + ( + "turn-0f3a", + "/meta/actors/human:alex/identities/0/id".into(), + ), ] { assert!( named.contains(&(step.to_string(), at.clone())), @@ -1762,6 +2101,18 @@ mod tests { // the extras map the pointer resolves against. ("turn-0f3a", "/change/claude:~1~1sess-abc/structural/type"), ("", "/meta/not_present"), + // The actor fields the map deliberately leaves out. An unnamed + // field must resolve to nothing, or the pass would report a + // surface no detector was ever handed. + ("", "/meta/actors/agent:claude-opus-5/model"), + ("", "/meta/actors/agent:claude-opus-5/provider"), + ("", "/meta/actors/human:alex/keys/0/fingerprint"), + ("", "/meta/actors/human:alex/keys/0/type"), + ("", "/meta/actors/no-such-actor/name"), + ("", "/meta/actors/human:alex/identities/9/id"), + ("turn-0f3a", "/meta/actors/agent:claude-opus-5/model"), + // `turn-deleg` has no metadata at all, so nothing under it can. + ("turn-deleg", "/meta/actors/human:alex/name"), ("", "/path/base/nope"), ("", "/meta/refs/9/href"), ("", "/meta/refs/0/rel"), From f26c5eefd732fd2d195257c7b76dc689636e46c4 Mon Sep 17 00:00:00 2001 From: ulmentflam Date: Fri, 31 Jul 2026 02:57:36 -0400 Subject: [PATCH 9/9] fix(redact): gate `p redact` off the emscripten target `cmd_redact` resolves detectors through `sync::build_detectors` and records the replay policy through `sync::record_redaction_policy`, but `sync` is itself `#[cfg(not(target_os = "emscripten"))]`, so both names were unresolved on that target. `deploy-site.yml` builds wasm32-unknown-emscripten on every pull request, so this was a red job rather than a hypothetical. Gated rather than made portable: redaction needs the document cache and the on-disk key store, neither of which exists on that target. Co-Authored-By: Claude Opus 5 (1M context) --- crates/path-cli/src/cmd_p.rs | 2 ++ crates/path-cli/src/lib.rs | 3 +++ 2 files changed, 5 insertions(+) diff --git a/crates/path-cli/src/cmd_p.rs b/crates/path-cli/src/cmd_p.rs index c37b4c2c..b83a939f 100644 --- a/crates/path-cli/src/cmd_p.rs +++ b/crates/path-cli/src/cmd_p.rs @@ -92,6 +92,7 @@ pub enum PCommand { op: crate::cmd_p_query::PQueryOp, }, /// Remove credentials from a Toolpath document via a reviewable plan-then-apply flow + #[cfg(not(target_os = "emscripten"))] Redact { #[command(flatten)] args: crate::cmd_redact::RedactArgs, @@ -116,6 +117,7 @@ pub fn run(command: PCommand, pretty: bool) -> Result<()> { PCommand::Incept { target } => crate::cmd_incept::run(target), PCommand::Track { op } => crate::cmd_track::run(op, pretty), PCommand::Query { op } => crate::cmd_p_query::run(op, pretty), + #[cfg(not(target_os = "emscripten"))] PCommand::Redact { args } => crate::cmd_redact::run(args), } } diff --git a/crates/path-cli/src/lib.rs b/crates/path-cli/src/lib.rs index 82c5a00c..47053ef7 100644 --- a/crates/path-cli/src/lib.rs +++ b/crates/path-cli/src/lib.rs @@ -17,6 +17,9 @@ mod cmd_p_query; mod cmd_pathbase; mod cmd_project; mod cmd_query; +// Resolves detectors and records the replay policy through `sync`, +// which is itself off for this target. +#[cfg(not(target_os = "emscripten"))] mod cmd_redact; mod cmd_render; #[cfg(not(target_os = "emscripten"))]