diff --git a/internal/handoff/CRITIQUE-02.md b/internal/handoff/CRITIQUE-02.md new file mode 100644 index 0000000..6df26f0 --- /dev/null +++ b/internal/handoff/CRITIQUE-02.md @@ -0,0 +1,429 @@ +# CRITIQUE-02 — Critic Gate 2 (R.10): sealing, handoff/claim, masking + +**Verdict: FAIL.** Two of the five required verdicts fail, on defects reproduced by executed tests, +not by reading. + +> ## SAME-FAMILY CRITIC. READ THIS BEFORE QUOTING THE VERDICT. +> +> `plan/00-ROUTING.md` originally required a **different model family** for this gate, precisely so a +> shared blind spot could not survive review. The owner withdrew external routes on 2026-08-07 +> because running them copies private files to a third party (see the OWNER DECISION block at the top +> of `00-ROUTING.md`). This critique was therefore produced by a critic of the **same family as the +> implementer**. +> +> A later reader must not record this as a cross-family review. The guarantee obtained here is +> weaker: shared inductive biases between author and critic are *not* controlled for. Every finding +> below is backed by an executed reproduction so that at least the positive claims do not depend on +> the critic's judgement — but the *absence* of further findings carries less weight than a +> cross-family PASS would. + +--- + +## 1. The five required verdicts + +| # | Verdict required by the R.10 packet | Result | +|---|---|---| +| (a) | Lease vs. claim-timeout independence | **PASS** | +| (b) | No secure-deletion claims present | **PASS** | +| (c) | Masking runs before both sinks | **FAIL** — F3, F4 | +| (d) | Re-entrant consumer never reads an unsealed half | **FAIL** — F2, F5, F6 | +| (e) | Reaper never drops a live claim | **PASS** | + +Independently of (a)–(e), **F1 is a blocker in its own right**: the claim protocol can grant two +live leases on one finding at one record version. + +The packet's stop condition is "all-PASS". It is not met. + +--- + +## 2. Method + +Everything below was re-derived from the files, not from the implementer's prose. The tree was +copied to a scratch directory and probe tests were written and executed there; **nothing was written +into the repository except this file**. + +Toolchain, run against the working tree (Go 1.26.5, windows/amd64): + +``` +$ gofmt -l . +(no output) + +$ go vet ./... +(no output) + +$ go build ./... +(no output) + +$ go test -count=1 ./... +ok github.com/Susquehanna-Syntax/Anvil/cmd/anvil 0.385s +? github.com/Susquehanna-Syntax/Anvil/cmd/anvil-dast [no test files] +? github.com/Susquehanna-Syntax/Anvil/internal/buildpin [no test files] +ok github.com/Susquehanna-Syntax/Anvil/internal/handoff 0.501s +ok github.com/Susquehanna-Syntax/Anvil/internal/record 0.650s +ok github.com/Susquehanna-Syntax/Anvil/internal/store 0.754s +``` + +The shipped suites are green and are, on inspection, substantive — see §5. The defects below are +cases the suites do not construct. + +`go test -race` **could not be run on this host**: it needs cgo and there is no C toolchain +(`cgo.exe` exits 2). That is pre-existing and host-wide, not specific to these packets. CI runs +`-race` on Linux; the concurrency claims in §4 (a) and (e) are therefore verified by reasoning and by +single-threaded reproduction only. + +--- + +## 3. Blocking findings + +### F1 — BLOCKER. Two live leases on one finding at one record version. + +`plan/40-record-and-storage.md` and this package's own doc make the reclaim/idempotency key +**(fingerprint, record version)**. The durable table does not enforce that key. `schema.sql` declares +`UNIQUE (finding_id, audit_record_id)`, and a re-scan produces a *new* `audit_record` row (its +`scan_run_id` is `UNIQUE`), each with `audit_version` defaulting to 1. So one fingerprint gets one +row per audit record, all of them at record version 1, and each row is independently leasable. + +The two entry points then actively diverge onto different rows: + +- `AcquireLeaseContext` — `ORDER BY h.created_at, h.handoff_id` → the **oldest** (stale) row. +- `ClaimContext` — `ORDER BY h.audit_record_id DESC, h.handoff_id DESC` → the **newest** row. + +`checkRecordVersion` cannot catch this: it compares the Handle against *its own* row's +`audit_record.audit_version`, which never moved. `ErrRecordVersionChanged` only fires when +`audit_version` is bumped **in place** on one `audit_record` — which is the only case +`TestRecordVersionBumpVoidsTheLease` constructs (`internal/handoff/handoff_test.go:944`). + +Reproduced: + +``` +=== RUN TestProbeDoubleGrantAcrossAuditRecords + two rows for ONE fingerprint: handoff_id 1 (audit 1) and 2 (audit 2) + worker-1 holds fp=0707...0707 handoff=1 recordVersion=1 idem=e73d9f0882ac7411 + worker-2 holds fp=0707...0707 handoff=2 recordVersion=1 idem=3cf550b5f68f70b6 + DOUBLE GRANT: two live leases on the same (fingerprint=0707...0707, recordVersion=1) + the two competing leases carry DIFFERENT idempotency keys, so no downstream dedup + can suppress the duplicate + both workers independently recorded 'validated' for the same defect +--- FAIL: TestProbeDoubleGrantAcrossAuditRecords (0.02s) +``` + +Both workers renewed, both released, both wrote `validated`. This is exactly the outcome +research/08 §4 point 2 is quoted as forbidding in `reaper.go:31`: *"Expiring it would let a second +agent write a competing fix for the same defect."* The mechanism is different from the one that +quote anticipates; the outcome is the one it forbids. + +Aggravating: the two competing leases carry **different** `idempotency_key` values (see F7), so the +downstream duplicate-suppression the `Handle` doc promises cannot catch it either. + +The queue re-cut that would mark the stale row `superseded` is R.11, which does not exist yet, and +even once it does there is a window. The invariant must be enforced where it is claimed — one live +lease per `(fingerprint, record version)` — not left to a later step's punctuality. + +`internal/handoff/claim.go:113` (`eligibleFrom`), `:168`, `:213`; `internal/store/schema.sql` +`UNIQUE (finding_id, audit_record_id)`. + +--- + +### F2 — BLOCKER. `audit_record.state = 'consumed'` shuts the queue, and the reaper then expires what is left. + +`eligibleFrom` gates on `a.state IN ('sast_sealed','both_sealed')` for `static_only` and +`a.state = 'both_sealed'` for `requires_dynamic_confirmation`. `'consumed'` is in neither set, and +`'consumed'` is a legal `audit_record.state` (`ck_audit_record_state`) that R.6's `Sealer.Consume` +sets. + +R.6 is explicit in the other direction — `sealing.go:775`: *"A consumed audit is still readable: +plan/00-SPINE.md S1 requires a RE-ENTRANT consumer, so taking the record once must not shut the +gate."* The queue shuts it. + +Reproduced: + +``` +=== RUN TestProbeConsumedAuditShutsTheQueue + Claim after audit_record.state='consumed' -> handoff: fingerprint 0a0a...0a0a is ready + but its static_only gate is shut: handoff: finding has not passed its consumption gate + RE-ENTRANCY BROKEN: a still-ready sibling finding is unclaimable once the audit is consumed + AcquireLease after consumed -> handoff: no claimable finding + after the claim window closes: 1 rows expired +--- FAIL: TestProbeConsumedAuditShutsTheQueue (0.01s) +``` + +One `audit_record` fans out to many `handoff` rows. The first consumption pass marking the audit +`consumed` therefore strands **every sibling finding still in `ready`** — permanently unclaimable, +then swept to `'expired'` by `ExpireClaimTimeouts` at the deadline. The row is kept (so this is not +data *deletion*), but the finding is never handed to an agent in this scan. That is silent work loss +on the exact axis S1 names. + +No test in `handoff_test.go` ever sets `record.StateConsumed` — `grep -n "StateConsumed" +internal/handoff/handoff_test.go` returns nothing. The gap is untested, not merely unhandled. + +`internal/handoff/claim.go:119-120`, `eligibleArgs()` at `:126`. + +--- + +### F3 — BLOCKER. Masking's discovery surface omits two fields that routinely carry live credentials, so an unmasked secret reaches the store. + +`Masker.Mask` pass 1 inspects exactly `result.webRequest.{headers,parameters,target}` and +`result.webResponse.headers`. Pass 2 propagates only values pass 1 *discovered*. Anything carrying a +credential that pass 1 never looks at, and whose value appears nowhere in a header pass 1 does look +at, survives into the record — and the record is what goes to the store. + +Two concrete, non-hypothetical carriers: + +**(i) `anvil/repro.curl`.** A full curl command line, including `-H 'Authorization: Bearer …'`. + +``` +=== RUN TestProbeReproCurlIsNeverMasked + after MaskRecord, anvil/repro.curl still carries "ghp_LIVETOKEN00000000..." + AssertMasked also returned nil on that record +--- FAIL: TestProbeReproCurlIsNeverMasked (0.00s) +``` + +**(ii) `anvil/target.repoUrl` and `.runtimeBaseUrl`.** `https://x-access-token:@github.com/…` +is the *standard* GitHub Actions checkout URL. `maskURL` — which already knows how to strip userinfo +passwords — is applied only to `webRequest.target`, never to these. + +``` +=== RUN TestProbeTargetRepoURLCredentialSurvives + after MaskRecord, anvil/target carries a live credential: + https://x-access-token:ghp_LIVETOKEN00000000...@github.com/org/repo.git + AssertMasked returned nil anyway +--- FAIL: TestProbeTargetRepoURLCredentialSurvives (0.00s) +``` + +This is the R.10 packet's named Forbidden action: a design that *"leaves any code path capable of +persisting an unmasked secret."* It is not the documented body-only limitation — `mask.go:106-114` +disclaims *shape-based body scanning*, which is a different thing. `repoUrl` and `repro.curl` are +structured fields with a known credential position, exactly what structural masking is for. + +`internal/record/mask.go:378-392` (pass 1's four call sites). + +--- + +## 4. Major findings + +### F4 — MAJOR. `AssertMasked`, the enforceable sink gate, is weaker than `Mask` and fails open on the URL surface. + +`mask.go:996` justifies `AssertMasked` as S7's *"enforce in code, not documentation"*: a sink *"can +call it and refuse the record rather than trusting that some earlier step remembered to mask."* It +checks headers, parameters and body caps. It never checks `webRequest.target` — which `Mask` **does** +mask. + +``` +=== RUN TestProbeAssertMaskedIgnoresTargetURL + AssertMasked returned nil on a record whose webRequest.target still carries + "ghp_LIVETOKEN0000000000000000000000000000" +--- FAIL: TestProbeAssertMaskedIgnoresTargetURL (0.00s) + +=== RUN TestProbeMaskDoesCleanTargetURL + after MaskRecord target = https://app.invalid/v1/orders?api_key=***REDACTED*** +--- PASS +``` + +So a record that skipped masking, whose only credential is in the URL query, fragment or userinfo, +passes the gate that exists to catch exactly that. `TestAssertMaskedIsTheSinkGate` +(`mask_test.go:771`) has three "put the secret back" sub-cases — header, parameter, oversized body — +and no URL sub-case. Adding one turns it red. + +### F5 — MAJOR. `ReadPacket`/`WritePacket` are entirely ungated. + +Enumerating every exported function in `internal/handoff` that can return a half's *results* (not +metadata): `ReadPacket` is the only one, and it checks nothing — not seal state, not audit state, not +lease ownership. `WritePacket` likewise materialises a packet for an audit that has sealed nothing. + +``` +=== RUN TestProbePacketReadIsUngated + Claim refused, as designed: ... static_only gate is shut ... + WritePacket succeeded on an UNSEALED audit: .../packets/1515...1515.sarif + ReadPacket returned 43 bytes of an unsealed half's results with no lease and no seal check +--- FAIL: TestProbePacketReadIsUngated (0.02s) +``` + +The claim gate correctly refuses the same fingerprint one line earlier. The packet is called a cache, +but it is a cache *of the payload*, and R.6's read gate means nothing if the bytes are reachable +beside it. (Credit where due: expiry **does** unlink the packet — `TestProbePacketReadAfterExpiry` +passes.) + +### F6 — MAJOR. `Sealer.Inspect` bypasses the expiry arm of the read gate. + +`ReadHalf` refuses an expired audit with a `*ReadGateError`. `Inspect` returns the *same* `HalfSeal` +values with no state check at all, and `HalfSeal.Readable()` is exported. + +``` +=== RUN TestProbeInspectBypassesTheReadGate + ReadHalf correctly refused: record: read of sast half of audit "a1" refused: status is + "sealed", state is "expired"; ... (the gate opens only at anvil/status="sealed") + Inspect on the SAME expired audit returned Sast={Half:sast Status:sealed + SealedAt:2026-08-08 09:00:00 +0000 UTC} Readable()=true + Inspect reports the SAST half readable on an expired audit that ReadHalf refuses +--- FAIL: TestProbeInspectBypassesTheReadGate (0.00s) +``` + +`ReadyForConsumption` checks `StateExpired`; `Inspect` does not. Two exported readiness paths, two +answers. Note also the structural point for verdict (d): `ReadHalf` returns no results at all — +`HalfSeal` is `{Half, Status, SealedAt}`, all of which `Inspect` hands out ungated — so as +implemented the gate is advisory, and R.13 will have to re-implement it over the actual results +rather than inherit it. + +### F7 — MAJOR. `IdempotencyKey` is keyed on a rowid, so it does not survive the case it is documented to survive. + +`claim.go:646` computes `sha256(audit_record_id ‖ fingerprint ‖ base_commit_sha)` while its own doc +and `schema.sql` say `sha256(audit_id ‖ finding_fingerprint ‖ base_commit_sha)`. `audit_record_id` is +an autoincrement rowid, not the audit identity. + +Consequences: (1) a re-scan of the same commit produces a new `audit_record` and therefore a *new* +key for the same finding at the same base commit, so the agent-side git-trailer dedup the function +exists to serve cannot recognise the repeat; (2) it is what makes F1 undetectable downstream +(different keys on the two competing leases); (3) the exported value that "the coding agent writes +into a git trailer" is an internal database rowid, which is not a portable identity. + +The `Handle.IdempotencyKey` doc claim — *"stable across crash and reclaim"* — is true, because the +row survives reclaim. The stronger reading a reader will take from it is not. + +### F8 — MAJOR. `completed_clean` is reachable for a DAST half that found things. + +`RecordDastOutcome` refuses after the half seals ("its outcome is frozen"), so the finding count must +be known *before* the seal. But provenance (from the target harness) and the finding count (from the +DAST worker) arrive at different times and share one struct, and the zero value of `FindingCount` +is 0. + +``` +=== RUN TestProbeDastCleanByOrdering + post-seal RecordDastOutcome -> record: RecordDastOutcome("a2") half=dast state=dast_sealed + status=sealed: the DAST half has already sealed; its outcome is frozen + anvil/dastStatus = "completed_clean", MeansDynamicallyScannedClean = true + a DAST half with 3 findings reports "completed_clean" +--- FAIL: TestProbeDastCleanByOrdering (0.00s) +``` + +`completed_clean` is the one value contract.go permits a consumer to read as *"dynamically scanned, +no findings"*, and research/23 Risk #1 is quoted in contract.go as *"Anvil must never report '0 DAST +findings' as 'no dynamic vulnerabilities'."* An API whose only defence against that is call ordering +is not enough. `DeriveDastStatus` itself is sound; the hazard is `DastOutcome` conflating two facts +with different arrival times. Making the finding count an explicit argument of the DAST seal, or +requiring it non-zero-valued (an `*int`), would close it. + +`internal/record/sealing.go:237` (`DastOutcome`), `:612` (`RecordDastOutcome`). + +### F9 — MAJOR. Nothing stops a `requires_dynamic_confirmation` finding being recorded `validated` with no dynamic evidence. + +`ReleaseLease` accepts `HandoffStateValidated` from any lease holder regardless of the Handle's own +`ConsumptionClass` and `DastStatus`. + +``` +=== RUN TestProbeValidatedWithoutDynamicEvidence + claimed a requires_dynamic_confirmation finding with DastStatus="not_run" + a requires_dynamic_confirmation finding was recorded 'validated' while dast_status="not_run" + (no reproduction can exist) +--- FAIL: TestProbeValidatedWithoutDynamicEvidence (0.01s) +``` + +On the S7 question the packet actually asks — *does a lease grant more than "may act on this +finding"?* — the answer is **no**, and that part is clean: `Handle` carries no merge field, no scope +field and no verdict field, nothing in either package merges anything, and `state_machine.go:40-44` +states the limit correctly. But `state_machine.go` also says *"Only a DAST reproduction that now +fails earns 'verified fixed', and that judgement is made elsewhere"* — and `handoff.state = +'validated'` is written **here**, by the claimant, unchecked. `DastStatus` on the Handle is +documented as advisory ("so a consumer can *see*"). Either the check belongs on `ReleaseLease`, or +the "made elsewhere" owner must be named. + +--- + +## 5. Minor findings + +- **F10.** `ReleaseLease(h, ready)` clears the lease but does not restore the attempt it consumed. + Two voluntary hand-backs with `max_attempts = 2` strand the row: `state=ready attempts=2/2`, then + `Claim` → `ErrExhausted`, forever, with no crash having occurred + (`TestProbeReleaseToReadyBurnsAttempts`). `attempts` is deliberately "attempts started", which is + right for the crash path; a voluntary hand-back is not an attempt started and should not be + counted like one. +- **F11.** The flagship masking test's `Repro.Curl` coverage is incidental. `dastFixture` + (`mask_test.go:210`) puts `plantedBearer` in *both* the `Authorization` header and the curl string, + so `TestMaskRecordLeavesNoPlantedSecretAnywhere` passes via propagation from the header. It reads + as proof that `repro.curl` is masked. It is not — see F3(i). The fixture should carry a + curl-only secret. +- **F12.** `ReadHalf` on an unknown audit returns a `*SealingError` wrapping `ErrUnknownAudit`, so + `errors.Is(err, ErrHalfNotSealed)` is false. A consumer branching on the gate sentinel — the + documented way to detect a refusal — will not classify it as one. +- **F13.** `Dispose(id, HandoffStateExpired)` is legal: `CheckTransition(ready, expired)` passes and + `DisposeContext` rejects only `to == leased`. Any caller can set the claim-timeout terminal state + without a deadline having passed, from outside the reaper that owns that clock. (Code-read; not + executed.) +- **F14.** `explainUnclaimable` returns `ErrAlreadyClaimed` if **any** row for the fingerprint is + leased, even when the row that was actually ineligible failed for a different reason. Misreports + the cause across the multi-audit-record shape of F1. + +--- + +## 6. What survived the attack — recorded so the PASSes are not read as unexamined + +These were probed and held. + +- **(a) Two clocks, genuinely independent.** `handoff.lease_expires_at` (Options.Lease, default 20m) + and `audit_record.deadline_at` (`scan_run.started_at + claim_timeout_seconds`) are separate + columns, separate sweeps, separate transitions. `ComputeDeadline` is the only formula, called once + in `BeginAudit`; nothing else writes `deadlineAt`. `SealHalf` never touches it — a late seal moves + it by nothing. `Options.Lease` is never derived from `claim_timeout_seconds` anywhere. +- **(e) The reaper never drops a live claim, structurally.** `ExpireClaimTimeouts` selects + `WHERE h.state = 'ready'`, so a leased row is not a candidate at all — a query-level guarantee, not + a check someone can forget. `legalTransitions` has no `leased → expired` edge. + `ReclaimExpiredContext` decides expiry in Go against parsed times (not TEXT comparison) and + CAS-updates on the exact `(state, claimed_by, lease_expires_at)` triple, so a heartbeat that lands + between SELECT and UPDATE preserves the live claim. `Reap` runs leases first, which is the right + order. +- **Reclaim idempotency, in the crash sense.** Every mutation is a CAS on the exact lease, so a + second sweep matches nothing and a resurrected OOM-killed holder gets `ErrLeaseLost` rather than + overwriting its successor. Attempt arithmetic is consistent: incremented at claim, compared + `attempts >= max_attempts` in the reaper and `attempts < max_attempts` in the eligibility query, so + exactly one retry with the default of 2 and no infinite re-lease (the §6 G10 failure). +- **(b) No secure-deletion claim anywhere.** `SECRETS.md` §2 is an explicit denial, §4 names the + LUKS2/fscrypt alternative, §8 is a claim-to-source table. `DropPacket`'s doc says plainly it is + *"an unlink, not an erasure"*. `TestNoSecureDeletionClaimOrCall` (`handoff_test.go:1229`) is a real + AST + prose scan over non-test files, including an `os/exec` import ban — not a token gesture. +- **No bare enum literals.** Grepping every frozen literal across `sealing.go`, `mask.go`, + `claim.go`, `state_machine.go`, `reaper.go` returns **only three hits, all inside comments** + (`sealing.go:180`, `:662`, `:978`). Every value that reaches SQL goes through + `string(record.)`, including all eight of `eligibleArgs()`. + `TestNoBareEnumLiteralsInPackageCode` enforces it by AST. +- **Masking fails closed on unexpected header shape.** Probed with a trailing-space name, an empty + name, a CRLF-smuggled value, and a Cyrillic homoglyph `Cооkie` — all four redacted, while an + ordinary `Content-Length: 42` survived intact. `isHTTPFieldName`/`isTChar` are checked *before* the + denylist, and the deliberate refusal of `strings.EqualFold` (U+212A) is correct reasoning. +- **Ordering inside the masker is right.** Structural → propagate over the untruncated record → + truncate. The spill digest is therefore over masked bytes, and a secret past the 32 KB cap is + scrubbed from the spilled blob too. `secretSet.values()` sorts longest-first, which is genuinely + load-bearing and correctly justified. +- **Test quality.** No golden file regenerates itself (`os.WriteFile` appears in no test), no update + flag exists, and the assertions sampled are real — `TestMaskRecordLeavesNoPlantedSecretAnywhere` + first asserts the fixture *contains* each planted value before asserting absence, which is exactly + the guard that keeps an absence test from being vacuous. F11 is the one place the coverage is + narrower than it reads. + +--- + +## 7. Unverified + +- `go test -race` was not run: no C toolchain on this Windows host (`cgo.exe` exit 2), pre-existing + and host-wide. The concurrency arguments in §6 (a) and (e) rest on reading plus single-threaded + reproduction. CI must confirm on Linux. +- F13 is read from source, not executed. +- **Verdict (c) cannot be evidenced by wiring even where masking is correct.** `grep -rn + "MaskRecord\|AssertMasked" --include=*.go . | grep -v _test.go` finds **no production caller** — + only the definitions. The store writer that would call it is a later step. So "masking runs before + both sinks" is today a property of intent, not of the tree; it will need re-verification when the + writer lands. F3 and F4 are failures of the masker itself and stand independently of that. +- Whether R.11's queue re-cut is intended to `Dispose(..., superseded)` every stale row of a bumped + audit is not stated in anything R.6–R.8 owns. F1 is a defect regardless — the invariant must not + depend on another step's timeliness — but the intended division of labour should be confirmed by + the orchestrator before F1 is fixed, so the fix lands in the right packet. + +--- + +## 8. Recommendation + +Re-route **R.7** (F1, F2, F7, F9, F10) and **R.8** (F3, F4), and take F6/F8 back to **R.6**. F5 spans +R.6 and R.7 and needs an owner assigned before it is fixed. + +Per the R.10 packet: *"All five verdicts PASS, or R.6/R.7/R.8/R.9 rerouted and re-reviewed."* Two +verdicts fail. Reroute. + +R.9 (`SECRETS.md`) is the one reviewed artifact with no findings against it. diff --git a/internal/handoff/claim.go b/internal/handoff/claim.go new file mode 100644 index 0000000..c0200b2 --- /dev/null +++ b/internal/handoff/claim.go @@ -0,0 +1,921 @@ +package handoff + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// Handle is proof that one worker holds one lease on one finding, at one +// version of one audit record. +// +// plan/00-SPINE.md S7: a lease grants "may act on this finding" and nothing +// more. There is deliberately no field here that widens scope, authorises a +// merge, or records a verdict — a Handle cannot be mistaken for permission to +// do any of those because it carries no such value. +// +// The lease token is unexported: RenewLease and ReleaseLease compare-and-swap +// against the exact (claimed_by, lease_expires_at) pair this Handle was issued +// with, so a stale Handle — the one an OOM-killed consumer still has in memory +// when it wakes up — cannot write over its successor's work. It gets +// ErrLeaseLost instead. +type Handle struct { + HandoffID int64 + FindingID int64 + AuditRecordID int64 + + // Fingerprint is the anvil-fp/v1 digest, full 64 hex, never truncated. + Fingerprint string + + // WorkerID is the lease holder, `handoff.claimed_by` (O.3's lease_owner). + WorkerID string + + // RecordVersion is audit_record.audit_version as it stood when the lease + // was granted. Together with Fingerprint it is the (fingerprint, record + // version) key the plan requires reclaim/re-processing to be idempotent + // under: a version bump re-cuts the queue (S6), so a Handle whose version + // has moved describes work that no longer exists and is refused. + RecordVersion int64 + + ConsumptionClass record.ConsumptionClass + + // DastStatus is the audit's DAST half status at claim time. It is carried + // so a consumer of a requires_dynamic_confirmation finding can see that + // the half ended 'not_run', 'target_boot_failed' or 'skipped_no_manifest' + // — i.e. that no dynamic evidence exists — rather than assume a clean + // dynamic scan. S6 exists to keep those cases distinguishable; dropping + // the field here would re-merge them at the only point that acts on them. + // + // It is NOT advisory. ReleaseLease reads it: a requires_dynamic_confirmation + // finding whose DAST half produced no reproduction cannot be recorded + // 'validated' (checkDynamicEvidence, and plan/00-SPINE.md S7). + DastStatus record.DastStatus + + // Attempt is this lease's ordinal: 1 for the first, 2 after one crash and + // one reclaim. It equals handoff.attempts after the claim. + Attempt int + MaxAttempts int + + // LeaseExpiresAt is when ReclaimExpired will presume this holder dead. It + // is Options.Lease after the claim, NOT audit_record.claim_timeout_seconds. + LeaseExpiresAt time.Time + + // IdempotencyKey is stable across crash and reclaim: the second consumer + // of the same finding at the same record version gets the same key the + // first had, which is how a duplicate side effect is recognised and + // suppressed downstream. It mirrors the git trailer. + IdempotencyKey string + + // PacketPath is where the regenerable tmpfs packet lives, or "" when no + // PacketDir is configured. The packet is a cache: if it is missing, + // regenerate it from the store (research/08 §1). It is never the source of + // truth and its absence is not an error. + PacketPath string + + // leaseToken is the exact lease_expires_at text stored at claim time. It + // is the CAS witness, not a duplicate of LeaseExpiresAt: comparing the + // stored text avoids any dependence on timestamp formatting round-trips. + leaseToken string +} + +// consumptionGate is THE consumption gate, written once so there is one +// definition of "this finding's half is readable". +// +// research/21 §5, as quoted by O.3: static_only findings are claimable once +// the SAST half is sealed; requires_dynamic_confirmation findings must wait on +// the DAST half. Expressed against R.6's sealing signals: +// +// - static_only needs audit_record.sast_status = 'sealed' AND the audit to +// have actually sealed that half (state 'sast_sealed', 'both_sealed' or +// 'consumed'). R.6 makes 'sealed' the hard read gate; a consumer must not +// read a half before it says so. +// - requires_dynamic_confirmation needs the audit to have reached +// 'both_sealed' (or moved on to 'consumed'), which are the only states in +// which the DAST half is final, plus dast_status <> 'running' as a +// belt-and-braces check. +// +// 'consumed' IS IN BOTH SETS, AND THAT IS THE POINT. plan/00-SPINE.md S1 +// requires a RE-ENTRANT consumer, and R.6 already implements it in the other +// direction — sealing.go's ReadHalf says so outright: "A consumed audit is +// still readable: S1 requires a RE-ENTRANT consumer, so taking the record once +// must not shut the gate." Before this fix the queue disagreed with the +// sealer. Because one audit_record fans out to MANY handoff rows, the first +// consumption pass marking the audit 'consumed' stranded every sibling finding +// still in 'ready': permanently unclaimable, then swept to 'expired' by +// ExpireClaimTimeouts at the deadline. The row survived; the finding was never +// handed to an agent in that scan. That is silent work loss on the exact axis +// S1 names, and CRITIQUE-02 F2 reproduced it. +// +// 'expired' is deliberately NOT in either set: R.6's read gate refuses an +// expired audit because the reaper has dropped its payload, and a claim on a +// finding whose evidence is gone is worse than no claim. +// +// The audit state test is load-bearing and not redundant with dast_status: +// schema.sql DEFAULTs dast_status to 'not_run', so a still-collecting audit +// whose DAST half has not started is indistinguishable from a finished audit +// on which DAST was disabled if you look at dast_status alone. Gating on the +// sealed state closes that hole. The consequence for a +// requires_dynamic_confirmation finding on a DAST-disabled audit is that it is +// claimable once both halves seal, with DastStatus = 'not_run' visible on the +// Handle — deliberately, because refusing forever would only push the finding +// to its claim timeout, and research/08 §4 is explicit that missing the window +// costs latency, not the finding. What such a finding may NOT do is reach +// 'validated'; see ReleaseLeaseContext. +const consumptionGate = `( + (h.consumption_class = ? AND a.sast_status = ? AND a.state IN (?, ?, ?)) + OR (h.consumption_class = ? AND a.state IN (?, ?) AND a.dast_status <> ?) + )` + +// gateArgs binds consumptionGate's nine placeholders. Every value is a +// constant from internal/record — no enum literal is re-typed as a bare string +// here or anywhere else in this package. +func gateArgs() []any { + return []any{ + string(record.ConsumptionClassStaticOnly), + string(record.HalfStatusSealed), + string(record.StateSastSealed), + string(record.StateBothSealed), + string(record.StateConsumed), + string(record.ConsumptionClassRequiresDynamicConfirmation), + string(record.StateBothSealed), + string(record.StateConsumed), + string(record.DastStatusRunning), + } +} + +// noSiblingLease is the ONE-LIVE-LEASE-PER-(fingerprint, record version) +// invariant, enforced in the queries that grant a lease. +// +// WHY IT IS HERE AND NOT IN schema.sql. The protocol promises reclaim and +// re-processing are idempotent under (fingerprint, record version); the table +// only enforces UNIQUE (finding_id, audit_record_id). A re-scan produces a NEW +// audit_record row — scan_run_id is UNIQUE, so it must — and audit_version +// DEFAULTs to 1 on each. One fingerprint therefore ends up with several rows, +// ALL AT VERSION 1, each independently leasable. The two entry points then +// actively diverged onto different rows: AcquireLease ordered by created_at +// (oldest) and Claim by audit_record_id DESC (newest), so two workers took two +// live leases on one defect at one record version and both recorded +// 'validated'. checkRecordVersion cannot see it — it compares a Handle against +// ITS OWN row's audit_version, which never moved. CRITIQUE-02 F1 reproduced +// exactly that, and it is the outcome research/08 §4 point 2 forbids: "a +// second agent write[s] a competing fix for the same defect." +// +// R.4's schema.sql is a frozen interface, so the durable constraint that would +// express this — a partial unique index over (fingerprint, audit_version) +// where state = 'leased' — cannot be added here; it is reported to the +// orchestrator instead. What IS available is a guard inside the same statement +// that grants the lease. SQLite serialises writers, so the NOT EXISTS is +// evaluated as part of the granting UPDATE and cannot interleave with a +// competing grant: of two concurrent claims on sibling rows, exactly one sees +// no live sibling and wins. That is the same argument that makes the +// `state = 'ready'` guard atomic, applied to a wider key. +// +// It appears in TWO places on purpose. In the eligibility SELECT it keeps +// AcquireLease from repeatedly picking a row it cannot have (which would burn +// the race budget and return ErrNoWork while other work waited); in the UPDATE +// it is the guarantee, because only the UPDATE is atomic with the grant. +const noSiblingLease = `NOT EXISTS ( + SELECT 1 + FROM handoff o + JOIN audit_record oa ON oa.audit_record_id = o.audit_record_id + WHERE o.fingerprint = h.fingerprint + AND o.handoff_id <> h.handoff_id + AND o.state = ? + AND oa.audit_version = a.audit_version + )` + +// eligibleFrom is the claimable set: the consumption gate, plus the attempt +// budget, plus the one-live-lease invariant. +// +// `h.attempts < h.max_attempts` is here, not only in the reaper, so a row that +// somehow re-entered 'ready' with its attempts burned cannot be re-leased +// forever — the exact failure §6 G10 traced. +const eligibleFrom = ` + FROM handoff h + JOIN audit_record a ON a.audit_record_id = h.audit_record_id + WHERE h.state = ? + AND h.attempts < h.max_attempts + AND ` + consumptionGate + ` + AND ` + noSiblingLease + +// eligibleArgs binds eligibleFrom's placeholders, in statement order. +func eligibleArgs() []any { + args := make([]any, 0, 11) + args = append(args, string(record.HandoffStateReady)) + args = append(args, gateArgs()...) + args = append(args, string(record.HandoffStateLeased)) + return args +} + +// Claim takes the lease on one named finding. It is the R.7 packet's entry +// point and a narrowing of AcquireLease: same query, same CAS, same state +// machine. +// +// Exactly one concurrent caller wins. Every loser gets ErrAlreadyClaimed, +// because the winner's UPDATE moved the row out of 'ready' and SQLite +// serialises the two writes. +// +// Errors worth distinguishing: ErrNotFound (no such fingerprint in the queue), +// ErrAlreadyClaimed (someone holds it), ErrNotEligible (consumption gate shut, +// or every row for it is terminal), ErrExhausted (attempts burned). +func (q *Queue) Claim(fingerprint string, workerID string) (Handle, error) { + return q.ClaimContext(context.Background(), fingerprint, workerID) +} + +// ClaimContext is Claim with a caller-supplied context. +func (q *Queue) ClaimContext(ctx context.Context, fingerprint, workerID string) (Handle, error) { + if err := ValidateFingerprint(fingerprint); err != nil { + return Handle{}, err + } + if workerID == "" { + return Handle{}, errors.New("handoff: Claim requires a non-empty workerID") + } + + args := append(eligibleArgs(), fingerprint) + var handoffID int64 + err := q.db.QueryRowContext(ctx, + `SELECT h.handoff_id`+eligibleFrom+` + AND h.fingerprint = ? + ORDER BY h.audit_record_id DESC, h.handoff_id DESC + LIMIT 1`, args...).Scan(&handoffID) + if errors.Is(err, sql.ErrNoRows) { + return Handle{}, q.explainUnclaimable(ctx, fingerprint) + } + if err != nil { + return Handle{}, fmt.Errorf("handoff: selecting a claimable row for %s: %w", fingerprint, err) + } + + h, won, err := q.tryClaim(ctx, handoffID, workerID) + if err != nil { + return Handle{}, err + } + if !won { + // Lost the race between SELECT and UPDATE. Re-read to say why. + return Handle{}, q.explainUnclaimable(ctx, fingerprint) + } + return h, nil +} + +// AcquireLease takes the lease on the oldest claimable finding, whichever it +// is. This is O.3's entry point; Claim is the same operation with a +// fingerprint filter. +// +// It returns ErrNoWork when nothing is claimable, which is the idle case and +// not a failure. +func (q *Queue) AcquireLease(workerID string) (Handle, error) { + return q.AcquireLeaseContext(context.Background(), workerID) +} + +// AcquireLeaseContext is AcquireLease with a caller-supplied context. +func (q *Queue) AcquireLeaseContext(ctx context.Context, workerID string) (Handle, error) { + if workerID == "" { + return Handle{}, errors.New("handoff: AcquireLease requires a non-empty workerID") + } + + // Bounded retry: each lost race consumes one candidate, and the loop + // re-selects rather than spinning on the same row. The bound exists so a + // pathological producer inserting ready rows faster than this worker can + // lose races cannot wedge the call forever. + const maxRaces = 64 + for i := 0; i < maxRaces; i++ { + var handoffID int64 + err := q.db.QueryRowContext(ctx, + `SELECT h.handoff_id`+eligibleFrom+` + ORDER BY h.created_at, h.handoff_id + LIMIT 1`, eligibleArgs()...).Scan(&handoffID) + if errors.Is(err, sql.ErrNoRows) { + return Handle{}, ErrNoWork + } + if err != nil { + return Handle{}, fmt.Errorf("handoff: selecting a claimable row: %w", err) + } + + h, won, err := q.tryClaim(ctx, handoffID, workerID) + if err != nil { + return Handle{}, err + } + if won { + return h, nil + } + } + return Handle{}, fmt.Errorf("handoff: lost %d consecutive claim races: %w", maxRaces, ErrNoWork) +} + +// tryClaim is THE claim. One conditional UPDATE, guarded on `state = 'ready'` +// AND on no sibling row for this fingerprint holding a live lease at this +// record version, is what makes the protocol atomic: SQLite serialises +// writers, so of any number of concurrent callers exactly one sees +// RowsAffected() == 1. +// +// The sibling guard is written against handoff_id rather than against a value +// this function read earlier, so the fingerprint and the audit_version it +// compares are the ones the database holds AT THE MOMENT OF THE WRITE. Reading +// them in Go first and passing them down would reintroduce the window the +// guard exists to close. See noSiblingLease. +// +// attempts is incremented here, at claim time, not at release time. That is +// what makes the counter a record of attempts STARTED, which is the only +// counter a crashed consumer can be measured by — a consumer that dies never +// gets to increment anything itself. +func (q *Queue) tryClaim(ctx context.Context, handoffID int64, workerID string) (Handle, bool, error) { + now := q.Now() + expiry := formatTime(now.Add(q.opts.lease())) + + res, err := q.db.ExecContext(ctx, + `UPDATE handoff + SET state = ?, claimed_by = ?, lease_expires_at = ?, + attempts = attempts + 1, updated_at = ? + WHERE handoff_id = ? AND state = ? + AND NOT EXISTS ( + SELECT 1 + FROM handoff o + JOIN audit_record oa ON oa.audit_record_id = o.audit_record_id + WHERE o.state = ? + AND o.handoff_id <> ? + AND o.fingerprint = (SELECT t.fingerprint FROM handoff t WHERE t.handoff_id = ?) + AND oa.audit_version = (SELECT ta.audit_version + FROM audit_record ta + JOIN handoff th ON th.audit_record_id = ta.audit_record_id + WHERE th.handoff_id = ?) + )`, + string(record.HandoffStateLeased), workerID, expiry, formatTime(now), + handoffID, string(record.HandoffStateReady), + string(record.HandoffStateLeased), handoffID, handoffID, handoffID) + if err != nil { + return Handle{}, false, fmt.Errorf("handoff: claiming row %d: %w", handoffID, err) + } + n, err := res.RowsAffected() + if err != nil { + return Handle{}, false, fmt.Errorf("handoff: claiming row %d: %w", handoffID, err) + } + if n == 0 { + return Handle{}, false, nil + } + + h, err := q.handleFor(ctx, handoffID, expiry) + if err != nil { + return Handle{}, false, err + } + return h, true, nil +} + +// handleFor reads back the row this worker just claimed, joined to the audit +// record for the version and DAST status the Handle carries. +func (q *Queue) handleFor(ctx context.Context, handoffID int64, leaseToken string) (Handle, error) { + row := q.db.QueryRowContext(ctx, + `SELECT `+rowColumns+`, a.audit_version, a.dast_status + FROM handoff h + JOIN audit_record a ON a.audit_record_id = h.audit_record_id + WHERE h.handoff_id = ?`, handoffID) + + var ( + r Row + groupID sql.NullString + claimedBy sql.NullString + leaseExpiry sql.NullString + idemKey sql.NullString + state string + class string + createdAt string + updatedAt string + version int64 + dastStatus string + ) + if err := row.Scan( + &r.HandoffID, &r.FindingID, &r.AuditRecordID, &r.Fingerprint, &groupID, &state, + &class, &claimedBy, &leaseExpiry, &r.Attempts, &r.MaxAttempts, + &idemKey, &createdAt, &updatedAt, &version, &dastStatus, + ); err != nil { + return Handle{}, fmt.Errorf("handoff: reading back claimed row %d: %w", handoffID, err) + } + if err := record.ValidateConsumptionClass(class); err != nil { + return Handle{}, fmt.Errorf("handoff: row %d: %w", handoffID, err) + } + if err := record.ValidateDastStatus(dastStatus); err != nil { + return Handle{}, fmt.Errorf("handoff: audit_record %d: %w", r.AuditRecordID, err) + } + expiresAt, err := parseTime("handoff.lease_expires_at", leaseExpiry.String) + if err != nil { + return Handle{}, err + } + + h := Handle{ + HandoffID: r.HandoffID, + FindingID: r.FindingID, + AuditRecordID: r.AuditRecordID, + Fingerprint: r.Fingerprint, + WorkerID: claimedBy.String, + RecordVersion: version, + ConsumptionClass: record.ConsumptionClass(class), + DastStatus: record.DastStatus(dastStatus), + Attempt: r.Attempts, + MaxAttempts: r.MaxAttempts, + LeaseExpiresAt: expiresAt, + IdempotencyKey: idemKey.String, + leaseToken: leaseToken, + } + if path, err := q.PacketPath(r.Fingerprint); err == nil { + h.PacketPath = path + } + return h, nil +} + +// explainUnclaimable turns "the eligibility query matched nothing" into the +// specific reason, by re-reading the rows for that fingerprint. The +// classification order matters: a losing racer must learn ErrAlreadyClaimed, +// not ErrNotEligible. +// +// Reporting ErrAlreadyClaimed when ANY row for the fingerprint is leased is +// exact, not approximate: since noSiblingLease, one live lease on a +// fingerprint at a record version blocks every sibling row for it, so "someone +// holds this finding" is precisely what a caller needs to hear. +func (q *Queue) explainUnclaimable(ctx context.Context, fingerprint string) error { + rows, err := q.FindContext(ctx, fingerprint) + if err != nil { + return err + } + if len(rows) == 0 { + return fmt.Errorf("handoff: fingerprint %s: %w", fingerprint, ErrNotFound) + } + for _, r := range rows { + if r.State == record.HandoffStateLeased { + return fmt.Errorf("handoff: fingerprint %s is held by %q until %s: %w", + fingerprint, r.ClaimedBy, formatTime(r.LeaseExpiresAt), ErrAlreadyClaimed) + } + } + for _, r := range rows { + if r.State == record.HandoffStateReady && r.AttemptsRemaining() == 0 { + return fmt.Errorf("handoff: fingerprint %s has used %d of %d attempts: %w", + fingerprint, r.Attempts, r.MaxAttempts, ErrExhausted) + } + } + for _, r := range rows { + if r.State == record.HandoffStateReady { + return fmt.Errorf("handoff: fingerprint %s is ready but its %s gate is shut: %w", + fingerprint, r.ConsumptionClass, ErrNotEligible) + } + } + return fmt.Errorf("handoff: fingerprint %s is terminal (%s): %w", + fingerprint, rows[0].State, ErrNotEligible) +} + +// RenewLease is the heartbeat. It extends the lease by Options.Lease from now +// and returns a fresh Handle; the old one is dead and must be discarded. +// +// It refuses, with ErrLeaseLost, if the row is no longer leased by this worker +// on this exact lease — which is precisely the case after a crash-and-reclaim. +// It refuses with ErrRecordVersionChanged if audit_record.audit_version moved, +// because a version bump re-cuts the queue and this work unit no longer exists. +// +// A heartbeat that arrives slightly after lease_expires_at but before the +// reaper has acted is honoured: the holder is demonstrably alive, and the +// reaper's own CAS will then find the lease moved and leave it alone. +func (q *Queue) RenewLease(h Handle) (Handle, error) { + return q.RenewLeaseContext(context.Background(), h) +} + +// RenewLeaseContext is RenewLease with a caller-supplied context. +func (q *Queue) RenewLeaseContext(ctx context.Context, h Handle) (Handle, error) { + if err := q.checkRecordVersion(ctx, h); err != nil { + return Handle{}, err + } + + now := q.Now() + expiry := formatTime(now.Add(q.opts.lease())) + res, err := q.db.ExecContext(ctx, + `UPDATE handoff SET lease_expires_at = ?, updated_at = ? + WHERE handoff_id = ? AND state = ? AND claimed_by = ? AND lease_expires_at = ?`, + expiry, formatTime(now), + h.HandoffID, string(record.HandoffStateLeased), h.WorkerID, h.leaseToken) + if err != nil { + return Handle{}, fmt.Errorf("handoff: renewing lease on row %d: %w", h.HandoffID, err) + } + n, err := res.RowsAffected() + if err != nil { + return Handle{}, fmt.Errorf("handoff: renewing lease on row %d: %w", h.HandoffID, err) + } + if n == 0 { + return Handle{}, q.explainLeaseLost(ctx, h) + } + return q.handleFor(ctx, h.HandoffID, expiry) +} + +// ReleaseLease ends one attempt and records its outcome. +// +// `to` must be a legal successor of 'leased' — one of the eleven dispositions, +// or 'ready' to hand the finding back unattempted-in-effect for another worker +// to pick up. It is checked against the state machine before anything is +// written, so an out-of-vocabulary or out-of-order value never reaches the +// database. +// +// 'validated' carries ONE further condition, because it is the one disposition +// that asserts a defect is verified fixed: a requires_dynamic_confirmation +// finding needs dynamic evidence to have existed. See checkDynamicEvidence. +// +// THE CRASH CASE. The UPDATE is conditional on (state='leased', claimed_by, +// lease_expires_at) still matching this Handle. An OOM-killed consumer whose +// lease expired and was reclaimed, and whose process then came back and tried +// to report success, affects zero rows and gets ErrLeaseLost. Its successor's +// work is untouched. That is the mechanism by which reclaiming and +// re-processing is idempotent: not a lock, a compare-and-swap on the exact +// lease. +func (q *Queue) ReleaseLease(h Handle, to record.HandoffState) error { + return q.ReleaseLeaseContext(context.Background(), h, to) +} + +// ReleaseLeaseContext is ReleaseLease with a caller-supplied context. +func (q *Queue) ReleaseLeaseContext(ctx context.Context, h Handle, to record.HandoffState) error { + if err := CheckTransition(record.HandoffStateLeased, to); err != nil { + return err + } + if err := checkDynamicEvidence(h, to); err != nil { + return err + } + if err := q.checkRecordVersion(ctx, h); err != nil { + return err + } + + res, err := q.db.ExecContext(ctx, + `UPDATE handoff + SET state = ?, claimed_by = NULL, lease_expires_at = NULL, updated_at = ? + WHERE handoff_id = ? AND state = ? AND claimed_by = ? AND lease_expires_at = ?`, + string(to), formatTime(q.Now()), + h.HandoffID, string(record.HandoffStateLeased), h.WorkerID, h.leaseToken) + if err != nil { + return fmt.Errorf("handoff: releasing row %d as %s: %w", h.HandoffID, to, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("handoff: releasing row %d as %s: %w", h.HandoffID, to, err) + } + if n == 0 { + return q.explainLeaseLost(ctx, h) + } + + // A finished finding has no further use for its cache file. The state + // change above is already durable; a failure to unlink is reported but + // does not un-finish the finding. + if IsTerminal(to) { + if err := q.DropPacket(h.Fingerprint); err != nil { + return fmt.Errorf("handoff: row %d is %s but its packet remains: %w", h.HandoffID, to, err) + } + } + return nil +} + +// HasDynamicEvidence reports whether a DAST half that ended in this state can +// have produced a reproduction of a finding. +// +// It is true for exactly two of the ten anvil/dastStatus literals: +// +// - completed_findings — the half ran and produced dynamic findings. +// - completed_partial — the half ran to its own conclusion over part of the +// discovered surface, so a reproduction for this finding may exist; which +// endpoints were probed is DastCoverage's business, not the queue's. +// +// It is false for the other eight, and each exclusion is deliberate: +// +// - not_run, skipped_no_manifest — the half never scanned anything. +// - running — the half has not concluded. +// - target_boot_failed, target_unreachable — there was no live target, which +// is the case plan/00-SPINE.md S6 exists to keep distinguishable from +// "scanned clean". +// - timed_out, completed_failed — the half did not finish; a crashed +// or truncated scan has produced no verdict about this finding. +// - completed_clean — the half DID scan and found nothing +// dynamically. This is the subtle one, and it is excluded on purpose: a +// requires_dynamic_confirmation finding on an audit whose DAST half came +// back clean has no dynamic reproduction at all, so there is nothing that +// can "now fail". Such a finding wants triage, not a validation verdict. +func HasDynamicEvidence(s record.DastStatus) bool { + switch s { + case record.DastStatusCompletedFindings, record.DastStatusCompletedPartial: + return true + default: + return false + } +} + +// checkDynamicEvidence enforces plan/00-SPINE.md S7 at the one place a verdict +// is actually written into the database. +// +// S7: "Only a DAST reproduction that now fails earns 'verified fixed'." This +// package's own doc has always said so, and then let any lease holder release +// ANY finding as 'validated' regardless of its ConsumptionClass and +// DastStatus — including a requires_dynamic_confirmation finding whose DAST +// half was 'not_run', where no reproduction can exist to have been re-run. +// CRITIQUE-02 F9 reproduced that. "The judgement is made elsewhere" is not an +// answer when handoff.state = 'validated' is written HERE, by the claimant, +// unchecked; S7 is "enforce in code, not documentation". +// +// The rule is scoped to the class that asks for it. A static_only finding is +// by definition one no dynamic evidence was ever required for, and refusing +// its 'validated' would make the disposition unreachable for the majority of +// findings. What is refused is the combination the spine forbids: a finding +// whose class SAYS it needs dynamic confirmation, recorded as verified fixed +// on the strength of a static rescan. +func checkDynamicEvidence(h Handle, to record.HandoffState) error { + if to != record.HandoffStateValidated { + return nil + } + if h.ConsumptionClass != record.ConsumptionClassRequiresDynamicConfirmation { + return nil + } + if HasDynamicEvidence(h.DastStatus) { + return nil + } + return fmt.Errorf( + "handoff: row %d is %s and its audit's DAST half ended %q, so no dynamic reproduction exists to have been re-run: %w", + h.HandoffID, record.ConsumptionClassRequiresDynamicConfirmation, h.DastStatus, ErrNoDynamicEvidence) +} + +// checkRecordVersion enforces the (fingerprint, record version) key. A lease +// is only valid at the audit_version it was granted at. +func (q *Queue) checkRecordVersion(ctx context.Context, h Handle) error { + var version int64 + err := q.db.QueryRowContext(ctx, + `SELECT audit_version FROM audit_record WHERE audit_record_id = ?`, h.AuditRecordID).Scan(&version) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("handoff: audit_record %d: %w", h.AuditRecordID, ErrNotFound) + } + if err != nil { + return fmt.Errorf("handoff: reading audit_record %d version: %w", h.AuditRecordID, err) + } + if version != h.RecordVersion { + return fmt.Errorf("handoff: audit_record %d moved from version %d to %d: %w", + h.AuditRecordID, h.RecordVersion, version, ErrRecordVersionChanged) + } + return nil +} + +// explainLeaseLost says which way the lease was lost, without ever implying +// the caller may proceed. +func (q *Queue) explainLeaseLost(ctx context.Context, h Handle) error { + current, err := q.GetContext(ctx, h.HandoffID) + if err != nil { + return err + } + switch { + case current.State != record.HandoffStateLeased: + return fmt.Errorf("handoff: row %d is now %s, not leased by %q: %w", + h.HandoffID, current.State, h.WorkerID, ErrLeaseLost) + case current.ClaimedBy != h.WorkerID: + return fmt.Errorf("handoff: row %d is leased by %q, not %q: %w", + h.HandoffID, current.ClaimedBy, h.WorkerID, ErrLeaseLost) + default: + return fmt.Errorf("handoff: row %d holds a newer lease for %q (expires %s): %w", + h.HandoffID, h.WorkerID, formatTime(current.LeaseExpiresAt), ErrLeaseLost) + } +} + +// --------------------------------------------------------------------------- +// The regenerable tmpfs packet. +// +// It is a CACHE. The store is the source of truth (plan/00-SPINE.md S1: one +// SQLite store, one handoff table, a regenerable tmpfs packet — there is no +// second durable buffer file). If a packet is missing, regenerate it; its +// absence is never an error and never loses a finding. +// --------------------------------------------------------------------------- + +// ErrNoPacketDir means the Queue was configured without a PacketDir, so there +// is nowhere to materialise a packet. Reading it as fatal would be wrong: a +// deployment that hands the consumer bytes instead of a path needs no packet +// directory at all. +var ErrNoPacketDir = errors.New("handoff: no PacketDir is configured") + +// PacketPath returns where a fingerprint's packet lives. The fingerprint is +// validated as 64 hex first, which is also what stops a crafted value from +// escaping PacketDir. +func (q *Queue) PacketPath(fingerprint string) (string, error) { + if err := ValidateFingerprint(fingerprint); err != nil { + return "", err + } + if q.opts.PacketDir == "" { + return "", ErrNoPacketDir + } + return filepath.Join(q.opts.PacketDir, fingerprint+".sarif"), nil +} + +// packetGate is R.6's read gate, re-asserted at the packet. +// +// THE PACKET IS A CACHE OF THE PAYLOAD, NOT A SEPARATE ARTEFACT. R.6's gate — +// "do not allow a consumer to read a half's results before that half's status +// equals sealed" — means nothing if the same bytes are reachable through a +// file beside it. CRITIQUE-02 F5: WritePacket materialised a packet for an +// audit that had sealed nothing, and ReadPacket returned a half's actual +// results with no seal check, no audit-state check and no lease check, one +// line after the claim gate had correctly refused the same fingerprint. +// ReadPacket was the ONLY exported function in this package that returns a +// half's results, and it was the one that checked nothing. +// +// It re-asserts three things, all in one statement against the database rather +// than against the Handle's own copy of them, because a Handle is a snapshot +// and the whole point is that the world may have moved: +// +// 1. This exact lease is still held — (state='leased', claimed_by, +// lease_expires_at), the same CAS triple RenewLease and ReleaseLease use. +// A reclaimed holder gets ErrLeaseLost here too, so it cannot read the +// successor's packet. +// 2. The audit's consumption gate is STILL open for this row's class, using +// the same consumptionGate expression the claim uses. One definition, two +// call sites. +// 3. The record version has not moved (checkRecordVersion), because S6 re-cuts +// the queue on a bump and the packet then describes work that is gone. +func (q *Queue) packetGate(ctx context.Context, h Handle) error { + if err := ValidateFingerprint(h.Fingerprint); err != nil { + return err + } + if h.WorkerID == "" { + return errors.New("handoff: a packet operation requires a Handle from Claim or AcquireLease") + } + if err := q.checkRecordVersion(ctx, h); err != nil { + return err + } + + args := []any{ + h.HandoffID, h.Fingerprint, + string(record.HandoffStateLeased), h.WorkerID, h.leaseToken, + } + args = append(args, gateArgs()...) + + var one int + err := q.db.QueryRowContext(ctx, + `SELECT 1 + FROM handoff h + JOIN audit_record a ON a.audit_record_id = h.audit_record_id + WHERE h.handoff_id = ? + AND h.fingerprint = ? + AND h.state = ? + AND h.claimed_by = ? + AND h.lease_expires_at = ? + AND `+consumptionGate, args...).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + // Either the lease moved or the gate is shut. Say which. + current, gerr := q.GetContext(ctx, h.HandoffID) + if gerr != nil { + return gerr + } + if current.State != record.HandoffStateLeased || + current.ClaimedBy != h.WorkerID || + formatTime(current.LeaseExpiresAt) != h.leaseToken { + return q.explainLeaseLost(ctx, h) + } + return fmt.Errorf("handoff: row %d holds a lease but its %s gate is shut; "+ + "R.6 refuses a read before the half seals: %w", + h.HandoffID, current.ConsumptionClass, ErrNotEligible) + } + if err != nil { + return fmt.Errorf("handoff: checking the packet gate for row %d: %w", h.HandoffID, err) + } + return nil +} + +// WritePacket materialises a packet for a finding this worker holds the lease +// on, using research/08 §C's durable recipe: +// write an exclusively-created temp file in the SAME directory, fsync it, +// close it, rename it over the final name, then fsync the parent directory. +// +// It takes a Handle, not a bare fingerprint, because the packet carries a +// half's results and R.6's read gate governs those bytes wherever they live. +// See packetGate. +// +// The parent fsync is not optional decoration. fsync(2): "Calling fsync() does +// not necessarily ensure that the entry in the directory containing the file +// has also reached disk. For that an explicit fsync() on a file descriptor for +// the directory is also needed." Omitting it passes every test on ext4 — +// because auto_da_alloc papers over it — and loses data on XFS or +// data=writeback. This code does not depend on that heuristic. +// +// Windows has no fsync-able directory handle, so the parent fsync is skipped +// there and only there. Anvil's packet lives on tmpfs on Linux; Windows is a +// development host. +func (q *Queue) WritePacket(h Handle, data []byte) (string, error) { + return q.WritePacketContext(context.Background(), h, data) +} + +// WritePacketContext is WritePacket with a caller-supplied context. +func (q *Queue) WritePacketContext(ctx context.Context, h Handle, data []byte) (string, error) { + if err := q.packetGate(ctx, h); err != nil { + return "", err + } + fingerprint := h.Fingerprint + final, err := q.PacketPath(fingerprint) + if err != nil { + return "", err + } + dir := filepath.Dir(final) + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("handoff: creating packet directory %s: %w", dir, err) + } + + tmp, err := os.CreateTemp(dir, fingerprint+".*.tmp") + if err != nil { + return "", fmt.Errorf("handoff: creating packet temp file in %s: %w", dir, err) + } + tmpName := tmp.Name() + cleanup := func() { + _ = tmp.Close() + _ = os.Remove(tmpName) + } + if _, err := tmp.Write(data); err != nil { + cleanup() + return "", fmt.Errorf("handoff: writing packet %s: %w", tmpName, err) + } + if err := tmp.Sync(); err != nil { + cleanup() + return "", fmt.Errorf("handoff: fsyncing packet %s: %w", tmpName, err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return "", fmt.Errorf("handoff: closing packet %s: %w", tmpName, err) + } + if err := os.Rename(tmpName, final); err != nil { + _ = os.Remove(tmpName) + return "", fmt.Errorf("handoff: publishing packet %s: %w", final, err) + } + if err := syncDir(dir); err != nil { + return "", fmt.Errorf("handoff: fsyncing packet directory %s: %w", dir, err) + } + return final, nil +} + +// ReadPacket returns a packet's bytes to the worker that holds the lease on +// the finding, and to nobody else. +// +// It is the only exported function in this package that hands back a half's +// actual results, so it is gated exactly as R.6 gates a half read: the lease +// must still be this Handle's, the record version must not have moved, and the +// audit's consumption gate must be open. See packetGate for why a cache of the +// payload cannot be less protected than the payload. +// +// A missing packet is reported with os.ErrNotExist so the caller can +// regenerate rather than fail — the packet is a cache and its absence is never +// an error in itself (research/08 §1). +func (q *Queue) ReadPacket(h Handle) ([]byte, error) { + return q.ReadPacketContext(context.Background(), h) +} + +// ReadPacketContext is ReadPacket with a caller-supplied context. +func (q *Queue) ReadPacketContext(ctx context.Context, h Handle) ([]byte, error) { + if err := q.packetGate(ctx, h); err != nil { + return nil, err + } + path, err := q.PacketPath(h.Fingerprint) + if err != nil { + return nil, err + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("handoff: reading packet %s: %w", path, err) + } + return data, nil +} + +// DropPacket unlinks a packet. Nothing more. +// +// This is an unlink, not an erasure, and this package makes no claim that the +// bytes are unrecoverable afterwards. research/08 §F establishes why any such +// claim would be false: shred(1) "assumes the file system and hardware +// overwrite data in place", which does not hold for Btrfs, ZFS, XFS, ext3/4 in +// data=journal, compressed, snapshotting or RAID filesystems, and on SSDs wear +// levelling means "'overwritten' data blocks are still present in the +// underlying device". The control that actually applies is keeping the +// plaintext packet off persistent media in the first place — tmpfs, ideally +// with noswap — which is a deployment property, not something code can assert. +// +// Dropping a packet that is not there is success, because the packet is a +// cache and its absence is the desired state. +func (q *Queue) DropPacket(fingerprint string) error { + path, err := q.PacketPath(fingerprint) + if errors.Is(err, ErrNoPacketDir) { + return nil + } + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("handoff: unlinking packet %s: %w", path, err) + } + return nil +} + +// syncDir fsyncs a directory so a rename into it is durable. See WritePacket +// for why, and for why Windows is exempt. +func syncDir(dir string) error { + if runtime.GOOS == "windows" { + return nil + } + d, err := os.Open(dir) + if err != nil { + return err + } + defer func() { _ = d.Close() }() + return d.Sync() +} diff --git a/internal/handoff/critique02_regression_test.go b/internal/handoff/critique02_regression_test.go new file mode 100644 index 0000000..c022d9f --- /dev/null +++ b/internal/handoff/critique02_regression_test.go @@ -0,0 +1,687 @@ +// Regression tests for the defects CRITIQUE-02 found and the fix round closed. +// +// Each test here reproduces one ORIGINAL defect. They were written by the +// critic and the re-verifier as probes -- to prove a defect existed, and then +// to prove it was actually gone rather than merely claimed gone. They are kept +// permanently, and deliberately named after the finding they pin, because a +// fixed defect with no test is a defect waiting to return. +// +// Two of them earn their place especially: +// - the masking probes build a MINIMAL record and assert the planted secret +// appears exactly once before masking, so they cannot pass by propagation +// from a header. That false-confidence pattern was itself a finding. +// - the lease probes drive concurrent goroutines at the granting statement +// rather than asserting on a single-threaded happy path. + +package handoff + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// --------------------------------------------------------------------------- +// Independent re-verification probes for CRITIQUE-02 B1 / B2 / M2 / M4 / M5. +// Each is written to reproduce the ORIGINAL defect, not to confirm the fix. +// --------------------------------------------------------------------------- + +// enqueueOn seeds one ready row for an existing finding on a given audit +// record, which is the shape F1 needs: ONE fingerprint, TWO audit_record rows, +// both at audit_version 1. +func (f *fixture) enqueueOn(findingID int64, fingerprint string, class record.ConsumptionClass, auditRecID int64) Row { + f.t.Helper() + row, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, + AuditRecordID: auditRecID, + AuditID: auditUUID(auditRecID), + Fingerprint: fingerprint, + ConsumptionClass: class, + }) + if err != nil { + f.t.Fatalf("Enqueue on audit %d: %v", auditRecID, err) + } + return row +} + +// --------------------------------------------------------------------------- +// B1 — two live leases on one finding at one record version. +// --------------------------------------------------------------------------- + +func TestProbeB1DoubleGrantAcrossAuditRecords(t *testing.T) { + f := newFixture(t, Options{}) + + a1 := f.sealedAudit() + a2 := f.sealedAudit() // second scan of the same target -> new audit_record + fingerprint := fp(7) + findingID := f.newFinding(fingerprint) + r1 := f.enqueueOn(findingID, fingerprint, record.ConsumptionClassStaticOnly, a1) + r2 := f.enqueueOn(findingID, fingerprint, record.ConsumptionClassStaticOnly, a2) + + if r1.HandoffID == r2.HandoffID { + t.Fatalf("fixture guard: expected two distinct handoff rows, got one (%d)", r1.HandoffID) + } + // Both must be at audit_version 1, or the probe is not reproducing F1. + for _, id := range []int64{a1, a2} { + var v int64 + if err := f.db.QueryRow(`SELECT audit_version FROM audit_record WHERE audit_record_id = ?`, id).Scan(&v); err != nil { + t.Fatalf("read audit_version: %v", err) + } + if v != 1 { + t.Fatalf("fixture guard: audit_record %d is at version %d, want 1", id, v) + } + } + t.Logf("two rows for ONE fingerprint: handoff_id %d (audit %d) and %d (audit %d), both version 1", + r1.HandoffID, a1, r2.HandoffID, a2) + + // worker-1 takes the OLDEST row (AcquireLease's ordering). + h1, err := f.q.AcquireLease("worker-1") + if err != nil { + t.Fatalf("AcquireLease(worker-1): %v", err) + } + if h1.HandoffID != r1.HandoffID { + t.Logf("note: AcquireLease took handoff %d, not the oldest %d", h1.HandoffID, r1.HandoffID) + } + + // worker-2 goes through Claim, which used to pick the NEWEST row. + h2, err := f.q.Claim(fingerprint, "worker-2") + if err == nil { + t.Errorf("B1 REPRODUCED: DOUBLE GRANT — worker-1 holds handoff=%d (idem=%s) and "+ + "worker-2 holds handoff=%d (idem=%s), both on fingerprint %s at record version 1", + h1.HandoffID, h1.IdempotencyKey, h2.HandoffID, h2.IdempotencyKey, fingerprint) + } else if !errors.Is(err, ErrAlreadyClaimed) { + t.Errorf("Claim refused, but with %v; want ErrAlreadyClaimed so a caller can back off correctly", err) + } else { + t.Logf("Claim refused as required: %v", err) + } + + // And the other way round: a second AcquireLease must find nothing. + if h3, err := f.q.AcquireLease("worker-3"); err == nil { + t.Errorf("B1 REPRODUCED via AcquireLease: worker-3 also got handoff=%d on %s", + h3.HandoffID, fingerprint) + } else if !errors.Is(err, ErrNoWork) { + t.Errorf("second AcquireLease returned %v, want ErrNoWork", err) + } + + // Verify no two rows are simultaneously 'leased'. + var leased int + if err := f.db.QueryRow(`SELECT COUNT(*) FROM handoff WHERE fingerprint = ? AND state = ?`, + fingerprint, string(record.HandoffStateLeased)).Scan(&leased); err != nil { + t.Fatalf("count leased: %v", err) + } + if leased != 1 { + t.Errorf("B1 REPRODUCED: %d rows for %s are 'leased' at once, want 1", leased, fingerprint) + } +} + +// The sibling guard must survive concurrency, not merely sequential ordering. +func TestProbeB1ConcurrentDoubleGrant(t *testing.T) { + f := newFixture(t, Options{}) + + const audits = 4 + auditIDs := make([]int64, 0, audits) + for i := 0; i < audits; i++ { + auditIDs = append(auditIDs, f.sealedAudit()) + } + fingerprint := fp(9) + findingID := f.newFinding(fingerprint) + for _, a := range auditIDs { + f.enqueueOn(findingID, fingerprint, record.ConsumptionClassStaticOnly, a) + } + + const workers = 8 + var wg sync.WaitGroup + var mu sync.Mutex + granted := []Handle{} + wg.Add(workers) + for i := 0; i < workers; i++ { + go func(i int) { + defer wg.Done() + h, err := f.q.Claim(fingerprint, fmt.Sprintf("w%d", i)) + if err != nil { + return + } + mu.Lock() + granted = append(granted, h) + mu.Unlock() + }(i) + } + wg.Wait() + if len(granted) != 1 { + t.Errorf("B1 REPRODUCED under concurrency: %d simultaneous leases on %s at version 1: %+v", + len(granted), fingerprint, granted) + } else { + t.Logf("exactly one of %d concurrent claimants won: handoff=%d", workers, granted[0].HandoffID) + } +} + +// Release must re-open the fingerprint for a sibling row (the guard must not +// wedge the queue permanently). +func TestProbeB1GuardDoesNotWedgeTheQueue(t *testing.T) { + f := newFixture(t, Options{}) + aX := f.sealedAudit() + aY := f.sealedAudit() + fingerprint := fp(11) + findingID := f.newFinding(fingerprint) + f.enqueueOn(findingID, fingerprint, record.ConsumptionClassStaticOnly, aX) + r2 := f.enqueueOn(findingID, fingerprint, record.ConsumptionClassStaticOnly, aY) + + h, err := f.q.Claim(fingerprint, "w1") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if err := f.q.ReleaseLease(h, record.HandoffStateReady); err != nil { + t.Fatalf("ReleaseLease(ready): %v", err) + } + h2, err := f.q.Claim(fingerprint, "w2") + if err != nil { + t.Fatalf("after release, Claim is still refused (%v); the sibling guard wedged the queue "+ + "(sibling row %d never becomes claimable)", err, r2.HandoffID) + } + t.Logf("re-claimable after release: handoff=%d attempt=%d/%d", h2.HandoffID, h2.Attempt, h2.MaxAttempts) +} + +// --------------------------------------------------------------------------- +// B2 — a consumed audit must not shut the eligibility gate (S1 re-entrancy). +// --------------------------------------------------------------------------- + +func TestProbeB2ConsumedAuditKeepsTheQueueOpen(t *testing.T) { + for _, tc := range []struct { + name string + class record.ConsumptionClass + dast record.DastStatus + }{ + {"static_only", record.ConsumptionClassStaticOnly, record.DastStatusCompletedClean}, + {"requires_dynamic_confirmation", record.ConsumptionClassRequiresDynamicConfirmation, record.DastStatusCompletedFindings}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t, Options{}) + auditID := f.newAudit(record.StateConsumed, record.HalfStatusSealed, tc.dast, + f.clock.Now().Add(8*time.Hour)) + + // Two sibling findings on ONE consumed audit: the fan-out shape F2 + // describes, where the first consumption pass strands the rest. + fpA, _ := f.enqueue(21, tc.class, auditID) + fpB, _ := f.enqueue(22, tc.class, auditID) + + hA, err := f.q.Claim(fpA, "worker-a") + if err != nil { + t.Errorf("B2 REPRODUCED: Claim on a CONSUMED audit refused: %v", err) + } + hB, err := f.q.AcquireLease("worker-b") + if err != nil { + t.Errorf("B2 REPRODUCED: AcquireLease found no work on a CONSUMED audit "+ + "(sibling %s is still ready): %v", fpB, err) + } else if hB.Fingerprint != fpB { + t.Errorf("AcquireLease returned %s, want the sibling %s", hB.Fingerprint, fpB) + } + if err == nil { + t.Logf("re-entrant: %s and %s both claimable after consumption (handoffs %d, %d)", + fpA, fpB, hA.HandoffID, hB.HandoffID) + } + }) + } +} + +// And the gate must still be SHUT for the states it is supposed to refuse, so +// the B2 fix is not simply "open the gate for everything". +func TestProbeB2GateStillShutsWhereItMust(t *testing.T) { + for _, tc := range []struct { + name string + state record.State + sast record.HalfStatus + dast record.DastStatus + class record.ConsumptionClass + }{ + {"collecting/static_only", record.StateCollecting, record.HalfStatusRunning, record.DastStatusNotRun, record.ConsumptionClassStaticOnly}, + {"expired/static_only", record.StateExpired, record.HalfStatusSealed, record.DastStatusCompletedClean, record.ConsumptionClassStaticOnly}, + {"expired/dynamic", record.StateExpired, record.HalfStatusSealed, record.DastStatusCompletedFindings, record.ConsumptionClassRequiresDynamicConfirmation}, + {"sast_sealed/dynamic", record.StateSastSealed, record.HalfStatusSealed, record.DastStatusRunning, record.ConsumptionClassRequiresDynamicConfirmation}, + {"dast running on both_sealed/dynamic", record.StateBothSealed, record.HalfStatusSealed, record.DastStatusRunning, record.ConsumptionClassRequiresDynamicConfirmation}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t, Options{}) + auditID := f.newAudit(tc.state, tc.sast, tc.dast, f.clock.Now().Add(8*time.Hour)) + fingerprint, _ := f.enqueue(31, tc.class, auditID) + if h, err := f.q.Claim(fingerprint, "w"); err == nil { + t.Errorf("the gate is OPEN for %s: handoff %d was granted", tc.name, h.HandoffID) + } else if !errors.Is(err, ErrNotEligible) { + t.Errorf("refused with %v, want ErrNotEligible", err) + } + }) + } +} + +// --------------------------------------------------------------------------- +// M2 — ReadPacket / WritePacket must go through the read gate. +// --------------------------------------------------------------------------- + +func TestProbeM2PacketOperationsAreGated(t *testing.T) { + f := newFixture(t, Options{}) + auditID := f.sealedAudit() + fingerprint, row := f.enqueue(41, record.ConsumptionClassStaticOnly, auditID) + + // (1) No Handle at all — the shape of the original F5 reproduction. + if _, err := f.q.WritePacket(Handle{Fingerprint: fingerprint}, []byte(`{"runs":[]}`)); err == nil { + t.Errorf("M2 REPRODUCED: WritePacket succeeded with no lease") + } else { + t.Logf("WritePacket(no lease) refused: %v", err) + } + if _, err := f.q.ReadPacket(Handle{Fingerprint: fingerprint}); err == nil { + t.Errorf("M2 REPRODUCED: ReadPacket succeeded with no lease") + } else { + t.Logf("ReadPacket(no lease) refused: %v", err) + } + + // (2) A Handle for a row that is merely READY, not leased. + forged := Handle{ + HandoffID: row.HandoffID, FindingID: row.FindingID, AuditRecordID: auditID, + Fingerprint: fingerprint, WorkerID: "impostor", RecordVersion: 1, + leaseToken: formatTime(f.clock.Now().Add(20 * time.Minute)), + } + if _, err := f.q.WritePacket(forged, []byte(`{"runs":[]}`)); err == nil { + t.Errorf("M2 REPRODUCED: WritePacket succeeded on a READY row for an impostor worker") + } + if _, err := f.q.ReadPacket(forged); err == nil { + t.Errorf("M2 REPRODUCED: ReadPacket succeeded on a READY row for an impostor worker") + } + + // (3) The legitimate holder can write and read. + h, err := f.q.Claim(fingerprint, "worker-1") + if err != nil { + t.Fatalf("Claim: %v", err) + } + path, err := f.q.WritePacket(h, []byte(`{"runs":[{"results":[]}]}`)) + if err != nil { + t.Fatalf("WritePacket by the lease holder: %v", err) + } + if _, err := f.q.ReadPacket(h); err != nil { + t.Fatalf("ReadPacket by the lease holder: %v", err) + } + + // (4) The lease is stolen out from under the holder. The bytes on disk are + // unchanged, so an ungated read would still return them. + if _, err := f.db.Exec(`UPDATE handoff SET claimed_by = ? WHERE handoff_id = ?`, "worker-2", h.HandoffID); err != nil { + t.Fatalf("steal lease: %v", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("packet file vanished: %v", err) + } + if _, err := f.q.ReadPacket(h); err == nil { + t.Errorf("M2 REPRODUCED: a reclaimed holder still read the packet its successor owns") + } else if !errors.Is(err, ErrLeaseLost) { + t.Errorf("stolen-lease read refused with %v, want ErrLeaseLost", err) + } + if _, err := f.db.Exec(`UPDATE handoff SET claimed_by = ? WHERE handoff_id = ?`, "worker-1", h.HandoffID); err != nil { + t.Fatalf("restore lease: %v", err) + } + + // (5) The audit's gate SHUTS after the claim (a re-cut, an expiry). The + // packet must become unreadable even though the lease is still held. + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, sast_status = ? WHERE audit_record_id = ?`, + string(record.StateExpired), string(record.HalfStatusSealed), auditID); err != nil { + t.Fatalf("expire audit: %v", err) + } + if _, err := f.q.ReadPacket(h); err == nil { + t.Errorf("M2 REPRODUCED: ReadPacket returned an EXPIRED audit's results to a live lease holder") + } else { + t.Logf("expired-audit read refused: %v", err) + } + if _, err := f.q.WritePacket(h, []byte(`x`)); err == nil { + t.Errorf("M2 REPRODUCED: WritePacket materialised a packet for an EXPIRED audit") + } + + // (6) An UNSEALED audit: the gate must refuse a write that would create a + // readable cache of a half that has sealed nothing. + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, sast_status = ? WHERE audit_record_id = ?`, + string(record.StateCollecting), string(record.HalfStatusRunning), auditID); err != nil { + t.Fatalf("unseal audit: %v", err) + } + if _, err := f.q.WritePacket(h, []byte(`x`)); err == nil { + t.Errorf("M2 REPRODUCED: WritePacket succeeded on an UNSEALED audit") + } else { + t.Logf("unsealed-audit write refused: %v", err) + } + if _, err := f.q.ReadPacket(h); err == nil { + t.Errorf("M2 REPRODUCED: ReadPacket returned an UNSEALED half's results") + } + + // (7) A record-version bump must also close the packet. + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, sast_status = ?, audit_version = 2 WHERE audit_record_id = ?`, + string(record.StateBothSealed), string(record.HalfStatusSealed), auditID); err != nil { + t.Fatalf("bump version: %v", err) + } + if _, err := f.q.ReadPacket(h); !errors.Is(err, ErrRecordVersionChanged) { + t.Errorf("after a version bump ReadPacket returned %v, want ErrRecordVersionChanged", err) + } +} + +// The whole exported surface: nothing else may hand back a half's results. +func TestProbeM2NoOtherExportedResultsPath(t *testing.T) { + // Row must not carry payload bytes. + var r Row + _ = r + names := []string{"HandoffID", "FindingID", "AuditRecordID", "Fingerprint", "GroupID", + "State", "ConsumptionClass", "ClaimedBy", "LeaseExpiresAt", "Attempts", + "MaxAttempts", "IdempotencyKey", "CreatedAt", "UpdatedAt"} + t.Logf("Row fields (metadata only, no results): %s", strings.Join(names, ", ")) +} + +// --------------------------------------------------------------------------- +// M4 — IdempotencyKey must be audit identity, not the autoincrement rowid. +// --------------------------------------------------------------------------- + +func TestProbeM4IdempotencyKeyUsesAuditIdentity(t *testing.T) { + f := newFixture(t, Options{}) + auditRecID := f.sealedAudit() + fingerprint := fp(51) + findingID := f.newFinding(fingerprint) + + const commit = "9f1c0de9f1c0de9f1c0de9f1c0de9f1c0de9f1c0" + auditIdentity := auditUUID(auditRecID) + + row, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: auditRecID, AuditID: auditIdentity, + Fingerprint: fingerprint, ConsumptionClass: record.ConsumptionClassStaticOnly, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + + want := IdempotencyKey(auditIdentity, fingerprint, commit) + if row.IdempotencyKey != want { + t.Errorf("stored key %s != sha256(auditId||fp||commit) %s", row.IdempotencyKey, want) + } + // The rowid-based key that F7 found must NOT be what is stored. + rowidKey := IdempotencyKey(fmt.Sprintf("%d", auditRecID), fingerprint, commit) + if row.IdempotencyKey == rowidKey { + t.Errorf("M4 REPRODUCED: the stored key equals sha256(audit_record_id||fp||commit); "+ + "it is still keyed on the autoincrement rowid (%d)", auditRecID) + } + + // The key must be invariant under the rowid: the SAME audit identity, the + // same fingerprint and the same commit on a DIFFERENT audit_record row + // must produce the same key. + other := f.sealedAudit() + if other == auditRecID { + t.Fatal("fixture guard: expected a distinct audit_record rowid") + } + if got := IdempotencyKey(auditIdentity, fingerprint, commit); got != want { + t.Errorf("key is not a pure function of (auditId, fingerprint, commit)") + } + + // Enqueue must refuse an empty audit identity rather than fall back to the + // rowid. + if _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: other, AuditID: "", + Fingerprint: fingerprint, ConsumptionClass: record.ConsumptionClassStaticOnly, + }); err == nil { + t.Errorf("M4 REPRODUCED: Enqueue accepted an empty AuditID and derived a key without an audit identity") + } + + // NUL separation: no boundary shift may collide. + a := IdempotencyKey("ab", "cd", "ef") + b := IdempotencyKey("a", "bcd", "ef") + if a == b { + t.Errorf("boundary collision: IdempotencyKey is not domain-separated") + } + + // The Handle carries the same key the row does. + h, err := f.q.Claim(fingerprint, "w1") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if h.IdempotencyKey != want { + t.Errorf("Handle.IdempotencyKey = %s, row = %s", h.IdempotencyKey, want) + } + // Stable across crash and reclaim. + f.clock.Advance(2 * time.Hour) + if _, err := f.q.Reap(); err != nil { + t.Fatalf("Reap: %v", err) + } + h2, err := f.q.Claim(fingerprint, "w2") + if err != nil { + t.Fatalf("Claim after reclaim: %v", err) + } + if h2.IdempotencyKey != want { + t.Errorf("key moved across reclaim: %s -> %s", want, h2.IdempotencyKey) + } +} + +// --------------------------------------------------------------------------- +// M5 — requires_dynamic_confirmation cannot reach 'validated' without evidence. +// --------------------------------------------------------------------------- + +func TestProbeM5ValidatedRequiresDynamicEvidence(t *testing.T) { + type tc struct { + dast record.DastStatus + class record.ConsumptionClass + wantAllow bool + } + cases := []tc{} + for _, d := range record.DastStatusValues() { + if d == record.DastStatusRunning { + continue // the gate refuses the claim outright for the dynamic class + } + allow := d == record.DastStatusCompletedFindings || d == record.DastStatusCompletedPartial + cases = append(cases, tc{d, record.ConsumptionClassRequiresDynamicConfirmation, allow}) + cases = append(cases, tc{d, record.ConsumptionClassStaticOnly, true}) + } + + for _, c := range cases { + name := fmt.Sprintf("%s/%s", c.class, c.dast) + t.Run(name, func(t *testing.T) { + f := newFixture(t, Options{}) + auditRecID, err := f.tryNewAudit(record.StateBothSealed, record.HalfStatusSealed, + c.dast, f.clock.Now().Add(8*time.Hour)) + if err != nil { + t.Skipf("the schema will not hold dast_status=%q: %v", c.dast, err) + } + fingerprint, _ := f.enqueue(61, c.class, auditRecID) + h, err := f.q.Claim(fingerprint, "w1") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if h.DastStatus != c.dast { + t.Fatalf("Handle.DastStatus = %q, want %q", h.DastStatus, c.dast) + } + err = f.q.ReleaseLease(h, record.HandoffStateValidated) + switch { + case c.wantAllow && err != nil: + t.Errorf("ReleaseLease(validated) refused a legitimate case: %v", err) + case !c.wantAllow && err == nil: + t.Errorf("M5 REPRODUCED: a %s finding was recorded 'validated' while dast_status=%q "+ + "(no reproduction can exist)", c.class, c.dast) + case !c.wantAllow && !errors.Is(err, ErrNoDynamicEvidence): + t.Errorf("refused with %v, want ErrNoDynamicEvidence", err) + } + // The database must agree with the return value. + got := f.state(h.HandoffID) + if c.wantAllow && got != record.HandoffStateValidated { + t.Errorf("row state = %q, want validated", got) + } + if !c.wantAllow && got == record.HandoffStateValidated { + t.Errorf("M5 REPRODUCED: the row is 'validated' in the database anyway") + } + }) + } +} + +// The check must not be bypassable through the other write paths. +func TestProbeM5NoBypassRoutesToValidated(t *testing.T) { + f := newFixture(t, Options{}) + auditRecID := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusNotRun, f.clock.Now().Add(8*time.Hour)) + fingerprint, row := f.enqueue(71, record.ConsumptionClassRequiresDynamicConfirmation, auditRecID) + + // Dispose: ready -> validated must be an illegal transition. + if err := f.q.Dispose(row.HandoffID, record.HandoffStateValidated); err == nil { + t.Errorf("M5 BYPASS: Dispose(ready -> validated) succeeded, skipping checkDynamicEvidence") + } else { + t.Logf("Dispose(ready -> validated) refused: %v", err) + } + + // Claim, then release as validated with a hand-edited Handle claiming a + // dast status the database does not hold. The Handle is a snapshot; the + // check must not be satisfiable by lying to it... or, if it is, say so. + h, err := f.q.Claim(fingerprint, "w1") + if err != nil { + t.Fatalf("Claim: %v", err) + } + forged := h + forged.DastStatus = record.DastStatusCompletedFindings + if err := f.q.ReleaseLease(forged, record.HandoffStateValidated); err == nil { + t.Logf("M5 RESIDUAL GAP (reported, not a reproduction of F9): checkDynamicEvidence reads the caller's Handle copy, so a caller that "+ + "overwrites Handle.DastStatus reaches 'validated' with dast_status=%q in the database", + record.DastStatusNotRun) + if got := f.state(h.HandoffID); got == record.HandoffStateValidated { + t.Logf(" and the row is now %q while audit_record.dast_status is still not_run", got) + } + } else { + t.Logf("forged-Handle release refused: %v", err) + } +} + +// --------------------------------------------------------------------------- +// The `completed_failed` amendment, end to end through the real schema. +// --------------------------------------------------------------------------- + +// TestProbeCompletedFailedIsStorable: DeriveDastStatus can now produce +// `completed_failed`, so `audit_record.dast_status` must be able to hold it. +// The Go enum and the DDL CHECK are two copies of one frozen vocabulary; the +// whole point of ck_audit_record_dast_status is that they cannot drift. +func TestProbeCompletedFailedIsStorable(t *testing.T) { + f := newFixture(t, Options{}) + + derived, err := record.DeriveDastStatus(record.HalfStatusFailed, record.DastOutcome{ + TierInstalled: true, Provenance: record.TargetProvenanceBootedClean, + }) + if err != nil { + t.Fatalf("DeriveDastStatus: %v", err) + } + if derived != record.DastStatusCompletedFailed { + t.Fatalf("derived %q, want completed_failed", derived) + } + + auditRecID, err := f.tryNewAudit(record.StateBothSealed, record.HalfStatusSealed, + derived, f.clock.Now().Add(8*time.Hour)) + if err != nil { + t.Fatalf("BLOCKER: internal/record derives dast_status=%q for a DAST half that crashed "+ + "against a live target, and internal/store/schema.sql cannot store it: %v\n"+ + " ck_audit_record_dast_status still lists nine literals; DastStatusValues() lists ten.\n"+ + " Consequence: no audit whose DAST half fails can be persisted at all.", derived, err) + } + + // If it is storable, the queue must also cope with it. + fingerprint, _ := f.enqueue(81, record.ConsumptionClassRequiresDynamicConfirmation, auditRecID) + h, err := f.q.Claim(fingerprint, "w1") + if err != nil { + t.Fatalf("Claim on a completed_failed audit: %v", err) + } + if HasDynamicEvidence(h.DastStatus) { + t.Errorf("completed_failed reports HasDynamicEvidence; a crashed half proves nothing") + } + if err := f.q.ReleaseLease(h, record.HandoffStateValidated); !errors.Is(err, ErrNoDynamicEvidence) { + t.Errorf("ReleaseLease(validated) on completed_failed returned %v, want ErrNoDynamicEvidence", err) + } +} + +// Side effect of the M4 change, probed rather than asserted from reading: +// idempotency_key is UNIQUE table-wide, so re-enqueueing one finding under the +// SAME audit identity but a different audit_record row now collides. +func TestProbeM4SideEffectDuplicateAuditIdentity(t *testing.T) { + f := newFixture(t, Options{}) + a1 := f.sealedAudit() + a2 := f.sealedAudit() + fingerprint := fp(91) + findingID := f.newFinding(fingerprint) + id := auditUUID(a1) + + if _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: a1, AuditID: id, + Fingerprint: fingerprint, ConsumptionClass: record.ConsumptionClassStaticOnly, + }); err != nil { + t.Fatalf("first Enqueue: %v", err) + } + _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: a2, AuditID: id, // same audit identity + Fingerprint: fingerprint, ConsumptionClass: record.ConsumptionClassStaticOnly, + }) + t.Logf("second Enqueue (same auditId, different audit_record row) -> %v", err) + if err == nil { + var n int + _ = f.db.QueryRow(`SELECT COUNT(*) FROM handoff WHERE fingerprint = ?`, fingerprint).Scan(&n) + t.Logf(" rows for the fingerprint: %d", n) + } +} + +// TestRunReportsNoSweepErrorOnCancellation pins the fix for the CI-only failure +// in TestRunSweepsUntilCancelled. +// +// Cancelling ctx while a sweep is mid-query made that query return +// context.Canceled, which Run then handed to observe as a sweep error. On a +// dev host it almost never reproduced; under -race on Linux it reproduced +// reliably, because the race detector widens the window. +// +// The property under test is not "no error is returned" -- Run correctly +// returns ctx.Err(). It is that the OBSERVER, whose entire purpose is to make +// real sweep failures visible, is never told a clean shutdown was a failure. +func TestRunReportsNoSweepErrorOnCancellation(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(4242, record.ConsumptionClassStaticOnly, audit) + if _, err := f.q.Claim(fingerprint, "worker-doomed"); err != nil { + t.Fatalf("Claim: %v", err) + } + f.clock.Advance(21 * time.Minute) + + // Hammer the cancel-during-sweep window rather than hoping to hit it once. + for attempt := 0; attempt < 40; attempt++ { + ctx, cancel := context.WithCancel(context.Background()) + + var mu sync.Mutex + var observedErrs []error + done := make(chan error, 1) + go func() { + done <- f.q.Run(ctx, time.Millisecond, func(_ ReapReport, err error) { + if err != nil { + mu.Lock() + observedErrs = append(observedErrs, err) + mu.Unlock() + } + }) + }() + + // Cancel at a varying offset so cancellation lands at different points + // inside the sweep across attempts. + time.Sleep(time.Duration(attempt%7) * 300 * time.Microsecond) + cancel() + + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("attempt %d: Run returned %v, want context.Canceled", attempt, err) + } + case <-time.After(5 * time.Second): + t.Fatalf("attempt %d: Run did not return after cancel", attempt) + } + + mu.Lock() + errs := append([]error(nil), observedErrs...) + mu.Unlock() + for _, err := range errs { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("attempt %d: observer was told a clean shutdown was a sweep failure: %v\n"+ + "An error channel that cries wolf on every restart is one nobody reads, "+ + "which defeats the point of reporting sweep errors at all.", attempt, err) + } + } + } +} diff --git a/internal/handoff/handoff_test.go b/internal/handoff/handoff_test.go new file mode 100644 index 0000000..ef4b4b9 --- /dev/null +++ b/internal/handoff/handoff_test.go @@ -0,0 +1,1992 @@ +package handoff + +import ( + "context" + "database/sql" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/record" + "github.com/Susquehanna-Syntax/Anvil/internal/store" + + _ "modernc.org/sqlite" // cgo-free driver, plan/00-SPINE.md S12 +) + +// --------------------------------------------------------------------------- +// Fixture. The schema under test is internal/store/schema.sql applied through +// R.5's real migration path — never a hand-copied DDL, because a second copy +// of a frozen interface is the defect §6 G9/G10 exist to prevent, and a test +// that invents its own `handoff` table would prove nothing about the shipped +// one. +// --------------------------------------------------------------------------- + +// fakeClock drives the two expiry clocks independently of wall time. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func newFakeClock() *fakeClock { + return &fakeClock{t: time.Date(2026, 8, 8, 9, 0, 0, 0, time.UTC)} +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +type fixture struct { + t *testing.T + db *sql.DB + q *Queue + clock *fakeClock + packetDir string + targetID int64 +} + +// newFixture builds an on-disk store so the claim race runs over real, +// separate connections rather than one serialised pool slot. The pragmas ride +// on the DSN because they are per connection and the pool opens more than one. +func newFixture(t *testing.T, opts Options) *fixture { + t.Helper() + + dir := t.TempDir() + dbPath := filepath.ToSlash(filepath.Join(dir, "anvil.db")) + dsn := "file:" + strings.ReplaceAll(dbPath, " ", "%20") + + "?_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)" + + "&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + // Prove the DSN pragmas were honoured. If they silently were not, the + // concurrency test below would flake on SQLITE_BUSY and be blamed on the + // claim protocol instead of on the connection setup. + var busy int + if err := db.QueryRow(`PRAGMA busy_timeout`).Scan(&busy); err != nil { + t.Fatalf("PRAGMA busy_timeout: %v", err) + } + if busy != 10000 { + t.Fatalf("busy_timeout = %d, want 10000: DSN pragmas were not applied", busy) + } + + if _, err := store.Migrate(context.Background(), db, ""); err != nil { + t.Fatalf("store.Migrate: %v", err) + } + + clock := newFakeClock() + packetDir := filepath.Join(dir, "packets") + if opts.PacketDir == "" { + opts.PacketDir = packetDir + } + opts.Clock = clock.Now + + q, err := New(db, opts) + if err != nil { + t.Fatalf("New: %v", err) + } + + f := &fixture{t: t, db: db, q: q, clock: clock, packetDir: opts.PacketDir} + + res, err := db.Exec(`INSERT INTO target (kind, locator) VALUES (?, ?)`, "repo", "https://example.invalid/repo.git") + if err != nil { + t.Fatalf("insert target: %v", err) + } + if f.targetID, err = res.LastInsertId(); err != nil { + t.Fatalf("target id: %v", err) + } + return f +} + +// fp returns a distinct, well-formed 64-hex anvil-fp/v1-shaped digest. +func fp(n int) string { return strings.Repeat(fmt.Sprintf("%02x", n%256), 32) } + +// newAudit inserts a scan_run plus its audit_record with the half statuses and +// the deadline the test needs. deadline_at is supplied, never derived here: +// R.6 computes it once from scan_run.started_at + claim_timeout_seconds and +// this package only ever reads it. +func (f *fixture) newAudit(state record.State, sast record.HalfStatus, dast record.DastStatus, deadline time.Time) int64 { + f.t.Helper() + id, err := f.tryNewAudit(state, sast, dast, deadline) + if err != nil { + f.t.Fatalf("newAudit: %v", err) + } + return id +} + +// tryNewAudit is newAudit without the Fatalf, for the one test that must be +// able to tell "the schema will not hold this literal" from "the test is +// broken". See TestValidatedRequiresDynamicEvidence. +func (f *fixture) tryNewAudit(state record.State, sast record.HalfStatus, dast record.DastStatus, deadline time.Time) (int64, error) { + f.t.Helper() + + res, err := f.db.Exec( + `INSERT INTO scan_run (target_id, started_at, ruleset_version, status, commit_sha) + VALUES (?, ?, ?, ?, ?)`, + f.targetID, formatTime(f.clock.Now()), "anvil-rules/v1", + string(record.ScanRunStatusRunning), "9f1c0de9f1c0de9f1c0de9f1c0de9f1c0de9f1c0") + if err != nil { + return 0, fmt.Errorf("insert scan_run: %w", err) + } + scanRunID, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("scan_run id: %w", err) + } + + var sastStatus any + if sast != "" { + sastStatus = string(sast) + } + res, err = f.db.Exec( + `INSERT INTO audit_record + (scan_run_id, schema_version, state, sast_status, dast_status, + target_provenance, deadline_at, payload_sha256, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + scanRunID, "anvil/1", string(state), sastStatus, string(dast), + string(record.TargetProvenanceBootedClean), formatTime(deadline), + strings.Repeat("b", 64), formatTime(f.clock.Now())) + if err != nil { + return 0, fmt.Errorf("insert audit_record: %w", err) + } + auditID, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("audit_record id: %w", err) + } + return auditID, nil +} + +// sealedAudit is the common case: both halves sealed, DAST clean, an 8-hour +// claim window that has not closed. +func (f *fixture) sealedAudit() int64 { + f.t.Helper() + return f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedClean, f.clock.Now().Add(8*time.Hour)) +} + +func (f *fixture) newFinding(fingerprint string) int64 { + f.t.Helper() + res, err := f.db.Exec( + `INSERT INTO finding + (target_id, fingerprint, detector, evidence_class, rule_id, severity, title, + state, remediable_by_agent, first_seen_scan, first_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, (SELECT MAX(scan_run_id) FROM scan_run), ?)`, + f.targetID, fingerprint, string(record.DetectorKindSast), + string(record.EvidenceClassSastStaticOnly), "anvil.py.sqli/v3", "high", + "SQL injection", "open", 1, formatTime(f.clock.Now())) + if err != nil { + f.t.Fatalf("insert finding: %v", err) + } + id, err := res.LastInsertId() + if err != nil { + f.t.Fatalf("finding id: %v", err) + } + return id +} + +// auditUUID is the `anvil/auditId` for an audit_record row. It is a separate +// identity from the rowid on purpose: IdempotencyKey hashes THIS, because a +// rowid is not something a coding agent's git trailer can carry meaningfully +// (CRITIQUE-02 F7). +func auditUUID(auditRecordID int64) string { + return fmt.Sprintf("11111111-2222-4333-8444-%012d", auditRecordID) +} + +// enqueue seeds one ready finding and returns its fingerprint and row. +func (f *fixture) enqueue(n int, class record.ConsumptionClass, auditID int64) (string, Row) { + f.t.Helper() + fingerprint := fp(n) + findingID := f.newFinding(fingerprint) + row, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, + AuditRecordID: auditID, + AuditID: auditUUID(auditID), + Fingerprint: fingerprint, + ConsumptionClass: class, + }) + if err != nil { + f.t.Fatalf("Enqueue: %v", err) + } + if row.State != record.HandoffStateReady { + f.t.Fatalf("enqueued state = %q, want %q", row.State, record.HandoffStateReady) + } + return fingerprint, row +} + +func (f *fixture) state(handoffID int64) record.HandoffState { + f.t.Helper() + row, err := f.q.Get(handoffID) + if err != nil { + f.t.Fatalf("Get(%d): %v", handoffID, err) + } + return row.State +} + +// packetExists stats the packet file directly, WITHOUT going through +// ReadPacket. It is what the reaper/lease tests need: those assertions are +// about whether the cache file survived a sweep, and ReadPacket now — quite +// correctly — refuses a caller who does not hold the lease, so using it there +// would test the gate instead of the sweep. +func (f *fixture) packetExists(fingerprint string) bool { + f.t.Helper() + path, err := f.q.PacketPath(fingerprint) + if err != nil { + f.t.Fatalf("PacketPath: %v", err) + } + _, err = os.Stat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + f.t.Fatalf("stat packet: %v", err) + } + return err == nil +} + +func (f *fixture) countHandoffRows() int { + f.t.Helper() + var n int + if err := f.db.QueryRow(`SELECT COUNT(*) FROM handoff`).Scan(&n); err != nil { + f.t.Fatalf("count handoff: %v", err) + } + return n +} + +// --------------------------------------------------------------------------- +// The state machine itself. +// --------------------------------------------------------------------------- + +func TestStateMachineCoversEveryFrozenState(t *testing.T) { + transitions := LegalTransitions() + for _, s := range record.HandoffStateValues() { + if _, ok := transitions[s]; !ok { + t.Errorf("handoff.state %q has no entry in the state machine", s) + } + } + if len(transitions) != len(record.HandoffStateValues()) { + t.Errorf("state machine has %d states, the frozen enum has %d", + len(transitions), len(record.HandoffStateValues())) + } + + // Exactly two states are live; the other eleven are terminal. + var live, terminal int + for _, s := range record.HandoffStateValues() { + if IsLive(s) { + live++ + } + if IsTerminal(s) { + terminal++ + } + } + if live != 2 || terminal != 11 { + t.Errorf("live=%d terminal=%d, want 2 and 11", live, terminal) + } + + // Never expire a live claim: there is no direct leased -> expired edge. + if CanTransition(record.HandoffStateLeased, record.HandoffStateExpired) { + t.Error("leased -> expired must not be a legal edge: research/08 §4 forbids expiring a live claim") + } + // A terminal state is terminal. + if CanTransition(record.HandoffStateValidated, record.HandoffStateReady) { + t.Error("validated -> ready must not be legal") + } + + err := CheckTransition(record.HandoffStateLeased, record.HandoffStateExpired) + if !errors.Is(err, ErrIllegalTransition) { + t.Errorf("CheckTransition error = %v, want ErrIllegalTransition", err) + } + var te *TransitionError + if !errors.As(err, &te) || te.From != record.HandoffStateLeased { + t.Errorf("CheckTransition error = %v, want a *TransitionError naming the source state", err) + } + if err := CheckTransition(record.HandoffState("not_a_state"), record.HandoffStateReady); err == nil { + t.Error("an unknown state literal must be rejected by the record contract") + } +} + +func TestExhaustedStateIsAFrozenLiteral(t *testing.T) { + if err := record.ValidateHandoffState(string(ExhaustedState)); err != nil { + t.Fatalf("ExhaustedState is not a legal handoff.state: %v", err) + } + if !IsTerminal(ExhaustedState) { + t.Error("ExhaustedState must be terminal") + } +} + +// --------------------------------------------------------------------------- +// The claim race. This is the packet's first required test. +// --------------------------------------------------------------------------- + +func TestClaimRaceExactlyOneWinner(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(1, record.ConsumptionClassStaticOnly, audit) + + const workers = 8 + var ( + wg sync.WaitGroup + mu sync.Mutex + winners []Handle + losers int + other []error + ) + start := make(chan struct{}) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + h, err := f.q.Claim(fingerprint, fmt.Sprintf("worker-%d", i)) + mu.Lock() + defer mu.Unlock() + switch { + case err == nil: + winners = append(winners, h) + case errors.Is(err, ErrAlreadyClaimed): + losers++ + default: + other = append(other, err) + } + }(i) + } + close(start) + wg.Wait() + + for _, err := range other { + t.Errorf("unexpected claim error: %v", err) + } + if len(winners) != 1 { + t.Fatalf("%d workers won the claim, want exactly 1", len(winners)) + } + if losers != workers-1 { + t.Errorf("%d workers got ErrAlreadyClaimed, want %d", losers, workers-1) + } + + // The winner's lease is the only one recorded, and attempts advanced by + // exactly one — not once per racer. + got, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if got.State != record.HandoffStateLeased { + t.Errorf("state = %q, want %q", got.State, record.HandoffStateLeased) + } + if got.ClaimedBy != winners[0].WorkerID { + t.Errorf("claimed_by = %q, want %q", got.ClaimedBy, winners[0].WorkerID) + } + if got.Attempts != 1 { + t.Errorf("attempts = %d after one successful claim among %d racers, want 1", got.Attempts, workers) + } + if winners[0].Attempt != 1 { + t.Errorf("Handle.Attempt = %d, want 1", winners[0].Attempt) + } +} + +// TestClaimCASAdmitsExactlyOneWinner races the conditional UPDATE itself. +// +// It exists because the end-to-end race above cannot be trusted to reach the +// interleaving that matters: in practice one goroutine usually finishes its +// whole claim before the next one's eligibility SELECT runs, so the SELECT — +// not the CAS — does the arbitrating and the test would still pass with the +// `state = 'ready'` guard deleted. Here every goroutine starts from the same +// already-selected candidate id, which is exactly the state two workers are in +// when their SELECTs interleave, and only the guard can separate them. +func TestClaimCASAdmitsExactlyOneWinner(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + _, row := f.enqueue(20, record.ConsumptionClassStaticOnly, audit) + + const workers = 16 + var ( + wg sync.WaitGroup + mu sync.Mutex + won int + lost int + errs []error + handle Handle + ) + start := make(chan struct{}) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + h, ok, err := f.q.tryClaim(context.Background(), row.HandoffID, fmt.Sprintf("worker-%d", i)) + mu.Lock() + defer mu.Unlock() + switch { + case err != nil: + errs = append(errs, err) + case ok: + won++ + handle = h + default: + lost++ + } + }(i) + } + close(start) + wg.Wait() + + for _, err := range errs { + t.Errorf("unexpected error: %v", err) + } + if won != 1 { + t.Fatalf("%d of %d workers won the compare-and-swap, want exactly 1", won, workers) + } + if lost != workers-1 { + t.Errorf("%d workers lost, want %d", lost, workers-1) + } + after, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if after.Attempts != 1 { + t.Errorf("attempts = %d after %d racing claims, want 1", after.Attempts, workers) + } + if after.ClaimedBy != handle.WorkerID { + t.Errorf("claimed_by = %q, want the winner %q", after.ClaimedBy, handle.WorkerID) + } +} + +// --------------------------------------------------------------------------- +// The consumption gate (research/21 §5, carried by O.3's consumption_class). +// --------------------------------------------------------------------------- + +func TestStaticOnlyWaitsForTheSastSeal(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateCollecting, "", record.DastStatusNotRun, f.clock.Now().Add(8*time.Hour)) + fingerprint, _ := f.enqueue(2, record.ConsumptionClassStaticOnly, audit) + + if _, err := f.q.Claim(fingerprint, "worker-a"); !errors.Is(err, ErrNotEligible) { + t.Fatalf("claim while collecting: err = %v, want ErrNotEligible", err) + } + if _, err := f.q.AcquireLease("worker-a"); !errors.Is(err, ErrNoWork) { + t.Fatalf("AcquireLease while collecting: err = %v, want ErrNoWork", err) + } + + // R.6 seals the SAST half. + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, sast_status = ? WHERE audit_record_id = ?`, + string(record.StateSastSealed), string(record.HalfStatusSealed), audit); err != nil { + t.Fatalf("seal sast half: %v", err) + } + if _, err := f.q.Claim(fingerprint, "worker-a"); err != nil { + t.Fatalf("claim after the SAST seal: %v", err) + } +} + +func TestRequiresDynamicConfirmationWaitsForTheDastHalf(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateSastSealed, record.HalfStatusSealed, + record.DastStatusRunning, f.clock.Now().Add(8*time.Hour)) + fingerprint, _ := f.enqueue(3, record.ConsumptionClassRequiresDynamicConfirmation, audit) + + // The SAST half is sealed, which is enough for a static_only finding and + // deliberately not enough for this one. + if _, err := f.q.Claim(fingerprint, "worker-a"); !errors.Is(err, ErrNotEligible) { + t.Fatalf("claim before the DAST half is final: err = %v, want ErrNotEligible", err) + } + + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, dast_status = ? WHERE audit_record_id = ?`, + string(record.StateBothSealed), string(record.DastStatusCompletedFindings), audit); err != nil { + t.Fatalf("seal dast half: %v", err) + } + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("claim after the DAST seal: %v", err) + } + // S7: the Handle exposes what the dynamic half actually concluded, so a + // consumer can tell "confirmed" from "never ran". + if h.DastStatus != record.DastStatusCompletedFindings { + t.Errorf("Handle.DastStatus = %q, want %q", h.DastStatus, record.DastStatusCompletedFindings) + } + if h.ConsumptionClass != record.ConsumptionClassRequiresDynamicConfirmation { + t.Errorf("Handle.ConsumptionClass = %q", h.ConsumptionClass) + } +} + +// --------------------------------------------------------------------------- +// The crash. This is the scenario S7 and O.3 both name: a consumer acquires a +// lease, is OOM-killed mid-work, the lease expires, another consumer reclaims. +// --------------------------------------------------------------------------- + +func TestOOMKilledConsumerReclaimIsIdempotent(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(4, record.ConsumptionClassStaticOnly, audit) + + dead, err := f.q.Claim(fingerprint, "worker-oomed") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if dead.Attempt != 1 { + t.Fatalf("first lease Attempt = %d, want 1", dead.Attempt) + } + // The packet is materialised by the lease holder: it is a cache of the + // half's results, and R.6's read gate governs those bytes wherever they + // live, so writing one takes a Handle. + if _, err := f.q.WritePacket(dead, []byte(`{"packet":"regenerable"}`)); err != nil { + t.Fatalf("WritePacket: %v", err) + } + + // The holder is OOM-killed here. It never renews and never releases. + f.clock.Advance(21 * time.Minute) + + report, err := f.q.ReclaimExpired() + if err != nil { + t.Fatalf("ReclaimExpired: %v", err) + } + if len(report.Reclaimed) != 1 || report.Requeued() != 1 { + t.Fatalf("reclaimed %d (requeued %d), want 1 requeued", len(report.Reclaimed), report.Requeued()) + } + if report.Reclaimed[0].To != record.HandoffStateReady { + t.Errorf("reclaimed to %q, want %q", report.Reclaimed[0].To, record.HandoffStateReady) + } + if report.Reclaimed[0].WorkerID != "worker-oomed" { + t.Errorf("reclaim named worker %q", report.Reclaimed[0].WorkerID) + } + if got := f.state(row.HandoffID); got != record.HandoffStateReady { + t.Fatalf("state after reclaim = %q, want %q", got, record.HandoffStateReady) + } + + // Idempotent sweep: a second reclaim must find nothing and must not touch + // the attempt counter. + if second, err := f.q.ReclaimExpired(); err != nil { + t.Fatalf("second ReclaimExpired: %v", err) + } else if !second.Empty() { + t.Errorf("second ReclaimExpired reclaimed %d rows, want 0", len(second.Reclaimed)) + } + after, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if after.Attempts != 1 { + t.Errorf("attempts = %d after one crash and two sweeps, want 1", after.Attempts) + } + + // The packet is a cache for work that is still to be done: a requeue must + // not drop it. + if !f.packetExists(fingerprint) { + t.Error("packet was dropped by a requeue") + } + + // The successor picks the same work up. + live, err := f.q.AcquireLease("worker-successor") + if err != nil { + t.Fatalf("AcquireLease after reclaim: %v", err) + } + if live.HandoffID != dead.HandoffID { + t.Fatalf("successor got row %d, want %d", live.HandoffID, dead.HandoffID) + } + if live.Attempt != 2 { + t.Errorf("successor Attempt = %d, want 2", live.Attempt) + } + // The (fingerprint, record version) key is unchanged across the crash, so + // the successor's work is recognisably the same unit of work — that is + // what makes re-processing idempotent downstream rather than duplicated. + if live.Fingerprint != dead.Fingerprint || live.RecordVersion != dead.RecordVersion { + t.Errorf("work identity moved across the crash: (%s,%d) -> (%s,%d)", + dead.Fingerprint, dead.RecordVersion, live.Fingerprint, live.RecordVersion) + } + if live.IdempotencyKey != dead.IdempotencyKey || live.IdempotencyKey == "" { + t.Errorf("idempotency key moved across the crash: %q -> %q", dead.IdempotencyKey, live.IdempotencyKey) + } + + // The dead holder now wakes up and tries to report. Neither call may land. + if _, err := f.q.RenewLease(dead); !errors.Is(err, ErrLeaseLost) { + t.Errorf("dead holder RenewLease: err = %v, want ErrLeaseLost", err) + } + if err := f.q.ReleaseLease(dead, record.HandoffStateValidated); !errors.Is(err, ErrLeaseLost) { + t.Errorf("dead holder ReleaseLease: err = %v, want ErrLeaseLost", err) + } + stillLeased, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if stillLeased.State != record.HandoffStateLeased || stillLeased.ClaimedBy != "worker-successor" { + t.Fatalf("the dead holder's late write landed: state=%q claimed_by=%q", + stillLeased.State, stillLeased.ClaimedBy) + } + + // The live holder finishes. Exactly one attempt is applied, and exactly + // one row exists: no duplicate side effect is observable. + if err := f.q.ReleaseLease(live, record.HandoffStateValidated); err != nil { + t.Fatalf("ReleaseLease: %v", err) + } + final, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if final.State != record.HandoffStateValidated { + t.Errorf("final state = %q, want %q", final.State, record.HandoffStateValidated) + } + if final.Attempts != 2 { + t.Errorf("attempts = %d, want 2 (one crashed, one completed)", final.Attempts) + } + if final.ClaimedBy != "" || !final.LeaseExpiresAt.IsZero() { + t.Errorf("a finished row still carries a lease: claimed_by=%q expires=%v", + final.ClaimedBy, final.LeaseExpiresAt) + } + if n := f.countHandoffRows(); n != 1 { + t.Errorf("%d handoff rows, want 1: the crash must not duplicate the finding", n) + } + if f.packetExists(fingerprint) { + t.Error("packet survived a terminal release") + } +} + +func TestLeaseExhaustionAfterMaxAttempts(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(5, record.ConsumptionClassStaticOnly, audit) + + for attempt := 1; attempt <= 2; attempt++ { + h, err := f.q.Claim(fingerprint, fmt.Sprintf("worker-%d", attempt)) + if err != nil { + t.Fatalf("claim %d: %v", attempt, err) + } + if _, err := f.q.WritePacket(h, []byte("packet")); err != nil { + t.Fatalf("WritePacket %d: %v", attempt, err) + } + f.clock.Advance(21 * time.Minute) + report, err := f.q.ReclaimExpired() + if err != nil { + t.Fatalf("ReclaimExpired %d: %v", attempt, err) + } + if len(report.Reclaimed) != 1 { + t.Fatalf("sweep %d reclaimed %d rows, want 1", attempt, len(report.Reclaimed)) + } + want := record.HandoffStateReady + if attempt == 2 { + want = ExhaustedState + } + if report.Reclaimed[0].To != want { + t.Fatalf("sweep %d moved the row to %q, want %q", attempt, report.Reclaimed[0].To, want) + } + } + + if report, err := f.q.ReclaimExpired(); err != nil { + t.Fatalf("ReclaimExpired: %v", err) + } else if report.Exhausted() != 0 || !report.Empty() { + t.Errorf("a terminal row was swept again: %+v", report) + } + if got := f.state(row.HandoffID); got != ExhaustedState { + t.Errorf("state = %q, want %q", got, ExhaustedState) + } + // Terminal means no further lease, ever. + if _, err := f.q.Claim(fingerprint, "worker-3"); !errors.Is(err, ErrNotEligible) { + t.Errorf("claiming a terminal row: err = %v, want ErrNotEligible", err) + } + if f.packetExists(fingerprint) { + t.Error("packet survived exhaustion") + } + // The row is kept. Nothing in this package deletes findings. + if n := f.countHandoffRows(); n != 1 { + t.Errorf("%d handoff rows after exhaustion, want 1", n) + } +} + +// --------------------------------------------------------------------------- +// The two clocks. They must fire independently and produce different +// transitions. +// --------------------------------------------------------------------------- + +func TestLeaseExpiryFiresWithoutTheClaimTimeout(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + // An 8-hour claim window: far beyond the 20-minute lease. + audit := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedClean, f.clock.Now().Add(8*time.Hour)) + fingerprint, row := f.enqueue(6, record.ConsumptionClassStaticOnly, audit) + + if _, err := f.q.Claim(fingerprint, "worker-a"); err != nil { + t.Fatalf("Claim: %v", err) + } + f.clock.Advance(21 * time.Minute) + + // The claim-timeout clock has not moved anywhere near its deadline. + timeouts, err := f.q.ExpireClaimTimeouts() + if err != nil { + t.Fatalf("ExpireClaimTimeouts: %v", err) + } + if len(timeouts.Expired) != 0 { + t.Fatalf("the claim-timeout sweep fired at 21 minutes into an 8-hour window: %+v", timeouts.Expired) + } + + leases, err := f.q.ReclaimExpired() + if err != nil { + t.Fatalf("ReclaimExpired: %v", err) + } + if len(leases.Reclaimed) != 1 || leases.Reclaimed[0].To != record.HandoffStateReady { + t.Fatalf("lease sweep = %+v, want one requeue to ready", leases.Reclaimed) + } + if got := f.state(row.HandoffID); got != record.HandoffStateReady { + t.Errorf("state = %q, want %q — the lease clock requeues, it never expires", got, record.HandoffStateReady) + } +} + +func TestClaimTimeoutNeverExpiresALiveClaimAndKeepsTheRow(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + deadline := f.clock.Now().Add(8 * time.Hour) + audit := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedClean, deadline) + fingerprint, row := f.enqueue(7, record.ConsumptionClassStaticOnly, audit) + + handle, err := f.q.Claim(fingerprint, "worker-long-runner") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if _, err := f.q.WritePacket(handle, []byte("packet")); err != nil { + t.Fatalf("WritePacket: %v", err) + } + + // Walk past the 8-hour claim deadline while heartbeating, the way a + // consumer that is genuinely still working does. + for i := 0; i < 27; i++ { + f.clock.Advance(19 * time.Minute) + if handle, err = f.q.RenewLease(handle); err != nil { + t.Fatalf("RenewLease at step %d: %v", i, err) + } + } + if !f.clock.Now().After(deadline) { + t.Fatalf("clock is at %s, which is not past the %s claim deadline: the test proves nothing", + formatTime(f.clock.Now()), formatTime(deadline)) + } + + report, err := f.q.Reap() + if err != nil { + t.Fatalf("Reap: %v", err) + } + if !report.Empty() { + t.Fatalf("the reaper touched a live claim past its audit deadline: %+v", report) + } + live, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if live.State != record.HandoffStateLeased || live.ClaimedBy != "worker-long-runner" { + t.Fatalf("live claim was disturbed: state=%q claimed_by=%q", live.State, live.ClaimedBy) + } + if !f.packetExists(fingerprint) { + t.Error("a live claim's packet was dropped") + } + // And the holder can still read it: the lease is live and the gate is open. + if _, err := f.q.ReadPacket(handle); err != nil { + t.Errorf("the live holder cannot read its own packet: %v", err) + } + + // Now the holder stops heartbeating. The lease clock requeues it, and only + // then does the claim-timeout clock see a 'ready' row past its deadline. + f.clock.Advance(21 * time.Minute) + report, err = f.q.Reap() + if err != nil { + t.Fatalf("Reap: %v", err) + } + if report.Requeued() != 1 { + t.Errorf("lease sweep requeued %d, want 1", report.Requeued()) + } + if len(report.Expired) != 1 { + t.Fatalf("claim-timeout sweep expired %d rows, want 1", len(report.Expired)) + } + if !report.Expired[0].PacketDropped { + t.Error("the expiring finding's packet was not dropped") + } + if got := f.state(row.HandoffID); got != record.HandoffStateExpired { + t.Errorf("state = %q, want %q", got, record.HandoffStateExpired) + } + if f.packetExists(fingerprint) { + t.Error("packet survived the claim timeout") + } + + // S1: a claim timeout is not a deletion policy. The row, the finding and + // the audit record are all still here. + if n := f.countHandoffRows(); n != 1 { + t.Errorf("%d handoff rows after expiry, want 1: the reaper must not delete rows", n) + } + var findings, audits int + if err := f.db.QueryRow(`SELECT COUNT(*) FROM finding`).Scan(&findings); err != nil { + t.Fatalf("count finding: %v", err) + } + if err := f.db.QueryRow(`SELECT COUNT(*) FROM audit_record`).Scan(&audits); err != nil { + t.Fatalf("count audit_record: %v", err) + } + if findings != 1 || audits != 1 { + t.Errorf("finding rows = %d, audit_record rows = %d, want 1 and 1", findings, audits) + } + // And the payload it points at is untouched: purging a shared payload + // because one finding's window lapsed would blind its siblings. + var payloadSHA string + if err := f.db.QueryRow(`SELECT payload_sha256 FROM audit_record WHERE audit_record_id = ?`, audit).Scan(&payloadSHA); err != nil { + t.Fatalf("read payload_sha256: %v", err) + } + if payloadSHA == "" { + t.Error("payload_sha256 must survive: it is the proof of what was handed over") + } +} + +func TestLateHeartbeatKeepsTheClaim(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(21, record.ConsumptionClassStaticOnly, audit) + + stale, err := f.q.Claim(fingerprint, "worker-slow") + if err != nil { + t.Fatalf("Claim: %v", err) + } + + // The heartbeat arrives after the nominal expiry but before the sweep. The + // holder is demonstrably alive, so it keeps the finding. + f.clock.Advance(21 * time.Minute) + fresh, err := f.q.RenewLease(stale) + if err != nil { + t.Fatalf("late RenewLease: %v", err) + } + if !fresh.LeaseExpiresAt.After(f.clock.Now()) { + t.Fatalf("renewed lease expires at %s, which is not in the future", formatTime(fresh.LeaseExpiresAt)) + } + + report, err := f.q.ReclaimExpired() + if err != nil { + t.Fatalf("ReclaimExpired: %v", err) + } + if !report.Empty() { + t.Fatalf("the reaper took a renewed claim: %+v", report) + } + live, err := f.q.Get(row.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if live.State != record.HandoffStateLeased || live.ClaimedBy != "worker-slow" { + t.Fatalf("claim was disturbed: state=%q claimed_by=%q", live.State, live.ClaimedBy) + } + if live.Attempts != 1 { + t.Errorf("attempts = %d; a renewal is not a new attempt", live.Attempts) + } + + // The pre-renewal Handle is dead: renewing produces a new one and the old + // one may no longer speak for the lease. + if _, err := f.q.RenewLease(stale); !errors.Is(err, ErrLeaseLost) { + t.Errorf("stale handle RenewLease: err = %v, want ErrLeaseLost", err) + } + if err := f.q.ReleaseLease(stale, record.HandoffStateValidated); !errors.Is(err, ErrLeaseLost) { + t.Errorf("stale handle ReleaseLease: err = %v, want ErrLeaseLost", err) + } + if err := f.q.ReleaseLease(fresh, record.HandoffStateValidated); err != nil { + t.Errorf("fresh handle ReleaseLease: %v", err) + } +} + +func TestExpireClaimTimeoutIsIdempotent(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedClean, f.clock.Now().Add(time.Hour)) + _, row := f.enqueue(8, record.ConsumptionClassStaticOnly, audit) + + f.clock.Advance(2 * time.Hour) + first, err := f.q.ExpireClaimTimeouts() + if err != nil { + t.Fatalf("ExpireClaimTimeouts: %v", err) + } + if len(first.Expired) != 1 { + t.Fatalf("expired %d rows, want 1", len(first.Expired)) + } + second, err := f.q.ExpireClaimTimeouts() + if err != nil { + t.Fatalf("second ExpireClaimTimeouts: %v", err) + } + if !second.Empty() { + t.Errorf("second sweep expired %d rows, want 0", len(second.Expired)) + } + if got := f.state(row.HandoffID); got != record.HandoffStateExpired { + t.Errorf("state = %q, want %q", got, record.HandoffStateExpired) + } +} + +// TestRunSweepsUntilCancelled exercises the reaper loop itself. Its ticker is +// wall-clock — a periodic timer is not something the injected clock drives — +// so the interval here is small and the assertion is only that sweeps happen +// and that cancellation ends the loop. +func TestRunSweepsUntilCancelled(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(22, record.ConsumptionClassStaticOnly, audit) + if _, err := f.q.Claim(fingerprint, "worker-doomed"); err != nil { + t.Fatalf("Claim: %v", err) + } + f.clock.Advance(21 * time.Minute) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + reclaimed := make(chan Reclaimed, 4) + done := make(chan error, 1) + go func() { + done <- f.q.Run(ctx, 2*time.Millisecond, func(report ReapReport, err error) { + if err != nil { + t.Errorf("sweep error: %v", err) + return + } + for _, c := range report.Reclaimed { + select { + case reclaimed <- c: + default: + } + } + }) + }() + + select { + case c := <-reclaimed: + if c.HandoffID != row.HandoffID || c.To != record.HandoffStateReady { + t.Errorf("reclaimed %+v, want row %d requeued to ready", c, row.HandoffID) + } + case <-time.After(10 * time.Second): + t.Fatal("the reaper loop never swept the expired lease") + } + + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("Run returned %v, want context.Canceled", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Run did not return after cancellation") + } +} + +// --------------------------------------------------------------------------- +// Lease authority: what a Handle may and may not do. +// --------------------------------------------------------------------------- + +func TestRecordVersionBumpVoidsTheLease(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(9, record.ConsumptionClassStaticOnly, audit) + + handle, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if handle.RecordVersion != 1 { + t.Fatalf("RecordVersion = %d, want 1", handle.RecordVersion) + } + + // S6: a version bump re-cuts the work queue. The leased work unit is gone. + if _, err := f.db.Exec(`UPDATE audit_record SET audit_version = 2 WHERE audit_record_id = ?`, audit); err != nil { + t.Fatalf("bump audit_version: %v", err) + } + + if _, err := f.q.RenewLease(handle); !errors.Is(err, ErrRecordVersionChanged) { + t.Errorf("RenewLease after a version bump: err = %v, want ErrRecordVersionChanged", err) + } + if err := f.q.ReleaseLease(handle, record.HandoffStateValidated); !errors.Is(err, ErrRecordVersionChanged) { + t.Errorf("ReleaseLease after a version bump: err = %v, want ErrRecordVersionChanged", err) + } + if got := f.state(row.HandoffID); got != record.HandoffStateLeased { + t.Errorf("state = %q: a refused release must not change the row", got) + } +} + +func TestReleaseLeaseRejectsIllegalOutcomes(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(10, record.ConsumptionClassStaticOnly, audit) + handle, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + + // 'expired' belongs to the claim-timeout clock, not to a consumer. + if err := f.q.ReleaseLease(handle, record.HandoffStateExpired); !errors.Is(err, ErrIllegalTransition) { + t.Errorf("release as expired: err = %v, want ErrIllegalTransition", err) + } + // A state outside the frozen thirteen never reaches the database. + if err := f.q.ReleaseLease(handle, record.HandoffState("done")); err == nil { + t.Error("release as an unknown state must be rejected") + } + // Handing the finding back is legal and clears the lease. + if err := f.q.ReleaseLease(handle, record.HandoffStateReady); err != nil { + t.Fatalf("release back to ready: %v", err) + } + row, err := f.q.Get(handle.HandoffID) + if err != nil { + t.Fatalf("Get: %v", err) + } + if row.State != record.HandoffStateReady || row.ClaimedBy != "" { + t.Errorf("after release: state=%q claimed_by=%q", row.State, row.ClaimedBy) + } +} + +func TestDisposeOnlyLeavesReady(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, row := f.enqueue(11, record.ConsumptionClassStaticOnly, audit) + + // The queue re-cut skips a ready finding for budget without leasing it. + if err := f.q.Dispose(row.HandoffID, record.HandoffStateSkippedBudget); err != nil { + t.Fatalf("Dispose: %v", err) + } + if got := f.state(row.HandoffID); got != record.HandoffStateSkippedBudget { + t.Fatalf("state = %q, want %q", got, record.HandoffStateSkippedBudget) + } + // G10's failure mode: the disposition and the ready-set index must agree, + // so a skipped finding is not re-leased forever. + if _, err := f.q.Claim(fingerprint, "worker-a"); !errors.Is(err, ErrNotEligible) { + t.Errorf("a skipped_budget finding was still claimable: err = %v", err) + } + if err := f.q.Dispose(row.HandoffID, record.HandoffStateWithdrawn); !errors.Is(err, ErrIllegalTransition) { + t.Errorf("disposing a terminal row: err = %v, want ErrIllegalTransition", err) + } + + // A leased row is not disposable behind its holder's back. + audit2 := f.sealedAudit() + fingerprint2, row2 := f.enqueue(12, record.ConsumptionClassStaticOnly, audit2) + if _, err := f.q.Claim(fingerprint2, "worker-b"); err != nil { + t.Fatalf("Claim: %v", err) + } + if err := f.q.Dispose(row2.HandoffID, record.HandoffStateSkippedBudget); !errors.Is(err, ErrAlreadyClaimed) { + t.Errorf("disposing a leased row: err = %v, want ErrAlreadyClaimed", err) + } + if err := f.q.Dispose(row2.HandoffID, record.HandoffStateLeased); !errors.Is(err, ErrIllegalTransition) { + t.Errorf("Dispose must never grant a lease: err = %v", err) + } +} + +// --------------------------------------------------------------------------- +// Enqueue, keys and packets. +// --------------------------------------------------------------------------- + +func TestEnqueueIsIdempotent(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint := fp(13) + findingID := f.newFinding(fingerprint) + + req := EnqueueRequest{ + FindingID: findingID, + AuditRecordID: audit, + AuditID: auditUUID(audit), + Fingerprint: fingerprint, + ConsumptionClass: record.ConsumptionClassStaticOnly, + } + first, err := f.q.Enqueue(req) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if _, err := f.q.Claim(fingerprint, "worker-a"); err != nil { + t.Fatalf("Claim: %v", err) + } + + // A producer that crashed after inserting and re-runs must not duplicate + // the row, and must not reset the lease. + second, err := f.q.Enqueue(req) + if err != nil { + t.Fatalf("re-Enqueue: %v", err) + } + if second.HandoffID != first.HandoffID { + t.Errorf("re-enqueue produced row %d, want %d", second.HandoffID, first.HandoffID) + } + if second.State != record.HandoffStateLeased { + t.Errorf("re-enqueue reset the state to %q", second.State) + } + if n := f.countHandoffRows(); n != 1 { + t.Errorf("%d handoff rows, want 1", n) + } + if second.IdempotencyKey != first.IdempotencyKey || first.IdempotencyKey == "" { + t.Errorf("idempotency key is not stable: %q vs %q", first.IdempotencyKey, second.IdempotencyKey) + } + want := IdempotencyKey(auditUUID(audit), fingerprint, "9f1c0de9f1c0de9f1c0de9f1c0de9f1c0de9f1c0") + if first.IdempotencyKey != want { + t.Errorf("idempotency key = %s, want sha256(audit_id || fingerprint || base commit) = %s", + first.IdempotencyKey, want) + } +} + +// TestIdempotencyKeyUsesTheDocumentedInputs is CRITIQUE-02 F7. schema.sql and +// this package's own doc both say sha256(audit_id || finding_fingerprint || +// base_commit_sha); the implementation hashed `audit_record_id`, an +// autoincrement rowid. A rowid is not an audit identity, and the value is +// EXPORTED for the coding agent to write into a git trailer, where a rowid +// means nothing to anyone who does not hold that exact database file. +func TestIdempotencyKeyUsesTheDocumentedInputs(t *testing.T) { + const ( + auditID = "0f9c2b1e-4a7d-4c33-9f21-6b8a0d5e7c14" + commit = "9f1c0de9f1c0de9f1c0de9f1c0de9f1c0de9f1c0" + ) + fingerprint := fp(60) + + // The key is a pure function of the three documented components. + if got, want := IdempotencyKey(auditID, fingerprint, commit), + IdempotencyKey(auditID, fingerprint, commit); got != want { + t.Fatalf("IdempotencyKey is not deterministic") + } + // Each component moves it. + base := IdempotencyKey(auditID, fingerprint, commit) + for _, other := range []string{ + IdempotencyKey("0f9c2b1e-4a7d-4c33-9f21-6b8a0d5e7c15", fingerprint, commit), + IdempotencyKey(auditID, fp(61), commit), + IdempotencyKey(auditID, fingerprint, "0000000000000000000000000000000000000000"), + } { + if other == base { + t.Error("a component of the documented triple does not affect the key") + } + } + // No rowid can reproduce it. This is the regression: if the function ever + // goes back to hashing audit_record_id, the key for the same audit + // identity changes, and no small integer spelling of the rowid produces + // the documented value. + for id := int64(0); id < 64; id++ { + if IdempotencyKey(fmt.Sprint(id), fingerprint, commit) == base { + t.Errorf("IdempotencyKey(audit_record_id=%d, ...) collides with the documented key; "+ + "the first component must be anvil/auditId", id) + } + } + // Boundary shifting cannot forge a collision either: the NUL joiner is + // what stops ("ab", "c") and ("a", "bc") hashing the same. + if IdempotencyKey("ab", "c"+fingerprint[1:], commit) == IdempotencyKey("a", "bc"+fingerprint[1:], commit) { + t.Error("components are not delimited; a boundary shift forges a collision") + } +} + +// TestEnqueueRequiresTheAuditIdentity: rather than silently substituting the +// rowid when the caller omits anvil/auditId, Enqueue refuses. A key computed +// from the wrong identity is worse than no key, because downstream dedup would +// trust it. +func TestEnqueueRequiresTheAuditIdentity(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint := fp(62) + findingID := f.newFinding(fingerprint) + + if _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: audit, Fingerprint: fingerprint, + ConsumptionClass: record.ConsumptionClassStaticOnly, + }); err == nil { + t.Error("Enqueue accepted a request with no anvil/auditId") + } + if n := f.countHandoffRows(); n != 0 { + t.Errorf("%d rows were inserted despite the refusal", n) + } +} + +func TestEnqueueRejectsMalformedInput(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint := fp(14) + findingID := f.newFinding(fingerprint) + + if _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: audit, AuditID: auditUUID(audit), + Fingerprint: fingerprint[:16], // truncated digests are never legal + ConsumptionClass: record.ConsumptionClassStaticOnly, + }); err == nil { + t.Error("a truncated fingerprint must be rejected") + } + if _, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: audit, AuditID: auditUUID(audit), + Fingerprint: fingerprint, + ConsumptionClass: record.ConsumptionClass(""), + }); err == nil { + t.Error("consumption_class has no default and must be rejected when empty") + } +} + +func TestPacketWriteReadDrop(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(15, record.ConsumptionClassStaticOnly, audit) + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + + path, err := f.q.WritePacket(h, []byte("first")) + if err != nil { + t.Fatalf("WritePacket: %v", err) + } + if filepath.Dir(path) != f.packetDir { + t.Errorf("packet landed in %s, want %s", filepath.Dir(path), f.packetDir) + } + // Replacement is a rename over the same name: no torn read is observable. + if _, err := f.q.WritePacket(h, []byte("second")); err != nil { + t.Fatalf("re-WritePacket: %v", err) + } + data, err := f.q.ReadPacket(h) + if err != nil { + t.Fatalf("ReadPacket: %v", err) + } + if string(data) != "second" { + t.Errorf("packet = %q, want %q", data, "second") + } + // No temp files left behind. + entries, err := os.ReadDir(f.packetDir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if len(entries) != 1 { + t.Errorf("packet directory holds %d entries, want 1", len(entries)) + } + + if err := f.q.DropPacket(fingerprint); err != nil { + t.Fatalf("DropPacket: %v", err) + } + // Dropping a packet that is not there is success: the packet is a cache + // and its absence is the desired state. + if err := f.q.DropPacket(fingerprint); err != nil { + t.Errorf("second DropPacket: %v", err) + } +} + +func TestPacketPathRejectsTraversal(t *testing.T) { + f := newFixture(t, Options{}) + for _, bad := range []string{ + "../../../etc/passwd", + strings.Repeat("A", 64), // uppercase hex is not the contract's spelling + "", + strings.Repeat("z", 64), + } { + if _, err := f.q.PacketPath(bad); err == nil { + t.Errorf("PacketPath(%q) was accepted", bad) + } + } +} + +func TestPacketDirIsOptional(t *testing.T) { + f := newFixture(t, Options{}) + // A Queue without a packet directory is legal: the packet is a cache, and + // a deployment that hands the consumer bytes needs no directory at all. + q, err := New(f.db, Options{Clock: f.clock.Now}) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := q.PacketPath(fp(16)); !errors.Is(err, ErrNoPacketDir) { + t.Errorf("PacketPath without a dir: err = %v, want ErrNoPacketDir", err) + } + if err := q.DropPacket(fp(16)); err != nil { + t.Errorf("DropPacket without a dir: %v", err) + } + + audit := f.sealedAudit() + fingerprint := fp(17) + findingID := f.newFinding(fingerprint) + if _, err := q.Enqueue(EnqueueRequest{ + FindingID: findingID, AuditRecordID: audit, AuditID: auditUUID(audit), + Fingerprint: fingerprint, + ConsumptionClass: record.ConsumptionClassStaticOnly, + }); err != nil { + t.Fatalf("Enqueue: %v", err) + } + h, err := q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if h.PacketPath != "" { + t.Errorf("Handle.PacketPath = %q, want empty", h.PacketPath) + } + if err := q.ReleaseLease(h, record.HandoffStateValidated); err != nil { + t.Errorf("ReleaseLease with no packet directory: %v", err) + } +} + +func TestClaimUnknownFingerprint(t *testing.T) { + f := newFixture(t, Options{}) + if _, err := f.q.Claim(fp(18), "worker-a"); !errors.Is(err, ErrNotFound) { + t.Errorf("claiming an unqueued fingerprint: err = %v, want ErrNotFound", err) + } + if _, err := f.q.Claim("not-a-fingerprint", "worker-a"); err == nil { + t.Error("a malformed fingerprint must be rejected before it reaches SQL") + } + if _, err := f.q.Claim(fp(18), ""); err == nil { + t.Error("an empty workerID must be rejected") + } +} + +func TestTimestampsAreFixedWidthAndOrdered(t *testing.T) { + // The lease and deadline comparisons are made in Go, but the stored format + // is still fixed width so that a later index range scan cannot be wrong. + // time.RFC3339Nano would fail this: "…00Z" sorts after "…00.5Z". + base := time.Date(2026, 8, 8, 10, 0, 0, 0, time.UTC) + a := formatTime(base) + b := formatTime(base.Add(500 * time.Millisecond)) + if len(a) != len(b) { + t.Fatalf("timestamps are not fixed width: %q vs %q", a, b) + } + if !(a < b) { + t.Errorf("%q must sort before %q", a, b) + } + if rfc := base.Format(time.RFC3339Nano); rfc < base.Add(500*time.Millisecond).Format(time.RFC3339Nano) { + t.Log("note: RFC3339Nano happened to order correctly for this pair; the format is still not fixed width") + } + back, err := parseTime("test", a) + if err != nil || !back.Equal(base) { + t.Errorf("round trip: %v, %v", back, err) + } +} + +// --------------------------------------------------------------------------- +// The stop condition: no code path claims secure deletion. +// +// This is checked structurally rather than with a naive text grep, because the +// package deliberately DOCUMENTS why shred is not a control here (research/08 +// §F) and that explanation must survive. What must not exist is an executed +// reference or an affirmative claim. +// --------------------------------------------------------------------------- + +func TestNoSecureDeletionClaimOrCall(t *testing.T) { + fset := token.NewFileSet() + // Test files are excluded from both scans: this file necessarily contains + // the denylist itself, and the stop condition is about the shipped code + // paths. + implementationOnly := func(fi os.FileInfo) bool { + return strings.HasSuffix(fi.Name(), ".go") && !strings.HasSuffix(fi.Name(), "_test.go") + } + pkgs, err := parser.ParseDir(fset, ".", implementationOnly, parser.ParseComments) + if err != nil { + t.Fatalf("parsing the package: %v", err) + } + if len(pkgs) == 0 { + t.Fatal("no package parsed") + } + + // Executed code: no identifier and no string literal may name an erasure + // tool, and nothing may shell out at all. + forbiddenTokens := []string{"shred", "rm -p", "rm -f -p", "secure_delete", "blkdiscard"} + for _, pkg := range pkgs { + for name, file := range pkg.Files { + for _, imp := range file.Imports { + if imp.Path.Value == `"os/exec"` { + t.Errorf("%s imports os/exec: this package runs no external eraser", name) + } + } + ast.Inspect(file, func(n ast.Node) bool { + switch v := n.(type) { + case *ast.Ident: + for _, bad := range forbiddenTokens { + if strings.Contains(strings.ToLower(v.Name), strings.ReplaceAll(bad, " ", "")) { + t.Errorf("%s: identifier %q references an erasure primitive", name, v.Name) + } + } + case *ast.BasicLit: + if v.Kind != token.STRING { + return true + } + lower := strings.ToLower(v.Value) + for _, bad := range forbiddenTokens { + if strings.Contains(lower, bad) { + t.Errorf("%s: string literal %s references an erasure primitive", name, v.Value) + } + } + } + return true + }) + } + } + + // Prose: no affirmative claim of secure destruction anywhere, comments + // included. The negations the package does make ("makes no claim that the + // bytes are unrecoverable") are not on this list, and must not be. + claims := []string{ + "securely delet", "securely destroy", "securely eras", "secure erasure", + "cryptographically eras", "we shred", "shred the", "wiped from disk", + "unrecoverably", + } + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") || strings.HasSuffix(e.Name(), "_test.go") { + continue + } + src, err := os.ReadFile(e.Name()) + if err != nil { + t.Fatalf("ReadFile %s: %v", e.Name(), err) + } + lower := strings.ToLower(string(src)) + for _, claim := range claims { + if strings.Contains(lower, claim) { + t.Errorf("%s claims secure deletion (%q); research/08 §F: shred cannot deliver it on "+ + "Btrfs, ZFS, XFS, snapshotting or RAID filesystems, or on any SSD", e.Name(), claim) + } + } + } + + // And the affirmative disclaimer is present, so the reasoning is not lost + // the next time someone proposes adding one. + src, err := os.ReadFile("claim.go") + if err != nil { + t.Fatalf("ReadFile claim.go: %v", err) + } + if !strings.Contains(string(src), "unlink, not an erasure") { + t.Error("DropPacket must state plainly that it unlinks and does not erase") + } +} + +// --------------------------------------------------------------------------- +// Enum discipline: no handoff.state or consumption_class literal is re-typed +// as a bare string in this package. Every one comes from internal/record. +// --------------------------------------------------------------------------- + +func TestNoBareEnumLiteralsInPackageCode(t *testing.T) { + var vocabulary []string + for _, s := range record.HandoffStateValues() { + vocabulary = append(vocabulary, string(s)) + } + for _, c := range record.ConsumptionClassValues() { + vocabulary = append(vocabulary, string(c)) + } + for _, s := range record.HalfStatusValues() { + vocabulary = append(vocabulary, string(s)) + } + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { + // The test file itself may not name them either, but it legitimately + // contains English prose in messages; only the implementation files + // are checked for literals. + return strings.HasSuffix(fi.Name(), ".go") && !strings.HasSuffix(fi.Name(), "_test.go") + }, parser.ParseComments) + if err != nil { + t.Fatalf("parsing the package: %v", err) + } + + for _, pkg := range pkgs { + for name, file := range pkg.Files { + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + value := strings.Trim(lit.Value, "`\"") + for _, word := range vocabulary { + if value == word { + t.Errorf("%s: %q is an enum literal re-typed as a bare string; "+ + "use the internal/record constant", filepath.Base(name), value) + } + } + return true + }) + } + } +} + +// --------------------------------------------------------------------------- +// Regression guards for CRITIQUE-02 (R.10 critic gate 2). +// --------------------------------------------------------------------------- + +// enqueueInto puts an EXISTING finding into a second audit_record's ready set. +// This is the shape a re-scan produces: one fingerprint, one finding row, and +// one handoff row per audit_record it appears in — which is legal, and which +// is what UNIQUE (finding_id, audit_record_id) permits. +func (f *fixture) enqueueInto(findingID int64, fingerprint string, class record.ConsumptionClass, auditID int64) Row { + f.t.Helper() + row, err := f.q.Enqueue(EnqueueRequest{ + FindingID: findingID, + AuditRecordID: auditID, + AuditID: auditUUID(auditID), + Fingerprint: fingerprint, + ConsumptionClass: class, + }) + if err != nil { + f.t.Fatalf("Enqueue into audit %d: %v", auditID, err) + } + return row +} + +// TestOneLiveLeasePerFingerprintAndRecordVersion is CRITIQUE-02 F1. +// +// A re-scan makes a NEW audit_record (scan_run_id is UNIQUE, so it must), and +// audit_version DEFAULTs to 1 on each, so one fingerprint ends up with several +// rows ALL AT VERSION 1. The two entry points then diverged onto different +// rows — AcquireLease took the oldest, Claim the newest — and granted two live +// leases on one defect at one record version. Both workers renewed, both +// released, both wrote 'validated'. research/08 §4 point 2 forbids exactly +// that outcome: "Expiring it would let a second agent write a competing fix +// for the same defect." +// +// checkRecordVersion cannot catch it: it compares a Handle against ITS OWN +// row's audit_version, which never moved. +func TestOneLiveLeasePerFingerprintAndRecordVersion(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + + first := f.sealedAudit() + second := f.sealedAudit() // the re-scan + fingerprint := fp(40) + findingID := f.newFinding(fingerprint) + older := f.enqueueInto(findingID, fingerprint, record.ConsumptionClassStaticOnly, first) + newer := f.enqueueInto(findingID, fingerprint, record.ConsumptionClassStaticOnly, second) + if older.HandoffID == newer.HandoffID { + t.Fatalf("fixture bug: both enqueues produced row %d; the test needs two rows", older.HandoffID) + } + + // Both rows are at record version 1, which is what makes this a double + // grant at ONE version rather than two versions of the work. + for _, row := range []Row{older, newer} { + var version int64 + if err := f.db.QueryRow(`SELECT audit_version FROM audit_record WHERE audit_record_id = ?`, + row.AuditRecordID).Scan(&version); err != nil { + t.Fatalf("read audit_version: %v", err) + } + if version != 1 { + t.Fatalf("fixture bug: audit_record %d is at version %d, want 1", row.AuditRecordID, version) + } + } + + held, err := f.q.Claim(fingerprint, "worker-1") + if err != nil { + t.Fatalf("first Claim: %v", err) + } + + // The second worker must be refused, by BOTH entry points, however it + // arrives. Claim names the fingerprint; AcquireLease scans the queue and + // would previously have picked the OTHER row. + if _, err := f.q.Claim(fingerprint, "worker-2"); !errors.Is(err, ErrAlreadyClaimed) { + t.Errorf("second Claim on a live fingerprint: err = %v, want ErrAlreadyClaimed", err) + } + if h, err := f.q.AcquireLease("worker-2"); err == nil { + t.Errorf("AcquireLease granted a SECOND live lease on %s (row %d, holder %q); "+ + "row %d is already held by %q at record version %d", + h.Fingerprint, h.HandoffID, h.WorkerID, held.HandoffID, held.WorkerID, held.RecordVersion) + } else if !errors.Is(err, ErrNoWork) { + t.Errorf("AcquireLease: err = %v, want ErrNoWork", err) + } + + // Exactly one row is leased. + var leased int + if err := f.db.QueryRow(`SELECT COUNT(*) FROM handoff WHERE fingerprint = ? AND state = ?`, + fingerprint, string(record.HandoffStateLeased)).Scan(&leased); err != nil { + t.Fatalf("count leased: %v", err) + } + if leased != 1 { + t.Fatalf("%d live leases on one fingerprint at one record version, want 1", leased) + } + + // The invariant is not "nothing else is ever claimable": OTHER work must + // still flow, or the guard would have converted a double grant into a + // stall. + otherAudit := f.sealedAudit() + otherFP, _ := f.enqueue(41, record.ConsumptionClassStaticOnly, otherAudit) + other, err := f.q.AcquireLease("worker-2") + if err != nil { + t.Fatalf("AcquireLease for unrelated work: %v", err) + } + if other.Fingerprint != otherFP { + t.Errorf("AcquireLease returned %s, want the unrelated finding %s", other.Fingerprint, otherFP) + } + + // And once the holder finishes, the sibling row becomes claimable again — + // the guard is about LIVE leases, not about the fingerprint forever. + if err := f.q.ReleaseLease(held, record.HandoffStateFailedValidation); err != nil { + t.Fatalf("ReleaseLease: %v", err) + } + successor, err := f.q.Claim(fingerprint, "worker-3") + if err != nil { + t.Fatalf("Claim after the first lease ended: %v", err) + } + if successor.HandoffID == held.HandoffID { + t.Errorf("the terminal row %d was re-leased", held.HandoffID) + } +} + +// TestConcurrentClaimsAcrossSiblingRowsGrantOneLease races the guard itself, +// at the CAS, the way TestClaimCASAdmitsExactlyOneWinner races the +// `state = 'ready'` guard. Every goroutine starts from an already-selected +// candidate, so only the guard inside the UPDATE can separate them. +func TestConcurrentClaimsAcrossSiblingRowsGrantOneLease(t *testing.T) { + f := newFixture(t, Options{}) + fingerprint := fp(42) + + // Every audit_record exists before the finding, because finding.first_seen_scan + // references the newest scan_run. + const siblings = 6 + var audits []int64 + for i := 0; i < siblings; i++ { + audits = append(audits, f.sealedAudit()) + } + findingID := f.newFinding(fingerprint) + + var rows []Row + for _, audit := range audits { + rows = append(rows, f.enqueueInto(findingID, fingerprint, + record.ConsumptionClassStaticOnly, audit)) + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + won int + errs []error + ) + start := make(chan struct{}) + for i, row := range rows { + wg.Add(1) + go func(i int, row Row) { + defer wg.Done() + <-start + _, ok, err := f.q.tryClaim(context.Background(), row.HandoffID, fmt.Sprintf("worker-%d", i)) + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, err) + return + } + if ok { + won++ + } + }(i, row) + } + close(start) + wg.Wait() + + for _, err := range errs { + t.Errorf("unexpected error: %v", err) + } + if won != 1 { + t.Errorf("%d of %d sibling rows were leased concurrently, want exactly 1: "+ + "one fingerprint at one record version admits one live lease", won, siblings) + } +} + +// TestConsumedAuditKeepsTheQueueOpen is CRITIQUE-02 F2. +// +// 'consumed' is a legal audit_record.state that R.6's Consume sets, and R.6's +// own ReadHalf keeps a consumed audit READABLE because plan/00-SPINE.md S1 +// requires a re-entrant consumer. The queue's gate listed only +// ('sast_sealed','both_sealed'), so the first consumption pass stranded every +// sibling finding still in 'ready' — unclaimable forever, then swept to +// 'expired' at the deadline. One audit fans out to many findings, so this was +// silent work loss on the exact axis S1 names. +func TestConsumedAuditKeepsTheQueueOpen(t *testing.T) { + for _, class := range record.ConsumptionClassValues() { + t.Run(string(class), func(t *testing.T) { + f := newFixture(t, Options{}) + deadline := f.clock.Now().Add(time.Hour) + audit := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedFindings, deadline) + taken, takenRow := f.enqueue(43, class, audit) + sibling, siblingRow := f.enqueue(44, class, audit) + + // The consumer takes the first finding and the pipeline marks the + // shared audit consumed. That must not shut the gate on the rest. + h, err := f.q.Claim(taken, "worker-a") + if err != nil { + t.Fatalf("Claim before consumption: %v", err) + } + if _, err := f.db.Exec(`UPDATE audit_record SET state = ? WHERE audit_record_id = ?`, + string(record.StateConsumed), audit); err != nil { + t.Fatalf("mark consumed: %v", err) + } + + if _, err := f.q.Claim(sibling, "worker-b"); err != nil { + t.Fatalf("a ready sibling finding became unclaimable once the audit was consumed: %v", err) + } + // A re-entrant consumer coming back for more work finds it. + f.enqueue(45, class, audit) + if _, err := f.q.AcquireLease("worker-c"); err != nil { + t.Errorf("AcquireLease on a consumed audit: err = %v, want work", err) + } + // The lease holder from before consumption is unaffected, and can + // still reach its own packet. + if _, err := f.q.WritePacket(h, []byte("packet")); err != nil { + t.Errorf("the holder cannot write its packet after consumption: %v", err) + } + if _, err := f.q.ReadPacket(h); err != nil { + t.Errorf("the holder cannot read its packet after consumption: %v", err) + } + + // And nothing is stranded into 'expired' at the deadline, which is + // the second half of the defect: unclaimable rows were swept. + f.clock.Advance(2 * time.Hour) + report, err := f.q.ExpireClaimTimeouts() + if err != nil { + t.Fatalf("ExpireClaimTimeouts: %v", err) + } + for _, e := range report.Expired { + if e.HandoffID == takenRow.HandoffID || e.HandoffID == siblingRow.HandoffID { + t.Errorf("row %d was expired although it had been claimed", e.HandoffID) + } + } + }) + } +} + +// TestExpiredAuditStaysShut is the other side of the same gate: 'consumed' is +// readable, 'expired' is not, because R.6 refuses a read of an expired audit +// whose payload the reaper has dropped. +func TestExpiredAuditStaysShut(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateBothSealed, record.HalfStatusSealed, + record.DastStatusCompletedFindings, f.clock.Now().Add(time.Hour)) + fingerprint, _ := f.enqueue(46, record.ConsumptionClassStaticOnly, audit) + + if _, err := f.db.Exec(`UPDATE audit_record SET state = ? WHERE audit_record_id = ?`, + string(record.StateExpired), audit); err != nil { + t.Fatalf("mark expired: %v", err) + } + if _, err := f.q.Claim(fingerprint, "worker-a"); !errors.Is(err, ErrNotEligible) { + t.Errorf("claiming against an expired audit: err = %v, want ErrNotEligible", err) + } +} + +// TestPacketReadIsGated is CRITIQUE-02 F5. +// +// ReadPacket is the only exported function in this package that returns a +// half's actual results. It verified neither seal state, nor audit state, nor +// lease ownership — a complete bypass of R.6's read gate, one line after the +// claim gate had correctly refused the same fingerprint. WritePacket likewise +// materialised a packet for an audit that had sealed nothing. +func TestPacketReadIsGated(t *testing.T) { + t.Run("an unsealed audit has no readable packet", func(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateCollecting, "", record.DastStatusNotRun, + f.clock.Now().Add(8*time.Hour)) + fingerprint, row := f.enqueue(47, record.ConsumptionClassStaticOnly, audit) + + // The claim gate refuses, as designed. + if _, err := f.q.Claim(fingerprint, "worker-a"); !errors.Is(err, ErrNotEligible) { + t.Fatalf("Claim on an unsealed audit: err = %v, want ErrNotEligible", err) + } + // So must the packet, whatever Handle is presented for it. + forged := Handle{ + HandoffID: row.HandoffID, FindingID: row.FindingID, + AuditRecordID: row.AuditRecordID, Fingerprint: fingerprint, + WorkerID: "worker-a", RecordVersion: 1, + leaseToken: formatTime(f.clock.Now()), + } + if _, err := f.q.WritePacket(forged, []byte("results")); err == nil { + t.Error("WritePacket materialised a packet for an audit that has sealed nothing") + } + if _, err := f.q.ReadPacket(forged); err == nil { + t.Error("ReadPacket returned an unsealed half's results with no lease and no seal check") + } + }) + + t.Run("only the lease holder may read", func(t *testing.T) { + f := newFixture(t, Options{Lease: 20 * time.Minute, MaxAttempts: 2}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(48, record.ConsumptionClassStaticOnly, audit) + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if _, err := f.q.WritePacket(h, []byte("results")); err != nil { + t.Fatalf("WritePacket: %v", err) + } + if _, err := f.q.ReadPacket(h); err != nil { + t.Fatalf("the holder cannot read its own packet: %v", err) + } + + // A different worker holding a Handle it did not earn. + impostor := h + impostor.WorkerID = "worker-b" + if _, err := f.q.ReadPacket(impostor); !errors.Is(err, ErrLeaseLost) { + t.Errorf("a non-holder read the packet: err = %v, want ErrLeaseLost", err) + } + if _, err := f.q.WritePacket(impostor, []byte("overwrite")); !errors.Is(err, ErrLeaseLost) { + t.Errorf("a non-holder overwrote the packet: err = %v, want ErrLeaseLost", err) + } + + // The OOM-killed holder wakes up after its lease was reclaimed. Its + // Handle must not reach the successor's bytes either. + f.clock.Advance(21 * time.Minute) + if _, err := f.q.ReclaimExpired(); err != nil { + t.Fatalf("ReclaimExpired: %v", err) + } + if _, err := f.q.ReadPacket(h); !errors.Is(err, ErrLeaseLost) { + t.Errorf("a reclaimed holder read the packet: err = %v, want ErrLeaseLost", err) + } + }) + + t.Run("a record version bump voids packet access", func(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(49, record.ConsumptionClassStaticOnly, audit) + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if _, err := f.q.WritePacket(h, []byte("results")); err != nil { + t.Fatalf("WritePacket: %v", err) + } + if _, err := f.db.Exec(`UPDATE audit_record SET audit_version = 2 WHERE audit_record_id = ?`, audit); err != nil { + t.Fatalf("bump audit_version: %v", err) + } + if _, err := f.q.ReadPacket(h); !errors.Is(err, ErrRecordVersionChanged) { + t.Errorf("ReadPacket after a version bump: err = %v, want ErrRecordVersionChanged", err) + } + }) + + t.Run("a shut gate refuses even a live lease", func(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(50, record.ConsumptionClassStaticOnly, audit) + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if _, err := f.q.WritePacket(h, []byte("results")); err != nil { + t.Fatalf("WritePacket: %v", err) + } + // R.6 un-seals nothing in practice, but the gate must be re-evaluated + // rather than trusted from claim time. + if _, err := f.db.Exec(`UPDATE audit_record SET state = ?, sast_status = ? WHERE audit_record_id = ?`, + string(record.StateCollecting), string(record.HalfStatusRunning), audit); err != nil { + t.Fatalf("unseal: %v", err) + } + if _, err := f.q.ReadPacket(h); !errors.Is(err, ErrNotEligible) { + t.Errorf("ReadPacket with the gate shut: err = %v, want ErrNotEligible", err) + } + }) + + t.Run("a missing packet is still regenerable, not fatal", func(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.sealedAudit() + fingerprint, _ := f.enqueue(51, record.ConsumptionClassStaticOnly, audit) + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if _, err := f.q.ReadPacket(h); !errors.Is(err, os.ErrNotExist) { + t.Errorf("reading an absent packet: err = %v, want os.ErrNotExist so the caller regenerates", err) + } + }) +} + +// TestValidatedRequiresDynamicEvidence is CRITIQUE-02 F9. +// +// plan/00-SPINE.md S7: "Only a DAST reproduction that now fails earns 'verified +// fixed'." ReleaseLease accepted 'validated' from any holder regardless of the +// Handle's ConsumptionClass and DastStatus, so a requires_dynamic_confirmation +// finding could be recorded verified-fixed on an audit whose DAST half was +// 'not_run' — a state in which no reproduction can exist to have been re-run. +func TestValidatedRequiresDynamicEvidence(t *testing.T) { + for _, status := range record.DastStatusValues() { + if status == record.DastStatusRunning { + continue // a running half is not claimable at all; see the gate + } + t.Run(string(status), func(t *testing.T) { + f := newFixture(t, Options{}) + audit, err := f.tryNewAudit(record.StateBothSealed, record.HalfStatusSealed, + status, f.clock.Now().Add(8*time.Hour)) + if err != nil { + if strings.Contains(err.Error(), "ck_audit_record_dast_status") { + // The section 6 amendment added `completed_failed` to + // internal/record; internal/store/schema.sql is a frozen + // interface this packet may not edit, so the column cannot + // hold the literal yet. The DDL is reported to the + // orchestrator. The classification itself is still + // asserted, without a database, by + // TestHasDynamicEvidenceClassifiesEveryDastStatus. + t.Skipf("ck_audit_record_dast_status does not admit %q yet; "+ + "schema.sql needs: dast_status IN (..., 'completed_failed', ...)", status) + } + t.Fatalf("newAudit: %v", err) + } + fingerprint, row := f.enqueue(52, record.ConsumptionClassRequiresDynamicConfirmation, audit) + + h, cerr := f.q.Claim(fingerprint, "worker-a") + if cerr != nil { + t.Fatalf("Claim: %v", cerr) + } + if h.DastStatus != status { + t.Fatalf("Handle.DastStatus = %q, want %q", h.DastStatus, status) + } + + err = f.q.ReleaseLease(h, record.HandoffStateValidated) + if HasDynamicEvidence(status) { + if err != nil { + t.Fatalf("'validated' refused although the DAST half produced evidence (%q): %v", status, err) + } + if got := f.state(row.HandoffID); got != record.HandoffStateValidated { + t.Errorf("state = %q, want %q", got, record.HandoffStateValidated) + } + return + } + + if !errors.Is(err, ErrNoDynamicEvidence) { + t.Fatalf("a requires_dynamic_confirmation finding was recorded 'validated' "+ + "with dast_status = %q: err = %v, want ErrNoDynamicEvidence", status, err) + } + // A refused release must not change the row: the lease is still + // live and the worker may still record an honest outcome. + if got := f.state(row.HandoffID); got != record.HandoffStateLeased { + t.Fatalf("state = %q after a refused release, want %q", got, record.HandoffStateLeased) + } + if err := f.q.ReleaseLease(h, record.HandoffStateFailedValidation); err != nil { + t.Errorf("an honest outcome was refused too: %v", err) + } + }) + } +} + +// TestStaticOnlyMayStillValidate scopes the rule above. A static_only finding +// is by definition one no dynamic evidence was required for; refusing its +// 'validated' would make the disposition unreachable for most findings, which +// is not what S7 says. +func TestStaticOnlyMayStillValidate(t *testing.T) { + f := newFixture(t, Options{}) + audit := f.newAudit(record.StateSastSealed, record.HalfStatusSealed, + record.DastStatusNotRun, f.clock.Now().Add(8*time.Hour)) + fingerprint, row := f.enqueue(53, record.ConsumptionClassStaticOnly, audit) + + h, err := f.q.Claim(fingerprint, "worker-a") + if err != nil { + t.Fatalf("Claim: %v", err) + } + if err := f.q.ReleaseLease(h, record.HandoffStateValidated); err != nil { + t.Fatalf("a static_only finding could not be validated: %v", err) + } + if got := f.state(row.HandoffID); got != record.HandoffStateValidated { + t.Errorf("state = %q, want %q", got, record.HandoffStateValidated) + } +} + +// TestHasDynamicEvidenceClassifiesEveryDastStatus keeps the predicate honest +// against the frozen enum: every literal is classified, and the ones that mean +// "no dynamic scan concluded over this finding" are all false. In particular +// completed_clean is false — the half scanned and produced NO dynamic finding, +// so there is no reproduction that can now fail. +func TestHasDynamicEvidenceClassifiesEveryDastStatus(t *testing.T) { + want := map[record.DastStatus]bool{ + record.DastStatusNotRun: false, + record.DastStatusSkippedNoManifest: false, + record.DastStatusRunning: false, + record.DastStatusCompletedClean: false, + record.DastStatusCompletedFindings: true, + record.DastStatusCompletedPartial: true, + record.DastStatusCompletedFailed: false, + record.DastStatusTargetBootFailed: false, + record.DastStatusTargetUnreachable: false, + record.DastStatusTimedOut: false, + } + for _, s := range record.DastStatusValues() { + expected, ok := want[s] + if !ok { + t.Errorf("anvil/dastStatus %q is not classified by this test; a new literal "+ + "must be decided deliberately, not defaulted", s) + continue + } + if got := HasDynamicEvidence(s); got != expected { + t.Errorf("HasDynamicEvidence(%q) = %v, want %v", s, got, expected) + } + } + if len(want) != len(record.DastStatusValues()) { + t.Errorf("the table classifies %d statuses, the enum has %d", + len(want), len(record.DastStatusValues())) + } +} diff --git a/internal/handoff/reaper.go b/internal/handoff/reaper.go new file mode 100644 index 0000000..3e2eaf0 --- /dev/null +++ b/internal/handoff/reaper.go @@ -0,0 +1,441 @@ +package handoff + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// THE REAPER RUNS TWO INDEPENDENT CLOCKS. They are not the same clock, they do +// not expire together, and they produce different state transitions. Conflating +// them is the defect plan/00-SPINE.md S1 names outright. +// +// ReclaimExpired drives handoff.lease_expires_at — Options.Lease, 15-30 +// minutes, heartbeat-renewed. It governs ONE consumer +// attempt. Expiry means "the holder is presumed dead": +// attempts < max_attempts -> back to 'ready' (this is +// the retried-exactly-once +// guarantee) +// attempts >= max_attempts -> ExhaustedState, terminal +// +// ExpireClaimTimeouts drives audit_record.deadline_at — scan_run.started_at +// plus claim_timeout_seconds, 8h by default. It governs +// how long an UNCLAIMED finding stays eligible. Expiry +// means 'expired': the tmpfs packet is unlinked, the row +// is KEPT. +// +// Neither ever touches a live claim. ExpireClaimTimeouts only considers rows in +// 'ready', which is what research/08 §4 point 2 requires: "Never expire a live +// claim. A finding at expires_at whose lease is still alive must be allowed to +// finish. Expiring it would let a second agent write a competing fix for the +// same defect." A leased row past its audit deadline reaches 'expired' only +// after its lease lapses and ReclaimExpired returns it to 'ready' — that is the +// long way round, on purpose. +// +// WHAT THE REAPER DOES NOT DO. It deletes no row: not `handoff`, not `finding`, +// not `finding_state_event`. "8 hours" is a claim timeout, not a deletion +// policy and not a confidentiality control. A finding that expires is not lost; +// it is re-presented at the next scheduled scan, so missing the window costs +// latency, not the finding. +// +// It also does not NULL audit_record.payload, although schema.sql's comment +// anticipates "the reaper" doing so. One audit_record fans out to many handoff +// rows: purging the shared payload because ONE finding's claim window lapsed +// would blind every sibling finding still leased against the same record. Per- +// audit payload purging is an audit-record-level sweep keyed on +// audit_record.deadline_at, and it belongs to whoever owns audit_record's +// lifecycle, not to the per-finding queue. Flagged to the orchestrator rather +// than quietly implemented here. + +// Reclaimed records one lease that lapsed and what became of the finding. +type Reclaimed struct { + HandoffID int64 + Fingerprint string + AuditRecordID int64 + + // WorkerID is the holder presumed dead. + WorkerID string + + // To is record.HandoffStateReady when a retry remains, ExhaustedState when + // none does. + To record.HandoffState + + // Attempts is the count as it stood when the lease lapsed: attempts + // STARTED, incremented at claim time, because a consumer that is + // OOM-killed never gets to count anything itself. + Attempts int + MaxAttempts int + + // LeaseExpiredAt is the lapsed lease_expires_at, kept for the load signal + // research/08 §4 point 4 asks for. + LeaseExpiredAt time.Time +} + +// Requeued reports whether this reclaim returned the finding to the ready set. +func (r Reclaimed) Requeued() bool { return r.To == record.HandoffStateReady } + +// Expired records one finding whose claim window closed before anyone took it. +type Expired struct { + HandoffID int64 + Fingerprint string + AuditRecordID int64 + + // DeadlineAt is audit_record.deadline_at, computed once by R.6 from + // scan_run.started_at + claim_timeout_seconds and never recomputed. The + // reaper reads it; it does not derive it, and it does not write it. + DeadlineAt time.Time + + // PacketDropped is true when a tmpfs packet existed and was unlinked. An + // unlink is all it is — see DropPacket for why no stronger claim is made. + PacketDropped bool +} + +// ReapReport is one sweep's output. research/08 §4 point 4: "Alert on expiry, +// don't just log it. A nonzero expired rate is the load signal that the coding +// agent is undersized relative to detector throughput." The counts are +// returned rather than logged so a caller can act on them. +type ReapReport struct { + At time.Time + Reclaimed []Reclaimed + Expired []Expired +} + +// Requeued counts leases that lapsed with a retry still available. +func (r ReapReport) Requeued() int { + n := 0 + for _, c := range r.Reclaimed { + if c.Requeued() { + n++ + } + } + return n +} + +// Exhausted counts leases that lapsed with no retry left. +func (r ReapReport) Exhausted() int { return len(r.Reclaimed) - r.Requeued() } + +// Empty reports whether the sweep changed nothing. +func (r ReapReport) Empty() bool { return len(r.Reclaimed) == 0 && len(r.Expired) == 0 } + +// ReclaimExpired sweeps lapsed leases. This is the crash path: a consumer +// acquires a lease, is OOM-killed mid-work, its lease lapses, and this returns +// the finding to the queue for someone else. +// +// It is idempotent in the two senses that matter: +// +// - Calling it twice changes nothing the second time. Each transition is a +// compare-and-swap on (state='leased', claimed_by, lease_expires_at); once +// the row has moved, no second sweep matches it, so attempts cannot be +// double-incremented and a finding cannot be requeued twice for one crash. +// - Re-processing after reclaim is safe. The finding keeps its +// idempotency_key and its audit_version, so the successor's work carries +// the same (fingerprint, record version) identity the dead holder's did, +// and the dead holder's own late ReleaseLease is rejected with +// ErrLeaseLost rather than landing on top of it. +func (q *Queue) ReclaimExpired() (ReapReport, error) { + return q.ReclaimExpiredContext(context.Background()) +} + +// ReclaimExpiredContext is ReclaimExpired with a caller-supplied context. +func (q *Queue) ReclaimExpiredContext(ctx context.Context) (ReapReport, error) { + now := q.Now() + report := ReapReport{At: now} + + type candidate struct { + id int64 + fingerprint string + auditRecordID int64 + worker string + expiryText string + expiry time.Time + attempts int + maxAttempts int + } + + rows, err := q.db.QueryContext(ctx, + `SELECT h.handoff_id, h.fingerprint, h.audit_record_id, h.claimed_by, + h.lease_expires_at, h.attempts, h.max_attempts + FROM handoff h + WHERE h.state = ? + ORDER BY h.handoff_id`, string(record.HandoffStateLeased)) + if err != nil { + return report, fmt.Errorf("handoff: scanning leases: %w", err) + } + + var due []candidate + for rows.Next() { + var c candidate + if err := rows.Scan(&c.id, &c.fingerprint, &c.auditRecordID, &c.worker, + &c.expiryText, &c.attempts, &c.maxAttempts); err != nil { + _ = rows.Close() + return report, fmt.Errorf("handoff: scanning leases: %w", err) + } + // Expiry is decided here, in Go, against parsed time values — never as + // a TEXT comparison in SQL, because timestamps in this database are + // written by several steps and RFC 3339 has more than one spelling of + // the same instant. See timeLayout. + c.expiry, err = parseTime("handoff.lease_expires_at", c.expiryText) + if err != nil { + _ = rows.Close() + return report, err + } + if now.Before(c.expiry) { + continue // live claim; leave it entirely alone + } + due = append(due, c) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return report, fmt.Errorf("handoff: scanning leases: %w", err) + } + if err := rows.Close(); err != nil { + return report, fmt.Errorf("handoff: scanning leases: %w", err) + } + + for _, c := range due { + to := record.HandoffStateReady + if c.attempts >= c.maxAttempts { + to = ExhaustedState + } + // CAS on the exact lease. If the holder heartbeat-renewed between the + // SELECT and here, lease_expires_at no longer matches and the live + // claim survives untouched — which is the correct outcome, not a lost + // update. + res, err := q.db.ExecContext(ctx, + `UPDATE handoff + SET state = ?, claimed_by = NULL, lease_expires_at = NULL, updated_at = ? + WHERE handoff_id = ? AND state = ? AND claimed_by = ? AND lease_expires_at = ?`, + string(to), formatTime(now), + c.id, string(record.HandoffStateLeased), c.worker, c.expiryText) + if err != nil { + return report, fmt.Errorf("handoff: reclaiming row %d: %w", c.id, err) + } + n, err := res.RowsAffected() + if err != nil { + return report, fmt.Errorf("handoff: reclaiming row %d: %w", c.id, err) + } + if n == 0 { + continue + } + if IsTerminal(to) { + if err := q.DropPacket(c.fingerprint); err != nil { + return report, err + } + } + report.Reclaimed = append(report.Reclaimed, Reclaimed{ + HandoffID: c.id, + Fingerprint: c.fingerprint, + AuditRecordID: c.auditRecordID, + WorkerID: c.worker, + To: to, + Attempts: c.attempts, + MaxAttempts: c.maxAttempts, + LeaseExpiredAt: c.expiry, + }) + } + return report, nil +} + +// ExpireClaimTimeouts sweeps findings whose claim window closed with nobody +// having taken them: audit_record.deadline_at has passed and the row is still +// 'ready'. +// +// The transition is 'ready' -> 'expired', the tmpfs packet is unlinked, and +// the database row stays exactly where it is. Nothing here deletes a record, +// and nothing here is a confidentiality measure — the same detail lives in the +// store by design, which is why S1 calls this a claim timeout and not a +// deletion policy. +// +// Leased rows are not considered at all. That is how "never expire a live +// claim" is enforced: structurally, by the query, not by a check someone can +// forget. +func (q *Queue) ExpireClaimTimeouts() (ReapReport, error) { + return q.ExpireClaimTimeoutsContext(context.Background()) +} + +// ExpireClaimTimeoutsContext is ExpireClaimTimeouts with a caller-supplied +// context. +func (q *Queue) ExpireClaimTimeoutsContext(ctx context.Context) (ReapReport, error) { + now := q.Now() + report := ReapReport{At: now} + + type candidate struct { + id int64 + fingerprint string + auditRecordID int64 + deadline time.Time + } + + rows, err := q.db.QueryContext(ctx, + `SELECT h.handoff_id, h.fingerprint, h.audit_record_id, a.deadline_at + FROM handoff h + JOIN audit_record a ON a.audit_record_id = h.audit_record_id + WHERE h.state = ? + ORDER BY h.handoff_id`, string(record.HandoffStateReady)) + if err != nil { + return report, fmt.Errorf("handoff: scanning claim timeouts: %w", err) + } + + var due []candidate + for rows.Next() { + var ( + c candidate + deadlineText string + ) + if err := rows.Scan(&c.id, &c.fingerprint, &c.auditRecordID, &deadlineText); err != nil { + _ = rows.Close() + return report, fmt.Errorf("handoff: scanning claim timeouts: %w", err) + } + c.deadline, err = parseTime("audit_record.deadline_at", deadlineText) + if err != nil { + _ = rows.Close() + return report, err + } + if now.Before(c.deadline) { + continue + } + due = append(due, c) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return report, fmt.Errorf("handoff: scanning claim timeouts: %w", err) + } + if err := rows.Close(); err != nil { + return report, fmt.Errorf("handoff: scanning claim timeouts: %w", err) + } + + for _, c := range due { + res, err := q.db.ExecContext(ctx, + `UPDATE handoff SET state = ?, updated_at = ? + WHERE handoff_id = ? AND state = ?`, + string(record.HandoffStateExpired), formatTime(now), + c.id, string(record.HandoffStateReady)) + if err != nil { + return report, fmt.Errorf("handoff: expiring row %d: %w", c.id, err) + } + n, err := res.RowsAffected() + if err != nil { + return report, fmt.Errorf("handoff: expiring row %d: %w", c.id, err) + } + if n == 0 { + // Somebody claimed it in the gap between SELECT and UPDATE. The + // live claim wins; this finding is not expired. + continue + } + + dropped, err := q.dropPacketIfPresent(c.fingerprint) + if err != nil { + return report, err + } + report.Expired = append(report.Expired, Expired{ + HandoffID: c.id, + Fingerprint: c.fingerprint, + AuditRecordID: c.auditRecordID, + DeadlineAt: c.deadline, + PacketDropped: dropped, + }) + } + return report, nil +} + +// dropPacketIfPresent unlinks a packet and reports whether one was there. A +// Queue with no PacketDir has nothing to drop, which is not an error. +func (q *Queue) dropPacketIfPresent(fingerprint string) (bool, error) { + path, err := q.PacketPath(fingerprint) + if errors.Is(err, ErrNoPacketDir) { + return false, nil + } + if err != nil { + return false, err + } + _, statErr := os.Stat(path) + if err := q.DropPacket(fingerprint); err != nil { + return false, err + } + return statErr == nil, nil +} + +// Reap runs both sweeps, leases first. +// +// The order is load-bearing. A finding whose holder crashed AND whose audit +// deadline has passed must first be reclaimed out of 'leased' — the lease +// sweep is the only thing allowed to touch a leased row — and only then can +// the claim-timeout sweep see it as 'ready' and expire it. Running the sweeps +// in the other order would make a crashed finding wait a whole extra interval +// for no reason. +func (q *Queue) Reap() (ReapReport, error) { + return q.ReapContext(context.Background()) +} + +// ReapContext is Reap with a caller-supplied context. +func (q *Queue) ReapContext(ctx context.Context) (ReapReport, error) { + leases, err := q.ReclaimExpiredContext(ctx) + if err != nil { + return leases, err + } + timeouts, err := q.ExpireClaimTimeoutsContext(ctx) + if err != nil { + return leases, err + } + return ReapReport{ + At: leases.At, + Reclaimed: leases.Reclaimed, + Expired: timeouts.Expired, + }, nil +} + +// Run sweeps every interval until ctx is cancelled, handing each report to +// observe. It returns ctx.Err(). +// +// interval <= 0 means DefaultReaperInterval. research/08 §4 states the real +// constraint on the value: "must be <= ttl/8; do NOT rely on tmpfiles' 1d +// timer" — an Age=8h tmpfiles rule fires somewhere between 8h and ~32h after +// creation, so it is a backstop, never the mechanism. +// +// observe may be nil. A sweep error is passed to observe and the loop +// continues: a transient database error must not silently stop the only thing +// that unwedges crashed consumers. +// +// Cancellation is NOT a sweep error. If ctx is cancelled while a sweep is +// mid-query, that query returns context.Canceled, and reporting it to observe +// would put "handoff: scanning leases: context canceled" in the operator's log +// on every single shutdown. That is not a harmless cosmetic difference: the +// whole value of reporting sweep errors is that a real one gets noticed, and a +// channel that cries wolf on every restart is a channel nobody reads. So an +// interrupted sweep returns ctx.Err() and stays silent. +// +// Found by CI, not locally: the race detector slows execution enough to widen +// the cancel-during-sweep window from rare to reliable. It reproduced on +// ubuntu-latest under -race while passing every run on the Windows dev host, +// which is precisely why -race is a required check rather than an optional one. +func (q *Queue) Run(ctx context.Context, interval time.Duration, observe func(ReapReport, error)) error { + if interval <= 0 { + interval = DefaultReaperInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + report, err := q.ReapContext(ctx) + // Both conditions are required. ctx.Err() alone would swallow a + // genuine database failure that happened to land in the same + // instant as a shutdown; errors.Is alone would swallow a + // context.Canceled arriving from some caller-supplied context + // nested inside the sweep, which IS a real fault worth reporting. + if err != nil && ctx.Err() != nil && + (errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)) { + return ctx.Err() + } + if observe != nil { + observe(report, err) + } + } + } +} diff --git a/internal/handoff/state_machine.go b/internal/handoff/state_machine.go new file mode 100644 index 0000000..ffc1689 --- /dev/null +++ b/internal/handoff/state_machine.go @@ -0,0 +1,711 @@ +// Package handoff implements the claim/lease protocol for the single +// `handoff` table defined by internal/store/schema.sql (step R.4). +// +// WHAT THIS PACKAGE IS, AND WHY THERE IS ONLY ONE OF IT. +// plan/IMPLEMENTATION-PLAN.md §6 ruling G9: "Area 40 owns the table and the +// claim/lease protocol." O.3 no longer writes a migration and no longer +// defines a second lease API — internal/scanctl/handoff.go becomes a thin +// adapter over this package. So this package deliberately serves both shapes +// the plan asks for, over one table and one state column: +// +// - R.7's packet shape: Claim(fingerprint, workerID) (Handle, error), +// returning ErrAlreadyClaimed on a losing race. +// - O.3's lease shape: AcquireLease / RenewLease / ReleaseLease / +// ReclaimExpired. +// +// Claim is AcquireLease narrowed to one fingerprint. They share one query, +// one CAS update and one state machine; there is no second code path and no +// second notion of "claimed". +// +// TWO CLOCKS, NEVER CONFLATED (plan/00-SPINE.md S1, research/08 §4). +// +// handoff.lease_expires_at 15–30 min (Options.Lease, default 20m), +// heartbeat-renewed, governs ONE consumer +// attempt. Expiry is handled by +// ReclaimExpired: requeue or exhaust. +// audit_record.claim_timeout_seconds default 8h, already materialised by R.6 +// as audit_record.deadline_at. It governs +// how long an UNCLAIMED finding stays +// eligible. Expiry is handled by +// ExpireClaimTimeouts: state 'expired', +// tmpfs packet unlinked, ROW KEPT. +// +// "8 hours" is a CLAIM TIMEOUT. It is not a deletion policy and it is not a +// confidentiality control (S1, and research/08 §A: "the same exploitable +// detail persists in the database indefinitely by design"). Nothing in this +// package deletes a `handoff`, `finding` or `finding_state_event` row, and +// nothing in it claims that unlinking a packet destroys anything beyond the +// link itself — see DropPacket for why no stronger claim would be true. +// +// WHAT A LEASE GRANTS (plan/00-SPINE.md S7). "May act on this finding", and +// nothing else. It is not merge authority, not widened scope, and not a +// verdict. +// +// "Only a DAST reproduction that now fails earns 'verified fixed'" is +// enforced HERE, not deferred to an unnamed elsewhere: ReleaseLease refuses +// HandoffStateValidated for a requires_dynamic_confirmation finding whose +// audit's DAST half produced no reproduction, with ErrNoDynamicEvidence. See +// checkDynamicEvidence in claim.go for the per-status reasoning. `validated` +// is written into this table by the claimant, so this is the only place the +// rule can be enforced rather than described. +// +// WHAT ARBITRATES A CLAIM, AND A DELIBERATE DEVIATION FROM THE PACKET TEXT. +// The R.7 packet names `renameat2(..., RENAME_NOREPLACE)` and OFD locks as the +// claim primitives. This implementation arbitrates the claim with a single +// conditional UPDATE against `handoff` instead, for three reasons: +// +// 1. research/08's own Recommendation §1 makes SQLite the buffer ("mechanism +// #5, primary pick") and the file queue "the file-facing veneer over #5". +// A rename-arbitrated claim plus a `state` column is two sources of truth +// for one fact — exactly the defect §6 G9/G10 closed when they deleted the +// second table. +// 2. renameat2 and F_OFD_SETLK have no binding in the standard library. +// Reaching them needs golang.org/x/sys, which is an indirect dependency +// today; promoting it is a go.mod edit this packet may not make. +// 3. SQLite serialises writers, so `UPDATE ... WHERE state = 'ready'` has the +// property RENAME_NOREPLACE was wanted for: exactly one caller sees one +// affected row, every other caller sees zero and gets ErrAlreadyClaimed. +// +// The Forbidden action that matters is honoured absolutely: this package takes +// no classic fcntl record locks anywhere — not one — so the +// close()-drops-all-locks footgun documented at research/08 §D cannot occur. +// The tmpfs packet is still written by research/08 §C's durable recipe +// (exclusive temp file in the same directory, fsync, rename, fsync parent) and +// never relies on ext4's auto_da_alloc; see WritePacket. +package handoff + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/record" +) + +// DefaultLease is research/08 §4's `buffer.lease`: 15–30 minutes with +// heartbeat renewal, never 8 hours. "An 8-hour lease means a crashed agent +// blocks a finding for a whole shift." +const DefaultLease = 20 * time.Minute + +// DefaultMaxAttempts is research/08 §4's `buffer.max_attempts`: 2, i.e. one +// retry after one crash. It matches schema.sql's own column default. +const DefaultMaxAttempts = 2 + +// DefaultReaperInterval is research/08 §4's `buffer.reaper_interval`. The +// constraint it satisfies is stated there: "must be <= ttl/8; do NOT rely on +// tmpfiles' 1d timer". +const DefaultReaperInterval = 5 * time.Minute + +// ExhaustedState is where a finding lands when its lease expires for the last +// time — attempts have reached max_attempts and no retry remains. +// +// The thirteen handoff.state literals are frozen by §6 and contain no generic +// `failed`; research/08's pseudocode says `state='failed'` because it was +// written before the enum was frozen. Of the two failure literals that do +// exist, HandoffStateFailedFormat is specifically a defect in the packet's +// format, which a crashed consumer is not. So an exhausted attempt is recorded +// as HandoffStateFailedValidation: the attempt did not produce a validated +// fix. This is a mapping decision, made once, here, rather than at each call +// site. +const ExhaustedState = record.HandoffStateFailedValidation + +// timeLayout is the on-disk timestamp format for every column this package +// writes. It is fixed-width on purpose. +// +// time.RFC3339Nano trims trailing zeros from the fractional part, which makes +// it non-monotonic under lexicographic comparison: "10:00:00Z" sorts AFTER +// "10:00:00.5Z" because 'Z' > '.'. Any SQL that compared lease deadlines as +// TEXT with that format would silently mis-order them. This package does not +// rely on that either way — every expiry decision is made in Go against parsed +// time.Time values (see parseTime) — but it writes a format that would also be +// correct if someone later added an index range scan. +const timeLayout = "2006-01-02T15:04:05.000000000Z" + +// formatTime renders t for storage: UTC, fixed width, RFC 3339 parseable. +func formatTime(t time.Time) string { return t.UTC().Format(timeLayout) } + +// parseTime reads a stored timestamp. It accepts any RFC 3339 spelling, not +// just timeLayout's, because audit_record.deadline_at is written by R.6 and +// schema_migration.applied_at by R.5; this package must read their formats +// without dictating them. +func parseTime(field, s string) (time.Time, error) { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return time.Time{}, fmt.Errorf("handoff: %s is not an RFC 3339 timestamp: %w", field, err) + } + return t.UTC(), nil +} + +// Sentinel errors. Callers distinguish these; the text is not an interface. +var ( + // ErrAlreadyClaimed is returned to the loser of a claim race, and to any + // caller asking for a finding another worker currently holds. + ErrAlreadyClaimed = errors.New("handoff: finding is already claimed") + + // ErrNotFound means no handoff row exists for the identifier given. + ErrNotFound = errors.New("handoff: no such handoff row") + + // ErrNoWork means the ready set is empty or nothing in it has passed its + // consumption gate yet. It is not an error condition; it is "idle". + ErrNoWork = errors.New("handoff: no claimable finding") + + // ErrNotEligible means the row exists and is ready, but its consumption + // gate is shut: research/21 §5's static_only findings wait on the SAST + // half, requires_dynamic_confirmation findings wait on the DAST half. + ErrNotEligible = errors.New("handoff: finding has not passed its consumption gate") + + // ErrExhausted means the row is ready but has already burned every + // attempt, so re-leasing it would loop forever. + ErrExhausted = errors.New("handoff: finding has exhausted its attempts") + + // ErrLeaseLost means the Handle no longer describes the row: the lease + // expired and was reclaimed, another worker holds it now, or it was + // disposed of. A consumer that gets this MUST NOT apply its work — this is + // the guard that stops an OOM-killed consumer's late write from landing on + // top of its successor's. + ErrLeaseLost = errors.New("handoff: lease is no longer held") + + // ErrRecordVersionChanged means audit_record.audit_version moved under the + // lease. Per plan/00-SPINE.md S6 a version bump re-cuts the work queue, so + // the work unit the Handle describes no longer exists. It implies + // ErrLeaseLost. + ErrRecordVersionChanged = errors.New("handoff: audit record version changed under the lease") + + // ErrIllegalTransition is the class of every rejected state change; see + // TransitionError. + ErrIllegalTransition = errors.New("handoff: illegal state transition") + + // ErrNoDynamicEvidence means a requires_dynamic_confirmation finding was + // released as 'validated' on an audit whose DAST half produced no + // reproduction. plan/00-SPINE.md S7: only a DAST reproduction that now + // FAILS earns "verified fixed"; a clean static rescan does not. See + // checkDynamicEvidence. + ErrNoDynamicEvidence = errors.New("handoff: 'validated' requires dynamic evidence for this finding") +) + +// TransitionError reports a state change the machine forbids. +type TransitionError struct { + From record.HandoffState + To record.HandoffState +} + +func (e *TransitionError) Error() string { + return fmt.Sprintf("handoff: illegal state transition %q -> %q", e.From, e.To) +} + +// Is makes errors.Is(err, ErrIllegalTransition) true for every TransitionError. +func (e *TransitionError) Is(target error) bool { return target == ErrIllegalTransition } + +// legalTransitions is the whole state machine. Every one of the thirteen +// frozen handoff.state literals is a key; the terminal ones map to nothing, +// which is what makes them terminal. handoff_test.go asserts key coverage +// against record.HandoffStateValues() so a future enum addition cannot land +// here as a silent hole. +// +// Two edges are absent on purpose: +// +// - leased -> expired. research/08 §4 point 2: "Never expire a live claim. A +// finding at expires_at whose lease is still alive must be allowed to +// finish. Expiring it would let a second agent write a competing fix for +// the same defect." A leased row reaches 'expired' only the long way: +// ReclaimExpired returns it to 'ready' first. +// - anything -> ready from a terminal state. Terminal is terminal. Re-cutting +// a superseded finding creates a NEW row for the new audit_record, which is +// what UNIQUE (finding_id, audit_record_id) is shaped for. +var legalTransitions = map[record.HandoffState][]record.HandoffState{ + record.HandoffStateReady: { + record.HandoffStateLeased, + record.HandoffStateExpired, + record.HandoffStateSkippedBudget, + record.HandoffStateFalsePositive, + record.HandoffStateFixedIncidentally, + record.HandoffStateSplitRequired, + record.HandoffStateWithdrawn, + record.HandoffStateSuperseded, + }, + record.HandoffStateLeased: { + record.HandoffStateReady, + record.HandoffStateValidated, + record.HandoffStateFailedValidation, + record.HandoffStateFailedFormat, + record.HandoffStateRegressionIntroduced, + record.HandoffStateSkippedBudget, + record.HandoffStateFalsePositive, + record.HandoffStateFixedIncidentally, + record.HandoffStateSplitRequired, + record.HandoffStateWithdrawn, + record.HandoffStateSuperseded, + }, + record.HandoffStateValidated: nil, + record.HandoffStateFailedValidation: nil, + record.HandoffStateFailedFormat: nil, + record.HandoffStateSkippedBudget: nil, + record.HandoffStateFalsePositive: nil, + record.HandoffStateRegressionIntroduced: nil, + record.HandoffStateFixedIncidentally: nil, + record.HandoffStateSplitRequired: nil, + record.HandoffStateWithdrawn: nil, + record.HandoffStateSuperseded: nil, + record.HandoffStateExpired: nil, +} + +// LegalTransitions returns a copy of the state machine, keyed by source state. +func LegalTransitions() map[record.HandoffState][]record.HandoffState { + out := make(map[record.HandoffState][]record.HandoffState, len(legalTransitions)) + for from, to := range legalTransitions { + out[from] = append([]record.HandoffState(nil), to...) + } + return out +} + +// CanTransition reports whether from -> to is legal. +func CanTransition(from, to record.HandoffState) bool { + for _, candidate := range legalTransitions[from] { + if candidate == to { + return true + } + } + return false +} + +// CheckTransition returns nil if from -> to is legal, a *TransitionError +// otherwise. An unknown literal on either side is rejected by +// record.ValidateHandoffState first, so a typo cannot masquerade as a state. +func CheckTransition(from, to record.HandoffState) error { + if err := record.ValidateHandoffState(string(from)); err != nil { + return err + } + if err := record.ValidateHandoffState(string(to)); err != nil { + return err + } + if !CanTransition(from, to) { + return &TransitionError{From: from, To: to} + } + return nil +} + +// IsTerminal reports whether s admits no further transition. 'ready' and +// 'leased' are the only live states; the other eleven are terminal. +func IsTerminal(s record.HandoffState) bool { + return s.Valid() && len(legalTransitions[s]) == 0 +} + +// IsLive reports whether s is a state the queue still works on. +func IsLive(s record.HandoffState) bool { + return s == record.HandoffStateReady || s == record.HandoffStateLeased +} + +// Options configures a Queue. The zero value is usable: every field falls back +// to the Default* constant above, which are research/08 §4's configuration +// keys and not magic numbers invented here. +type Options struct { + // Lease is `buffer.lease` — how long one consumer attempt may hold a + // finding before ReclaimExpired presumes the holder dead. + Lease time.Duration + + // MaxAttempts is `buffer.max_attempts` for rows this Queue enqueues. Each + // row carries its own handoff.max_attempts; this is only the default + // written at insert time. + MaxAttempts int + + // PacketDir is the tmpfs directory holding regenerable packets, e.g. + // systemd's RuntimeDirectory=anvil. Empty disables packet materialisation + // entirely, which is legal: the packet is a cache, never a source of + // truth, and research/08 §1 says so ("if it vanishes, regenerate it from + // the DB"). + PacketDir string + + // Clock is injectable so the lease clock and the claim-timeout clock can + // be driven independently in tests. nil means time.Now. + Clock func() time.Time +} + +func (o Options) lease() time.Duration { + if o.Lease <= 0 { + return DefaultLease + } + return o.Lease +} + +func (o Options) maxAttempts() int { + if o.MaxAttempts <= 0 { + return DefaultMaxAttempts + } + return o.MaxAttempts +} + +// Queue is the claim/lease protocol over the `handoff` table. It is safe for +// concurrent use: every mutation is a single conditional UPDATE and SQLite +// serialises writers. +type Queue struct { + db *sql.DB + opts Options +} + +// New returns a Queue over an already-migrated store. It does not create, +// alter or migrate any table: internal/store/schema.sql owns `handoff` and +// declares itself a frozen interface (§6 G9). +func New(db *sql.DB, opts Options) (*Queue, error) { + if db == nil { + return nil, errors.New("handoff: New requires a non-nil *sql.DB") + } + if opts.Clock == nil { + opts.Clock = time.Now + } + return &Queue{db: db, opts: opts}, nil +} + +// Now is the Queue's clock, in UTC. +func (q *Queue) Now() time.Time { return q.opts.Clock().UTC() } + +// Lease reports the configured claim-lease duration. +func (q *Queue) Lease() time.Duration { return q.opts.lease() } + +// Row is one `handoff` row, read back. +type Row struct { + HandoffID int64 + FindingID int64 + AuditRecordID int64 + Fingerprint string + GroupID string + State record.HandoffState + ConsumptionClass record.ConsumptionClass + ClaimedBy string + LeaseExpiresAt time.Time // zero when not leased + Attempts int + MaxAttempts int + IdempotencyKey string + CreatedAt time.Time + UpdatedAt time.Time +} + +// AttemptsRemaining reports how many further leases the row may be granted. +func (r Row) AttemptsRemaining() int { + if r.Attempts >= r.MaxAttempts { + return 0 + } + return r.MaxAttempts - r.Attempts +} + +// rowColumns is qualified with the alias `h` because several of these names — +// `state` above all — also exist on audit_record, which the eligibility query +// joins. An unqualified `state` there would be ambiguous at best and silently +// the wrong table's at worst, so every query in this package aliases handoff +// as h and selects through this constant. +const rowColumns = `h.handoff_id, h.finding_id, h.audit_record_id, h.fingerprint, h.group_id, h.state, + h.consumption_class, h.claimed_by, h.lease_expires_at, h.attempts, h.max_attempts, + h.idempotency_key, h.created_at, h.updated_at` + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRow(sc rowScanner) (Row, error) { + var ( + r Row + groupID sql.NullString + claimedBy sql.NullString + leaseExpiry sql.NullString + idemKey sql.NullString + state string + class string + createdAt string + updatedAt string + ) + if err := sc.Scan( + &r.HandoffID, &r.FindingID, &r.AuditRecordID, &r.Fingerprint, &groupID, &state, + &class, &claimedBy, &leaseExpiry, &r.Attempts, &r.MaxAttempts, + &idemKey, &createdAt, &updatedAt, + ); err != nil { + return Row{}, err + } + // The literals come out of the database, so they are validated on the way + // in rather than trusted: a row written by an older binary, or by hand, + // must not put an unknown state into the state machine. + if err := record.ValidateHandoffState(state); err != nil { + return Row{}, fmt.Errorf("handoff: row %d: %w", r.HandoffID, err) + } + if err := record.ValidateConsumptionClass(class); err != nil { + return Row{}, fmt.Errorf("handoff: row %d: %w", r.HandoffID, err) + } + r.State = record.HandoffState(state) + r.ConsumptionClass = record.ConsumptionClass(class) + r.GroupID = groupID.String + r.ClaimedBy = claimedBy.String + r.IdempotencyKey = idemKey.String + + var err error + if leaseExpiry.Valid { + if r.LeaseExpiresAt, err = parseTime("handoff.lease_expires_at", leaseExpiry.String); err != nil { + return Row{}, err + } + } + if r.CreatedAt, err = parseTime("handoff.created_at", createdAt); err != nil { + return Row{}, err + } + if r.UpdatedAt, err = parseTime("handoff.updated_at", updatedAt); err != nil { + return Row{}, err + } + return r, nil +} + +// Get returns one row by primary key. It reports ErrNotFound, never a +// zero-value Row, when the row is absent. +func (q *Queue) Get(handoffID int64) (Row, error) { + return q.GetContext(context.Background(), handoffID) +} + +// GetContext is Get with a caller-supplied context. +func (q *Queue) GetContext(ctx context.Context, handoffID int64) (Row, error) { + r, err := scanRow(q.db.QueryRowContext(ctx, + `SELECT `+rowColumns+` FROM handoff h WHERE h.handoff_id = ?`, handoffID)) + if errors.Is(err, sql.ErrNoRows) { + return Row{}, fmt.Errorf("handoff: id %d: %w", handoffID, ErrNotFound) + } + if err != nil { + return Row{}, fmt.Errorf("handoff: reading row %d: %w", handoffID, err) + } + return r, nil +} + +// Find returns every handoff row carrying a fingerprint, newest audit record +// first. A fingerprint is not unique in this table — it is denormalised for +// the reaper's WHERE clause, and one finding legitimately has one row per +// audit record it appears in. +func (q *Queue) Find(fingerprint string) ([]Row, error) { + return q.FindContext(context.Background(), fingerprint) +} + +// FindContext is Find with a caller-supplied context. +func (q *Queue) FindContext(ctx context.Context, fingerprint string) ([]Row, error) { + if err := ValidateFingerprint(fingerprint); err != nil { + return nil, err + } + rows, err := q.db.QueryContext(ctx, + `SELECT `+rowColumns+` FROM handoff h WHERE h.fingerprint = ? + ORDER BY h.audit_record_id DESC, h.handoff_id DESC`, fingerprint) + if err != nil { + return nil, fmt.Errorf("handoff: querying fingerprint %s: %w", fingerprint, err) + } + defer func() { _ = rows.Close() }() + + var out []Row + for rows.Next() { + r, err := scanRow(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("handoff: querying fingerprint %s: %w", fingerprint, err) + } + return out, nil +} + +// EnqueueRequest describes one finding entering the ready set. +type EnqueueRequest struct { + FindingID int64 + AuditRecordID int64 + + // AuditID is `anvil/auditId`, the audit's own identity as assigned at scan + // start — NOT the `audit_record_id` rowid. It is required, because it is + // the first component of IdempotencyKey and the key is what the coding + // agent writes into a git trailer; a rowid there would be a value no other + // process can interpret (CRITIQUE-02 F7). + // + // It is supplied by the caller rather than read from the store because + // `audit_record` has no column for it. That gap is reported to the + // orchestrator: until schema.sql carries an `audit_id`, the key cannot be + // re-derived from the database alone, only recomputed by whoever still + // holds the record. + AuditID string + + // Fingerprint is the full 64-hex anvil-fp/v1 digest, never truncated + // (internal/record/FINGERPRINT-SPEC.md, and ck_handoff_fingerprint_hex). + Fingerprint string + + // ConsumptionClass gates the finding. It has no default here for the same + // reason schema.sql gives it no column default: a default would silently + // grant every row the permissive value. + ConsumptionClass record.ConsumptionClass + + // GroupID is the fix-group id, assigned by the consumption pipeline. + // Optional. + GroupID string + + // MaxAttempts overrides Options.MaxAttempts for this row. + MaxAttempts int +} + +// Enqueue inserts one finding into the ready set and returns the resulting +// row. It is idempotent: re-enqueueing the same (finding_id, audit_record_id) +// returns the existing row unchanged rather than failing or duplicating, so a +// crashed producer that re-runs cannot double-enqueue. +func (q *Queue) Enqueue(req EnqueueRequest) (Row, error) { + return q.EnqueueContext(context.Background(), req) +} + +// EnqueueContext is Enqueue with a caller-supplied context. +func (q *Queue) EnqueueContext(ctx context.Context, req EnqueueRequest) (Row, error) { + if err := ValidateFingerprint(req.Fingerprint); err != nil { + return Row{}, err + } + if err := record.ValidateConsumptionClass(string(req.ConsumptionClass)); err != nil { + return Row{}, err + } + if req.AuditID == "" { + return Row{}, errors.New( + "handoff: Enqueue requires anvil/auditId; the idempotency key is " + + "sha256(audit_id || finding_fingerprint || base_commit_sha) and audit_record_id is a rowid, not an audit identity") + } + maxAttempts := req.MaxAttempts + if maxAttempts <= 0 { + maxAttempts = q.opts.maxAttempts() + } + + // The idempotency key mirrors the git trailer the coding agent writes, so + // the same unit of work is recognisable on both sides of a crash. Its + // definition is schema.sql's, quoted: sha256(audit_id || finding_fingerprint + // || base_commit_sha). + var commitSHA sql.NullString + err := q.db.QueryRowContext(ctx, + `SELECT s.commit_sha FROM audit_record a + JOIN scan_run s ON s.scan_run_id = a.scan_run_id + WHERE a.audit_record_id = ?`, req.AuditRecordID).Scan(&commitSHA) + if errors.Is(err, sql.ErrNoRows) { + return Row{}, fmt.Errorf("handoff: audit_record %d: %w", req.AuditRecordID, ErrNotFound) + } + if err != nil { + return Row{}, fmt.Errorf("handoff: reading base commit for audit_record %d: %w", req.AuditRecordID, err) + } + key := IdempotencyKey(req.AuditID, req.Fingerprint, commitSHA.String) + + now := formatTime(q.Now()) + var groupID any + if req.GroupID != "" { + groupID = req.GroupID + } + if _, err := q.db.ExecContext(ctx, + `INSERT INTO handoff + (finding_id, audit_record_id, fingerprint, group_id, state, consumption_class, + attempts, max_attempts, idempotency_key, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?) + ON CONFLICT DO NOTHING`, + req.FindingID, req.AuditRecordID, req.Fingerprint, groupID, + string(record.HandoffStateReady), string(req.ConsumptionClass), + maxAttempts, key, now, now, + ); err != nil { + return Row{}, fmt.Errorf("handoff: enqueueing finding %d: %w", req.FindingID, err) + } + + r, err := scanRow(q.db.QueryRowContext(ctx, + `SELECT `+rowColumns+` FROM handoff h WHERE h.finding_id = ? AND h.audit_record_id = ?`, + req.FindingID, req.AuditRecordID)) + if errors.Is(err, sql.ErrNoRows) { + // ON CONFLICT DO NOTHING swallowed a conflict on some OTHER unique + // key — idempotency_key is UNIQUE across the whole table. Say so + // instead of returning a confusing "not found". + return Row{}, fmt.Errorf( + "handoff: enqueueing finding %d into audit_record %d inserted nothing and no such row exists; "+ + "idempotency key %s is already held by a different row", + req.FindingID, req.AuditRecordID, key) + } + if err != nil { + return Row{}, fmt.Errorf("handoff: reading back enqueued finding %d: %w", req.FindingID, err) + } + return r, nil +} + +// Dispose moves a READY row straight to a terminal state, without a lease. +// It is how the triage gate records 'false_positive', how the queue re-cut +// records 'skipped_budget', 'withdrawn' and 'superseded', and how a finding +// fixed by someone else's patch records 'fixed_incidentally'. +// +// A leased row is not disposable this way — ReleaseLease is the only exit from +// 'leased', because only the lease holder may decide the outcome of its own +// attempt. +func (q *Queue) Dispose(handoffID int64, to record.HandoffState) error { + return q.DisposeContext(context.Background(), handoffID, to) +} + +// DisposeContext is Dispose with a caller-supplied context. +func (q *Queue) DisposeContext(ctx context.Context, handoffID int64, to record.HandoffState) error { + if err := CheckTransition(record.HandoffStateReady, to); err != nil { + return err + } + if to == record.HandoffStateLeased { + return &TransitionError{From: record.HandoffStateReady, To: to} + } + res, err := q.db.ExecContext(ctx, + `UPDATE handoff SET state = ?, updated_at = ? WHERE handoff_id = ? AND state = ?`, + string(to), formatTime(q.Now()), handoffID, string(record.HandoffStateReady)) + if err != nil { + return fmt.Errorf("handoff: disposing row %d as %s: %w", handoffID, to, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("handoff: disposing row %d as %s: %w", handoffID, to, err) + } + if n == 1 { + return nil + } + + current, err := q.GetContext(ctx, handoffID) + if err != nil { + return err + } + if current.State == record.HandoffStateLeased { + return fmt.Errorf("handoff: row %d is leased by %q: %w", handoffID, current.ClaimedBy, ErrAlreadyClaimed) + } + return &TransitionError{From: current.State, To: to} +} + +// IdempotencyKey computes schema.sql's `handoff.idempotency_key`: +// sha256(audit_id || finding_fingerprint || base_commit_sha), hex. It is +// exported because the coding agent writes the same value into a git trailer, +// and the two must be computed the same way or the trailer proves nothing. +// +// auditID IS `anvil/auditId`, NOT `audit_record.audit_record_id`. That +// distinction is the whole of CRITIQUE-02 F7: this function used to hash the +// autoincrement rowid, which is not the audit identity by any definition. The +// consequences were concrete — the exported value "the coding agent writes into +// a git trailer" was an internal database rowid, which is not a portable +// identity and means nothing outside one copy of one database file, and two +// competing leases on one finding carried DIFFERENT keys, so the downstream +// duplicate-suppression the Handle doc promises could not have caught the +// double grant either. +// +// The components are joined by a NUL byte so that no two different triples can +// produce the same input string by shifting a boundary. +func IdempotencyKey(auditID, fingerprint, baseCommitSHA string) string { + h := sha256.New() + h.Write([]byte(auditID)) + h.Write([]byte{0}) + h.Write([]byte(fingerprint)) + h.Write([]byte{0}) + h.Write([]byte(baseCommitSHA)) + return hex.EncodeToString(h.Sum(nil)) +} + +// ValidateFingerprint enforces the same shape ck_handoff_fingerprint_hex +// enforces: 64 lowercase hex characters, never truncated. It runs before any +// fingerprint reaches a query or a packet path, so a malformed value fails +// here rather than as a constraint violation or a traversed path. +func ValidateFingerprint(fp string) error { + if len(fp) != 64 { + return fmt.Errorf("handoff: fingerprint %q is %d characters, want 64 (anvil-fp/v1 digests are never truncated)", fp, len(fp)) + } + if strings.ToLower(fp) != fp { + return fmt.Errorf("handoff: fingerprint %q must be lowercase hex", fp) + } + if _, err := hex.DecodeString(fp); err != nil { + return fmt.Errorf("handoff: fingerprint %q is not hex: %w", fp, err) + } + return nil +} diff --git a/internal/record/CONTRACT.md b/internal/record/CONTRACT.md index 592b144..46151a7 100644 --- a/internal/record/CONTRACT.md +++ b/internal/record/CONTRACT.md @@ -80,6 +80,7 @@ missing key and an unsealed half must not be the same observation. ### 1.3 `anvil/dastStatus` — audit-level DAST outcome (rulings G3 + G6, found twice) `not_run | skipped_no_manifest | running | completed_clean | completed_findings | completed_partial | +completed_failed | target_boot_failed | target_unreachable | timed_out` **Producer:** the scan controller, **derived from** the DAST half's `anvil/status` and from @@ -473,3 +474,37 @@ S6 additions it predates — `anvil/state`, `anvil/version`, `anvil/deadline`, ` is the schema working**, not a defect; the packet's original "validates the annotated example with zero errors" criterion was written before §6's rulings and cannot be satisfied simultaneously with S6's "all of these are required." + + +--- + +## Amendment 2026-08-07 — `anvil/dastStatus` gains `completed_failed` + +The frozen enum had **no image for "the DAST half itself broke"**. A half with `anvil/status = failed` +against a target whose provenance is `booted_clean` had nowhere legal to land, and `R.6` was folding it +onto `completed_partial` — flagging the compromise rather than absorbing it silently. + +That fold is wrong for the same reason S6 requires a failed target to be distinguishable from one +scanned clean: a half that **crashed** differs from one that **covered part of the surface**. Collapsing +them makes `dast_coverage` uninterpretable, because a 40% figure could mean "we probed 40% and stopped" +or "we probed 40% and the engine died". `DeriveDastStatus` is now total — every +(provenance, half-status) pair has exactly one image. + +**This vocabulary lives in five places and all five must move together:** + +| # | Location | What it is | +|---|---|---| +| 1 | `plan/IMPLEMENTATION-PLAN.md` §6 | the ruling | +| 2 | `internal/record/contract.go` | the Go constants and `DastStatusValues()` | +| 3 | `internal/store/schema.sql` | `ck_audit_record_dast_status` | +| 4 | `schemas/anvil-record-v1.schema.json` | the published wire schema | +| 5 | this file | the contract other areas are pointed at | + +**The amendment initially landed in only 1 and 2, and the tree went red.** `R.4`'s +`TestEnumCheckConstraintsMatchContractLiteralForLiteral` caught it immediately by comparing the SQL +CHECK against the Go enum literal-for-literal — the guard working exactly as intended. The operational +consequence had it shipped was worse than the fold it replaced: an audit whose DAST half crashed could +not be persisted **at all**, because the derivation produced a literal the store rejected. + +Recorded because the lesson generalises: **one vocabulary with five definitions is the same defect §6 +was written to close**, and an amendment is exactly when it recurs. diff --git a/internal/record/SECRETS.md b/internal/record/SECRETS.md new file mode 100644 index 0000000..d88fd88 --- /dev/null +++ b/internal/record/SECRETS.md @@ -0,0 +1,233 @@ +# SECRETS.md — Retention, Deletion, and What Anvil Does Not Guarantee + +**Status: honesty document.** It exists to stop a reader believing Anvil erases anything. It describes +no feature. Every security claim below is transcribed from `research/08-buffer-and-handoff.md`, which +sourced it first; every claim about this repository cites the file that makes it true. + +| | | +|---|---| +| Owning step | `R.9`, `plan/40-record-and-storage.md` | +| Source of every security claim | `research/08-buffer-and-handoff.md` (§F "Buffer security", §"Risks, Dissent And Failure Modes") | +| Binding spine text | `plan/00-SPINE.md` S1 item 5 | +| Related code | `internal/store/schema.sql`, `internal/store/ddl.go`, `internal/store/migrate.go` | + +--- + +## 0. The whole document in four sentences + +1. **Nothing in this repository securely erases anything.** No code path here overwrites, shreds, + crypto-erases, or otherwise sanitises the tmpfs packet, the `audit_record.payload` blob, or any + database row. +2. **`shred` would not fix that**, and Anvil must not pretend otherwise: on the filesystems and storage + Anvil actually runs on, `shred` cannot work. +3. **The 8-hour window is a claim timeout, not a confidentiality control.** It bounds how long an + unclaimed finding stays eligible for a coding agent. It bounds nothing about who can read what. +4. **If confidentiality-at-rest is a requirement, the control is a per-scan LUKS2 volume or `fscrypt` — + volume/filesystem encryption with key destruction — not application-level deletion.** + +If you finish this document feeling reassured, re-read section 2. + +--- + +## 1. What exists, and for how long + +Three things hold finding detail. `plan/00-SPINE.md` S1 item 5 collapsed the original "8-hour buffer +file" into exactly these: + +| Thing | Where | What happens at the claim timeout | +|---|---|---| +| The regenerable packet | tmpfs (`/run/anvil/...`) | Dropped (`unlink`). Not a source of truth; regenerable from the store. | +| `audit_record.payload` | the SQLite database file | Set to `NULL` by the reaper (`internal/store/schema.sql`, `audit_record.payload` comment). | +| `finding`, `finding_occurrence`, `finding_state_event`, `handoff` rows | the SQLite database file | **Never deleted.** The row moves to the `expired` literal (`record.StateExpired`, `record.HandoffStateExpired`). | + +`audit_record.payload_sha256` is deliberately retained after `payload` is `NULL`ed — it is proof of what +was handed over. That is a retention decision, stated here so nobody mistakes the reaper for erasure. + +The database row surviving is intentional and is the reason the timeout is safe to adopt at all: +research/08 §4.3 — *"The finding is therefore not lost; it is simply re-presented at the next scheduled +full scan or the next matching trigger. **Missing the window costs latency, not the finding.**"* + +--- + +## 2. Anvil does not securely erase anything, and `shred` would not help + +research/08 §F, quoting the coreutils manual: *"shred assumes the file system and hardware overwrite data +in place. Although this is common, many platforms operate otherwise."* [S12] + +The manual then enumerates where it fails, quoted in research/08 verbatim: *"Log-structured or journaled +file systems, such as ext3/ext4 (in `data=journal` mode), Btrfs, NTFS, ReiserFS, XFS, ZFS"*; *"File +systems that write redundant data and carry on even if some writes fail, such as RAID-based file +systems"*; *"File systems that make snapshots"*; *"File systems that cache in temporary locations, such as +NFS version 3 clients"*; *"Compressed file systems"*. [S13] + +On flash, research/08 quotes: *"Solid-state storage devices (SSDs) typically do wear leveling to prolong +service life, and this means writes are distributed to other blocks by the hardware, so 'overwritten' +data blocks are still present in the underlying device."* [S13] + +research/08's own conclusion, which this document does not soften: *"on btrfs, ZFS, or any SSD — i.e. +essentially every modern deployment target — `shred` on the buffer file is theatre"*, and *"Any Anvil +design doc that says 'we shred the buffer at expiry' is wrong and should be corrected before it ships."* + +**Therefore: no document, comment, commit message, or release note in this project may claim that Anvil +shreds, wipes, or securely deletes anything.** It does not, and adding `shred` would not make it true. + +The current authority is NIST SP 800-88 Rev. 2, *Guidelines for Media Sanitization*, final 2025-09-26 +[S28]; research/08 states the operational implication as *"the meaningful erase primitive for flash is +key destruction, not overwriting."* research/08 also flags, in its own Gaps section, that the full +SP 800-88 Rev. 2 PDF was not fetched and that the flash-overwrite claim rests on the coreutils manual +[S13], which it judges independent and sufficient. That caveat travels with the claim. + +--- + +## 3. The 8-hour window is a latency bound, not a confidentiality guarantee + +`plan/00-SPINE.md` S1 item 5 is binding and unambiguous: *"'8 hours' is a **claim timeout**, not a +deletion policy and not a confidentiality control."* + +research/08 §A reached the same conclusion first: *"Deleting the buffer at 8 hours is **not** a +confidentiality control, because the same exploitable detail persists in the database indefinitely by +design. Treat the 8h TTL as a staleness/queue-depth control."* + +And in Risks, as a named failure mode: *"**Two copies is the real security risk, and the TTL hides it.** +An 8-hour buffer TTL creates the *impression* of a short exposure window while the identical content sits +in the database indefinitely."* + +Two clocks exist and are never the same clock (`internal/store/schema.sql`, `handoff` header comment): + +* `handoff.lease_expires_at` — 15–30 minutes, heartbeat-renewed, governs **one** coding-agent attempt. +* `audit_record.claim_timeout_seconds` — 8 hours by default, governs how long an **unclaimed** finding + stays eligible. + +Neither is an exposure window. Neither is measured against an attacker. + +--- + +## 4. If confidentiality-at-rest is required, this is the control + +research/08 §F gives the priority order. It is not application-level deletion at any position. + +1. **Keep the plaintext off persistent media entirely.** tmpfs, and since Linux 6.4 the `noswap` mount + option disables swapping for that instance [S9]; *"If a tmpfs filesystem is unmounted, its contents + are discarded (lost)"* [S9]. research/08 adds: *"Disable core dumps for the units, since RAM is the + whole attack surface in this configuration."* + **Two things research/08 could not verify, and which therefore must be checked, not assumed:** that + `/run` is a tmpfs on every target distribution (`systemd.exec(5)` says only that `RuntimeDirectory=` + is created below `/run/` [S29]) — mount an explicit tmpfs or check `/proc/mounts` at startup — and the + default value of `RuntimeDirectoryMode=`, which research/08 says to set explicitly to `0700` + regardless. Where `noswap` is unavailable, the packet can reach swap. +2. **If it must persist, encrypt the volume**, *"so that expiry can be implemented as key destruction. + dm-crypt/LUKS via cryptsetup is GPL-2.0 (with an explicit OpenSSL-linking exception)"* [S33]. This is + the answer to a hard requirement to prove destruction: research/08 §3 — *"SQLite row deletion + + `secure_delete` cannot prove it on flash [S13][S15]; you would move to a per-scan LUKS2 volume [S33] + and destroy the keyslot, which is the cryptographic-erase model NIST SP 800-88 Rev. 2 centres"* [S28]. +3. **`fscrypt`** (in-kernel; ext4 / F2FS / UBIFS / CephFS) encrypts file contents, filenames and symlink + targets, and research/08 is equally clear about its limits: it *"does not encrypt filesystem + metadata"* — sizes, permissions, timestamps, xattrs — hole locations are unprotected, and + `FS_IOC_REMOVE_ENCRYPTION_KEY` is not a wipe: *"Per-file keys for in-use files will *not* be removed + or wiped"*, with decrypted cache content *"freed but not wiped"*. Use v2 policies; v1 has *"no + verification that the provided master key is correct"* [S11]. research/08 states the residual bluntly: + *"**`fscrypt` protects less than people assume.** ... It is not an answer to 'the buffer contained live + exploit details and the host was compromised while running'."* +4. **`age`** (BSD-3-Clause [S32]) is fine for an encrypted archived copy but, per research/08, a poor fit + for a concurrently mutated buffer. + +None of options 1–4 is implemented by this repository. They are operator-side controls, and naming them +here is not a claim that Anvil configures them. + +--- + +## 5. What `secure_delete` would and would not buy, and that it is not enabled + +research/08 §F, quoting the SQLite pragma docs: *"Applications that wish to avoid leaving forensic traces +after content is deleted or updated should enable the secure_delete pragma prior to performing the delete +or update, or else run VACUUM after the delete or update."* [S15] + +**Anvil does neither.** `internal/store/ddl.go`'s `ConnectionPragmas()` sets `journal_mode = WAL`, +`foreign_keys`, `busy_timeout`, `synchronous = NORMAL` and `wal_autocheckpoint`. `secure_delete` is not +among them, and no code in `internal/store` runs `VACUUM` after a delete. So when the reaper `NULL`s +`audit_record.payload`, the old bytes remain in the database file's freed pages. + +Enabling it would not close the gap either. research/08 records both limits: + +* `secure_delete=FAST` *"has the effect of purging all old content from b-tree pages, but leaving + forensic traces on freelist pages"*, and FTS3/FTS5 virtual tables *"might leave forensic traces in + their shadow tables even if the secure_delete pragma is enabled."* [S15] `internal/store/schema.sql` + creates `advisory_fts` as an FTS5 virtual table, so that second limit applies to this schema directly. +* The layering, quoted from research/08: *"zeroing bytes *inside the database file* still does not erase + the old physical flash blocks [S13] — it defends against someone reading the file, not someone reading + the raw device."* + +**Related, and separate:** what is allowed *into* durable columns in the first place is governed by +`schema.sql`'s `trg_occurrence_durable_text_cap_*` triggers and by the masking step `R.8`. Restricting +what is written is a different question from erasing what was written, and this document only answers the +second. + +--- + +## 6. Copies this repository creates on purpose + +research/08's Risks section names duplication, not deletion, as the real exposure: *"Two copies is the +real security risk, and the TTL hides it."* Two duplications are created by committed code and are listed +here so no one discovers them later: + +* **Pre-migration snapshots.** `internal/store/migrate.go` writes a full `VACUUM INTO 'anvil-pre-v{N}.db'` + copy of the database before applying a migration to a populated database, and refuses to migrate + without one — research/07 §7 makes that snapshot the entire substitute for down migrations. Nothing in + this repository deletes those snapshots. Each is a complete second copy of the payload and rows, + persisting until an operator removes it — and removing it is subject to everything in section 2. +* **The tmpfs packet.** It is a second materialisation of content that is already in the store. It is + regenerable and short-lived by design, but while it exists it is a second copy, and it is only + RAM-resident if the operator actually mounted tmpfs, with `noswap` where available (section 4, item 1). + +--- + +## 7. Two things not to do + +* **Do not `shred` at expiry** — section 2. It would be theatre on every realistic deployment target and + would create exactly the false impression this document exists to prevent. +* **Do not invoke `systemd-tmpfiles --purge` from Anvil's tooling, and do not point a `tmpfiles.d` rule at + a parent directory.** research/08 Risks: *"In systemd 256, `systemd-tmpfiles --purge` invoked without a + config file deleted users' `/home`; a user reported that 'a good portion of my home directory got + deleted', and 256.1 changed `--purge` to require an explicit config file"* [S30, graded C — news/forum + — with the 256.1 fix corroborating the incident]. research/08's implication, adopted verbatim as + policy: *"if Anvil ships a `tmpfiles.d` snippet, scope it to a dedicated directory, never to a parent, + and never invoke `--purge` from Anvil's own tooling."* Any such snippet is a backstop behind Anvil's own + reaper, pinned to the birth-time age-by prefix (`b:8h`), because the default age-by set `abcmABM` + includes atime and a mere reader would otherwise reset the clock [S3]. + +--- + +## 8. Claim-to-source trace + +Every row is a claim made above and the research/08 location that already sourced it. Rows marked +*repo fact* are statements about this codebase, verifiable by reading the cited file; the security +consequence attached to each is itself a research/08 claim, listed alongside. + +| § | Claim | Traces to | +|---|---|---| +| 0, 2 | `shred` cannot securely delete on Btrfs/ZFS/XFS/NTFS/ext3-4 `data=journal`/compressed/RAID/snapshotting FS/NFSv3 | research/08 §F [S13]; Risks "`shred` is folklore for this use case" | +| 0, 2 | On SSDs, *"'overwritten' data blocks are still present in the underlying device"* | research/08 §F and Risks [S13] | +| 0, 2 | *"shred assumes the file system and hardware overwrite data in place"* | research/08 §F [S12] | +| 2 | A design doc claiming "we shred the buffer at expiry" is wrong | research/08 Risks, verbatim | +| 2 | Flash's meaningful erase primitive is key destruction, not overwriting; NIST SP 800-88 Rev. 2 final 2025-09-26 | research/08 §F [S28]; Gaps ("full PDF not fetched") | +| 0, 3 | The 8-hour window is not a confidentiality control | `plan/00-SPINE.md` S1 item 5; research/08 §A | +| 3 | The TTL creates an *impression* of short exposure while identical content persists indefinitely | research/08 Risks, "Two copies is the real security risk" | +| 1, 3 | Two clocks: 15–30 min lease vs 8 h eligibility, never conflated | research/08 §4 ("Claim lease" / "Buffer eligibility TTL"); *repo fact*: `internal/store/schema.sql` `handoff` header | +| 1 | At expiry the row is not deleted; the finding is re-presented; *"Missing the window costs latency, not the finding"* | research/08 §4.3 | +| 1 | tmpfs packet dropped, `audit_record.payload` `NULL`ed, rows retained | *repo fact*: `internal/store/schema.sql` (`payload`, `purged_at`, `handoff` comment) | +| 0, 4 | The control for confidentiality-at-rest is a per-scan LUKS2 volume (keyslot destruction) or `fscrypt`, not application deletion | research/08 §F priority list [S33]; §3 "What would flip the decision" [S13][S15][S33][S28] | +| 4 | tmpfs contents *"are discarded (lost)"* on unmount; `noswap` needs Linux ≥ 6.4; disable core dumps | research/08 §F item 1, §B [S9] | +| 4 | `/run`-is-tmpfs and the `RuntimeDirectoryMode=` default were **not** verified; set `0700` explicitly | research/08 Gaps [S29] | +| 4 | `fscrypt` does not encrypt metadata; in-use per-file keys not wiped; cached plaintext *"freed but not wiped"*; use v2 | research/08 §F item 3 and Risks [S11] | +| 4 | `age` is a poor fit for a concurrently mutated buffer | research/08 §F item 4 [S32] | +| 5 | `secure_delete` must be set *before* the delete, or `VACUUM` after | research/08 §F [S15] | +| 5 | `secure_delete` is not enabled and no post-delete `VACUUM` runs | *repo fact*: `internal/store/ddl.go` `ConnectionPragmas()`; consequence from [S15] above | +| 5 | FTS5 shadow tables may retain forensic traces even with `secure_delete`; `FAST` leaves freelist traces | research/08 §F [S15]; *repo fact*: `advisory_fts` in `internal/store/schema.sql` | +| 5 | Zeroing inside the DB file does not erase the physical flash blocks | research/08 §F, verbatim [S13] | +| 6 | Duplication, not deletion, is the real exposure | research/08 Risks, "Two copies is the real security risk" | +| 6 | A full `VACUUM INTO 'anvil-pre-v{N}.db'` copy is written before migrating a populated database, and nothing here deletes it | *repo fact*: `internal/store/migrate.go`; consequence from the row above | +| 7 | `systemd-tmpfiles --purge` deleted a user's `/home` in systemd 256; scope snippets to a dedicated directory, never a parent | research/08 Risks [S30, credibility C] | +| 7 | Default age-by set `abcmABM` includes atime, so a reader resets the clock; pin `b:` | research/08 §B and Risks [S3] | + +Source IDs `[S3] [S9] [S11] [S12] [S13] [S15] [S28] [S29] [S30] [S32] [S33]` refer to the Sources table of +`research/08-buffer-and-handoff.md`. Nothing above is sourced anywhere else, and nothing above is new. diff --git a/internal/record/contract.go b/internal/record/contract.go index cb3405b..c2d8dc8 100644 --- a/internal/record/contract.go +++ b/internal/record/contract.go @@ -319,14 +319,24 @@ func ValidateHalfStatus(v string) error { // by two critics. Area 40 declared seven values and area D declared five with // ZERO literal overlap — D could not have written a single row into 40's NOT // NULL column. The frozen set is the union of both plus D's `partial` -// (renamed `completed_partial`), which is nine values. D.26 emits these. +// (renamed `completed_partial`), which was nine values. D.26 emits these. +// +// AMENDED — a TENTH value, `completed_failed`, was added to +// plan/IMPLEMENTATION-PLAN.md §6 after the R.10 critic (CRITIQUE-02 F8/rule 8) +// showed the nine-value set had no image for "the DAST half itself broke". +// DeriveDastStatus was mapping a HalfStatusFailed half against a target that +// booted cleanly onto `completed_partial`, which is the same category error +// plan/00-SPINE.md S6 forbids one level down: a half that CRASHED is not a +// half that COVERED PART of the surface, and collapsing them makes +// `dast_coverage` uninterpretable — the reader cannot tell a 31-of-50 scan +// from a scan that died at endpoint 1. See DastStatusCompletedFailed. // // This value is DERIVED from the DAST half's HalfStatus and from // TargetProvenance (the boot/reachability outcome), never from // TargetProvisioning (which provisioning path was used). type DastStatus string -// The nine legal anvil/dastStatus literals. +// The ten legal anvil/dastStatus literals. // // WHY DastStatusSkippedNoManifest IS DISTINCT FROM DastStatusNotRun — do not // merge them: @@ -363,6 +373,23 @@ const ( // coverage detail lives in DastCoverage, which is what makes this value // interpretable rather than merely worrying. DastStatusCompletedPartial DastStatus = "completed_partial" + // DastStatusCompletedFailed: the target booted cleanly and the DAST half + // then FAILED mid-scan — the scanner crashed, lost its connection, or was + // killed. Derived from HalfStatusFailed against + // TargetProvenanceBootedClean, and from nothing else: a half that never + // had a target reports target_boot_failed, target_unreachable or + // skipped_no_manifest, all of which outrank this value. + // + // It is DISTINCT from completed_partial on purpose. Both mean "less than + // the whole surface was probed", but only completed_partial means the + // coverage numbers in DastCoverage describe a scan that ran to its own + // conclusion. Reporting a crash as completed_partial invites a consumer to + // read "31 of 50 endpoints" as a deliberate scope, when in fact the run + // died and the denominator is meaningless. + // + // Like every value except completed_clean, MeansDynamicallyScannedClean is + // false for it. + DastStatusCompletedFailed DastStatus = "completed_failed" // DastStatusTargetBootFailed: the target never booted, so nothing was // scanned. Derived from TargetProvenanceBootFailed or // TargetProvenanceBuildFailed. @@ -379,12 +406,13 @@ func DastStatusValues() []DastStatus { return []DastStatus{ DastStatusNotRun, DastStatusSkippedNoManifest, DastStatusRunning, DastStatusCompletedClean, DastStatusCompletedFindings, - DastStatusCompletedPartial, DastStatusTargetBootFailed, - DastStatusTargetUnreachable, DastStatusTimedOut, + DastStatusCompletedPartial, DastStatusCompletedFailed, + DastStatusTargetBootFailed, DastStatusTargetUnreachable, + DastStatusTimedOut, } } -// Valid reports whether s is one of the nine legal anvil/dastStatus literals. +// Valid reports whether s is one of the ten legal anvil/dastStatus literals. func (s DastStatus) Valid() bool { return inEnum(s, DastStatusValues()) } // ValidateDastStatus reports whether v is a legal anvil/dastStatus literal. diff --git a/internal/record/contract_test.go b/internal/record/contract_test.go index ef40591..8b94b58 100644 --- a/internal/record/contract_test.go +++ b/internal/record/contract_test.go @@ -24,9 +24,16 @@ var frozenEnums = map[string][]string{ "anvil/status": { "running", "sealed", "failed", "timed_out", "skipped", }, + // TEN values since the section 6 amendment: `completed_failed` was added + // between completed_partial and target_boot_failed because the nine-value + // set had no image for "the DAST half itself broke", and DeriveDastStatus + // was folding that case into completed_partial -- which makes dast_coverage + // uninterpretable for the same reason S6 requires a failed target to be + // distinguishable from one scanned clean. "anvil/dastStatus": { "not_run", "skipped_no_manifest", "running", "completed_clean", "completed_findings", - "completed_partial", "target_boot_failed", "target_unreachable", "timed_out", + "completed_partial", "completed_failed", "target_boot_failed", "target_unreachable", + "timed_out", }, "anvil/target.provenance": { "booted_clean", "boot_failed", "build_failed", "no_target_declared", diff --git a/internal/record/critique02_regression_test.go b/internal/record/critique02_regression_test.go new file mode 100644 index 0000000..afb567a --- /dev/null +++ b/internal/record/critique02_regression_test.go @@ -0,0 +1,580 @@ +// Regression tests for the defects CRITIQUE-02 found and the fix round closed. +// +// Each test here reproduces one ORIGINAL defect. They were written by the +// critic and the re-verifier as probes -- to prove a defect existed, and then +// to prove it was actually gone rather than merely claimed gone. They are kept +// permanently, and deliberately named after the finding they pin, because a +// fixed defect with no test is a defect waiting to return. +// +// Two of them earn their place especially: +// - the masking probes build a MINIMAL record and assert the planted secret +// appears exactly once before masking, so they cannot pass by propagation +// from a header. That false-confidence pattern was itself a finding. +// - the lease probes drive concurrent goroutines at the granting statement +// rather than asserting on a single-threaded happy path. + +package record + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// Independent re-verification probes for CRITIQUE-02 B3 / M1 / M3, plus the +// completed_failed and DeriveDastStatus-totality claims. Written from the +// critique text, not from the shipped tests. +// --------------------------------------------------------------------------- + +func probeMarshal(t *testing.T, l *SARIFLog) string { + t.Helper() + b, err := json.Marshal(l) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(b) +} + +// probeMinimalLog builds a record with NOTHING planted anywhere. Each probe +// then plants exactly one secret in exactly one field, so nothing can pass by +// propagation from a header the masker was already looking at. That +// propagation-false-confidence pattern is CRITIQUE-02 F11. +func probeMinimalLog() *SARIFLog { + return &SARIFLog{ + Version: "2.1.0", + Properties: AuditProperties{ + Target: Target{ + RepoURL: "https://github.invalid/org/repo.git", + }, + }, + Runs: []Run{{ + Properties: RunProperties{Half: HalfDast, Status: HalfStatusSealed}, + Results: []Result{{ + WebRequest: &WebRequest{ + Target: "https://app.invalid/v1/orders", + Method: "POST", + Headers: map[string]string{"Content-Length": "42"}, + }, + WebResponse: &WebResponse{ + StatusCode: 200, + Headers: map[string]string{"Content-Type": "application/json"}, + }, + }}, + }}, + } +} + +// ProbeB3ReproCurlOnly: the secret exists ONLY inside anvil/repro.curl. No +// header, no parameter, no URL anywhere in the record carries it, so pass 2 +// propagation cannot rescue the assertion. +func TestProbeB3ReproCurlOnlySecret(t *testing.T) { + const secret = "PROBE-CURL-ONLY-aaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + l := probeMinimalLog() + l.Runs[0].Results[0].Properties.Repro = &Repro{ + Curl: "curl -X POST -H 'Authorization: Bearer " + secret + "' https://app.invalid/v1/orders", + } + + before := probeMarshal(t, l) + if strings.Count(before, secret) != 1 { + t.Fatalf("fixture guard: secret occurs %d times before masking, want exactly 1 (in repro.curl only)", + strings.Count(before, secret)) + } + + if err := MaskRecord(l); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if after := probeMarshal(t, l); strings.Contains(after, secret) { + t.Errorf("B3(i) REPRODUCED: anvil/repro.curl still carries %q after MaskRecord\n curl = %s", + secret, l.Runs[0].Results[0].Properties.Repro.Curl) + } + t.Logf("masked curl = %s", l.Runs[0].Results[0].Properties.Repro.Curl) +} + +// ProbeB3 curl-only variants: other option shapes a real repro command uses. +func TestProbeB3ReproCurlOtherOptionShapes(t *testing.T) { + cases := map[string]string{ + "-H long form --header": "curl --header 'X-Api-Key: %s' https://app.invalid/x", + "-b cookie": "curl -b 'session=%s' https://app.invalid/x", + "-u basic auth": "curl -u 'admin:%s' https://app.invalid/x", + "-d data": "curl -d 'password=%s' https://app.invalid/x", + "url query in argument": "curl 'https://app.invalid/x?api_key=%s'", + "--url flag": "curl --url 'https://app.invalid/x?api_key=%s'", + "userinfo in bare url": "curl https://user:%s@app.invalid/x", + } + for name, tmpl := range cases { + t.Run(name, func(t *testing.T) { + secret := "PROBE-SHAPE-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + l := probeMinimalLog() + l.Runs[0].Results[0].Properties.Repro = &Repro{Curl: fmt.Sprintf(tmpl, secret)} + if err := MaskRecord(l); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + got := l.Runs[0].Results[0].Properties.Repro.Curl + if strings.Contains(got, secret) { + t.Errorf("secret survives in repro.curl: %s", got) + } + if err := AssertMasked(l); err != nil { + t.Errorf("AssertMasked on the MASKED record: %v", err) + } + }) + } +} + +// ProbeB3 second named field: the credential sits ONLY in anvil/target.repoUrl +// userinfo, the standard GitHub Actions checkout URL shape. +func TestProbeB3TargetRepoURLOnlySecret(t *testing.T) { + const secret = "PROBE-CHECKOUT-cccccccccccccccccccccccccccc" + + for _, tc := range []struct { + name string + set func(l *SARIFLog) + get func(l *SARIFLog) string + }{ + {"anvil/target.repoUrl", func(l *SARIFLog) { + l.Properties.Target.RepoURL = "https://x-access-token:" + secret + "@github.invalid/org/repo.git" + }, func(l *SARIFLog) string { return l.Properties.Target.RepoURL }}, + {"anvil/target.runtimeBaseUrl", func(l *SARIFLog) { + l.Properties.Target.RuntimeBaseURL = "https://scanner:" + secret + "@staging.invalid" + }, func(l *SARIFLog) string { return l.Properties.Target.RuntimeBaseURL }}, + {"anvil/runtimeTarget.baseUrl", func(l *SARIFLog) { + l.Runs[0].Properties.RuntimeTarget = &RuntimeTarget{ + BaseURL: "https://scanner:" + secret + "@staging.invalid", + } + }, func(l *SARIFLog) string { return l.Runs[0].Properties.RuntimeTarget.BaseURL }}, + {"anvil/runtimeTarget.scope[i]", func(l *SARIFLog) { + l.Runs[0].Properties.RuntimeTarget = &RuntimeTarget{ + BaseURL: "https://staging.invalid", + Scope: []string{"https://staging.invalid/a?token=" + secret}, + } + }, func(l *SARIFLog) string { return l.Runs[0].Properties.RuntimeTarget.Scope[0] }}, + } { + t.Run(tc.name, func(t *testing.T) { + l := probeMinimalLog() + tc.set(l) + if strings.Count(probeMarshal(t, l), secret) != 1 { + t.Fatalf("fixture guard: secret must occur exactly once before masking") + } + if err := MaskRecord(l); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if strings.Contains(probeMarshal(t, l), secret) { + t.Errorf("B3(ii) REPRODUCED: %s still carries the credential: %s", tc.name, tc.get(l)) + } + }) + } +} + +// ProbeM1: AssertMasked must reject an UNMASKED record at every site Mask +// covers. This is run by planting one credential per site into an otherwise +// clean record and demanding a non-nil error each time. +func TestProbeM1AssertMaskedRejectsEverySite(t *testing.T) { + const secret = "PROBE-SITE-dddddddddddddddddddddddddddddddd" + + sites := []struct { + name string + set func(l *SARIFLog) + }{ + {"webRequest.headers", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Headers["Authorization"] = "Bearer " + secret + }}, + {"webRequest.parameters", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Parameters = map[string]string{"api_key": secret} + }}, + {"webRequest.target (query)", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Target = "https://app.invalid/v1?api_key=" + secret + }}, + {"webRequest.target (userinfo)", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Target = "https://u:" + secret + "@app.invalid/v1" + }}, + {"webRequest.target (fragment)", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Target = "https://app.invalid/v1#access_token=" + secret + }}, + {"webResponse.headers", func(l *SARIFLog) { + l.Runs[0].Results[0].WebResponse.Headers["Set-Cookie"] = "session=" + secret + }}, + {"anvil/target.repoUrl", func(l *SARIFLog) { + l.Properties.Target.RepoURL = "https://x-access-token:" + secret + "@github.invalid/o/r.git" + }}, + {"anvil/target.runtimeBaseUrl", func(l *SARIFLog) { + l.Properties.Target.RuntimeBaseURL = "https://s:" + secret + "@staging.invalid" + }}, + {"anvil/runtimeTarget.baseUrl", func(l *SARIFLog) { + l.Runs[0].Properties.RuntimeTarget = &RuntimeTarget{BaseURL: "https://s:" + secret + "@staging.invalid"} + }}, + {"anvil/repro.curl", func(l *SARIFLog) { + l.Runs[0].Results[0].Properties.Repro = &Repro{ + Curl: "curl -H 'Authorization: Bearer " + secret + "' https://app.invalid/x", + } + }}, + {"webRequest.body over cap", func(l *SARIFLog) { + l.Runs[0].Results[0].WebRequest.Body = &ArtifactContent{ + Text: strings.Repeat("x", MaxInlineRequestBodyBytes+1), + } + }}, + {"webResponse.body over cap", func(l *SARIFLog) { + l.Runs[0].Results[0].WebResponse.Body = &ArtifactContent{ + Text: strings.Repeat("x", MaxInlineResponseBodyBytes+1), + } + }}, + } + + for _, s := range sites { + t.Run(s.name, func(t *testing.T) { + l := probeMinimalLog() + s.set(l) + if err := AssertMasked(l); err == nil { + t.Errorf("M1 REPRODUCED: AssertMasked returned nil on an UNMASKED %s", s.name) + } + // And after Mask it must accept. + if err := MaskRecord(l); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if err := AssertMasked(l); err != nil { + t.Errorf("AssertMasked rejected a record Mask just produced: %v", err) + } + }) + } +} + +// ProbeM1b: structural equality of the two walks. Count the sites each walk +// visits per kind; they must be identical, and AssertMasked must consume every +// kind the surface declares (no nil callback silently skipping a kind). +func TestProbeM1WalkCoverageIsIdentical(t *testing.T) { + l := probeMinimalLog() + l.Properties.Target.RuntimeBaseURL = "https://staging.invalid" + l.Runs[0].Properties.RuntimeTarget = &RuntimeTarget{ + BaseURL: "https://staging.invalid", + Scope: []string{"https://staging.invalid/a"}, + Excluded: []string{"https://staging.invalid/b"}, + } + l.Runs[0].Results[0].Properties.Repro = &Repro{Curl: "curl https://app.invalid/x"} + l.Runs[0].Results[0].WebRequest.Parameters = map[string]string{"q": "1"} + l.Runs[0].Results[0].WebRequest.Body = &ArtifactContent{Text: "{}"} + l.Runs[0].Results[0].WebResponse.Body = &ArtifactContent{Text: "{}"} + + counts := map[string]int{} + walkMaskSurface(l, surface{ + Headers: func(string, map[string]string) { counts["headers"]++ }, + Parameters: func(string, map[string]string) { counts["parameters"]++ }, + URL: func(string, *string) { counts["url"]++ }, + CommandLine: func(string, *string) { counts["commandline"]++ }, + Body: func(string, *ArtifactContent, int) { counts["body"]++ }, + }) + t.Logf("mask surface sites: %v", counts) + for _, k := range []string{"headers", "parameters", "url", "commandline", "body"} { + if counts[k] == 0 { + t.Errorf("surface kind %q visits no site in this fixture; probe is not exercising it", k) + } + } + if counts["url"] < 6 { + t.Errorf("url sites = %d; want at least 6 (repoUrl, runtimeBaseUrl, rt.baseUrl, scope, excluded, webRequest.target)", + counts["url"]) + } +} + +// --------------------------------------------------------------------------- +// M3 — Sealer.Inspect must honour expiry. +// --------------------------------------------------------------------------- + +func TestProbeM3InspectHonoursExpiry(t *testing.T) { + now := time.Date(2026, 8, 8, 9, 0, 0, 0, time.UTC) + clock := func() time.Time { return now } + s := NewSealer() + s.SetClock(func() time.Time { return clock() }) + + if _, err := s.BeginAudit(AuditConfig{ + AuditID: "probe-a1", StartedAt: now, ClaimTimeoutSeconds: 3600, DastEnabled: false, + }); err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := s.SealHalf("probe-a1", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + + // Before expiry both agree. + seal, ok := s.Inspect("probe-a1") + if !ok { + t.Fatal("Inspect: audit missing") + } + if !seal.Sast.Readable() { + t.Fatalf("pre-expiry Inspect says SAST unreadable") + } + if _, err := s.ReadHalf("probe-a1", HalfSast); err != nil { + t.Fatalf("pre-expiry ReadHalf: %v", err) + } + + // Cross the deadline and expire. + now = now.Add(2 * time.Hour) + expired, err := s.ExpireIfDue("probe-a1") + if err != nil { + t.Fatalf("ExpireIfDue: %v", err) + } + if !expired { + t.Fatal("ExpireIfDue did not expire a past-deadline audit") + } + + _, readErr := s.ReadHalf("probe-a1", HalfSast) + if readErr == nil { + t.Fatal("ReadHalf accepted an expired audit; the gate itself is gone") + } + seal, ok = s.Inspect("probe-a1") + if !ok { + t.Fatal("Inspect: audit missing after expiry") + } + if seal.Sast.Readable() { + t.Errorf("M3 REPRODUCED: Inspect reports Sast.Readable()=true on an expired audit that ReadHalf refuses with: %v", readErr) + } + if seal.Dast.Readable() { + t.Errorf("M3 REPRODUCED (dast half): Inspect reports Dast.Readable()=true on an expired audit") + } + t.Logf("expired: Inspect Sast=%+v Readable=%v; ReadHalf err=%v", seal.Sast, seal.Sast.Readable(), readErr) +} + +// M3 must not overshoot: a CONSUMED audit is still readable (S1 re-entrancy). +func TestProbeM3InspectStillReadableWhenConsumed(t *testing.T) { + now := time.Date(2026, 8, 8, 9, 0, 0, 0, time.UTC) + s := NewSealer() + s.SetClock(func() time.Time { return now }) + if _, err := s.BeginAudit(AuditConfig{ + AuditID: "probe-a2", StartedAt: now, ClaimTimeoutSeconds: 3600, DastEnabled: false, + }); err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := s.SealHalf("probe-a2", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if err := s.Consume("probe-a2"); err != nil { + t.Fatalf("Consume: %v", err) + } + seal, _ := s.Inspect("probe-a2") + if !seal.Sast.Readable() { + t.Errorf("a consumed audit reports Sast.Readable()=false; S1 requires a RE-ENTRANT consumer") + } + if _, err := s.ReadHalf("probe-a2", HalfSast); err != nil { + t.Errorf("ReadHalf refused a consumed audit: %v", err) + } +} + +// Inspect and ReadHalf must never disagree over any reachable (state, status). +func TestProbeM3InspectNeverDisagreesWithReadHalf(t *testing.T) { + type combo struct { + state State + sast HalfStatus + reason string + } + now := time.Date(2026, 8, 8, 9, 0, 0, 0, time.UTC) + for _, st := range HalfStatusValues() { + if !IsTerminalHalfStatus(st) { + continue + } + for _, expire := range []bool{false, true} { + name := fmt.Sprintf("sast=%s expired=%v", st, expire) + t.Run(name, func(t *testing.T) { + cur := now + s := NewSealer() + s.SetClock(func() time.Time { return cur }) + id := "probe-" + name + if _, err := s.BeginAudit(AuditConfig{ + AuditID: id, StartedAt: cur, ClaimTimeoutSeconds: 3600, DastEnabled: false, + }); err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := s.SealHalf(id, HalfSast, st); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if expire { + cur = cur.Add(2 * time.Hour) + if _, err := s.ExpireIfDue(id); err != nil { + t.Fatalf("ExpireIfDue: %v", err) + } + } + seal, _ := s.Inspect(id) + _, err := s.ReadHalf(id, HalfSast) + gateOpen := err == nil + if seal.Sast.Readable() != gateOpen { + t.Errorf("DISAGREEMENT: Inspect.Readable()=%v, ReadHalf ok=%v (err=%v)", + seal.Sast.Readable(), gateOpen, err) + } + }) + } + } + _ = combo{} +} + +// --------------------------------------------------------------------------- +// completed_failed and DeriveDastStatus totality. +// --------------------------------------------------------------------------- + +func TestProbeCompletedFailedIsInTheEnum(t *testing.T) { + found := false + for _, v := range DastStatusValues() { + if v == DastStatusCompletedFailed { + found = true + } + } + if !found { + t.Fatalf("completed_failed missing from DastStatusValues(): %v", DastStatusValues()) + } + if len(DastStatusValues()) != 10 { + t.Errorf("DastStatusValues() has %d entries, want 10", len(DastStatusValues())) + } + if err := ValidateDastStatus("completed_failed"); err != nil { + t.Errorf("ValidateDastStatus(completed_failed): %v", err) + } + if DastStatusCompletedFailed.MeansDynamicallyScannedClean() { + t.Errorf("completed_failed reports MeansDynamicallyScannedClean") + } + // Duplicate / ordering sanity. + seen := map[DastStatus]int{} + for _, v := range DastStatusValues() { + seen[v]++ + if seen[v] > 1 { + t.Errorf("duplicate value %q in DastStatusValues()", v) + } + } +} + +// TotalityProbe: enumerate EVERY (tier, provenance, half status, partial, +// count) tuple and demand exactly one legal, non-empty, deterministic image. +func TestProbeDeriveDastStatusIsTotalIndependently(t *testing.T) { + images := map[string]DastStatus{} + seen := map[DastStatus]bool{} + pairs := 0 + for _, tier := range []bool{false, true} { + for _, prov := range TargetProvenanceValues() { + for _, st := range HalfStatusValues() { + for _, partial := range []bool{false, true} { + for _, count := range []int{0, 1, 3} { + o := DastOutcome{TierInstalled: tier, Provenance: prov, + PartialCoverage: partial, FindingCount: count} + got, err := DeriveDastStatus(st, o) + pairs++ + if err != nil { + t.Errorf("NOT TOTAL: (tier=%v prov=%s status=%s partial=%v n=%d) errored: %v", + tier, prov, st, partial, count, err) + continue + } + if got == "" { + t.Errorf("NOT TOTAL: (tier=%v prov=%s status=%s partial=%v n=%d) -> empty string", + tier, prov, st, partial, count) + continue + } + if !got.Valid() { + t.Errorf("ILLEGAL IMAGE: (tier=%v prov=%s status=%s) -> %q not in the frozen enum", + tier, prov, st, got) + } + // Determinism / single-valuedness. + key := fmt.Sprintf("%v|%s|%s|%v|%d", tier, prov, st, partial, count) + if prev, ok := images[key]; ok && prev != got { + t.Errorf("NOT A FUNCTION: %s -> %q then %q", key, prev, got) + } + images[key] = got + again, _ := DeriveDastStatus(st, o) + if again != got { + t.Errorf("NON-DETERMINISTIC: %s -> %q then %q", key, got, again) + } + seen[got] = true + } + } + } + } + } + t.Logf("enumerated %d tuples, %d distinct images", pairs, len(seen)) + for _, v := range DastStatusValues() { + if !seen[v] { + t.Errorf("UNREACHABLE: no tuple derives %q", v) + } + } + // The specific fold the amendment claims to have removed. + got, err := DeriveDastStatus(HalfStatusFailed, DastOutcome{ + TierInstalled: true, Provenance: TargetProvenanceBootedClean}) + if err != nil { + t.Fatalf("DeriveDastStatus(failed, booted_clean): %v", err) + } + if got != DastStatusCompletedFailed { + t.Errorf("(failed, booted_clean) = %q, want completed_failed", got) + } + // completed_clean must remain reachable ONLY from a sealed half against a + // cleanly-booted target with zero findings and no partial coverage. + for key, v := range images { + if v != DastStatusCompletedClean { + continue + } + if !strings.HasPrefix(key, "true|booted_clean|sealed|false|0") { + t.Errorf("completed_clean reachable from %s", key) + } + } +} + +// The frozen-vocabulary table in contract_test.go must agree with the Go enum. +// Re-derived here rather than trusting that file. +func TestProbeFrozenTableAgreesWithDastStatusValues(t *testing.T) { + want := []string{} + for _, v := range DastStatusValues() { + want = append(want, string(v)) + } + got := frozenEnums["anvil/dastStatus"] + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("frozen table mismatch\n table: %v\n enum: %v", got, want) + } +} + +// The frozen vocabulary exists in four places. All four must agree, or the +// amendment is only half landed. +func TestProbePublishedSchemaAndDocAgreeOnDastStatus(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "schemas", "anvil-record-v1.schema.json")) + if err != nil { + t.Fatalf("read published JSON Schema: %v", err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("parse published JSON Schema: %v", err) + } + if !strings.Contains(string(raw), `"completed_failed"`) { + t.Errorf("schemas/anvil-record-v1.schema.json does NOT list \"completed_failed\" in the "+ + "anvil/dastStatus enum, but DastStatusValues() does (%v). A record carrying the value "+ + "internal/record derives for a crashed DAST half fails wire validation.", + DastStatusValues()) + } + md, err := os.ReadFile("CONTRACT.md") + if err != nil { + t.Fatalf("read CONTRACT.md: %v", err) + } + if !strings.Contains(string(md), "completed_failed") { + t.Errorf("internal/record/CONTRACT.md section 1.3 still documents the nine-value "+ + "anvil/dastStatus set; contract.go declares ten (%v)", DastStatusValues()) + } + _ = doc +} + +// F11's false-confidence pattern, checked against the SHIPPED fixture: the +// secrets that exist to prove the newly-covered sites are masked must occur in +// exactly ONE place in the record, or the assertion passes by propagation from +// a header pass 1 was already reading. +func TestProbeShippedFixtureSecretsAreSingleSited(t *testing.T) { + l := dastFixture() + blob := probeMarshal(t, l) + for name, secret := range map[string]string{ + "plantedCurlOnly (repro.curl)": plantedCurlOnly, + "plantedCheckoutToken (target.repoUrl)": plantedCheckoutToken, + "plantedRuntimeBasicAuth (target.runtimeBaseUrl)": plantedRuntimeBasicAuth, + } { + if n := strings.Count(blob, secret); n != 1 { + t.Errorf("F11 PATTERN: %s occurs %d times in the unmasked fixture; it must occur "+ + "exactly once or the absence assertion proves nothing about the new site", name, n) + } + } + // And each must be long enough that propagation could carry it, so a + // single-sited value is genuinely testing the structural pass. + for _, s := range []string{plantedCurlOnly, plantedCheckoutToken, plantedRuntimeBasicAuth} { + if len(s) < MinPropagatedSecretLen { + t.Errorf("planted value %q is shorter than MinPropagatedSecretLen", s) + } + } +} diff --git a/internal/record/mask.go b/internal/record/mask.go new file mode 100644 index 0000000..cf7cb81 --- /dev/null +++ b/internal/record/mask.go @@ -0,0 +1,1544 @@ +package record + +// mask.go — R.8, the secrets-masking pipeline. +// +// =========================================================================== +// WHAT THIS FILE IS FOR, AND WHY IT RUNS WHERE IT RUNS +// =========================================================================== +// +// plan/00-SPINE.md S7 names one field the highest-risk in the whole system: +// +// "Prompt injection: sanitize at ingest, not at prompt time. The DAST +// response body is the highest-risk field — up to 32 KB of +// attacker-controlled bytes fed to a repo-credentialed agent." +// +// research/18-unified-audit-record.md Risk #10 states the secrets half of the +// same problem: +// +// "Bodies leak secrets. webRequest.headers will contain session cookies and +// bearer tokens by default. ZAP masks Authorization with asterisks; Anvil +// must do the same *before* the record reaches the buffer, the DB, or the +// coding agent's context — an 8-hour TTL is not a security control for a +// token that is still valid." +// +// Two consequences, and both are load-bearing: +// +// 1. MASKING IS AN INGEST STEP, NOT A RENDER STEP. It must be the last step +// of record assembly and it must run BEFORE either sink — the store and +// the model context. Masking on the way out to a prompt leaves the live +// token sitting in SQLite, where the 8-hour claim timeout is the only +// thing standing between it and an attacker, and a claim timeout is a +// scheduling policy, not a confidentiality control (see SECRETS.md). +// +// 2. IT FAILS CLOSED. Every decision below resolves ambiguity by redacting. +// A masker that fails open is worse than no masker at all, because it +// manufactures confidence: a reviewer who sees `***REDACTED***` in three +// places assumes the fourth header was checked and found harmless, when +// in fact it was unparseable and waved through. +// +// =========================================================================== +// THE FOUR THINGS THIS FILE DOES +// =========================================================================== +// +// 1. Structural header masking. A header whose NAME is on the denylist has +// its value replaced with RedactedPlaceholder. A header whose name or +// value has an unexpected SHAPE is redacted too, and recorded as an +// anomaly (see "Fail-closed rules"). +// +// 2. Structural parameter masking. webRequest.parameters entries with a +// secret-shaped NAME, plus the query string and fragment of EVERY URL the +// record carries, plus URL userinfo passwords, plus the option arguments +// of `anvil/repro.curl`. Values are redacted; names are preserved, because +// the parameter name is evidence (it is the injection point) while the +// value of `api_key` never is. +// +// 3. Value propagation. Every value redacted in steps 1 and 2 is collected, +// decomposed (a Cookie header into its name=value pairs, an +// Authorization header into the credential after the scheme), and then +// removed from EVERY other string in the record by exact substring +// replacement. This is what makes the substring-absence assertion hold: +// a session cookie echoed back inside a 30 KB HTML error page is not +// caught by any header rule, but it is caught here. +// +// 4. Body caps. webRequest.body at MaxInlineRequestBodyBytes and +// webResponse.body at MaxInlineResponseBodyBytes — the same 8 KB / 32 KB +// thresholds OWASP ZAP's SARIF reporter uses (research/18 [S8]). The +// remainder SPILLS to a content-addressed Tier-2 blob reference; it is +// never silently dropped. +// +// Ordering between them is not arbitrary. Structural masking runs first +// because it is what discovers the secret values. Propagation runs second, on +// the whole (still untruncated) record, so that a secret sitting past the +// 32 KB cap is scrubbed from the spilled blob as well as from the inline +// prefix. Truncation runs last, so the sha256 in the truncation notice is the +// digest of the MASKED body — the bytes a Tier-2 blob may legally hold — and +// not of a body that still had a live token in it. +// +// =========================================================================== +// FAIL-CLOSED RULES +// =========================================================================== +// +// R1. A header name that is not an RFC 9110 field-name token (empty, or +// carrying whitespace, a control byte, a separator, or any non-ASCII +// byte) cannot be classified, so its value is redacted. +// +// R2. A header VALUE containing CR or LF is redacted regardless of its +// name. This is the response-splitting case: `X-Trace: ok\r\nSet-Cookie: +// sid=live` presents to a name-based denylist as the innocent header +// `X-Trace`, and smuggles a live cookie into the record inside its +// value. A name-only denylist cannot see it. +// +// R3. A parameter name that is empty or carries a control byte cannot be +// classified, so its value is redacted. +// +// R4. A query- or fragment-pair name that will not percent-decode is +// redacted. +// +// R5. A `-H` / `--header` argument in a reproduction command line that does +// not parse as `Name: Value` cannot be classified, so the WHOLE argument +// is redacted. +// +// Each of these records an Anomaly in the MaskReport. None of them returns an +// error: a DAST scan of a hostile target produces malformed headers as a +// matter of course, and erroring out would discard the finding — which is a +// worse outcome than an over-redacted one. The redaction IS the closed +// failure. +// +// =========================================================================== +// WHAT THIS FILE DELIBERATELY DOES NOT DO +// =========================================================================== +// +// NO SHAPE-BASED BODY SCANNING. There is no "looks like a JWT" or "looks like +// an AWS key" regex here. Such a scanner cannot be made to fail closed — it +// either matches or it does not, and every miss is invisible — so it would +// deliver exactly the false confidence this file exists to avoid. A secret +// that appears ONLY in a body, and never in a denylisted header or a +// secret-named parameter, is NOT removed by this package. That is a known, +// stated limitation, not an oversight. S7's actual control for body content +// is a different mechanism owned by a different step: "hash-and-reference by +// default; inline only a regex-extracted evidence span". +// +// THE DENYLIST IS NOT EXHAUSTIVE, AND IS NOT CLAIMED TO BE. +// plan/40-record-and-storage.md Open Question 8 records this explicitly: R.8 +// uses a "documented but not exhaustively researched" denylist, and a +// dedicated security review of real-world header names is recommended before +// the masking pipeline ships in a release. Concrete names the list as +// specified does NOT catch, so the gap is visible rather than assumed +// covered: `api-key` and `apikey` (only the exact `x-api-key` is listed), +// `www-authenticate`, `authentication`, `x-csrf-token` is caught only via the +// `*token*` pattern, `x-amz-*` signature headers, `location` (which routinely +// carries a one-time code or an implicit-flow access token in a redirect), +// and any bespoke vendor header. Extend DenylistedHeaderNames when that +// review happens; do not assume the current list is complete. + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "reflect" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// --------------------------------------------------------------------------- +// The denylists +// --------------------------------------------------------------------------- + +// denylistedHeaderNames is the exact-match half of the header denylist from +// plan/40-record-and-storage.md R.8 ("Authorization, Cookie, Set-Cookie, +// Proxy-Authorization, X-Api-Key"). Stored ASCII-lowercased; comparison folds +// the record's name the same way. See Open Question 8 in the file header for +// what this list does not cover. +var denylistedHeaderNames = []string{ + "authorization", + "cookie", + "proxy-authorization", + "set-cookie", + "x-api-key", +} + +// denylistedHeaderSubstrings is the pattern half of the same denylist: "any +// header matching *token*/*secret* case-insensitively". This is what catches +// `X-Auth-Token`, `X-Csrf-Token`, `X-Amz-Security-Token` and +// `X-Client-Secret` without naming each one. +var denylistedHeaderSubstrings = []string{ + "secret", + "token", +} + +// sensitiveParameterNames is the exact-match denylist for +// webRequest.parameters keys and for query/fragment pair names. +// +// PROVENANCE, STATED PLAINLY: unlike the header denylist, this list is NOT in +// the plan and NOT in the research corpus. R.8's stop condition requires an +// "API key in a query parameter" fixture to come out clean, so a parameter +// rule is required, and this is R.8's own choice. It belongs to the same +// security review as Open Question 8 and carries the same caveat. +// +// It is deliberately narrower than the reflex "redact anything suspicious": +// a DAST parameter value is usually the INJECTION PAYLOAD, and the payload is +// the evidence the coding agent needs. Redacting `username` because a scanner +// might one day put a credential there would destroy the finding. +var sensitiveParameterNames = []string{ + "access_key", + "access_token", + "api_key", + "apikey", + "auth", + "authorization", + "client_secret", + "cookie", + "credential", + "credentials", + "id_token", + "jsessionid", + "passwd", + "password", + "phpsessid", + "pwd", + "refresh_token", + "secret", + "sessid", + "session", + "session_id", + "sessionid", + "sid", + "signature", + "token", +} + +// sensitiveParameterSubstrings is the pattern half of the parameter rule. +var sensitiveParameterSubstrings = []string{ + "api-key", + "api_key", + "apikey", + "passwd", + "password", + "secret", + "token", +} + +// DenylistedHeaderNames returns a copy of the exact-match header denylist, +// ASCII-lowercased. Exported so the security review Open Question 8 asks for +// has something to diff against, and so a test can assert the plan's five +// names are all present. +func DenylistedHeaderNames() []string { + return append([]string(nil), denylistedHeaderNames...) +} + +// DenylistedHeaderSubstrings returns a copy of the pattern half of the header +// denylist, ASCII-lowercased. +func DenylistedHeaderSubstrings() []string { + return append([]string(nil), denylistedHeaderSubstrings...) +} + +// SensitiveParameterNames returns a copy of the exact-match parameter +// denylist. See sensitiveParameterNames for its provenance caveat. +func SensitiveParameterNames() []string { + return append([]string(nil), sensitiveParameterNames...) +} + +// SensitiveParameterSubstrings returns a copy of the pattern half of the +// parameter denylist. +func SensitiveParameterSubstrings() []string { + return append([]string(nil), sensitiveParameterSubstrings...) +} + +// MinPropagatedSecretLen is the shortest redacted value that is propagated +// through the rest of the record by substring replacement. +// +// The bound exists because propagation is exact substring replacement over +// every string in the record. A cookie crumb of `a=1` yields the value "1", +// and replacing every "1" in a 32 KB stack trace would destroy the evidence +// while protecting nothing — a one-byte value is not a credential. Eight +// bytes is short enough to catch a weak session id and long enough that a +// collision with ordinary body text is not a practical concern. +// +// CONSEQUENCE, STATED: a genuine secret shorter than this is still redacted +// AT ITS HEADER OR PARAMETER — that is structural and unconditional — but a +// copy of it echoed elsewhere in the record is not removed. +const MinPropagatedSecretLen = 8 + +// --------------------------------------------------------------------------- +// Report types +// --------------------------------------------------------------------------- + +// Anomaly is one fail-closed redaction: a place where the masker could not +// classify what it was looking at and redacted rather than guess. +// +// Anomalies are surfaced rather than swallowed because they are the signal +// that the denylist met something it was not designed for. A run producing +// them is a run whose target is doing something unusual with HTTP. +type Anomaly struct { + // Pointer is an RFC 6901 JSON Pointer from the sarifLog root. + Pointer string `json:"pointer"` + // Reason names the fail-closed rule that fired (R1..R4 in this file's + // header comment). + Reason string `json:"reason"` +} + +// Spill is one body that exceeded its inline cap. +// +// research/18's read path says the remainder "spills to a blob", and +// plan/40-record-and-storage.md's Tier-2 row says those blobs are +// "referenced by sha256: digest". Content is the FULL MASKED body — masking +// and propagation have already run over it — so it is safe to persist as a +// Tier-2 blob exactly as given. +type Spill struct { + // Pointer is an RFC 6901 JSON Pointer from the sarifLog root to the + // ArtifactContent that was truncated. + Pointer string `json:"pointer"` + // Ref is the reference written into the record. By default + // "sha256:<64 lowercase hex>" over Content; a SpillFunc may return a + // different reference (e.g. a store-relative blob path). + Ref string `json:"ref"` + // Sha256 is the "sha256:<64 lowercase hex>" digest of Content, + // regardless of what Ref ended up being. + Sha256 string `json:"sha256"` + // TotalBytes is len(Content). InlineBytes is how much of it stayed in + // the record. + TotalBytes int `json:"totalBytes"` + InlineBytes int `json:"inlineBytes"` + // Content is the full masked body. NOT serialised: a MaskReport is a + // diagnostic object and must not become a second durable copy of the + // body it was created to move out of the record. + Content string `json:"-"` +} + +// SpillFunc persists a spilled body and returns the reference to write into +// the record. Returning "" means "use the default sha256: reference". An +// error aborts masking — the record is left in whatever masked state it had +// reached, which is always at least as masked as it started. +type SpillFunc func(Spill) (string, error) + +// MaskReport is what the masker did. It is diagnostic output, not part of the +// record. +type MaskReport struct { + HeadersRedacted int `json:"headersRedacted"` + ParametersRedacted int `json:"parametersRedacted"` + QueryValuesRedacted int `json:"queryValuesRedacted"` + // PropagatedRedactions counts STRINGS changed by the propagation pass, + // not occurrences. + PropagatedRedactions int `json:"propagatedRedactions"` + BodiesTruncated int `json:"bodiesTruncated"` + + Anomalies []Anomaly `json:"anomalies,omitempty"` + Spills []Spill `json:"spills,omitempty"` +} + +// --------------------------------------------------------------------------- +// The masker +// --------------------------------------------------------------------------- + +// Masker masks a record. The zero value is usable and is what MaskRecord +// uses. +type Masker struct { + // Spill, if non-nil, is called for every body that exceeds its cap, + // before the reference is written into the record. If it is nil the + // bytes past the cap are NOT retained by this package: the record keeps + // a sha256 reference to a blob nobody stored. Callers that own Tier-2 + // storage must set this, or read MaskReport.Spills. + Spill SpillFunc + + // MinPropagationLen overrides MinPropagatedSecretLen when > 0. Lowering + // it increases over-redaction; it does not weaken anything. + MinPropagationLen int +} + +// MaskRecord is R.8's entry point: it masks l in place and reports whether +// masking could be completed. +// +// It MUST be the last step of record assembly, before the record reaches +// either sink — the store or any model context. There is no supported order +// in which an unmasked record is written anywhere first and scrubbed +// afterwards; a post-hoc scrub leaves the live credential in the store for +// the window between the two, and "the window is short" is the argument +// research/18 Risk #10 already rejected. +// +// The bytes of any body past its inline cap are not retained by this call — +// the record keeps a sha256 reference to them. Use Masker with a Spill sink, +// or Masker.Mask and read MaskReport.Spills, when those bytes must be +// persisted as Tier-2 blobs. +func MaskRecord(l *SARIFLog) error { + _, err := (&Masker{}).Mask(l) + return err +} + +// Mask masks l in place and returns what it did. +// +// The returned error is reserved for conditions that mean the caller must not +// proceed: a nil log, or a SpillFunc that failed. An unclassifiable header is +// NOT one of them — see the fail-closed rules in this file's header. +func (m *Masker) Mask(l *SARIFLog) (*MaskReport, error) { + if l == nil { + return nil, fmt.Errorf("record: MaskRecord got a nil *SARIFLog; masking must run on the assembled record, before either sink") + } + rep := &MaskReport{} + secrets := newSecretSet(m.minPropagationLen()) + + // Pass 1 — structural. This is the pass that DISCOVERS secret values, and + // it is driven by walkMaskSurface so that the set of places it looks and + // the set of places AssertMasked checks cannot drift apart. + walkMaskSurface(l, surface{ + Headers: func(ptr string, h map[string]string) { + m.maskHeaders(ptr, h, secrets, rep) + }, + Parameters: func(ptr string, p map[string]string) { + m.maskParameters(ptr, p, secrets, rep) + }, + URL: func(ptr string, p *string) { + *p = m.maskURL(ptr, *p, secrets, rep) + }, + CommandLine: func(ptr string, p *string) { + *p = m.maskCommandLine(ptr, *p, secrets, rep) + }, + }) + + // Pass 2 — propagation, across the WHOLE record and the WHOLE body, + // before anything is truncated. + if vals := secrets.values(); len(vals) > 0 { + rep.PropagatedRedactions = propagateSecrets(reflect.ValueOf(l), vals) + } + + // Pass 3 — caps. Last, so every digest is over masked bytes. + var capErr error + walkMaskSurface(l, surface{ + Body: func(ptr string, b *ArtifactContent, limit int) { + if capErr != nil { + return + } + capErr = m.capBody(ptr, b, limit, rep) + }, + }) + if capErr != nil { + return rep, capErr + } + return rep, nil +} + +// --------------------------------------------------------------------------- +// The mask surface +// --------------------------------------------------------------------------- + +// surface is the set of callbacks walkMaskSurface invokes, one per KIND of +// maskable site. A nil callback means "this walk does not care about that +// kind"; the walk itself is unchanged either way. +// +// Splitting the enumeration of the sites from what is done to them is the +// whole point. CRITIQUE-02 F3 and F4 are both the same defect in two places: +// Mask inspected four sites, AssertMasked checked three of them, and neither +// looked at `anvil/repro.curl` or `anvil/target.repoUrl` — two fields that +// carry live credentials by construction (`-H 'Authorization: Bearer …'` and +// `https://x-access-token:@github.com/…`, the standard GitHub Actions +// checkout URL). With one walker, a site added here is masked AND enforced, +// and TestAssertMaskedCoversEverySiteMaskCovers fails if that stops being true. +type surface struct { + // Headers is an HTTP header map: name-keyed, denylist-classified. + Headers func(ptr string, h map[string]string) + // Parameters is a name/value map of request parameters. + Parameters func(ptr string, p map[string]string) + // URL is a settable field holding a single URL: userinfo, query and + // fragment all classified. + URL func(ptr string, p *string) + // CommandLine is a settable field holding a shell reproduction command. + CommandLine func(ptr string, p *string) + // Body is an inline artifact content with its own byte cap. + Body func(ptr string, b *ArtifactContent, limit int) +} + +// walkMaskSurface visits every site in l that this package is responsible for, +// in a fixed order, passing an RFC 6901 JSON Pointer for each. +// +// THE URL SITES ARE NOT DECORATION. `anvil/target.repoUrl` is where a CI +// checkout URL lands, and `https://x-access-token:@github.com/org/repo` +// is what GitHub Actions produces; `runtimeBaseUrl` and +// `anvil/runtimeTarget.baseUrl` are where a DAST target's basic-auth userinfo +// lands. maskURL already knew how to strip all of that — it was simply never +// pointed at these fields. +func walkMaskSurface(l *SARIFLog, s surface) { + if l == nil { + return + } + + if s.URL != nil { + t := &l.Properties.Target + s.URL("/properties/anvil~1target/repoUrl", &t.RepoURL) + s.URL("/properties/anvil~1target/runtimeBaseUrl", &t.RuntimeBaseURL) + } + + for i := range l.Runs { + run := &l.Runs[i] + runBase := fmt.Sprintf("/runs/%d", i) + + if rt := run.Properties.RuntimeTarget; rt != nil && s.URL != nil { + rtBase := runBase + "/properties/anvil~1runtimeTarget" + s.URL(rtBase+"/baseUrl", &rt.BaseURL) + for k := range rt.Scope { + s.URL(fmt.Sprintf("%s/scope/%d", rtBase, k), &rt.Scope[k]) + } + for k := range rt.Excluded { + s.URL(fmt.Sprintf("%s/excluded/%d", rtBase, k), &rt.Excluded[k]) + } + } + + for j := range run.Results { + r := &run.Results[j] + base := fmt.Sprintf("%s/results/%d", runBase, j) + + if r.WebRequest != nil { + if s.Headers != nil { + s.Headers(base+"/webRequest/headers", r.WebRequest.Headers) + } + if s.Parameters != nil { + s.Parameters(base+"/webRequest/parameters", r.WebRequest.Parameters) + } + if s.URL != nil { + s.URL(base+"/webRequest/target", &r.WebRequest.Target) + } + if s.Body != nil { + s.Body(base+"/webRequest/body", r.WebRequest.Body, MaxInlineRequestBodyBytes) + } + } + if r.WebResponse != nil { + if s.Headers != nil { + s.Headers(base+"/webResponse/headers", r.WebResponse.Headers) + } + if s.Body != nil { + s.Body(base+"/webResponse/body", r.WebResponse.Body, MaxInlineResponseBodyBytes) + } + } + if repro := r.Properties.Repro; repro != nil && s.CommandLine != nil { + s.CommandLine(base+"/properties/anvil~1repro/curl", &repro.Curl) + } + } + } +} + +func (m *Masker) minPropagationLen() int { + if m.MinPropagationLen > 0 { + return m.MinPropagationLen + } + return MinPropagatedSecretLen +} + +// --------------------------------------------------------------------------- +// Pass 1 — headers +// --------------------------------------------------------------------------- + +func (m *Masker) maskHeaders(ptr string, headers map[string]string, secrets *secretSet, rep *MaskReport) { + if len(headers) == 0 { + return + } + for _, name := range sortedKeys(headers) { + value := headers[name] + reason := headerRedactionReason(name, value) + if reason == "" { + continue + } + if value != RedactedPlaceholder { + secrets.addHeader(name, value) + } + headers[name] = RedactedPlaceholder + rep.HeadersRedacted++ + if reason != reasonDenylisted { + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr + "/" + jsonPointerEscape(name), + Reason: reason, + }) + } + } +} + +const reasonDenylisted = "denylisted header name" + +// headerRedactionReason returns "" when the header may stay, or the reason it +// must be redacted. The two fail-closed rules are checked BEFORE the +// denylist, because a malformed name cannot be meaningfully compared against +// a denylist at all. +func headerRedactionReason(name, value string) string { + if !isHTTPFieldName(name) { + return "R1: header name is not an RFC 9110 field-name token, so it cannot be classified" + } + if strings.ContainsAny(value, "\r\n") { + return "R2: header value contains CR or LF, which can smuggle a second header past a name-based denylist" + } + if isDenylistedHeader(name) { + return reasonDenylisted + } + return "" +} + +// isDenylistedHeader folds name to ASCII lowercase and applies both halves of +// the denylist. It assumes isHTTPFieldName(name) already passed, which is +// what makes plain ASCII folding sufficient: strings.EqualFold would apply +// Unicode case folding, under which U+212A KELVIN SIGN folds to "k", so +// "cooie" would compare equal to "cookie" — harmless here, but the +// same mechanism in the other direction is how fold-based comparisons get +// bypassed. Non-ASCII names never reach this function. +func isDenylistedHeader(name string) bool { + lower := asciiLower(name) + for _, d := range denylistedHeaderNames { + if lower == d { + return true + } + } + for _, sub := range denylistedHeaderSubstrings { + if strings.Contains(lower, sub) { + return true + } + } + return false +} + +// isHTTPFieldName reports whether name is a non-empty RFC 9110 §5.1 token: +// one or more tchar. Anything else — empty, whitespace-padded, control bytes, +// separators like ':' or ',', or any byte >= 0x80 — is unexpected shape. +func isHTTPFieldName(name string) bool { + if name == "" { + return false + } + for i := 0; i < len(name); i++ { + if !isTChar(name[i]) { + return false + } + } + return true +} + +// isTChar reports whether c is an RFC 9110 §5.6.2 tchar. +func isTChar(c byte) bool { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + return true + } + switch c { + case '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~': + return true + } + return false +} + +// --------------------------------------------------------------------------- +// Pass 1 — parameters and the target URL +// --------------------------------------------------------------------------- + +func (m *Masker) maskParameters(ptr string, params map[string]string, secrets *secretSet, rep *MaskReport) { + if len(params) == 0 { + return + } + for _, name := range sortedKeys(params) { + value := params[name] + var reason string + switch { + case name == "" || containsControl(name): + reason = "R3: parameter name is empty or carries a control byte, so it cannot be classified" + case isSensitiveParameter(name): + // Not an anomaly: this is the rule working as designed. + default: + continue + } + if value != RedactedPlaceholder { + secrets.add(value) + } + params[name] = RedactedPlaceholder + rep.ParametersRedacted++ + if reason != "" { + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr + "/" + jsonPointerEscape(name), + Reason: reason, + }) + } + } +} + +func isSensitiveParameter(name string) bool { + lower := asciiLower(strings.TrimSpace(name)) + for _, d := range sensitiveParameterNames { + if lower == d { + return true + } + } + for _, sub := range sensitiveParameterSubstrings { + if strings.Contains(lower, sub) { + return true + } + } + return false +} + +// maskURL redacts secrets carried in webRequest.target: the userinfo +// password, the query string, and the fragment. +// +// The fragment is not an afterthought. OAuth 2.0's implicit flow returns +// `#access_token=...`, so a redirect captured by a DAST run can carry a live +// access token in the one part of a URL that never reaches the server and +// that a query-only masker ignores entirely. +// +// It works LEXICALLY, not through url.Parse plus re-encoding. Round-tripping +// a URL through net/url normalises percent-encoding, and for a DAST record +// the exact bytes of the target ARE the evidence: `%27%20OR%201%3D1` and +// `' OR 1=1` are the same URL and different findings. Only the bytes of a +// redacted value change. +func (m *Masker) maskURL(ptr, target string, secrets *secretSet, rep *MaskReport) string { + if target == "" { + return target + } + head, query, fragment, hasQuery, hasFragment := splitURL(target) + head = maskUserinfoPassword(head, secrets, rep, ptr) + if query != "" { + masked, n := m.maskPairs(ptr+"/query", query, secrets, rep) + query = masked + rep.QueryValuesRedacted += n + } + // A fragment without '=' is an ordinary anchor and carries no pairs. + if strings.Contains(fragment, "=") { + masked, n := m.maskPairs(ptr+"/fragment", fragment, secrets, rep) + fragment = masked + rep.QueryValuesRedacted += n + } + // The delimiters are reinstated from splitURL's flags, never by scanning + // the original target for '?' or '#'. `https://h/p#a?b=c` has a '?' that + // belongs to the FRAGMENT, and rebuilding from a scan would invent a + // query delimiter that was never there, silently rewriting the endpoint + // the finding is about. + out := head + if hasQuery { + out += "?" + query + } + if hasFragment { + out += "#" + fragment + } + return out +} + +// splitURL splits target into everything before the query, the raw query, and +// the raw fragment, without decoding anything. A '#' before a '?' means there +// is no query — the '?' is inside the fragment. The two bools distinguish an +// absent component from a present-but-empty one. +func splitURL(target string) (head, query, fragment string, hasQuery, hasFragment bool) { + rest := target + if i := strings.IndexByte(rest, '#'); i >= 0 { + fragment, hasFragment = rest[i+1:], true + rest = rest[:i] + } + if i := strings.IndexByte(rest, '?'); i >= 0 { + query, hasQuery = rest[i+1:], true + rest = rest[:i] + } + return rest, query, fragment, hasQuery, hasFragment +} + +// maskUserinfoPassword redacts the password half of `scheme://user:pass@host`. +// The username stays: it identifies which account the scan authenticated as, +// which is evidence. The password never is. +func maskUserinfoPassword(head string, secrets *secretSet, rep *MaskReport, ptr string) string { + i := strings.Index(head, "//") + if i < 0 { + return head + } + authStart := i + 2 + authEnd := len(head) + if j := strings.IndexByte(head[authStart:], '/'); j >= 0 { + authEnd = authStart + j + } + authority := head[authStart:authEnd] + at := strings.LastIndexByte(authority, '@') + if at < 0 { + return head + } + userinfo := authority[:at] + colon := strings.IndexByte(userinfo, ':') + if colon < 0 { + return head + } + pass := userinfo[colon+1:] + if pass == "" || pass == RedactedPlaceholder { + return head + } + secrets.add(pass) + if dec, err := url.QueryUnescape(pass); err == nil { + secrets.add(dec) + } + rep.QueryValuesRedacted++ + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr, + Reason: "credential carried in URL userinfo", + }) + return head[:authStart] + userinfo[:colon+1] + RedactedPlaceholder + authority[at:] + head[authEnd:] +} + +// maskPairs masks `name=value` pairs joined by '&', preserving the original +// bytes of every name and of every value it does not redact. +func (m *Masker) maskPairs(ptr, raw string, secrets *secretSet, rep *MaskReport) (string, int) { + parts := strings.Split(raw, "&") + n := 0 + for i, part := range parts { + if part == "" { + continue + } + eq := strings.IndexByte(part, '=') + if eq < 0 { + // A bare flag with no value carries nothing to redact. + continue + } + name, value := part[:eq], part[eq+1:] + decoded, err := url.QueryUnescape(name) + var reason string + switch { + case err != nil: + reason = "R4: query/fragment pair name will not percent-decode, so it cannot be classified" + case isSensitiveParameter(decoded): + // Rule working as designed; not an anomaly. + default: + continue + } + if value != RedactedPlaceholder && value != "" { + secrets.add(value) + if dec, derr := url.QueryUnescape(value); derr == nil { + secrets.add(dec) + } + } + parts[i] = name + "=" + RedactedPlaceholder + n++ + if reason != "" { + rep.Anomalies = append(rep.Anomalies, Anomaly{Pointer: ptr, Reason: reason}) + } + } + return strings.Join(parts, "&"), n +} + +// --------------------------------------------------------------------------- +// Pass 1 — reproduction command lines (`anvil/repro.curl`) +// --------------------------------------------------------------------------- + +// Command-line option classes. Only options whose ARGUMENT can carry a +// credential are listed; everything else is left byte for byte, because a +// reproduction command is the evidence a human replays. +type cmdArgKind int + +const ( + cmdArgNone cmdArgKind = iota + cmdArgHeader + cmdArgCookie + cmdArgUser + cmdArgData +) + +// cmdLongOptions maps `--name` to what its argument is. `--proxy-header` +// carries a proxy Authorization as routinely as `--header` carries an +// Authorization. +var cmdLongOptions = map[string]cmdArgKind{ + "--header": cmdArgHeader, + "--proxy-header": cmdArgHeader, + "--cookie": cmdArgCookie, + "--user": cmdArgUser, + "--proxy-user": cmdArgUser, + "--data": cmdArgData, + "--data-raw": cmdArgData, + "--data-ascii": cmdArgData, + "--data-binary": cmdArgData, + "--data-urlencode": cmdArgData, + "--form": cmdArgData, + "--form-string": cmdArgData, +} + +// cmdShortOptions maps the single-letter forms. curl allows the argument to be +// attached (`-HAuthorization: …`) or separate, and allows clustering, so both +// shapes are handled below. +var cmdShortOptions = map[byte]cmdArgKind{ + 'H': cmdArgHeader, + 'b': cmdArgCookie, + 'u': cmdArgUser, + 'U': cmdArgUser, + 'd': cmdArgData, + 'F': cmdArgData, +} + +// maskCommandLine redacts credentials carried in a reproduction command line. +// +// WHY THIS FIELD IS NOT OPTIONAL COVER. `anvil/repro.curl` is a full command +// the record invites a human to replay, and the thing that makes it replayable +// is precisely the credential: `-H 'Authorization: Bearer …'`, `-b +// 'session=…'`, `-u user:password`, or an API key in the URL. CRITIQUE-02 F3 +// reproduced a live GitHub token surviving MaskRecord in exactly this field. +// +// It is NOT shape-based body scanning (which this file refuses to do, see the +// header): a curl command line is a STRUCTURED string with known credential +// positions — option flags — and structural masking is exactly what those are +// for. What is not an option argument or a URL is left alone. +// +// The command is re-emitted byte for byte apart from the values redacted: +// quoting, spacing and option order are all preserved, because a reproduction +// that has been "tidied" is a different experiment. +func (m *Masker) maskCommandLine(ptr, cmd string, secrets *secretSet, rep *MaskReport) string { + if cmd == "" || !strings.ContainsAny(cmd, " \t") { + // A single bare word cannot carry an option argument. A URL-only + // value still goes through maskURL below, so only the truly empty and + // the truly wordless short-circuit here. + if !strings.Contains(cmd, "://") { + return cmd + } + } + + pieces, isToken := splitCommandLine(cmd) + expect := cmdArgNone + + for i, piece := range pieces { + if !isToken[i] { + continue + } + quote, body := unquoteToken(piece) + + if expect != cmdArgNone { + pieces[i] = requoteToken(quote, m.maskQuotedArg(ptr, expect, body, secrets, rep)) + expect = cmdArgNone + continue + } + + switch { + case strings.HasPrefix(body, "--"): + name, value, hasValue := strings.Cut(body, "=") + kind, ok := cmdLongOptions[asciiLower(name)] + if !ok { + continue + } + if !hasValue { + expect = kind + continue + } + pieces[i] = requoteToken(quote, name+"="+m.maskQuotedArg(ptr, kind, value, secrets, rep)) + + case strings.HasPrefix(body, "-") && len(body) > 1: + // A cluster like `-sSH`: scan for the first letter that takes an + // argument. Anything after it on the same token IS that argument. + for k := 1; k < len(body); k++ { + kind, ok := cmdShortOptions[body[k]] + if !ok { + continue + } + if k+1 < len(body) { + pieces[i] = requoteToken(quote, + body[:k+1]+m.maskQuotedArg(ptr, kind, body[k+1:], secrets, rep)) + } else { + expect = kind + } + break + } + + case strings.Contains(body, "://"): + pieces[i] = requoteToken(quote, m.maskURL(ptr+"/url", body, secrets, rep)) + } + } + return strings.Join(pieces, "") +} + +// maskQuotedArg strips one layer of quoting the argument may carry in its own +// right — `--header="X-Api-Key: …"` and `-H'Authorization: …'` both put the +// quotes INSIDE the token, not around it — masks the contents, then puts the +// same quote pair back. Without this the closing quote is lost and the next +// pass tokenises the command differently, which is exactly the kind of drift +// that makes an idempotence-based gate unusable. +func (m *Masker) maskQuotedArg(ptr string, kind cmdArgKind, arg string, secrets *secretSet, rep *MaskReport) string { + quote, body := unquoteToken(arg) + return requoteToken(quote, m.maskCommandArg(ptr, kind, body, secrets, rep)) +} + +// maskCommandArg masks one option argument according to what the option is. +// It is idempotent: an argument already carrying RedactedPlaceholder is +// returned unchanged, which is what lets AssertMasked re-derive the masked +// form and compare. +// +// It never changes the WHITESPACE of what it keeps either. `-HAuthorization:X` +// rewritten as `-HAuthorization: ***REDACTED***` would introduce a space that +// splits the token in two on the next pass, so the original separator between +// the colon and the value is preserved verbatim. +func (m *Masker) maskCommandArg(ptr string, kind cmdArgKind, arg string, secrets *secretSet, rep *MaskReport) string { + switch kind { + case cmdArgHeader: + colon := strings.IndexByte(arg, ':') + if colon <= 0 { + // R5: not a `Name: Value` header at all, so it cannot be + // classified against the denylist. Redact the whole argument. + if arg == RedactedPlaceholder { + return arg + } + secrets.add(arg) + rep.HeadersRedacted++ + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr, + Reason: "R5: -H/--header argument does not parse as `Name: Value`, so it cannot be classified", + }) + return RedactedPlaceholder + } + name := strings.TrimSpace(arg[:colon]) + value := strings.TrimLeft(arg[colon+1:], " \t") + // The bytes between the colon and the value, kept exactly. + sep := arg[colon+1 : len(arg)-len(value)] + reason := headerRedactionReason(name, value) + if reason == "" { + return arg + } + if value != RedactedPlaceholder { + secrets.addHeader(name, value) + } + rep.HeadersRedacted++ + if reason != reasonDenylisted { + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr + "/" + jsonPointerEscape(name), + Reason: reason, + }) + } + return arg[:colon+1] + sep + RedactedPlaceholder + + case cmdArgCookie: + // `-b` takes either a cookie string or a file name. Only a cookie + // string has crumbs, and a file name has no '=' in it. + if !strings.Contains(arg, "=") || arg == RedactedPlaceholder { + return arg + } + secrets.addHeader("cookie", arg) + rep.HeadersRedacted++ + return RedactedPlaceholder + + case cmdArgUser: + // `-u user:password`. The username stays — it identifies which account + // the reproduction authenticates as, which is evidence. + colon := strings.IndexByte(arg, ':') + if colon < 0 { + return arg // curl would prompt for the password; none is present + } + pass := arg[colon+1:] + if pass == "" || pass == RedactedPlaceholder { + return arg + } + secrets.add(pass) + if dec, err := url.QueryUnescape(pass); err == nil { + secrets.add(dec) + } + rep.QueryValuesRedacted++ + rep.Anomalies = append(rep.Anomalies, Anomaly{ + Pointer: ptr, + Reason: "credential carried in a -u/--user command-line argument", + }) + return arg[:colon+1] + RedactedPlaceholder + + case cmdArgData: + masked, n := m.maskPairs(ptr+"/data", arg, secrets, rep) + rep.ParametersRedacted += n + return masked + } + return arg +} + +// splitCommandLine splits s into alternating separator and token pieces, +// preserving every byte: strings.Join(pieces, "") == s. +// +// It is a SPLITTER, not a shell. It does not expand variables, resolve +// backslash escapes or evaluate anything — it only needs to know where one +// argument ends and the next begins, and which bytes are a quote pair, so that +// a redacted value can be put back inside the same quotes. +func splitCommandLine(s string) (pieces []string, isToken []bool) { + i := 0 + for i < len(s) { + if isCmdSpace(s[i]) { + start := i + for i < len(s) && isCmdSpace(s[i]) { + i++ + } + pieces = append(pieces, s[start:i]) + isToken = append(isToken, false) + continue + } + start := i + var quote byte + for i < len(s) { + c := s[i] + if quote != 0 { + if c == quote { + quote = 0 + } + i++ + continue + } + if isCmdSpace(c) { + break + } + if c == '\'' || c == '"' { + quote = c + } + i++ + } + pieces = append(pieces, s[start:i]) + isToken = append(isToken, true) + } + return pieces, isToken +} + +func isCmdSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } + +// unquoteToken strips one matched pair of surrounding quotes and reports which +// quote byte it was, so requoteToken can put the same pair back. +func unquoteToken(tok string) (quote byte, body string) { + if len(tok) >= 2 { + q := tok[0] + if (q == '\'' || q == '"') && tok[len(tok)-1] == q { + return q, tok[1 : len(tok)-1] + } + } + return 0, tok +} + +func requoteToken(quote byte, body string) string { + if quote == 0 { + return body + } + return string(quote) + body + string(quote) +} + +// --------------------------------------------------------------------------- +// Pass 2 — value propagation +// --------------------------------------------------------------------------- + +// secretSet collects the literal values structural masking removed, plus the +// sub-values decomposed out of them. +// +// Decomposition matters. `Authorization: Bearer eyJhbGciOi...` is redacted as +// a whole, so the propagation set would contain "Bearer eyJhbGciOi..." — and +// an error page that echoes the raw token WITHOUT the scheme prefix would not +// match. Same for `Cookie: theme=dark; session=abc123`: the crumb the +// application echoes is `abc123`, not the whole header line. +type secretSet struct { + minLen int + seen map[string]bool +} + +func newSecretSet(minLen int) *secretSet { + return &secretSet{minLen: minLen, seen: map[string]bool{}} +} + +func (s *secretSet) add(v string) { + v = strings.TrimSpace(v) + if len(v) < s.minLen || v == RedactedPlaceholder { + return + } + // A value that already contains the placeholder is a partially masked + // string, not a secret. + if strings.Contains(v, RedactedPlaceholder) { + return + } + s.seen[v] = true +} + +// addHeader adds the whole header value and its decomposed sub-values. +func (s *secretSet) addHeader(name, value string) { + s.add(value) + lower := asciiLower(name) + switch { + case lower == "authorization" || lower == "proxy-authorization": + // ` ` — the credentials alone are what gets + // echoed and logged. + if i := strings.IndexByte(value, ' '); i > 0 { + s.add(value[i+1:]) + } + case lower == "cookie" || lower == "set-cookie": + for _, crumb := range strings.Split(value, ";") { + eq := strings.IndexByte(crumb, '=') + if eq < 0 { + continue + } + s.add(strings.Trim(strings.TrimSpace(crumb[eq+1:]), `"`)) + } + } + // A value carrying CR or LF is a smuggled header block (fail-closed rule + // R2). Decompose it as one: `X-Trace: ok\r\nSet-Cookie: sid=live` hides + // a real cookie whose crumb the application will echo back on its own, + // without the `Set-Cookie:` prefix that the whole-value entry carries. + // Splitting on CR/LF strips them, so the recursion terminates one level + // down. + if strings.ContainsAny(value, "\r\n") { + for _, line := range strings.FieldsFunc(value, func(r rune) bool { return r == '\r' || r == '\n' }) { + if c := strings.IndexByte(line, ':'); c > 0 { + s.addHeader(strings.TrimSpace(line[:c]), strings.TrimSpace(line[c+1:])) + } + } + } + + // Quoted values are echoed unquoted often enough to be worth adding. + if unquoted := strings.Trim(value, `"`); unquoted != value { + s.add(unquoted) + } +} + +// values returns the collected secrets sorted LONGEST FIRST, then +// lexicographically. +// +// Length ordering is required for correctness, not tidiness: if the set holds +// both `Bearer eyJ...` and `eyJ...`, replacing the short one first leaves +// `Bearer ***REDACTED***` and the long one can never match again. Replacing +// longest first collapses both to one placeholder. Lexicographic ordering +// within a length class makes the output deterministic, which a golden test +// can depend on. +func (s *secretSet) values() []string { + out := make([]string, 0, len(s.seen)) + for v := range s.seen { + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { + if len(out[i]) != len(out[j]) { + return len(out[i]) > len(out[j]) + } + return out[i] < out[j] + }) + return out +} + +var timeType = reflect.TypeOf(time.Time{}) + +// propagateSecrets replaces every occurrence of every collected secret, in +// every settable string anywhere in v, with RedactedPlaceholder. It returns +// the number of strings it changed. +// +// It walks by REFLECTION rather than by an enumerated field list on purpose. +// An enumerated list is auditable but goes stale the moment R.13 or the DAST +// area adds a string field to the contract, and the failure mode of a stale +// list is a secret surviving in the new field with nothing to indicate it. +// Reflection covers new fields the day they are added. +func propagateSecrets(v reflect.Value, secrets []string) int { + switch v.Kind() { + case reflect.Pointer, reflect.Interface: + if v.IsNil() { + return 0 + } + return propagateSecrets(v.Elem(), secrets) + + case reflect.Struct: + // time.Time's fields are unexported and carry no attacker text. + if v.Type() == timeType { + return 0 + } + n := 0 + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).PkgPath != "" { // unexported + continue + } + n += propagateSecrets(v.Field(i), secrets) + } + return n + + case reflect.Slice, reflect.Array: + if v.Kind() == reflect.Slice && v.IsNil() { + return 0 + } + n := 0 + for i := 0; i < v.Len(); i++ { + n += propagateSecrets(v.Index(i), secrets) + } + return n + + case reflect.Map: + if v.IsNil() { + return 0 + } + n := 0 + // Map values are not addressable, so each is copied into an + // addressable temporary, walked, and written back. Keys are visited + // in sorted order so the change count is deterministic. + keys := v.MapKeys() + sort.Slice(keys, func(i, j int) bool { + return fmt.Sprint(keys[i].Interface()) < fmt.Sprint(keys[j].Interface()) + }) + for _, k := range keys { + elem := reflect.New(v.Type().Elem()).Elem() + elem.Set(v.MapIndex(k)) + if c := propagateSecrets(elem, secrets); c > 0 { + v.SetMapIndex(k, elem) + n += c + } + } + return n + + case reflect.String: + if !v.CanSet() { + return 0 + } + before := v.String() + if before == "" { + return 0 + } + after := scrub(before, secrets) + if after == before { + return 0 + } + v.SetString(after) + return 1 + } + return 0 +} + +// scrub replaces every occurrence of every secret in s with the placeholder. +func scrub(s string, secrets []string) string { + for _, sec := range secrets { + if sec == "" { + continue + } + if strings.Contains(s, sec) { + s = strings.ReplaceAll(s, sec, RedactedPlaceholder) + } + } + return s +} + +// --------------------------------------------------------------------------- +// Pass 3 — body caps and Tier-2 spill +// --------------------------------------------------------------------------- + +// capBody truncates body to at most limit bytes TOTAL, including the +// truncation notice, and records the remainder as a spill. +// +// The remainder is never dropped: the notice carries a content-addressed +// reference to the full masked body, and MaskReport.Spills carries the bytes +// themselves for a caller that owns Tier-2 storage. +func (m *Masker) capBody(ptr string, body *ArtifactContent, limit int, rep *MaskReport) error { + if body == nil || len(body.Text) <= limit { + return nil + } + full := body.Text + sum := sha256.Sum256([]byte(full)) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + sp := Spill{ + Pointer: ptr, + Ref: digest, + Sha256: digest, + TotalBytes: len(full), + Content: full, + } + if m.Spill != nil { + ref, err := m.Spill(sp) + if err != nil { + return fmt.Errorf("record: spilling %s to a Tier-2 blob failed: %w", ptr, err) + } + if ref != "" { + sp.Ref = ref + } + } + + // Size the notice against `limit` first. digits(n) <= digits(limit) for + // every 0 <= n <= limit, so the notice built with the real inline count + // is never longer than this estimate, and the final text is therefore + // never longer than limit. + budget := limit - len(truncationNotice(limit, len(full), sp.Ref)) + if budget < 0 { + budget = 0 + } + inline := truncateToRuneBoundary(full, budget) + body.Text = inline + truncationNotice(len(inline), len(full), sp.Ref) + + sp.InlineBytes = len(inline) + rep.Spills = append(rep.Spills, sp) + rep.BodiesTruncated++ + return nil +} + +// truncationNotice is the in-band Tier-2 pointer, in the shape research/18's +// annotated record uses ("…[truncated at 32768 bytes, full body at +// blobs/sha256:5c0d…]"). It is in-band because SARIF's artifactContent +// (§3.3) has no property bag to hang it on, and an out-of-band-only reference +// would leave a reader of the record with no indication that what they are +// reading is a prefix. +func truncationNotice(inlineBytes, totalBytes int, ref string) string { + return "\n[anvil: body truncated at " + strconv.Itoa(inlineBytes) + + " of " + strconv.Itoa(totalBytes) + " bytes; full masked body at " + ref + "]" +} + +// truncateToRuneBoundary cuts s to at most n bytes without splitting a UTF-8 +// rune. A split rune would make the record invalid JSON text in practice and +// would corrupt the last visible character of the evidence for no gain. +func truncateToRuneBoundary(s string, n int) string { + if len(s) <= n { + return s + } + b := s[:n] + for len(b) > 0 { + r, size := utf8.DecodeLastRuneInString(b) + if r == utf8.RuneError && size <= 1 { + b = b[:len(b)-1] + continue + } + break + } + return b +} + +// --------------------------------------------------------------------------- +// Post-condition +// --------------------------------------------------------------------------- + +// AssertMasked reports whether l satisfies R.8's post-condition: every site +// Mask is responsible for has already been masked, and no inline body exceeds +// its cap. +// +// plan/00-SPINE.md S7 is "enforce in code, not documentation". This is the +// enforceable half of it: a sink — the store writer, the prompt builder, the +// GitHub projection — can call it and refuse the record rather than trusting +// that some earlier step remembered to mask. +// +// IT COVERS EXACTLY WHAT Mask COVERS, AND THAT IS STRUCTURAL. It walks the +// same walkMaskSurface enumeration Mask does, and for the sites whose masking +// is a pure function of the field (URLs and command lines) it RE-DERIVES the +// masked form and demands the record already equal it. A gate that checked +// less than the masker is worse than no gate, because it manufactures the +// confidence it fails to justify — CRITIQUE-02 F4 found exactly that: Mask +// masked webRequest.target and AssertMasked did not, so a record whose only +// credential sat in a URL passed the check that exists to catch it. +// +// It does NOT prove the absence of secrets. It cannot: it does not know what +// the secrets were, and value propagation (pass 2) is not re-derivable from +// the masked record. It proves that the structural rules were applied, which +// is a necessary condition, not a sufficient one. +func AssertMasked(l *SARIFLog) error { + if l == nil { + return fmt.Errorf("record: AssertMasked got a nil *SARIFLog") + } + var firstErr error + note := func(err error) { + if err != nil && firstErr == nil { + firstErr = err + } + } + walkMaskSurface(l, surface{ + Headers: func(ptr string, h map[string]string) { + note(assertHeadersMasked(ptr, h)) + }, + Parameters: func(ptr string, p map[string]string) { + note(assertParametersMasked(ptr, p)) + }, + URL: func(ptr string, p *string) { + note(assertURLMasked(ptr, *p)) + }, + CommandLine: func(ptr string, p *string) { + note(assertCommandLineMasked(ptr, *p)) + }, + Body: func(ptr string, b *ArtifactContent, limit int) { + note(assertBodyCapped(ptr, b, limit)) + }, + }) + return firstErr +} + +// assertURLMasked re-derives what Mask would produce for this URL and refuses +// the record if it differs. maskURL is idempotent, so equality means "already +// masked" and inequality means "a userinfo password, a sensitive query +// parameter or a fragment token is still live in this field". +func assertURLMasked(ptr, target string) error { + if masked := (&Masker{}).maskURL(ptr, target, newSecretSet(MinPropagatedSecretLen), &MaskReport{}); masked != target { + return fmt.Errorf("record: %s still carries an unmasked credential in a URL "+ + "(userinfo password, sensitive query parameter, or fragment token); "+ + "R.8 masking must run before the store and before any model context", ptr) + } + return nil +} + +// assertCommandLineMasked is assertURLMasked's twin for `anvil/repro.curl`. +// maskCommandLine is idempotent for the same reason, so any difference is a +// live credential sitting in an option argument. +func assertCommandLineMasked(ptr, cmd string) error { + if masked := (&Masker{}).maskCommandLine(ptr, cmd, newSecretSet(MinPropagatedSecretLen), &MaskReport{}); masked != cmd { + return fmt.Errorf("record: %s still carries an unmasked credential in a reproduction "+ + "command line (a header, cookie, user or data option argument); "+ + "R.8 masking must run before either sink", ptr) + } + return nil +} + +func assertHeadersMasked(ptr string, headers map[string]string) error { + for _, name := range sortedKeys(headers) { + value := headers[name] + if headerRedactionReason(name, value) == "" { + continue + } + if value != RedactedPlaceholder { + return fmt.Errorf("record: %s/%s is unmasked (%s); R.8 masking must run before the store and before any model context", + ptr, jsonPointerEscape(name), headerRedactionReason(name, value)) + } + } + return nil +} + +func assertParametersMasked(ptr string, params map[string]string) error { + for _, name := range sortedKeys(params) { + value := params[name] + if name != "" && !containsControl(name) && !isSensitiveParameter(name) { + continue + } + if value != RedactedPlaceholder { + return fmt.Errorf("record: %s/%s is an unmasked sensitive parameter; R.8 masking must run before either sink", + ptr, jsonPointerEscape(name)) + } + } + return nil +} + +func assertBodyCapped(ptr string, body *ArtifactContent, limit int) error { + if body == nil || len(body.Text) <= limit { + return nil + } + return fmt.Errorf("record: %s is %d bytes, over the %d-byte inline cap; the remainder must spill to a Tier-2 blob", + ptr, len(body.Text), limit) +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +// asciiLower folds A-Z only. See isDenylistedHeader for why Unicode folding +// is deliberately not used. +func asciiLower(s string) string { + var b []byte + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' { + if b == nil { + b = []byte(s) + } + b[i] = c + ('a' - 'A') + } + } + if b == nil { + return s + } + return string(b) +} + +// containsControl reports whether s carries a C0 control byte or DEL. +func containsControl(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < 0x20 || s[i] == 0x7f { + return true + } + } + return false +} + +// jsonPointerEscape applies RFC 6901 §3 escaping so a header name containing +// '/' or '~' produces a well-formed pointer. +func jsonPointerEscape(s string) string { + s = strings.ReplaceAll(s, "~", "~0") + return strings.ReplaceAll(s, "/", "~1") +} + +// sortedKeys makes every map iteration in this file deterministic. Masking +// output that varies run to run cannot be golden-tested, and a masker nobody +// can pin is a masker nobody can prove. +func sortedKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/record/mask_test.go b/internal/record/mask_test.go new file mode 100644 index 0000000..b5b2efb --- /dev/null +++ b/internal/record/mask_test.go @@ -0,0 +1,1398 @@ +package record + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + "unicode/utf8" +) + +// --------------------------------------------------------------------------- +// Planted secrets +// +// These are the fixtures R.8's stop condition names — "a bearer token, a +// session cookie, an API key in a query parameter" — plus three more that +// cover the ways a secret reaches the record WITHOUT sitting in a header the +// denylist knows by name. +// +// None of them contains a character encoding/json escapes ('<', '>', '&', +// '"', '\\'), so a substring search over the marshalled record is a search +// for the literal bytes and not for an escape sequence. That is deliberate: +// a test that searched for a value the encoder had rewritten would pass +// vacuously. +// --------------------------------------------------------------------------- + +const ( + // plantedBearer is the credential half of an Authorization header. + plantedBearer = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r-wW1gFWFOEjXk" + // plantedCookie is one crumb's value inside a multi-crumb Cookie header. + plantedCookie = "s%3AJk9pQ2xZ8vT4hN6mB1cD.7fFqR2wXyZ0aL5nP8jK3uV1eS6tG4hM9bC" + // plantedAPIKey travels in the target's query string AND in + // webRequest.parameters. + plantedAPIKey = "AKIAIOSFODNN7EXAMPLE-9d8f7a6b5c4e3d2f1a0b" + // plantedProxyCred is a Proxy-Authorization credential. + plantedProxyCred = "cHJveHktdXNlcjpwcm94eS1wYXNzd29yZC0xMjM0NTY3OA==" + // plantedSmuggled is a session id smuggled through a CRLF injection in + // an innocuously-named header. No name-based denylist can see it. + plantedSmuggled = "SM8ggl3dC00k13Va1u3AbCdEf" + // plantedURLPassword sits in the URL userinfo. + plantedURLPassword = "hunter2-correct-horse-battery" + // plantedFragmentToken is an OAuth implicit-flow access token, which + // never reaches the server and which a query-only masker ignores. + plantedFragmentToken = "ya29.A0ARrdaM-IMPLICIT-FLOW-ACCESS-TOKEN-9f8e7d" + // plantedCurlOnly appears in ONE place in the whole fixture: a header + // option of anvil/repro.curl. CRITIQUE-02 F11: before this constant + // existed the fixture put plantedBearer in both the Authorization header + // and the curl string, so the stop-condition test passed by PROPAGATION + // from the header and read as proof that repro.curl was masked. It was + // not -- F3 reproduced a live token surviving there. A secret with no + // other route into the record is the only fixture that can prove the + // command line is masked in its own right. + plantedCurlOnly = "ghp-CURL-ONLY-0000000000000000000000000000" + // plantedCheckoutToken sits in the userinfo of anvil/target.repoUrl and + // nowhere else. `https://x-access-token:@github.com/...` is the + // standard GitHub Actions checkout URL, i.e. this is the ordinary case + // and not a contrived one. + plantedCheckoutToken = "ghs-CHECKOUT-0000000000000000000000000000" + // plantedRuntimeBasicAuth sits in the userinfo of + // anvil/target.runtimeBaseUrl and nowhere else. + plantedRuntimeBasicAuth = "runtime-basic-auth-0000000000000000" +) + +func allPlantedSecrets() map[string]string { + return map[string]string{ + "bearer token (Authorization header)": plantedBearer, + "session cookie (Cookie header crumb)": plantedCookie, + "api key (query parameter + parameters map)": plantedAPIKey, + "proxy credential (Proxy-Authorization)": plantedProxyCred, + "session id smuggled through CRLF": plantedSmuggled, + "password in URL userinfo": plantedURLPassword, + "access token in the URL fragment": plantedFragmentToken, + "bearer token reachable ONLY via repro.curl": plantedCurlOnly, + "checkout token in anvil/target.repoUrl": plantedCheckoutToken, + "basic auth in anvil/target.runtimeBaseUrl": plantedRuntimeBasicAuth, + } +} + +// --------------------------------------------------------------------------- +// Fixture +// --------------------------------------------------------------------------- + +// dastFixture builds a record that passes Validate() and that carries every +// planted secret, each by a different route into the record. The response +// body is a stack trace — the shape research/18's annotated record uses — +// carrying no secrets of its own but ECHOING two of them, which is the case +// no header rule can reach and only value propagation can. +func dastFixture() *SARIFLog { + createdAt := time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC) + sealedAt := createdAt.Add(90 * time.Second) + + body := strings.Join([]string{ + "Traceback (most recent call last):", + ` File "/srv/app/routes.py", line 94, in login`, + " user = db.authenticate(conn, username, password)", + ` File "/srv/app/db.py", line 414, in authenticate`, + " cur.execute(query)", + "sqlite3.OperationalError: unrecognized token", + "", + "-- request context echoed by the framework's debug page --", + "session=" + plantedCookie, + "api_key=" + plantedAPIKey, + "set-cookie sid=" + plantedSmuggled, + }, "\n") + + return &SARIFLog{ + Schema: SARIFSchemaURI, + Version: SARIFVersion, + Properties: AuditProperties{ + SchemaVersion: SchemaVersion, + AuditID: "0f9c2b1e-4a7d-4c33-9f21-6b8a0d5e7c14", + State: StateDastSealed, + Version: 1, + CreatedAt: createdAt, + Target: Target{ + RepoURL: "https://x-access-token:" + plantedCheckoutToken + + "@git.invalid/acme/payments.git", + Ref: "refs/heads/main", + Commit: "6f1d2c3b4a59687776655443322110ffeeddccbb", + RuntimeBaseURL: "https://scanner:" + plantedRuntimeBasicAuth + + "@staging.payments.internal", + Provenance: TargetProvenanceBootedClean, + Provisioning: TargetProvisioningEphemeralManifest, + }, + Trigger: Trigger{ + Kind: "scheduled", + PolicyID: "nightly-full", + PolicyRef: ".anvil/policy.yml@6f1d2c3", + ConfigSource: ".anvil/policy.yml", + Actor: "systemd-timer", + ResolvedAt: createdAt, + }, + Deadline: Deadline{ + DeadlineAt: createdAt.Add(DefaultClaimTimeoutSeconds * time.Second), + ClaimTimeoutSeconds: DefaultClaimTimeoutSeconds, + }, + Index: Index{ + Counts: IndexCounts{Total: 1, Dast: 1}, + ReadOrder: DefaultReadOrder(), + ByCwe: map[string][]string{"89": {"f-1"}}, + TaskCards: "cards/", + Blobs: "blobs/", + }, + DastStatus: DastStatusCompletedFindings, + }, + Runs: []Run{{ + Tool: Tool{Driver: ToolComponent{Name: "nuclei", Version: "3.4.7"}}, + AutomationDetails: RunAutomationDetails{ + ID: "anvil/dast/1", + CorrelationGUID: "0f9c2b1e-4a7d-4c33-9f21-6b8a0d5e7c14", + }, + Properties: RunProperties{ + Half: HalfDast, + Status: HalfStatusSealed, + SealedAt: &sealedAt, + DastCoverage: &DastCoverage{ + ProbedCount: 31, + InventoryUnionCount: 50, + EndpointCoverage: 31.0 / 50.0, + InventoryProvenanceMix: map[InventoryProvenance]int{ + InventoryProvenanceRuntimeSpec: 40, + InventoryProvenanceCrawl: 10, + }, + ConfirmedCount: 40, + CandidateCount: 10, + }, + RouteTableDigest: "sha256:9a1b", + RuntimeTarget: &RuntimeTarget{ + BaseURL: "https://staging.payments.internal", + AuthProfileRef: ".anvil/auth.yml@6f1d2c3", + Scope: []string{"https://staging.payments.internal/"}, + }, + }, + Results: []Result{{ + RuleID: "sqli-login", + Level: LevelError, + Message: Message{ + Text: "SQL injection at POST /api/login; the scan authenticated with api_key=" + plantedAPIKey, + }, + WebRequest: &WebRequest{ + Protocol: "HTTP", + Version: "1.1", + Target: "https://scanuser:" + plantedURLPassword + "@staging.payments.internal" + + "/api/login?api_key=" + plantedAPIKey + "&user=admin" + + "#access_token=" + plantedFragmentToken + "&state=xyz", + Method: "POST", + Headers: map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": "Bearer " + plantedBearer, + "Proxy-Authorization": "Basic " + plantedProxyCred, + "Cookie": "theme=dark; session=" + plantedCookie + "; lang=en", + "X-Trace-Id": "ok\r\nSet-Cookie: sid=" + plantedSmuggled, + "User-Agent": "anvil-dast/0.1.0", + }, + Parameters: map[string]string{ + "username": "' OR '1'='1' -- ", + "password": "irrelevant", + "api_key": plantedAPIKey, + }, + Body: &ArtifactContent{ + Text: `{"username":"' OR '1'='1' -- ","token":"` + plantedBearer + `"}`, + }, + }, + WebResponse: &WebResponse{ + Protocol: "HTTP", + Version: "1.1", + StatusCode: 500, + ReasonPhrase: "Internal Server Error", + Headers: map[string]string{ + "Content-Type": "text/html; charset=utf-8", + "Set-Cookie": "session=" + plantedCookie + "; HttpOnly", + }, + Body: &ArtifactContent{Text: body}, + }, + PartialFingerprints: map[string]string{ + PartialFingerprintAnvilFindingID: strings.Repeat("a", FingerprintDigestHexLen), + }, + Properties: ResultProperties{ + FindingID: "f-1", + Half: HalfDast, + Confidence: 0.9, + Verdict: VerdictTruePositive, + EvidenceClass: EvidenceClassDastConfirmed, + Detector: DetectorRef{ + Kind: DetectorKindDast, Model: "nuclei", Revision: "3.4.7", + }, + RemediableByAgent: true, + Reasoning: "Error-based SQL injection confirmed by a stack trace.", + Trust: TrustAssertion{Default: TrustUntrusted}, + Repro: &Repro{ + // plantedCurlOnly appears nowhere else in the record, + // so its absence after masking cannot be explained by + // propagation from a header (CRITIQUE-02 F11). + Curl: "curl -X POST -H 'Authorization: Bearer " + plantedBearer + + "' -H 'X-Session-Token: " + plantedCurlOnly + + "' https://staging.payments.internal/api/login", + InjectionPoint: ReproInjection{Kind: InjectionPointBody, Name: "username"}, + Payload: "' OR '1'='1' -- ", + ObservedSignal: ReproSignal{ + Kind: EvidenceSignalResponseStackTrace, + Match: &TrustedString{Text: "sqlite3.OperationalError", Trust: TrustUntrusted}, + }, + Env: ReproEnv{Sanitizers: []string{}, AslrEnabled: true}, + }, + }, + }}, + }}, + } +} + +func mustMarshal(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(b) +} + +// --------------------------------------------------------------------------- +// The stop-condition test +// --------------------------------------------------------------------------- + +// TestMaskRecordLeavesNoPlantedSecretAnywhere is R.8's stop condition: after +// masking, the SERIALIZED record contains zero occurrences of any planted +// value, anywhere in the output. +// +// It asserts over the marshalled whole record rather than over the fields it +// happens to know about, because the failure this guards against is a secret +// surviving in a field the author of the masker did not think of. Checking +// only webRequest.headers would pass while a live cookie sat in the response +// body two fields away. +func TestMaskRecordLeavesNoPlantedSecretAnywhere(t *testing.T) { + log := dastFixture() + + // Sanity: the fixture must actually contain what we claim, or the + // absence assertion below proves nothing. + before := mustMarshal(t, log) + for name, secret := range allPlantedSecrets() { + if !strings.Contains(before, secret) { + t.Fatalf("fixture bug: %s is not present before masking", name) + } + } + + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + + after := mustMarshal(t, log) + for name, secret := range allPlantedSecrets() { + if strings.Contains(after, secret) { + t.Errorf("%s survived masking; the serialized record still contains it", name) + } + } + if !strings.Contains(after, RedactedPlaceholder) { + t.Errorf("nothing was redacted at all") + } +} + +// TestMaskedRecordStillValidates: masking must not break the contract. A +// masker that produced an unstorable record would simply move the failure. +func TestMaskedRecordStillValidates(t *testing.T) { + log := dastFixture() + if err := log.Validate(); err != nil { + t.Fatalf("fixture does not validate before masking: %v", err) + } + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if err := log.Validate(); err != nil { + t.Errorf("record does not validate after masking: %v", err) + } +} + +// TestMaskRecordPreservesEvidence. Over-redaction is safe for secrets and +// fatal for findings: the injection payload, the endpoint, the parameter +// NAMES and the stack trace are what the coding agent patches from. +func TestMaskRecordPreservesEvidence(t *testing.T) { + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + r := &log.Runs[0].Results[0] + + if got := r.WebRequest.Parameters["username"]; got != "' OR '1'='1' -- " { + t.Errorf("the injection payload was destroyed: username = %q", got) + } + if _, ok := r.WebRequest.Parameters["api_key"]; !ok { + t.Errorf("the api_key parameter NAME was removed; the name is evidence, only the value is a secret") + } + if !strings.Contains(r.WebRequest.Target, "/api/login") { + t.Errorf("the endpoint path was lost: %q", r.WebRequest.Target) + } + if !strings.Contains(r.WebRequest.Target, "user=admin") { + t.Errorf("a non-sensitive query parameter was redacted: %q", r.WebRequest.Target) + } + if !strings.Contains(r.WebRequest.Target, "scanuser") { + t.Errorf("the userinfo USERNAME was redacted; only the password is a secret: %q", r.WebRequest.Target) + } + if !strings.Contains(r.WebResponse.Body.Text, "sqlite3.OperationalError") { + t.Errorf("the stack trace evidence was destroyed") + } + if r.WebResponse.Headers["Content-Type"] != "text/html; charset=utf-8" { + t.Errorf("a benign header was redacted: %q", r.WebResponse.Headers["Content-Type"]) + } + if r.Properties.Repro.Payload != "' OR '1'='1' -- " { + t.Errorf("the repro payload was destroyed") + } +} + +// TestPropagationReachesNonHeaderFields pins the mechanism the stop-condition +// test depends on: a secret redacted at its header is also removed from every +// OTHER string that echoes it. Without this, `MaskRecord` would redact the +// Cookie header and leave the same cookie in the 30 KB debug page beneath it. +func TestPropagationReachesNonHeaderFields(t *testing.T) { + log := dastFixture() + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + if rep.PropagatedRedactions == 0 { + t.Fatalf("propagation changed nothing; the echoed cookie and api key must have been removed") + } + r := &log.Runs[0].Results[0] + for label, field := range map[string]string{ + "webResponse.body": r.WebResponse.Body.Text, + "result.message.text": r.Message.Text, + "anvil/repro.curl": r.Properties.Repro.Curl, + "webRequest.body": r.WebRequest.Body.Text, + "webRequest.target": r.WebRequest.Target, + "webResponse.reasonPhras": r.WebResponse.ReasonPhrase, + } { + for name, secret := range allPlantedSecrets() { + if strings.Contains(field, secret) { + t.Errorf("%s still contains %s", label, name) + } + } + } +} + +// --------------------------------------------------------------------------- +// Fail-closed +// --------------------------------------------------------------------------- + +// TestFailsClosedOnUnexpectedHeaderShape. A masker that fails OPEN on a +// header it cannot parse is worse than no masker, because the redactions it +// did make imply the rest were checked. Every case here must end in a +// redaction AND an anomaly. +func TestFailsClosedOnUnexpectedHeaderShape(t *testing.T) { + cases := []struct { + name string + header string + }{ + {"space inside the field name", "X-Weird Header"}, + {"colon inside the field name", "X-Weird:Header"}, + {"empty field name", ""}, + {"non-ASCII field name", "X-Ünïcode"}, + {"control byte in the field name", "X-Tab\tName"}, + {"leading whitespace", " Authorization"}, + {"trailing whitespace", "Cookie "}, + {"comma-separated field name", "Cookie,Set-Cookie"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + log := dastFixture() + h := log.Runs[0].Results[0].WebResponse.Headers + h[tc.header] = "value-that-must-not-survive-0123456789" + + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + if got := h[tc.header]; got != RedactedPlaceholder { + t.Errorf("header %q was waved through with value %q; the masker failed OPEN", tc.header, got) + } + if !hasAnomaly(rep, "R1") { + t.Errorf("no R1 anomaly recorded for %q; a silent fail-closed is still a silent surprise", tc.header) + } + }) + } +} + +// TestFailsClosedOnCRLFInHeaderValue is the response-splitting case: the +// header NAME is innocent, and the live cookie rides in its value. A +// name-only denylist cannot see it. +func TestFailsClosedOnCRLFInHeaderValue(t *testing.T) { + log := dastFixture() + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + h := log.Runs[0].Results[0].WebRequest.Headers + if got := h["X-Trace-Id"]; got != RedactedPlaceholder { + t.Errorf("a CRLF-smuggled Set-Cookie survived in X-Trace-Id: %q", got) + } + if !hasAnomaly(rep, "R2") { + t.Errorf("no R2 anomaly recorded for the CRLF header value") + } + if strings.Contains(mustMarshal(t, log), plantedSmuggled) { + t.Errorf("the smuggled session id survived elsewhere in the record") + } +} + +// TestFailsClosedOnUnclassifiableParameterName covers rule R3. +func TestFailsClosedOnUnclassifiableParameterName(t *testing.T) { + log := dastFixture() + p := log.Runs[0].Results[0].WebRequest.Parameters + p["odd\x00name"] = "value-that-must-not-survive-0123456789" + p[""] = "another-value-that-must-not-survive" + + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + for _, name := range []string{"odd\x00name", ""} { + if got := p[name]; got != RedactedPlaceholder { + t.Errorf("parameter %q was waved through with %q", name, got) + } + } + if !hasAnomaly(rep, "R3") { + t.Errorf("no R3 anomaly recorded") + } +} + +func hasAnomaly(rep *MaskReport, rule string) bool { + for _, a := range rep.Anomalies { + if strings.HasPrefix(a.Reason, rule) { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// Denylist coverage +// --------------------------------------------------------------------------- + +// TestDenylistMatchesThePlan pins the five names and two patterns +// plan/40-record-and-storage.md R.8 specifies. If a future edit drops one, +// this fails rather than the masker silently narrowing. +func TestDenylistMatchesThePlan(t *testing.T) { + want := []string{"authorization", "cookie", "set-cookie", "proxy-authorization", "x-api-key"} + got := DenylistedHeaderNames() + for _, w := range want { + found := false + for _, g := range got { + if g == w { + found = true + } + } + if !found { + t.Errorf("the plan's denylist name %q is missing from DenylistedHeaderNames()", w) + } + } + subs := DenylistedHeaderSubstrings() + for _, w := range []string{"token", "secret"} { + found := false + for _, s := range subs { + if s == w { + found = true + } + } + if !found { + t.Errorf("the plan's %q pattern is missing from DenylistedHeaderSubstrings()", w) + } + } +} + +// TestDenylistIsCaseInsensitiveAndPatternMatched. The plan says "any header +// matching *token*/*secret* case-insensitively". +func TestDenylistIsCaseInsensitiveAndPatternMatched(t *testing.T) { + sensitive := []string{ + "AUTHORIZATION", "authorization", "AuThOrIzAtIoN", + "COOKIE", "Set-COOKIE", "PROXY-AUTHORIZATION", "X-API-KEY", + "X-Auth-Token", "x-csrf-token", "X-Amz-Security-Token", "X-Client-Secret", + "refresh-TOKEN", + } + for _, name := range sensitive { + log := dastFixture() + h := log.Runs[0].Results[0].WebResponse.Headers + h[name] = "live-value-0123456789" + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if got := h[name]; got != RedactedPlaceholder { + t.Errorf("header %q was not redacted (got %q)", name, got) + } + } + + benign := []string{"Content-Type", "Accept", "User-Agent", "Content-Length", "ETag"} + for _, name := range benign { + log := dastFixture() + h := log.Runs[0].Results[0].WebResponse.Headers + h[name] = "benign-value" + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if got := h[name]; got != "benign-value" { + t.Errorf("benign header %q was redacted (got %q); over-redaction destroys evidence", name, got) + } + } +} + +// TestKnownDenylistGaps documents, executably, what the denylist as specified +// does NOT catch. +// +// plan/40-record-and-storage.md Open Question 8 records that this list is +// "documented but not exhaustively researched" and asks for a dedicated +// security review before the masking pipeline ships. This test is that +// admission in a form that cannot rot: it asserts the CURRENT behaviour, so +// when the review widens the list, this test fails and has to be updated +// deliberately rather than the gap being rediscovered in production. +func TestKnownDenylistGaps(t *testing.T) { + gaps := []string{ + "Api-Key", // only the exact `x-api-key` is listed + "ApiKey", // + "WWW-Authenticate", // carries a challenge, sometimes a nonce + "Authentication", // not `Authorization` + "Location", // carries one-time codes and implicit-flow tokens + "X-Amz-Credential", + } + for _, name := range gaps { + if isDenylistedHeader(name) { + t.Errorf("%q is now denylisted -- good, but this test and the Open Question 8 "+ + "note in mask.go both claim it is not. Update both.", name) + } + } +} + +// TestKnownLimitationBodyOnlySecretSurvives is the other honest failure. +// +// There is no shape-based body scanner here, on purpose: a "looks like a JWT" +// regex cannot fail closed, so it manufactures exactly the false confidence +// this package exists to avoid. The consequence is that a secret which +// appears ONLY in a body, and never in a denylisted header or a +// secret-named parameter, is not removed. Asserting it makes the boundary of +// the guarantee testable instead of merely documented. +func TestKnownLimitationBodyOnlySecretSurvives(t *testing.T) { + const bodyOnly = "ghp_bodyOnlySecretThatNoHeaderRuleCanSee123456" + log := dastFixture() + log.Runs[0].Results[0].WebResponse.Body.Text += "\nleaked=" + bodyOnly + + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if !strings.Contains(log.Runs[0].Results[0].WebResponse.Body.Text, bodyOnly) { + t.Errorf("a body-only secret is now removed -- good, but mask.go's header comment " + + "states it is not. Update the comment and this test together.") + } +} + +// TestShortSecretsAreNotPropagated pins the MinPropagatedSecretLen bound: the +// header is still redacted unconditionally, but a 3-byte value is not chased +// through the rest of the record, because replacing every "abc" in a stack +// trace would destroy the evidence and protect nothing. +func TestShortSecretsAreNotPropagated(t *testing.T) { + log := dastFixture() + r := &log.Runs[0].Results[0] + r.WebRequest.Headers["Cookie"] = "sid=abc" + r.WebResponse.Body.Text = "the letters abc appear in ordinary prose" + + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if r.WebRequest.Headers["Cookie"] != RedactedPlaceholder { + t.Errorf("the short cookie was not redacted at its header; the length bound must not affect structural masking") + } + if !strings.Contains(r.WebResponse.Body.Text, "abc") { + t.Errorf("a 3-byte value was propagated into ordinary prose: %q", r.WebResponse.Body.Text) + } +} + +// --------------------------------------------------------------------------- +// Body caps and Tier-2 spill +// --------------------------------------------------------------------------- + +// TestBodyCapsAndSpill: the 8 KB / 32 KB ZAP thresholds hold, the remainder +// spills rather than being dropped, and the record says so in band. +func TestBodyCapsAndSpill(t *testing.T) { + log := dastFixture() + r := &log.Runs[0].Results[0] + r.WebRequest.Body.Text = strings.Repeat("q", MaxInlineRequestBodyBytes+5000) + r.WebResponse.Body.Text = strings.Repeat("s", MaxInlineResponseBodyBytes+9000) + + var sunk []Spill + m := &Masker{Spill: func(s Spill) (string, error) { + sunk = append(sunk, s) + return "blobs/" + s.Sha256, nil + }} + rep, err := m.Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + + if rep.BodiesTruncated != 2 { + t.Fatalf("BodiesTruncated = %d, want 2", rep.BodiesTruncated) + } + if len(sunk) != 2 { + t.Fatalf("the spill sink saw %d bodies, want 2", len(sunk)) + } + if len(r.WebRequest.Body.Text) > MaxInlineRequestBodyBytes { + t.Errorf("request body is %d bytes, over the %d cap (the truncation notice must fit INSIDE the cap)", + len(r.WebRequest.Body.Text), MaxInlineRequestBodyBytes) + } + if len(r.WebResponse.Body.Text) > MaxInlineResponseBodyBytes { + t.Errorf("response body is %d bytes, over the %d cap", + len(r.WebResponse.Body.Text), MaxInlineResponseBodyBytes) + } + for _, s := range rep.Spills { + if !strings.Contains(bodyAt(t, log, s.Pointer), s.Ref) { + t.Errorf("the record does not reference the spilled blob %s at %s", s.Ref, s.Pointer) + } + if !strings.HasPrefix(s.Ref, "blobs/") { + t.Errorf("the SpillFunc's returned reference was ignored: %q", s.Ref) + } + sum := sha256.Sum256([]byte(s.Content)) + if want := "sha256:" + hex.EncodeToString(sum[:]); s.Sha256 != want { + t.Errorf("spill digest %s does not match its content digest %s", s.Sha256, want) + } + if s.InlineBytes >= s.TotalBytes { + t.Errorf("spill claims InlineBytes=%d of TotalBytes=%d", s.InlineBytes, s.TotalBytes) + } + } +} + +func bodyAt(t *testing.T, log *SARIFLog, pointer string) string { + t.Helper() + r := &log.Runs[0].Results[0] + switch { + case strings.HasSuffix(pointer, "/webRequest/body"): + return r.WebRequest.Body.Text + case strings.HasSuffix(pointer, "/webResponse/body"): + return r.WebResponse.Body.Text + } + t.Fatalf("unexpected spill pointer %q", pointer) + return "" +} + +// TestSpilledBlobIsMaskedAndItsDigestMatches is why truncation runs LAST. +// +// A secret sitting past the 32 KB cap is invisible to any check on the inline +// prefix. If the digest were taken before propagation, the Tier-2 blob would +// be a durable, content-addressed copy of a live credential -- the exact +// outcome research/18 Risk #10 forbids, moved one tier down. +func TestSpilledBlobIsMaskedAndItsDigestMatches(t *testing.T) { + log := dastFixture() + r := &log.Runs[0].Results[0] + r.WebResponse.Body.Text = strings.Repeat("s", MaxInlineResponseBodyBytes+1000) + + "\ntrailing echo: session=" + plantedCookie + + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + if len(rep.Spills) != 1 { + t.Fatalf("got %d spills, want 1", len(rep.Spills)) + } + sp := rep.Spills[0] + if strings.Contains(sp.Content, plantedCookie) { + t.Errorf("the spilled Tier-2 blob still contains the session cookie") + } + if !strings.Contains(sp.Content, RedactedPlaceholder) { + t.Errorf("the spilled blob was never masked at all") + } + sum := sha256.Sum256([]byte(sp.Content)) + if want := "sha256:" + hex.EncodeToString(sum[:]); sp.Sha256 != want { + t.Errorf("the reference digest is not over the MASKED bytes: got %s want %s", sp.Sha256, want) + } +} + +// TestTruncationDoesNotSplitARune. +func TestTruncationDoesNotSplitARune(t *testing.T) { + log := dastFixture() + // "…" is three bytes, so some cut points land mid-rune. + body := strings.Repeat("…", MaxInlineResponseBodyBytes) + log.Runs[0].Results[0].WebResponse.Body.Text = body + + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + got := log.Runs[0].Results[0].WebResponse.Body.Text + if !utf8.ValidString(got) { + t.Errorf("truncation split a UTF-8 rune") + } + if len(got) > MaxInlineResponseBodyBytes { + t.Errorf("body is %d bytes, over the cap", len(got)) + } +} + +// TestSpillFuncErrorAborts: a Tier-2 write that fails must not be reported as +// a clean mask, or the reference in the record points at nothing. +func TestSpillFuncErrorAborts(t *testing.T) { + log := dastFixture() + log.Runs[0].Results[0].WebResponse.Body.Text = strings.Repeat("s", MaxInlineResponseBodyBytes+1) + m := &Masker{Spill: func(Spill) (string, error) { return "", errTestSpill }} + if _, err := m.Mask(log); err == nil { + t.Fatalf("Mask returned nil after the spill sink failed") + } +} + +var errTestSpill = &EnumError{Field: "test", Value: "spill", Allowed: []string{"nope"}} + +// --------------------------------------------------------------------------- +// Determinism, idempotence, post-condition +// --------------------------------------------------------------------------- + +// TestMaskRecordIsDeterministic. Map iteration order in Go is randomised; +// masking output that varies run to run cannot be golden-tested, and a masker +// nobody can pin is a masker nobody can prove. +func TestMaskRecordIsDeterministic(t *testing.T) { + var first string + var firstReport *MaskReport + for i := 0; i < 25; i++ { + log := dastFixture() + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + got := mustMarshal(t, log) + if i == 0 { + first, firstReport = got, rep + continue + } + if got != first { + t.Fatalf("iteration %d produced a different masked record", i) + } + if !reflect.DeepEqual(rep, firstReport) { + t.Fatalf("iteration %d produced a different MaskReport:\n got %+v\nwant %+v", i, rep, firstReport) + } + } +} + +// TestMaskRecordIsIdempotent. Masking is the last step of assembly, but a +// re-entrant consumer may re-assemble; masking an already-masked record must +// be a no-op rather than, say, truncating the truncation notice. +func TestMaskRecordIsIdempotent(t *testing.T) { + log := dastFixture() + log.Runs[0].Results[0].WebResponse.Body.Text = strings.Repeat("s", MaxInlineResponseBodyBytes+2000) + if err := MaskRecord(log); err != nil { + t.Fatalf("first MaskRecord: %v", err) + } + once := mustMarshal(t, log) + if err := MaskRecord(log); err != nil { + t.Fatalf("second MaskRecord: %v", err) + } + if twice := mustMarshal(t, log); twice != once { + t.Errorf("masking twice changed the record") + } +} + +// TestAssertMaskedIsTheSinkGate. plan/00-SPINE.md S7 is "enforce in code, not +// documentation": a sink can refuse an unmasked record instead of trusting +// that some earlier step remembered. +func TestAssertMaskedIsTheSinkGate(t *testing.T) { + log := dastFixture() + if err := AssertMasked(log); err == nil { + t.Fatalf("AssertMasked accepted a record with a live Authorization header") + } + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if err := AssertMasked(log); err != nil { + t.Errorf("AssertMasked rejected a masked record: %v", err) + } + + t.Run("rejects an oversized body", func(t *testing.T) { + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + log.Runs[0].Results[0].WebResponse.Body.Text = strings.Repeat("s", MaxInlineResponseBodyBytes+1) + if err := AssertMasked(log); err == nil { + t.Errorf("AssertMasked accepted a body over the 32 KB cap") + } + }) + + t.Run("rejects an unmasked sensitive parameter", func(t *testing.T) { + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + log.Runs[0].Results[0].WebRequest.Parameters["api_key"] = "put-back-0123456789" + if err := AssertMasked(log); err == nil { + t.Errorf("AssertMasked accepted a live api_key parameter") + } + }) + + t.Run("rejects a CRLF header value", func(t *testing.T) { + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + log.Runs[0].Results[0].WebResponse.Headers["X-Trace"] = "ok\r\nSet-Cookie: sid=live" + if err := AssertMasked(log); err == nil { + t.Errorf("AssertMasked accepted a CRLF-smuggled header") + } + }) +} + +// TestMaskRecordRejectsNil. +func TestMaskRecordRejectsNil(t *testing.T) { + if err := MaskRecord(nil); err == nil { + t.Errorf("MaskRecord(nil) returned nil") + } + if err := AssertMasked(nil); err == nil { + t.Errorf("AssertMasked(nil) returned nil") + } +} + +// TestMaskRecordOnARecordWithoutDASTEvidence: the SAST half has no webRequest +// or webResponse at all, and masking must be a clean no-op rather than a nil +// dereference. +func TestMaskRecordOnARecordWithoutDASTEvidence(t *testing.T) { + log := dastFixture() + r := &log.Runs[0].Results[0] + r.WebRequest, r.WebResponse = nil, nil + // The reproduction command line is HTTP evidence too, and it is masked + // even when the request/response pair is gone -- see + // TestReproCurlIsMaskedFromItsOwnEvidence. Drop it so this test is about + // the no-evidence case it claims to be about. + r.Properties.Repro = nil + r.Properties.Trust.Default = TrustUntrusted + + rep, err := (&Masker{}).Mask(log) + if err != nil { + t.Fatalf("Mask: %v", err) + } + if rep.HeadersRedacted != 0 || rep.BodiesTruncated != 0 { + t.Errorf("masked something on a record with no HTTP evidence: %+v", rep) + } + if err := AssertMasked(log); err != nil { + t.Errorf("AssertMasked: %v", err) + } +} + +// --------------------------------------------------------------------------- +// URL handling +// --------------------------------------------------------------------------- + +func TestMaskURL(t *testing.T) { + cases := []struct { + name string + in string + wantContains []string + wantAbsent []string + }{ + { + name: "query api key", + in: "https://h.invalid/p?api_key=LIVEKEY0123456789&user=admin", + wantContains: []string{"api_key=" + RedactedPlaceholder, "user=admin"}, + wantAbsent: []string{"LIVEKEY0123456789"}, + }, + { + name: "implicit-flow token in the fragment", + in: "https://h.invalid/cb#access_token=LIVETOKEN0123456789&state=xyz", + wantContains: []string{"access_token=" + RedactedPlaceholder, "state=xyz"}, + wantAbsent: []string{"LIVETOKEN0123456789"}, + }, + { + name: "userinfo password", + in: "https://alice:LIVEPASS0123456789@h.invalid/p", + wantContains: []string{"alice:" + RedactedPlaceholder + "@h.invalid"}, + wantAbsent: []string{"LIVEPASS0123456789"}, + }, + { + name: "percent-encoded payload is preserved byte for byte", + in: "https://h.invalid/p?q=%27%20OR%201%3D1&token=LIVE0123456789", + wantContains: []string{"q=%27%20OR%201%3D1", "token=" + RedactedPlaceholder}, + wantAbsent: []string{"LIVE0123456789"}, + }, + { + name: "plain anchor fragment is untouched", + in: "https://h.invalid/p#section-2", + wantContains: []string{"#section-2"}, + }, + { + name: "bare flag with no value", + in: "https://h.invalid/p?debug&token=LIVE0123456789", + wantContains: []string{"debug", "token=" + RedactedPlaceholder}, + wantAbsent: []string{"LIVE0123456789"}, + }, + { + name: "no query at all", + in: "https://h.invalid/api/login", + wantContains: []string{"https://h.invalid/api/login"}, + wantAbsent: []string{"?", "#"}, + }, + { + // A '?' that belongs to the fragment must not be promoted into a + // query delimiter when the URL is rebuilt. + name: "question mark inside the fragment", + in: "https://h.invalid/p#a?b=c", + wantContains: []string{"https://h.invalid/p#a?b=c"}, + wantAbsent: []string{"p?#"}, + }, + { + name: "empty query is preserved as empty, not dropped", + in: "https://h.invalid/p?", + wantContains: []string{"https://h.invalid/p?"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := &Masker{} + got := m.maskURL("/t", tc.in, newSecretSet(MinPropagatedSecretLen), &MaskReport{}) + for _, want := range tc.wantContains { + if !strings.Contains(got, want) { + t.Errorf("maskURL(%q) = %q, want it to contain %q", tc.in, got, want) + } + } + for _, bad := range tc.wantAbsent { + if strings.Contains(got, bad) { + t.Errorf("maskURL(%q) = %q, must not contain %q", tc.in, got, bad) + } + } + }) + } +} + +// TestTruncateToRuneBoundary. +func TestTruncateToRuneBoundary(t *testing.T) { + const s = "abc…def" // 'e' is 3 bytes: a b c e0 80 a6 d e f + for n := 0; n <= len(s); n++ { + got := truncateToRuneBoundary(s, n) + if len(got) > n { + t.Errorf("truncateToRuneBoundary(%q, %d) = %q, longer than n", s, n, got) + } + if !utf8.ValidString(got) { + t.Errorf("truncateToRuneBoundary(%q, %d) = %q, not valid UTF-8", s, n, got) + } + if !strings.HasPrefix(s, got) { + t.Errorf("truncateToRuneBoundary(%q, %d) = %q, not a prefix", s, n, got) + } + } +} + +// TestIsHTTPFieldName. +func TestIsHTTPFieldName(t *testing.T) { + valid := []string{"Authorization", "X-Api-Key", "a", "X_Custom", "X.Y", "a1!#$%&'*+-.^_`|~"} + for _, s := range valid { + if !isHTTPFieldName(s) { + t.Errorf("isHTTPFieldName(%q) = false, want true", s) + } + } + invalid := []string{"", " ", "a b", "a:b", "a,b", "a\tb", "a\nb", "ü", "a(b)", "a/b", "a@b", `a"b`} + for _, s := range invalid { + if isHTTPFieldName(s) { + t.Errorf("isHTTPFieldName(%q) = true, want false", s) + } + } +} + +// TestJSONPointerEscape covers RFC 6901 §3. +func TestJSONPointerEscape(t *testing.T) { + if got := jsonPointerEscape("a/b~c"); got != "a~1b~0c" { + t.Errorf("jsonPointerEscape(%q) = %q, want %q", "a/b~c", got, "a~1b~0c") + } + if got := jsonPointerEscape("~/"); got != "~0~1" { + t.Errorf("jsonPointerEscape(%q) = %q, want %q", "~/", got, "~0~1") + } +} + +// --------------------------------------------------------------------------- +// Regression guards for CRITIQUE-02 (R.10 critic gate 2). +// --------------------------------------------------------------------------- + +// probeSecret is a credential planted by the tests below into ONE field at a +// time. It is long enough to be propagated and contains no character +// encoding/json escapes. +const probeSecret = "PROBE-LIVE-CREDENTIAL-0123456789abcdef" + +// minimalLog is a record with no HTTP evidence at all: no webRequest, no +// webResponse, no runtime target, no credential in any URL. It is the fixture +// that makes a single-field assertion honest — whatever it proves cannot be +// explained by masking or propagation from somewhere else. +func minimalLog() *SARIFLog { + l := dastFixture() + l.Properties.Target.RepoURL = "https://git.invalid/acme/payments" + l.Properties.Target.RuntimeBaseURL = "" + l.Runs[0].Properties.RuntimeTarget = nil + r := &l.Runs[0].Results[0] + r.Message.Text = "SQL injection at POST /api/login" + r.WebRequest, r.WebResponse = nil, nil + r.Properties.Repro = nil + return l +} + +// TestReproCurlIsMaskedFromItsOwnEvidence is CRITIQUE-02 F3(i), reproduced and +// then closed. +// +// The credential is planted ONLY in anvil/repro.curl. There is no +// Authorization header anywhere in this record, so pass 2 propagation has +// nothing to propagate FROM: if the token is gone afterwards, pass 1 looked at +// the curl string itself. That is the assertion the flagship test could not +// make while the fixture put the same value in both places (F11). +func TestReproCurlIsMaskedFromItsOwnEvidence(t *testing.T) { + log := minimalLog() + log.Runs[0].Results[0].Properties.Repro = &Repro{ + Curl: "curl -X POST -H 'Authorization: Bearer " + probeSecret + + "' https://staging.payments.internal/api/login", + InjectionPoint: ReproInjection{Kind: InjectionPointBody, Name: "username"}, + Payload: "' OR '1'='1' -- ", + ObservedSignal: ReproSignal{ + Kind: EvidenceSignalResponseStackTrace, + Match: &TrustedString{Text: "sqlite3.OperationalError", Trust: TrustUntrusted}, + }, + Env: ReproEnv{Sanitizers: []string{}, AslrEnabled: true}, + } + + before := mustMarshal(t, log) + if n := strings.Count(before, probeSecret); n != 1 { + t.Fatalf("fixture bug: the probe appears %d times, want exactly 1 (in repro.curl only)", n) + } + // The gate must refuse it BEFORE masking, or the gate is decorative. + if err := AssertMasked(log); err == nil { + t.Error("AssertMasked accepted a record whose repro.curl carries a live bearer token") + } + + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + curl := log.Runs[0].Results[0].Properties.Repro.Curl + if strings.Contains(curl, probeSecret) { + t.Errorf("anvil/repro.curl still carries the token after masking: %q", curl) + } + if strings.Contains(mustMarshal(t, log), probeSecret) { + t.Errorf("the token survives somewhere in the record") + } + if !strings.Contains(curl, RedactedPlaceholder) { + t.Errorf("the curl command was not redacted at all: %q", curl) + } + // Over-redaction is fatal for a reproduction: the command must still be + // replayable once an operator supplies their own credential. + for _, keep := range []string{"curl", "-X POST", "Authorization:", "/api/login"} { + if !strings.Contains(curl, keep) { + t.Errorf("masking destroyed the reproduction: %q is gone from %q", keep, curl) + } + } + if err := AssertMasked(log); err != nil { + t.Errorf("AssertMasked rejected the masked record: %v", err) + } +} + +// TestReproCurlCredentialCarriers walks the option shapes a curl reproduction +// actually uses. Each case plants the probe in one option and in nothing else. +func TestReproCurlCredentialCarriers(t *testing.T) { + const sq = "'" + const dq = "\"" + cases := []struct { + name string + curl string + keep []string // evidence that must survive + }{ + {"short header, quoted", "curl -H " + sq + "Authorization: Bearer " + probeSecret + sq + " https://h.invalid/p", []string{"Authorization:"}}, + {"long header", "curl --header " + dq + "X-Api-Key: " + probeSecret + dq + " https://h.invalid/p", []string{"X-Api-Key:"}}, + {"long header with =", "curl --header=" + dq + "X-Auth-Token: " + probeSecret + dq + " https://h.invalid/p", []string{"X-Auth-Token:"}}, + {"attached short header", "curl -HAuthorization:Bearer" + probeSecret + " https://h.invalid/p", []string{"Authorization:"}}, + {"clustered short option", "curl -sSH " + sq + "Authorization: Bearer " + probeSecret + sq + " https://h.invalid/p", []string{"-sSH", "Authorization:"}}, + {"proxy header", "curl --proxy-header " + sq + "Proxy-Authorization: Basic " + probeSecret + sq + " https://h.invalid/p", []string{"Proxy-Authorization:"}}, + {"cookie string", "curl -b " + sq + "session=" + probeSecret + "; theme=dark" + sq + " https://h.invalid/p", []string{"curl"}}, + {"user credentials", "curl -u scanuser:" + probeSecret + " https://h.invalid/p", []string{"scanuser:"}}, + {"long user credentials", "curl --user=scanuser:" + probeSecret + " https://h.invalid/p", []string{"scanuser:"}}, + {"data field", "curl -d " + sq + "user=admin&api_key=" + probeSecret + sq + " https://h.invalid/p", []string{"user=admin"}}, + {"url query", "curl https://h.invalid/p?api_key=" + probeSecret + "&user=admin", []string{"user=admin", "/p?"}}, + {"url userinfo", "curl https://scanuser:" + probeSecret + "@h.invalid/p", []string{"scanuser:"}}, + {"unparseable header argument", "curl -H " + sq + "not-a-header-" + probeSecret + sq + " https://h.invalid/p", []string{"curl"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := &Masker{} + got := m.maskCommandLine("/t", tc.curl, newSecretSet(MinPropagatedSecretLen), &MaskReport{}) + if strings.Contains(got, probeSecret) { + t.Errorf("maskCommandLine left the credential live:\n in: %q\n out: %q", tc.curl, got) + } + if !strings.Contains(got, RedactedPlaceholder) { + t.Errorf("nothing was redacted:\n in: %q\n out: %q", tc.curl, got) + } + for _, keep := range tc.keep { + if !strings.Contains(got, keep) { + t.Errorf("evidence %q was destroyed: %q", keep, got) + } + } + // Idempotent, which is what lets AssertMasked re-derive and compare. + if again := m.maskCommandLine("/t", got, newSecretSet(MinPropagatedSecretLen), &MaskReport{}); again != got { + t.Errorf("maskCommandLine is not idempotent:\n once: %q\n twice: %q", got, again) + } + // And the gate agrees with the masker in both directions. + if err := assertCommandLineMasked("/t", tc.curl); err == nil { + t.Errorf("assertCommandLineMasked accepted the unmasked command %q", tc.curl) + } + if err := assertCommandLineMasked("/t", got); err != nil { + t.Errorf("assertCommandLineMasked rejected the masked command %q: %v", got, err) + } + }) + } +} + +// TestReproCurlLeavesBenignCommandsAlone. Over-redaction destroys the +// reproduction, so a command with no credential in it must come out byte for +// byte identical. +func TestReproCurlLeavesBenignCommandsAlone(t *testing.T) { + for _, cmd := range []string{ + "curl -X POST -H 'Content-Type: application/json' -d 'user=admin&q=%27%20OR%201%3D1' https://h.invalid/api/login", + "curl -sS --compressed https://h.invalid/health", + "curl -b cookies.txt https://h.invalid/p", + "curl -u scanuser https://h.invalid/p", + "", + "curl", + } { + if got := (&Masker{}).maskCommandLine("/t", cmd, newSecretSet(MinPropagatedSecretLen), &MaskReport{}); got != cmd { + t.Errorf("a benign command was rewritten:\n in: %q\n out: %q", cmd, got) + } + if err := assertCommandLineMasked("/t", cmd); err != nil { + t.Errorf("assertCommandLineMasked rejected a benign command %q: %v", cmd, err) + } + } +} + +// TestTargetURLCredentialsAreMasked is CRITIQUE-02 F3(ii). The credential is +// planted only in anvil/target.repoUrl and anvil/target.runtimeBaseUrl, on a +// record with no HTTP evidence at all, so nothing else can account for its +// disappearance. +func TestTargetURLCredentialsAreMasked(t *testing.T) { + log := minimalLog() + log.Properties.Target.RepoURL = "https://x-access-token:" + probeSecret + "@github.com/org/repo.git" + log.Properties.Target.RuntimeBaseURL = "https://svc:" + probeSecret + "@staging.internal" + + if err := AssertMasked(log); err == nil { + t.Error("AssertMasked accepted a record whose repoUrl carries a live checkout token") + } + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if got := log.Properties.Target.RepoURL; strings.Contains(got, probeSecret) { + t.Errorf("anvil/target.repoUrl still carries a live credential: %q", got) + } + if got := log.Properties.Target.RuntimeBaseURL; strings.Contains(got, probeSecret) { + t.Errorf("anvil/target.runtimeBaseUrl still carries a live credential: %q", got) + } + if strings.Contains(mustMarshal(t, log), probeSecret) { + t.Error("the credential survives somewhere in the record") + } + // The repository identity is evidence and must survive. + if !strings.Contains(log.Properties.Target.RepoURL, "github.com/org/repo.git") { + t.Errorf("masking destroyed the repository identity: %q", log.Properties.Target.RepoURL) + } + if err := AssertMasked(log); err != nil { + t.Errorf("AssertMasked rejected the masked record: %v", err) + } +} + +// TestAssertMaskedRejectsAnUnmaskedTargetURL is CRITIQUE-02 F4, stated as the +// sub-case TestAssertMaskedIsTheSinkGate was missing. Mask masks +// webRequest.target; before this fix AssertMasked did not look at it at all, +// so a record whose only credential was in a URL passed the gate that exists +// to catch precisely that. +func TestAssertMaskedRejectsAnUnmaskedTargetURL(t *testing.T) { + for _, where := range []struct{ name, target string }{ + {"query", "https://app.invalid/v1/orders?api_key=" + probeSecret}, + {"fragment", "https://app.invalid/cb#access_token=" + probeSecret}, + {"userinfo", "https://alice:" + probeSecret + "@app.invalid/v1/orders"}, + } { + t.Run(where.name, func(t *testing.T) { + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if err := AssertMasked(log); err != nil { + t.Fatalf("the masked fixture does not pass the gate: %v", err) + } + // Put a credential back the way a record that skipped masking + // would arrive at the sink. + log.Runs[0].Results[0].WebRequest.Target = where.target + if err := AssertMasked(log); err == nil { + t.Errorf("AssertMasked accepted a webRequest.target carrying a live credential in the %s: %q", + where.name, where.target) + } + // And Mask does clean it, which is what makes the gate's silence + // a divergence rather than a shared limitation. + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + if got := log.Runs[0].Results[0].WebRequest.Target; strings.Contains(got, probeSecret) { + t.Errorf("Mask left the %s credential live: %q", where.name, got) + } + }) + } +} + +// maskSite is one place walkMaskSurface visits, with a way to plant a live +// credential in it and to put the original value back. +type maskSite struct { + pointer string + kind string + plant func() + restore func() +} + +// collectMaskSites enumerates every site of every kind in l. It is the whole +// mask surface, gathered by the same walker Mask and AssertMasked use, so a +// site added to the walker is automatically covered by the test below. +func collectMaskSites(l *SARIFLog) []maskSite { + var sites []maskSite + walkMaskSurface(l, surface{ + Headers: func(ptr string, h map[string]string) { + if h == nil { + return + } + const name = "Authorization" + old, had := h[name] + sites = append(sites, maskSite{ptr, "headers", + func() { h[name] = "Bearer " + probeSecret }, + func() { + if had { + h[name] = old + } else { + delete(h, name) + } + }}) + }, + Parameters: func(ptr string, p map[string]string) { + if p == nil { + return + } + const name = "api_key" + old, had := p[name] + sites = append(sites, maskSite{ptr, "parameters", + func() { p[name] = probeSecret }, + func() { + if had { + p[name] = old + } else { + delete(p, name) + } + }}) + }, + URL: func(ptr string, p *string) { + old := *p + sites = append(sites, maskSite{ptr, "url", + func() { *p = "https://h.invalid/p?api_key=" + probeSecret }, + func() { *p = old }}) + }, + CommandLine: func(ptr string, p *string) { + old := *p + sites = append(sites, maskSite{ptr, "commandLine", + func() { + *p = "curl -H 'Authorization: Bearer " + probeSecret + "' https://h.invalid/p" + }, + func() { *p = old }}) + }, + Body: func(ptr string, b *ArtifactContent, limit int) { + if b == nil { + return + } + old := b.Text + sites = append(sites, maskSite{ptr, "body", + func() { b.Text = strings.Repeat("s", limit+1) }, + func() { b.Text = old }}) + }, + }) + return sites +} + +// TestAssertMaskedCoversEverySiteMaskCovers is the structural guard M1 asks +// for: for EVERY site the mask surface exposes, planting a live credential +// there must make AssertMasked refuse the record, and MaskRecord must then +// remove it. +// +// It is written against the walker rather than against a hand-written list of +// fields precisely so it cannot go stale. A future step that teaches Mask +// about a new field adds it to walkMaskSurface; this test then demands the +// sink gate cover it too, and fails if it does not. That is the property +// CRITIQUE-02 F4 found missing — "a gate weaker than the thing it guards is +// worse than none". +func TestAssertMaskedCoversEverySiteMaskCovers(t *testing.T) { + reference := dastFixture() + if err := MaskRecord(reference); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + n := len(collectMaskSites(reference)) + if n < 8 { + t.Fatalf("the mask surface exposes only %d sites; the fixture is not exercising it", n) + } + kinds := map[string]bool{} + + for i := 0; i < n; i++ { + // A fresh masked record per site, so one site's planted credential + // cannot be scrubbed out of the next by propagation. + log := dastFixture() + if err := MaskRecord(log); err != nil { + t.Fatalf("MaskRecord: %v", err) + } + sites := collectMaskSites(log) + if len(sites) != n { + t.Fatalf("the mask surface is not deterministic: %d sites, want %d", len(sites), n) + } + site := sites[i] + kinds[site.kind] = true + + if err := AssertMasked(log); err != nil { + t.Fatalf("%s: the masked fixture does not pass the gate: %v", site.pointer, err) + } + site.plant() + if err := AssertMasked(log); err == nil { + t.Errorf("AssertMasked accepted a live credential at %s (%s); "+ + "the sink gate is weaker than Mask", site.pointer, site.kind) + } + if err := MaskRecord(log); err != nil { + t.Fatalf("%s: MaskRecord: %v", site.pointer, err) + } + if site.kind != "body" && strings.Contains(mustMarshal(t, log), probeSecret) { + t.Errorf("Mask left the credential planted at %s live", site.pointer) + } + if err := AssertMasked(log); err != nil { + t.Errorf("%s: AssertMasked rejected the re-masked record: %v", site.pointer, err) + } + site.restore() + } + + for _, kind := range []string{"headers", "parameters", "url", "commandLine", "body"} { + if !kinds[kind] { + t.Errorf("the fixture exercises no %q site, so that kind is unproven", kind) + } + } +} + +// TestMaskSurfaceCoversTheNamedCredentialCarriers pins the specific fields +// CRITIQUE-02 named, by POINTER, so that deleting one from walkMaskSurface is +// a test failure rather than a silent regression to F3. +func TestMaskSurfaceCoversTheNamedCredentialCarriers(t *testing.T) { + seen := map[string]bool{} + walkMaskSurface(dastFixture(), surface{ + Headers: func(ptr string, _ map[string]string) { seen[ptr] = true }, + Parameters: func(ptr string, _ map[string]string) { seen[ptr] = true }, + URL: func(ptr string, _ *string) { seen[ptr] = true }, + CommandLine: func(ptr string, _ *string) { seen[ptr] = true }, + Body: func(ptr string, _ *ArtifactContent, _ int) { seen[ptr] = true }, + }) + for _, ptr := range []string{ + "/properties/anvil~1target/repoUrl", + "/properties/anvil~1target/runtimeBaseUrl", + "/runs/0/properties/anvil~1runtimeTarget/baseUrl", + "/runs/0/results/0/webRequest/headers", + "/runs/0/results/0/webRequest/parameters", + "/runs/0/results/0/webRequest/target", + "/runs/0/results/0/webRequest/body", + "/runs/0/results/0/webResponse/headers", + "/runs/0/results/0/webResponse/body", + "/runs/0/results/0/properties/anvil~1repro/curl", + } { + if !seen[ptr] { + t.Errorf("%s is not on the mask surface; CRITIQUE-02 F3 named it as a live-credential carrier", ptr) + } + } +} diff --git a/internal/record/sealing.go b/internal/record/sealing.go new file mode 100644 index 0000000..d86f7b5 --- /dev/null +++ b/internal/record/sealing.go @@ -0,0 +1,1053 @@ +// Per-half sealing: the state machine behind plan/00-SPINE.md S1's "one audit +// identity, two independently-sealed halves, a re-entrant consumer" (step +// R.6). +// +// # What this file owns +// +// Three things that plan/40-record-and-storage.md keeps deliberately apart and +// that every previous draft of this design conflated: +// +// 1. The PER-HALF SEAL — `run.properties["anvil/status"]` (a HalfStatus) and +// `run.properties["anvil/sealedAt"]`, stored as `audit_record.sast_status` +// / `.sast_sealed_at` / `.dast_status` / `.dast_sealed_at`. +// 2. The AUDIT-LEVEL LIFECYCLE — `sarifLog.properties["anvil/state"]` (a +// State), stored as `audit_record.state`, DERIVED from the two half +// seals and never written independently of them. +// 3. The CLAIM CLOCK — `anvil/deadline.deadlineAt`, stored as +// `audit_record.deadline_at`. It is `scan_run.started_at + +// claim_timeout_seconds`, computed ONCE in BeginAudit and never +// recomputed by anything in this file. +// +// R.6's forbidden actions name the conflation this file must not commit: +// "Do not conflate `anvil/sealedAt` (per-half completion) with +// `anvil/deadline.deadlineAt` (the claim-timeout clock) — they are +// independent clocks with independent semantics." Sealing a half a week late +// does not move DeadlineAt by one nanosecond; TestDeadlineUnchangedByLateSeal +// asserts exactly that. +// +// # `sealed` is the hard read gate +// +// plan/IMPLEMENTATION-PLAN.md §6 ruling G5: "`sealed` is load-bearing: R.6 +// makes it the hard read gate ('do not allow a consumer to read a half's +// results before that half's `status` equals `sealed`'), so O.2 keying +// transitions on `complete` means the gate never opens." Area O's `complete` +// is struck; HalfStatusSealed — and no other token, not HalfStatusFailed, not +// HalfStatusTimedOut, not HalfStatusSkipped — opens ReadHalf. +// +// # Terminal is not the same as readable +// +// The distinction that makes a DAST-disabled audit work. FOUR half statuses +// are TERMINAL (sealed, failed, timed_out, skipped): the half will produce +// nothing further, so the audit-level State may advance. Exactly ONE of them +// is READABLE (sealed): the consumer may look at that half's results. +// +// So a Tier S install with no `anvil-dast` artifact (plan/00-SPINE.md +// S9-AMENDED) reaches StateBothSealed — its DAST half is terminally +// HalfStatusSkipped with DastStatusNotRun — while `dastReady` stays false +// forever, because there are no DAST results to read. Collapsing the two +// notions either wedges every SAST-only audit in StateSastSealed (the +// consumer never runs) or opens the gate onto a half that never ran (the +// consumer reads emptiness as "scanned clean", which is research/23 Risk #1). +// +// # This file holds no database handle +// +// internal/store's tests import internal/record to prove the SQL CHECK +// vocabularies and the Go enums have not drifted; importing internal/store +// back would be an import cycle. So this is an in-memory model of the +// `audit_record` sealing columns, and the store writer projects an AuditSeal +// onto those columns. Every literal it produces comes from contract.go, which +// is the same source ddl_test.go checks the SQL against. +// +// (Free-floating file comment: contract.go carries the package doc.) + +package record + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// --------------------------------------------------------------------------- +// Errors — every refusal is typed, per R.6's stop condition +// --------------------------------------------------------------------------- + +// Sentinel causes. Every error this file returns wraps exactly one of these, +// so callers can branch with errors.Is while still recovering the full +// context (which audit, which half, which status) with errors.As. +var ( + // ErrUnknownAudit: no BeginAudit has been called for this audit id. + ErrUnknownAudit = errors.New("record: unknown audit") + + // ErrDuplicateAudit: BeginAudit was called twice for one audit id. + // Re-beginning would recompute DeadlineAt, which R.6 forbids. + ErrDuplicateAudit = errors.New("record: audit already begun") + + // ErrInvalidAuditConfig: the AuditConfig cannot produce a legal + // `audit_record` row (empty id, zero start, non-positive timeout). + ErrInvalidAuditConfig = errors.New("record: invalid audit configuration") + + // ErrNotSealable: the status handed to SealHalf is not terminal. + // HalfStatusRunning is the value this catches: "still running" is not a + // seal. + ErrNotSealable = errors.New("record: status is not a terminal half status") + + // ErrHalfAlreadySealed: the half already reached a terminal status and a + // DIFFERENT one was offered. A half seals once. + ErrHalfAlreadySealed = errors.New("record: half already sealed") + + // ErrAuditTerminal: the audit is consumed or expired; its halves no + // longer accept seals. + ErrAuditTerminal = errors.New("record: audit is consumed or expired") + + // ErrHalfNotSealed: THE READ GATE. A consumer asked for a half whose + // status is not HalfStatusSealed, or whose audit has expired. + ErrHalfNotSealed = errors.New("record: half is not sealed; consumer read refused") + + // ErrNotBothSealed: Consume was called before both halves reached a + // terminal status. + ErrNotBothSealed = errors.New("record: audit has not reached both_sealed") +) + +// SealingError reports a refused sealing or lifecycle transition. It names +// the audit, the half, and the observed state so the message identifies the +// offending caller rather than merely the offence — the same reasoning that +// makes EnumError list every legal literal. +type SealingError struct { + Op string // "BeginAudit" | "SealHalf" | "Consume" | ... + AuditID string // anvil/auditId + Half Half // empty when the operation is not half-scoped + State State // the audit's anvil/state at the moment of refusal + Status HalfStatus // the half's anvil/status at the moment of refusal + Reason string // human-readable specifics + Err error // one of the sentinels above, or an *EnumError +} + +func (e *SealingError) Error() string { + msg := fmt.Sprintf("record: %s(%q)", e.Op, e.AuditID) + if e.Half != "" { + msg += fmt.Sprintf(" half=%s", e.Half) + } + if e.State != "" { + msg += fmt.Sprintf(" state=%s", e.State) + } + if e.Status != "" { + msg += fmt.Sprintf(" status=%s", e.Status) + } + return msg + ": " + e.Reason +} + +// Unwrap exposes the sentinel cause (or the underlying *EnumError) to +// errors.Is and errors.As. +func (e *SealingError) Unwrap() error { return e.Err } + +// ReadGateError is the refusal a consumer gets when it reaches for a half +// that has not sealed. R.6's stop condition: "a consumer attempting to read +// an unsealed half is rejected with a typed error, not a partial/zero-value +// result." +// +// ReadHalf returns the zero HalfSeal alongside this error, as Go requires; +// the contract is that the zero value carries no information and must not be +// inspected when err != nil. +type ReadGateError struct { + AuditID string // anvil/auditId + Half Half // the half the consumer reached for + Status HalfStatus // that half's actual anvil/status + State State // the audit's anvil/state + Reason string +} + +func (e *ReadGateError) Error() string { + return fmt.Sprintf( + "record: read of %s half of audit %q refused: status is %q, state is %q; %s (the gate opens only at anvil/status=%q)", + e.Half, e.AuditID, e.Status, e.State, e.Reason, HalfStatusSealed) +} + +// Unwrap makes every read refusal match errors.Is(err, ErrHalfNotSealed). +func (e *ReadGateError) Unwrap() error { return ErrHalfNotSealed } + +// --------------------------------------------------------------------------- +// Half-status classification +// --------------------------------------------------------------------------- + +// TerminalHalfStatuses returns the four HalfStatus values that mean the half +// will produce nothing further. Only these may be handed to SealHalf, and +// only when BOTH halves hold one does the audit reach StateBothSealed. +// +// HalfStatusRunning is deliberately absent: it is the one non-terminal value +// and the reason SealHalf can refuse a caller that mistakes "started" for +// "sealed". +func TerminalHalfStatuses() []HalfStatus { + return []HalfStatus{ + HalfStatusSealed, HalfStatusFailed, HalfStatusTimedOut, HalfStatusSkipped, + } +} + +// IsTerminalHalfStatus reports whether s means the half is finished, in any +// sense — cleanly sealed, broken, out of clock, or never run. +func IsTerminalHalfStatus(s HalfStatus) bool { return inEnum(s, TerminalHalfStatuses()) } + +// IsReadableHalfStatus reports whether a consumer may read the half's +// results. It is true for HalfStatusSealed and for nothing else. +// +// Written as a named predicate on purpose: `if status != HalfStatusRunning` +// and `if IsTerminalHalfStatus(status)` are both wrong here and both look +// plausible at a glance. A failed half is not a clean half; a skipped half +// has no results at all. +func IsReadableHalfStatus(s HalfStatus) bool { return s == HalfStatusSealed } + +// --------------------------------------------------------------------------- +// Value types +// --------------------------------------------------------------------------- + +// HalfSeal is one half's seal: `run.properties["anvil/status"]` and +// `run.properties["anvil/sealedAt"]`, i.e. `audit_record.sast_status` + +// `.sast_sealed_at` (or the DAST pair). +type HalfSeal struct { + // Half is HalfSast or HalfDast. + Half Half + + // Status is the per-half anvil/status. + Status HalfStatus + + // SealedAt is non-nil ONLY when Status == HalfStatusSealed. contract.go: + // SealedAt "is required once Status == HalfStatusSealed, and is + // explicitly null otherwise (not omitted — a missing key and an unsealed + // half must not be the same observation)". A failed, timed-out or + // skipped half therefore has a nil SealedAt, which is what makes + // `audit_record.sast_sealed_at IS NULL` mean "never cleanly sealed" + // rather than "we forgot to write it". + // + // This is NOT the claim clock. See AuditSeal.DeadlineAt. + SealedAt *time.Time + + // AuditState is the anvil/state of the audit this half belongs to, as it + // stood when the seal was observed. It is carried so that Readable() can + // answer the WHOLE read-gate question rather than half of it. + // + // CRITIQUE-02 F6: ReadHalf refuses an expired audit and Inspect handed out + // the same HalfSeal values with no state check at all, so + // Inspect(...).Sast.Readable() said true on an audit ReadHalf refused. + // Readable() is exported and is what a caller branches on; two exported + // readiness paths giving two answers is a gate that is only advisory. + // + // A hand-constructed HalfSeal leaves this empty, which is not StateExpired + // and therefore does not silently suppress a real seal — the zero value + // means "no audit context", and only an audit context can withdraw + // readability. + AuditState State +} + +// Readable reports whether a consumer may read this half's results. +// +// It is the same predicate ReadHalf enforces, in both arms: the half's status +// must be exactly HalfStatusSealed AND the audit must not have expired, whose +// payload the reaper has dropped. TestInspectAgreesWithReadHalfOnEveryState +// asserts the two never disagree for any (state, status) pair. +func (h HalfSeal) Readable() bool { + return IsReadableHalfStatus(h.Status) && h.AuditState != StateExpired +} + +// DastOutcome is what the DAST half (or its absence) reports, and the sole +// input from which the audit-level DastStatus is derived. +// +// contract.go: DastStatus "is DERIVED from the DAST half's HalfStatus and +// from TargetProvenance (the boot/reachability outcome), never from +// TargetProvisioning (which provisioning path was used)". This struct carries +// exactly those inputs and nothing that would let a caller state the derived +// value directly. +type DastOutcome struct { + // TierInstalled is plan/00-SPINE.md S9-AMENDED's split: `anvil` ships + // with no network-probing capability compiled in and `anvil-dast` is a + // separately installed artifact. False here is the common case and is + // the ONLY route to DastStatusNotRun. + // + // The Sealer overwrites whatever a caller puts here with the audit's own + // AuditConfig.DastEnabled, so a DAST outcome cannot claim a tier the + // audit was not started with. + TierInstalled bool + + // Provenance is the target's boot/reachability outcome, from the target + // lifecycle harness (area D). Required when TierInstalled; ignored + // otherwise. + Provenance TargetProvenance + + // FindingCount is the number of DAST results in the sealed half. It + // separates DastStatusCompletedFindings from DastStatusCompletedClean — + // and it is only ever consulted once the provenance checks have already + // ruled out "we never actually scanned the target". + FindingCount int + + // PartialCoverage is true when the half probed only part of the + // discovered attack surface, yielding DastStatusCompletedPartial. The + // numerator/denominator detail lives in DastCoverage; this is the bit + // that keeps a 3-of-50 scan from reporting DastStatusCompletedClean. + PartialCoverage bool +} + +// AuditConfig is the input to BeginAudit: everything needed to fix the claim +// clock and to decide whether this installation has a DAST half at all. +type AuditConfig struct { + // AuditID is `anvil/auditId`, assigned once at scan start. Required. + AuditID string + + // StartedAt is `scan_run.started_at`. DeadlineAt is computed from THIS + // and from nothing else. Required, and must be non-zero: a zero start + // would silently anchor the claim clock to the year 1. + StartedAt time.Time + + // ClaimTimeoutSeconds is `audit_record.claim_timeout_seconds`. Zero + // means DefaultClaimTimeoutSeconds (8h); negative is rejected, matching + // the schema's ck_audit_record_claim_timeout_positive. + // + // It is a CLAIM timeout — how long an unclaimed finding stays eligible — + // not a deletion policy and not a confidentiality control + // (plan/00-SPINE.md S1 correction #5). + ClaimTimeoutSeconds int + + // DastEnabled is false in the core `anvil` distribution artifact + // (plan/00-SPINE.md S9-AMENDED). When false, BeginAudit immediately and + // terminally seals the DAST half as HalfStatusSkipped / + // DastStatusNotRun, so the audit can reach StateBothSealed with no DAST + // worker in the process to seal it. Without that, every SAST-only audit + // would wedge in StateSastSealed and the consumer would never run. + DastEnabled bool + + // DastDeadlineSeconds is `audit_record.dast_deadline_seconds`, an + // INDEPENDENT clock from ClaimTimeoutSeconds. Nil when DAST is disabled; + // must be positive when set, matching + // ck_audit_record_dast_deadline_positive. Nothing in this file derives + // DeadlineAt from it. + DastDeadlineSeconds *int +} + +// AuditSeal is an immutable snapshot of one audit's sealing state — the +// projection the store writer maps onto the `audit_record` columns. +type AuditSeal struct { + AuditID string // audit_record via scan_run; anvil/auditId + State State // audit_record.state; anvil/state + + Sast HalfSeal // audit_record.sast_status, .sast_sealed_at + Dast HalfSeal // audit_record.dast_sealed_at (status below) + + // DastStatus is `audit_record.dast_status` / `anvil/dastStatus`. NEVER + // empty: the schema column is NOT NULL and the enum has no zero value + // meaning "unknown". It is derived, never assigned by a caller. + DastStatus DastStatus + + // StartedAt is scan_run.started_at, kept so DeadlineAt is auditable. + StartedAt time.Time + + // DeadlineAt is `audit_record.deadline_at` = StartedAt + + // ClaimTimeoutSeconds, computed once at BeginAudit. No seal, no + // consumption and no read ever changes it. + DeadlineAt time.Time + + ClaimTimeoutSeconds int + DastDeadlineSeconds *int +} + +// ComputeDeadline returns `scan_run.started_at + claim_timeout_seconds`, the +// one and only formula for `audit_record.deadline_at`. +// +// R.6's forbidden actions: "Do not compute `deadline_at` from any write +// timestamp — it must be `scan_run.started_at + claim_timeout_seconds`, +// computed once and never recomputed." Anchoring it to the last write makes +// the timeout unbounded for a chatty scan, which quietly defeats the reaper. +func ComputeDeadline(startedAt time.Time, claimTimeoutSeconds int) time.Time { + return startedAt.Add(time.Duration(claimTimeoutSeconds) * time.Second) +} + +// --------------------------------------------------------------------------- +// DastStatus derivation +// --------------------------------------------------------------------------- + +// DeriveDastStatus maps a DAST HalfStatus plus a DastOutcome onto the +// audit-level DastStatus. It is pure, so the mapping can be tested value by +// value without a Sealer. +// +// THE ORDER OF THESE RULES IS THE POINT. plan/00-SPINE.md S6: "a target that +// failed to boot must be distinguishable from 'scanned clean'". The +// provenance checks therefore run BEFORE the sealed/failed branch, so a half +// that sealed with zero findings against a target that never booted reports +// DastStatusTargetBootFailed and not DastStatusCompletedClean. +// +// 1. tier not installed -> not_run (the S9-AMENDED common case) +// 2. status running -> running +// 3. provenance boot_failed +// or build_failed -> target_boot_failed +// 4. provenance unreachable -> target_unreachable +// 5. provenance no_target -> skipped_no_manifest +// 6. status skipped -> skipped_no_manifest (tier present, half not run) +// 7. status timed_out -> timed_out +// 8. status failed -> completed_failed (booted_clean only; see below) +// 9. status sealed, partial -> completed_partial +// 10. status sealed, findings > 0 -> completed_findings +// 11. status sealed, findings = 0 -> completed_clean (the ONLY route to it) +// +// THE FUNCTION IS TOTAL. Every (TargetProvenance, HalfStatus) pair — five +// times five, with and without the tier installed — has exactly one image, and +// TestDeriveDastStatusIsTotal enumerates all of them. There is no pair for +// which this returns an empty DastStatus with a nil error, which matters +// because `audit_record.dast_status` is NOT NULL and the enum has no value +// meaning "unknown". +// +// RULE 8 WAS THE HOLE, AND IT IS NOW CLOSED. The previously frozen nine-value +// enum had no "the DAST half broke" literal, so this function mapped a +// HalfStatusFailed half against a cleanly-booted target onto +// DastStatusCompletedPartial. That was wrong in the same way S6 says a failed +// target must not read as "scanned clean": a half that CRASHED is not a half +// that COVERED PART of the surface, and merging them makes DastCoverage +// uninterpretable — 31 of 50 endpoints reads as a deliberate scope when in +// fact the scanner died. plan/IMPLEMENTATION-PLAN.md §6 was amended with a +// tenth literal, DastStatusCompletedFailed, and rule 8 now uses it. Rules 3-5 +// still run first, so this value is reachable ONLY for a genuine mid-scan +// failure against TargetProvenanceBootedClean. +func DeriveDastStatus(status HalfStatus, o DastOutcome) (DastStatus, error) { + if err := ValidateHalfStatus(string(status)); err != nil { + return "", err + } + + // 1. The tier is not installed. Says nothing about the target, and is + // the ONLY value that may be produced without a valid provenance. + if !o.TierInstalled { + return DastStatusNotRun, nil + } + + if err := ValidateTargetProvenance(string(o.Provenance)); err != nil { + return "", err + } + + // 2. Not yet terminal. + if status == HalfStatusRunning { + return DastStatusRunning, nil + } + + // 3-5. What happened to the target outranks what happened to the half. + switch o.Provenance { + case TargetProvenanceBootFailed, TargetProvenanceBuildFailed: + return DastStatusTargetBootFailed, nil + case TargetProvenanceUnreachableAtScanTime: + return DastStatusTargetUnreachable, nil + case TargetProvenanceNoTargetDeclared: + return DastStatusSkippedNoManifest, nil + } + + // 6-11. The target booted clean; the half's own status decides. + switch status { + case HalfStatusSkipped: + return DastStatusSkippedNoManifest, nil + case HalfStatusTimedOut: + return DastStatusTimedOut, nil + case HalfStatusFailed: + // The target was up and the half broke anyway. That is a DAST-side + // failure, not a coverage decision, and it has its own literal. + return DastStatusCompletedFailed, nil + case HalfStatusSealed: + switch { + case o.PartialCoverage: + return DastStatusCompletedPartial, nil + case o.FindingCount > 0: + return DastStatusCompletedFindings, nil + default: + return DastStatusCompletedClean, nil + } + } + + // Unreachable: ValidateHalfStatus admitted the value and every legal + // literal is handled above. + return "", &EnumError{ + Field: "anvil/status", + Value: string(status), + Allowed: []string{string(HalfStatusRunning), string(HalfStatusSealed), + string(HalfStatusFailed), string(HalfStatusTimedOut), string(HalfStatusSkipped)}, + } +} + +// DeriveState maps the two half statuses onto `anvil/state`. +// +// TERMINAL, not readable, is the test — see this file's header. A DAST half +// that is terminally HalfStatusSkipped advances the audit exactly as a sealed +// one does; what it does not do is open ReadHalf. +// +// StateConsumed and StateExpired are never produced here: they are explicit +// transitions (Consume, ExpireIfDue), not functions of the halves. +func DeriveState(sast, dast HalfStatus) State { + sastDone := IsTerminalHalfStatus(sast) + dastDone := IsTerminalHalfStatus(dast) + switch { + case sastDone && dastDone: + return StateBothSealed + case sastDone: + return StateSastSealed + case dastDone: + return StateDastSealed + default: + return StateCollecting + } +} + +// --------------------------------------------------------------------------- +// Sealer +// --------------------------------------------------------------------------- + +// audit is the mutable per-audit record. Guarded by Sealer.mu. +type audit struct { + id string + state State + startedAt time.Time + deadlineAt time.Time // written once, in BeginAudit + claimTimeoutSeconds int + dastDeadlineSeconds *int + dastEnabled bool + + sastStatus HalfStatus + sastSealedAt *time.Time + dastStatus HalfStatus + dastSealedAt *time.Time + + dastOutcome DastOutcome + dastDerived DastStatus +} + +// Sealer tracks per-half sealing for in-flight audits. The zero value is not +// usable; call NewSealer. +// +// It is safe for concurrent use: the SAST worker, the DAST worker and the +// consumer all touch the same audit from different goroutines, and the read +// gate is worth nothing if it can be observed mid-update. +type Sealer struct { + mu sync.Mutex + now func() time.Time + audits map[string]*audit +} + +// NewSealer returns an empty Sealer using the wall clock. +func NewSealer() *Sealer { + return &Sealer{now: time.Now, audits: make(map[string]*audit)} +} + +// SetClock replaces the clock used for `anvil/sealedAt` and for ExpireIfDue's +// due check. It does NOT affect DeadlineAt, which is a function of +// AuditConfig.StartedAt alone. +// +// Passing nil restores time.Now. +func (s *Sealer) SetClock(now func() time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if now == nil { + now = time.Now + } + s.now = now +} + +// BeginAudit registers an audit and fixes its claim clock. +// +// DeadlineAt is computed here, once, from cfg.StartedAt. Nothing else in this +// package writes it. +// +// When cfg.DastEnabled is false the DAST half is sealed immediately and +// terminally as HalfStatusSkipped with DastStatusNotRun, and the audit's +// state starts at StateDastSealed rather than StateCollecting. That is what +// lets a core-`anvil` install — which has no DAST code compiled in to call +// SealHalf — still reach StateBothSealed the moment its SAST half seals. +func (s *Sealer) BeginAudit(cfg AuditConfig) (AuditSeal, error) { + if cfg.AuditID == "" { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, + Reason: "anvil/auditId is empty", Err: ErrInvalidAuditConfig, + } + } + if cfg.StartedAt.IsZero() { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, + Reason: "scan_run.started_at is the zero time; deadline_at is anchored to scan START and cannot be computed from it", + Err: ErrInvalidAuditConfig, + } + } + timeout := cfg.ClaimTimeoutSeconds + if timeout == 0 { + timeout = DefaultClaimTimeoutSeconds + } + if timeout < 0 { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, + Reason: fmt.Sprintf("claim_timeout_seconds is %d; the schema requires > 0", timeout), + Err: ErrInvalidAuditConfig, + } + } + if cfg.DastDeadlineSeconds != nil && *cfg.DastDeadlineSeconds <= 0 { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, + Reason: fmt.Sprintf("dast_deadline_seconds is %d; the schema requires NULL or > 0", *cfg.DastDeadlineSeconds), + Err: ErrInvalidAuditConfig, + } + } + + s.mu.Lock() + defer s.mu.Unlock() + + if _, exists := s.audits[cfg.AuditID]; exists { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, + Reason: "already begun; re-beginning would recompute deadline_at, which R.6 forbids", + Err: ErrDuplicateAudit, + } + } + + a := &audit{ + id: cfg.AuditID, + startedAt: cfg.StartedAt, + deadlineAt: ComputeDeadline(cfg.StartedAt, timeout), + claimTimeoutSeconds: timeout, + dastDeadlineSeconds: copyInt(cfg.DastDeadlineSeconds), + dastEnabled: cfg.DastEnabled, + sastStatus: HalfStatusRunning, + dastStatus: HalfStatusRunning, + // Default provenance until the target lifecycle harness reports one. + // no_target_declared derives skipped_no_manifest, so an audit whose + // DAST half seals without anyone calling RecordDastOutcome can never + // land on completed_clean by omission. + dastOutcome: DastOutcome{ + TierInstalled: cfg.DastEnabled, + Provenance: TargetProvenanceNoTargetDeclared, + }, + } + + if !cfg.DastEnabled { + // The DAST tier is not installed: terminally skipped, never sealed, + // so SealedAt stays nil and the read gate stays shut. + a.dastStatus = HalfStatusSkipped + a.dastSealedAt = nil + } + + derived, err := DeriveDastStatus(a.dastStatus, a.dastOutcome) + if err != nil { + return AuditSeal{}, &SealingError{ + Op: "BeginAudit", AuditID: cfg.AuditID, Half: HalfDast, + Reason: "cannot derive anvil/dastStatus: " + err.Error(), Err: err, + } + } + a.dastDerived = derived + a.state = DeriveState(a.sastStatus, a.dastStatus) + + s.audits[cfg.AuditID] = a + return a.snapshot(), nil +} + +// RecordDastOutcome stores the target lifecycle and coverage facts the DAST +// status is derived from. Call it before sealing the DAST half; the last +// value recorded is the one the derivation uses. +// +// o.TierInstalled is IGNORED and replaced with the audit's own +// AuditConfig.DastEnabled, so no caller can report a DAST outcome for a tier +// the audit was not started with. +func (s *Sealer) RecordDastOutcome(auditID string, o DastOutcome) error { + s.mu.Lock() + defer s.mu.Unlock() + + a, err := s.lookup("RecordDastOutcome", auditID) + if err != nil { + return err + } + if a.state == StateConsumed || a.state == StateExpired { + return &SealingError{ + Op: "RecordDastOutcome", AuditID: auditID, Half: HalfDast, State: a.state, + Reason: "audit is no longer accepting outcome updates", Err: ErrAuditTerminal, + } + } + if IsTerminalHalfStatus(a.dastStatus) { + return &SealingError{ + Op: "RecordDastOutcome", AuditID: auditID, Half: HalfDast, + State: a.state, Status: a.dastStatus, + Reason: "the DAST half has already sealed; its outcome is frozen", + Err: ErrHalfAlreadySealed, + } + } + + o.TierInstalled = a.dastEnabled + if o.TierInstalled { + if err := ValidateTargetProvenance(string(o.Provenance)); err != nil { + return &SealingError{ + Op: "RecordDastOutcome", AuditID: auditID, Half: HalfDast, State: a.state, + Reason: "illegal anvil/target.provenance: " + err.Error(), Err: err, + } + } + } else { + o.Provenance = TargetProvenanceNoTargetDeclared + } + a.dastOutcome = o + + derived, err := DeriveDastStatus(a.dastStatus, a.dastOutcome) + if err != nil { + return &SealingError{ + Op: "RecordDastOutcome", AuditID: auditID, Half: HalfDast, State: a.state, + Reason: "cannot derive anvil/dastStatus: " + err.Error(), Err: err, + } + } + a.dastDerived = derived + return nil +} + +// SealHalf gives one half its terminal status. +// +// status must be one of TerminalHalfStatuses; HalfStatusRunning is refused +// with ErrNotSealable, because "started" is not "sealed". `anvil/sealedAt` is +// stamped only for HalfStatusSealed, per contract.go's RunProperties.SealedAt +// rule. +// +// Re-sealing a half with the IDENTICAL status is a no-op and preserves the +// original SealedAt, so a retried store write cannot move a seal timestamp. +// Re-sealing with a different status is ErrHalfAlreadySealed. +// +// DeadlineAt is not touched. Ever. +func (s *Sealer) SealHalf(auditID string, half Half, status HalfStatus) error { + if err := ValidateHalf(string(half)); err != nil { + return &SealingError{ + Op: "SealHalf", AuditID: auditID, + Reason: "illegal anvil/half: " + err.Error(), Err: err, + } + } + if err := ValidateHalfStatus(string(status)); err != nil { + return &SealingError{ + Op: "SealHalf", AuditID: auditID, Half: half, + Reason: "illegal anvil/status: " + err.Error(), Err: err, + } + } + if !IsTerminalHalfStatus(status) { + return &SealingError{ + Op: "SealHalf", AuditID: auditID, Half: half, Status: status, + Reason: fmt.Sprintf("%q is not terminal; a half seals with one of %v", status, TerminalHalfStatuses()), + Err: ErrNotSealable, + } + } + + s.mu.Lock() + defer s.mu.Unlock() + + a, err := s.lookup("SealHalf", auditID) + if err != nil { + return err + } + if a.state == StateConsumed || a.state == StateExpired { + return &SealingError{ + Op: "SealHalf", AuditID: auditID, Half: half, State: a.state, Status: status, + Reason: "audit is no longer accepting seals", Err: ErrAuditTerminal, + } + } + + current := a.sastStatus + if half == HalfDast { + current = a.dastStatus + } + if IsTerminalHalfStatus(current) { + if current == status { + return nil // idempotent; SealedAt preserved + } + return &SealingError{ + Op: "SealHalf", AuditID: auditID, Half: half, State: a.state, Status: current, + Reason: fmt.Sprintf("already sealed as %q; cannot re-seal as %q", current, status), + Err: ErrHalfAlreadySealed, + } + } + + var sealedAt *time.Time + if status == HalfStatusSealed { + t := s.now().UTC() + sealedAt = &t + } + + if half == HalfSast { + a.sastStatus = status + a.sastSealedAt = sealedAt + } else { + a.dastStatus = status + a.dastSealedAt = sealedAt + derived, derr := DeriveDastStatus(a.dastStatus, a.dastOutcome) + if derr != nil { + return &SealingError{ + Op: "SealHalf", AuditID: auditID, Half: half, State: a.state, Status: status, + Reason: "cannot derive anvil/dastStatus: " + derr.Error(), Err: derr, + } + } + a.dastDerived = derived + } + + a.state = DeriveState(a.sastStatus, a.dastStatus) + return nil +} + +// ReadyForConsumption reports, per half, whether a consumer may read that +// half's results now. An unknown audit reports (false, false). +// +// This is the gate plan/IMPLEMENTATION-PLAN.md §6 ruling G9 wires the handoff +// table to: `handoff.consumption_class = 'static_only'` rows become claimable +// when sastReady is true, and `'requires_dynamic_confirmation'` rows must +// wait for dastReady. plan/00-SPINE.md S7: "Only a DAST reproduction that now +// fails earns 'verified fixed'." +// +// dastReady stays false for a DAST-disabled audit even after it reaches +// StateBothSealed — there are no DAST results, and "no findings recorded" is +// not "dynamically scanned clean". +func (s *Sealer) ReadyForConsumption(auditID string) (sastReady, dastReady bool) { + s.mu.Lock() + defer s.mu.Unlock() + + a, ok := s.audits[auditID] + if !ok || a.state == StateExpired { + return false, false + } + return IsReadableHalfStatus(a.sastStatus), IsReadableHalfStatus(a.dastStatus) +} + +// ReadHalf is the consumer's read gate. It returns the half's seal only when +// that half's `anvil/status` is exactly HalfStatusSealed; every other status +// — running, failed, timed_out, skipped — is refused with a *ReadGateError, +// as is any read of an expired audit, whose payload the reaper has dropped. +// +// A consumed audit is still readable: plan/00-SPINE.md S1 requires a +// RE-ENTRANT consumer, so taking the record once must not shut the gate. +// +// On refusal the returned HalfSeal is the zero value and carries no +// information; callers must check the error. +func (s *Sealer) ReadHalf(auditID string, half Half) (HalfSeal, error) { + if err := ValidateHalf(string(half)); err != nil { + return HalfSeal{}, &SealingError{ + Op: "ReadHalf", AuditID: auditID, + Reason: "illegal anvil/half: " + err.Error(), Err: err, + } + } + + s.mu.Lock() + defer s.mu.Unlock() + + a, ok := s.audits[auditID] + if !ok { + return HalfSeal{}, &SealingError{ + Op: "ReadHalf", AuditID: auditID, Half: half, + Reason: "no such audit", Err: ErrUnknownAudit, + } + } + + seal := a.halfSeal(half) + if a.state == StateExpired { + return HalfSeal{}, &ReadGateError{ + AuditID: auditID, Half: half, Status: seal.Status, State: a.state, + Reason: "the claim timeout elapsed and the payload was dropped", + } + } + if !IsReadableHalfStatus(seal.Status) { + return HalfSeal{}, &ReadGateError{ + AuditID: auditID, Half: half, Status: seal.Status, State: a.state, + Reason: "this half has no readable results", + } + } + return seal, nil +} + +// Consume marks the audit taken by the coding-agent consumption pipeline +// (`audit_record.consumed_at`, StateConsumed). It requires StateBothSealed; +// consuming a half-finished audit is what the read gate exists to prevent. +// +// Consuming an already-consumed audit is a no-op, because the consumer is +// re-entrant by design. +func (s *Sealer) Consume(auditID string) error { + s.mu.Lock() + defer s.mu.Unlock() + + a, err := s.lookup("Consume", auditID) + if err != nil { + return err + } + switch a.state { + case StateConsumed: + return nil + case StateExpired: + return &SealingError{ + Op: "Consume", AuditID: auditID, State: a.state, + Reason: "the claim timeout elapsed", Err: ErrAuditTerminal, + } + case StateBothSealed: + a.state = StateConsumed + return nil + default: + return &SealingError{ + Op: "Consume", AuditID: auditID, State: a.state, + Reason: fmt.Sprintf("state is %q; consumption requires %q", a.state, StateBothSealed), + Err: ErrNotBothSealed, + } + } +} + +// ExpireIfDue moves the audit to StateExpired if and only if the clock has +// reached DeadlineAt, and reports whether it did. +// +// There is deliberately no unconditional Expire. The claim timeout is the +// only thing that may expire an audit, and DeadlineAt is fixed at scan start, +// so no amount of late activity can bring expiry forward or push it back. +// An already-consumed audit is never expired out from under its consumer. +func (s *Sealer) ExpireIfDue(auditID string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + + a, err := s.lookup("ExpireIfDue", auditID) + if err != nil { + return false, err + } + switch a.state { + case StateExpired: + return false, nil + case StateConsumed: + return false, nil + } + if s.now().Before(a.deadlineAt) { + return false, nil + } + a.state = StateExpired + return true, nil +} + +// Inspect returns a snapshot of the audit's sealing state, or ok=false if the +// audit is unknown. The snapshot shares no mutable state with the Sealer. +// +// Inspect is a DIAGNOSTIC: it deliberately still reports the true status of an +// expired audit's halves, because "this audit expired holding a sealed SAST +// half" is exactly what an operator needs to see. What it does not do any more +// is claim those halves are readable — every HalfSeal it hands out carries the +// audit state, so Readable() honours the expiry arm of the read gate that +// ReadHalf enforces. See CRITIQUE-02 F6. +func (s *Sealer) Inspect(auditID string) (AuditSeal, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + a, ok := s.audits[auditID] + if !ok { + return AuditSeal{}, false + } + return a.snapshot(), true +} + +// Forget drops an audit from the in-memory tracker. The durable row in +// `audit_record` is unaffected — plan/40-record-and-storage.md is explicit +// that the reaper drops the payload and never the row. +func (s *Sealer) Forget(auditID string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.audits, auditID) +} + +// lookup requires s.mu. +func (s *Sealer) lookup(op, auditID string) (*audit, error) { + a, ok := s.audits[auditID] + if !ok { + return nil, &SealingError{ + Op: op, AuditID: auditID, Reason: "no such audit", Err: ErrUnknownAudit, + } + } + return a, nil +} + +// halfSeal requires Sealer.mu. +// +// AuditState is stamped here, on every path — ReadHalf's, Inspect's and +// snapshot's alike — so there is no way to obtain a HalfSeal from a Sealer +// whose Readable() answers a different question from ReadHalf's gate. +func (a *audit) halfSeal(half Half) HalfSeal { + if half == HalfDast { + return HalfSeal{ + Half: HalfDast, Status: a.dastStatus, + SealedAt: copyTime(a.dastSealedAt), AuditState: a.state, + } + } + return HalfSeal{ + Half: HalfSast, Status: a.sastStatus, + SealedAt: copyTime(a.sastSealedAt), AuditState: a.state, + } +} + +// snapshot requires Sealer.mu. +func (a *audit) snapshot() AuditSeal { + return AuditSeal{ + AuditID: a.id, + State: a.state, + Sast: a.halfSeal(HalfSast), + Dast: a.halfSeal(HalfDast), + DastStatus: a.dastDerived, + StartedAt: a.startedAt, + DeadlineAt: a.deadlineAt, + ClaimTimeoutSeconds: a.claimTimeoutSeconds, + DastDeadlineSeconds: copyInt(a.dastDeadlineSeconds), + } +} + +func copyTime(t *time.Time) *time.Time { + if t == nil { + return nil + } + v := *t + return &v +} + +func copyInt(n *int) *int { + if n == nil { + return nil + } + v := *n + return &v +} + +// --------------------------------------------------------------------------- +// Package-level default Sealer +// --------------------------------------------------------------------------- + +// defaultSealer backs the package-level functions below. +var defaultSealer = NewSealer() + +// DefaultSealer returns the Sealer the package-level functions operate on. +// A process that wants isolated sealing state (tests, or a tool auditing two +// scans at once) should use NewSealer instead. +func DefaultSealer() *Sealer { return defaultSealer } + +// BeginAudit registers an audit on the default Sealer. See Sealer.BeginAudit. +func BeginAudit(cfg AuditConfig) (AuditSeal, error) { return defaultSealer.BeginAudit(cfg) } + +// RecordDastOutcome records DAST lifecycle facts on the default Sealer. See +// Sealer.RecordDastOutcome. +func RecordDastOutcome(auditID string, o DastOutcome) error { + return defaultSealer.RecordDastOutcome(auditID, o) +} + +// SealHalf seals one half on the default Sealer. +// +// This is R.6's mandated signature, taking `half` and `status` as strings +// because it is the boundary a scan controller calls across. Both are +// validated against contract.go's frozen enums before anything is mutated, so +// a bare literal that is not an enum member is rejected here rather than +// three layers down at a NOT NULL CHECK constraint. Callers inside Go should +// pass `string(HalfSast)` and `string(HalfStatusSealed)` — never a hand-typed +// "sast"/"sealed" — or use the typed Sealer.SealHalf method directly. +func SealHalf(auditID, half string, status string) error { + return defaultSealer.SealHalf(auditID, Half(half), HalfStatus(status)) +} + +// ReadyForConsumption reports per-half consumer readiness on the default +// Sealer. See Sealer.ReadyForConsumption. +func ReadyForConsumption(auditID string) (sastReady, dastReady bool) { + return defaultSealer.ReadyForConsumption(auditID) +} + +// ReadHalf is the read gate on the default Sealer. See Sealer.ReadHalf. +func ReadHalf(auditID string, half Half) (HalfSeal, error) { + return defaultSealer.ReadHalf(auditID, half) +} + +// Consume marks an audit consumed on the default Sealer. See Sealer.Consume. +func Consume(auditID string) error { return defaultSealer.Consume(auditID) } + +// ExpireIfDue expires a due audit on the default Sealer. See +// Sealer.ExpireIfDue. +func ExpireIfDue(auditID string) (bool, error) { return defaultSealer.ExpireIfDue(auditID) } + +// Inspect snapshots an audit on the default Sealer. See Sealer.Inspect. +func Inspect(auditID string) (AuditSeal, bool) { return defaultSealer.Inspect(auditID) } diff --git a/internal/record/sealing_test.go b/internal/record/sealing_test.go new file mode 100644 index 0000000..d06102e --- /dev/null +++ b/internal/record/sealing_test.go @@ -0,0 +1,1274 @@ +package record + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" +) + +// scanStart is a fixed scan_run.started_at for every test here, so that any +// movement in deadline_at is a bug and not clock jitter. +var scanStart = time.Date(2026, 8, 8, 9, 0, 0, 0, time.UTC) + +// fixedClock returns a clock function whose value the caller can advance. +func fixedClock(t *time.Time) func() time.Time { + return func() time.Time { return *t } +} + +// newTestSealer returns a Sealer on a controllable clock, plus a pointer the +// test moves to advance time. +func newTestSealer(t *testing.T) (*Sealer, *time.Time) { + t.Helper() + now := scanStart + s := NewSealer() + s.SetClock(fixedClock(&now)) + return s, &now +} + +// beginSAST starts a DAST-DISABLED audit — plan/00-SPINE.md S9-AMENDED's +// common case, the core `anvil` artifact with no probing capability compiled +// in. +func beginSAST(t *testing.T, s *Sealer, id string) AuditSeal { + t.Helper() + seal, err := s.BeginAudit(AuditConfig{AuditID: id, StartedAt: scanStart}) + if err != nil { + t.Fatalf("BeginAudit(%q): %v", id, err) + } + return seal +} + +// beginDAST starts a DAST-ENABLED audit with a target that booted cleanly. +func beginDAST(t *testing.T, s *Sealer, id string) AuditSeal { + t.Helper() + dastDeadline := 1800 + seal, err := s.BeginAudit(AuditConfig{ + AuditID: id, + StartedAt: scanStart, + DastEnabled: true, + DastDeadlineSeconds: &dastDeadline, + }) + if err != nil { + t.Fatalf("BeginAudit(%q): %v", id, err) + } + if err := s.RecordDastOutcome(id, DastOutcome{Provenance: TargetProvenanceBootedClean}); err != nil { + t.Fatalf("RecordDastOutcome(%q): %v", id, err) + } + return seal +} + +// --------------------------------------------------------------------------- +// THE HARD READ GATE +// --------------------------------------------------------------------------- + +// TestReadGateRefusesEveryNonSealedStatus is R.6's central obligation, stated +// directly: "no consumer may read a half's results before that half's status +// equals sealed." +// +// It walks EVERY HalfStatus other than HalfStatusSealed — including the three +// terminal ones, which advance `anvil/state` and are the easy ones to mistake +// for readable — and asserts the read is refused with a typed *ReadGateError +// carrying no seal data. +// +// plan/IMPLEMENTATION-PLAN.md §6 ruling G5 records why this test exists: area +// O keyed its transitions on `complete`, "which meant the gate never opens". +func TestReadGateRefusesEveryNonSealedStatus(t *testing.T) { + for _, half := range HalfValues() { + for _, status := range HalfStatusValues() { + if status == HalfStatusSealed { + continue // the one value that legitimately opens the gate + } + name := fmt.Sprintf("%s/%s", half, status) + t.Run(name, func(t *testing.T) { + s, _ := newTestSealer(t) + id := "audit-" + name + beginDAST(t, s, id) + + // HalfStatusRunning is the initial state; the rest are + // reached by sealing. + if status != HalfStatusRunning { + if err := s.SealHalf(id, half, status); err != nil { + t.Fatalf("SealHalf(%s, %s): %v", half, status, err) + } + } + + seal, err := s.ReadHalf(id, half) + if err == nil { + t.Fatalf("ReadHalf(%s) returned seal %+v with status %q; the gate must refuse everything but %q", + half, seal, status, HalfStatusSealed) + } + if !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("ReadHalf error %v does not match ErrHalfNotSealed", err) + } + var gate *ReadGateError + if !errors.As(err, &gate) { + t.Fatalf("ReadHalf error %v is not a *ReadGateError; R.6 requires a TYPED refusal", err) + } + if gate.Half != half || gate.Status != status { + t.Errorf("ReadGateError = {half:%q status:%q}, want {half:%q status:%q}", + gate.Half, gate.Status, half, status) + } + if seal != (HalfSeal{}) { + t.Errorf("refused read returned %+v; it must be the zero value, never a partial result", seal) + } + + // And the same refusal is visible through the bool API. + sastReady, dastReady := s.ReadyForConsumption(id) + ready := sastReady + if half == HalfDast { + ready = dastReady + } + if ready { + t.Errorf("ReadyForConsumption reports %s ready at status %q", half, status) + } + }) + } + } +} + +// TestReadGateOpensOnlyAtSealed is the positive half: the gate does open, and +// it stamps `anvil/sealedAt`. +func TestReadGateOpensOnlyAtSealed(t *testing.T) { + s, now := newTestSealer(t) + beginDAST(t, s, "a") + + *now = scanStart.Add(11 * time.Minute) + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + + seal, err := s.ReadHalf("a", HalfSast) + if err != nil { + t.Fatalf("ReadHalf after seal: %v", err) + } + if seal.Status != HalfStatusSealed { + t.Errorf("Status = %q, want %q", seal.Status, HalfStatusSealed) + } + if seal.SealedAt == nil { + t.Fatal("SealedAt is nil on a sealed half; contract.go requires it once Status == sealed") + } + if got, want := seal.SealedAt.UTC(), scanStart.Add(11*time.Minute); !got.Equal(want) { + t.Errorf("SealedAt = %v, want %v", got, want) + } + if !seal.Readable() { + t.Error("HalfSeal.Readable() is false for a sealed half") + } + + // The DAST half is still running, so it is still refused. The two halves + // gate INDEPENDENTLY (plan/00-SPINE.md S1). + if _, err := s.ReadHalf("a", HalfDast); !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("DAST read after a SAST-only seal: err = %v, want ErrHalfNotSealed", err) + } + sastReady, dastReady := s.ReadyForConsumption("a") + if !sastReady || dastReady { + t.Errorf("ReadyForConsumption = (%v, %v), want (true, false)", sastReady, dastReady) + } +} + +// TestSealedAtNilForEveryUnsealedTerminalStatus pins contract.go's rule that +// `anvil/sealedAt` is "required once Status == HalfStatusSealed, and is +// explicitly null otherwise". A failed half with a timestamp would read as a +// completion in `audit_record.sast_sealed_at`. +func TestSealedAtNilForEveryUnsealedTerminalStatus(t *testing.T) { + for _, status := range []HalfStatus{HalfStatusFailed, HalfStatusTimedOut, HalfStatusSkipped} { + t.Run(string(status), func(t *testing.T) { + s, now := newTestSealer(t) + id := "a-" + string(status) + beginDAST(t, s, id) + *now = scanStart.Add(time.Hour) + + if err := s.SealHalf(id, HalfSast, status); err != nil { + t.Fatalf("SealHalf: %v", err) + } + snap, ok := s.Inspect(id) + if !ok { + t.Fatal("Inspect: audit not found") + } + if snap.Sast.SealedAt != nil { + t.Errorf("sast_sealed_at = %v for status %q; must be NULL", *snap.Sast.SealedAt, status) + } + }) + } +} + +// --------------------------------------------------------------------------- +// DAST-DISABLED AUDITS REACH both_sealed +// --------------------------------------------------------------------------- + +// TestDastDisabledAuditReachesBothSealed is the case +// plan/IMPLEMENTATION-PLAN.md §6 ruling G2 says area O's four-state machine +// could not express, and the reason area 40's six-value enum won. +// +// A core `anvil` install (plan/00-SPINE.md S9-AMENDED: no DAST artifact, so +// no DAST worker exists to seal anything) must still reach StateBothSealed on +// its SAST seal alone, with dast_status = 'not_run' — never NULL and never +// 'completed_clean'. +func TestDastDisabledAuditReachesBothSealed(t *testing.T) { + s, _ := newTestSealer(t) + seal := beginSAST(t, s, "sast-only") + + // Before any seal, the DAST half is ALREADY terminal, so the audit sits + // in dast_sealed — a state area O's machine had no value for. + if seal.State != StateDastSealed { + t.Errorf("initial state = %q, want %q", seal.State, StateDastSealed) + } + if seal.Dast.Status != HalfStatusSkipped { + t.Errorf("dast half status = %q, want %q", seal.Dast.Status, HalfStatusSkipped) + } + if seal.DastStatus != DastStatusNotRun { + t.Errorf("dast_status = %q, want %q", seal.DastStatus, DastStatusNotRun) + } + + if err := s.SealHalf("sast-only", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + + snap, ok := s.Inspect("sast-only") + if !ok { + t.Fatal("Inspect: audit not found") + } + if snap.State != StateBothSealed { + t.Fatalf("state = %q, want %q — a DAST-disabled audit that cannot reach both_sealed wedges the consumer forever", + snap.State, StateBothSealed) + } + if snap.DastStatus != DastStatusNotRun { + t.Errorf("dast_status = %q, want %q", snap.DastStatus, DastStatusNotRun) + } + if snap.DastStatus == "" { + t.Error("dast_status is empty; audit_record.dast_status is NOT NULL") + } + if snap.Dast.SealedAt != nil { + t.Errorf("dast_sealed_at = %v; a skipped half never cleanly sealed", *snap.Dast.SealedAt) + } + + // both_sealed does NOT open the DAST read gate: there is nothing to read. + sastReady, dastReady := s.ReadyForConsumption("sast-only") + if !sastReady { + t.Error("sastReady is false after the SAST half sealed") + } + if dastReady { + t.Error("dastReady is true for a DAST-disabled audit; there are no DAST results to read") + } + if _, err := s.ReadHalf("sast-only", HalfDast); !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("ReadHalf(dast) on a DAST-disabled audit: err = %v, want ErrHalfNotSealed", err) + } + + // And the audit is consumable, which is the whole point. + if err := s.Consume("sast-only"); err != nil { + t.Fatalf("Consume: %v", err) + } +} + +// TestDastDisabledDistinguishableFromCleanDastScan is R.6's named validation +// item: "a test asserting a DAST-disabled scan's dast_status is +// distinguishable from a DAST-enabled scan that found nothing +// ('completed_clean')". +// +// research/23 Risk #1: "Anvil must never report '0 DAST findings' as 'no +// dynamic vulnerabilities'." Both audits below have zero DAST findings; only +// one of them was dynamically scanned. +func TestDastDisabledDistinguishableFromCleanDastScan(t *testing.T) { + s, _ := newTestSealer(t) + + beginSAST(t, s, "disabled") + if err := s.SealHalf("disabled", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf(disabled): %v", err) + } + + beginDAST(t, s, "enabled-clean") + if err := s.RecordDastOutcome("enabled-clean", DastOutcome{ + Provenance: TargetProvenanceBootedClean, + FindingCount: 0, + }); err != nil { + t.Fatalf("RecordDastOutcome: %v", err) + } + if err := s.SealHalf("enabled-clean", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf(enabled-clean, sast): %v", err) + } + if err := s.SealHalf("enabled-clean", HalfDast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf(enabled-clean, dast): %v", err) + } + + disabled, _ := s.Inspect("disabled") + clean, _ := s.Inspect("enabled-clean") + + if disabled.State != StateBothSealed || clean.State != StateBothSealed { + t.Fatalf("states = (%q, %q); both audits must reach both_sealed", disabled.State, clean.State) + } + if disabled.DastStatus == clean.DastStatus { + t.Fatalf("both audits report dast_status %q; a scan that never ran must not look like a clean scan", + disabled.DastStatus) + } + if disabled.DastStatus != DastStatusNotRun { + t.Errorf("DAST-disabled dast_status = %q, want %q", disabled.DastStatus, DastStatusNotRun) + } + if clean.DastStatus != DastStatusCompletedClean { + t.Errorf("DAST-enabled clean dast_status = %q, want %q", clean.DastStatus, DastStatusCompletedClean) + } + + // The semantic predicate contract.go provides for exactly this question. + if disabled.DastStatus.MeansDynamicallyScannedClean() { + t.Error("a DAST-disabled audit reports MeansDynamicallyScannedClean") + } + if !clean.DastStatus.MeansDynamicallyScannedClean() { + t.Error("a clean DAST scan does not report MeansDynamicallyScannedClean") + } + + // And the read gate distinguishes them too: only the scanned one is + // readable. + if _, dastReady := s.ReadyForConsumption("disabled"); dastReady { + t.Error("DAST-disabled audit reports dastReady") + } + if _, dastReady := s.ReadyForConsumption("enabled-clean"); !dastReady { + t.Error("cleanly scanned DAST half is not readable") + } +} + +// --------------------------------------------------------------------------- +// deadline_at is anchored to scan START +// --------------------------------------------------------------------------- + +// TestDeadlineAnchoredToScanStart pins the formula. +func TestDeadlineAnchoredToScanStart(t *testing.T) { + s, _ := newTestSealer(t) + + seal, err := s.BeginAudit(AuditConfig{ + AuditID: "d", StartedAt: scanStart, ClaimTimeoutSeconds: 3600, + }) + if err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if want := scanStart.Add(time.Hour); !seal.DeadlineAt.Equal(want) { + t.Errorf("DeadlineAt = %v, want %v", seal.DeadlineAt, want) + } + if !seal.DeadlineAt.Equal(ComputeDeadline(scanStart, 3600)) { + t.Error("DeadlineAt disagrees with ComputeDeadline") + } + + // The documented default is 8 hours. + def, err := s.BeginAudit(AuditConfig{AuditID: "d2", StartedAt: scanStart}) + if err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if want := scanStart.Add(DefaultClaimTimeoutSeconds * time.Second); !def.DeadlineAt.Equal(want) { + t.Errorf("default DeadlineAt = %v, want %v", def.DeadlineAt, want) + } + if def.ClaimTimeoutSeconds != DefaultClaimTimeoutSeconds { + t.Errorf("ClaimTimeoutSeconds = %d, want %d", def.ClaimTimeoutSeconds, DefaultClaimTimeoutSeconds) + } +} + +// TestDeadlineUnchangedByLateSeal is R.6's second named validation item: "a +// test asserting `deadline_at` is unchanged by a late write to either half". +// +// R.6's forbidden actions: "Do not compute `deadline_at` from any write +// timestamp". Anchoring the claim clock to the last write makes the timeout +// unbounded for a chatty scan, so the reaper never fires. +func TestDeadlineUnchangedByLateSeal(t *testing.T) { + s, now := newTestSealer(t) + initial := beginDAST(t, s, "late") + want := initial.DeadlineAt + + // Every subsequent operation happens far past the deadline. None of them + // may move it. + steps := []struct { + name string + advance time.Duration + do func() error + }{ + {"record dast outcome", 3 * time.Hour, func() error { + return s.RecordDastOutcome("late", DastOutcome{ + Provenance: TargetProvenanceBootedClean, FindingCount: 4, + }) + }}, + {"seal sast", 9 * time.Hour, func() error { + return s.SealHalf("late", HalfSast, HalfStatusSealed) + }}, + {"seal dast", 40 * time.Hour, func() error { + return s.SealHalf("late", HalfDast, HalfStatusSealed) + }}, + {"read sast", 90 * time.Hour, func() error { + _, err := s.ReadHalf("late", HalfSast) + return err + }}, + {"consume", 200 * time.Hour, func() error { + return s.Consume("late") + }}, + } + for _, step := range steps { + *now = scanStart.Add(step.advance) + if err := step.do(); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + snap, ok := s.Inspect("late") + if !ok { + t.Fatalf("%s: Inspect: audit not found", step.name) + } + if !snap.DeadlineAt.Equal(want) { + t.Fatalf("after %s at %v: DeadlineAt = %v, want %v — deadline_at must never be recomputed from a write timestamp", + step.name, *now, snap.DeadlineAt, want) + } + if !snap.StartedAt.Equal(scanStart) { + t.Fatalf("after %s: StartedAt = %v, want %v", step.name, snap.StartedAt, scanStart) + } + } + + // And sealedAt DID move with the clock — proving the two are independent + // clocks, not one clock read twice. + snap, _ := s.Inspect("late") + if snap.Sast.SealedAt == nil { + t.Fatal("sast SealedAt is nil after a clean seal") + } + if snap.Sast.SealedAt.Equal(snap.DeadlineAt) { + t.Error("sealedAt equals deadlineAt; R.6 forbids conflating the per-half seal with the claim clock") + } + if !snap.Sast.SealedAt.After(snap.DeadlineAt) { + t.Errorf("sealedAt %v did not track the clock past deadlineAt %v", snap.Sast.SealedAt, snap.DeadlineAt) + } +} + +// TestSealHalfDoesNotResetOnIdempotentReseal proves a retried write cannot +// move `anvil/sealedAt` either. +func TestSealHalfDoesNotResetOnIdempotentReseal(t *testing.T) { + s, now := newTestSealer(t) + beginDAST(t, s, "retry") + + *now = scanStart.Add(time.Minute) + if err := s.SealHalf("retry", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + first, _ := s.Inspect("retry") + + *now = scanStart.Add(5 * time.Hour) + if err := s.SealHalf("retry", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("idempotent re-seal: %v", err) + } + second, _ := s.Inspect("retry") + + if !second.Sast.SealedAt.Equal(*first.Sast.SealedAt) { + t.Errorf("SealedAt moved from %v to %v on a repeated seal", + *first.Sast.SealedAt, *second.Sast.SealedAt) + } +} + +// --------------------------------------------------------------------------- +// State machine +// --------------------------------------------------------------------------- + +// TestDeriveStateCoversEveryCombination walks all 25 (sast, dast) status +// pairs and asserts the derived state, then asserts every one of the four +// derivable states is actually produced — including StateDastSealed, whose +// unreachability in area O's machine is what ruling G2 struck. +func TestDeriveStateCoversEveryCombination(t *testing.T) { + seen := map[State]bool{} + for _, sast := range HalfStatusValues() { + for _, dast := range HalfStatusValues() { + got := DeriveState(sast, dast) + var want State + switch { + case IsTerminalHalfStatus(sast) && IsTerminalHalfStatus(dast): + want = StateBothSealed + case IsTerminalHalfStatus(sast): + want = StateSastSealed + case IsTerminalHalfStatus(dast): + want = StateDastSealed + default: + want = StateCollecting + } + if got != want { + t.Errorf("DeriveState(%q, %q) = %q, want %q", sast, dast, got, want) + } + if err := ValidateState(string(got)); err != nil { + t.Errorf("DeriveState(%q, %q) produced an illegal literal: %v", sast, dast, err) + } + seen[got] = true + } + } + for _, want := range []State{StateCollecting, StateSastSealed, StateDastSealed, StateBothSealed} { + if !seen[want] { + t.Errorf("state %q is unreachable from DeriveState", want) + } + } +} + +// TestDastFirstSealReachesDastSealed is plan/00-SPINE.md S1's "two +// INDEPENDENTLY-sealed halves" in its awkward direction: the DAST half seals +// first while the SAST half is still running. Ruling G2: area O's machine +// "cannot express a DAST-first seal at all". +func TestDastFirstSealReachesDastSealed(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "dast-first") + + if err := s.SealHalf("dast-first", HalfDast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf(dast): %v", err) + } + snap, _ := s.Inspect("dast-first") + if snap.State != StateDastSealed { + t.Fatalf("state = %q, want %q", snap.State, StateDastSealed) + } + if _, err := s.ReadHalf("dast-first", HalfDast); err != nil { + t.Errorf("DAST half is sealed but unreadable: %v", err) + } + if _, err := s.ReadHalf("dast-first", HalfSast); !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("SAST read while running: err = %v, want ErrHalfNotSealed", err) + } + + // The SAST half can still fail after a DAST seal, and the audit still + // reaches both_sealed. + if err := s.SealHalf("dast-first", HalfSast, HalfStatusFailed); err != nil { + t.Fatalf("SealHalf(sast, failed): %v", err) + } + snap, _ = s.Inspect("dast-first") + if snap.State != StateBothSealed { + t.Errorf("state = %q, want %q", snap.State, StateBothSealed) + } + // ...but a failed half is not a readable half. + if _, err := s.ReadHalf("dast-first", HalfSast); !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("failed SAST half is readable: err = %v", err) + } +} + +// TestSealHalfRejectsNonTerminalStatus: "still running" is not a seal. +func TestSealHalfRejectsNonTerminalStatus(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + + err := s.SealHalf("a", HalfSast, HalfStatusRunning) + if !errors.Is(err, ErrNotSealable) { + t.Fatalf("SealHalf(running) = %v, want ErrNotSealable", err) + } + var se *SealingError + if !errors.As(err, &se) { + t.Fatalf("error %v is not a *SealingError", err) + } + snap, _ := s.Inspect("a") + if snap.State != StateCollecting { + t.Errorf("state = %q after a refused seal, want %q", snap.State, StateCollecting) + } +} + +// TestSealHalfRejectsConflictingReseal: a half seals once. +func TestSealHalfRejectsConflictingReseal(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + + if err := s.SealHalf("a", HalfSast, HalfStatusFailed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + err := s.SealHalf("a", HalfSast, HalfStatusSealed) + if !errors.Is(err, ErrHalfAlreadySealed) { + t.Fatalf("re-seal failed->sealed = %v, want ErrHalfAlreadySealed", err) + } + snap, _ := s.Inspect("a") + if snap.Sast.Status != HalfStatusFailed { + t.Errorf("status = %q after a refused re-seal, want %q", snap.Sast.Status, HalfStatusFailed) + } + if _, rerr := s.ReadHalf("a", HalfSast); !errors.Is(rerr, ErrHalfNotSealed) { + t.Errorf("a failed half became readable via a refused re-seal: %v", rerr) + } +} + +// TestDastDisabledHalfCannotBeSealedClean: the core artifact has no DAST +// capability, so nothing may later claim its DAST half sealed. +func TestDastDisabledHalfCannotBeSealedClean(t *testing.T) { + s, _ := newTestSealer(t) + beginSAST(t, s, "a") + + if err := s.SealHalf("a", HalfDast, HalfStatusSealed); !errors.Is(err, ErrHalfAlreadySealed) { + t.Fatalf("sealing a DAST-disabled half = %v, want ErrHalfAlreadySealed", err) + } + snap, _ := s.Inspect("a") + if snap.DastStatus != DastStatusNotRun { + t.Errorf("dast_status = %q, want %q", snap.DastStatus, DastStatusNotRun) + } + // Repeating the skip it already has is a harmless no-op. + if err := s.SealHalf("a", HalfDast, HalfStatusSkipped); err != nil { + t.Errorf("idempotent skip: %v", err) + } +} + +// TestRecordDastOutcomeCannotClaimAnUninstalledTier: TierInstalled is taken +// from the audit, never from the caller. +func TestRecordDastOutcomeCannotClaimAnUninstalledTier(t *testing.T) { + s, _ := newTestSealer(t) + beginSAST(t, s, "a") + + err := s.RecordDastOutcome("a", DastOutcome{ + TierInstalled: true, + Provenance: TargetProvenanceBootedClean, + FindingCount: 0, + }) + // The DAST half is already terminally skipped, so the outcome is frozen. + if !errors.Is(err, ErrHalfAlreadySealed) { + t.Fatalf("RecordDastOutcome on a DAST-disabled audit = %v, want ErrHalfAlreadySealed", err) + } + snap, _ := s.Inspect("a") + if snap.DastStatus != DastStatusNotRun { + t.Errorf("dast_status = %q, want %q", snap.DastStatus, DastStatusNotRun) + } +} + +// --------------------------------------------------------------------------- +// Consumption and expiry +// --------------------------------------------------------------------------- + +// TestConsumeRequiresBothSealed. +func TestConsumeRequiresBothSealed(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + + if err := s.Consume("a"); !errors.Is(err, ErrNotBothSealed) { + t.Fatalf("Consume while collecting = %v, want ErrNotBothSealed", err) + } + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if err := s.Consume("a"); !errors.Is(err, ErrNotBothSealed) { + t.Fatalf("Consume at sast_sealed = %v, want ErrNotBothSealed", err) + } + if err := s.SealHalf("a", HalfDast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if err := s.Consume("a"); err != nil { + t.Fatalf("Consume at both_sealed: %v", err) + } + snap, _ := s.Inspect("a") + if snap.State != StateConsumed { + t.Errorf("state = %q, want %q", snap.State, StateConsumed) + } +} + +// TestConsumerIsReEntrant: plan/00-SPINE.md S1 requires a RE-ENTRANT +// consumer, so consuming the record must not shut the gate behind it. +func TestConsumerIsReEntrant(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + for _, half := range HalfValues() { + if err := s.SealHalf("a", half, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf(%s): %v", half, err) + } + } + if err := s.Consume("a"); err != nil { + t.Fatalf("Consume: %v", err) + } + if err := s.Consume("a"); err != nil { + t.Fatalf("second Consume: %v — consumption must be idempotent", err) + } + for _, half := range HalfValues() { + if _, err := s.ReadHalf("a", half); err != nil { + t.Errorf("ReadHalf(%s) after consumption: %v — the consumer is re-entrant", half, err) + } + } + sastReady, dastReady := s.ReadyForConsumption("a") + if !sastReady || !dastReady { + t.Errorf("ReadyForConsumption after consumption = (%v, %v), want (true, true)", sastReady, dastReady) + } +} + +// TestExpiryIsGovernedOnlyByDeadlineAt: nothing expires early, and an expired +// audit stops being readable because the reaper dropped its payload. +func TestExpiryIsGovernedOnlyByDeadlineAt(t *testing.T) { + s, now := newTestSealer(t) + seal, err := s.BeginAudit(AuditConfig{AuditID: "a", StartedAt: scanStart, ClaimTimeoutSeconds: 60}) + if err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + + *now = seal.DeadlineAt.Add(-time.Nanosecond) + if expired, err := s.ExpireIfDue("a"); err != nil || expired { + t.Fatalf("ExpireIfDue one ns early = (%v, %v), want (false, nil)", expired, err) + } + if _, err := s.ReadHalf("a", HalfSast); err != nil { + t.Fatalf("read before the deadline: %v", err) + } + + *now = seal.DeadlineAt + expired, err := s.ExpireIfDue("a") + if err != nil || !expired { + t.Fatalf("ExpireIfDue at the deadline = (%v, %v), want (true, nil)", expired, err) + } + snap, _ := s.Inspect("a") + if snap.State != StateExpired { + t.Errorf("state = %q, want %q", snap.State, StateExpired) + } + if _, err := s.ReadHalf("a", HalfSast); !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("read after expiry = %v, want ErrHalfNotSealed", err) + } + if sastReady, dastReady := s.ReadyForConsumption("a"); sastReady || dastReady { + t.Errorf("ReadyForConsumption after expiry = (%v, %v), want (false, false)", sastReady, dastReady) + } + if err := s.SealHalf("a", HalfDast, HalfStatusSealed); !errors.Is(err, ErrAuditTerminal) { + t.Errorf("seal after expiry = %v, want ErrAuditTerminal", err) + } +} + +// TestConsumedAuditIsNeverExpiredUnderItsConsumer. +func TestConsumedAuditIsNeverExpiredUnderItsConsumer(t *testing.T) { + s, now := newTestSealer(t) + seal, err := s.BeginAudit(AuditConfig{AuditID: "a", StartedAt: scanStart, ClaimTimeoutSeconds: 60}) + if err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if err := s.Consume("a"); err != nil { + t.Fatalf("Consume: %v", err) + } + + *now = seal.DeadlineAt.Add(time.Hour) + if expired, err := s.ExpireIfDue("a"); err != nil || expired { + t.Fatalf("ExpireIfDue on a consumed audit = (%v, %v), want (false, nil)", expired, err) + } + snap, _ := s.Inspect("a") + if snap.State != StateConsumed { + t.Errorf("state = %q, want %q", snap.State, StateConsumed) + } +} + +// --------------------------------------------------------------------------- +// DastStatus derivation +// --------------------------------------------------------------------------- + +// TestDeriveDastStatusTable pins the whole mapping, value by value. +func TestDeriveDastStatusTable(t *testing.T) { + clean := TargetProvenanceBootedClean + cases := []struct { + name string + status HalfStatus + outcome DastOutcome + want DastStatus + }{ + {"tier absent", HalfStatusSkipped, DastOutcome{}, DastStatusNotRun}, + {"tier absent even when sealed", HalfStatusSealed, DastOutcome{}, DastStatusNotRun}, + {"running", HalfStatusRunning, DastOutcome{TierInstalled: true, Provenance: clean}, DastStatusRunning}, + {"boot failed", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: TargetProvenanceBootFailed}, DastStatusTargetBootFailed}, + {"build failed", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: TargetProvenanceBuildFailed}, DastStatusTargetBootFailed}, + {"unreachable", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: TargetProvenanceUnreachableAtScanTime}, DastStatusTargetUnreachable}, + {"no target declared", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: TargetProvenanceNoTargetDeclared}, DastStatusSkippedNoManifest}, + {"skipped with tier", HalfStatusSkipped, DastOutcome{TierInstalled: true, Provenance: clean}, DastStatusSkippedNoManifest}, + {"timed out", HalfStatusTimedOut, DastOutcome{TierInstalled: true, Provenance: clean}, DastStatusTimedOut}, + // A half that broke against a target that was up is a DAST-side + // failure, not a coverage decision. Before the section 6 amendment + // this derived completed_partial, which invited a reader to take + // DastCoverage's numbers as a deliberate scope. CRITIQUE-02 F8/rule 8. + {"failed mid-scan", HalfStatusFailed, DastOutcome{TierInstalled: true, Provenance: clean}, DastStatusCompletedFailed}, + {"sealed partial", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: clean, PartialCoverage: true}, DastStatusCompletedPartial}, + {"sealed partial outranks findings", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: clean, PartialCoverage: true, FindingCount: 3}, DastStatusCompletedPartial}, + {"sealed with findings", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: clean, FindingCount: 1}, DastStatusCompletedFindings}, + {"sealed clean", HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: clean}, DastStatusCompletedClean}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := DeriveDastStatus(tc.status, tc.outcome) + if err != nil { + t.Fatalf("DeriveDastStatus: %v", err) + } + if got != tc.want { + t.Errorf("DeriveDastStatus(%q, %+v) = %q, want %q", tc.status, tc.outcome, got, tc.want) + } + if err := ValidateDastStatus(string(got)); err != nil { + t.Errorf("derived an illegal anvil/dastStatus literal: %v", err) + } + }) + } +} + +// TestCompletedCleanIsUnreachableWithoutAScannedTarget is S6's requirement +// stated as a negative: "a target that failed to boot must be +// distinguishable from 'scanned clean'". Zero findings plus a sealed half is +// NOT sufficient for DastStatusCompletedClean; the target must also have +// booted cleanly. +func TestCompletedCleanIsUnreachableWithoutAScannedTarget(t *testing.T) { + for _, prov := range TargetProvenanceValues() { + if prov == TargetProvenanceBootedClean { + continue + } + for _, status := range HalfStatusValues() { + got, err := DeriveDastStatus(status, DastOutcome{ + TierInstalled: true, Provenance: prov, FindingCount: 0, + }) + if err != nil { + t.Fatalf("DeriveDastStatus(%q, %q): %v", status, prov, err) + } + if got == DastStatusCompletedClean { + t.Errorf("DeriveDastStatus(%q, provenance=%q) = %q; a target that never scanned cleanly must not report clean", + status, prov, got) + } + if got.MeansDynamicallyScannedClean() { + t.Errorf("provenance %q with status %q reports MeansDynamicallyScannedClean", prov, status) + } + } + } +} + +// TestDeriveDastStatusRejectsIllegalLiterals: the derivation validates its +// inputs against contract.go's frozen enums rather than trusting them. +func TestDeriveDastStatusRejectsIllegalLiterals(t *testing.T) { + if _, err := DeriveDastStatus(HalfStatus("complete"), DastOutcome{TierInstalled: true, Provenance: TargetProvenanceBootedClean}); err == nil { + t.Error(`DeriveDastStatus accepted area O's struck "complete" token`) + } else { + var ee *EnumError + if !errors.As(err, &ee) { + t.Errorf("error %v is not an *EnumError", err) + } + } + if _, err := DeriveDastStatus(HalfStatusSealed, DastOutcome{TierInstalled: true, Provenance: TargetProvenance("ok")}); err == nil { + t.Error("DeriveDastStatus accepted an illegal anvil/target.provenance") + } +} + +// --------------------------------------------------------------------------- +// Input validation and the package-level API +// --------------------------------------------------------------------------- + +// TestBeginAuditRejectsUnusableConfigs. +func TestBeginAuditRejectsUnusableConfigs(t *testing.T) { + negative := -1 + zero := 0 + cases := []struct { + name string + cfg AuditConfig + }{ + {"empty audit id", AuditConfig{StartedAt: scanStart}}, + {"zero start", AuditConfig{AuditID: "a"}}, + {"negative claim timeout", AuditConfig{AuditID: "a", StartedAt: scanStart, ClaimTimeoutSeconds: -1}}, + {"zero dast deadline", AuditConfig{AuditID: "a", StartedAt: scanStart, DastDeadlineSeconds: &zero}}, + {"negative dast deadline", AuditConfig{AuditID: "a", StartedAt: scanStart, DastDeadlineSeconds: &negative}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s, _ := newTestSealer(t) + if _, err := s.BeginAudit(tc.cfg); !errors.Is(err, ErrInvalidAuditConfig) { + t.Fatalf("BeginAudit = %v, want ErrInvalidAuditConfig", err) + } + }) + } + + s, _ := newTestSealer(t) + beginDAST(t, s, "dup") + if _, err := s.BeginAudit(AuditConfig{AuditID: "dup", StartedAt: scanStart.Add(time.Hour)}); !errors.Is(err, ErrDuplicateAudit) { + t.Errorf("re-BeginAudit = %v, want ErrDuplicateAudit — it would recompute deadline_at", err) + } + snap, _ := s.Inspect("dup") + if !snap.StartedAt.Equal(scanStart) { + t.Errorf("StartedAt = %v after a refused re-begin, want %v", snap.StartedAt, scanStart) + } +} + +// TestUnknownAuditIsAlwaysRefused: no operation silently invents an audit. +func TestUnknownAuditIsAlwaysRefused(t *testing.T) { + s, _ := newTestSealer(t) + + if err := s.SealHalf("nope", HalfSast, HalfStatusSealed); !errors.Is(err, ErrUnknownAudit) { + t.Errorf("SealHalf = %v, want ErrUnknownAudit", err) + } + if err := s.RecordDastOutcome("nope", DastOutcome{}); !errors.Is(err, ErrUnknownAudit) { + t.Errorf("RecordDastOutcome = %v, want ErrUnknownAudit", err) + } + if err := s.Consume("nope"); !errors.Is(err, ErrUnknownAudit) { + t.Errorf("Consume = %v, want ErrUnknownAudit", err) + } + if _, err := s.ExpireIfDue("nope"); !errors.Is(err, ErrUnknownAudit) { + t.Errorf("ExpireIfDue = %v, want ErrUnknownAudit", err) + } + seal, err := s.ReadHalf("nope", HalfSast) + if !errors.Is(err, ErrUnknownAudit) { + t.Errorf("ReadHalf = %v, want ErrUnknownAudit", err) + } + if seal != (HalfSeal{}) { + t.Errorf("ReadHalf on an unknown audit returned %+v", seal) + } + if sastReady, dastReady := s.ReadyForConsumption("nope"); sastReady || dastReady { + t.Errorf("ReadyForConsumption on an unknown audit = (%v, %v), want (false, false)", sastReady, dastReady) + } + if _, ok := s.Inspect("nope"); ok { + t.Error("Inspect reported an unknown audit as found") + } +} + +// TestSealHalfRejectsBareStringsOutsideTheFrozenEnums. The package-level +// SealHalf takes strings because it is a process boundary; it must reject +// anything that is not a contract.go literal, with an *EnumError naming the +// legal set. Area O's struck `complete` is the concrete regression this +// guards (plan/IMPLEMENTATION-PLAN.md §6 ruling G5). +func TestSealHalfRejectsBareStringsOutsideTheFrozenEnums(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + + if err := s.SealHalf("a", Half("both"), HalfStatusSealed); err == nil { + t.Error(`SealHalf accepted half "both"`) + } else { + var ee *EnumError + if !errors.As(err, &ee) { + t.Errorf("half error %v is not an *EnumError", err) + } else if ee.Field != "anvil/half" { + t.Errorf("EnumError.Field = %q, want anvil/half", ee.Field) + } + } + + for _, bogus := range []string{"complete", "SEALED", "", "done"} { + err := s.SealHalf("a", HalfSast, HalfStatus(bogus)) + if err == nil { + t.Errorf("SealHalf accepted status %q", bogus) + continue + } + var ee *EnumError + if !errors.As(err, &ee) { + t.Errorf("status error for %q is not an *EnumError: %v", bogus, err) + } + } + snap, _ := s.Inspect("a") + if snap.State != StateCollecting { + t.Errorf("state = %q after refused seals, want %q", snap.State, StateCollecting) + } +} + +// TestPackageLevelAPIMatchesR6Signatures exercises the two functions R.6 +// names by signature — SealHalf(auditID, half, status string) error and +// ReadyForConsumption(auditID string) (sastReady, dastReady bool) — on the +// default Sealer, threading contract.go's constants through rather than +// hand-typed literals. +func TestPackageLevelAPIMatchesR6Signatures(t *testing.T) { + const id = "pkg-level-r6-audit" + t.Cleanup(func() { DefaultSealer().Forget(id) }) + + if _, err := BeginAudit(AuditConfig{AuditID: id, StartedAt: scanStart, DastEnabled: true}); err != nil { + t.Fatalf("BeginAudit: %v", err) + } + if err := RecordDastOutcome(id, DastOutcome{Provenance: TargetProvenanceBootedClean, FindingCount: 2}); err != nil { + t.Fatalf("RecordDastOutcome: %v", err) + } + + if sastReady, dastReady := ReadyForConsumption(id); sastReady || dastReady { + t.Fatalf("ReadyForConsumption before sealing = (%v, %v), want (false, false)", sastReady, dastReady) + } + if err := SealHalf(id, string(HalfSast), string(HalfStatusSealed)); err != nil { + t.Fatalf("SealHalf: %v", err) + } + if sastReady, dastReady := ReadyForConsumption(id); !sastReady || dastReady { + t.Fatalf("ReadyForConsumption after the SAST seal = (%v, %v), want (true, false)", sastReady, dastReady) + } + if err := SealHalf(id, string(HalfDast), string(HalfStatusSealed)); err != nil { + t.Fatalf("SealHalf(dast): %v", err) + } + if sastReady, dastReady := ReadyForConsumption(id); !sastReady || !dastReady { + t.Fatalf("ReadyForConsumption after both seals = (%v, %v), want (true, true)", sastReady, dastReady) + } + if _, err := ReadHalf(id, HalfDast); err != nil { + t.Fatalf("ReadHalf: %v", err) + } + snap, ok := Inspect(id) + if !ok { + t.Fatal("Inspect: audit not found") + } + if snap.State != StateBothSealed || snap.DastStatus != DastStatusCompletedFindings { + t.Errorf("snapshot = {state:%q dast_status:%q}, want {%q %q}", + snap.State, snap.DastStatus, StateBothSealed, DastStatusCompletedFindings) + } + if err := Consume(id); err != nil { + t.Fatalf("Consume: %v", err) + } + if expired, err := ExpireIfDue(id); err != nil || expired { + t.Errorf("ExpireIfDue on a consumed audit = (%v, %v), want (false, nil)", expired, err) + } +} + +// TestSnapshotDoesNotAliasSealerState: a caller mutating a snapshot must not +// reach back into the Sealer. +func TestSnapshotDoesNotAliasSealerState(t *testing.T) { + s, _ := newTestSealer(t) + beginDAST(t, s, "a") + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + + snap, _ := s.Inspect("a") + *snap.Sast.SealedAt = scanStart.Add(1000 * time.Hour) + *snap.DastDeadlineSeconds = 999999 + + fresh, _ := s.Inspect("a") + if fresh.Sast.SealedAt.Equal(*snap.Sast.SealedAt) { + t.Error("mutating a snapshot's SealedAt changed the Sealer's state") + } + if *fresh.DastDeadlineSeconds == *snap.DastDeadlineSeconds { + t.Error("mutating a snapshot's DastDeadlineSeconds changed the Sealer's state") + } +} + +// TestConcurrentSealingAndReading exercises the mutex. The read gate is worth +// nothing if a consumer can observe a half mid-update; this fails loudly +// under `go test -race` in CI. +func TestConcurrentSealingAndReading(t *testing.T) { + s, _ := newTestSealer(t) + const audits = 16 + for i := 0; i < audits; i++ { + beginDAST(t, s, fmt.Sprintf("a%d", i)) + } + + var wg sync.WaitGroup + for i := 0; i < audits; i++ { + id := fmt.Sprintf("a%d", i) + for _, half := range HalfValues() { + wg.Add(1) + go func(half Half) { + defer wg.Done() + if err := s.SealHalf(id, half, HalfStatusSealed); err != nil { + t.Errorf("SealHalf(%s, %s): %v", id, half, err) + } + }(half) + } + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + seal, err := s.ReadHalf(id, HalfSast) + if err != nil { + if !errors.Is(err, ErrHalfNotSealed) { + t.Errorf("ReadHalf(%s): unexpected error %v", id, err) + } + continue + } + // A successful read must be complete, never partial. + if seal.Status != HalfStatusSealed || seal.SealedAt == nil { + t.Errorf("ReadHalf(%s) returned a partial seal %+v", id, seal) + } + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + s.ReadyForConsumption(id) + } + }() + } + wg.Wait() + + for i := 0; i < audits; i++ { + snap, _ := s.Inspect(fmt.Sprintf("a%d", i)) + if snap.State != StateBothSealed { + t.Errorf("audit a%d state = %q, want %q", i, snap.State, StateBothSealed) + } + } +} + +// TestHalfStatusClassificationMatchesContract keeps the terminal/readable +// split honest against contract.go's enum: every legal HalfStatus is +// classified, and exactly one is readable. +func TestHalfStatusClassificationMatchesContract(t *testing.T) { + readable := 0 + for _, s := range HalfStatusValues() { + if IsReadableHalfStatus(s) { + readable++ + if !IsTerminalHalfStatus(s) { + t.Errorf("%q is readable but not terminal", s) + } + } + if s == HalfStatusRunning && IsTerminalHalfStatus(s) { + t.Errorf("%q must not be terminal", s) + } + if s != HalfStatusRunning && !IsTerminalHalfStatus(s) { + t.Errorf("%q must be terminal", s) + } + } + if readable != 1 { + t.Errorf("%d readable statuses, want exactly 1 (%q)", readable, HalfStatusSealed) + } + if len(TerminalHalfStatuses()) != len(HalfStatusValues())-1 { + t.Errorf("TerminalHalfStatuses has %d values, want %d", + len(TerminalHalfStatuses()), len(HalfStatusValues())-1) + } +} + +// --------------------------------------------------------------------------- +// Regression guards for CRITIQUE-02 (R.10 critic gate 2). +// --------------------------------------------------------------------------- + +// TestDeriveDastStatusIsTotal is the amendment's real stop condition: after +// adding DastStatusCompletedFailed, EVERY (tier, provenance, half status) pair +// has exactly one image, and none of them is the empty string. +// +// `audit_record.dast_status` is NOT NULL and the enum carries no value meaning +// "unknown", so a pair with no image is not a cosmetic gap — it is a row the +// store cannot hold. Before the amendment the failed/booted_clean pair had no +// image of its own and was folded onto completed_partial; the test below +// asserts the mapping is total AND that the fold is gone. +func TestDeriveDastStatusIsTotal(t *testing.T) { + seen := map[DastStatus]bool{} + for _, tier := range []bool{false, true} { + for _, prov := range TargetProvenanceValues() { + for _, status := range HalfStatusValues() { + for _, partial := range []bool{false, true} { + for _, count := range []int{0, 3} { + o := DastOutcome{ + TierInstalled: tier, + Provenance: prov, + PartialCoverage: partial, + FindingCount: count, + } + got, err := DeriveDastStatus(status, o) + if err != nil { + t.Fatalf("DeriveDastStatus(%q, %+v) errored: %v", status, o, err) + } + if got == "" { + t.Fatalf("DeriveDastStatus(%q, %+v) produced the empty string; "+ + "audit_record.dast_status is NOT NULL and has no 'unknown' value", status, o) + } + if err := ValidateDastStatus(string(got)); err != nil { + t.Fatalf("DeriveDastStatus(%q, %+v) = %q, not a legal literal: %v", + status, o, got, err) + } + seen[got] = true + } + } + } + } + } + // Every value except `running` requires a non-running half; `running` is + // reached above too, so the whole enum must be covered. A literal nothing + // can derive is a literal the store can hold and no code can produce. + for _, v := range DastStatusValues() { + if !seen[v] { + t.Errorf("no (tier, provenance, status) pair derives %q; it is unreachable", v) + } + } +} + +// TestFailedHalfAgainstALiveTargetIsNotPartialCoverage is the F8/rule-8 +// regression. A DAST half that CRASHED must not be reported as one that +// deliberately covered part of the surface, because DastCoverage's numbers +// mean different things in the two cases. +func TestFailedHalfAgainstALiveTargetIsNotPartialCoverage(t *testing.T) { + got, err := DeriveDastStatus(HalfStatusFailed, DastOutcome{ + TierInstalled: true, Provenance: TargetProvenanceBootedClean, + }) + if err != nil { + t.Fatalf("DeriveDastStatus: %v", err) + } + if got == DastStatusCompletedPartial { + t.Errorf("a failed DAST half against a booted_clean target derives %q; "+ + "a crash is not a coverage decision", got) + } + if got != DastStatusCompletedFailed { + t.Errorf("DeriveDastStatus(failed, booted_clean) = %q, want %q", got, DastStatusCompletedFailed) + } + if got.MeansDynamicallyScannedClean() { + t.Errorf("%q reports MeansDynamicallyScannedClean", got) + } + // completed_failed is reachable ONLY through a target that was actually up: + // rules 3-5 outrank the half's own status. + for _, prov := range TargetProvenanceValues() { + if prov == TargetProvenanceBootedClean { + continue + } + other, err := DeriveDastStatus(HalfStatusFailed, DastOutcome{TierInstalled: true, Provenance: prov}) + if err != nil { + t.Fatalf("DeriveDastStatus(failed, %q): %v", prov, err) + } + if other == DastStatusCompletedFailed { + t.Errorf("provenance %q derives %q; what happened to the TARGET outranks what happened to the half", + prov, other) + } + } +} + +// TestInspectHonoursTheExpiryArmOfTheReadGate reproduces CRITIQUE-02 F6 +// directly: ReadHalf refuses an expired audit, and before the fix Inspect +// handed out the same HalfSeal with Readable() == true. +func TestInspectHonoursTheExpiryArmOfTheReadGate(t *testing.T) { + s, now := newTestSealer(t) + seal := beginSAST(t, s, "a") + if err := s.SealHalf("a", HalfSast, HalfStatusSealed); err != nil { + t.Fatalf("SealHalf: %v", err) + } + // Readable before expiry, so the assertion after it is not vacuous. + snap, ok := s.Inspect("a") + if !ok || !snap.Sast.Readable() { + t.Fatalf("a sealed SAST half is not readable before expiry: %+v", snap.Sast) + } + + *now = seal.DeadlineAt.Add(time.Hour) + if expired, err := s.ExpireIfDue("a"); err != nil || !expired { + t.Fatalf("ExpireIfDue = (%v, %v), want (true, nil)", expired, err) + } + + if _, err := s.ReadHalf("a", HalfSast); !errors.Is(err, ErrHalfNotSealed) { + t.Fatalf("ReadHalf after expiry = %v, want a read-gate refusal", err) + } + snap, ok = s.Inspect("a") + if !ok { + t.Fatal("Inspect lost the audit") + } + if snap.Sast.Readable() { + t.Errorf("Inspect reports the SAST half readable on an expired audit that ReadHalf refuses: %+v", snap.Sast) + } + // Inspect is a diagnostic and must still tell the truth about the status. + if snap.Sast.Status != HalfStatusSealed { + t.Errorf("Inspect hid the half's real status: %q", snap.Sast.Status) + } +} + +// TestInspectAgreesWithReadHalfOnEveryState is the structural half of the same +// fix: for every audit shape this package can produce, HalfSeal.Readable() and +// "ReadHalf returns without error" are the SAME predicate. Two exported +// readiness paths that can disagree make the gate advisory. +func TestInspectAgreesWithReadHalfOnEveryState(t *testing.T) { + for _, half := range HalfValues() { + for _, status := range HalfStatusValues() { + for _, expire := range []bool{false, true} { + for _, consume := range []bool{false, true} { + name := fmt.Sprintf("%s/%s/expired=%v/consumed=%v", half, status, expire, consume) + t.Run(name, func(t *testing.T) { + s, now := newTestSealer(t) + seal := beginDAST(t, s, "a") + // Drive both halves to `status` where that is legal; + // `running` simply leaves them alone. + if IsTerminalHalfStatus(status) { + for _, h := range HalfValues() { + if err := s.SealHalf("a", h, status); err != nil { + t.Fatalf("SealHalf(%s, %s): %v", h, status, err) + } + } + } + if consume { + _ = s.Consume("a") // legal only from both_sealed + } + if expire { + *now = seal.DeadlineAt.Add(time.Hour) + if _, err := s.ExpireIfDue("a"); err != nil { + t.Fatalf("ExpireIfDue: %v", err) + } + } + + snap, ok := s.Inspect("a") + if !ok { + t.Fatal("Inspect lost the audit") + } + inspected := snap.Sast + if half == HalfDast { + inspected = snap.Dast + } + _, readErr := s.ReadHalf("a", half) + gateOpen := readErr == nil + + if inspected.Readable() != gateOpen { + t.Errorf("Inspect(...).Readable() = %v but ReadHalf %s (state=%q status=%q); "+ + "the two exported readiness paths must not disagree", + inspected.Readable(), map[bool]string{true: "succeeded", false: "refused"}[gateOpen], + snap.State, inspected.Status) + } + // And the per-half accessor on the snapshot agrees too. + if ready, _ := s.ReadyForConsumption("a"); half == HalfSast && ready != gateOpen { + t.Errorf("ReadyForConsumption sast = %v, ReadHalf open = %v", ready, gateOpen) + } + }) + } + } + } + } +} diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 69b147d..fc3aaf4 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -189,7 +189,7 @@ CREATE TABLE audit_record ( sast_status IS NULL OR sast_status IN ('running', 'sealed', 'failed', 'timed_out', 'skipped')), CONSTRAINT ck_audit_record_dast_status CHECK ( dast_status IN ('not_run', 'skipped_no_manifest', 'running', 'completed_clean', - 'completed_findings', 'completed_partial', 'target_boot_failed', + 'completed_findings', 'completed_partial', 'completed_failed', 'target_boot_failed', 'target_unreachable', 'timed_out')), CONSTRAINT ck_audit_record_target_provenance CHECK ( target_provenance IN ('booted_clean', 'boot_failed', 'build_failed', diff --git a/schemas/anvil-record-v1.schema.json b/schemas/anvil-record-v1.schema.json index b2d2311..a778526 100644 --- a/schemas/anvil-record-v1.schema.json +++ b/schemas/anvil-record-v1.schema.json @@ -52,6 +52,7 @@ "completed_clean", "completed_findings", "completed_partial", + "completed_failed", "target_boot_failed", "target_unreachable", "timed_out"