From 9dd4faa6bef2947900c9737b34ae5e3b49331729 Mon Sep 17 00:00:00 2001 From: Konrad Heimel Date: Sat, 15 Aug 2026 17:52:29 +0200 Subject: [PATCH 1/4] :sparkles: feat(docs): gate example pack and format claims against dogfood Paper-gate both polarities so README cannot name a pack or input format that is not shipped under examples/packs, and a dogfooded pack cannot go undocumented. Wired through task docs-gates. --- Taskfile.yml | 1 + examples/README.md | 1 + hack/docs/example_format_inventory_test.sh | 248 +++++++++++++++++++++ hack/docs/truthlag_pins_test.sh | 15 ++ 4 files changed, 265 insertions(+) create mode 100755 hack/docs/example_format_inventory_test.sh diff --git a/Taskfile.yml b/Taskfile.yml index 29a20f69..8ae9fc3b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -140,6 +140,7 @@ tasks: cmds: - bash hack/docs/readme_smoke_test.sh - bash hack/docs/truthlag_pins_test.sh + - bash hack/docs/example_format_inventory_test.sh compare-exitgate-test: desc: "PCS-S09 exit gate: full suite + E6 seed dir + schema drift guard (REQ-PCS-S09-01..03)" diff --git a/examples/README.md b/examples/README.md index 0f61604c..6b45abf8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,6 +8,7 @@ don't run are lies, so the runnable ones are executed by the gates, not just rea - [`packs/`](packs/) — complete adopter policy trees (`topic-registry`, `service-catalog`, `infra-vars`). Each is a repo root: `assent lint ` is clean and `assent test ` passes, both under `task check`. Start here. + Input formats: yaml, json, tfvars. - [`policies/declarative/`](policies/declarative/) — standalone envelope rules with `assert` predicates - [`policies/rego/`](policies/rego/) — the tier-2 escape hatch for the same archetype. diff --git a/hack/docs/example_format_inventory_test.sh b/hack/docs/example_format_inventory_test.sh new file mode 100755 index 00000000..61d543e3 --- /dev/null +++ b/hack/docs/example_format_inventory_test.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# REQ-EX-S01-01..05 — examples pack/format inventory paper-gate (both polarities). +# +# Enumerates examples/packs/* (every immediate child is a pack), records each +# pack's governed format from class match.paths extensions (.yaml / .json / +# .tfvars / .tf), and asserts examples/README.md names exactly those packs and +# claims exactly those formats. A pack directory without .assent/tests/ is an +# incomplete tree (hard error). Claiming cue / kafka-acl / HCL before a fixture +# exists must go red; omitting a real pack from the README must go red the +# other way. +# +# WIRED: `task docs-gates` runs this script (REQ-EX-S01-05). Deleting that +# invocation reddens the wiring pin here and in truthlag_pins_test.sh. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +fails=0 +pass() { echo "PASS $1"; } +fail() { echo "FAIL $1" >&2; fails=$((fails + 1)); } + +# --- discovery ---------------------------------------------------------------- + +# Immediate children of examples/packs. Every one is a pack candidate. +list_pack_dirs() { + local root="$1" + local d + for d in "$root"/examples/packs/*; do + [[ -d "$d" ]] || continue + basename "$d" + done | LC_ALL=C sort +} + +# Formats governed by class match.paths. .tfvars is not .tf. +# Reads .assent/config.yaml; environment match globs without an extension are ignored. +pack_formats() { + local pack_dir="$1" + local cfg="$pack_dir/.assent/config.yaml" + [[ -f "$cfg" ]] || return 0 + # Prefer the classes: block so env path globs cannot contribute a false extension. + awk ' + $0 ~ /^classes:[[:space:]]*$/ { in_cls = 1; next } + in_cls && /^[a-zA-Z]/ { in_cls = 0 } + in_cls { print } + ' "$cfg" | grep -oE '\*\.[A-Za-z0-9]+' | sed 's/^\*\.//' | while read -r ext; do + case "$ext" in + tfvars) echo tfvars ;; + tf) echo tf ;; + yaml|yml) echo yaml ;; + json) echo json ;; + esac + done | LC_ALL=C sort -u +} + +all_pack_formats() { + local root="$1" + local name + while IFS= read -r name; do + [[ -z "$name" ]] && continue + pack_formats "$root/examples/packs/$name" + done < <(list_pack_dirs "$root") | LC_ALL=C sort -u +} + +# Backtick pack names on the [`packs/`] list item (and its wrapped continuation). +readme_packs() { + local readme="$1" + awk ' + /\[`packs\/`\]/ { in_packs = 1 } + in_packs { print } + in_packs && /^[[:space:]]*- \[`/ && !/\[`packs\/`\]/ { in_packs = 0 } + in_packs && /^$/ { in_packs = 0 } + ' "$readme" | grep -oE '`[a-z0-9-]+`' | tr -d '`' | grep -vx 'packs' | LC_ALL=C sort -u +} + +# Tokens on the dedicated "Input formats:" sentence (yaml, json, tfvars, tf, hcl, cue, …). +readme_formats() { + local readme="$1" + local line + line="$(grep -E '^[[:space:]]*Input formats:' "$readme" || true)" + [[ -n "$line" ]] || return 0 + printf '%s\n' "$line" | grep -oE '\b(yaml|yml|json|tfvars|tf|hcl|cue)\b' | sed 's/^yml$/yaml/' | LC_ALL=C sort -u +} + +# Returns 0 when README and filesystem agree. Prints a PACKS=/FORMATS= report. +inventory_ok() { + local root="$1" + local readme="$2" + local name + local incomplete=0 + local packs_fs packs_doc fmts_fs fmts_doc + + packs_fs="$(list_pack_dirs "$root" | tr '\n' ' ' | sed 's/[[:space:]]*$//')" + while IFS= read -r name; do + [[ -z "$name" ]] && continue + if [[ ! -d "$root/examples/packs/$name/.assent/tests" ]]; then + echo "incomplete pack (no .assent/tests/): $name" >&2 + incomplete=1 + fi + done < <(list_pack_dirs "$root") + if [[ "$incomplete" -ne 0 ]]; then + return 1 + fi + + packs_doc="$(readme_packs "$readme" | tr '\n' ' ' | sed 's/[[:space:]]*$//')" + fmts_fs="$(all_pack_formats "$root" | tr '\n' ' ' | sed 's/[[:space:]]*$//')" + fmts_doc="$(readme_formats "$readme" | tr '\n' ' ' | sed 's/[[:space:]]*$//')" + + echo "PACKS: $packs_fs" + echo "FORMATS: $fmts_fs" + + if [[ -z "$packs_fs" ]]; then + echo "no example packs discovered" >&2 + return 1 + fi + if [[ "$packs_doc" != "$packs_fs" ]]; then + echo "README packs [$packs_doc] != filesystem [$packs_fs]" >&2 + return 1 + fi + if [[ -z "$fmts_doc" ]]; then + echo "README has no 'Input formats:' sentence (or no recognised tokens)" >&2 + return 1 + fi + if [[ "$fmts_doc" != "$fmts_fs" ]]; then + echo "README formats [$fmts_doc] != pack class match.paths [$fmts_fs]" >&2 + return 1 + fi + return 0 +} + +docs_gates_invokes_inventory() { + local taskfile="$1" + awk ' + $0 == " docs-gates:" { inblk = 1; next } + inblk && /^ [A-Za-z0-9_.:-]+:[[:space:]]*$/ { inblk = 0 } + inblk { print } + ' "$taskfile" | grep -q 'hack/docs/example_format_inventory_test.sh' +} + +# --- polarity mutations (must go red) ----------------------------------------- + +GOOD_README="$WORK/readme.good.md" +cat >"$GOOD_README" <<'EOF' +# Examples +- [`packs/`](packs/) — complete adopter policy trees (`topic-registry`, `service-catalog`, + `infra-vars`). +Input formats: yaml, json, tfvars. +EOF + +if inventory_ok "$ROOT" "$GOOD_README" >"$WORK/good.out" 2>"$WORK/good.err"; then + pass "synthetic README matching the three shipped packs is green" +else + fail "synthetic matching README should be green: $(cat "$WORK/good.err")" +fi + +EXTRA="$WORK/readme.extra.md" +sed 's/`infra-vars`/`infra-vars`, `kafka-acl`/' "$GOOD_README" >"$EXTRA" +if inventory_ok "$ROOT" "$EXTRA" >"$WORK/extra.out" 2>"$WORK/extra.err"; then + fail "claimed-but-missing pack kafka-acl stayed green (REQ-EX-S01-02 vacuous)" +else + pass "REQ-EX-S01-02: extra pack name kafka-acl reddens" +fi + +OMIT="$WORK/readme.omit.md" +sed 's/`topic-registry`, //' "$GOOD_README" >"$OMIT" +if inventory_ok "$ROOT" "$OMIT" >"$WORK/omit.out" 2>"$WORK/omit.err"; then + fail "omitting topic-registry from README stayed green (REQ-EX-S01-03 vacuous)" +else + pass "REQ-EX-S01-03: omitting topic-registry reddens" +fi + +CUE="$WORK/readme.cue.md" +sed 's/tfvars\./tfvars, cue./' "$GOOD_README" >"$CUE" +if inventory_ok "$ROOT" "$CUE" >"$WORK/cue.out" 2>"$WORK/cue.err"; then + fail "claiming cue stayed green (REQ-EX-S01-04 vacuous)" +else + pass "REQ-EX-S01-04: claiming cue reddens" +fi + +HCL="$WORK/readme.hcl.md" +sed 's/tfvars\./tfvars, hcl./' "$GOOD_README" >"$HCL" +if inventory_ok "$ROOT" "$HCL" >"$WORK/hcl.out" 2>"$WORK/hcl.err"; then + fail "claiming hcl before a .tf fixture stayed green" +else + pass "claiming hcl / .tf before S05 reddens" +fi + +# Incomplete tree: a fourth pack dir with .assent/ but no tests. +INCOMPLETE="$WORK/incomplete-root" +mkdir -p "$INCOMPLETE/examples/packs" +for name in infra-vars service-catalog topic-registry; do + mkdir -p "$INCOMPLETE/examples/packs/$name/.assent/tests" + cp "$ROOT/examples/packs/$name/.assent/config.yaml" "$INCOMPLETE/examples/packs/$name/.assent/config.yaml" +done +mkdir -p "$INCOMPLETE/examples/packs/orphan/.assent" +printf 'classes:\n - name: x\n match: { paths: ["x/**/*.yaml"] }\n' >"$INCOMPLETE/examples/packs/orphan/.assent/config.yaml" +if inventory_ok "$INCOMPLETE" "$GOOD_README" >"$WORK/inc.out" 2>"$WORK/inc.err"; then + fail "pack without .assent/tests/ stayed green" +else + pass "pack directory without .assent/tests/ reddens" +fi + +# --- happy path against the real tree (REQ-EX-S01-01) ------------------------- + +if inventory_ok "$ROOT" "$ROOT/examples/README.md" >"$WORK/real.out" 2>"$WORK/real.err"; then + report="$(tr '\n' ' ' <"$WORK/real.out")" + echo "$report" + if echo "$report" | grep -q 'PACKS: infra-vars service-catalog topic-registry' \ + && echo "$report" | grep -q 'FORMATS: json tfvars yaml'; then + if echo "$report" | grep -qw 'tf' && ! echo "$report" | grep -q 'tfvars'; then + fail "REQ-EX-S01-01: reported bare tf before the S05 fixture" + else + pass "REQ-EX-S01-01: real tree reports the three packs and yaml/json/tfvars" + fi + else + fail "REQ-EX-S01-01: unexpected report: $report" + fi +else + fail "REQ-EX-S01-01: real examples/README.md vs packs is red: $(cat "$WORK/real.err")" +fi + +# --- wiring (REQ-EX-S01-05) --------------------------------------------------- + +if docs_gates_invokes_inventory "$ROOT/Taskfile.yml"; then + pass "REQ-EX-S01-05: task docs-gates invokes example_format_inventory_test.sh" +else + fail "REQ-EX-S01-05: Taskfile.yml docs-gates does not run hack/docs/example_format_inventory_test.sh" +fi + +MUT_TF="$WORK/Taskfile.no-inventory.yml" +# Drop only the inventory invocation; prove the assertion goes red. +grep -v 'hack/docs/example_format_inventory_test.sh' "$ROOT/Taskfile.yml" >"$MUT_TF" +if grep -q 'hack/docs/example_format_inventory_test.sh' "$MUT_TF"; then + fail "REQ-EX-S01-05: mutation did not remove the inventory invocation" +elif docs_gates_invokes_inventory "$MUT_TF"; then + fail "REQ-EX-S01-05: wiring assertion stayed green after deleting the invocation (vacuous)" +else + pass "REQ-EX-S01-05: deleting the docs-gates inventory line reddens the wiring pin" +fi + +if [[ "$fails" -ne 0 ]]; then + echo "FAILED: $fails example-format inventory check(s)" >&2 + exit 1 +fi +echo "OK: example format inventory (EX-S01)" diff --git a/hack/docs/truthlag_pins_test.sh b/hack/docs/truthlag_pins_test.sh index ead8fa4a..2297506a 100755 --- a/hack/docs/truthlag_pins_test.sh +++ b/hack/docs/truthlag_pins_test.sh @@ -26,6 +26,8 @@ # WIRED (D-124/D-125): `task docs-gates` runs this and readme_smoke_test.sh, and # `task check` runs `docs-gates`. A docs edit that reopens one of these findings reds # the gate every developer runs before every commit. +# REQ-EX-S01-05: docs-gates also runs example_format_inventory_test.sh (inventory +# paper-gate). Deleting that line from Taskfile.yml reddens the pin below. set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -213,6 +215,19 @@ else pass "ADR index: $adr_checked ADR status rows agree with their files" fi +# REQ-EX-S01-05 — docs-gates must invoke the example format inventory script. +# Deleting that cmds line from Taskfile.yml makes this grep fail (non-vacuity). +docs_gates_body="$(awk ' + $0 == " docs-gates:" { inblk = 1; next } + inblk && /^ [A-Za-z0-9_.:-]+:[[:space:]]*$/ { inblk = 0 } + inblk { print } +' Taskfile.yml)" +if printf '%s\n' "$docs_gates_body" | grep -q 'hack/docs/example_format_inventory_test.sh'; then + pass "EX-S01: docs-gates runs example_format_inventory_test.sh" +else + fail "EX-S01: docs-gates does not run hack/docs/example_format_inventory_test.sh" +fi + if [[ "$fails" -ne 0 ]]; then echo "FAILED: $fails truth-lag pin(s) reopened" >&2 exit 1 From d52584657aeaf22238244c2ffce476318461e688 Mon Sep 17 00:00:00 2001 From: Konrad Heimel Date: Sat, 15 Aug 2026 17:53:55 +0200 Subject: [PATCH 2/4] :wrench: chore(release): regenerate CHANGELOG.md after EX-S01 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21bf533e..843fcea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,9 @@ repository still gets a decision, never by following the link; no release carrie - :memo: docs(openspec): specify Scorecard fuzzing and Best Practices residuals - :memo: specs(p5-ex): decompose complex in-tree examples epic +### Features +- :sparkles: feat(docs): gate example pack and format claims against dogfood + ### Fixes - :bug: fix(ci): pin ci-audit-test in the AUD-S18 check-stage list - :bug: specs(p5-ex): make S10 schema freeze and S07 fence non-vacuous From 5ad726e9154e08cc3370d7866929a36b5e849f6d Mon Sep 17 00:00:00 2001 From: Konrad Heimel Date: Sun, 16 Aug 2026 00:58:28 +0200 Subject: [PATCH 3/4] :bug: fix(docs-gates): fail on unmapped format tokens and de-confound the tests-dir mutation (EX-S01 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1-F1: readme_formats tokenized only a hard-coded whitelist, so an unrecognised claim (e.g. toml) was silently dropped and the gate stayed green (REQ-EX-S01-04 fail-open). Now the ENTIRE comma-separated list after 'Input formats:' is tokenized and any token with no matching pack class match.paths extension reddens with a dedicated message. New toml polarity assertion added red-first. P1-F2: the orphan (no .assent/tests/) mutation used GOOD_README, which omits orphan, so deleting the tests-dir check still reddened via the packs-list mismatch — the mutation could not detect removal of the check. The mutation now uses a README variant that lists orphan and asserts the redden reason is the tests-dir message. Verified: deleting the check now flips this assertion to FAIL. P3: deleted the dead bare-tf branch after the exact FORMATS match — the match guarantees tfvars is present, so the branch could never fire. --- hack/docs/example_format_inventory_test.sh | 52 +++++++++++++++++----- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/hack/docs/example_format_inventory_test.sh b/hack/docs/example_format_inventory_test.sh index 61d543e3..cc28d3c0 100755 --- a/hack/docs/example_format_inventory_test.sh +++ b/hack/docs/example_format_inventory_test.sh @@ -76,13 +76,19 @@ readme_packs() { ' "$readme" | grep -oE '`[a-z0-9-]+`' | tr -d '`' | grep -vx 'packs' | LC_ALL=C sort -u } -# Tokens on the dedicated "Input formats:" sentence (yaml, json, tfvars, tf, hcl, cue, …). +# ALL comma-separated tokens on the dedicated "Input formats:" sentence. +# No whitelist: an unrecognised claim (toml, ini, …) must surface as a token so +# the doc-vs-filesystem comparison reddens instead of silently dropping it. readme_formats() { local readme="$1" local line line="$(grep -E '^[[:space:]]*Input formats:' "$readme" || true)" [[ -n "$line" ]] || return 0 - printf '%s\n' "$line" | grep -oE '\b(yaml|yml|json|tfvars|tf|hcl|cue)\b' | sed 's/^yml$/yaml/' | LC_ALL=C sort -u + printf '%s\n' "$line" \ + | sed -E 's/^[[:space:]]*Input formats:[[:space:]]*//; s/\.?[[:space:]]*$//' \ + | tr ',' '\n' \ + | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//; s/^yml$/yaml/' \ + | grep -v '^$' | LC_ALL=C sort -u } # Returns 0 when README and filesystem agree. Prints a PACKS=/FORMATS= report. @@ -121,7 +127,18 @@ inventory_ok() { return 1 fi if [[ -z "$fmts_doc" ]]; then - echo "README has no 'Input formats:' sentence (or no recognised tokens)" >&2 + echo "README has no 'Input formats:' sentence (or no tokens on it)" >&2 + return 1 + fi + local tok unmapped="" + for tok in $fmts_doc; do + case " $fmts_fs " in + *" $tok "*) ;; + *) unmapped="$unmapped $tok" ;; + esac + done + if [[ -n "$unmapped" ]]; then + echo "README claims format(s) with no pack class match.paths extension:$unmapped" >&2 return 1 fi if [[ "$fmts_doc" != "$fmts_fs" ]]; then @@ -180,6 +197,14 @@ else pass "REQ-EX-S01-04: claiming cue reddens" fi +TOML="$WORK/readme.toml.md" +sed 's/tfvars\./tfvars, toml./' "$GOOD_README" >"$TOML" +if inventory_ok "$ROOT" "$TOML" >"$WORK/toml.out" 2>"$WORK/toml.err"; then + fail "claiming toml stayed green (unrecognised token silently dropped — REQ-EX-S01-04 fail-open)" +else + pass "REQ-EX-S01-04: claiming toml (token outside any whitelist) reddens" +fi + HCL="$WORK/readme.hcl.md" sed 's/tfvars\./tfvars, hcl./' "$GOOD_README" >"$HCL" if inventory_ok "$ROOT" "$HCL" >"$WORK/hcl.out" 2>"$WORK/hcl.err"; then @@ -188,7 +213,10 @@ else pass "claiming hcl / .tf before S05 reddens" fi -# Incomplete tree: a fourth pack dir with .assent/ but no tests. +# Incomplete tree: a fourth pack dir with .assent/ but no tests. The README +# variant DOES list `orphan` (and its yaml format is already claimed), so the +# missing-tests check is the ONLY thing that can redden this — deleting that +# check must flip this assertion, not the packs-list comparison. INCOMPLETE="$WORK/incomplete-root" mkdir -p "$INCOMPLETE/examples/packs" for name in infra-vars service-catalog topic-registry; do @@ -197,10 +225,16 @@ for name in infra-vars service-catalog topic-registry; do done mkdir -p "$INCOMPLETE/examples/packs/orphan/.assent" printf 'classes:\n - name: x\n match: { paths: ["x/**/*.yaml"] }\n' >"$INCOMPLETE/examples/packs/orphan/.assent/config.yaml" -if inventory_ok "$INCOMPLETE" "$GOOD_README" >"$WORK/inc.out" 2>"$WORK/inc.err"; then +INCOMPLETE_README="$WORK/readme.incomplete.md" +sed 's/`infra-vars`/`infra-vars`, `orphan`/' "$GOOD_README" >"$INCOMPLETE_README" +if inventory_ok "$INCOMPLETE" "$INCOMPLETE_README" >"$WORK/inc.out" 2>"$WORK/inc.err"; then fail "pack without .assent/tests/ stayed green" else - pass "pack directory without .assent/tests/ reddens" + if grep -q 'incomplete pack (no .assent/tests/): orphan' "$WORK/inc.err"; then + pass "pack directory without .assent/tests/ reddens" + else + fail "orphan mutation reddened for the wrong reason (confounded): $(cat "$WORK/inc.err")" + fi fi # --- happy path against the real tree (REQ-EX-S01-01) ------------------------- @@ -210,11 +244,7 @@ if inventory_ok "$ROOT" "$ROOT/examples/README.md" >"$WORK/real.out" 2>"$WORK/re echo "$report" if echo "$report" | grep -q 'PACKS: infra-vars service-catalog topic-registry' \ && echo "$report" | grep -q 'FORMATS: json tfvars yaml'; then - if echo "$report" | grep -qw 'tf' && ! echo "$report" | grep -q 'tfvars'; then - fail "REQ-EX-S01-01: reported bare tf before the S05 fixture" - else - pass "REQ-EX-S01-01: real tree reports the three packs and yaml/json/tfvars" - fi + pass "REQ-EX-S01-01: real tree reports the three packs and yaml/json/tfvars" else fail "REQ-EX-S01-01: unexpected report: $report" fi From b3180af09c1bb8f16ac54a515864df7f4048cc47 Mon Sep 17 00:00:00 2001 From: Konrad Heimel Date: Sun, 16 Aug 2026 00:58:48 +0200 Subject: [PATCH 4/4] :wrench: chore(release): regenerate CHANGELOG.md after the EX-S01 review fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843fcea4..09070e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,7 @@ repository still gets a decision, never by following the link; no release carrie ### Fixes - :bug: fix(ci): pin ci-audit-test in the AUD-S18 check-stage list - :bug: specs(p5-ex): make S10 schema freeze and S07 fence non-vacuous +- :bug: fix(docs-gates): fail on unmapped format tokens and de-confound the tests-dir mutation (EX-S01 review) ### Testing - :white_check_mark: test(release): anchor the D-120 note check on its header sentence, not the bare token