Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions .github/workflows/verify.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,8 @@ jobs:
- name: coverage gate (D-010 — threshold single-sourced from Taskfile.yml)
run: task coverage
- run: CGO_ENABLED=0 go build ./...
- name: dogfood examples (P5-E6-S08 — every non-locked pack gates itself green under the real `assent test` CLI)
run: |
CGO_ENABLED=0 go build -o bin/assent ./cmd/assent
for pack in service-catalog infra-vars topic-registry; do
./bin/assent test "examples/packs/$pack"
./bin/assent test --coverage "examples/packs/$pack"
done
- name: dogfood examples (EX-S08 — shared discovery script; every examples/packs/<name> with .assent/tests gates itself green under the real `assent test` CLI)
run: bash hack/dogfood-examples.sh
- name: comparison corpus dogfood (PCS-S08 — examples/comparison validates and runs green)
run: go test ./examples/comparison/...
- name: PCS compare exit gate (RELSE-03 — REQ-PCS-S09)
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ repository still gets a decision, never by following the link; no release carrie
- :memo: docs(adr): add ADR-0011 Amendment 4 — Rego/OPA rule-7 capability boundary
- :memo: docs(adr): add ADR-0015 host-side credential resolver amendment
- :memo: docs(adr): retcon ADR narrative to read as planned phases
- :memo: docs(changelog): regenerate CHANGELOG.md for PR #59's operator-ruling commits

### Features
- :sparkles: feat(docs): gate example pack and format claims against dogfood
Expand All @@ -124,6 +125,9 @@ repository still gets a decision, never by following the link; no release carrie
- :bug: specs(p5-ex): make S10 schema freeze and S07 fence non-vacuous
- :bug: fix(docs-gates): fail on unmapped format tokens and de-confound the tests-dir mutation (EX-S01 review)

### Refactoring
- :recycle: refactor(examples): discover dogfood packs instead of a hardcoded 3-name loop

### Testing
- :white_check_mark: test(release): anchor the D-120 note check on its header sentence, not the bare token
- :white_check_mark: test(release): key the merge-skip proof on commit shape, not subject prefix
Expand Down
24 changes: 16 additions & 8 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,10 @@ tasks:
- CGO_ENABLED=0 go build -ldflags "-X main.version=${ASSENT_VERSION:-0.0.0-dev}" -o bin/assent ./cmd/assent

dogfood-examples:
desc: "Dogfood gate (P5-E6-S08): every non-locked example pack gates itself green under the real `assent test` (+ --coverage) CLI"
desc: "Dogfood gate (EX-S08): every examples/packs/<name> with .assent/tests gates itself green under the real `assent test` (+ --coverage) CLI — discovered, not hardcoded"
deps: [build]
cmds:
- |
set -e
for pack in service-catalog infra-vars topic-registry; do
echo "== assent test examples/packs/$pack =="
./bin/assent test "examples/packs/$pack"
./bin/assent test --coverage "examples/packs/$pack"
done
- bash hack/dogfood-examples.sh

dogfood-comparison:
desc: "Dogfood gate (PCS-S08): comparison corpus validates and runs green under assent compare --suite"
Expand All @@ -83,6 +77,11 @@ tasks:
- task: test
- task: coverage
- task: build
# EX-S08: dogfood the shipped example packs through the real `assent test`
# CLI, sequential and AFTER build (D-124 lesson: a deps: race with fmt
# caused a prior bug — this stage runs the built binary, so it belongs
# after build like docs-gates runs after the tree is otherwise settled).
- task: dogfood-examples
- task: dogfood-comparison
- task: compare-exitgate-test
# AUD-S02 (REQ-AUD-S02-02): CHANGELOG.md drift gate. It is the LOCAL half of the
Expand Down Expand Up @@ -116,6 +115,10 @@ tasks:
# .github/workflows/**, so without this line an unpinned workflow is caught
# only by CI — and only for as long as the CI step itself survives.
- task: lint-workflow-pins-test
# EX-S08 (REQ-EX-S08-03): the dogfood-examples gate's own guard. Without
# this, deleting `- task: dogfood-examples` from check: above passes
# every wired gate — the same manual-gate defect D-124 closed elsewhere.
- task: dogfood-wiring-test
# Operator 2026-08-13: Dependabot is the updater; a reintroduced
# renovate.json would otherwise merge green.
- task: ci-audit-test
Expand All @@ -125,6 +128,11 @@ tasks:
cmds:
- bash hack/release/ci_audit_test.sh

dogfood-wiring-test:
desc: "EX-S08 gate: task check runs dogfood-examples after build, and Taskfile.yml/verify.yaml both delegate to hack/dogfood-examples.sh (REQ-EX-S08-02/03)"
cmds:
- bash hack/examples/dogfood_wiring_test.sh

lint-workflow-pins-test:
desc: "AUD-S09/S14 gate: workflow supply-chain pins are present and the checks can fail (REQ-AUD-S09-01, REQ-AUD-S14-01/02)"
cmds:
Expand Down
151 changes: 141 additions & 10 deletions cmd/assent/test_corpus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ import (
"bytes"
"os"
"path/filepath"
"reflect"
"regexp"
"sort"
"strings"
"testing"

Expand All @@ -42,18 +45,52 @@ import (

// greenExamplePacks are the non-locked example packs that gate themselves green under
// `assent test`. rego is locked (D-012) and carries no .assent/tests/** at all.
// Keep Taskfile.yml dogfood-examples + .github/workflows/verify.yaml dogfood step in
// sync (TestDogfoodScriptsIncludeGreenExamplePacks).
var greenExamplePacks = []string{"service-catalog", "infra-vars", "topic-registry"}
//
// EX-S08: this used to be a hardcoded three-name literal, duplicated by hand in
// Taskfile.yml's dogfood-examples task and .github/workflows/verify.yaml's dogfood
// step — a proven skew risk (a pack green in one, silently absent in another). It is
// now DISCOVERED from the filesystem via discoverGreenExamplePacks, walking the same
// examples/packs/*/.assent/tests contract that hack/dogfood-examples.sh (the script
// Taskfile.yml and verify.yaml both now call) and examples/README.md's inventory gate
// (EX-S01, hack/docs/example_format_inventory_test.sh) use — a fourth pack is picked
// up here without editing this file.
var greenExamplePacks = discoverGreenExamplePacks()

// discoverGreenExamplePacks walks examples/packs/*/.assent/tests: every immediate
// child of examples/packs/ with a .assent/tests/ subdirectory is a pack this gate
// dogfoods. Mirrors hack/dogfood-examples.sh's discovery rule exactly.
func discoverGreenExamplePacks() []string {
matches, err := filepath.Glob(filepath.Join(examplesPacksDir, "*", ".assent", "tests"))
if err != nil {
// filepath.Glob only errors on a malformed pattern; the pattern above is a
// compile-time constant, so this is unreachable short of test corruption.
panic("discoverGreenExamplePacks: " + err.Error())
}
packs := make([]string, 0, len(matches))
for _, m := range matches {
// m == examplesPacksDir/<pack>/.assent/tests
packs = append(packs, filepath.Base(filepath.Dir(filepath.Dir(m))))
}
sort.Strings(packs)
return packs
}

// brokenPackDir is the DELIBERATELY-broken fixture (a valid pack whose expect.yaml pins
// the wrong decision). It lives under testdata, never examples/packs, so the shipped
// corpus stays green while the failure path is still proven.
const brokenPackDir = "testdata/broken-pack"

// TestDogfoodScriptsIncludeGreenExamplePacks pins the CLI dogfood loops
// (Taskfile dogfood-examples + verify.yaml) to the same pack set as greenExamplePacks
// so unpinning a pack (EFE-S04 / topic-registry) cannot leave the shell loops stale.
// hardcodedPackLoopPattern matches the OLD `for pack in <name1> <name2> ...` shell
// loop shape (2+ space-separated tokens) that EX-S08 replaced. Mirrors
// hack/examples/dogfood_wiring_test.sh's hardcoded_pack_loop check.
var hardcodedPackLoopPattern = regexp.MustCompile(`for pack in [A-Za-z0-9_-]+ [A-Za-z0-9_-]+`)

// TestDogfoodScriptsIncludeGreenExamplePacks is REQ-EX-S08-02: Taskfile.yml's
// dogfood-examples task and verify.yaml's dogfood step both call the SHARED
// hack/dogfood-examples.sh discovery script rather than each carrying its own
// hardcoded pack-name loop. Before EX-S08 this test grepped for the three literal
// pack names in each file — passing on the redundant-but-correct loop AND on a
// hardcoded loop of DIFFERENT names, which is exactly the skew this story closes.
func TestDogfoodScriptsIncludeGreenExamplePacks(t *testing.T) {
files := []string{
filepath.Join("..", "..", "Taskfile.yml"),
Expand All @@ -65,13 +102,107 @@ func TestDogfoodScriptsIncludeGreenExamplePacks(t *testing.T) {
t.Fatalf("read %s: %v", f, err)
}
body := string(raw)
for _, pack := range greenExamplePacks {
// Match the shell `for pack in …` token list, not incidental prose.
if !strings.Contains(body, pack) {
t.Errorf("%s: dogfood loop missing green pack %q", f, pack)
if !strings.Contains(body, "hack/dogfood-examples.sh") {
t.Errorf("%s: does not invoke the shared hack/dogfood-examples.sh discovery script", f)
}
if hardcodedPackLoopPattern.MatchString(body) {
t.Errorf("%s: re-hardcodes a pack-name loop instead of delegating to hack/dogfood-examples.sh", f)
}
}
}

// TestGreenExamplePacksIsFilesystemDerived is REQ-EX-S08-05: greenExamplePacks is
// exactly the set of examples/packs/* directories with a .assent/tests/
// subdirectory — not a hardcoded literal that happens to match today's corpus. A
// hand-reverted `var greenExamplePacks = []string{...}` would still compile and
// would still pass every other test in this file (they only iterate the slice), so
// this pin independently recomputes the glob and compares.
func TestGreenExamplePacksIsFilesystemDerived(t *testing.T) {
matches, err := filepath.Glob(filepath.Join(examplesPacksDir, "*", ".assent", "tests"))
if err != nil {
t.Fatalf("glob: %v", err)
}
want := make([]string, 0, len(matches))
for _, m := range matches {
want = append(want, filepath.Base(filepath.Dir(filepath.Dir(m))))
}
sort.Strings(want)
if len(want) == 0 {
t.Fatal("filesystem glob discovered zero packs — this assertion would be vacuous")
}
got := append([]string(nil), greenExamplePacks...)
sort.Strings(got)
if !reflect.DeepEqual(got, want) {
t.Errorf("greenExamplePacks = %v, want filesystem-derived %v", got, want)
}
}

// TestBrokenPackFixtureIsNotDiscovered is the REQ-EX-S08-04 edge: the
// deliberately-broken CLI fixture (brokenPackDir) lives under cmd/assent/testdata,
// never under examples/packs/, precisely so the shipped corpus stays green while
// the failure path is still proven. Discovery walks examples/packs/* only, so it
// can never pick brokenPackDir up by construction — pinned explicitly so a future
// move of the fixture under examples/packs/ is caught here rather than silently
// turning every dogfood run red.
func TestBrokenPackFixtureIsNotDiscovered(t *testing.T) {
for _, pack := range greenExamplePacks {
if pack == "broken-pack" {
t.Fatalf("greenExamplePacks discovered the testdata broken-pack fixture: %v", greenExamplePacks)
}
}
if _, err := os.Stat(filepath.Join(examplesPacksDir, "broken-pack")); err == nil {
t.Fatal("broken-pack must not exist under examples/packs/ — the deliberately-broken fixture belongs under cmd/assent/testdata only")
}
}

// readmePackNames extracts the backtick-quoted pack names from examples/README.md's
// `[`packs/`]` bullet (and its wrapped continuation line), mirroring
// hack/docs/example_format_inventory_test.sh's readme_packs() exactly.
func readmePackNames(t *testing.T, readmePath string) []string {
t.Helper()
raw, err := os.ReadFile(readmePath) //nolint:gosec // fixed in-repo docs path.
if err != nil {
t.Fatalf("read %s: %v", readmePath, err)
}
backtick := regexp.MustCompile("`([a-z0-9-]+)`")
inPacks := false
var names []string
for _, line := range strings.Split(string(raw), "\n") {
if strings.Contains(line, "[`packs/`]") {
inPacks = true
} else if inPacks && (strings.TrimSpace(line) == "" ||
(strings.HasPrefix(strings.TrimSpace(line), "- [`"))) {
break
}
if !inPacks {
continue
}
for _, m := range backtick.FindAllStringSubmatch(line, -1) {
if m[1] != "packs" {
names = append(names, m[1])
}
}
}
sort.Strings(names)
return names
}

// TestExampleCorpusDiscoveryMatchesReadmeInventory is the second half of
// REQ-EX-S08-02/05 ("the walk matches what README/inventory claims"): the
// filesystem-discovered greenExamplePacks must equal the pack names
// examples/README.md's `[`packs/`]` bullet advertises — the same equality EX-S01's
// hack/docs/example_format_inventory_test.sh proves from the shell side. Two
// independent readers (bash awk vs. Go regex) agreeing on the same filesystem
// contract is the point: a drift only one of them would catch is exactly what this
// duplication is meant to rule out.
func TestExampleCorpusDiscoveryMatchesReadmeInventory(t *testing.T) {
readmePath := filepath.Join("..", "..", "examples", "README.md")
docPacks := readmePackNames(t, readmePath)
fsPacks := append([]string(nil), greenExamplePacks...)
sort.Strings(fsPacks)
if !reflect.DeepEqual(docPacks, fsPacks) {
t.Errorf("examples/README.md packs %v != filesystem discovery %v (EX-S01 inventory drift)", docPacks, fsPacks)
}
}

// TestAllExamplePacksGreenUnderAssentTest is REQ-E6-S08-01: every non-locked
Expand Down
55 changes: 55 additions & 0 deletions hack/dogfood-examples.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# hack/dogfood-examples.sh — EX-S08 shared discovery script.
#
# Before this script existed, "run every example pack through `assent test`"
# was a hardcoded `for pack in service-catalog infra-vars topic-registry` loop
# duplicated in THREE places: Taskfile.yml's dogfood-examples task,
# .github/workflows/verify.yaml's dogfood step, and the greenExamplePacks Go
# slice in cmd/assent/test_corpus_test.go. Adding a pack meant editing all
# three in sync — a proven skew risk (a pack green in one, silently absent in
# another). Taskfile.yml and verify.yaml now both call THIS script; the Go
# corpus test discovers packs the same way (filepath.Glob over
# examples/packs/*/.assent/tests), so all three halves walk the same
# filesystem contract instead of carrying independent copies of a name list.
#
# Discovery rule (matches examples/README.md's inventory gate,
# hack/docs/example_format_inventory_test.sh, EX-S01): every immediate child
# of examples/packs/ that has a .assent/tests/ subdirectory is a pack to
# dogfood. A pack directory WITHOUT .assent/tests/ is not silently skipped —
# hack/docs/example_format_inventory_test.sh treats that as a hard error
# (incomplete tree) — so this script only ever sees complete packs, and an
# incomplete/red one that DOES have .assent/tests/ fails loudly below instead
# of being filtered out by name.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"

BIN="${ASSENT_BIN:-bin/assent}"
if [[ ! -x "$BIN" ]]; then
echo "== building $BIN =="
CGO_ENABLED=0 go build -o "$BIN" ./cmd/assent
fi

packs=()
for dir in examples/packs/*/; do
[[ -d "$dir" ]] || continue
name="$(basename "$dir")"
if [[ -d "${dir}.assent/tests" ]]; then
packs+=("$name")
fi
done

if [[ "${#packs[@]}" -eq 0 ]]; then
echo "no example packs with .assent/tests/ discovered under examples/packs/" >&2
exit 1
fi

echo "discovered packs: ${packs[*]}"
for pack in "${packs[@]}"; do
echo "== assent test examples/packs/$pack =="
"$BIN" test "examples/packs/$pack"
"$BIN" test --coverage "examples/packs/$pack"
done

echo "OK: dogfooded ${#packs[@]} example pack(s): ${packs[*]}"
Loading