diff --git a/.agents/skills/sdlc/SKILL.md b/.agents/skills/sdlc/SKILL.md index 1e67627..e9ef3e1 100644 --- a/.agents/skills/sdlc/SKILL.md +++ b/.agents/skills/sdlc/SKILL.md @@ -33,6 +33,7 @@ Use this skill for implementation, bug-fix, refactor, testing, release, publish, Reviewer role: inspect the frozen diff and return prioritized code-review findings only; do not edit, implement, run tests, re-plan, or perform follow-up work. The builder owns every correction through the normal SDLC loop. `review_model` controls native Codex review model selection but does not set review reasoning independently. `auto_review` is for eligible approval prompts, not code-diff review. Do not require `/autoreview` unless the current Codex host exposes it as a verified feature. At each coherent green slice, author-review the exact incremental diff before committing. Once the cumulative candidate is stable, freeze it, run one fresh broad proof, and review the full base-to-candidate diff once. A relevant correction invalidates that completion proof; use narrow delta checks while fixing, then run a fresh final proof. + Incremental checkpoint: use affected proof, exact-diff author review, and at most one risk-based reviewer before committing a coherent green slice. During the ten-delivery pilot, the completion boundary reviews the whole base-to-candidate diff with Sol High and then Fable High; outside the pilot, use Fable only when cross-model policy requires it. A finding produces one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. Record ten-delivery pilot outcomes in `benchmarks/review-cadence.csv` before making this cadence permanent. Severity ladder: P0 stops the line; P1 blocks completion; P2 is a bounded fix now or a follow-up issue; P3 never blocks and is recorded only when worthwhile. When two reviewers are required, they assess the same frozen candidate independently, exchange compact findings once, and return a joint ledger. Allow at most two corrective rounds. If P0/P1 remains, decompose, abandon, or escalate; never waive it or continue an unbounded review loop. Run Fable High only after Sol is clean and only when cross-model policy requires it: `node .codex/hooks/fable-review.cjs --base --consent-subscription-quota`. The explicit consent acknowledges Claude subscription-quota use; the wrapper rejects API-key and alternate-provider lanes, disables tools/MCP/session persistence, reuses the current proof, and binds its receipt to the frozen staged candidate. diff --git a/README.md b/README.md index 7be9611..96bf91c 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,12 @@ node .codex/hooks/fable-review.cjs --base main --consent-subscription-quota The consent flag is required because the review consumes Claude subscription quota. The wrapper verifies Claude first-party subscription auth, refuses API keys and alternate providers, disables tools/MCP/session persistence, reuses the current SDLC proof, and writes a candidate-bound receipt under Git metadata. It does not create a metered API-key charge when the verified subscription lane is used. +### Incremental checkpoints and the completion boundary + +For each coherent green slice, run affected proof, author-review the exact incremental diff, and use at most one risk-based reviewer before committing. During the ten-delivery pilot, the completion boundary is deliberately broader: freeze the candidate, run the broad proof once, and have Sol High and then Fable High review the whole base-to-candidate diff. Outside the pilot, use Fable only when cross-model policy requires it. Fix a blocker as one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. + +This cadence is a measured ten-delivery pilot, not permanent ceremony. Record delivery, duplicate-proof, per-reviewer disposition and confidence, reconciliation, quota/token cost, correction, tripwire, CI, milestone, and release outcomes in `benchmarks/review-cadence.csv`, then run `bash scripts/summarize-review-cadence.sh`. After ten eligible deliveries across at least two strategies, a human compares the arms and chooses whether to keep, tune, or sunset it. + ## Repo-Scoped Skills `install.sh` and `setup.sh` scaffold repo-local Codex skills under `.agents/skills`. diff --git a/SDLC-LOOP.md b/SDLC-LOOP.md index c1e925c..46e5f40 100644 --- a/SDLC-LOOP.md +++ b/SDLC-LOOP.md @@ -21,6 +21,7 @@ Codex does not have a native `/sdlc` command. This file is the honest replacemen Author-review the exact incremental diff, note risks, and remove junk before each coherent green commit. 7. Commit only after proof Commit coherent green slices after focused proof. Freeze the cumulative completion candidate and run one fresh broad proof before final review; relevant changes invalidate it and require a fresh final proof. + Incremental checkpoint: use affected proof, exact-diff author review, and at most one risk-based reviewer before each coherent green commit. During the ten-delivery pilot, the completion boundary reviews the whole base-to-candidate diff with Sol High and then Fable High; outside the pilot, use Fable only when cross-model policy requires it. Fix a blocker as one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. Record the ten-delivery pilot in `benchmarks/review-cadence.csv` before making this cadence permanent. 8. Review to a decision Review the full base-to-candidate diff once after it is stable. Severity ladder: P0 stops the line; P1 blocks completion; P2 is a bounded fix now or a follow-up issue; P3 never blocks and is recorded only when worthwhile. Run one broad proof run total on the frozen candidate through `node .codex/hooks/git-guard.cjs prove --reviewed`; do not run the suite directly and then rerun it through the guard. diff --git a/benchmarks/review-cadence.csv b/benchmarks/review-cadence.csv new file mode 100644 index 0000000..98f638c --- /dev/null +++ b/benchmarks/review-cadence.csv @@ -0,0 +1 @@ +delivery_id,repo,issue_id,strategy,eligible,stable_base,candidate_tree,diff_files,broad_proof_runs,duplicate_broad_proof_runs,sol_review_minutes,fable_review_minutes,sol_pre_confidence,fable_pre_confidence,sol_post_confidence,fable_post_confidence,sol_pre_disposition,fable_pre_disposition,sol_post_disposition,fable_post_disposition,sol_disposition_change_reason,fable_disposition_change_reason,reconciliation_rounds,reconciliation_skipped,reconciliation_ledger_entries,unique_second_reviewer_blockers,corrective_rounds,tripwire_count,red_on_main,sol_quota_cost,fable_quota_cost,sol_token_cost,fable_token_cost,issue_closed,milestone_closed,release_shipped,delivery_minutes,notes diff --git a/scripts/summarize-review-cadence.sh b/scripts/summarize-review-cadence.sh new file mode 100755 index 0000000..1b1b0fe --- /dev/null +++ b/scripts/summarize-review-cadence.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Summarize the bounded ten-delivery review-cadence pilot. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CSV_PATH="${1:-$REPO_DIR/benchmarks/review-cadence.csv}" +EXPECTED_HEADER='delivery_id,repo,issue_id,strategy,eligible,stable_base,candidate_tree,diff_files,broad_proof_runs,duplicate_broad_proof_runs,sol_review_minutes,fable_review_minutes,sol_pre_confidence,fable_pre_confidence,sol_post_confidence,fable_post_confidence,sol_pre_disposition,fable_pre_disposition,sol_post_disposition,fable_post_disposition,sol_disposition_change_reason,fable_disposition_change_reason,reconciliation_rounds,reconciliation_skipped,reconciliation_ledger_entries,unique_second_reviewer_blockers,corrective_rounds,tripwire_count,red_on_main,sol_quota_cost,fable_quota_cost,sol_token_cost,fable_token_cost,issue_closed,milestone_closed,release_shipped,delivery_minutes,notes' + +if [ ! -f "$CSV_PATH" ]; then + echo "missing ledger: $CSV_PATH" >&2 + exit 1 +fi + +IFS= read -r actual_header < "$CSV_PATH" +if [ "$actual_header" != "$EXPECTED_HEADER" ]; then + echo "review cadence ledger header does not match the required schema" >&2 + exit 2 +fi + +awk -F, -v expected_header="$EXPECTED_HEADER" ' + NR == 1 { + expected_fields = split(expected_header, required_header, ",") + for (field_index = 1; field_index <= NF; field_index++) column[$field_index] = field_index + next + } + + NF != expected_fields { + printf "malformed review cadence row %d: expected %d fields, got %d; free-text fields must not contain commas\n", NR, expected_fields, NF > "/dev/stderr" + exit 2 + } + + $(column["eligible"]) == 1 { + strategy = $(column["strategy"]) + strategies[strategy] = 1 + strategy_count[strategy]++ + eligible++ + delivered += $(column["issue_closed"]) + milestone_closed += $(column["milestone_closed"]) + release_shipped += $(column["release_shipped"]) + duplicate_proof += $(column["duplicate_broad_proof_runs"]) + unique_blockers += $(column["unique_second_reviewer_blockers"]) + corrective_rounds += $(column["corrective_rounds"]) + tripwires += $(column["tripwire_count"]) + red_on_main += $(column["red_on_main"]) + + strategy_review_minutes[strategy] += $(column["sol_review_minutes"]) + $(column["fable_review_minutes"]) + strategy_reconciliation_rounds[strategy] += $(column["reconciliation_rounds"]) + strategy_quota_cost[strategy] += $(column["sol_quota_cost"]) + $(column["fable_quota_cost"]) + strategy_token_cost[strategy] += $(column["sol_token_cost"]) + $(column["fable_token_cost"]) + + if ($(column["delivery_minutes"]) != "") { + delivery_minutes += $(column["delivery_minutes"]) + delivery_minutes_count++ + } + + if ($(column["sol_pre_confidence"]) != "" && $(column["sol_post_confidence"]) != "" && + $(column["fable_pre_confidence"]) != "" && $(column["fable_post_confidence"]) != "") { + strategy_confidence_delta[strategy] += (($(column["sol_post_confidence"]) - $(column["sol_pre_confidence"])) + ($(column["fable_post_confidence"]) - $(column["fable_pre_confidence"]))) / 2 + strategy_confidence_count[strategy]++ + } + } + + END { + printf "eligible_delivery_count: %d\n", eligible + printf "delivered_count: %d\n", delivered + if (eligible == 0) print "delivery_rate: n/a" + else printf "delivery_rate: %.2f%%\n", (delivered / eligible) * 100 + printf "milestone_closed_count: %d\n", milestone_closed + printf "release_shipped_count: %d\n", release_shipped + printf "duplicate_broad_proof_runs: %d\n", duplicate_proof + printf "unique_second_reviewer_blockers: %d\n", unique_blockers + printf "corrective_round_count: %d\n", corrective_rounds + printf "tripwire_count: %d\n", tripwires + printf "red_on_main_count: %d\n", red_on_main + if (delivery_minutes_count == 0) print "average_delivery_minutes: n/a" + else printf "average_delivery_minutes: %.2f\n", delivery_minutes / delivery_minutes_count + + strategy_total = 0 + for (strategy in strategies) { + strategy_total++ + printf "strategy.%s.eligible_delivery_count: %d\n", strategy, strategy_count[strategy] + printf "strategy.%s.average_review_minutes: %.2f\n", strategy, strategy_review_minutes[strategy] / strategy_count[strategy] + printf "strategy.%s.reconciliation_round_count: %d\n", strategy, strategy_reconciliation_rounds[strategy] + if (strategy_confidence_count[strategy] == 0) printf "strategy.%s.average_confidence_delta: n/a\n", strategy + else printf "strategy.%s.average_confidence_delta: %.2f\n", strategy, strategy_confidence_delta[strategy] / strategy_confidence_count[strategy] + printf "strategy.%s.total_quota_cost: %.2f\n", strategy, strategy_quota_cost[strategy] + printf "strategy.%s.total_token_cost: %.0f\n", strategy, strategy_token_cost[strategy] + } + + if (eligible >= 10 && strategy_total >= 2) { + print "recommendation: human-evaluate-pilot" + print "reason: ten eligible deliveries across at least two strategies are recorded; a human must choose keep, tune, or sunset" + } else { + print "recommendation: continue-pilot" + print "reason: record at least ten eligible deliveries across at least two strategies" + } + } +' "$CSV_PATH" diff --git a/skill-sources/sdlc/SKILL.template.md b/skill-sources/sdlc/SKILL.template.md index d7b9307..8107526 100644 --- a/skill-sources/sdlc/SKILL.template.md +++ b/skill-sources/sdlc/SKILL.template.md @@ -109,6 +109,8 @@ Reviewer role: inspect the frozen diff and return prioritized code-review findin At each coherent green slice, author-review the exact incremental diff before committing. Once the cumulative candidate is stable, freeze it, run one fresh broad proof, and review the full base-to-candidate diff once. A relevant correction invalidates that completion proof; use narrow delta checks while fixing, then run a fresh final proof. +Incremental checkpoint: use affected proof, exact-diff author review, and at most one risk-based reviewer before committing a coherent green slice. During the ten-delivery pilot, the completion boundary reviews the whole base-to-candidate diff with Sol High and then Fable High; outside the pilot, use Fable only when cross-model policy requires it. A finding produces one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. Record ten-delivery pilot outcomes in `benchmarks/review-cadence.csv` before making this cadence permanent. + Severity ladder: P0 stops the line; P1 blocks completion; P2 is a bounded fix now or a follow-up issue; P3 never blocks and is recorded only when worthwhile. When two reviewers are required, they assess the same frozen candidate independently, exchange compact findings once, and return a joint ledger. Allow at most two corrective rounds. If P0/P1 remains, decompose, abandon, or escalate; never waive it or continue an unbounded review loop. diff --git a/templates/AGENTS.baseline.md b/templates/AGENTS.baseline.md index 4254028..ebdb37f 100644 --- a/templates/AGENTS.baseline.md +++ b/templates/AGENTS.baseline.md @@ -12,6 +12,7 @@ Read `TESTING.md` and `ARCHITECTURE.md` when present and relevant. If `GOALS.md` 4. Run focused checks, the broader relevant suite, and a self-review before commit. 5. Never claim completion without fresh proof. 6. Author-review and commit coherent green slices. Freeze the cumulative candidate for one fresh broad proof and completion review. + Incremental checkpoint: use affected proof, exact-diff author review, and at most one risk-based reviewer before a coherent green commit. During the ten-delivery pilot, the completion boundary reviews the whole base-to-candidate diff with Sol High and then Fable High; outside the pilot, use Fable only when cross-model policy requires it. A blocker becomes one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. Record ten-delivery pilot outcomes in `benchmarks/review-cadence.csv` before making the cadence permanent. 7. Severity ladder: P0 stops the line; P1 blocks completion; P2 is a bounded fix or follow-up issue; P3 never blocks. When two reviewers are required, they exchange compact findings once. Allow at most two corrective rounds; unresolved P0/P1 requires decomposition, abandonment, or escalation. Run one broad proof run total on the frozen candidate through `node .codex/hooks/git-guard.cjs prove --reviewed`; do not run the suite directly and then rerun it through the guard. Use a prompt-only review when supplying custom proof-aware instructions. A custom prompt must not be combined with `--uncommitted`, `--base`, or `--commit`; those predefined target flags are for reviews without a custom prompt. Include the exact base identity, frozen candidate tree identity, proof command, and result and say `Do not rerun tests`. Targeted verification is allowed only for a concrete suspected defect; never rerun the broad suite. Stale proof is a blocker to report, not permission to launch another broad suite. diff --git a/templates/AGENTS.md.tmpl b/templates/AGENTS.md.tmpl index d6c6629..e8378f0 100644 --- a/templates/AGENTS.md.tmpl +++ b/templates/AGENTS.md.tmpl @@ -39,6 +39,7 @@ Use skills for the visible workflow contract, let hooks enforce silently, and ke 4. **Verify incrementally:** run focused proof for each coherent green slice; run the full required proof on the frozen completion candidate 5. **Active goals:** When `GOALS.md` exists, complete that active scope before claiming the run is done; do not confuse active goal completion with roadmap completion. 6. **Review to a decision:** author-review each incremental diff, then review the stable cumulative candidate once. Severity ladder: P0 stops the line; P1 blocks completion; P2 is a bounded fix or follow-up issue; P3 never blocks. When two reviewers are required, they exchange compact findings once. Allow at most two corrective rounds; unresolved P0/P1 requires decomposition, abandonment, or escalation. + - Incremental checkpoint: use affected proof, exact-diff author review, and at most one risk-based reviewer before a coherent green commit. During the ten-delivery pilot, the completion boundary reviews the whole base-to-candidate diff with Sol High and then Fable High; outside the pilot, use Fable only when cross-model policy requires it. A blocker becomes one bounded corrective delta with targeted proof. A third same-plan correction means stop; human approval may authorize a replan with newly scoped work, not silently extend the exhausted plan. Record ten-delivery pilot outcomes in `benchmarks/review-cadence.csv` before making the cadence permanent. - Run one broad proof run total on the frozen candidate through `node .codex/hooks/git-guard.cjs prove --reviewed`; do not run the suite directly and then rerun it through the guard. - Use a prompt-only review when supplying custom proof-aware instructions. A custom prompt must not be combined with `--uncommitted`, `--base`, or `--commit`; those predefined target flags are for reviews without a custom prompt. Include the exact base identity, frozen candidate tree identity, proof command, and result and say `Do not rerun tests`. Targeted verification is allowed only for a concrete suspected defect; never rerun the broad suite. Stale proof is a blocker to report, not permission to launch another broad suite. - Reviewer role: inspect the frozen diff and return prioritized code-review findings only; do not edit, implement, run tests, re-plan, or perform follow-up work. The builder owns every correction through the normal SDLC loop. diff --git a/tests/test-benchmark.sh b/tests/test-benchmark.sh index 759581b..73e48a0 100644 --- a/tests/test-benchmark.sh +++ b/tests/test-benchmark.sh @@ -9,6 +9,8 @@ LEDGER="$REPO_DIR/benchmarks/model-experiment.csv" SUMMARY_SCRIPT="$REPO_DIR/scripts/summarize-model-experiment.sh" PILOT_LEDGER="$REPO_DIR/benchmarks/pilot-rollout.csv" PILOT_SUMMARY_SCRIPT="$REPO_DIR/scripts/summarize-pilot-rollout.sh" +REVIEW_LEDGER="$REPO_DIR/benchmarks/review-cadence.csv" +REVIEW_SUMMARY_SCRIPT="$REPO_DIR/scripts/summarize-review-cadence.sh" PASSED=0 FAILED=0 @@ -209,6 +211,103 @@ CSV rm -rf "$ws" } +test_review_cadence_ledger_exists_with_required_headers() { + local has_file=true + local header expected_header + + [ -f "$REVIEW_LEDGER" ] || has_file=false + header="$(head -n 1 "$REVIEW_LEDGER" 2>/dev/null || true)" + expected_header='delivery_id,repo,issue_id,strategy,eligible,stable_base,candidate_tree,diff_files,broad_proof_runs,duplicate_broad_proof_runs,sol_review_minutes,fable_review_minutes,sol_pre_confidence,fable_pre_confidence,sol_post_confidence,fable_post_confidence,sol_pre_disposition,fable_pre_disposition,sol_post_disposition,fable_post_disposition,sol_disposition_change_reason,fable_disposition_change_reason,reconciliation_rounds,reconciliation_skipped,reconciliation_ledger_entries,unique_second_reviewer_blockers,corrective_rounds,tripwire_count,red_on_main,sol_quota_cost,fable_quota_cost,sol_token_cost,fable_token_cost,issue_closed,milestone_closed,release_shipped,delivery_minutes,notes' + + if [ "$has_file" = "true" ] && [ "$header" = "$expected_header" ]; then + pass "Review cadence ledger exists with the ten-delivery pilot schema" + else + fail "Review cadence ledger is missing or does not have the required pilot schema" + fi +} + +test_review_cadence_summary_rejects_schema_drift() { + local ws malformed missing_header + ws="$(mktemp -d)" + malformed="$ws/malformed.csv" + missing_header="$ws/missing-header.csv" + + head -n 1 "$REVIEW_LEDGER" > "$malformed" + printf '%s\n' 'd01,repo,1,incremental,1,a,b,4,1,0,2,2,90,90,90,90,clean,clean,clean,clean,reason,with-comma,reason,0,1,,0,0,0,0,1,1,100,100,1,0,0,20,note' >> "$malformed" + printf '%s\n' 'delivery_id,repo,issue_id' > "$missing_header" + + if "$REVIEW_SUMMARY_SCRIPT" "$malformed" >/dev/null 2>&1 || + "$REVIEW_SUMMARY_SCRIPT" "$missing_header" >/dev/null 2>&1; then + fail "Review cadence summary accepts malformed rows or missing required headers" + else + pass "Review cadence summary rejects malformed rows and missing required headers" + fi + + rm -rf "$ws" +} + +test_review_cadence_summary_default_is_repo_relative() { + local ws + ws="$(mktemp -d)" + + if (cd "$ws" && "$REVIEW_SUMMARY_SCRIPT" >/dev/null); then + pass "Review cadence summary resolves its default ledger outside the repo root" + else + fail "Review cadence summary default ledger depends on the caller working directory" + fi + + rm -rf "$ws" +} + +test_review_cadence_summary_script_exists() { + if [ -x "$REVIEW_SUMMARY_SCRIPT" ]; then + pass "Review cadence summary script exists and is executable" + else + fail "Review cadence summary script is missing or not executable" + fi +} + +test_review_cadence_summary_reports_delivery_outcomes() { + local ws fixture output + ws="$(mktemp -d)" + fixture="$ws/review-cadence.csv" + + cat > "$fixture" <<'CSV' +delivery_id,repo,issue_id,strategy,eligible,stable_base,candidate_tree,diff_files,broad_proof_runs,duplicate_broad_proof_runs,sol_review_minutes,fable_review_minutes,sol_pre_confidence,fable_pre_confidence,sol_post_confidence,fable_post_confidence,sol_pre_disposition,fable_pre_disposition,sol_post_disposition,fable_post_disposition,sol_disposition_change_reason,fable_disposition_change_reason,reconciliation_rounds,reconciliation_skipped,reconciliation_ledger_entries,unique_second_reviewer_blockers,corrective_rounds,tripwire_count,red_on_main,sol_quota_cost,fable_quota_cost,sol_token_cost,fable_token_cost,issue_closed,milestone_closed,release_shipped,delivery_minutes,notes +d01,repo,1,monolithic,1,a,b,4,1,0,4,3,90,91,90,91,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,1000,800,1,0,0,30,clean +d02,repo,2,incremental,1,a,b,4,1,0,2,2,80,70,87,85,block,block,clean,clean,fixed-a,fixed-b,1,0,ledger-1,1,1,0,0,1,1,900,700,1,0,0,25,reconciled +d03,repo,3,monolithic,1,a,b,4,1,0,4,3,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,1000,800,1,0,0,31,clean +d04,repo,4,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,22,clean +d05,repo,5,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,23,clean +d06,repo,6,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,24,clean +d07,repo,7,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,25,clean +d08,repo,8,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,26,clean +d09,repo,9,incremental,1,a,b,4,1,0,2,2,,,,,clean,clean,clean,clean,,,0,1,,0,0,0,0,1,1,900,700,1,0,0,27,clean +d10,repo,10,incremental,1,a,b,4,1,0,2,2,,,,,block,block,block,block,,,0,1,,0,2,1,0,1,1,900,700,0,0,0,30,breaker +CSV + + output="$("$REVIEW_SUMMARY_SCRIPT" "$fixture")" + + if echo "$output" | grep -q 'eligible_delivery_count: 10' && + echo "$output" | grep -q 'delivered_count: 9' && + echo "$output" | grep -q 'duplicate_broad_proof_runs: 0' && + echo "$output" | grep -q 'red_on_main_count: 0' && + echo "$output" | grep -q 'tripwire_count: 1' && + echo "$output" | grep -q 'strategy.monolithic.eligible_delivery_count: 2' && + echo "$output" | grep -q 'strategy.monolithic.average_review_minutes: 7.00' && + echo "$output" | grep -q 'strategy.incremental.eligible_delivery_count: 8' && + echo "$output" | grep -q 'strategy.incremental.average_review_minutes: 4.00' && + echo "$output" | grep -q 'strategy.incremental.reconciliation_round_count: 1' && + echo "$output" | grep -q 'strategy.incremental.average_confidence_delta: 11.00' && + echo "$output" | grep -q 'recommendation: human-evaluate-pilot'; then + pass "Review cadence summary reports delivery, waste, and termination outcomes" + else + fail "Review cadence summary does not report the ten-delivery decision metrics" + fi + + rm -rf "$ws" +} + test_benchmark_ledger_exists_with_required_headers test_benchmark_summary_script_exists test_benchmark_summary_script_reports_thresholds_and_recommendation @@ -217,6 +316,11 @@ test_pilot_rollout_ledger_exists_with_required_headers test_pilot_rollout_summary_script_exists test_pilot_rollout_summary_recommends_default_use_when_gate_is_met test_pilot_rollout_summary_holds_default_use_when_reusable_bug_count_is_too_high +test_review_cadence_ledger_exists_with_required_headers +test_review_cadence_summary_script_exists +test_review_cadence_summary_rejects_schema_drift +test_review_cadence_summary_default_is_repo_relative +test_review_cadence_summary_reports_delivery_outcomes echo "" echo "=== Results: $PASSED passed, $FAILED failed ===" diff --git a/tests/test-skill.sh b/tests/test-skill.sh index 02e00ba..cf3e45f 100644 --- a/tests/test-skill.sh +++ b/tests/test-skill.sh @@ -522,6 +522,29 @@ test_sdlc_documents_bounded_fable_review() { fi } +test_sdlc_documents_incremental_completion_cadence() { + local file + local valid=true + + for file in "$REPO_SDLC_SKILL" "$SHIPPED_SDLC_SKILL" "$SDLC_LOOP" "$AGENTS_BASELINE" "$AGENTS_TEMPLATE"; do + grep -Fqi 'incremental checkpoint' "$file" || valid=false + grep -Eqi 'at most one risk-based reviewer|one risk-based reviewer at most' "$file" || valid=false + grep -Fqi 'completion boundary' "$file" || valid=false + grep -Eqi 'during (this |the )?ten-delivery pilot|when cross-model policy requires it' "$file" || valid=false + grep -Eqi 'whole base-to-candidate|complete base-to-candidate' "$file" || valid=false + grep -Fqi 'corrective delta' "$file" || valid=false + grep -Eqi 'third same-plan correction.*stop|third correction.*stop' "$file" || valid=false + grep -Eqi 'human approval.*replan|human.*authoriz.*new plan' "$file" || valid=false + grep -Fqi 'ten-delivery' "$file" || valid=false + done + + if [ "$valid" = "true" ]; then + pass "SDLC workflow distinguishes incremental checkpoints from the final completion gate" + else + fail "SDLC workflow is missing the incremental checkpoint, completion, correction, or pilot contract" + fi +} + test_skill_manifest_exists test_plugin_skill_resolves_bundled_scripts_from_plugin_root test_plugin_skill_handles_legacy_standalone_install @@ -541,6 +564,7 @@ test_repo_scoped_sdlc_skill_documents_native_review test_sdlc_workflow_is_bounded_and_repairable test_sdlc_review_reuses_one_broad_proof test_sdlc_documents_bounded_fable_review +test_sdlc_documents_incremental_completion_cadence echo "" echo "=== Results: $PASSED passed, $FAILED failed ==="