Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,22 @@ documentation files it did not read and which directories they are in, so a clea
over two files in a project with three hundred cannot be mistaken for a clean report over
three hundred. See [what gets read](#what-gets-read).

**That sentence was printed in the header and the verdict was at the bottom, which is not the
same promise.** A run would end `Nothing contradicted. 149 claims checked` with the coverage
note forty lines above it, and the line anyone quotes from a CI log is the last one. So the
verdict now carries it:

```
1 broken, 149 checked, 497 not judged.
This judged 11 of 383 documentation file(s). 372 were never read, so this verdict
covers 2% of the documentation in this project.
```

That is a real run against a real repository. It was measured: over nine public repositories
docproof read **972 of 3,782** documentation files under the default scope, and re-running two
of them across the whole tree took one from 1 finding to 19 and another from 23 to 111. Those
runs were never clean; they were narrow, and only the header said so.

## What it checks

Two things, each done properly:
Expand Down
5 changes: 3 additions & 2 deletions src/docproof/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,8 @@ def main(argv: Sequence[str] | None = None) -> int:
outcomes.append(outcome)
report = Report(project=project, outcomes=outcomes)
print(f"docproof {__version__} — {project.root.name}, {len(documents)} document(s)")
report_coverage(project, unread_documents(project.root, in_scope, tracked))
unread = unread_documents(project.root, in_scope, tracked)
report_coverage(project, unread)
report_set_aside(historical, disclaimed)
# Same principle, one level down, and it applies harder: nobody asked for this rule.
# A `Before:` label is the tool deciding by itself that a block is not a claim, so the
Expand All @@ -306,7 +307,7 @@ def main(argv: Sequence[str] | None = None) -> int:
if superseded:
print(f" labelled superseded by the prose above, not judged: {', '.join(sorted(superseded))}")
print()
print(report.render(show_skips=args.show_skips))
print(report.render(show_skips=args.show_skips, read=len(documents), unread=len(unread)))
if args.exit_zero and not report.stopped_checking:
return 0
return report.exit_code
Expand Down
25 changes: 24 additions & 1 deletion src/docproof/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,21 @@ def _silence_line(self, outcome: Outcome) -> str:
marker = WARN if verdict.alarming or verdict.kind is Silence.UNKNOWN else DASH
return f"{marker} {outcome.verifier}: found nothing to check — {verdict.detail}."

def render(self, *, show_skips: bool = False) -> str:
def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) -> str:
"""`read` and `unread` put the document-level coverage INTO the verdict.

**Measured defect, 2026-08-19.** The coverage note prints in the header and the verdict
prints at the bottom, so the last line of a run said `Nothing contradicted. 28 claims
checked` while 18 of that project's 26 documentation files had never been opened. The
README promises the opposite in as many words: *"a clean report over two files in a
project with three hundred cannot be mistaken for a clean report over three hundred."*
Printed forty lines apart, it can be, and the line people quote is the last one.

The measurement that forced this: across nine real repositories docproof read **972 of
3,782** documentation files, 25.7%. Re-run over the whole tree, langwatch went from 1
broken to 19 and cherry-studio from 23 to 111. Those runs were not clean, they were
narrow, and only the header said so.
"""
lines: list[str] = []
root = self.project.root

Expand Down Expand Up @@ -129,4 +143,13 @@ def render(self, *, show_skips: bool = False) -> str:
lines.append(f"Nothing contradicted. {self.checked} claims checked, {self.skipped} not judged.")
if self.skipped and not show_skips:
lines.append("Run with --show-skips to see what was left unjudged and why.")
# Attached to the verdict itself, and to the BROKEN verdict too: "19 broken" over a
# fifth of the tree is as easy to misread as "nothing contradicted" over a fifth.
if unread:
total = read + unread
lines.append(
f"This judged {read} of {total} documentation file(s). "
f"{unread} were never read, so this verdict covers "
f"{100 * read // total}% of the documentation in this project."
)
return "\n".join(lines)
56 changes: 56 additions & 0 deletions tests/test_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,59 @@ def test_the_historical_list_is_capped_and_says_the_remainder(make_repo: Callabl
# The whole block stays on one line, so its length is the thing that was wrong before.
line = next(ln for ln in out.splitlines() if "describing the past" in ln)
assert len(line) < 500, line[:200]


def test_the_verdict_itself_says_how_much_it_covered(make_repo: Callable[..., Path], capsys) -> None:
"""**The coverage note was in the header and the verdict was at the bottom.**

A run ended `Nothing contradicted. 28 claims checked` while 18 of that project's 26
documentation files had never been opened. The README promises the opposite in as many
words: *"a clean report over two files in a project with three hundred cannot be mistaken
for a clean report over three hundred."* Forty lines apart, it can be, and the line people
quote from a CI log is the last one.

Measured before it was changed: across nine real repositories docproof read **972 of 3,782**
documentation files, 25.7 per cent. Re-run over the whole tree, langwatch went from 1 broken
to 19 and cherry-studio from 23 to 111. Those runs were not clean; they were narrow.
"""
files = {"README.md": README, "src/thing.py": "x = 1\n"}
for n in range(9):
files[f"elsewhere/note{n}.md"] = "# Note\n"
repo = make_repo(files)
main([str(repo)])
out = capsys.readouterr().out
assert "Nothing contradicted." in out
verdict = out.strip().splitlines()[-1]
assert "9 were never read" in verdict
assert "10% of the documentation" in verdict


def test_a_broken_verdict_carries_the_coverage_too(make_repo: Callable[..., Path], capsys) -> None:
""" "19 broken" over a fifth of the tree is as easy to misread as "nothing contradicted"
over a fifth, so the sentence is attached to both verdicts rather than only the clean one."""
files = {"src/thing.py": "x = 1\n"}
for n in range(4):
files[f"elsewhere/note{n}.md"] = "# Note\n"
# Real drift, not an illustration: the README claims the path in the SAME commit that
# still has it, and a later commit removes it. The fixture's own docstring is emphatic
# that this distinction is the whole rule, and my first attempt at this test ignored it
# and produced a clean run I then asserted was broken.
repo = make_repo(
files,
documented_before={"README.md": "# A project\n\nSee `src/gone.py` for the details.\n"},
deleted={"src/gone.py": "x = 1\n"},
)
main([str(repo)])
out = capsys.readouterr().out
assert " broken, " in out
assert "4 were never read" in out.strip().splitlines()[-1]


def test_full_coverage_adds_no_sentence(make_repo: Callable[..., Path], capsys) -> None:
"""A project whose documentation was entirely read must not gain a line telling it so.
The point is to mark a narrow verdict, and a reassurance printed on every clean run is the
kind of noise that teaches a reader to skip the whole block."""
repo = make_repo({"README.md": README, "src/thing.py": "x = 1\n"})
main([str(repo)])
out = capsys.readouterr().out
assert "were never read" not in out
Loading