Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/docproof/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,14 @@ def survive_a_narrow_console() -> None:
SET_ASIDE_NAMES = 10


def report_coverage(project: Project, unread: list[Path]) -> None:
def report_coverage(project: Project, unread: list[Path]) -> str | None:
"""Say how much of the tree was in scope at all, whether or not any of it was missed.

Returns the directory a reader should widen to, or None when nothing unread is named
like documentation, so the VERDICT at the bottom can carry the same judgement this
header already makes. It used to be computed here and thrown away, and the two ends of
one report then said different things about the same number - see `Report.render`.

**This prints on every run, including the clean one, and that is the whole point.** The
other skip reports in this file stay quiet when they have nothing to say, which is right
for them: the count of documents they set aside is visible in the header line beside the
Expand All @@ -142,7 +147,7 @@ def report_coverage(project: Project, unread: list[Path]) -> None:
"""
if not unread:
print(" every documentation file in the tree was in scope")
return
return None
groups = by_directory(project.root, unread)
shown = groups[:TOP_DIRECTORIES]
rest = groups[TOP_DIRECTORIES:]
Expand Down Expand Up @@ -170,6 +175,7 @@ def report_coverage(project: Project, unread: list[Path]) -> None:
" none of them is named like a documentation tree; if one is, widen with "
"--docs 'DIR/**/*.md' or [tool.docproof] docs = [\"DIR/**/*.md\"]"
)
return widen


def report_set_aside(historical: list[str], disclaimed: dict[tuple[str, str], list[str]]) -> None:
Expand Down Expand Up @@ -310,7 +316,7 @@ def main(argv: Sequence[str] | None = None) -> int:
report = Report(project=project, outcomes=outcomes)
print(f"docproof {__version__} — {project.root.name}, {len(documents)} document(s)")
unread = unread_documents(project.root, in_scope, tracked)
report_coverage(project, unread)
docs_directory = report_coverage(project, unread)
report_set_aside(historical, disclaimed)
# Same principle, one level down, and it applies harder: nobody asked for this rule.
# A `Before:` label is the tool deciding by itself that a block is not a claim, so the
Expand All @@ -324,7 +330,14 @@ def main(argv: Sequence[str] | None = None) -> int:
if superseded:
print(f" labelled superseded by the prose above, not judged: {', '.join(sorted(superseded))}")
print()
print(report.render(show_skips=args.show_skips, read=len(documents), unread=len(unread)))
print(
report.render(
show_skips=args.show_skips,
read=len(documents),
unread=len(unread),
docs_directory=docs_directory,
)
)
if args.exit_zero and not report.stopped_checking:
return 0
return report.exit_code
Expand Down
52 changes: 46 additions & 6 deletions src/docproof/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,14 @@ def _silence_line(self, outcome: Outcome) -> str:
marker = WARN if verdict.alarming or verdict.kind is Silence.UNKNOWN else DASH
return f"{marker} {outcome.verifier}: found nothing to check — {verdict.detail}."

def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) -> str:
def render(
self,
*,
show_skips: bool = False,
read: int = 0,
unread: int = 0,
docs_directory: str | None = None,
) -> str:
"""`read` and `unread` put the document-level coverage INTO the verdict.

**Measured defect, 2026-08-19.** The coverage note prints in the header and the verdict
Expand All @@ -93,6 +100,31 @@ def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) ->
3,782** documentation files, 25.7%. Re-run over the whole tree, langwatch went from 1
broken to 19 and cherry-studio from 23 to 111. Those runs were not clean, they were
narrow, and only the header said so.

**`docs_directory` was added because the two ends of one report disagreed.** The
header already asks `likeliest_docs_directory` whether anything unread is NAMED like
documentation, and prints either a widen suggestion or *"none of them is named like a
documentation tree"*. This line then called every one of those files
"documentation file(s)" and divided by them. On `zhukunpenglinyutong/desktop-cc-gui`
the same report said, forty lines apart, that none of the 5,812 unread files looks
like documentation and that the verdict *"covers 1% of the documentation in this
project"*. 5,271 of them are an OpenSpec change-proposal tree.

Measured over 39 sweep captures, 24 of which print a coverage line: coverage runs
**1% to 99%, median 27.5%**, and seven repositories are under 10%. The number that
matters is the other one - **a directory named like documentation was found unread in
4 of those 24.** So in twenty cases the tool divided by a denominator it had itself
just judged to hold no documentation tree. One of the fourteen commonest
biggest-unread directories is documentation: the rest are `skills/`, `src/`,
`.agents/`, `crates/`, `libs/`, `ts/`, `tools/`, `mobile/`, `examples/`,
`benchmarks/`, `scripts/`, per-package READMEs and agent instruction files, which
`find_docs` excludes ON PURPOSE and whose docstring argues why.

So the percentage stays - a narrow pass must never read as a clean one, which is the
whole point of this paragraph existing - and the false assertion goes. The denominator
is markdown in the tree, not "the documentation in this project", and when something
unread IS named like documentation the line says so instead of leaving the reader to
scroll up for it.
"""
lines: list[str] = []
root = self.project.root
Expand Down Expand Up @@ -165,9 +197,17 @@ def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) ->
# fifth of the tree is as easy to misread as "nothing contradicted" over a fifth.
if unread:
total = read + unread
lines.append(
f"This judged {read} of {total} documentation file(s). "
f"{unread} were never read, so this verdict covers "
f"{100 * read // total}% of the documentation in this project."
)
said = f"This judged {read} of {total} documentation file(s) in the tree, {100 * read // total}%."
if docs_directory:
said += (
f" The {unread} it did not read include `{docs_directory}/`, which is "
f"named like a documentation tree, so this verdict is narrower than it "
f"looks."
)
else:
said += (
f" The {unread} it did not read are outside the default scope and listed "
f"above; none of them is in a directory named like documentation."
)
lines.append(said)
return "\n".join(lines)
44 changes: 41 additions & 3 deletions tests/test_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,13 @@ def test_the_verdict_itself_says_how_much_it_covered(make_repo: Callable[..., Pa
out = capsys.readouterr().out
assert "Nothing contradicted." in out
verdict = out.strip().splitlines()[-1]
assert "9 were never read" in verdict
assert "10% of the documentation" in verdict
assert "1 of 10 documentation file(s) in the tree, 10%" in verdict
# **The denominator is markdown in the tree, NOT "the documentation in this project".**
# `elsewhere/` is not named like documentation and the header two lines up already says
# so, in as many words. Asserting the old phrasing back would restore a report whose two
# ends disagreed about the same nine files.
assert "of the documentation in this project" not in verdict
assert "none of them is in a directory named like documentation" in verdict


def test_a_broken_verdict_carries_the_coverage_too(make_repo: Callable[..., Path], capsys) -> None:
Expand All @@ -194,7 +199,7 @@ def test_a_broken_verdict_carries_the_coverage_too(make_repo: Callable[..., Path
main([str(repo)])
out = capsys.readouterr().out
assert " broken, " in out
assert "4 were never read" in out.strip().splitlines()[-1]
assert "1 of 5 documentation file(s) in the tree, 20%" in out.strip().splitlines()[-1]


def test_full_coverage_adds_no_sentence(make_repo: Callable[..., Path], capsys) -> None:
Expand Down Expand Up @@ -246,3 +251,36 @@ def test_no_documentation_tree_means_no_confident_suggestion(make_repo: Callable
assert "none of them is named like a documentation tree" in out
assert "--docs 'apps/**/*.md'" not in out
assert "--docs" in out and "[tool.docproof] docs" in out


def test_the_verdict_names_an_unread_documentation_tree(make_repo: Callable[..., Path], capsys) -> None:
"""The two ends of one report used to disagree about the same files.

The header asks `likeliest_docs_directory` whether anything unread is NAMED like
documentation and prints either a widen suggestion or "none of them is named like a
documentation tree". The verdict then called all of them "the documentation in this
project" and divided by them. On `zhukunpenglinyutong/desktop-cc-gui` the same run said,
forty lines apart, that none of the 5,812 unread files looks like documentation and that
it covered "1% of the documentation in this project" - and 5,271 of them are an OpenSpec
change-proposal tree.

Measured over the sweep captures: coverage runs 1% to 99%, median near 27%, and the
biggest unread directory is `libs/`, `crates/`, `apps/`, `src/`, `packages/`, `mobile/`,
`examples/`, `benchmarks/`, `tools/`, `.agents/` or `.claude/` far more often than it is
documentation. So the percentage stays and the assertion about what those files ARE goes.

This is the other branch: when something unread really is a documentation tree, the
verdict has to say so, because the reader who scrolls to the last line is the one this
whole paragraph exists for.
"""
files = {"README.md": README, "src/thing.py": "x = 1\n"}
for n in range(7):
files[f"handbook/page{n}.md"] = "# Page\n"
repo = make_repo(files)
main([str(repo)])
out = capsys.readouterr().out
verdict = out.strip().splitlines()[-1]
assert "`handbook/`" in verdict
assert "narrower than it looks" in verdict
# And the header still gives the actionable form, so the two ends now agree.
assert "--docs 'handbook/**/*.md'" in out
Loading