From 7f4124e9951051a66acb1a6184e4ddfed805eac3 Mon Sep 17 00:00:00 2001 From: melbinjp Date: Wed, 19 Aug 2026 20:35:58 +0530 Subject: [PATCH] The two ends of one report disagreed about the same files The header asks `likeliest_docs_directory` whether anything unread is NAMED like documentation, and prints either a widen suggestion or "none of them is named like a documentation tree". The verdict then called every one of those files "documentation file(s)" and divided by them. On `zhukunpenglinyutong/desktop-cc-gui` the same run says, forty lines apart, that none of the 5,812 unread files looks like documentation and that the verdict "covers 1% of the documentation in this project". 5,271 of them are an OpenSpec change-proposal tree, which nobody would want checked against HEAD. Measured over 39 sweep captures, 24 with a coverage line: 1% to 99%, median 27.5%, seven under 10%. A directory named like documentation was found unread in 4 of the 24 - so in twenty cases the tool divided by a denominator it had itself just judged to hold no documentation. One of the fourteen commonest biggest-unread directories is documentation; the rest are skills/, src/, .agents/, crates/, libs/, ts/, tools/, mobile/, examples/, benchmarks/. The percentage STAYS. A narrow pass must never read as a clean one and that is the entire reason this line exists. What goes is the assertion about what those files are. And when something unread IS a documentation tree the verdict now names it, instead of leaving the reader who only reads the last line to scroll up for it. 209 tests. Two rewritten to the new wording and one new, covering the branch the change exists for. --- src/docproof/cli.py | 21 +++++++++++++---- src/docproof/report.py | 52 +++++++++++++++++++++++++++++++++++++----- tests/test_coverage.py | 44 ++++++++++++++++++++++++++++++++--- 3 files changed, 104 insertions(+), 13 deletions(-) diff --git a/src/docproof/cli.py b/src/docproof/cli.py index a1c6c53..40ba09e 100644 --- a/src/docproof/cli.py +++ b/src/docproof/cli.py @@ -126,9 +126,14 @@ def survive_a_narrow_console() -> None: SET_ASIDE_NAMES = 10 -def report_coverage(project: Project, unread: list[Path]) -> None: +def report_coverage(project: Project, unread: list[Path]) -> str | None: """Say how much of the tree was in scope at all, whether or not any of it was missed. + Returns the directory a reader should widen to, or None when nothing unread is named + like documentation, so the VERDICT at the bottom can carry the same judgement this + header already makes. It used to be computed here and thrown away, and the two ends of + one report then said different things about the same number - see `Report.render`. + **This prints on every run, including the clean one, and that is the whole point.** The other skip reports in this file stay quiet when they have nothing to say, which is right for them: the count of documents they set aside is visible in the header line beside the @@ -142,7 +147,7 @@ def report_coverage(project: Project, unread: list[Path]) -> None: """ if not unread: print(" every documentation file in the tree was in scope") - return + return None groups = by_directory(project.root, unread) shown = groups[:TOP_DIRECTORIES] rest = groups[TOP_DIRECTORIES:] @@ -170,6 +175,7 @@ def report_coverage(project: Project, unread: list[Path]) -> None: " none of them is named like a documentation tree; if one is, widen with " "--docs 'DIR/**/*.md' or [tool.docproof] docs = [\"DIR/**/*.md\"]" ) + return widen def report_set_aside(historical: list[str], disclaimed: dict[tuple[str, str], list[str]]) -> None: @@ -310,7 +316,7 @@ def main(argv: Sequence[str] | None = None) -> int: report = Report(project=project, outcomes=outcomes) print(f"docproof {__version__} — {project.root.name}, {len(documents)} document(s)") unread = unread_documents(project.root, in_scope, tracked) - report_coverage(project, unread) + docs_directory = report_coverage(project, unread) report_set_aside(historical, disclaimed) # Same principle, one level down, and it applies harder: nobody asked for this rule. # A `Before:` label is the tool deciding by itself that a block is not a claim, so the @@ -324,7 +330,14 @@ def main(argv: Sequence[str] | None = None) -> int: if superseded: print(f" labelled superseded by the prose above, not judged: {', '.join(sorted(superseded))}") print() - print(report.render(show_skips=args.show_skips, read=len(documents), unread=len(unread))) + print( + report.render( + show_skips=args.show_skips, + read=len(documents), + unread=len(unread), + docs_directory=docs_directory, + ) + ) if args.exit_zero and not report.stopped_checking: return 0 return report.exit_code diff --git a/src/docproof/report.py b/src/docproof/report.py index 619fa19..2ba312b 100644 --- a/src/docproof/report.py +++ b/src/docproof/report.py @@ -79,7 +79,14 @@ def _silence_line(self, outcome: Outcome) -> str: marker = WARN if verdict.alarming or verdict.kind is Silence.UNKNOWN else DASH return f"{marker} {outcome.verifier}: found nothing to check — {verdict.detail}." - def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) -> str: + def render( + self, + *, + show_skips: bool = False, + read: int = 0, + unread: int = 0, + docs_directory: str | None = None, + ) -> str: """`read` and `unread` put the document-level coverage INTO the verdict. **Measured defect, 2026-08-19.** The coverage note prints in the header and the verdict @@ -93,6 +100,31 @@ def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) -> 3,782** documentation files, 25.7%. Re-run over the whole tree, langwatch went from 1 broken to 19 and cherry-studio from 23 to 111. Those runs were not clean, they were narrow, and only the header said so. + + **`docs_directory` was added because the two ends of one report disagreed.** The + header already asks `likeliest_docs_directory` whether anything unread is NAMED like + documentation, and prints either a widen suggestion or *"none of them is named like a + documentation tree"*. This line then called every one of those files + "documentation file(s)" and divided by them. On `zhukunpenglinyutong/desktop-cc-gui` + the same report said, forty lines apart, that none of the 5,812 unread files looks + like documentation and that the verdict *"covers 1% of the documentation in this + project"*. 5,271 of them are an OpenSpec change-proposal tree. + + Measured over 39 sweep captures, 24 of which print a coverage line: coverage runs + **1% to 99%, median 27.5%**, and seven repositories are under 10%. The number that + matters is the other one - **a directory named like documentation was found unread in + 4 of those 24.** So in twenty cases the tool divided by a denominator it had itself + just judged to hold no documentation tree. One of the fourteen commonest + biggest-unread directories is documentation: the rest are `skills/`, `src/`, + `.agents/`, `crates/`, `libs/`, `ts/`, `tools/`, `mobile/`, `examples/`, + `benchmarks/`, `scripts/`, per-package READMEs and agent instruction files, which + `find_docs` excludes ON PURPOSE and whose docstring argues why. + + So the percentage stays - a narrow pass must never read as a clean one, which is the + whole point of this paragraph existing - and the false assertion goes. The denominator + is markdown in the tree, not "the documentation in this project", and when something + unread IS named like documentation the line says so instead of leaving the reader to + scroll up for it. """ lines: list[str] = [] root = self.project.root @@ -165,9 +197,17 @@ def render(self, *, show_skips: bool = False, read: int = 0, unread: int = 0) -> # fifth of the tree is as easy to misread as "nothing contradicted" over a fifth. if unread: total = read + unread - lines.append( - f"This judged {read} of {total} documentation file(s). " - f"{unread} were never read, so this verdict covers " - f"{100 * read // total}% of the documentation in this project." - ) + said = f"This judged {read} of {total} documentation file(s) in the tree, {100 * read // total}%." + if docs_directory: + said += ( + f" The {unread} it did not read include `{docs_directory}/`, which is " + f"named like a documentation tree, so this verdict is narrower than it " + f"looks." + ) + else: + said += ( + f" The {unread} it did not read are outside the default scope and listed " + f"above; none of them is in a directory named like documentation." + ) + lines.append(said) return "\n".join(lines) diff --git a/tests/test_coverage.py b/tests/test_coverage.py index a8b867f..d0390ca 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -172,8 +172,13 @@ def test_the_verdict_itself_says_how_much_it_covered(make_repo: Callable[..., Pa out = capsys.readouterr().out assert "Nothing contradicted." in out verdict = out.strip().splitlines()[-1] - assert "9 were never read" in verdict - assert "10% of the documentation" in verdict + assert "1 of 10 documentation file(s) in the tree, 10%" in verdict + # **The denominator is markdown in the tree, NOT "the documentation in this project".** + # `elsewhere/` is not named like documentation and the header two lines up already says + # so, in as many words. Asserting the old phrasing back would restore a report whose two + # ends disagreed about the same nine files. + assert "of the documentation in this project" not in verdict + assert "none of them is in a directory named like documentation" in verdict def test_a_broken_verdict_carries_the_coverage_too(make_repo: Callable[..., Path], capsys) -> None: @@ -194,7 +199,7 @@ def test_a_broken_verdict_carries_the_coverage_too(make_repo: Callable[..., Path main([str(repo)]) out = capsys.readouterr().out assert " broken, " in out - assert "4 were never read" in out.strip().splitlines()[-1] + assert "1 of 5 documentation file(s) in the tree, 20%" in out.strip().splitlines()[-1] def test_full_coverage_adds_no_sentence(make_repo: Callable[..., Path], capsys) -> None: @@ -246,3 +251,36 @@ def test_no_documentation_tree_means_no_confident_suggestion(make_repo: Callable assert "none of them is named like a documentation tree" in out assert "--docs 'apps/**/*.md'" not in out assert "--docs" in out and "[tool.docproof] docs" in out + + +def test_the_verdict_names_an_unread_documentation_tree(make_repo: Callable[..., Path], capsys) -> None: + """The two ends of one report used to disagree about the same files. + + The header asks `likeliest_docs_directory` whether anything unread is NAMED like + documentation and prints either a widen suggestion or "none of them is named like a + documentation tree". The verdict then called all of them "the documentation in this + project" and divided by them. On `zhukunpenglinyutong/desktop-cc-gui` the same run said, + forty lines apart, that none of the 5,812 unread files looks like documentation and that + it covered "1% of the documentation in this project" - and 5,271 of them are an OpenSpec + change-proposal tree. + + Measured over the sweep captures: coverage runs 1% to 99%, median near 27%, and the + biggest unread directory is `libs/`, `crates/`, `apps/`, `src/`, `packages/`, `mobile/`, + `examples/`, `benchmarks/`, `tools/`, `.agents/` or `.claude/` far more often than it is + documentation. So the percentage stays and the assertion about what those files ARE goes. + + This is the other branch: when something unread really is a documentation tree, the + verdict has to say so, because the reader who scrolls to the last line is the one this + whole paragraph exists for. + """ + files = {"README.md": README, "src/thing.py": "x = 1\n"} + for n in range(7): + files[f"handbook/page{n}.md"] = "# Page\n" + repo = make_repo(files) + main([str(repo)]) + out = capsys.readouterr().out + verdict = out.strip().splitlines()[-1] + assert "`handbook/`" in verdict + assert "narrower than it looks" in verdict + # And the header still gives the actionable form, so the two ends now agree. + assert "--docs 'handbook/**/*.md'" in out