diff --git a/packages/testmap/src/testmap/cli.py b/packages/testmap/src/testmap/cli.py index 4e9e3aa..bb04369 100644 --- a/packages/testmap/src/testmap/cli.py +++ b/packages/testmap/src/testmap/cli.py @@ -8,6 +8,7 @@ from pathlib import Path from testmap.discover import discover +from testmap.gitdiff import changed_files from testmap.report import build_report, load_config, render # The skill directory name; also the slash-command name (`/testmap-do`). @@ -80,6 +81,15 @@ def main() -> None: default=Path("pyproject.toml"), help="pyproject.toml holding [tool.testmap] (default: ./pyproject.toml)", ) + report.add_argument( + "--diff", + nargs="?", + const="HEAD", + default=None, + metavar="REF", + help="only gate generated features whose source changed since REF " + "(default: HEAD; requires the config directory to be a git repo)", + ) args = parser.parse_args() if args.command == "install-skill": @@ -90,10 +100,13 @@ def main() -> None: config = load_config(args.config) tests = _load_records(args.path, args.config) - result = build_report(tests, config) + changed = changed_files(args.diff, args.config.resolve().parent) if args.diff else None + result = build_report(tests, config, changed) print(json.dumps(result, indent=2) if args.json else render(result, config)) - # Exit non-zero when any feature is missing a required kind, so `testmap + # Exit non-zero when a gated feature is missing a required kind, so `testmap # report` doubles as a coverage gate (CI, pre-commit) with no extra flag. - if any(not data["complete"] for data in result["features"].values()): + # Without --diff every feature is gated; with it, only features whose + # generated source changed count, so unrelated legacy gaps don't block a PR. + if any(data["gated"] and not data["complete"] for data in result["features"].values()): raise SystemExit(1) diff --git a/packages/testmap/src/testmap/discover.py b/packages/testmap/src/testmap/discover.py index 83f13d0..2e35fde 100644 --- a/packages/testmap/src/testmap/discover.py +++ b/packages/testmap/src/testmap/discover.py @@ -58,20 +58,22 @@ def _records_from_file(path: Path, root: Path) -> list[dict[str, str]]: return records -def generate_features(generators: Iterable[dict[str, str]], root: Path) -> list[str]: - """Derive feature names from source by globbing files and reading their AST. +def generate_feature_sources(generators: Iterable[dict[str, str]], root: Path) -> dict[str, str]: + """Derive features from source by globbing files and reading their AST. Each generator is `{select, from, where}`: `select` is "functions" or "classes", `from` is a glob (Linux rules, `**` allowed) resolved against `root`, and `where` filters by visibility ("public", "private", "all"). Only top-level definitions count — the natural notion of a file's public API. - The feature name is the symbol name; the returned list is sorted and deduped. + Returns `{feature_name: source_file}`, the file given as a `root`-relative + posix path so it lines up with `git diff --name-only` output for + diff-scoped gating (the first file a duplicated name is found in wins). """ node_types = { "functions": (ast.FunctionDef, ast.AsyncFunctionDef), "classes": (ast.ClassDef,), } - features: set[str] = set() + sources: dict[str, str] = {} for gen in generators: select, pattern = gen["select"], gen["from"] if select not in node_types: @@ -93,8 +95,13 @@ def generate_features(generators: Iterable[dict[str, str]], root: Path) -> list[ private = node.name.startswith("_") if (where == "public" and private) or (where == "private" and not private): continue - features.add(node.name) - return sorted(features) + sources.setdefault(node.name, file.relative_to(root).as_posix()) + return sources + + +def generate_features(generators: Iterable[dict[str, str]], root: Path) -> list[str]: + """Feature names derived from source (see `generate_feature_sources`).""" + return sorted(generate_feature_sources(generators, root)) def discover(paths: Iterable[Path], root: Path | None = None) -> list[dict[str, str]]: diff --git a/packages/testmap/src/testmap/gitdiff.py b/packages/testmap/src/testmap/gitdiff.py new file mode 100644 index 0000000..d5d24a6 --- /dev/null +++ b/packages/testmap/src/testmap/gitdiff.py @@ -0,0 +1,22 @@ +"""Changed-file discovery for `testmap report --diff`.""" + +import subprocess +from pathlib import Path + + +def changed_files(against: str, root: Path) -> set[str]: + """Paths (root-relative, posix) changed since `against`, tracked + untracked. + + Untracked files count as "changed" too, since a brand-new feature's source + has nothing to diff against yet but should still be in scope. + """ + + def git(*args: str) -> list[str]: + out = subprocess.run( + ["git", *args], cwd=root, capture_output=True, text=True, check=True + ).stdout + return [line for line in out.splitlines() if line] + + tracked = git("diff", "--name-only", against) + untracked = git("ls-files", "--others", "--exclude-standard") + return {*tracked, *untracked} diff --git a/packages/testmap/src/testmap/report.py b/packages/testmap/src/testmap/report.py index c5adbeb..acbe49e 100644 --- a/packages/testmap/src/testmap/report.py +++ b/packages/testmap/src/testmap/report.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path -from testmap.discover import generate_features +from testmap.discover import generate_feature_sources # The Status column symbols, used when [tool.testmap.statuses] is absent. At # least one of each state so the table renders without any config. @@ -25,6 +25,7 @@ class Config: excludes: dict[str, list[str]] # per-feature kinds dropped from requirements statuses: dict[str, str] # Status-column symbol per state (complete/incomplete) generated: list[str] # feature names derived from source via [tool.testmap.generate] + generated_sources: dict[str, str] # generated feature -> its source file (for --diff) def excluded_for(self, feature: str) -> list[str]: return self.excludes.get(feature, []) @@ -39,6 +40,15 @@ def required_for(self, feature: str) -> list[str]: def status_symbol(self, complete: bool) -> str: return self.statuses["complete" if complete else "incomplete"] + def in_diff_scope(self, feature: str, changed: set[str]) -> bool: + """Whether `feature`'s source changed, for `--diff` gating. + + A feature with no known source (not from [tool.testmap.generate]) can't + be scoped, so it stays in scope rather than silently skipping its gate. + """ + source = self.generated_sources.get(feature) + return source is None or source in changed + def load_config(pyproject: Path) -> Config: """Load `[tool.testmap]` from a pyproject.toml. @@ -88,7 +98,9 @@ def load_config(pyproject: Path) -> Config: # Generators derive the expected feature universe from source, anchored to # the pyproject's directory, so a feature with no tests at all still surfaces # (as missing every required kind) instead of being invisible. - generated = generate_features(data.get("generate", []), pyproject.resolve().parent) + generated_sources = generate_feature_sources( + data.get("generate", []), pyproject.resolve().parent + ) return Config( kinds=kinds, @@ -96,16 +108,23 @@ def load_config(pyproject: Path) -> Config: features=features, excludes=excludes, statuses=statuses, - generated=generated, + generated=sorted(generated_sources), + generated_sources=generated_sources, ) -def build_report(tests: list[dict[str, str]], config: Config) -> dict: +def build_report( + tests: list[dict[str, str]], config: Config, changed: set[str] | None = None +) -> dict: """Aggregate `{feature, kind}` records into the feature x kind matrix. Raises on a kind not declared in `config.kinds` (no silent fallback). Only features with at least one test appear; a feature is complete when none of its required kinds are missing. + + `changed` is the `--diff` file set: when given, a feature is "gated" (counts + toward the exit-code gate) only if its source changed, so unrelated legacy + gaps don't block a PR that never touched them. """ matrix: dict[str, dict[str, int]] = {} # Seed source-derived features first so those with zero tests still appear @@ -128,6 +147,7 @@ def build_report(tests: list[dict[str, str]], config: Config) -> dict: "missing": missing, "excluded": excluded, "complete": not missing, + "gated": changed is None or config.in_diff_scope(feature, changed), } return {"features": features} @@ -155,6 +175,7 @@ def fmt(cells: list[str]) -> str: missing = [ f" • {feature}: {kind}" for feature, data in report["features"].items() + if data["gated"] for kind in data["missing"] ] if missing: diff --git a/packages/testmap/tests/test_cli.py b/packages/testmap/tests/test_cli.py index 9dce977..755ac34 100644 --- a/packages/testmap/tests/test_cli.py +++ b/packages/testmap/tests/test_cli.py @@ -1,5 +1,6 @@ import json import runpy +import subprocess from pathlib import Path import pytest @@ -73,6 +74,73 @@ def test_default_paths_missing_pyproject(tmp_path) -> None: assert _default_paths(tmp_path / "nope.toml") == [Path(".")] +DIFF_PYPROJECT = ( + "[tool.pytest.ini_options]\n" + 'testpaths = ["tests"]\n' + "[tool.testmap]\n" + 'kinds = ["unit", "integration"]\n' + 'required = ["unit", "integration"]\n' + "[[tool.testmap.generate]]\n" + 'select = "functions"\nfrom = "src/**/*.py"\nwhere = "public"\n' +) + +DIFF_SUITE = ( + "from pytest_testmap import testmap\n\n" + '@testmap(feature="parse", kind="unit")\n' + "def test_a(): ...\n\n" + '@testmap(feature="parse", kind="integration")\n' + "def test_b(): ...\n" +) + + +@pytest.fixture +def diff_project(tmp_path, monkeypatch): + # `render` is left with zero tests, so it's incomplete from the first commit + # onward — a "legacy gap" --diff should ignore once its source is untouched. + (tmp_path / "pyproject.toml").write_text(DIFF_PYPROJECT) + (tmp_path / "src").mkdir() + (tmp_path / "src" / "mod.py").write_text("def parse(): ...\ndef render(): ...\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_suite.py").write_text(DIFF_SUITE) + + def git(*args: str) -> None: + subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True) + + git("init") + git("config", "user.email", "test@example.com") + git("config", "user.name", "Test") + git("add", "-A") + git("commit", "-m", "initial") + + monkeypatch.chdir(tmp_path) + return tmp_path + + +def test_report_diff_ignores_unchanged_incomplete_feature(diff_project, monkeypatch) -> None: + # render is incomplete but untouched since HEAD -> --diff lets the gate pass. + monkeypatch.setattr("sys.argv", ["testmap", "report", "--diff"]) + main() # no SystemExit -> exit code 0 + + +def test_report_without_diff_still_gates_everything(diff_project, monkeypatch) -> None: + monkeypatch.setattr("sys.argv", ["testmap", "report"]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 1 + + +def test_report_diff_gates_changed_source(diff_project, monkeypatch) -> None: + # Touching mod.py brings render back into scope for --diff. + (diff_project / "src" / "mod.py").write_text( + "def parse(): ...\ndef render(): ...\ndef validate(): ...\n" + ) + monkeypatch.setattr("sys.argv", ["testmap", "report", "--diff"]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 1 + + def test_report_ingests_json_file(project, capsys, monkeypatch) -> None: records = {"tests": [{"feature": "parser", "kind": "integration"}]} (project / "in.json").write_text(json.dumps(records)) diff --git a/packages/testmap/tests/test_discover.py b/packages/testmap/tests/test_discover.py index 5b33557..c72f532 100644 --- a/packages/testmap/tests/test_discover.py +++ b/packages/testmap/tests/test_discover.py @@ -1,5 +1,5 @@ import pytest -from testmap.discover import discover, generate_features +from testmap.discover import discover, generate_feature_sources, generate_features SOURCE = """ from pytest_testmap import testmap @@ -108,3 +108,12 @@ def test_generate_features_globs_recursively_and_dedupes(tmp_path) -> None: def test_generate_features_rejects_unknown_select(tmp_path) -> None: with pytest.raises(ValueError, match="unknown select"): generate_features([{"select": "modules", "from": "*.py"}], tmp_path) + + +def test_generate_feature_sources_maps_feature_to_file(tmp_path) -> None: + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "mod.py").write_text("def parse(): ...\n") + + sources = generate_feature_sources([{"select": "functions", "from": "**/*.py"}], tmp_path) + + assert sources == {"parse": "pkg/mod.py"} diff --git a/packages/testmap/tests/test_report.py b/packages/testmap/tests/test_report.py index e0bbfba..291d3a4 100644 --- a/packages/testmap/tests/test_report.py +++ b/packages/testmap/tests/test_report.py @@ -8,6 +8,7 @@ excludes={}, statuses={"complete": "✓", "incomplete": "✗"}, generated=[], + generated_sources={}, ) TESTS = [ @@ -115,6 +116,7 @@ def test_exclude_makes_feature_complete_and_renders_na() -> None: excludes={"auth": ["perf"]}, statuses={"complete": "✓", "incomplete": "✗"}, generated=[], + generated_sources={}, ) tests = [{"feature": "auth", "kind": "unit"}, {"feature": "auth", "kind": "integration"}] report = build_report(tests, config) @@ -183,7 +185,43 @@ def test_load_config_generates_features_from_source(tmp_path) -> None: ) config = load_config(pyproject) assert config.generated == ["parse"] + assert config.generated_sources == {"parse": "src/mod.py"} # A generated feature with no tests surfaces as an all-missing row. report = build_report([], config)["features"] assert report["parse"]["missing"] == ["unit"] assert report["parse"]["complete"] is False + + +def test_in_diff_scope_gates_only_changed_generated_features() -> None: + config = Config( + kinds=["unit"], + required=["unit"], + features={}, + excludes={}, + statuses={"complete": "✓", "incomplete": "✗"}, + generated=["parse"], + generated_sources={"parse": "src/mod.py"}, + ) + assert config.in_diff_scope("parse", {"src/mod.py"}) is True + assert config.in_diff_scope("parse", {"src/other.py"}) is False + # A feature with no known source (not from [tool.testmap.generate]) can't + # be scoped, so it always stays in scope. + assert config.in_diff_scope("manual", set()) is True + + +def test_build_report_marks_ungated_features_when_diff_scoped() -> None: + config = Config( + kinds=["unit"], + required=["unit"], + features={}, + excludes={}, + statuses={"complete": "✓", "incomplete": "✗"}, + generated=["parse", "render"], + generated_sources={"parse": "src/a.py", "render": "src/b.py"}, + ) + report = build_report([], config, changed={"src/a.py"})["features"] + assert report["parse"]["gated"] is True + assert report["render"]["gated"] is False + # render is still shown as missing in the table, just excluded from the + # Missing section / exit-code gate since its source wasn't touched. + assert "render: unit" not in render({"features": report}, config)