Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,69 @@ doesn't need — those show as `n/a` in the matrix and never count as missing.
The `[tool.testmap.statuses]` symbols shown in the Status column default to
`✓` / `✗`; override either independently (the table is optional).

### Generate features from source

By default a feature only exists once a test tags it — a feature with *zero*
tests is invisible. `[[tool.testmap.generate]]` closes that gap: it derives the
expected feature universe straight from your source, so anything with no tests
shows up as an all-missing row instead of silently dropping out.

```toml
[[tool.testmap.generate]]
select = "functions" # "functions" or "classes"
from = "src/**/*.py" # glob (Linux rules, ** allowed), relative to pyproject.toml
where = "public" # "public", "private", or "all" (default)
```

Each generator is a small `select` / `from` / `where` query. `select` picks the
node kind, `from` globs the source files, and `where` filters by visibility
(leading-underscore = private). Only **top-level** definitions count — the
natural notion of a file's public API — and the feature name is the symbol name.
List multiple `[[tool.testmap.generate]]` blocks to combine sets (e.g. public
functions *and* public classes); the results are deduped. Generated features
take the global `required` kinds unless a `[tool.testmap.features]` entry
overrides them.

For example, given `src/mod.py`:

```python
def parse(): ...
def _helper(): ... # private, skipped
class Sender: ...
```

and this config:

```toml
[tool.testmap]
kinds = ["unit", "integration"]
required = ["unit"]

[[tool.testmap.generate]]
select = "functions"
from = "src/**/*.py"
where = "public"

[[tool.testmap.generate]]
select = "classes"
from = "src/**/*.py"
where = "public"
```

`testmap report` seeds the matrix with `parse` and `Sender` even though no test
tags them yet, so they surface as uncovered instead of being invisible:

```
Feature Unit Integration Status
----------------------------------
Sender 0 0 ✗
parse 0 0 ✗

Missing:
• Sender: unit
• parse: unit
```

## Packages

This is a [uv workspace](https://docs.astral.sh/uv/concepts/projects/workspaces/)
Expand Down
39 changes: 39 additions & 0 deletions packages/testmap/src/testmap/discover.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,45 @@ def _records_from_file(path: Path, root: Path) -> list[dict[str, str]]:
return records


def generate_features(generators: Iterable[dict[str, str]], root: Path) -> list[str]:
"""Derive feature names from source by globbing files and reading their AST.

Each generator is `{select, from, where}`: `select` is "functions" or
"classes", `from` is a glob (Linux rules, `**` allowed) resolved against
`root`, and `where` filters by visibility ("public", "private", "all").
Only top-level definitions count — the natural notion of a file's public API.
The feature name is the symbol name; the returned list is sorted and deduped.
"""
node_types = {
"functions": (ast.FunctionDef, ast.AsyncFunctionDef),
"classes": (ast.ClassDef,),
}
features: set[str] = set()
for gen in generators:
select, pattern = gen["select"], gen["from"]
if select not in node_types:
raise ValueError(
f"[tool.testmap.generate] unknown select {select!r} (functions/classes)"
)
where = gen.get("where", "all")
if where not in ("public", "private", "all"):
raise ValueError(
f"[tool.testmap.generate] unknown where {where!r} (public/private/all)"
)
for file in sorted(root.glob(pattern)):
if not file.is_file():
continue
tree = ast.parse(file.read_text(encoding="utf-8"))
for node in tree.body: # top-level only
if not isinstance(node, node_types[select]):
continue
private = node.name.startswith("_")
if (where == "public" and private) or (where == "private" and not private):
continue
features.add(node.name)
return sorted(features)


def discover(paths: Iterable[Path], root: Path | None = None) -> list[dict[str, str]]:
"""Scan `paths` (files or directories) for `@testmap`-tagged tests.

Expand Down
26 changes: 23 additions & 3 deletions packages/testmap/src/testmap/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from dataclasses import dataclass
from pathlib import Path

from testmap.discover import generate_features

# The Status column symbols, used when [tool.testmap.statuses] is absent. At
# least one of each state so the table renders without any config.
DEFAULT_STATUSES = {"complete": "✓", "incomplete": "✗"}
Expand All @@ -22,6 +24,7 @@ class Config:
features: dict[str, list[str]] # per-feature required-kind overrides
excludes: dict[str, list[str]] # per-feature kinds dropped from requirements
statuses: dict[str, str] # Status-column symbol per state (complete/incomplete)
generated: list[str] # feature names derived from source via [tool.testmap.generate]

def excluded_for(self, feature: str) -> list[str]:
return self.excludes.get(feature, [])
Expand All @@ -42,7 +45,8 @@ def load_config(pyproject: Path) -> Config:

A `[tool.testmap.features]` entry is either a list (the required kinds for
that feature) or a table taking `required` and/or `exclude`; excluded kinds
are dropped from that feature's requirements.
are dropped from that feature's requirements. The `__features__` list is
shorthand for features that just take the global `required` kinds.

Invariant: every referenced kind (global, per-feature, excluded) is one of
`kinds`; a stray kind is a config bug, so we raise rather than silently drop it.
Expand All @@ -54,7 +58,9 @@ def load_config(pyproject: Path) -> Config:
kinds: list[str] = data["kinds"]
required: list[str] = data.get("required", kinds)

features: dict[str, list[str]] = {}
# `__features__` is shorthand: each listed feature defaults to the global
# `required` kinds. An explicit [tool.testmap.features] entry below overrides it.
features: dict[str, list[str]] = {name: required for name in data.get("__features__", [])}
excludes: dict[str, list[str]] = {}
for name, entry in data.get("features", {}).items():
if isinstance(entry, dict):
Expand All @@ -79,8 +85,18 @@ def load_config(pyproject: Path) -> Config:
f"[tool.testmap.statuses] unknown states {unknown_states} "
f"(valid: {sorted(DEFAULT_STATUSES)})"
)
# Generators derive the expected feature universe from source, anchored to
# the pyproject's directory, so a feature with no tests at all still surfaces
# (as missing every required kind) instead of being invisible.
generated = generate_features(data.get("generate", []), pyproject.resolve().parent)

return Config(
kinds=kinds, required=required, features=features, excludes=excludes, statuses=statuses
kinds=kinds,
required=required,
features=features,
excludes=excludes,
statuses=statuses,
generated=generated,
)


Expand All @@ -92,6 +108,10 @@ def build_report(tests: list[dict[str, str]], config: Config) -> dict:
its required kinds are missing.
"""
matrix: dict[str, dict[str, int]] = {}
# Seed source-derived features first so those with zero tests still appear
# (as an all-missing row) rather than dropping out of the matrix entirely.
for feature in config.generated:
matrix[feature] = {k: 0 for k in config.kinds}
for test in tests:
feature, kind = test["feature"], test["kind"]
if kind not in config.kinds:
Expand Down
40 changes: 39 additions & 1 deletion packages/testmap/tests/test_discover.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import pytest
from testmap.discover import discover
from testmap.discover import discover, generate_features

SOURCE = """
from pytest_testmap import testmap
Expand Down Expand Up @@ -70,3 +70,41 @@ def test_discover_requires_feature_and_kind(tmp_path) -> None:

with pytest.raises(ValueError, match="requires string feature and kind"):
discover([tmp_path], root=tmp_path)


def test_generate_features_selects_by_kind_and_visibility(tmp_path) -> None:
(tmp_path / "mod.py").write_text(
"def parse(): ...\ndef _helper(): ...\nclass Sender: ...\nclass _Private: ...\n"
)

funcs = generate_features(
[{"select": "functions", "from": "*.py", "where": "public"}], tmp_path
)
classes = generate_features(
[{"select": "classes", "from": "*.py", "where": "public"}], tmp_path
)
private = generate_features(
[{"select": "functions", "from": "*.py", "where": "private"}], tmp_path
)

assert funcs == ["parse"]
assert classes == ["Sender"]
assert private == ["_helper"]


def test_generate_features_globs_recursively_and_dedupes(tmp_path) -> None:
(tmp_path / "pkg").mkdir()
(tmp_path / "pkg" / "a.py").write_text("def parse(): ...\n")
(tmp_path / "pkg" / "b.py").write_text(
"def parse(): ...\nclass Nested:\n def inner(self): ...\n"
)

# `**` recurses; top-level only (Nested.inner is skipped); duplicate `parse` deduped.
features = generate_features([{"select": "functions", "from": "**/*.py"}], tmp_path)

assert features == ["parse"]


def test_generate_features_rejects_unknown_select(tmp_path) -> None:
with pytest.raises(ValueError, match="unknown select"):
generate_features([{"select": "modules", "from": "*.py"}], tmp_path)
34 changes: 34 additions & 0 deletions packages/testmap/tests/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
features={"processor": ["unit", "integration", "property"]},
excludes={},
statuses={"complete": "✓", "incomplete": "✗"},
generated=[],
)

TESTS = [
Expand Down Expand Up @@ -113,6 +114,7 @@ def test_exclude_makes_feature_complete_and_renders_na() -> None:
features={},
excludes={"auth": ["perf"]},
statuses={"complete": "✓", "incomplete": "✗"},
generated=[],
)
tests = [{"feature": "auth", "kind": "unit"}, {"feature": "auth", "kind": "integration"}]
report = build_report(tests, config)
Expand Down Expand Up @@ -146,10 +148,42 @@ def test_load_config_feature_table_with_required(tmp_path) -> None:
assert config.required_for("auth") == ["unit"]


def test_load_config_shorthand_features(tmp_path) -> None:
pyproject = tmp_path / "pyproject.toml"
pyproject.write_text(
'[tool.testmap]\nkinds = ["unit", "integration"]\n'
'required = ["unit", "integration"]\n'
'__features__ = ["upcase", "lowcase"]\n'
# An explicit table entry still wins over the shorthand default.
'[tool.testmap.features]\nlowcase = ["unit"]\n'
)
config = load_config(pyproject)
assert config.required_for("upcase") == ["unit", "integration"]
assert config.features["upcase"] == ["unit", "integration"]
assert config.required_for("lowcase") == ["unit"]


def test_load_config_rejects_unknown_excluded_kind(tmp_path) -> None:
pyproject = tmp_path / "pyproject.toml"
pyproject.write_text(
'[tool.testmap]\nkinds = ["unit"]\n[tool.testmap.features]\nauth = { exclude = ["perf"] }\n'
)
with pytest.raises(ValueError, match="unknown kinds"):
load_config(pyproject)


def test_load_config_generates_features_from_source(tmp_path) -> None:
(tmp_path / "src").mkdir()
(tmp_path / "src" / "mod.py").write_text("def parse(): ...\ndef _hidden(): ...\n")
pyproject = tmp_path / "pyproject.toml"
pyproject.write_text(
'[tool.testmap]\nkinds = ["unit"]\nrequired = ["unit"]\n'
"[[tool.testmap.generate]]\n"
'select = "functions"\nfrom = "src/**/*.py"\nwhere = "public"\n'
)
config = load_config(pyproject)
assert config.generated == ["parse"]
# A generated feature with no tests surfaces as an all-missing row.
report = build_report([], config)["features"]
assert report["parse"]["missing"] == ["unit"]
assert report["parse"]["complete"] is False
Loading