diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..7df1d35 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,56 @@ +name: pages + +# The catalog page is built from the commit it is published from, never from a +# checked-in copy. There is no way for the page to fall behind main, and no +# generated HTML in the repository to review. + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +# One deployment at a time, and never cancel one half-done: an interrupted +# deploy would leave the published catalog in an unknown state. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + # configure-pages turns Pages on for this repository the first time it + # runs, so publishing needs no manual repository-settings change. + pages: write + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + python-version: "3.12" + - run: uv sync --frozen --extra package --extra dev + - name: Check the page against the manifests before publishing it + run: uv run --no-sync pytest -q tests/test_site.py + - run: uv run --no-sync python tools/generate_site.py --output site + - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 + with: + enablement: true + - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.gitignore b/.gitignore index 75c6875..281a495 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /.artifacts/ /.venv/ +/site/ /lua55 /luac55 /.pytest_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8469893..67ae333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ Driver versions follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html ## [Unreleased] +### Added +- **A published driver catalog at [srcfl.github.io/device-drivers](https://srcfl.github.io/device-drivers/)** — all 80 drivers, searchable by manufacturer or model number and filterable by device type, protocol, tier and control. `tools/generate_site.py` builds it from the manifests and the Lua sources on every push to `main`, so no copy of this data is checked in and the page cannot fall behind what a host installs. It reports hardware evidence the way the repository does rather than the way a catalog usually does: 5 drivers state they have been confirmed against physical hardware and 28 state that they have not, and `tests/test_site.py` fails the build if a driver's `verification_status` is ever rendered as a stronger claim than the driver makes + ### Added - **The flap sungrow 1.5.7 fixed is measured across the catalog, and it is not one driver — it is eleven.** `drivers/tests/refused_write_probe.lua` refuses every write a device is asked to take, then calls `driver_default_mode` eight times and counts what the driver does about it. That path runs on a timer — lease expiry, the telemetry watchdog, the stale-meter standdown, shutdown — and nothing reaching it can say no, so a driver that writes whatever the device answers keeps writing for the life of the session. **atmoce, deye, ferroamp_modbus, huawei, pixii, sigenergy, solaredge, solaredge_legacy, solaredge_pv, solinteg and solis** all reissue the same refused write on every call and never stop. sungrow is the only one that settles, and only because a customer's SG12RT forced it - `drivers/tests/test_refused_write_settles.py` ratchets against `refused-write-baseline.json`, the way #36 did for reads: a count that rises fails, a driver not listed must be clean, and a count that falls fails until the file is updated, so the debt cannot be quietly re-borrowed. `make refused-write-report ID=` prints it for one driver diff --git a/Makefile b/Makefile index 7f351fd..0325b99 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,12 @@ LEVEL ?= patch .PHONY: bootstrap new-driver test-driver package-driver check boundary \ refused-write-report absent-register-report \ sync-manifests bump-driver history ftw-baseline ftw-baseline-report \ - host-api + host-api site + +# Build the public driver catalog page into site/, exactly as GitHub Pages +# publishes it. Open site/index.html to review a change before it ships. +site: + uv run --frozen --extra package --extra dev python tools/generate_site.py --output site # Does any driver call a host function no host provides? host-api: diff --git a/README.md b/README.md index 3d1ed88..a7ddc8b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,21 @@ Device Support may later consume an exact public commit to build packages for other products or a higher support level. It does not own a second editable copy of the source and is not FTW's normal driver source. +## Browse the catalog + +**[Every driver, and what stands behind it →](https://srcfl.github.io/device-drivers/)** + +The catalog page is generated from this repository and republished on every push +to `main`, so it cannot fall behind the source. The manifests supply versions, +tiers, protocols and tested models; each Lua source supplies its own description +and verification record. Search by manufacturer or model number, filter by device +type or protocol, or take the whole catalog as +[drivers.json](https://srcfl.github.io/device-drivers/drivers.json). + +Being listed is not an install claim. The page states the same evidence the +repository does, including how few drivers have been confirmed against physical +hardware. + ## Contribute a driver Start from **[`blueprint/BLUEPRINT.lua`](blueprint/BLUEPRINT.lua)**. It is a diff --git a/tests/test_site.py b/tests/test_site.py new file mode 100644 index 0000000..3f930f7 --- /dev/null +++ b/tests/test_site.py @@ -0,0 +1,127 @@ +"""The catalog page must not be able to say something the repository doesn't. + +The page is the first thing most people see about these drivers, and it is the +one artefact where a stale number would be invisible: nobody diffs a rendered +page against 80 manifests. So the generator is held to the same rule as the +other generated files here — it may only restate what the repository already +says, and it must restate all of it. + +The claim that matters most is hardware coverage. Five drivers say they have +run on real hardware and 28 say they have not; a page that blurred that +distinction would be worse than no page. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import generate_site # noqa: E402 +from manifest_parser import parse_yaml_simple # noqa: E402 + + +@pytest.fixture(scope="module") +def catalog() -> dict: + return generate_site.build_catalog() + + +@pytest.fixture(scope="module") +def rendered(tmp_path_factory) -> Path: + out = tmp_path_factory.mktemp("site") + generate_site.write_site(out) + return out + + +def manifest_ids() -> set[str]: + return {path.stem for path in (ROOT / "manifests").glob("*.yaml")} + + +def test_every_manifest_reaches_the_page(catalog: dict) -> None: + """A driver that exists but is missing from the catalog is invisible.""" + listed = {driver["id"] for driver in catalog["drivers"]} + assert listed == manifest_ids(), ( + "the page and manifests/ disagree about which drivers exist: " + f"{listed ^ manifest_ids()}") + + +def test_versions_match_the_manifests(catalog: dict) -> None: + """The page states an installable version; it has to be the real one.""" + for driver in catalog["drivers"]: + manifest = parse_yaml_simple( + (ROOT / "manifests" / f"{driver['id']}.yaml").read_text(encoding="utf-8")) + assert driver["version"] == manifest.get("version"), driver["id"] + assert driver["tier"] == manifest.get("tier", "community"), driver["id"] + assert driver["control"] == bool(manifest.get("control", False)), driver["id"] + + +def test_hardware_claims_come_only_from_the_driver_source(catalog: dict) -> None: + """`hardware_verified` may only be true for a status the driver itself sets. + + Every other status — experimental, beta, alpha — describes a driver that has + not been confirmed against the device, and must not be shown as if it had. + """ + for driver in catalog["drivers"]: + verification = driver["verification"] + if driver["hardware_verified"]: + assert verification is not None, driver["id"] + assert verification["status"] in generate_site.HARDWARE_VERIFIED, driver["id"] + assert verification["level"] == "confirmed", driver["id"] + elif verification: + assert verification["level"] != "confirmed", ( + f"{driver['id']} is shown as confirmed but its source says " + f"{verification['status']}") + + +def test_every_verification_status_has_wording(catalog: dict) -> None: + """An unmapped status would reach the page as a bare enum value.""" + for driver in catalog["drivers"]: + verification = driver["verification"] + if verification: + assert verification["status"] in generate_site.VERIFICATION, ( + f"{driver['id']}: verification_status " + f"{verification['status']!r} has no wording in VERIFICATION") + + +def test_tested_devices_survive_intact(catalog: dict) -> None: + """The model list is the reason to visit the page; it must be complete.""" + total = sum(len(d["tested_devices"]) for d in catalog["drivers"]) + assert total >= 170, f"only {total} tested-device entries reached the page" + for driver in catalog["drivers"]: + for device in driver["tested_devices"]: + assert device["manufacturer"], driver["id"] + assert device["model_family"], driver["id"] + + +def test_every_driver_says_something(catalog: dict) -> None: + """A row with no prose tells a reader nothing the id didn't already.""" + silent = [d["id"] for d in catalog["drivers"] + if not d["description"] and not d["source_note"]] + assert not silent, f"no description and no source note: {silent}" + + +def test_page_is_written_and_self_consistent(rendered: Path) -> None: + """The embedded payload has to survive being inlined into a script tag.""" + assert (rendered / ".nojekyll").exists() + sidecar = json.loads((rendered / "drivers.json").read_text(encoding="utf-8")) + + html = (rendered / "index.html").read_text(encoding="utf-8") + match = re.search(r"window\.__CATALOG__ = (\{.*?\});", html, re.DOTALL) + assert match, "the page no longer embeds a catalog the browser can read" + assert "" not in match.group(1), "payload can close its own script tag" + embedded = json.loads(match.group(1)) + assert embedded == sidecar, "the page and drivers.json disagree" + + +def test_page_lists_every_driver_without_javascript(rendered: Path) -> None: + """Search engines and reader modes never run the filter.""" + html = (rendered / "index.html").read_text(encoding="utf-8") + noscript = html[html.index("")] + for driver_id in sorted(manifest_ids()): + assert driver_id in noscript, f"{driver_id} is missing from the noscript list" diff --git a/tools/generate_site.py b/tools/generate_site.py new file mode 100644 index 0000000..36ca766 --- /dev/null +++ b/tools/generate_site.py @@ -0,0 +1,966 @@ +#!/usr/bin/env python3 +"""Generate the public driver catalog site from the repository's own data. + +Everything on the page is derived here, never typed by hand: the manifests are +the source for versions, tiers, protocols and tested devices; the Lua sources +supply the driver's own description, homepage and field-verification record; +support-status.json supplies per-target state and driver-history.json supplies +the published version history. + +Nothing is written back into the repository. The site is a rendering of the +repository at one commit, built in CI and deployed to GitHub Pages, so a page +that disagrees with `main` cannot exist. + +Usage: + uv run python tools/generate_site.py --output site +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from manifest_parser import parse_tested_devices, parse_yaml_simple + +ROOT = Path(__file__).resolve().parents[1] +REPO_URL = "https://github.com/srcfl/device-drivers" + +# What a DER label means to somebody who has not read the contracts. The page +# is the first thing a new reader sees, so it spells the vocabulary out. +DER_LABELS = { + "battery": "Battery", + "ev": "EV charger", + "heatpump": "Heat pump", + "meter": "Meter", + "pv": "Solar PV", + "v2x_charger": "V2X charger", + "vehicle": "Vehicle", +} +PROTOCOL_LABELS = { + "modbus": "Modbus", + "mqtt": "MQTT", + "http": "HTTP", + "serial": "Serial", + "": "Unspecified", +} +TIER_LABELS = { + "core": "Core", + "community": "Community", + "oem": "OEM", +} +# A driver's own verification_status is the only hardware claim on the page, so +# it is never flattened into a single "verified" mark. Most drivers are ported +# from a reference implementation and say so; the page has to say so too. +VERIFICATION = { + "production": ("Running in production", "confirmed"), + "tested": ("Verified on hardware", "confirmed"), + "beta": ("Partly verified", "partial"), + "alpha": ("Early verification", "partial"), + "experimental": ("Not verified on hardware", "unverified"), +} +HARDWARE_VERIFIED = {"production", "tested"} +MAX_SOURCE_NOTE_LINES = 12 + + +def git_output(*args: str) -> str: + """Read a value from git, returning "" when git cannot answer.""" + try: + result = subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=True) + except (OSError, subprocess.CalledProcessError): + return "" + return result.stdout.strip() + + +def lua_string(body: str, key: str) -> str: + """Read a `key = "value"` field out of a Lua table body.""" + match = re.search( + rf'^\s*{key}\s*=\s*"((?:[^"\\]|\\.)*)"', body, re.MULTILINE) + if not match: + return "" + return match.group(1).replace('\\"', '"').replace("\\\\", "\\") + + +def header_comment(text: str) -> list[str]: + """Return the leading `--` comment block, dashes removed, indent kept.""" + lines: list[str] = [] + for raw in text.splitlines(): + match = re.match(r"^\s*--(.*)$", raw.rstrip()) + if match: + body = match.group(1) + lines.append(body[1:] if body.startswith(" ") else body) + continue + if not raw.strip(): + if lines: + break + continue + break + while lines and not lines[-1].strip(): + lines.pop() + return lines + + +def clean_headline(line: str, driver_id: str) -> str: + """Turn a header comment's first line into a title. + + The comments were written for someone reading the file, so they carry a + filename, a trailing "Driver" and sometimes a "(community, untested)" aside + that the tier and verification fields already state properly. + """ + line = re.sub(rf"^{re.escape(driver_id)}\.lua\s*[–—:-]*\s*", "", + line.strip(), flags=re.IGNORECASE) + line = re.sub(r"\s*\((?:community|untested|experimental|beta|alpha)[^)]*\)\s*$", + "", line, flags=re.IGNORECASE) + line = re.sub(r"\s+(?:Lua\s+)?Driver$", "", line, flags=re.IGNORECASE) + return line.strip() + + +def read_driver_source(path: Path) -> dict: + """Pull the driver's self-description out of its Lua source. + + 38 of the drivers carry a `DRIVER = { ... }` table with a description, + homepage and verification record. The other 42 describe themselves in the + header comment only, so that comment supplies the title and, quoted + verbatim, the summary. Both are the driver's own words; neither is restated + in a second place that could drift from the code. + """ + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8") + info: dict = {} + + table = re.search(r'^DRIVER\s*=\s*\{(.*?)^\}', text, re.MULTILINE | re.DOTALL) + if table: + body = table.group(1) + for key in ("name", "manufacturer", "description", "homepage"): + value = lua_string(body, key) + if value: + info[key] = value + status = lua_string(body, "verification_status") + if status: + info["verification"] = { + "status": status, + "at": lua_string(body, "verified_at"), + "notes": lua_string(body, "verification_notes"), + } + + comment = header_comment(text) + if "name" not in info: + for index, line in enumerate(comment): + headline = clean_headline(line, path.stem) + if headline: + info["name"] = headline + comment = comment[index + 1:] + break + if "description" not in info and comment: + note = comment[:MAX_SOURCE_NOTE_LINES] + while note and not note[0].strip(): + note.pop(0) + while note and not note[-1].strip(): + note.pop() + if note: + info["source_note"] = "\n".join(note) + return info + + +def collect_drivers() -> list[dict]: + """Build one record per driver from the manifest, the source and the status.""" + support = json.loads( + (ROOT / "support-status.json").read_text(encoding="utf-8")) + status_by_driver = { + entry["driver_id"]: entry for entry in support.get("drivers", []) + } + history = json.loads( + (ROOT / "driver-history.json").read_text(encoding="utf-8")) + history_by_driver = history.get("drivers", {}) + + drivers = [] + for manifest_path in sorted((ROOT / "manifests").glob("*.yaml")): + text = manifest_path.read_text(encoding="utf-8") + data = parse_yaml_simple(text) + driver_id = data.get("name", manifest_path.stem) + source = read_driver_source(ROOT / "drivers" / "lua" / f"{driver_id}.lua") + + tested = [] + for device in parse_tested_devices(text): + tested.append({ + "manufacturer": device.get("manufacturer", "Unknown"), + "model_family": device.get( + "model_family", device.get("model", "Unknown")), + "variants": device.get("variants", []) or [], + "regions": device.get("regions", []) or [], + "firmware_versions": device.get("firmware_versions", ""), + "notes": device.get("notes", ""), + "min_driver_version": device.get("min_driver_version", ""), + }) + + manufacturers = [] + for name in [source.get("manufacturer", "")] + [d["manufacturer"] for d in tested]: + if name and name not in manufacturers: + manufacturers.append(name) + + targets = status_by_driver.get(driver_id, {}).get("targets", {}) + releases = history_by_driver.get(driver_id, []) + + verification = source.get("verification") + if verification: + label, level = VERIFICATION.get( + verification["status"], (verification["status"], "partial")) + verification["label"] = label + verification["level"] = level + + drivers.append({ + "id": driver_id, + "title": source.get("name", "") or driver_id, + "description": source.get("description", ""), + "source_note": source.get("source_note", ""), + "homepage": source.get("homepage", ""), + "manufacturers": manufacturers, + "version": data.get("version", ""), + "tier": data.get("tier", "community"), + "protocol": data.get("protocol", ""), + "ders": data.get("ders", []) or [], + "control": bool(data.get("control", False)), + "author": data.get("author", ""), + "min_host_version": data.get("min_host_version", ""), + "size_bytes": data.get("size_bytes", 0), + "sha256": data.get("sha256", ""), + "tested_devices": tested, + "verification": verification, + "hardware_verified": bool( + verification and verification["status"] in HARDWARE_VERIFIED), + "targets": { + name: { + "conformance": target.get("target_conformance", "not_assessed"), + "hil": target.get("hil", "not_recorded"), + "control_enabled": bool(target.get("control_enabled", False)), + "stable": target.get("stable_package_version"), + "beta": target.get("historical_signed_beta_version"), + "note": target.get("note", ""), + } + for name, target in sorted(targets.items()) + }, + "releases": [ + {"version": r.get("version", ""), "date": (r.get("date", "") or "")[:10]} + for r in releases + ], + "source_url": f"{REPO_URL}/blob/main/drivers/lua/{driver_id}.lua", + "manifest_url": f"{REPO_URL}/blob/main/manifests/{driver_id}.yaml", + }) + return drivers + + +def build_catalog() -> dict: + """Assemble the whole payload the page renders, plus its provenance.""" + drivers = collect_drivers() + + families: set[tuple[str, str]] = set() + manufacturers: set[str] = set() + variants = 0 + for driver in drivers: + manufacturers.update(driver["manufacturers"]) + for device in driver["tested_devices"]: + families.add((device["manufacturer"], device["model_family"])) + variants += len(device["variants"]) + + commit = git_output("rev-parse", "HEAD") + return { + "drivers": drivers, + "labels": { + "ders": DER_LABELS, + "protocols": PROTOCOL_LABELS, + "tiers": TIER_LABELS, + }, + "totals": { + "drivers": len(drivers), + "manufacturers": len(manufacturers), + "model_families": len(families), + "variants": variants, + "control": sum(1 for d in drivers if d["control"]), + "hardware_verified": sum(1 for d in drivers if d["hardware_verified"]), + }, + "source": { + "repository": REPO_URL, + "commit": commit, + "commit_short": commit[:7], + "commit_date": git_output("log", "-1", "--format=%cs"), + }, + } + + +def embed_json(payload: dict) -> str: + """Serialise for a + + + +""" + + +def write_site(out: Path) -> dict: + """Render the site into `out` and return the catalog it was built from.""" + catalog = build_catalog() + out.mkdir(parents=True, exist_ok=True) + (out / "index.html").write_text(render_html(catalog), encoding="utf-8") + # The same payload as a file, for anyone who wants the catalog as data. + (out / "drivers.json").write_text( + json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8") + # GitHub Pages must serve the files as generated, not run Jekyll over them. + (out / ".nojekyll").write_text("", encoding="utf-8") + return catalog + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", default="site", + help="directory to write the site into (default: site)") + args = parser.parse_args() + + out = Path(args.output) + if not out.is_absolute(): + out = ROOT / out + catalog = write_site(out) + + totals = catalog["totals"] + print(f"Generated {out}/index.html: {totals['drivers']} drivers, " + f"{totals['manufacturers']} manufacturers, " + f"{totals['model_families']} model families.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())