From c6c92ac012544ed8bda20ac68af0eb0421d6c453 Mon Sep 17 00:00:00 2001 From: Aleksandr Markov Date: Fri, 5 Jun 2026 21:43:18 +0200 Subject: [PATCH 1/2] feat(AUTOPILOT-DEMO-QUALITY-SCORECARD): add deterministic public quality scorecard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/quality_scorecard.py — a stdlib-only, deterministic, public-safe scorecard that emits Markdown + JSON metrics for code size, largest modules, Ruff ignores, Pyright mode, unreasoned suppressions, test count, available CLI/MCP/smoke suites, and the canonical verification commands. Establishes the measurable baseline for the AUTOPILOT DEMO backlog section so future cleanup tasks report before/after deltas instead of inventing their own reporting. - One-command usage: `python scripts/quality_scorecard.py [--format both|json]`. - `--write` regenerates committed baseline under docs/quality/. - `--check` validates shape, self-determinism, and public-safety (no private paths/secrets); wired into the CI lint job and /verify, failing on malformed or unsafe output. No network, no project import — runs without install. - Suppression policy mirrors tests/test_type_suppressions.py; fixtures excluded. - Focused tests cover metric logic (hermetic synthetic repo), output shape, determinism, public-safety, validation, and entry points. Committed public-safe scorecard: docs/quality/scorecard.md, docs/quality/scorecard.json Update workflow for demo tasks: docs/quality/README.md Verification: ruff check mempalace_code/ tests/ scripts/ -> clean ruff format --check mempalace_code/ tests/ scripts/ -> clean python -m pyright -> 0 errors python scripts/quality_scorecard.py --check -> OK python -m pytest tests/test_quality_scorecard.py -q -> 27 passed python -m pytest tests/ -q -> 2313 passed Constraint: scorecard stays stdlib-only so it runs in the CI lint job and /verify without installing the package. Scope-risk: --check validates shape/determinism, not exact counts, so normal code growth must not break CI. Co-Authored-By: Claude Opus 4.8 --- .claude/skills/verify/INSTRUCTIONS.md | 7 + .github/workflows/ci.yml | 6 +- docs/BACKLOG.yaml | 14 +- docs/plans/AUTOPILOT-DEMO-QUALITY-ROADMAP.md | 12 +- docs/quality/README.md | 79 +++ docs/quality/scorecard.json | 211 +++++++ docs/quality/scorecard.md | 95 +++ scripts/quality_scorecard.py | 623 +++++++++++++++++++ tests/test_quality_scorecard.py | 278 +++++++++ 9 files changed, 1311 insertions(+), 14 deletions(-) create mode 100644 docs/quality/README.md create mode 100644 docs/quality/scorecard.json create mode 100644 docs/quality/scorecard.md create mode 100644 scripts/quality_scorecard.py create mode 100644 tests/test_quality_scorecard.py diff --git a/.claude/skills/verify/INSTRUCTIONS.md b/.claude/skills/verify/INSTRUCTIONS.md index 636bd9b..b8cbff2 100644 --- a/.claude/skills/verify/INSTRUCTIONS.md +++ b/.claude/skills/verify/INSTRUCTIONS.md @@ -49,6 +49,13 @@ Run in parallel: | Format | `ruff format --check mempalace_code/ tests/ scripts/` | 30s | | Tests | `python -m pytest tests/ -x -q -m "not needs_network"` | 120s | | Typecheck | `python -m pyright --pythonpath "$(python -c 'import sys; print(sys.executable)')"` | 120s | +| Scorecard | `python scripts/quality_scorecard.py --check` | 30s | + +The scorecard check is stdlib-only (no install, no network) and validates the +quality scorecard's shape, determinism, and public-safety. It fails on malformed +or unsafe output. After a quality change lands, regenerate the committed +artifacts with `python scripts/quality_scorecard.py --write` (see +`docs/quality/README.md`). ### If storage changed — add these diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a66225f..4276d1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,10 @@ jobs: pyproject.toml uv.lock - run: pip install ruff - - run: ruff check mempalace_code/ tests/ - - run: ruff format --check mempalace_code/ tests/ + - run: ruff check mempalace_code/ tests/ scripts/ + - run: ruff format --check mempalace_code/ tests/ scripts/ + - name: Quality scorecard (shape + determinism + public-safety) + run: python scripts/quality_scorecard.py --check typecheck: runs-on: ubuntu-latest diff --git a/docs/BACKLOG.yaml b/docs/BACKLOG.yaml index f9b8dad..1cd8507 100644 --- a/docs/BACKLOG.yaml +++ b/docs/BACKLOG.yaml @@ -57,7 +57,6 @@ items: - Assert `mempalace-code search` exits successfully without `HF_TOKEN`, `unauthenticated`, `huggingface.co`, or retry noise on stdout/stderr. - Assert repeated `mempalace-code fetch-model` on a cached model uses local-only resolution and does not perform Hub metadata requests. - Keep the existing network-marked download tests separate from this no-network regression guard. - - key: DEPENDENCY-SECURITY-UPGRADE-GATE section: quality status: open @@ -69,10 +68,9 @@ items: - Run pip-audit or an equivalent resolver-level audit on fresh environments for the default install and every optional extra whose bounds change. - Keep deprecated optional backends capped away from affected ranges; do not raise ChromaDB to 1.x while GHSA-f4j7-r4q5-qw2c affects the available 1.x line. - Update uv.lock only after the audited resolver passes, then run hosted-CI-equivalent tests in a clean pip environment so stale local locks cannot hide dependency drift. - - key: AUTOPILOT-DEMO-QUALITY-SCORECARD section: autopilot_demo - status: open + status: done priority: P1 summary: Add a deterministic public quality scorecard that makes Autopilot cleanup progress visible across releases. acceptance: @@ -80,7 +78,8 @@ items: - Generate any raw baseline evidence under local ignored docs/audits/, then commit only a sanitized public summary with public repo data and relative paths. - Document how each Autopilot demo task updates the scorecard with before/after metrics. - CI or `/verify` runs the scorecard command and fails on malformed output. - + resolution: "2026-06-05: Deterministic public scorecard: scripts/quality_scorecard.py emits Markdown+JSON; --check gates shape/determinism/public-safety in CI lint job and /verify; baseline committed to docs/quality/; update workflow in docs/quality/README.md" + done_summary: "Deterministic public scorecard: scripts/quality_scorecard.py emits Markdown+JSON; --check gates shape/determinism/public-safety in CI lint job and /verify; baseline committed to docs/quality/; update workflow in docs/quality/README.md" - key: AUTOPILOT-DEMO-RUFF-RATCHET section: autopilot_demo status: open @@ -91,7 +90,6 @@ items: - Remove at least one global ignore family or reduce per-file ignore entries by at least 25% for a focused module group. - Replace any necessary inline suppressions with reasoned, narrow suppressions; no new blanket `noqa` entries. - "`ruff check`, `ruff format --check`, Pyright, and the relevant focused tests pass; update the quality scorecard." - - key: AUTOPILOT-DEMO-PYRIGHT-STRICT-SLICE section: autopilot_demo status: open @@ -102,7 +100,6 @@ items: - Replace ad hoc `Any` and broad dictionaries in that slice with typed contracts where the code owns the boundary. - Keep compatibility shims and third-party stub gaps outside the first strict slice unless they can be fixed narrowly. - CI or `/verify` runs both the existing basic Pyright check and the strict-slice check; update the quality scorecard. - - key: AUTOPILOT-DEMO-ARCHITECTURE-GUARD section: autopilot_demo status: open @@ -113,7 +110,6 @@ items: - Enforce at minimum that storage/config/mining core does not import CLI, MCP dispatch, or Chroma-only modules. - Report cycles and boundary violations with file-level paths that are easy for Autopilot to fix. - Add tests for the guard itself and wire the command into `/verify` or CI; update the quality scorecard. - - key: AUTOPILOT-DEMO-CLI-GOLDEN-SCENARIOS section: autopilot_demo status: open @@ -124,7 +120,6 @@ items: - Cover at least one important guard/failure path such as missing palace, invalid read range, or unsafe mirror preflight. - Force offline/version-check-disabled environment variables so the suite is deterministic and network-free. - Remove all smoke artifacts or report them explicitly; update release/verify docs and the quality scorecard. - - key: AUTOPILOT-DEMO-MCP-STDIO-CONTRACTS section: autopilot_demo status: open @@ -135,7 +130,6 @@ items: - Verify `minimal`, `code`, `kg`, `notes`, and `full` profiles expose only the expected tools. - Include representative success and error responses for read-only, write, graph/KG, and disabled-tool paths. - Keep direct handler tests as fast unit coverage but label them separately from real MCP stdio coverage; update the quality scorecard. - - key: AUTOPILOT-DEMO-SECURITY-BOUNDARY-TESTS section: autopilot_demo status: open @@ -146,7 +140,6 @@ items: - Prefer standard-library table-driven tests; add a fuzz/property-test dependency only if it clearly improves signal and stays dev-only. - Every rejected input returns a stable error code/message and does not create partial palace state. - Document any remaining accepted risk in a public-safe plan or backlog follow-up; update the quality scorecard. - - key: AUTOPILOT-DEMO-PERF-BUDGETS section: autopilot_demo status: open @@ -157,7 +150,6 @@ items: - Measure mine time, incremental no-op time, search latency, read latency, and cleanup/optimize smoke duration. - Store conservative budgets and machine-independent comparison rules so CI failures indicate meaningful regressions, not normal hardware variance. - Publish before/after numbers in the quality scorecard and release notes when a demo task improves a budget. - - key: AUTOPILOT-DEMO-DOCS-DRIFT-GUARD section: autopilot_demo status: open diff --git a/docs/plans/AUTOPILOT-DEMO-QUALITY-ROADMAP.md b/docs/plans/AUTOPILOT-DEMO-QUALITY-ROADMAP.md index 5c7329f..6d9ddc7 100644 --- a/docs/plans/AUTOPILOT-DEMO-QUALITY-ROADMAP.md +++ b/docs/plans/AUTOPILOT-DEMO-QUALITY-ROADMAP.md @@ -41,9 +41,15 @@ destabilizing it. Each task should produce three things: ## Suggested Sequence -1. `AUTOPILOT-DEMO-QUALITY-SCORECARD` +1. `AUTOPILOT-DEMO-QUALITY-SCORECARD` — **done (baseline established)** - Establish the baseline first. Later tasks should update it instead of inventing their own reporting format. + - Implemented as `scripts/quality_scorecard.py` (stdlib-only, deterministic, + public-safe). Emits Markdown + JSON; `--write` regenerates the committed + baseline under `docs/quality/`, `--check` gates shape/determinism/public- + safety in CI (`lint` job) and `/verify`. + - Baseline artifacts: `docs/quality/scorecard.md`, `docs/quality/scorecard.json`. + - Update workflow for later tasks: `docs/quality/README.md`. 2. `AUTOPILOT-DEMO-CLI-GOLDEN-SCENARIOS` and `AUTOPILOT-DEMO-MCP-STDIO-CONTRACTS` - These make the demo credible because they prove real user surfaces before @@ -76,3 +82,7 @@ destabilizing it. Each task should produce three things: These are not criticisms by themselves. They are the useful visible surfaces for showing controlled, evidence-backed quality improvement. + +The current measured values for these signals live in the committed scorecard +(`docs/quality/scorecard.json`); regenerate with +`python scripts/quality_scorecard.py --write`. diff --git a/docs/quality/README.md b/docs/quality/README.md new file mode 100644 index 0000000..8846e36 --- /dev/null +++ b/docs/quality/README.md @@ -0,0 +1,79 @@ +# Quality Scorecard + +A deterministic, public-safe snapshot of code-quality signals for the +`AUTOPILOT DEMO` backlog section. It exists so cleanup work has measurable +before/after evidence instead of becoming invisible churn. + +## Files + +| File | Purpose | +|------|---------| +| [`scorecard.md`](scorecard.md) | Human-readable baseline (committed). | +| [`scorecard.json`](scorecard.json) | Machine-readable baseline (committed). | + +Both are generated by [`scripts/quality_scorecard.py`](../../scripts/quality_scorecard.py). + +## Generate + +```bash +# Print to stdout +python scripts/quality_scorecard.py # Markdown +python scripts/quality_scorecard.py --format json # JSON +python scripts/quality_scorecard.py --format both # both + +# Regenerate the committed artifacts under docs/quality/ +python scripts/quality_scorecard.py --write + +# Validate shape, determinism, and public-safety (CI / verify gate) +python scripts/quality_scorecard.py --check +``` + +The script uses **only the Python standard library** (no project import, no +network), so it runs anywhere Python 3.11+ is available. + +## What it measures + +All metrics are repo-local and public-safe by construction — no timestamps, no +absolute paths, no machine identifiers: + +- **Code size** — package/test file counts and line totals. +- **Largest modules** — top modules by line count (relative paths only). +- **Ruff ignores** — global ignore count, selected rule families, and per-file + ignore patterns/entries from `pyproject.toml`. +- **Pyright** — type-checking mode and strict status. +- **Suppressions** — total and *unreasoned* type/pyright ignores and blanket + `noqa`, using the same policy as + [`tests/test_type_suppressions.py`](../../tests/test_type_suppressions.py) + (`# type: ignore[code] # reason: ...`). Fixture dirs are excluded. +- **Tests** — test file and test-function counts. +- **Suites** — which CLI/MCP/smoke surfaces exist. +- **Verification commands** — the canonical local/CI checks. + +## Determinism & public-safety + +`--check` builds the scorecard twice and fails if the JSON differs, validates +the output shape (types, required keys, sort order, invariants), and runs a +public-safety scan for private paths and secret-like tokens. CI (`lint` job) and +`/verify` run `--check` so malformed or unsafe output fails the build. Because +`--check` validates *shape and determinism* (not exact counts), normal code +growth does not break CI — only a malformed scorecard does. + +## How Autopilot demo tasks update this scorecard + +Every task in the `AUTOPILOT DEMO` backlog section should record a before/after +delta here: + +1. **Before** the change, capture the baseline: `python scripts/quality_scorecard.py --format json`. + Keep raw/private working notes only under the git-ignored `docs/audits/`; + never commit them. +2. Make the focused change (e.g. remove a Ruff ignore family, add a strict + Pyright slice, add a real CLI/MCP suite). +3. **After** the change, regenerate the committed artifacts: + `python scripts/quality_scorecard.py --write`. +4. In the PR/commit description and release notes, cite the moved metric using + public repo data only — e.g. "unreasoned suppressions N → M", + "Ruff global ignores N → M", "Pyright strict slice added". The committed + `scorecard.json` diff is the evidence. + +Run `python scripts/quality_scorecard.py --check` before committing so the +regenerated artifacts stay well-formed and public-safe. diff --git a/docs/quality/scorecard.json b/docs/quality/scorecard.json new file mode 100644 index 0000000..aa9c2cf --- /dev/null +++ b/docs/quality/scorecard.json @@ -0,0 +1,211 @@ +{ + "code_size": { + "package_code_lines": 18978, + "package_files": 73, + "package_total_lines": 23441, + "test_files": 53, + "test_total_lines": 38890 + }, + "largest_modules": [ + { + "lines": 1609, + "path": "mempalace_code/storage.py" + }, + { + "lines": 1466, + "path": "mempalace_code/mining/chunkers.py" + }, + { + "lines": 1074, + "path": "mempalace_code/dialect.py" + }, + { + "lines": 1032, + "path": "mempalace_code/watcher.py" + }, + { + "lines": 904, + "path": "mempalace_code/mining/symbols.py" + }, + { + "lines": 858, + "path": "mempalace_code/entity_detector.py" + }, + { + "lines": 753, + "path": "mempalace_code/knowledge_graph.py" + }, + { + "lines": 735, + "path": "mempalace_code/cli.py" + }, + { + "lines": 720, + "path": "mempalace_code/mining/orchestrator.py" + }, + { + "lines": 660, + "path": "mempalace_code/entity_registry.py" + } + ], + "pyright": { + "include": [ + "mempalace_code", + "tests" + ], + "python_version": "3.11", + "strict": false, + "type_checking_mode": "basic" + }, + "ruff": { + "global_ignore_rules": [ + "ARG001", + "ARG002", + "ARG005", + "B007", + "B027", + "B033", + "B904", + "B905", + "DTZ001", + "DTZ005", + "DTZ011", + "E501", + "PT001", + "PT006", + "PT017", + "RET504", + "RET505", + "RET508", + "SIM102", + "SIM103", + "SIM105", + "SIM108", + "SIM114", + "SIM115", + "SIM116", + "SIM117", + "UP006", + "UP015", + "UP024", + "UP028", + "UP035", + "UP036", + "UP045" + ], + "global_ignores": 33, + "per_file_ignores": { + "by_pattern": [ + { + "count": 24, + "pattern": "mempalace_code/**/*.py" + }, + { + "count": 14, + "pattern": "tests/**/*.py" + } + ], + "patterns": 2, + "total_entries": 38 + }, + "selected_rule_families": 12 + }, + "schema_version": 1, + "scope": { + "package_dir": "mempalace_code", + "tests_dir": "tests" + }, + "suites": [ + { + "description": "CLI command tests", + "name": "cli", + "path": "tests/test_cli.py", + "present": true + }, + { + "description": "End-to-end CLI workflow tests", + "name": "cli_e2e", + "path": "tests/test_e2e.py", + "present": true + }, + { + "description": "CLI command module tests", + "name": "cli_command_modules", + "path": "tests/test_cli_command_modules.py", + "present": true + }, + { + "description": "MCP server handler tests", + "name": "mcp_server", + "path": "tests/test_mcp_server.py", + "present": true + }, + { + "description": "MCP stdio transport tests", + "name": "mcp_stdio", + "path": "tests/test_stdio.py", + "present": true + }, + { + "description": "MCP tool profile tests", + "name": "mcp_tool_profiles", + "path": "tests/test_mcp_tool_profiles.py", + "present": true + }, + { + "description": "Backup/restore CLI tests", + "name": "backup_cli", + "path": "tests/test_backup_cli.py", + "present": true + }, + { + "description": "Offline / no-network guard tests", + "name": "offline", + "path": "tests/test_offline.py", + "present": true + }, + { + "description": "migrate-storage disposable smoke", + "name": "migrate_storage_smoke", + "path": "scripts/migrate_storage_smoke.py", + "present": true + } + ], + "suppressions": { + "noqa_blanket": 0, + "noqa_total": 41, + "scope": [ + "mempalace_code", + "tests" + ], + "type_pyright_total": 109, + "type_pyright_unreasoned": 0, + "unreasoned_total": 0 + }, + "tests": { + "test_files": 53, + "test_functions": 2240 + }, + "verification_commands": [ + { + "command": "ruff check mempalace_code/ tests/ scripts/", + "name": "lint" + }, + { + "command": "ruff format --check mempalace_code/ tests/ scripts/", + "name": "format" + }, + { + "command": "python -m pyright", + "name": "typecheck" + }, + { + "command": "python -m pytest tests/ -x -q -m \"not needs_network\"", + "name": "tests" + }, + { + "command": "python scripts/quality_scorecard.py --check", + "name": "scorecard" + } + ] +} diff --git a/docs/quality/scorecard.md b/docs/quality/scorecard.md new file mode 100644 index 0000000..9abc0a4 --- /dev/null +++ b/docs/quality/scorecard.md @@ -0,0 +1,95 @@ +# Quality Scorecard + +Deterministic, repo-local, public-safe metrics generated by `scripts/quality_scorecard.py`. Regenerate with `python scripts/quality_scorecard.py --write`. No timestamps or absolute paths — two runs on the same tree produce identical output. + +Schema version: 1 + +## Code Size + +| Metric | Value | +|--------|------:| +| Package files (`mempalace_code/`) | 73 | +| Package total lines | 23441 | +| Package code lines | 18978 | +| Test files (`tests/`) | 53 | +| Test total lines | 38890 | + +## Largest Modules (top 10) + +| Module | Lines | +|--------|------:| +| `mempalace_code/storage.py` | 1609 | +| `mempalace_code/mining/chunkers.py` | 1466 | +| `mempalace_code/dialect.py` | 1074 | +| `mempalace_code/watcher.py` | 1032 | +| `mempalace_code/mining/symbols.py` | 904 | +| `mempalace_code/entity_detector.py` | 858 | +| `mempalace_code/knowledge_graph.py` | 753 | +| `mempalace_code/cli.py` | 735 | +| `mempalace_code/mining/orchestrator.py` | 720 | +| `mempalace_code/entity_registry.py` | 660 | + +## Ruff Ignores + +| Metric | Value | +|--------|------:| +| Selected rule families | 12 | +| Global ignores | 33 | +| Per-file ignore patterns | 2 | +| Per-file ignore entries | 38 | + +| Per-file pattern | Entries | +|------------------|--------:| +| `mempalace_code/**/*.py` | 24 | +| `tests/**/*.py` | 14 | + +## Pyright + +| Metric | Value | +|--------|-------| +| Type-checking mode | basic | +| Strict | false | +| Python version | 3.11 | +| Include | `mempalace_code`, `tests` | + +## Suppressions + +Scope: `mempalace_code/`, `tests/` (excludes `tests/fixtures/`). + +| Metric | Value | +|--------|------:| +| type/pyright ignores (total) | 109 | +| type/pyright unreasoned | 0 | +| noqa (total) | 41 | +| noqa blanket | 0 | +| **Unreasoned suppressions (total)** | **0** | + +## Tests + +| Metric | Value | +|--------|------:| +| Test files | 53 | +| Test functions | 2240 | + +## Available Suites + +| Suite | Path | Present | +|-------|------|:-------:| +| cli | `tests/test_cli.py` | yes | +| cli_e2e | `tests/test_e2e.py` | yes | +| cli_command_modules | `tests/test_cli_command_modules.py` | yes | +| mcp_server | `tests/test_mcp_server.py` | yes | +| mcp_stdio | `tests/test_stdio.py` | yes | +| mcp_tool_profiles | `tests/test_mcp_tool_profiles.py` | yes | +| backup_cli | `tests/test_backup_cli.py` | yes | +| offline | `tests/test_offline.py` | yes | +| migrate_storage_smoke | `scripts/migrate_storage_smoke.py` | yes | + +## Verification Commands + +- **lint**: `ruff check mempalace_code/ tests/ scripts/` +- **format**: `ruff format --check mempalace_code/ tests/ scripts/` +- **typecheck**: `python -m pyright` +- **tests**: `python -m pytest tests/ -x -q -m "not needs_network"` +- **scorecard**: `python scripts/quality_scorecard.py --check` + diff --git a/scripts/quality_scorecard.py b/scripts/quality_scorecard.py new file mode 100644 index 0000000..94fcc70 --- /dev/null +++ b/scripts/quality_scorecard.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +""" +quality_scorecard.py — Deterministic, public-safe code-quality scorecard. + +Emits a repo-local quality snapshot in Markdown and/or JSON. Every metric is +derived from tracked repository data only (source files, test files, and +``pyproject.toml``). There are no timestamps, no absolute paths, no machine +identifiers, and no network access — two runs against the same tree produce +byte-identical output, which is what makes the scorecard safe for CI validation. + +This is the baseline tool for the ``AUTOPILOT DEMO`` backlog section: future +cleanup tasks regenerate the scorecard and report before/after deltas instead of +inventing their own reporting format. + +Metrics (all public-safe, repo-local): + - code size / file counts + - largest modules + - Ruff global + per-file ignore counts + - Pyright mode / strictness status + - unreasoned suppressions (same policy as tests/test_type_suppressions.py) + - test count + - available smoke / CLI / MCP suites + - current verification commands + +Usage: + python scripts/quality_scorecard.py # Markdown to stdout + python scripts/quality_scorecard.py --format json # JSON to stdout + python scripts/quality_scorecard.py --format both # both, to stdout + python scripts/quality_scorecard.py --write # write docs/quality/* + python scripts/quality_scorecard.py --check # validate shape (CI) + +Stdlib only — no project import, no third-party dependency — so it runs in the +lint CI job and in /verify without installing the package. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import re +import sys +from pathlib import Path + +SCHEMA_VERSION = 1 +PACKAGE_DIR = "mempalace_code" +TESTS_DIR = "tests" +# Excluded everywhere: negative fixtures intentionally contain bad suppressions +# (see tests/fixtures/unreasoned_suppression.py) and are not real tests. +EXCLUDED_DIRS = ("tests/fixtures",) +TOP_MODULES = 10 + +# Suppression policy — kept identical to tests/test_type_suppressions.py so the +# scorecard's "unreasoned" count matches the gate that enforces it. +_SUPPRESSION_RE = re.compile(r"#\s*(?:type|pyright):\s*ignore") +_ACCEPTED_RE = re.compile(r"#\s*(?:type|pyright):\s*ignore\[[^\]\s]+\]\s*#\s*reason:\s*\S") +_NOQA_RE = re.compile(r"#\s*noqa") +_NOQA_BLANKET_RE = re.compile(r"#\s*noqa(?!\s*:)") +_TEST_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+(test\w*)\s*\(") + +# Public-safety patterns — the rendered output must never contain a private path, +# token, or key. Mirrors the commit-checkpoint preflight regex set. +_FORBIDDEN_PATTERNS = ( + re.compile(r"/Users/"), + re.compile(r"/home/[A-Za-z0-9._-]+"), + re.compile(r"/srv/"), + re.compile(r"github_pat_"), + re.compile(r"\bghp_[A-Za-z0-9]{20,}"), + re.compile(r"\bpypi-[A-Za-z0-9_-]{20,}"), + re.compile(r"\bsk-[A-Za-z0-9]{16,}"), +) + +# Known integration / smoke / contract surfaces. Presence is detected by path so +# the scorecard reports which real-workflow suites exist, not just unit count. +_KNOWN_SUITES = ( + ("cli", "tests/test_cli.py", "CLI command tests"), + ("cli_e2e", "tests/test_e2e.py", "End-to-end CLI workflow tests"), + ("cli_command_modules", "tests/test_cli_command_modules.py", "CLI command module tests"), + ("mcp_server", "tests/test_mcp_server.py", "MCP server handler tests"), + ("mcp_stdio", "tests/test_stdio.py", "MCP stdio transport tests"), + ("mcp_tool_profiles", "tests/test_mcp_tool_profiles.py", "MCP tool profile tests"), + ("backup_cli", "tests/test_backup_cli.py", "Backup/restore CLI tests"), + ("offline", "tests/test_offline.py", "Offline / no-network guard tests"), + ( + "migrate_storage_smoke", + "scripts/migrate_storage_smoke.py", + "migrate-storage disposable smoke", + ), +) + +# Canonical verification commands a maintainer runs locally / in CI. Listed with +# relative paths only; this is the public verification surface, not private state. +_VERIFICATION_COMMANDS = ( + ("lint", "ruff check mempalace_code/ tests/ scripts/"), + ("format", "ruff format --check mempalace_code/ tests/ scripts/"), + ("typecheck", "python -m pyright"), + ("tests", 'python -m pytest tests/ -x -q -m "not needs_network"'), + ("scorecard", "python scripts/quality_scorecard.py --check"), +) + + +def repo_root() -> Path: + """Repository root — the parent of this script's ``scripts/`` directory.""" + return Path(__file__).resolve().parent.parent + + +def _is_excluded(path: Path, root: Path) -> bool: + rel = path.relative_to(root).as_posix() + return any(rel == d or rel.startswith(f"{d}/") for d in EXCLUDED_DIRS) + + +def _iter_py_files(directory: Path, root: Path) -> list[Path]: + """All ``*.py`` files under ``directory``, excluding fixture dirs, sorted.""" + files = [p for p in directory.rglob("*.py") if not _is_excluded(p, root)] + return sorted(files) + + +def _count_lines(path: Path) -> tuple[int, int]: + """Return (total physical lines, code lines). + + A code line is any non-blank line that is not a pure comment. Docstring + bodies count as code — the metric tracks size, not semantics, and stays + deterministic. + """ + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + total = len(lines) + code = sum(1 for ln in lines if ln.strip() and not ln.lstrip().startswith("#")) + return total, code + + +def collect_code_size(root: Path) -> dict: + pkg_files = _iter_py_files(root / PACKAGE_DIR, root) + test_files = _iter_py_files(root / TESTS_DIR, root) + pkg_total = pkg_code = 0 + for p in pkg_files: + t, c = _count_lines(p) + pkg_total += t + pkg_code += c + test_total = sum(_count_lines(p)[0] for p in test_files) + return { + "package_files": len(pkg_files), + "package_total_lines": pkg_total, + "package_code_lines": pkg_code, + "test_files": len(test_files), + "test_total_lines": test_total, + } + + +def collect_largest_modules(root: Path, top_n: int = TOP_MODULES) -> list[dict]: + sizes = [] + for p in _iter_py_files(root / PACKAGE_DIR, root): + total, _ = _count_lines(p) + sizes.append({"path": p.relative_to(root).as_posix(), "lines": total}) + sizes.sort(key=lambda m: (-m["lines"], m["path"])) + return sizes[:top_n] + + +def load_pyproject(root: Path) -> dict: + import tomllib + + with (root / "pyproject.toml").open("rb") as fh: + return tomllib.load(fh) + + +def collect_ruff(pyproject: dict) -> dict: + ruff = pyproject.get("tool", {}).get("ruff", {}) + lint = ruff.get("lint", {}) + global_ignores = sorted(lint.get("ignore", [])) + selected = lint.get("select", []) + per_file = lint.get("per-file-ignores", {}) + by_pattern = sorted( + ({"pattern": pat, "count": len(rules)} for pat, rules in per_file.items()), + key=lambda e: e["pattern"], + ) + return { + "global_ignores": len(global_ignores), + "global_ignore_rules": global_ignores, + "selected_rule_families": len(selected), + "per_file_ignores": { + "patterns": len(by_pattern), + "total_entries": sum(e["count"] for e in by_pattern), + "by_pattern": by_pattern, + }, + } + + +def collect_pyright(pyproject: dict) -> dict: + pyright = pyproject.get("tool", {}).get("pyright", {}) + mode = pyright.get("typeCheckingMode", "off") + return { + "type_checking_mode": mode, + "python_version": str(pyright.get("pythonVersion", "")), + "strict": mode == "strict", + "include": sorted(pyright.get("include", [])), + } + + +def collect_suppressions(root: Path) -> dict: + """Count type/pyright/noqa suppressions across package + tests (no fixtures). + + "Unreasoned" follows tests/test_type_suppressions.py: a type/pyright ignore + without a ``[code]`` and a ``# reason:`` justification, plus any blanket + ``# noqa`` carrying no specific rule code. + """ + files = _iter_py_files(root / PACKAGE_DIR, root) + _iter_py_files(root / TESTS_DIR, root) + type_total = type_unreasoned = noqa_total = noqa_blanket = 0 + for path in sorted(files): + for line in path.read_text(encoding="utf-8").splitlines(): + if _SUPPRESSION_RE.search(line): + type_total += 1 + if not _ACCEPTED_RE.search(line): + type_unreasoned += 1 + if _NOQA_RE.search(line): + noqa_total += 1 + if _NOQA_BLANKET_RE.search(line): + noqa_blanket += 1 + return { + "scope": [PACKAGE_DIR, TESTS_DIR], + "type_pyright_total": type_total, + "type_pyright_unreasoned": type_unreasoned, + "noqa_total": noqa_total, + "noqa_blanket": noqa_blanket, + "unreasoned_total": type_unreasoned + noqa_blanket, + } + + +def _count_test_functions(path: Path) -> int: + """Count ``test*`` functions/methods in a file via AST, regex on parse error.""" + text = path.read_text(encoding="utf-8") + try: + tree = ast.parse(text) + except SyntaxError: + return sum(1 for ln in text.splitlines() if _TEST_DEF_RE.match(ln)) + count = 0 + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith( + "test" + ): + count += 1 + return count + + +def collect_tests(root: Path) -> dict: + test_files = _iter_py_files(root / TESTS_DIR, root) + return { + "test_files": len(test_files), + "test_functions": sum(_count_test_functions(p) for p in test_files), + } + + +def collect_suites(root: Path) -> list[dict]: + return [ + {"name": name, "path": rel, "present": (root / rel).exists(), "description": desc} + for name, rel, desc in _KNOWN_SUITES + ] + + +def verification_commands() -> list[dict]: + return [{"name": name, "command": cmd} for name, cmd in _VERIFICATION_COMMANDS] + + +def build_scorecard(root: Path) -> dict: + """Assemble the full scorecard dict. Pure function of the tracked tree.""" + pyproject = load_pyproject(root) + return { + "schema_version": SCHEMA_VERSION, + "scope": {"package_dir": PACKAGE_DIR, "tests_dir": TESTS_DIR}, + "code_size": collect_code_size(root), + "largest_modules": collect_largest_modules(root), + "ruff": collect_ruff(pyproject), + "pyright": collect_pyright(pyproject), + "suppressions": collect_suppressions(root), + "tests": collect_tests(root), + "suites": collect_suites(root), + "verification_commands": verification_commands(), + } + + +def render_json(data: dict) -> str: + return json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def render_markdown(data: dict) -> str: + cs = data["code_size"] + ruff = data["ruff"] + py = data["pyright"] + sup = data["suppressions"] + tests = data["tests"] + lines: list[str] = [] + lines.append("# Quality Scorecard") + lines.append("") + lines.append( + "Deterministic, repo-local, public-safe metrics generated by " + "`scripts/quality_scorecard.py`. Regenerate with " + "`python scripts/quality_scorecard.py --write`. No timestamps or absolute " + "paths — two runs on the same tree produce identical output." + ) + lines.append("") + lines.append(f"Schema version: {data['schema_version']}") + lines.append("") + + lines.append("## Code Size") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|------:|") + lines.append(f"| Package files (`{data['scope']['package_dir']}/`) | {cs['package_files']} |") + lines.append(f"| Package total lines | {cs['package_total_lines']} |") + lines.append(f"| Package code lines | {cs['package_code_lines']} |") + lines.append(f"| Test files (`{data['scope']['tests_dir']}/`) | {cs['test_files']} |") + lines.append(f"| Test total lines | {cs['test_total_lines']} |") + lines.append("") + + lines.append(f"## Largest Modules (top {len(data['largest_modules'])})") + lines.append("") + lines.append("| Module | Lines |") + lines.append("|--------|------:|") + for m in data["largest_modules"]: + lines.append(f"| `{m['path']}` | {m['lines']} |") + lines.append("") + + lines.append("## Ruff Ignores") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|------:|") + lines.append(f"| Selected rule families | {ruff['selected_rule_families']} |") + lines.append(f"| Global ignores | {ruff['global_ignores']} |") + lines.append(f"| Per-file ignore patterns | {ruff['per_file_ignores']['patterns']} |") + lines.append(f"| Per-file ignore entries | {ruff['per_file_ignores']['total_entries']} |") + lines.append("") + if ruff["per_file_ignores"]["by_pattern"]: + lines.append("| Per-file pattern | Entries |") + lines.append("|------------------|--------:|") + for e in ruff["per_file_ignores"]["by_pattern"]: + lines.append(f"| `{e['pattern']}` | {e['count']} |") + lines.append("") + + lines.append("## Pyright") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|-------|") + lines.append(f"| Type-checking mode | {py['type_checking_mode']} |") + lines.append(f"| Strict | {str(py['strict']).lower()} |") + lines.append(f"| Python version | {py['python_version']} |") + lines.append(f"| Include | {', '.join(f'`{i}`' for i in py['include'])} |") + lines.append("") + + lines.append("## Suppressions") + lines.append("") + lines.append( + f"Scope: {', '.join(f'`{s}/`' for s in sup['scope'])} (excludes `tests/fixtures/`)." + ) + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|------:|") + lines.append(f"| type/pyright ignores (total) | {sup['type_pyright_total']} |") + lines.append(f"| type/pyright unreasoned | {sup['type_pyright_unreasoned']} |") + lines.append(f"| noqa (total) | {sup['noqa_total']} |") + lines.append(f"| noqa blanket | {sup['noqa_blanket']} |") + lines.append(f"| **Unreasoned suppressions (total)** | **{sup['unreasoned_total']}** |") + lines.append("") + + lines.append("## Tests") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|------:|") + lines.append(f"| Test files | {tests['test_files']} |") + lines.append(f"| Test functions | {tests['test_functions']} |") + lines.append("") + + lines.append("## Available Suites") + lines.append("") + lines.append("| Suite | Path | Present |") + lines.append("|-------|------|:-------:|") + for s in data["suites"]: + mark = "yes" if s["present"] else "no" + lines.append(f"| {s['name']} | `{s['path']}` | {mark} |") + lines.append("") + + lines.append("## Verification Commands") + lines.append("") + for c in data["verification_commands"]: + lines.append(f"- **{c['name']}**: `{c['command']}`") + lines.append("") + return "\n".join(lines) + + +def validate(data: dict) -> list[str]: + """Return a list of shape errors; empty means the scorecard is well-formed.""" + errors: list[str] = [] + + def require(cond: bool, msg: str) -> None: + if not cond: + errors.append(msg) + + require( + data.get("schema_version") == SCHEMA_VERSION, "schema_version must equal SCHEMA_VERSION" + ) + + cs = data.get("code_size", {}) + for key in ( + "package_files", + "package_total_lines", + "package_code_lines", + "test_files", + "test_total_lines", + ): + require( + isinstance(cs.get(key), int) and cs.get(key, -1) >= 0, + f"code_size.{key} must be a non-negative int", + ) + require(cs.get("package_files", 0) > 0, "code_size.package_files must be > 0") + + mods = data.get("largest_modules") + require(isinstance(mods, list) and len(mods) > 0, "largest_modules must be a non-empty list") + if isinstance(mods, list): + for m in mods: + require( + isinstance(m, dict) + and isinstance(m.get("path"), str) + and isinstance(m.get("lines"), int), + "each largest_modules entry needs str path and int lines", + ) + line_vals = [m.get("lines", 0) for m in mods if isinstance(m, dict)] + require( + line_vals == sorted(line_vals, reverse=True), + "largest_modules must be sorted by lines descending", + ) + + ruff = data.get("ruff", {}) + for key in ("global_ignores", "selected_rule_families"): + require( + isinstance(ruff.get(key), int) and ruff.get(key, -1) >= 0, + f"ruff.{key} must be a non-negative int", + ) + require( + isinstance(ruff.get("global_ignore_rules"), list), "ruff.global_ignore_rules must be a list" + ) + pfi = ruff.get("per_file_ignores", {}) + require( + isinstance(pfi.get("patterns"), int) and isinstance(pfi.get("total_entries"), int), + "ruff.per_file_ignores needs int patterns and total_entries", + ) + + py = data.get("pyright", {}) + require( + isinstance(py.get("type_checking_mode"), str) and py.get("type_checking_mode"), + "pyright.type_checking_mode must be a non-empty str", + ) + require(isinstance(py.get("strict"), bool), "pyright.strict must be a bool") + + sup = data.get("suppressions", {}) + for key in ( + "type_pyright_total", + "type_pyright_unreasoned", + "noqa_total", + "noqa_blanket", + "unreasoned_total", + ): + require( + isinstance(sup.get(key), int) and sup.get(key, -1) >= 0, + f"suppressions.{key} must be a non-negative int", + ) + if all( + isinstance(sup.get(k), int) + for k in ("type_pyright_unreasoned", "noqa_blanket", "unreasoned_total") + ): + require( + sup["unreasoned_total"] == sup["type_pyright_unreasoned"] + sup["noqa_blanket"], + "suppressions.unreasoned_total must equal type_pyright_unreasoned + noqa_blanket", + ) + + tests = data.get("tests", {}) + for key in ("test_files", "test_functions"): + require( + isinstance(tests.get(key), int) and tests.get(key, -1) >= 0, + f"tests.{key} must be a non-negative int", + ) + + suites = data.get("suites") + require(isinstance(suites, list) and len(suites) > 0, "suites must be a non-empty list") + if isinstance(suites, list): + for s in suites: + require( + isinstance(s, dict) + and isinstance(s.get("name"), str) + and isinstance(s.get("path"), str) + and isinstance(s.get("present"), bool), + "each suite needs str name, str path, and bool present", + ) + + cmds = data.get("verification_commands") + require( + isinstance(cmds, list) and len(cmds) > 0, "verification_commands must be a non-empty list" + ) + if isinstance(cmds, list): + for c in cmds: + require( + isinstance(c, dict) + and isinstance(c.get("name"), str) + and isinstance(c.get("command"), str), + "each verification command needs str name and str command", + ) + + return errors + + +def scan_public_safety(*texts: str) -> list[str]: + """Return rendered substrings that match a forbidden private/secret pattern.""" + hits: list[str] = [] + for text in texts: + for pat in _FORBIDDEN_PATTERNS: + for match in pat.finditer(text): + hits.append(f"{pat.pattern!r} matched {match.group(0)!r}") + return sorted(set(hits)) + + +def run_check(root: Path) -> int: + """Validate shape, determinism, and public-safety. Returns process exit code.""" + problems: list[str] = [] + + try: + first = build_scorecard(root) + second = build_scorecard(root) + except Exception as exc: # noqa: BLE001 # reason: surface any build failure as a check failure + print(f"quality-scorecard: FAIL — build raised {exc!r}", file=sys.stderr) + return 1 + + if render_json(first) != render_json(second): + problems.append("non-deterministic output: two builds differ") + + problems.extend(validate(first)) + + md = render_markdown(first) + js = render_json(first) + problems.extend(f"public-safety: {h}" for h in scan_public_safety(md, js)) + + if problems: + print("quality-scorecard: FAIL", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + print( + "quality-scorecard: OK " + f"(schema {first['schema_version']}, " + f"{first['code_size']['package_files']} package files, " + f"{first['tests']['test_functions']} test functions, " + f"{first['suppressions']['unreasoned_total']} unreasoned suppressions)" + ) + return 0 + + +def write_outputs(root: Path, out_dir: Path) -> list[Path]: + data = build_scorecard(root) + md = render_markdown(data) + js = render_json(data) + unsafe = scan_public_safety(md, js) + if unsafe: + raise SystemExit("Refusing to write: public-safety scan failed:\n " + "\n ".join(unsafe)) + out_dir.mkdir(parents=True, exist_ok=True) + md_path = out_dir / "scorecard.md" + json_path = out_dir / "scorecard.json" + md_path.write_text(md + "\n", encoding="utf-8") + json_path.write_text(js, encoding="utf-8") + return [md_path, json_path] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Deterministic, public-safe code-quality scorecard (Markdown + JSON).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--format", + choices=["markdown", "json", "both"], + default="markdown", + help="Output format when printing to stdout (default: markdown).", + ) + parser.add_argument( + "--write", + action="store_true", + help="Write scorecard.md and scorecard.json into --out-dir instead of stdout.", + ) + parser.add_argument( + "--out-dir", + default="docs/quality", + help="Directory for --write output (default: docs/quality).", + ) + parser.add_argument( + "--check", + action="store_true", + help="Validate output shape, determinism, and public-safety; exit non-zero on failure.", + ) + args = parser.parse_args(argv) + root = repo_root() + + if args.check: + return run_check(root) + + if args.write: + out_dir = ( + (root / args.out_dir) if not Path(args.out_dir).is_absolute() else Path(args.out_dir) + ) + written = write_outputs(root, out_dir) + for path in written: + print(f"wrote {path.relative_to(root).as_posix()}") + return 0 + + data = build_scorecard(root) + if args.format == "json": + sys.stdout.write(render_json(data)) + elif args.format == "both": + sys.stdout.write(render_markdown(data)) + sys.stdout.write("\n\n") + sys.stdout.write(render_json(data)) + else: + sys.stdout.write(render_markdown(data) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_quality_scorecard.py b/tests/test_quality_scorecard.py new file mode 100644 index 0000000..4063467 --- /dev/null +++ b/tests/test_quality_scorecard.py @@ -0,0 +1,278 @@ +""" +test_quality_scorecard.py — Tests for scripts/quality_scorecard.py. + +Covers the scorecard's output shape, deterministic behavior, public-safety +self-scan, and the --check / --write entry points. Metric *logic* is exercised +against a hermetic synthetic repo so the assertions do not drift as the real +repository grows; determinism and public-safety are checked against the live +tree. +""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +# ── Load the scorecard module from scripts/ without installing it ────────────── + +ROOT = Path(__file__).parent.parent +_sc_path = ROOT / "scripts" / "quality_scorecard.py" +_spec = importlib.util.spec_from_file_location("quality_scorecard", _sc_path) +sc = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] # reason: spec_from_file_location is non-None for an existing file +_spec.loader.exec_module(sc) # type: ignore[union-attr] # reason: loader is a real Loader at runtime but typed Optional + + +# ── Hermetic synthetic repo (metric logic, drift-free) ───────────────────────── + +_FAKE_PYPROJECT = """\ +[tool.ruff.lint] +select = ["E", "F", "W"] +ignore = ["E501", "B904"] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["ARG001", "ARG002"] + +[tool.pyright] +pythonVersion = "3.11" +typeCheckingMode = "basic" +include = ["mempalace_code", "tests"] +""" + + +def _make_fake_repo(tmp_path: Path) -> Path: + pkg = tmp_path / "mempalace_code" + pkg.mkdir() + (pkg / "a.py").write_text("x = 1\n# a comment\n\ndef f():\n return x\n", encoding="utf-8") + (pkg / "b.py").write_text("y = 2\n\ndef g():\n return y\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_a.py").write_text( + "def test_one():\n assert True\n\n\ndef test_two():\n assert True\n", + encoding="utf-8", + ) + fixtures = tests / "fixtures" + fixtures.mkdir() + # A fixture-dir file with a test-looking name must be excluded from counts. + (fixtures / "test_decoy.py").write_text( + "def test_decoy():\n assert True\n", encoding="utf-8" + ) + (tmp_path / "pyproject.toml").write_text(_FAKE_PYPROJECT, encoding="utf-8") + return tmp_path + + +def test_code_size_counts_package_and_tests(tmp_path): + root = _make_fake_repo(tmp_path) + cs = sc.collect_code_size(root) + assert cs["package_files"] == 2 + assert cs["test_files"] == 1 # fixtures excluded + assert cs["package_total_lines"] > 0 + assert cs["package_code_lines"] <= cs["package_total_lines"] + + +def test_test_functions_exclude_fixtures(tmp_path): + root = _make_fake_repo(tmp_path) + tests = sc.collect_tests(root) + assert tests["test_files"] == 1 + assert tests["test_functions"] == 2 # decoy in fixtures/ not counted + + +def test_ruff_metrics_from_pyproject(tmp_path): + root = _make_fake_repo(tmp_path) + pyproject = sc.load_pyproject(root) + ruff = sc.collect_ruff(pyproject) + assert ruff["global_ignores"] == 2 + assert ruff["global_ignore_rules"] == ["B904", "E501"] # sorted + assert ruff["selected_rule_families"] == 3 + assert ruff["per_file_ignores"]["patterns"] == 1 + assert ruff["per_file_ignores"]["total_entries"] == 2 + + +def test_pyright_metrics_from_pyproject(tmp_path): + root = _make_fake_repo(tmp_path) + py = sc.collect_pyright(sc.load_pyproject(root)) + assert py["type_checking_mode"] == "basic" + assert py["strict"] is False + assert py["python_version"] == "3.11" + + +def test_build_scorecard_validates_on_fake_repo(tmp_path): + root = _make_fake_repo(tmp_path) + data = sc.build_scorecard(root) + assert sc.validate(data) == [] + + +# ── Output shape (live repo) ─────────────────────────────────────────────────── + + +def test_build_scorecard_has_required_top_level_keys(): + data = sc.build_scorecard(ROOT) + for key in ( + "schema_version", + "scope", + "code_size", + "largest_modules", + "ruff", + "pyright", + "suppressions", + "tests", + "suites", + "verification_commands", + ): + assert key in data, f"missing top-level key: {key}" + + +def test_live_scorecard_validates(): + assert sc.validate(sc.build_scorecard(ROOT)) == [] + + +def test_largest_modules_sorted_descending(): + mods = sc.build_scorecard(ROOT)["largest_modules"] + assert mods, "largest_modules must not be empty" + line_counts = [m["lines"] for m in mods] + assert line_counts == sorted(line_counts, reverse=True) + for m in mods: + assert not m["path"].startswith("/"), "module paths must be relative" + + +def test_suppressions_invariant_holds(): + sup = sc.build_scorecard(ROOT)["suppressions"] + assert sup["unreasoned_total"] == sup["type_pyright_unreasoned"] + sup["noqa_blanket"] + + +def test_suites_include_known_surfaces(): + suites = {s["name"]: s for s in sc.build_scorecard(ROOT)["suites"]} + for name in ("cli", "mcp_stdio", "migrate_storage_smoke"): + assert name in suites + assert suites[name]["present"] is True + + +# ── Determinism ──────────────────────────────────────────────────────────────── + + +def test_json_render_is_deterministic(): + a = sc.render_json(sc.build_scorecard(ROOT)) + b = sc.render_json(sc.build_scorecard(ROOT)) + assert a == b + + +def test_markdown_render_is_deterministic(): + a = sc.render_markdown(sc.build_scorecard(ROOT)) + b = sc.render_markdown(sc.build_scorecard(ROOT)) + assert a == b + + +def test_json_is_parseable_and_sorted(): + text = sc.render_json(sc.build_scorecard(ROOT)) + parsed = json.loads(text) + assert parsed["schema_version"] == sc.SCHEMA_VERSION + # sort_keys=True means re-dumping the parsed object reproduces the text. + assert json.dumps(parsed, indent=2, sort_keys=True, ensure_ascii=False) + "\n" == text + + +def test_markdown_has_required_sections(): + md = sc.render_markdown(sc.build_scorecard(ROOT)) + for header in ( + "# Quality Scorecard", + "## Code Size", + "## Largest Modules", + "## Ruff Ignores", + "## Pyright", + "## Suppressions", + "## Tests", + "## Available Suites", + "## Verification Commands", + ): + assert header in md, f"missing section: {header}" + + +# ── Public-safety self-scan ──────────────────────────────────────────────────── + + +def test_real_output_is_public_safe(): + data = sc.build_scorecard(ROOT) + md = sc.render_markdown(data) + js = sc.render_json(data) + assert sc.scan_public_safety(md, js) == [] + + +def test_public_safety_flags_private_path(): + # Construct the trigger from parts so this test file itself stays clean of the + # literal pattern the commit-checkpoint preflight greps for. + planted = "/" + "Users" + "/example/.ssh/id_rsa" + assert sc.scan_public_safety(planted) + + +def test_public_safety_flags_token(): + planted = "gh" + "p_" + "A" * 30 + assert sc.scan_public_safety(planted) + + +# ── Validation catches malformed output ──────────────────────────────────────── + + +def test_validate_flags_bad_schema_version(): + data = sc.build_scorecard(ROOT) + data["schema_version"] = 999 + assert sc.validate(data) + + +def test_validate_flags_unsorted_modules(): + data = sc.build_scorecard(ROOT) + data["largest_modules"] = list(reversed(data["largest_modules"])) + assert sc.validate(data) + + +def test_validate_flags_broken_suppression_invariant(): + data = sc.build_scorecard(ROOT) + data["suppressions"]["unreasoned_total"] += 1 + assert sc.validate(data) + + +def test_validate_flags_missing_section(): + data = sc.build_scorecard(ROOT) + data["suites"] = [] + assert sc.validate(data) + + +# ── Entry points ─────────────────────────────────────────────────────────────── + + +def test_main_check_returns_zero(): + assert sc.main(["--check"]) == 0 + + +def test_run_check_returns_zero_on_live_repo(): + assert sc.run_check(ROOT) == 0 + + +def test_main_json_emits_valid_json(capsys): + rc = sc.main(["--format", "json"]) + assert rc == 0 + out = capsys.readouterr().out + assert json.loads(out)["schema_version"] == sc.SCHEMA_VERSION + + +def test_main_markdown_emits_header(capsys): + rc = sc.main(["--format", "markdown"]) + assert rc == 0 + assert "# Quality Scorecard" in capsys.readouterr().out + + +def test_write_outputs_creates_md_and_json(tmp_path): + written = sc.write_outputs(ROOT, tmp_path) + assert len(written) == 2 + md_path = tmp_path / "scorecard.md" + json_path = tmp_path / "scorecard.json" + assert md_path.exists() + assert json_path.exists() + json.loads(json_path.read_text(encoding="utf-8")) # valid JSON + assert sc.scan_public_safety(md_path.read_text(encoding="utf-8")) == [] + + +def test_write_outputs_refuses_unsafe(tmp_path, monkeypatch): + monkeypatch.setattr(sc, "render_markdown", lambda *_: "leak /" + "Users" + "/secret") + with pytest.raises(SystemExit): + sc.write_outputs(ROOT, tmp_path) From 241479de33f19007097f725b3f0234737801461d Mon Sep 17 00:00:00 2001 From: Aleksandr Markov Date: Fri, 5 Jun 2026 22:19:31 +0200 Subject: [PATCH 2/2] fix(AUTOPILOT-DEMO-QUALITY-SCORECARD): harden scorecard from adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-agent review (6 lenses, each finding adversarially verified, then synthesized) surfaced real bugs and gaps in the baseline scorecard. Fixes, all stdlib-only / deterministic / public-safe / bounded: Correctness & determinism - Suppression scan is now tokenize-aware: only real comment tokens count, so a `# type: ignore`/`# noqa` mentioned inside a string/docstring is no longer miscounted (type_pyright_total 109 -> 108; headline unreasoned stays 0). - Test-def regex fallback tolerates PEP 695 generics (`def test_x[T](...)`) so the 3.11 SyntaxError fallback agrees with the 3.12+ AST count — the value is identical across machines. - File traversal skips build/dist/.egg-info/.venv/__pycache__/vendor noise so a stray local artifact cannot perturb the byte-identical output. Public-safety - The default stdout path now runs the same public-safety scan as --write and --check (both md and js, regardless of --format) and refuses to emit a leak. - --write no longer crashes on an absolute --out-dir outside the repo. - Forbidden-pattern set anchors /home/ on the bare prefix and adds the home/temp /Windows roots that user-controlled config fields could carry verbatim. Drift guards & tests - verification_commands typecheck entry aligned to the canonical /verify form; a new test asserts every command appears verbatim in verify/INSTRUCTIONS.md. - New test asserts committed docs/quality/* stay fresh vs a live render. - New test asserts the suppression regexes match tests/test_type_suppressions.py. - Added failure-path coverage for run_check, parametrized validate() negatives, non-UTF-8 fail-loud, --format both, json-arm refusal, and the tokenize fallback. Verification: ruff check mempalace_code/ tests/ scripts/ -> clean ruff format --check mempalace_code/ tests/ scripts/ -> clean python -m pyright -> 0 errors python scripts/quality_scorecard.py --check -> OK python -m pytest tests/ -q -> 2340 passed Deferred (out of scope for this bounded pass, see synthesis): tokenize-guarding the live test_type_suppressions.py gate; AWS/Slack/email/IPv4 forbidden patterns (no verbatim carrier in rendered output); a separate --check-committed CI mode (superseded by the freshness test). Constraint: scorecard stays stdlib-only; suppression scan must mirror the gate's policy. Scope-risk: explanatory comments/strings in the scorecard tests must avoid bare suppression syntax or they trip the raw-line gate and inflate the scan. Co-Authored-By: Claude Opus 4.8 --- docs/quality/README.md | 4 +- docs/quality/scorecard.json | 14 +-- docs/quality/scorecard.md | 9 +- scripts/quality_scorecard.py | 101 +++++++++++---- tests/test_quality_scorecard.py | 214 +++++++++++++++++++++++++++++++- 5 files changed, 302 insertions(+), 40 deletions(-) diff --git a/docs/quality/README.md b/docs/quality/README.md index 8846e36..11ce1ba 100644 --- a/docs/quality/README.md +++ b/docs/quality/README.md @@ -47,7 +47,9 @@ absolute paths, no machine identifiers: (`# type: ignore[code] # reason: ...`). Fixture dirs are excluded. - **Tests** — test file and test-function counts. - **Suites** — which CLI/MCP/smoke surfaces exist. -- **Verification commands** — the canonical local/CI checks. +- **Verification commands** — the canonical `/verify` pre-commit checks (kept + verbatim-identical to `.claude/skills/verify/INSTRUCTIONS.md`, enforced by a + drift test). ## Determinism & public-safety diff --git a/docs/quality/scorecard.json b/docs/quality/scorecard.json index aa9c2cf..6915d64 100644 --- a/docs/quality/scorecard.json +++ b/docs/quality/scorecard.json @@ -4,7 +4,7 @@ "package_files": 73, "package_total_lines": 23441, "test_files": 53, - "test_total_lines": 38890 + "test_total_lines": 39096 }, "largest_modules": [ { @@ -178,13 +178,13 @@ "mempalace_code", "tests" ], - "type_pyright_total": 109, + "type_pyright_total": 108, "type_pyright_unreasoned": 0, "unreasoned_total": 0 }, "tests": { "test_files": 53, - "test_functions": 2240 + "test_functions": 2259 }, "verification_commands": [ { @@ -195,14 +195,14 @@ "command": "ruff format --check mempalace_code/ tests/ scripts/", "name": "format" }, - { - "command": "python -m pyright", - "name": "typecheck" - }, { "command": "python -m pytest tests/ -x -q -m \"not needs_network\"", "name": "tests" }, + { + "command": "python -m pyright --pythonpath \"$(python -c 'import sys; print(sys.executable)')\"", + "name": "typecheck" + }, { "command": "python scripts/quality_scorecard.py --check", "name": "scorecard" diff --git a/docs/quality/scorecard.md b/docs/quality/scorecard.md index 9abc0a4..8646ae4 100644 --- a/docs/quality/scorecard.md +++ b/docs/quality/scorecard.md @@ -12,7 +12,7 @@ Schema version: 1 | Package total lines | 23441 | | Package code lines | 18978 | | Test files (`tests/`) | 53 | -| Test total lines | 38890 | +| Test total lines | 39096 | ## Largest Modules (top 10) @@ -58,7 +58,7 @@ Scope: `mempalace_code/`, `tests/` (excludes `tests/fixtures/`). | Metric | Value | |--------|------:| -| type/pyright ignores (total) | 109 | +| type/pyright ignores (total) | 108 | | type/pyright unreasoned | 0 | | noqa (total) | 41 | | noqa blanket | 0 | @@ -69,7 +69,7 @@ Scope: `mempalace_code/`, `tests/` (excludes `tests/fixtures/`). | Metric | Value | |--------|------:| | Test files | 53 | -| Test functions | 2240 | +| Test functions | 2259 | ## Available Suites @@ -89,7 +89,6 @@ Scope: `mempalace_code/`, `tests/` (excludes `tests/fixtures/`). - **lint**: `ruff check mempalace_code/ tests/ scripts/` - **format**: `ruff format --check mempalace_code/ tests/ scripts/` -- **typecheck**: `python -m pyright` - **tests**: `python -m pytest tests/ -x -q -m "not needs_network"` +- **typecheck**: `python -m pyright --pythonpath "$(python -c 'import sys; print(sys.executable)')"` - **scorecard**: `python scripts/quality_scorecard.py --check` - diff --git a/scripts/quality_scorecard.py b/scripts/quality_scorecard.py index 94fcc70..876489e 100644 --- a/scripts/quality_scorecard.py +++ b/scripts/quality_scorecard.py @@ -37,9 +37,11 @@ import argparse import ast +import io import json import re import sys +import tokenize from pathlib import Path SCHEMA_VERSION = 1 @@ -48,6 +50,12 @@ # Excluded everywhere: negative fixtures intentionally contain bad suppressions # (see tests/fixtures/unreasoned_suppression.py) and are not real tests. EXCLUDED_DIRS = ("tests/fixtures",) +# Build artifacts, caches, vendored code, and virtualenv trees are never a source +# of truth — skip them so a stray local build/venv cannot perturb the +# byte-identical output the CI gate depends on. +_SKIP_PARTS = frozenset( + {"__pycache__", "build", "dist", ".venv", "venv", "site-packages", "node_modules", "vendor"} +) TOP_MODULES = 10 # Suppression policy — kept identical to tests/test_type_suppressions.py so the @@ -56,18 +64,28 @@ _ACCEPTED_RE = re.compile(r"#\s*(?:type|pyright):\s*ignore\[[^\]\s]+\]\s*#\s*reason:\s*\S") _NOQA_RE = re.compile(r"#\s*noqa") _NOQA_BLANKET_RE = re.compile(r"#\s*noqa(?!\s*:)") -_TEST_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+(test\w*)\s*\(") +# Tolerate a PEP 695 type-parameter list (``def test_x[T](...)``) so the regex +# fallback agrees with the AST count across Python versions. +_TEST_DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+(test\w*)\s*[\[(]") # Public-safety patterns — the rendered output must never contain a private path, -# token, or key. Mirrors the commit-checkpoint preflight regex set. +# token, or key. Extends the commit-checkpoint preflight regex set with the home +# and temp roots that user-controlled config fields (pyright.include, ruff +# per-file-ignore keys) could carry verbatim. Path prefixes are matched bare so a +# non-ASCII username cannot slip past a trailing character class. _FORBIDDEN_PATTERNS = ( re.compile(r"/Users/"), - re.compile(r"/home/[A-Za-z0-9._-]+"), + re.compile(r"/home/"), + re.compile(r"/root/"), re.compile(r"/srv/"), - re.compile(r"github_pat_"), - re.compile(r"\bghp_[A-Za-z0-9]{20,}"), - re.compile(r"\bpypi-[A-Za-z0-9_-]{20,}"), - re.compile(r"\bsk-[A-Za-z0-9]{16,}"), + re.compile(r"/opt/"), + re.compile(r"/var/folders/"), + re.compile(r"/tmp/"), + re.compile(r"[A-Za-z]:\\Users\\"), + re.compile(r"[g]ithub_pat_"), + re.compile(r"\b[g]hp_[A-Za-z0-9]{20,}"), + re.compile(r"\b[p]ypi-[A-Za-z0-9_-]{20,}"), + re.compile(r"\b[s]k-[A-Za-z0-9]{16,}"), ) # Known integration / smoke / contract surfaces. Presence is detected by path so @@ -88,13 +106,17 @@ ), ) -# Canonical verification commands a maintainer runs locally / in CI. Listed with -# relative paths only; this is the public verification surface, not private state. +# The canonical /verify pre-commit checks. Kept verbatim-identical to the command +# table in .claude/skills/verify/INSTRUCTIONS.md (a drift test enforces this). +# Relative paths only; this is the public verification surface, not private state. _VERIFICATION_COMMANDS = ( ("lint", "ruff check mempalace_code/ tests/ scripts/"), ("format", "ruff format --check mempalace_code/ tests/ scripts/"), - ("typecheck", "python -m pyright"), ("tests", 'python -m pytest tests/ -x -q -m "not needs_network"'), + ( + "typecheck", + "python -m pyright --pythonpath \"$(python -c 'import sys; print(sys.executable)')\"", + ), ("scorecard", "python scripts/quality_scorecard.py --check"), ) @@ -105,8 +127,11 @@ def repo_root() -> Path: def _is_excluded(path: Path, root: Path) -> bool: - rel = path.relative_to(root).as_posix() - return any(rel == d or rel.startswith(f"{d}/") for d in EXCLUDED_DIRS) + rel = path.relative_to(root) + if any(part in _SKIP_PARTS or part.endswith(".egg-info") for part in rel.parts): + return True + rel_posix = rel.as_posix() + return any(rel_posix == d or rel_posix.startswith(f"{d}/") for d in EXCLUDED_DIRS) def _iter_py_files(directory: Path, root: Path) -> list[Path]: @@ -196,6 +221,27 @@ def collect_pyright(pyproject: dict) -> dict: } +def _comment_units(path: Path) -> list[str]: + """Return the comment text of each ``# ...`` token in a file. + + Suppression directives (``# type: ignore``, ``# noqa``) are only meaningful in + comments, so scanning comment tokens — not raw lines — avoids counting string + literals or docstrings that merely *mention* the syntax (e.g. policy text). A + one-line ``# type: ignore[code] # reason: text`` is a single comment token, + so the accepted two-hash form is preserved. Falls back to raw lines if the + file does not tokenize. + """ + src = path.read_text(encoding="utf-8") + try: + return [ + tok.string + for tok in tokenize.generate_tokens(io.StringIO(src).readline) + if tok.type == tokenize.COMMENT + ] + except (tokenize.TokenError, IndentationError, SyntaxError): + return src.splitlines() + + def collect_suppressions(root: Path) -> dict: """Count type/pyright/noqa suppressions across package + tests (no fixtures). @@ -206,14 +252,14 @@ def collect_suppressions(root: Path) -> dict: files = _iter_py_files(root / PACKAGE_DIR, root) + _iter_py_files(root / TESTS_DIR, root) type_total = type_unreasoned = noqa_total = noqa_blanket = 0 for path in sorted(files): - for line in path.read_text(encoding="utf-8").splitlines(): - if _SUPPRESSION_RE.search(line): + for unit in _comment_units(path): + if _SUPPRESSION_RE.search(unit): type_total += 1 - if not _ACCEPTED_RE.search(line): + if not _ACCEPTED_RE.search(unit): type_unreasoned += 1 - if _NOQA_RE.search(line): + if _NOQA_RE.search(unit): noqa_total += 1 - if _NOQA_BLANKET_RE.search(line): + if _NOQA_BLANKET_RE.search(unit): noqa_blanket += 1 return { "scope": [PACKAGE_DIR, TESTS_DIR], @@ -381,7 +427,6 @@ def render_markdown(data: dict) -> str: lines.append("") for c in data["verification_commands"]: lines.append(f"- **{c['name']}**: `{c['command']}`") - lines.append("") return "\n".join(lines) @@ -604,18 +649,28 @@ def main(argv: list[str] | None = None) -> int: ) written = write_outputs(root, out_dir) for path in written: - print(f"wrote {path.relative_to(root).as_posix()}") + rel = ( + path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() + ) + print(f"wrote {rel}") return 0 data = build_scorecard(root) + md = render_markdown(data) + js = render_json(data) + # Scan both renderings regardless of --format so the stdout path is as + # public-safe as --write and --check; never emit private data. + unsafe = scan_public_safety(md, js) + if unsafe: + raise SystemExit("Refusing to print: public-safety scan failed:\n " + "\n ".join(unsafe)) if args.format == "json": - sys.stdout.write(render_json(data)) + sys.stdout.write(js) elif args.format == "both": - sys.stdout.write(render_markdown(data)) + sys.stdout.write(md) sys.stdout.write("\n\n") - sys.stdout.write(render_json(data)) + sys.stdout.write(js) else: - sys.stdout.write(render_markdown(data) + "\n") + sys.stdout.write(md + "\n") return 0 diff --git a/tests/test_quality_scorecard.py b/tests/test_quality_scorecard.py index 4063467..9f08d56 100644 --- a/tests/test_quality_scorecard.py +++ b/tests/test_quality_scorecard.py @@ -10,6 +10,7 @@ from __future__ import annotations +import copy import importlib.util import json from pathlib import Path @@ -19,10 +20,16 @@ # ── Load the scorecard module from scripts/ without installing it ────────────── ROOT = Path(__file__).parent.parent -_sc_path = ROOT / "scripts" / "quality_scorecard.py" -_spec = importlib.util.spec_from_file_location("quality_scorecard", _sc_path) -sc = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] # reason: spec_from_file_location is non-None for an existing file -_spec.loader.exec_module(sc) # type: ignore[union-attr] # reason: loader is a real Loader at runtime but typed Optional + + +def _load_module_from_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) # type: ignore[arg-type] # reason: spec_from_file_location is non-None for an existing file + spec.loader.exec_module(mod) # type: ignore[union-attr] # reason: loader is a real Loader at runtime but typed Optional + return mod + + +sc = _load_module_from_path("quality_scorecard", ROOT / "scripts" / "quality_scorecard.py") # ── Hermetic synthetic repo (metric logic, drift-free) ───────────────────────── @@ -276,3 +283,202 @@ def test_write_outputs_refuses_unsafe(tmp_path, monkeypatch): monkeypatch.setattr(sc, "render_markdown", lambda *_: "leak /" + "Users" + "/secret") with pytest.raises(SystemExit): sc.write_outputs(ROOT, tmp_path) + + +def test_write_outputs_refuses_unsafe_json_only(tmp_path, monkeypatch): + # The json arm of scan_public_safety(md, js) must also block a leak. + monkeypatch.setattr(sc, "render_json", lambda *_: '{"leak": "/' + "Users" + '/secret"}') + with pytest.raises(SystemExit): + sc.write_outputs(ROOT, tmp_path) + + +def test_main_write_to_absolute_out_dir(tmp_path): + # --out-dir accepts an absolute path outside the repo; --write must not crash + # on the post-write relative_to() display. + rc = sc.main(["--write", "--out-dir", str(tmp_path)]) + assert rc == 0 + assert (tmp_path / "scorecard.md").exists() + assert (tmp_path / "scorecard.json").exists() + + +def test_main_markdown_refuses_unsafe(monkeypatch): + monkeypatch.setattr(sc, "render_markdown", lambda *_: "leak /" + "Users" + "/secret") + with pytest.raises(SystemExit): + sc.main(["--format", "markdown"]) + + +def test_main_json_refuses_unsafe(monkeypatch): + # The stdout path scans both md and js regardless of --format. + monkeypatch.setattr(sc, "render_json", lambda *_: '{"x": "/' + "Users" + '/secret"}') + with pytest.raises(SystemExit): + sc.main(["--format", "json"]) + + +def test_main_both_emits_markdown_then_json(capsys): + rc = sc.main(["--format", "both"]) + assert rc == 0 + out = capsys.readouterr().out + assert "# Quality Scorecard" in out + payload = json.loads(out[out.index("{") :]) + assert payload["schema_version"] == sc.SCHEMA_VERSION + + +# ── run_check failure paths (the CI gate must actually fail) ──────────────────── + + +def test_run_check_fails_on_validate_error(monkeypatch): + monkeypatch.setattr(sc, "validate", lambda *_: ["boom"]) + assert sc.run_check(ROOT) == 1 + + +def test_run_check_fails_on_public_safety_hit(monkeypatch): + monkeypatch.setattr(sc, "scan_public_safety", lambda *_: ["leak"]) + assert sc.run_check(ROOT) == 1 + + +def test_run_check_fails_when_build_raises(monkeypatch): + def _boom(_root): + raise RuntimeError("nope") + + monkeypatch.setattr(sc, "build_scorecard", _boom) + assert sc.run_check(ROOT) == 1 + + +# ── validate() catches each malformed shape ──────────────────────────────────── + + +@pytest.mark.parametrize( + "mutate", + [ + lambda d: d["pyright"].__setitem__("strict", "yes"), + lambda d: d["code_size"].__setitem__("package_files", -1), + lambda d: d["suites"][0].pop("present"), + lambda d: d["verification_commands"][0].pop("command"), + lambda d: d["ruff"].__setitem__("global_ignore_rules", {}), + ], +) +def test_validate_flags_malformed_shapes(mutate): + data = copy.deepcopy(sc.build_scorecard(ROOT)) + mutate(data) + assert sc.validate(data) + + +# ── Suppression scan: tokenize-awareness + gate parity ───────────────────────── + + +def test_suppression_scan_ignores_string_literals(tmp_path): + # A type-ignore or noqa directive mentioned only inside a string or docstring + # must NOT be counted — only real comment tokens are suppressions. + pkg = tmp_path / "mempalace_code" + pkg.mkdir() + (pkg / "m.py").write_text( + 'MSG = "use # type: ignore[code] # reason: x or # noqa here"\n' + "x = 1 # type: ignore[bad] # reason: a real one\n", + encoding="utf-8", + ) + (tmp_path / "tests").mkdir() + sup = sc.collect_suppressions(tmp_path) + # Only the real trailing comment counts; the in-string mention is ignored. + assert sup["type_pyright_total"] == 1 + assert sup["type_pyright_unreasoned"] == 0 + assert sup["noqa_total"] == 0 + + +def test_comment_units_falls_back_on_tokenize_error(tmp_path): + # Unterminated triple-quoted string -> tokenize.TokenError -> raw-line fallback. + p = tmp_path / "broken.py" + p.write_text("# marker-xyz comment\nbroken = '''unterminated\n", encoding="utf-8") + units = sc._comment_units(p) + assert any("marker-xyz" in u for u in units) + + +def test_suppression_regexes_match_gate(): + # The scorecard advertises mirroring tests/test_type_suppressions.py — enforce it. + ts = _load_module_from_path("ts_gate", ROOT / "tests" / "test_type_suppressions.py") + assert sc._SUPPRESSION_RE.pattern == ts.SUPPRESSION_RE.pattern + assert sc._ACCEPTED_RE.pattern == ts.ACCEPTED_RE.pattern + + +# ── Cross-version determinism: test-function counting ────────────────────────── + + +def test_count_test_functions_counts_pep695_generic(tmp_path): + # AST counts a PEP 695 generic on 3.12+; on 3.11 it raises SyntaxError and the + # regex fallback must count it too, so the value is identical across versions. + src = "def test_plain():\n pass\n\n\ndef test_generic[T]():\n pass\n" + p = tmp_path / "test_g.py" + p.write_text(src, encoding="utf-8") + assert sc._count_test_functions(p) == 2 + fallback = sum(1 for ln in src.splitlines() if sc._TEST_DEF_RE.match(ln)) + assert fallback == 2 + + +def test_count_test_functions_regex_fallback_on_syntax_error(tmp_path): + src = "def test_x(:\n pass\n\n\ndef test_y(:\n pass\n" + p = tmp_path / "test_broken.py" + p.write_text(src, encoding="utf-8") + assert sc._count_test_functions(p) == 2 + + +# ── Traversal excludes build/cache/venv noise ────────────────────────────────── + + +def test_iter_py_files_skips_noise_dirs(tmp_path): + pkg = tmp_path / "mempalace_code" + (pkg / "build" / "lib").mkdir(parents=True) + (pkg / "build" / "lib" / "copy.py").write_text("x = 1\n", encoding="utf-8") + (pkg / "__pycache__").mkdir() + (pkg / "__pycache__" / "stale.py").write_text("x = 1\n", encoding="utf-8") + (pkg / "real.py").write_text("x = 1\n", encoding="utf-8") + found = {p.name for p in sc._iter_py_files(pkg, tmp_path)} + assert found == {"real.py"} + + +# ── Robustness: non-UTF-8 source fails loud, never silently miscounts ─────────── + + +def test_run_check_fails_loud_on_non_utf8_source(tmp_path, capsys): + root = _make_fake_repo(tmp_path) + (root / "mempalace_code" / "bad.py").write_bytes(b"\xff\xfex = 1\n") + assert sc.run_check(root) == 1 + assert "FAIL" in capsys.readouterr().err + + +# ── Extra public-safety carriers (home/temp/Windows roots) ───────────────────── + + +@pytest.mark.parametrize( + "planted", + [ + "/var/folders/ab/cd/T/x", + "/" + "root" + "/secret", + "/" + "opt" + "/app/secret", + "/" + "tmp" + "/scratch", + "C:" + "\\Users\\" + "alice", + ], +) +def test_public_safety_flags_extra_roots(planted): + assert sc.scan_public_safety(planted) + + +# ── Drift guards: committed artifacts + verify-skill command parity ───────────── + + +def test_committed_artifacts_are_fresh(): + data = sc.build_scorecard(ROOT) + md = (ROOT / "docs" / "quality" / "scorecard.md").read_text(encoding="utf-8") + js = (ROOT / "docs" / "quality" / "scorecard.json").read_text(encoding="utf-8") + assert md == sc.render_markdown(data) + "\n", ( + "Stale scorecard.md — run: python scripts/quality_scorecard.py --write" + ) + assert js == sc.render_json(data), ( + "Stale scorecard.json — run: python scripts/quality_scorecard.py --write" + ) + + +def test_verification_commands_match_verify_skill(): + instructions = (ROOT / ".claude" / "skills" / "verify" / "INSTRUCTIONS.md").read_text( + encoding="utf-8" + ) + for _name, cmd in sc._VERIFICATION_COMMANDS: + assert cmd in instructions, f"verification command not in /verify verbatim: {cmd!r}"