Parity apparatus efficiency: shard R cache, close automation loop, de…

.gitignore

@@ -16,7 +16,7 @@ __pycache__/
 
 # R cache lock and fresh-regeneration backup
 tests/_r_cache.lock
-tests/_r_cache.json.bak
+tests/_r_cache.bak/
 
 # Vendored R NNS local-install build artifacts
 tools/NNS/src/*.o

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+# Local hooks run `python -m ...` with language: system so they use the
+# project environment's interpreter — the same one that has numpy and the
+# package installed. A mypy/ruff binary from a different environment produces
+# spurious import-not-found noise (numpy stubs missing) that masks real errors
+# CI would catch; running through the project env keeps local == CI.
+repos:
+  - repo: local
+    hooks:
+      - id: ruff
+        name: ruff check (project env)
+        entry: python -m ruff check --force-exclude
+        language: system
+        types_or: [python, pyi]
+        require_serial: true
+      - id: mypy
+        name: mypy (project env)
+        entry: python -m mypy
+        language: system
+        types_or: [python, pyi]
+        pass_filenames: false
+        require_serial: true

docs/install.md

@@ -50,6 +50,15 @@ uv run ruff check .
 uv run mypy
 ```
 
+Optionally install the pre-commit hooks, which run `ruff` and `mypy` through
+the project environment's interpreter so local results match CI (a `mypy`
+binary from another environment lacks the project's dependencies and reports
+spurious import errors):
+
+```bash
+uv run pre-commit install
+```
+
 The default parity suite is cache-backed and does not require `Rscript`.
 `Rscript` and the R `NNS` package are needed only when regenerating parity caches
 or running live R comparison scripts.

docs/parity.md

@@ -9,8 +9,9 @@ and remains the native C++ foundation for accelerated partial-moment routines.
 Full package parity is **not** claimed. Parity is bounded by the committed tests
 and cache:
 
-- `tests/_r_cache.json` — cache-only R result fixtures (2,406 keyed entries,
-  schema version `1`, `nns_version == "13.0"`),
+- `tests/_r_cache/` — cache-only R result fixtures, sharded into one JSON
+  file per R function label (keys are `<label>|<sha256>`, schema version
+  `2`, every shard stamped with the generating `nns_version`),
 - `tests/parity/` — public behavior parity checks,
 - `tests/invariants/` — Python-native contracts and invariants, and
 - `tests/fixtures/original_tests_expected.json` — adopted original R tests
@@ -85,9 +86,10 @@ If full regeneration is slow or unstable, regenerate deterministic chunks one
 file at a time, for example
 `python scripts/regenerate_r_cache.py -- -n 0 tests/parity/test_core.py`, then
 continue through the remaining parity files. The committed result must remain a
-single valid `tests/_r_cache.json` with `nns_version == "13.0"`,
-`schema_version == 1`, and non-empty `entries`; `scripts/regenerate_r_cache.py`
-enforces those guardrails after the pytest run.
+valid `tests/_r_cache/` shard directory: every shard carries the same
+`nns_version`, `schema_version == 2`, a `label` matching its filename, and
+non-empty `entries`; `scripts/regenerate_r_cache.py` enforces those
+guardrails after the pytest run.
 
 Validate the regenerated cache offline:
 
@@ -130,7 +132,7 @@ NNS R API change
 ### Fix rules (applied by the maintainer; later by the agent)
 
 - Edit **`src/nns/**` only**. Never edit `extern/NNS-core/**`, `tools/NNS/**`,
-  or `tests/_r_cache.json` to make a check pass.
+  or `tests/_r_cache/**` to make a check pass.
 - Classify the root cause and act accordingly:
   - **Python port bug** → fix in `src/nns/**`.
   - **R changed behavior** → do not chase a cache value; cache regeneration is a

docs/plot_parity_policy.md

@@ -36,7 +36,7 @@ colors and which element they sit on*, never rendered images.
 
 - Numeric return values (scalars, vectors, matrices, nested result dicts) from
   every ported function, against committed R fixtures and the committed R cache
-  (`tests/_r_cache.json`).
+  (`tests/_r_cache/`).
 - Structural contracts (result keys, shapes, dtypes, finiteness) via the
   invariant suite.
 

pyproject.toml

@@ -42,6 +42,7 @@ dependencies = [
 dev = [
     "hypothesis",
     "mypy",
+    "pre-commit",
     "pytest",
     "pytest-benchmark",
     "pytest-cov",

scripts/regenerate_r_cache.py

@@ -29,10 +29,10 @@
 from typing import Any
 
 _REPO_ROOT = Path(__file__).resolve().parents[1]
-_CACHE_PATH = _REPO_ROOT / "tests" / "_r_cache.json"
-_CACHE_BACKUP_PATH = _CACHE_PATH.with_suffix(".json.bak")
+_CACHE_DIR = _REPO_ROOT / "tests" / "_r_cache"
+_CACHE_BACKUP_DIR = _REPO_ROOT / "tests" / "_r_cache.bak"
 _R_HELPER_PATH = _REPO_ROOT / "tests" / "_r.py"
-_SCHEMA_VERSION = 1
+_SCHEMA_VERSION = 2
 _VERSION_ASSIGNMENT = re.compile(
     r'^_NNS_VERSION\s*=\s*["\'][^"\']+["\']\s*$',
     flags=re.MULTILINE,
@@ -48,69 +48,98 @@
 
 
 def _validate_cache(expected_version: str | None = None) -> int:
-    if not _CACHE_PATH.exists():
-        print(f"ERROR: R cache validation failed: {_CACHE_PATH} does not exist.", file=sys.stderr)
-        return 1
-    if _CACHE_PATH.stat().st_size == 0:
-        print(f"ERROR: R cache validation failed: {_CACHE_PATH} is empty.", file=sys.stderr)
-        return 1
-
-    try:
-        cache: Any = json.loads(_CACHE_PATH.read_text(encoding="utf-8"))
-    except json.JSONDecodeError as exc:
+    shard_paths = sorted(_CACHE_DIR.glob("*.json")) if _CACHE_DIR.is_dir() else []
+    if not shard_paths:
         print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} is not valid JSON: {exc}.",
+            f"ERROR: R cache validation failed: {_CACHE_DIR} contains no shard files.",
             file=sys.stderr,
         )
         return 1
 
-    if not isinstance(cache, dict):
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} top-level value is not an object.",
-            file=sys.stderr,
-        )
-        return 1
+    total_entries = 0
+    versions: set[str] = set()
+    for shard_path in shard_paths:
+        try:
+            shard: Any = json.loads(shard_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} is not valid JSON: {exc}.",
+                file=sys.stderr,
+            )
+            return 1
 
-    cache_version = cache.get("nns_version")
-    if not isinstance(cache_version, str) or not cache_version.strip():
+        if not isinstance(shard, dict):
+            print(
+                f"ERROR: R cache validation failed: {shard_path} top-level value "
+                "is not an object.",
+                file=sys.stderr,
+            )
+            return 1
+        if shard.get("schema_version") != _SCHEMA_VERSION:
+            print(
+                "ERROR: R cache validation failed: "
+                f"{shard_path} expected schema_version {_SCHEMA_VERSION!r}, "
+                f"got {shard.get('schema_version')!r}.",
+                file=sys.stderr,
+            )
+            return 1
+
+        label = shard.get("label")
+        if not isinstance(label, str) or _shard_filename(label) != shard_path.name:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} label {label!r} "
+                "does not match its filename.",
+                file=sys.stderr,
+            )
+            return 1
+
+        shard_version = shard.get("nns_version")
+        if not isinstance(shard_version, str) or not shard_version.strip():
+            print(
+                f"ERROR: R cache validation failed: {shard_path} nns_version "
+                "must be a non-empty string.",
+                file=sys.stderr,
+            )
+            return 1
+        versions.add(shard_version)
+
+        entries = shard.get("entries")
+        if not isinstance(entries, dict) or not entries:
+            print(
+                f"ERROR: R cache validation failed: {shard_path} entries object "
+                "is missing or empty.",
+                file=sys.stderr,
+            )
+            return 1
+        total_entries += len(entries)
+
+    if len(versions) != 1:
         print(
-            "ERROR: R cache validation failed: nns_version must be a non-empty string.",
+            "ERROR: R cache validation failed: shards disagree on nns_version "
+            f"({sorted(versions)}).",
             file=sys.stderr,
         )
         return 1
+    cache_version = versions.pop()
     if expected_version is not None and cache_version != expected_version:
         print(
             "ERROR: R cache validation failed: "
             f"expected nns_version {expected_version!r}, got {cache_version!r}.",
             file=sys.stderr,
         )
         return 1
-    if cache.get("schema_version") != _SCHEMA_VERSION:
-        print(
-            "ERROR: R cache validation failed: "
-            f"expected schema_version {_SCHEMA_VERSION!r}, got {cache.get('schema_version')!r}.",
-            file=sys.stderr,
-        )
-        return 1
 
-    entries = cache.get("entries")
-    if not isinstance(entries, dict):
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} entries value is not an object.",
-            file=sys.stderr,
-        )
-        return 1
-    if not entries:
-        print(
-            f"ERROR: R cache validation failed: {_CACHE_PATH} entries object is empty.",
-            file=sys.stderr,
-        )
-        return 1
-
-    print(f"OK: {_CACHE_PATH} contains {len(entries)} entries for NNS {cache_version}.")
+    print(
+        f"OK: {_CACHE_DIR} contains {total_entries} entries across "
+        f"{len(shard_paths)} shards for NNS {cache_version}."
+    )
     return 0
 
 
+def _shard_filename(label: str) -> str:
+    return label.replace("/", "_").replace("\\", "_") + ".json"
+
+
 def _running_in_ci() -> bool:
     return os.environ.get("CI", "").lower() in {"1", "true", "yes"} or bool(
         os.environ.get("GITHUB_ACTIONS")
@@ -186,7 +215,7 @@ def main() -> int:
         action="store_true",
         help=(
             "Detect the installed R NNS version, update the cache marker, move the old "
-            "cache to tests/_r_cache.json.bak, and regenerate every entry from live R."
+            "cache to tests/_r_cache.bak/, and regenerate every entry from live R."
         ),
     )
     parser.add_argument(
@@ -220,9 +249,11 @@ def main() -> int:
         if _set_cache_version(expected_version):
             return 1
 
-        if _CACHE_PATH.exists():
-            _CACHE_PATH.replace(_CACHE_BACKUP_PATH)
-            print(f"Moved existing cache to {_CACHE_BACKUP_PATH}; starting from empty cache.")
+        if _CACHE_DIR.is_dir():
+            if _CACHE_BACKUP_DIR.exists():
+                shutil.rmtree(_CACHE_BACKUP_DIR)
+            _CACHE_DIR.replace(_CACHE_BACKUP_DIR)
+            print(f"Moved existing cache to {_CACHE_BACKUP_DIR}; starting from empty cache.")
         else:
             print("No existing cache found; starting from empty cache.")
 

scripts/run_live_r_parity_for_changed_api.py

@@ -10,9 +10,9 @@
 from typing import Any
 
 REPORT_DEFAULT = Path("sync/last_live_r_parity_report.md")
-R_CACHE_PATH = Path("tests/_r_cache.json")
-# Backup suffix is gitignored (see .gitignore) so it never lands in a PR.
-R_CACHE_BACKUP = R_CACHE_PATH.with_suffix(".json.bak")
+R_CACHE_PATH = Path("tests/_r_cache")
+# Backup directory is gitignored (see .gitignore) so it never lands in a PR.
+R_CACHE_BACKUP = Path("tests/_r_cache.bak")
 
 # Toggles that force tests/_r.py into offline/cache-only mode. Clearing them lets
 # a parity test actually shell out to live R on a cache miss.
@@ -98,7 +98,7 @@ def run_live_subset(out: Path, header: list[str], parity_tests: list[str]) -> No
         code = run(cmd, clear_offline=True)
     finally:
         if R_CACHE_PATH.exists():
-            R_CACHE_PATH.unlink()
+            shutil.rmtree(R_CACHE_PATH)
         if moved:
             shutil.move(str(R_CACHE_BACKUP), str(R_CACHE_PATH))
 

scripts/sync_nns_core_snapshot.py

@@ -60,7 +60,7 @@ def main() -> None:
         "vendored_core_path": "extern/NNS-core",
         "vendored_r_path": "tools/NNS",
         "vendored_r_tarball": f"tools/NNS_{args.r_version}.tar.gz",
-        "r_cache_path": "tests/_r_cache.json",
+        "r_cache_path": "tests/_r_cache",
         "notes": (
             "NNS-python consumes accepted NNS-core snapshots for native code and "
             "verifies public Python behavior against live or cached R NNS."

scripts/sync_r_nns_snapshot.py

@@ -93,7 +93,7 @@ def main() -> None:
             "python_commit": None,
             "vendored_r_path": "tools/NNS",
             "vendored_r_tarball": str(tarball),
-            "r_cache_path": "tests/_r_cache.json",
+            "r_cache_path": "tests/_r_cache",
         }
     )
     manifest.setdefault("core_repo", "OVVO-Financial/NNS-core")

sync/nns_source.json

@@ -10,6 +10,6 @@
   "vendored_core_path": "extern/NNS-core",
   "vendored_r_path": "tools/NNS",
   "vendored_r_tarball": "tools/NNS_13.0.tar.gz",
-  "r_cache_path": "tests/_r_cache.json",
+  "r_cache_path": "tests/_r_cache",
   "notes": "NNS-python consumes accepted NNS-core snapshots for native code and verifies public Python behavior against live or cached R NNS. r_commit/r_version record the 13.2 Beta commit that generated the committed parity cache (NNS_13.2.tar.gz). The vendored native snapshot (tools/NNS, r_src_tree_hash, tarball) and NNS-core commit are left at their prior values; re-syncing the native snapshot to this R commit is a separate NNS-core snapshot update and is not part of this cache/behavior reconciliation."
 }

sync/r_api_map.json

@@ -57,7 +57,7 @@
     "python_modules": [
       "pyproject.toml",
       "tools/NNS",
-      "tests/_r_cache.json"
+      "tests/_r_cache"
     ],
     "parity_tests": ["tests/parity"],
     "requires_fresh_cache": true