From dc12ba62c60892e0c95a291e6d5504354da3c15f Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sat, 9 May 2026 15:42:46 -0400 Subject: [PATCH 01/22] feat(api): add JVSPATIAL_DOCS_DISABLED + per-path docs CSP - AppBuilder honors JVSPATIAL_DOCS_DISABLED (1/true/yes/on): unpublishes /docs, /redoc, /openapi.json, /docs/oauth2-redirect (404, no spec leak). - SecurityHeadersMiddleware emits a relaxed CSP on docs paths so Swagger UI / ReDoc render via cdn.jsdelivr.net. App routes keep strict. --- .env.example | 6 ++ CHANGELOG.md | 8 ++ docs/md/environment-keys-reference.md | 1 + docs/md/production-deployment.md | 20 +++++ jvspatial/api/components/app_builder.py | 29 +++++- jvspatial/api/middleware/manager.py | 50 ++++++++++- jvspatial/version.py | 2 +- .../test_app_builder_docs_disable.py | 86 ++++++++++++++++++ tests/api/middleware/test_security_headers.py | 88 +++++++++++++++++++ 9 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 tests/api/components/test_app_builder_docs_disable.py create mode 100644 tests/api/middleware/test_security_headers.py diff --git a/.env.example b/.env.example index b140634..7274753 100644 --- a/.env.example +++ b/.env.example @@ -75,6 +75,12 @@ # Default: API built with jvspatial framework # JVSPATIAL_API_DESCRIPTION=API built with jvspatial framework +# Unpublish documentation surface in production. +# When truthy, /docs, /redoc, /openapi.json, and /docs/oauth2-redirect +# are not registered (return 404 — no Swagger HTML, no OpenAPI JSON). +# Truthy values: 1, true, yes, on (case-insensitive). Default: published. +# JVSPATIAL_DOCS_DISABLED=1 + # ----------------------------------------------------------------------------- # CORS CONFIGURATION # ----------------------------------------------------------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index 647b46a..0eadac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `JVSPATIAL_DOCS_DISABLED` env var (truthy `1`/`true`/`yes`/`on`) — when set, `AppBuilder.create_app` constructs FastAPI with `docs_url=None`, `redoc_url=None`, `openapi_url=None`, and `swagger_ui_oauth2_redirect_url=None` so the documentation surface is fully unpublished (404 with no spec leak). Recommended for production. + +### Fixed + +- Security headers middleware now emits a relaxed Content-Security-Policy on `/docs`, `/redoc`, `/openapi.json` (and sub-paths) that permits `cdn.jsdelivr.net` so FastAPI's bundled Swagger UI / ReDoc pages render. The previous strict default blocked the CDN-hosted JS/CSS and the docs loaded blank. Application routes keep the strict default policy. + ## [0.0.7] - 2026-05-08 ### Security diff --git a/docs/md/environment-keys-reference.md b/docs/md/environment-keys-reference.md index e96763e..932db0f 100644 --- a/docs/md/environment-keys-reference.md +++ b/docs/md/environment-keys-reference.md @@ -31,6 +31,7 @@ For full examples and default values, see: - `JVSPATIAL_API_HEALTH` - Health route path. - `JVSPATIAL_API_ROOT` - Root route path. - `JVSPATIAL_GRAPH_ENDPOINT_ENABLED` - Enables graph REST endpoint. +- `JVSPATIAL_DOCS_DISABLED` - When truthy (`1`/`true`/`yes`/`on`), unpublishes the documentation surface entirely: `/docs`, `/redoc`, `/openapi.json`, and `/docs/oauth2-redirect` are not registered (return 404). Recommended for production. ### CORS - `JVSPATIAL_CORS_ENABLED` - Enables CORS middleware. diff --git a/docs/md/production-deployment.md b/docs/md/production-deployment.md index 16706c9..8e4613d 100644 --- a/docs/md/production-deployment.md +++ b/docs/md/production-deployment.md @@ -51,6 +51,25 @@ Security headers (X-Content-Type-Options, X-Frame-Options, X-XSS-Protection) are server = Server(security=dict(security_headers_enabled=False)) ``` +The `Content-Security-Policy` header is strict by default on application +routes. The bundled Swagger UI / ReDoc pages (`/docs`, `/redoc`, +`/openapi.json`) get a relaxed CSP that permits `cdn.jsdelivr.net` so the +docs render — no extra config needed in dev. + +#### Unpublishing the docs surface in production + +Set a single env var to remove `/docs`, `/redoc`, `/openapi.json`, and +`/docs/oauth2-redirect` from the registered routes entirely (FastAPI +returns 404 — no Swagger HTML, no OpenAPI JSON): + +``` +JVSPATIAL_DOCS_DISABLED=1 +``` + +Truthy values: `1`, `true`, `yes`, `on` (case-insensitive). Unset or any +other value keeps the docs published. Use this in production when the API +surface should not be self-documenting. + ### 4. CORS Configuration The default CORS configuration allows localhost origins. For production, restrict to your actual frontend domain(s): @@ -104,6 +123,7 @@ JVSPATIAL_WEBHOOK_HMAC_SECRET=your-webhook-secret-minimum-32-chars | `JVSPATIAL_CORS_ORIGINS` | Your frontend domain(s), not `*` | | `JVSPATIAL_WEBHOOK_HMAC_SECRET` | Unique per environment, 32+ chars | | `JVSPATIAL_WEBHOOK_HTTPS_REQUIRED` | `true` | +| `JVSPATIAL_DOCS_DISABLED` | `1` (unpublishes `/docs`, `/redoc`, `/openapi.json`) | ## Kubernetes / Container Deployment diff --git a/jvspatial/api/components/app_builder.py b/jvspatial/api/components/app_builder.py index 0c29fc4..94c2ac7 100644 --- a/jvspatial/api/components/app_builder.py +++ b/jvspatial/api/components/app_builder.py @@ -5,6 +5,7 @@ """ import logging +import os from pathlib import Path from typing import Any, Dict, List, Optional @@ -45,13 +46,30 @@ def create_app(self, lifespan: Optional[Any] = None) -> FastAPI: Returns: Configured FastAPI application instance + + Production unpublish: setting ``JVSPATIAL_DOCS_DISABLED=1`` (or + ``true``/``yes``/``on``) forces ``docs_url``, ``redoc_url``, + ``openapi_url``, and ``swagger_ui_oauth2_redirect_url`` to + ``None``. FastAPI then never registers any of those routes — they + return 404 with no body, the OpenAPI schema is not served, and no + Swagger UI HTML is emitted. Use this in production when the API + surface should not be self-documenting. """ - app_kwargs = { + docs_disabled = ( + os.getenv("JVSPATIAL_DOCS_DISABLED") or "" + ).strip().lower() in { + "1", + "true", + "yes", + "on", + } + + app_kwargs: Dict[str, Any] = { "title": self.config.title, "description": self.config.description, "version": self.config.version, - "docs_url": self.config.docs_url, - "redoc_url": self.config.redoc_url, + "docs_url": None if docs_disabled else self.config.docs_url, + "redoc_url": None if docs_disabled else self.config.redoc_url, "debug": self.config.debug, # Configure OpenAPI tags order - ensure "App" appears after "default" # Tags are ordered by their appearance in this list @@ -60,6 +78,11 @@ def create_app(self, lifespan: Optional[Any] = None) -> FastAPI: {"name": "App", "description": "Application-specific endpoints"}, ], } + if docs_disabled: + # Explicitly nuke the OpenAPI JSON spec and the Swagger OAuth2 + # redirect endpoint too — FastAPI defaults them on otherwise. + app_kwargs["openapi_url"] = None + app_kwargs["swagger_ui_oauth2_redirect_url"] = None # Add lifespan if provided if lifespan is not None: diff --git a/jvspatial/api/middleware/manager.py b/jvspatial/api/middleware/manager.py index 8b0e10c..a1a1c01 100644 --- a/jvspatial/api/middleware/manager.py +++ b/jvspatial/api/middleware/manager.py @@ -20,13 +20,59 @@ from jvspatial.api.server import Server +# Strict default — applied to application routes. +_DEFAULT_CSP = "default-src 'self'; frame-ancestors 'none'" + +# Relaxed CSP for FastAPI's bundled Swagger UI / ReDoc pages. Those pages +# pull swagger-ui-dist + redoc bundles from cdn.jsdelivr.net and a favicon +# from fastapi.tiangolo.com, plus run a small inline bootstrap script. The +# strict default blocks all of that and the docs render blank. Scoped to +# documentation paths only — application routes keep the strict policy. +# +# The single production knob is `JVSPATIAL_DOCS_DISABLED=1` (read at app +# build time in `AppBuilder.create_app`); that flag unpublishes /docs, +# /redoc, and /openapi.json entirely so this CSP never matters in prod. +# When docs ARE published (dev / staging) this relaxation makes Swagger +# UI render — no further env knobs needed. +_DOCS_CSP = ( + "default-src 'self' https://cdn.jsdelivr.net; " + "base-uri 'self'; " + "frame-ancestors 'none'; " + "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " + "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " + "img-src 'self' data: blob: https://cdn.jsdelivr.net https://fastapi.tiangolo.com; " + "font-src 'self' data: https://cdn.jsdelivr.net; " + "connect-src 'self' https://cdn.jsdelivr.net; " + "worker-src 'self' blob:; " + "object-src 'none'" +) + +# Documentation-page path prefixes. Both `/docs` and `/redoc` may carry +# trailing path segments (Swagger's oauth2-redirect, ReDoc assets); match +# exact path or `/...`. `/openapi.json` is the JSON spec consumed +# by both UIs — it's CSP-irrelevant on its own but we group it for symmetry. +_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json") + + +def _is_docs_path(path: str) -> bool: + """True for FastAPI Swagger / ReDoc / OpenAPI surfaces.""" + return any(path == p or path.startswith(p + "/") for p in _DOCS_PATH_PREFIXES) + + class SecurityHeadersMiddleware(BaseHTTPMiddleware): """Middleware that adds security headers to all responses. Headers applied: - X-Content-Type-Options: nosniff (MIME sniffing prevention) - X-Frame-Options: DENY (clickjacking prevention) - - Content-Security-Policy: default-src 'self'; frame-ancestors 'none' + - Content-Security-Policy: strict on app routes; a relaxed variant + that permits ``cdn.jsdelivr.net`` is emitted for ``/docs``, + ``/redoc``, and ``/openapi.json`` so FastAPI's bundled Swagger UI + and ReDoc pages render. Without this scoped relaxation the docs + load blank because the strict default blocks the CDN-hosted JS/CSS. + For production lockdown set ``JVSPATIAL_DOCS_DISABLED=1`` — + ``AppBuilder.create_app`` then unpublishes the docs surface entirely + and this CSP never applies (no routes registered). - Strict-Transport-Security: max-age=31536000; includeSubDomains (if enabled) HSTS is only applied when the server configures hsts_enabled=True @@ -43,7 +89,7 @@ async def dispatch(self, request, call_next): response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["Content-Security-Policy"] = ( - "default-src 'self'; frame-ancestors 'none'" + _DOCS_CSP if _is_docs_path(request.url.path) else _DEFAULT_CSP ) if self._hsts_enabled: response.headers["Strict-Transport-Security"] = ( diff --git a/jvspatial/version.py b/jvspatial/version.py index 2b835ec..d3e0d82 100644 --- a/jvspatial/version.py +++ b/jvspatial/version.py @@ -9,4 +9,4 @@ # - MAJOR: Breaking changes # - MINOR: New features, backward compatible # - PATCH: Bug fixes, backward compatible -__version__ = "0.0.7" +__version__ = "0.0.8" diff --git a/tests/api/components/test_app_builder_docs_disable.py b/tests/api/components/test_app_builder_docs_disable.py new file mode 100644 index 0000000..4069138 --- /dev/null +++ b/tests/api/components/test_app_builder_docs_disable.py @@ -0,0 +1,86 @@ +"""AppBuilder — JVSPATIAL_DOCS_DISABLED unpublish behavior. + +Verifies that the single env knob fully removes the documentation surface +(`/docs`, `/redoc`, `/openapi.json`, `/docs/oauth2-redirect`) without +otherwise affecting the application. +""" + +from __future__ import annotations + +from typing import Optional + +import pytest +from fastapi.testclient import TestClient + +from jvspatial.api.components.app_builder import AppBuilder +from jvspatial.api.config import ServerConfig + + +def _build_client(config: Optional[ServerConfig] = None) -> TestClient: + """Construct an AppBuilder-built FastAPI app + add a probe route.""" + cfg = config or ServerConfig() + app = AppBuilder(cfg).create_app() + + @app.get("/__probe") + def _probe() -> dict: + return {"ok": True} + + return TestClient(app) + + +def test_docs_published_by_default() -> None: + """Without the env flag, /docs and /openapi.json render.""" + client = _build_client() + assert client.get("/docs").status_code == 200 + assert client.get("/redoc").status_code == 200 + assert client.get("/openapi.json").status_code == 200 + + +@pytest.mark.parametrize("flag", ["1", "true", "True", "yes", "on"]) +def test_docs_unpublished_via_env_flag(monkeypatch, flag: str) -> None: + """`JVSPATIAL_DOCS_DISABLED=` returns 404 on every doc surface.""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", flag) + client = _build_client() + assert client.get("/docs").status_code == 404 + assert client.get("/redoc").status_code == 404 + assert client.get("/openapi.json").status_code == 404 + # Swagger's OAuth2 redirect helper is also stripped. + assert client.get("/docs/oauth2-redirect").status_code == 404 + + +def test_app_routes_unaffected_when_docs_disabled(monkeypatch) -> None: + """Disabling docs leaves application routes alone.""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", "1") + client = _build_client() + r = client.get("/__probe") + assert r.status_code == 200 + assert r.json() == {"ok": True} + + +@pytest.mark.parametrize("flag", ["", "0", "false", "no", "off", "garbage"]) +def test_falsey_or_unrelated_values_keep_docs_published(monkeypatch, flag: str) -> None: + """Only the documented truthy values disable docs.""" + if flag: + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", flag) + else: + monkeypatch.delenv("JVSPATIAL_DOCS_DISABLED", raising=False) + client = _build_client() + assert client.get("/docs").status_code == 200 + assert client.get("/openapi.json").status_code == 200 + + +def test_disabled_state_strips_openapi_schema_endpoint(monkeypatch) -> None: + """`/openapi.json` is the metadata leak that matters most in prod — + confirm it is gone (not just inaccessible).""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", "1") + cfg = ServerConfig() + app = AppBuilder(cfg).create_app() + + # FastAPI exposes the openapi mount via app.openapi_url; should be None. + assert app.openapi_url is None + # And the spec generator returns nothing usable when openapi_url is off. + schema = app.openapi() + # FastAPI still synthesizes a schema in-memory, but the route is not + # registered — the public surface is what matters and we already + # asserted 404 above. Sanity-check the schema is a dict (no crash). + assert isinstance(schema, dict) diff --git a/tests/api/middleware/test_security_headers.py b/tests/api/middleware/test_security_headers.py new file mode 100644 index 0000000..b719bc1 --- /dev/null +++ b/tests/api/middleware/test_security_headers.py @@ -0,0 +1,88 @@ +"""SecurityHeadersMiddleware — per-path CSP behavior. + +Production unpublish lives elsewhere: `JVSPATIAL_DOCS_DISABLED=1` in +`AppBuilder.create_app` removes the routes outright, so this middleware +only ever sees app routes. When docs ARE published (dev/staging) the +relaxed `_DOCS_CSP` permits the Swagger UI / ReDoc CDN bundles. +""" + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from jvspatial.api.middleware.manager import ( + _DEFAULT_CSP, + _DOCS_CSP, + SecurityHeadersMiddleware, +) + + +def _build_client() -> TestClient: + """Spin a minimal FastAPI app + a docs sub-path probe.""" + app = FastAPI() + app.add_middleware(SecurityHeadersMiddleware) + + @app.get("/health") + def _health() -> dict: + return {"ok": True} + + @app.get("/docs/extra") + def _docs_subpath() -> dict: + return {"docs": True} + + return TestClient(app) + + +def test_app_route_gets_strict_csp() -> None: + """Application routes receive the locked-down default CSP.""" + client = _build_client() + r = client.get("/health") + assert r.headers["content-security-policy"] == _DEFAULT_CSP + + +def test_docs_route_gets_relaxed_csp() -> None: + """`/docs` (FastAPI default) gets the CDN-permitting CSP so Swagger UI loads.""" + app = FastAPI() # FastAPI auto-mounts /docs and /openapi.json + app.add_middleware(SecurityHeadersMiddleware) + client = TestClient(app) + + r = client.get("/docs") + assert r.status_code == 200 + assert r.headers["content-security-policy"] == _DOCS_CSP + + r2 = client.get("/openapi.json") + assert r2.status_code == 200 + assert r2.headers["content-security-policy"] == _DOCS_CSP + + +def test_docs_subpath_also_relaxed() -> None: + """Docs path matching includes sub-paths (e.g. /docs/oauth2-redirect).""" + client = _build_client() + r = client.get("/docs/extra") + assert r.headers["content-security-policy"] == _DOCS_CSP + + +def test_constant_security_headers_always_present() -> None: + """X-Content-Type-Options + X-Frame-Options ride along on every response.""" + client = _build_client() + r = client.get("/health") + assert r.headers["x-content-type-options"] == "nosniff" + assert r.headers["x-frame-options"] == "DENY" + + +def test_hsts_off_by_default() -> None: + client = _build_client() + r = client.get("/health") + assert "strict-transport-security" not in {k.lower() for k in r.headers} + + +def test_hsts_on_when_constructor_flag_enabled() -> None: + app = FastAPI() + app.add_middleware(SecurityHeadersMiddleware, hsts_enabled=True) + + @app.get("/x") + def _x() -> dict: + return {"ok": True} + + client = TestClient(app) + r = client.get("/x") + assert "max-age=31536000" in r.headers["strict-transport-security"] From a3964ab4510198e7afa7ac769c306ccc4916b62f Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Tue, 12 May 2026 13:08:36 -0400 Subject: [PATCH 02/22] upd: added ability to customize entity name nofr nodes regardless of class name (optional) using __entity_name__ = 'Name' --- jvgraph-ui/package-lock.json | 32 ++++---- jvspatial/core/context.py | 8 +- jvspatial/core/entities/edge.py | 4 +- jvspatial/core/entities/object.py | 23 +++++- jvspatial/core/utils.py | 26 +++++- tests/core/test_entity_name_override.py | 101 ++++++++++++++++++++++++ 6 files changed, 164 insertions(+), 30 deletions(-) create mode 100644 tests/core/test_entity_name_override.py diff --git a/jvgraph-ui/package-lock.json b/jvgraph-ui/package-lock.json index 86832be..b7d9fcd 100644 --- a/jvgraph-ui/package-lock.json +++ b/jvgraph-ui/package-lock.json @@ -1280,12 +1280,12 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz", - "integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", + "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.11", + "follow-redirects": "^1.16.0", "form-data": "^4.0.5", "proxy-from-env": "^2.1.0" } @@ -1637,9 +1637,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", @@ -1843,9 +1843,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/loose-envify": { @@ -1965,9 +1965,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", + "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", "dev": true, "funding": [ { @@ -2211,9 +2211,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", + "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", "dev": true, "license": "MIT", "peer": true, diff --git a/jvspatial/core/context.py b/jvspatial/core/context.py index c4ead7d..11b1eb8 100644 --- a/jvspatial/core/context.py +++ b/jvspatial/core/context.py @@ -899,7 +899,7 @@ async def find( return await self.find_nodes(entity_class, query, limit=limit) collection = self._get_collection_name(entity_type_code) - db_query = {"entity": entity_class.__name__, **query} + db_query = {"entity": entity_class._entity_name(), **query} results = await self.database.find(collection, db_query, limit=limit) entities = [] @@ -1263,7 +1263,7 @@ async def find_nodes( collection = self._get_collection_name(self._get_entity_type_code(node_class)) # Add class name filter to query for type safety - db_query = {"entity": node_class.__name__, **query} + db_query = {"entity": node_class._entity_name(), **query} db = self.database results = await db.find(collection, db_query) @@ -1319,7 +1319,7 @@ async def ensure_indexes(self, entity_class: Type[T]) -> None: collection = self._get_collection_name(type_code) # Check if we've already ensured indexes for this collection - collection_key = f"{collection}:{entity_class.__name__}" + collection_key = f"{collection}:{entity_class._entity_name()}" if collection_key in _ensured_indexes: return # Already ensured @@ -1438,7 +1438,7 @@ async def _deserialize_entity( from .utils import find_subclass_by_name # Use entity field for class identification - stored_entity = data.get("entity", entity_class.__name__) + stored_entity = data.get("entity", entity_class._entity_name()) entity_type_code = self._get_entity_type_code(entity_class) # Prefer requested class subtree, then (for Nodes) scan the entire Node hierarchy. diff --git a/jvspatial/core/entities/edge.py b/jvspatial/core/entities/edge.py index 2f8d391..6502472 100644 --- a/jvspatial/core/entities/edge.py +++ b/jvspatial/core/entities/edge.py @@ -165,7 +165,7 @@ def __init__( # Don't override ID if already provided if "id" not in kwargs: - kwargs["id"] = generate_id("e", self.__class__.__name__) + kwargs["id"] = generate_id("e", self.__class__._entity_name()) kwargs.update( {"source": source, "target": target, "bidirectional": bidirectional} @@ -321,7 +321,7 @@ async def all(cls: Type["Edge"]) -> List["Object"]: bidirectional = data["context"].get("bidirectional", True) # Handle subclass instantiation based on stored entity - stored_entity = data.get("entity", cls.__name__) + stored_entity = data.get("entity", cls._entity_name()) target_class = find_subclass_by_name(cls, stored_entity) or cls context_data = { diff --git a/jvspatial/core/entities/object.py b/jvspatial/core/entities/object.py index 72af6d5..0a89b74 100644 --- a/jvspatial/core/entities/object.py +++ b/jvspatial/core/entities/object.py @@ -1,7 +1,7 @@ """Base Object class for jvspatial entities.""" import inspect -from typing import Any, Dict, List, Optional, Set, Type, Union +from typing import Any, ClassVar, Dict, List, Optional, Set, Type, Union from pydantic import BaseModel, ConfigDict @@ -28,6 +28,21 @@ class Object(AttributeMixin, BaseModel): model_config = ConfigDict(extra="ignore") + # Per-class override of the persisted ``entity`` discriminator. Defaults to + # ``cls.__name__``. Set this on a subclass when two unrelated ``Object`` + # subtrees share a Python class name (e.g. host-app ``App`` vs library + # ``App``) and must remain distinguishable at the storage layer. + __entity_name__: ClassVar[Optional[str]] = None + + @classmethod + def _entity_name(cls) -> str: + """Return the persisted entity discriminator for this class. + + Honors ``__entity_name__`` when set on the class (not inherited from a + parent that happens to set it), otherwise falls back to ``cls.__name__``. + """ + return cls.__dict__.get("__entity_name__") or cls.__name__ + id: str = attribute( protected=True, transient=True, description="Unique identifier for the object" ) @@ -82,10 +97,10 @@ def __init__(self: "Object", **kwargs: Any) -> None: type_code = type_code_field.default else: type_code = "o" # Default type code - kwargs["id"] = generate_id(type_code, self.__class__.__name__) + kwargs["id"] = generate_id(type_code, self.__class__._entity_name()) # Set entity to class name if not provided (protected attribute) if "entity" not in kwargs: - kwargs["entity"] = self.__class__.__name__ + kwargs["entity"] = self.__class__._entity_name() # Call super().__init__() first to initialize Pydantic model (including __pydantic_private__) # The _initializing attribute defaults to True, so it's already set during Pydantic initialization @@ -592,7 +607,7 @@ def _collect_class_names(cls: Type["Object"]) -> Set[str]: Returns: Set of class names including the class itself and all imported subclasses """ - names: Set[str] = {cls.__name__} + names: Set[str] = {cls._entity_name()} # Add all imported subclasses recursively for subclass in cls.__subclasses__(): diff --git a/jvspatial/core/utils.py b/jvspatial/core/utils.py index 24e7365..103d172 100644 --- a/jvspatial/core/utils.py +++ b/jvspatial/core/utils.py @@ -1,5 +1,6 @@ """Utility functions for jvspatial core module.""" +import contextlib import uuid from typing import Dict, Optional, Tuple, Type @@ -39,14 +40,31 @@ async def generate_id_async(type_: str, class_name: str) -> str: _subclass_cache: Dict[Tuple[Type, str], Optional[Type]] = {} +def _class_entity_name(cls: Type) -> str: + """Return the persisted entity discriminator for ``cls``. + + Mirrors ``Object._entity_name()`` but is safe to call on arbitrary types — + falls back to ``cls.__name__`` when ``_entity_name`` is absent. Lets + ``find_subclass_by_name`` honor ``__entity_name__`` overrides without + forcing every caller to pass an ``Object`` descendant. + """ + fn = getattr(cls, "_entity_name", None) + if callable(fn): + with contextlib.suppress(Exception): + return fn() + return cls.__name__ + + def find_subclass_by_name(base_class: Type, name: str) -> Optional[Type]: """Find a subclass by name recursively with caching. - Returns the base class if it matches the name, otherwise returns - the first matching subclass found. Uses caching for performance. + Matches against each class's ``_entity_name()`` (which honors the + ``__entity_name__`` override) and falls back to ``cls.__name__``. + Returns the base class if it matches, otherwise the first matching + subclass found. Uses caching for performance. """ # Check base class first - if base_class.__name__ == name: + if _class_entity_name(base_class) == name: return base_class # Check cache @@ -56,7 +74,7 @@ def find_subclass_by_name(base_class: Type, name: str) -> Optional[Type]: def find_subclass(cls: Type) -> Optional[Type]: for subclass in cls.__subclasses__(): - if subclass.__name__ == name: + if _class_entity_name(subclass) == name: return subclass found = find_subclass(subclass) if found: diff --git a/tests/core/test_entity_name_override.py b/tests/core/test_entity_name_override.py new file mode 100644 index 0000000..264c56d --- /dev/null +++ b/tests/core/test_entity_name_override.py @@ -0,0 +1,101 @@ +"""Tests for ``__entity_name__`` per-class override of the persisted entity discriminator. + +Covers the case where two unrelated ``Node`` (or ``Object``) subclasses share a +Python class name and must remain distinguishable at the storage layer (e.g. +host-app ``App`` vs library ``App``). Setting ``__entity_name__`` on one of +them decouples ``cls.__name__`` from the ``entity`` field jvspatial uses to +discriminate rows. +""" + +import tempfile +import uuid +from unittest.mock import patch + +import pytest + +from jvspatial.core.context import GraphContext +from jvspatial.core.entities import Node +from jvspatial.core.utils import _class_entity_name, find_subclass_by_name +from jvspatial.db import create_database + + +# Two classes with the same Python ``__name__`` but distinct entity discriminators. +class App(Node): + """Host-app App. Persists with the override entity name.""" + + __entity_name__ = "HostApp" + title: str = "" + + +# Define a second class also literally named ``App`` in this module's local scope +# by reusing the class statement. Pytest's collector preserves the binding so we +# rebind via type() to keep both alive. +_LibApp = type( + "App", + (Node,), + { + "__module__": __name__, + "__doc__": "Library App. Persists with default entity name 'App'.", + "__annotations__": {"label": str}, + "label": "", + }, +) + + +@pytest.fixture +def json_context(): + with tempfile.TemporaryDirectory() as tmpdir: + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + yield GraphContext(database=database) + + +def test_entity_name_helper_honors_override(): + assert _class_entity_name(App) == "HostApp" + assert _class_entity_name(_LibApp) == "App" + + # Subclasses without their own override inherit ``__name__`` semantics, + # NOT the parent's override. + class ChildOfHost(App): + pass + + assert _class_entity_name(ChildOfHost) == "ChildOfHost" + + +def test_find_subclass_by_name_routes_by_entity_name(): + # ``App`` (host) matches lookup for "HostApp", not "App". + assert find_subclass_by_name(Node, "HostApp") is App + # ``_LibApp`` (the other class also named App in Python) matches "App". + assert find_subclass_by_name(Node, "App") is _LibApp + + +@pytest.mark.asyncio +async def test_persisted_entity_field_uses_override(json_context): + host = App(title="host-side") + lib = _LibApp(label="lib-side") + + # ``entity`` attribute on the live instance reflects the override at create-time. + assert host.entity == "HostApp" + assert lib.entity == "App" + + # ID prefix follows entity name (``n..``). + assert host.id.startswith("n.HostApp.") + assert lib.id.startswith("n.App.") + + await json_context.save(host) + await json_context.save(lib) + + with patch("jvspatial.core.context.get_default_context", return_value=json_context): + # Cross-class query isolation: each class only sees its own rows. + hosts = await App.find({}) + libs = await _LibApp.find({}) + + assert len(hosts) == 1 + assert len(libs) == 1 + assert hosts[0].id == host.id + assert libs[0].id == lib.id + # Deserialization picks the right class for each entity discriminator. + assert isinstance(hosts[0], App) + assert isinstance(libs[0], _LibApp) + assert hosts[0].title == "host-side" + assert libs[0].label == "lib-side" From 3a6526cf853a9d3ad24ae8b4fff88f182af23f6b Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:12:57 -0400 Subject: [PATCH 03/22] docs: replace legacy SPEC with authoritative technical contract Renames the legacy code-pattern cookbook (formerly SPEC.md, 5284 lines labeled "Language Model Coding Guide") to LLM-CODING-GUIDE.md, since that is its actual purpose. Adds a new SPEC.md that captures the technical contract jvspatial guarantees: identity model, async contract, persistence layer, walker semantics, GraphContext, API surface, auth, configuration, serverless constraints, storage, caching, observability, security boundaries, extension points, error taxonomy, and stability tiers. Every claim cites a source-of-truth file:line. Companions PRD.md, ROADMAP.md, CLAUDE.md, and the docs index land in follow-up commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- LLM-CODING-GUIDE.md | 5284 ++++++++++++++++++++++++++++++++++++++++ SPEC.md | 5555 ++++--------------------------------------- 2 files changed, 5812 insertions(+), 5027 deletions(-) create mode 100644 LLM-CODING-GUIDE.md diff --git a/LLM-CODING-GUIDE.md b/LLM-CODING-GUIDE.md new file mode 100644 index 0000000..a80c749 --- /dev/null +++ b/LLM-CODING-GUIDE.md @@ -0,0 +1,5284 @@ +# jvspatial Language Model Coding Guide + +This document provides concise guidance for AI language models to generate code that follows jvspatial library standards and conventions. + +## 🎯 Core Philosophy + +jvspatial emphasizes **entity-centric design** with unified MongoDB-style queries across database backends (JSON, MongoDB). The library distinguishes between: + +- **Objects** - For standalone data entities (users, settings, logs) that don't require graph relationships +- **Nodes** - For graph entities that are interconnected by Edges and traversed by Walkers +- **Edges** - For relationships between Nodes in the graph +- **Walkers** - For traversing and processing graph structures + +**Key Principle**: Use Objects for simple data storage, use Nodes when you need graph traversal and relationships. + +## 🔧 Environment Setup + +### Essential Configuration +```python +from dotenv import load_dotenv +load_dotenv() # Load .env file + +from jvspatial.core import GraphContext +from jvspatial.db import create_database, get_database_manager + +# Option 1: Create database explicitly +db = create_database("json", base_path="./jvdb") +ctx = GraphContext(database=db) + +# Option 2: Use current database from manager (defaults to prime) +manager = get_database_manager() +ctx = GraphContext(database=manager.get_current_database()) +``` + +### Environment Variables +```env +# Choose backend +JVSPATIAL_DB_TYPE=json # or 'mongodb' + +# JSON backend (default) +JVSPATIAL_JSONDB_PATH=./jvdb/dev + +# MongoDB backend +JVSPATIAL_MONGODB_URI=mongodb://localhost:27017 +JVSPATIAL_MONGODB_DB_NAME=jvspatial_dev + +# Caching (optional) +JVSPATIAL_CACHE_BACKEND=memory # 'memory', 'redis', or 'layered' +JVSPATIAL_CACHE_SIZE=1000 # Max cached entities (0 to disable) +``` + +## 📝 Entity-Centric Code Patterns + +### Objects vs Nodes: When to Use Each + +**✅ Objects** - For standalone entities without graph relationships: +```python +from jvspatial.core import Object + +class UserProfile(Object): + name: str = "" + email: str = "" + settings: Dict[str, Any] = {} + +# Use for: user profiles, configuration, logs, simple data +profile = await UserProfile.create(name="Alice", email="alice@company.com") +``` + +**✅ Nodes** - For graph entities with relationships and traversal: +```python +from jvspatial.core import Node + +class User(Node): + name: str = "" + department: str = "" + +class City(Node): + name: str = "" + population: int = 0 + +# Use for: entities that connect to other entities via Edges +user = await User.create(name="Alice", department="engineering") +city = await City.create(name="San Francisco", population=800000) +``` + +### Entity Operations +```python +# Entity creation (no save() needed, automatically cached) +entity = await Entity.create(name="value", field="data") + +# Entity retrieval (uses cache after first access) +entity = await Entity.get(entity_id) # Cached by ID +entities = await Entity.find({"context.active": True}) # Not cached + +# Entity updates (save() only needed after property modification, updates cache) +entity = await Entity.get(entity_id) +entity.name = "Updated Name" # Property modified +await entity.save() # save() required to persist changes + update cache + +# Entity deletion (removes from cache) +await entity.delete() + +# Counting and aggregation (not cached) +# Note: Object.count() doesn't exist - use len() with find() instead +results = await Entity.find({"context.department": "engineering"}) +count = len(results) + +# For distinct values, query and extract manually +all_entities = await Entity.find({}) +departments = set(e.department for e in all_entities if hasattr(e, 'department')) +``` + +**Note**: Caching is automatic and transparent. Individual entity retrievals by ID (`Entity.get(id)`) are cached. Queries (`find()`, `all()`) always hit the database as they can change frequently. For counting, use `Entity.count(query)` for efficient counting without loading all records. + +### save() Operation Rules +**✅ save() is ONLY required when:** +1. You modify entity properties after retrieval: `entity.field = "new_value"` +2. You create entities without using `.create()` method + +**❌ save() is NOT needed when:** +1. Using `.create()` method (automatically persists) +2. Using `.delete()` method (automatically persists deletion) +3. Just reading/querying entities + +**❌ AVOID: Direct database access (use entity methods instead)** +```python +# Don't do this - use entity methods instead +from jvspatial.db import create_database +db = create_database("json") +entities = await db.find("object", {"name": "Entity"}) + +# ✅ Do this instead - use entity-centric methods +entities = await Entity.find({"context.name": "Entity"}) +``` + +## 🗄️ Multi-Database Support + +jvspatial supports managing multiple databases within the same application, with a prime database for core persistence operations (authentication, session management) and additional databases for application-specific data. + +### Basic Multi-Database Usage + +```python +from jvspatial.db import ( + create_database, + get_database_manager, + get_prime_database, + get_current_database, + switch_database, + unregister_database, +) +from jvspatial.core.context import GraphContext + +# Get database manager (singleton) +manager = get_database_manager() + +# Prime database is automatically created for core operations +prime_db = get_prime_database() # Used for auth, sessions, system data + +# Create and register additional database +app_db = create_database( + "json", + base_path="./app_data", + register=True, + name="app" +) + +# Switch to application database +switch_database("app") +current_db = get_current_database() # Now returns app_db + +# Use with GraphContext +app_ctx = GraphContext(database=current_db) + +# Switch back to prime database +switch_database("prime") + +# Unregister non-prime database when no longer needed +unregister_database("app") +``` + +### Prime Database + +The prime database is always used for: +- User authentication +- Session management +- System-level configuration +- Core persistence operations + +It cannot be unregistered and is always available as the default. + +### Database Isolation + +Each database maintains complete isolation: +- Entities in one database are not visible in another +- Switching databases changes the context for all operations +- Prime database ensures core operations always have a stable database + +**📖 For comprehensive multi-database documentation:** [Graph Context Guide](docs/md/graph-context.md) and [Multi-Database Example](examples/database/multi_database_example.py) + +## 🔧 Custom Database Integration + +jvspatial supports seamless extension with custom database backends through a registration system. + +### Registering Custom Database Types + +```python +from jvspatial.db import Database, register_database_type, create_database, list_database_types + +# Define custom database implementation +class CustomDatabase(Database): + async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: + # Implementation + pass + + async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: + # Implementation + pass + + async def delete(self, collection: str, id: str) -> None: + # Implementation + pass + + async def find(self, collection: str, query: Dict[str, Any]) -> List[Dict[str, Any]]: + # Implementation + pass + +# Factory function for creating instances +def create_custom_db(**kwargs: Any) -> CustomDatabase: + return CustomDatabase(**kwargs) + +# Register the custom database type +register_database_type("custom", create_custom_db) + +# Now use it like built-in types +db = create_database("custom", connection_string="custom://...") + +# List all available database types +types = list_database_types() +# Returns: {"json": "JSON file-based database", "mongodb": "MongoDB database", "custom": "Custom database: create_custom_db"} +``` + +### Custom Database Requirements + +Custom databases must: +1. Inherit from `Database` abstract base class +2. Implement all abstract methods: `save()`, `get()`, `delete()`, `find()` +3. Optionally implement `count()` and `find_one()` (default implementations provided) +4. Provide a factory function for creation + +**📖 For comprehensive custom database documentation:** [Custom Database Guide](docs/md/custom-database-guide.md) and [Custom Database Example](examples/database/custom_database_example.py) + +## 🔍 MongoDB-Style Query Patterns + +Always use dot notation for nested fields with `context.` prefix: + +```python +# Comparison operators +users = await User.find({"context.age": {"$gte": 35}}) +users = await User.find({"context.role": {"$ne": "admin"}}) + +# Logical operators +users = await User.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.active": True} + ] +}) + +# Array operations +users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) + +# Regular expressions +users = await User.find({ + "context.name": {"$regex": "Johnson", "$options": "i"} +}) +``` + +## 🔒 Attribute Annotations (Protected & Transient) + +jvspatial provides `@protected` and `@transient` decorators for controlling attribute behavior: + +### Protected Attributes +Protected attributes cannot be modified after initialization (ideal for IDs and immutable config): + +```python +from pydantic import Field +from jvspatial.core.annotations import protected + +class Entity(Node): + # id is already protected in Node + uuid: str = protected("", description="Immutable UUID") + created_at: datetime = protected(Field(default_factory=datetime.now)) + +# ✓ Can set during initialization +entity = await Entity.create(uuid="abc-123") + +# ✗ Cannot modify after creation +entity.uuid = "new-uuid" # Raises AttributeProtectionError +``` + +### Transient Attributes +Transient attributes are excluded from database exports (ideal for runtime caches): + +```python +from jvspatial.core.annotations import transient + +class Entity(Node): + data: str = "" + cache: dict = transient(Field(default_factory=dict)) # Not persisted + temp_count: int = transient(Field(default=0)) # Not persisted + +entity.cache["key"] = "value" # Works at runtime +data = await entity.export() # cache excluded from export +``` + +### Compound Decorators +Combine both for internal state that's neither modifiable nor persisted: + +```python +# Both protected AND transient +_internal: dict = protected(transient(Field(default_factory=dict))) +``` + +**Key Points:** +- All `id` fields in `Object`, `Node`, `Edge`, and `Walker` are automatically protected +- Always use `Field(default_factory=dict)` syntax with `@transient` +### Private Attributes +Private attributes are excluded from serialization and database operations (ideal for internal state): + +```python +from jvspatial.core.annotations import private + +class Entity(Node): + _cache: dict = private(default_factory=dict) # Not serialized + _internal_counter: int = private(default=0) # Not serialized + +entity._cache["key"] = "value" # Works at runtime +data = await entity.export() # _cache excluded from export +``` + +### Compound Decorators +Combine decorators for complex behaviors: +```python +# Private AND transient +_internal: dict = private(transient(Field(default_factory=dict))) +``` +- See [Attribute Annotations](docs/md/attribute-annotations.md) for full documentation + +## 🏢 Type Annotations & Error Handling + +### Required Typing Pattern +```python +from typing import List, Optional, Dict, Any +from jvspatial.core import Node, Object +from jvspatial.exceptions import NodeNotFoundError, ValidationError + +class User(Node): + name: str = "" + email: str = "" + age: int = 0 + roles: List[str] = [] + active: bool = True + metadata: Dict[str, Any] = {} + +async def get_user_by_id(user_id: str) -> Optional[User]: + """Get user by ID, returning None if not found.""" + try: + return await User.get(user_id) + except NodeNotFoundError: + return None +``` + +### Error Handling Patterns + +**🎯 Always catch specific exceptions first:** + +```python +import logging +from jvspatial.exceptions import ( + JVSpatialError, + ValidationError, + EntityNotFoundError, + NodeNotFoundError, + DatabaseError, + ConnectionError +) + +logger = logging.getLogger(__name__) + +# Entity operations with error handling +async def safe_user_operation(user_id: str) -> Optional[User]: + try: + user = await User.get(user_id) + return user + except NodeNotFoundError as e: + logger.warning(f"User not found: {e.entity_id}") + return None + except ValidationError as e: + logger.error(f"Validation failed: {e.message}") + if e.field_errors: + for field, error in e.field_errors.items(): + logger.error(f" {field}: {error}") + return None + except DatabaseError as e: + logger.error(f"Database error: {e.message}") + raise # Re-raise for higher-level handling + except JVSpatialError as e: + logger.error(f"jvspatial error: {e.message}") + return None +``` + +**🔄 Database operations with fallback:** + +```python +from jvspatial.exceptions import ConnectionError, QueryError + +async def robust_user_search(query: Dict[str, Any]) -> List[User]: + try: + # Try complex query + return await User.find(query) + except QueryError as e: + logger.warning(f"Complex query failed: {e.message}") + # Fallback to simple query + try: + all_users = await User.all() + # Apply filtering in Python + return [u for u in all_users if u.active] + except Exception: + logger.error("All query methods failed") + return [] + except ConnectionError as e: + logger.error(f"Database connection failed: {e.database_type}") + return [] # Graceful degradation +``` + +**⚠️ Walker error handling:** + +```python +from jvspatial.exceptions import WalkerExecutionError, WalkerTimeoutError + +class SafeUserProcessor(Walker): + @on_visit(User) + async def process_user(self, here: User): + try: + # Potentially risky operation + result = await external_api_call(here) + self.report(result) + except Exception as e: + # Don't let individual errors stop traversal + logger.warning(f"Failed to process user {here.id}: {e}") + self.report({"error": str(e), "user_id": here.id}) + +async def run_safe_walker(): + try: + walker = SafeUserProcessor() + result = await walker.spawn(start_user) + + # Get results and errors + report = await result.get_report() + errors = [r for r in report if isinstance(r, dict) and "error" in r] + logger.info(f"Processed with {len(errors)} errors") + + except WalkerTimeoutError as e: + logger.error(f"Walker timed out after {e.timeout_seconds}s") + # Access partial results if needed + except WalkerExecutionError as e: + logger.error(f"Walker failed: {e.walker_class} - {e.message}") +``` + +## 📄 ObjectPager for Large Datasets + +### Basic Pagination +```python +from jvspatial.core.pager import paginate_objects, ObjectPager + +# Simple pagination +users = await paginate_objects(User, page=1, page_size=20) + +# With filters +active_users = await paginate_objects( + User, + page=1, + page_size=10, + filters={"context.active": True} +) + +# Advanced pager +pager = ObjectPager( + User, + page_size=25, + filters={"context.department": "engineering"}, + order_by="name" +) +users = await pager.get_page(1) +``` + +## ⏰ Scheduler Integration + +### Task Scheduling +```python +from jvspatial.core.scheduler import Scheduler, ScheduledTask +from datetime import datetime, timedelta + +# Create scheduler +scheduler = Scheduler() + +# Schedule recurring tasks +@scheduler.task(interval=timedelta(hours=1)) +async def cleanup_expired_sessions(): + """Clean up expired user sessions hourly.""" + expired = await UserSession.find({ + "context.expires_at": {"$lt": datetime.now()} + }) + for session in expired: + await session.delete() + print(f"Cleaned up {len(expired)} expired sessions") + +# Schedule one-time tasks +@scheduler.task(run_at=datetime.now() + timedelta(minutes=30)) +async def send_reminder_emails(): + """Send reminder emails.""" + users = await User.find({"context.reminder_due": True}) + for user in users: + await send_email(user.email, "Reminder") + user.reminder_due = False + await user.save() + +# Start scheduler +await scheduler.start() +``` + +### Walker-Based Scheduled Tasks +```python +from jvspatial.core import Walker +from jvspatial.core.entities import on_visit + +@scheduler.walker_task(interval=timedelta(days=1)) +class DailyMaintenanceWalker(Walker): + """Perform daily maintenance tasks via graph traversal.""" + + @on_visit("User") + async def check_user_activity(self, here: Node): + """Check user activity and update status.""" + if here.last_active < datetime.now() - timedelta(days=30): + here.status = "inactive" + await here.save() + self.report({"deactivated_user": here.id}) + + @on_visit("DataNode") + async def cleanup_old_data(self, here: Node): + """Remove old data nodes.""" + if here.created_at < datetime.now() - timedelta(days=90): + await here.delete() + self.report({"deleted_data_node": here.id}) + +# Start scheduled walker +await scheduler.start_walker_task(DailyMaintenanceWalker) +``` + +## 🌐 API Server with Server Class + +### Basic Server Setup +```python +from jvspatial.api import Server + +# Create server instance +server = Server( + title="My Spatial API", + description="Graph-based spatial data management API", + version="1.0.0", + host="0.0.0.0", + port=8000 +) + +# Run server +if __name__ == "__main__": + server.run() +``` + +### Walker Endpoints +```python +from jvspatial.api import endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit + +@endpoint("/api/users/process", methods=["POST"]) +class ProcessUser(Walker): + """Process user data with graph traversal.""" + + user_name: str = EndpointField( + description="Name of user to process", + examples=["John Doe"] + ) + + department: str = EndpointField( + default="general", + description="User department" + ) + + @on_visit("User") + async def process_user(self, here: Node): + """Process user nodes - use 'here' for visited node.""" + if here.name == self.user_name: + self.report({ + "found_user": { + "id": here.id, + "name": here.name, + "department": here.department + } + }) + + # Get connected nodes and continue traversal + colleagues = await here.nodes( + node=['User'], + department=self.department + ) + await self.visit(colleagues) +``` + +### Function Endpoints +```python +from jvspatial.api import endpoint + +@endpoint("/api/users/count", methods=["GET"]) +async def get_user_count() -> Dict[str, int]: + """Get total user count.""" + users = await User.all() + return {"total_users": len(users)} + +@endpoint("/api/users/{user_id}", methods=["GET"]) +async def get_user(user_id: str, endpoint) -> Any: + """Get user with semantic response.""" + user = await User.get(user_id) + if not user: + return endpoint.not_found( + message="User not found", + details={"user_id": user_id} + ) + + return endpoint.success( + data={"id": user.id, "name": user.name, "email": user.email} + ) +``` + +### Server with Scheduler Integration +```python +from jvspatial.api import Server +from jvspatial.core.scheduler import Scheduler + +# Create integrated server with scheduler +server = Server(title="Scheduled API", port=8000) +scheduler = Scheduler() + +# Add scheduled tasks +@scheduler.task(interval=timedelta(minutes=5)) +async def periodic_health_check(): + """Check system health every 5 minutes.""" + # Health check logic here + pass + +@server.on_startup +async def startup_tasks(): + """Start scheduler when server starts.""" + await scheduler.start() + print("✅ Server and scheduler started") + +@server.on_shutdown +async def shutdown_tasks(): + """Stop scheduler when server shuts down.""" + await scheduler.stop() + print("🛑 Server and scheduler stopped") + +# Run integrated server +if __name__ == "__main__": + server.run() +``` + +## 🔗 Webhook Integration + +### Basic Webhook Handler +```python +from jvspatial.api import endpoint +from fastapi import Request + +@endpoint("/webhook/{service}/{auth_token}", methods=["POST"], webhook=True) +async def webhook_handler(request: Request) -> Dict[str, Any]: + """Process webhooks with automatic payload parsing.""" + raw_body = request.state.raw_body + content_type = request.state.content_type + current_user = get_current_user(request) + + # Always return 200 for webhooks + try: + # Process webhook logic here + return {"status": "success", "processed_at": datetime.now().isoformat()} + except Exception as e: + logger.error(f"Webhook error: {e}") + return {"status": "received", "error": "logged"} +``` + +## 🏗️ Core Architecture & Walker Patterns + +### Entity Hierarchy +- **Object** - Base class for all entities with unified query interface +- **Node** - Graph nodes with spatial/contextual data (extends Object) +- **Edge** - Relationships between nodes +- **Walker** - Graph traversal and processing logic +- **GraphContext** - Low-level database interface (use sparingly) + +### Walker Traversal (CRITICAL PATTERNS) + +#### Naming Convention for @on_visit Methods +**ALWAYS** use these parameter names: +- **`here`** - The visited node/edge (current location) +- **`visitor`** - The visiting walker (when accessing from node context) + +```python +@on_visit("User") +async def process_user(self, here: Node): + """Use 'here' for visited node.""" + connected_users = await here.nodes(node=['User']) + await self.visit(connected_users) + +@on_visit("City") +async def process_city(self, here: Node): + """Process cities with filtering.""" + # Skip small cities + if here.population < 10000: + self.skip() # Skip to next node + return + + # Get large connected cities + large_cities = await here.nodes( + node=['City'], + population={"$gte": 500000} + ) + await self.visit(large_cities) +``` + +#### Walker Control Flow +```python +class DataWalker(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.max_items = 100 + + @on_visit("Document") + async def process_document(self, here: Node): + """Process with control flow.""" + # Skip invalid documents + if here.status == "invalid": + self.skip() # Continue to next node + + # Stop at limit + if self.processed_count >= self.max_items: + await self.disengage() # Permanently halt walker + return + + # Pause for rate limiting + if self.processed_count % 50 == 0: + self.pause("Rate limit pause") + + # Normal processing + self.processed_count += 1 + next_docs = await here.nodes(node=['Document']) + await self.visit(next_docs) + + @on_exit + async def cleanup(self): + """Called when walker completes/pauses/disengages.""" + print(f"Processed {self.processed_count} documents") +``` + +## 📋 Quick Reference Checklist + +### Entity Operations +- ✅ Use `await Entity.create(**kwargs)` +- ✅ Use `await Entity.find(query_dict)` +- ✅ Use `await entity.save()` only after property modification +- ✅ Use Objects for standalone data, Nodes for graph entities +- ✅ Use `await node1.disconnect(node2)` to remove connections +- ❌ Avoid direct GraphContext database calls + +### Disconnecting Nodes +To remove connections between nodes, use the `disconnect()` method. This removes edges between nodes and deletes the edge objects. + +```python +# Disconnect two nodes +success = await node1.disconnect(node2) + +# Disconnect with specific edge type +success = await node1.disconnect(node2, edge_type=SpecialEdge) +``` + +### Query Patterns +- ✅ Use `"context.field"` dot notation for nested fields +- ✅ Use MongoDB operators: `$gte`, `$in`, `$regex`, `$and`, `$or` +- ✅ Combine dict filters with kwargs: `node=[{'User': {...}}], active=True` + +### Walker Patterns +- ✅ Use `here` parameter for visited nodes +- ✅ Use `await here.nodes()` to get connected nodes +- ✅ Use `await self.visit(nodes)` to continue traversal +- ✅ Use `self.skip()` to skip current node +- ✅ Use `await self.disengage()` to permanently halt +- ✅ Use `self.pause()` for temporary suspension + +### API Patterns +- ✅ Use `@endpoint` for both graph processing and simple functions +- ✅ Use `EndpointField` for parameter configuration +- ✅ Use `endpoint.success()`, `endpoint.not_found()` for responses +- ✅ Always return 200 for webhooks with try/catch + +### Type Safety +- ✅ Always include proper type annotations +- ✅ Import from `typing` and `jvspatial.exceptions` +- ✅ Handle `NodeNotFoundError`, `ValidationError`, `DatabaseError` +- ✅ Use structured error logging + +This guide provides the essential patterns for generating jvspatial-compliant code. Focus on entity-centric operations, proper typing, and following the established naming conventions for walker traversal. + +The **jvspatial webhook system** provides secure, flexible webhook endpoints with built-in authentication, HMAC verification, idempotency keys, and automatic payload processing. Webhooks integrate seamlessly with the FastAPI server and support both function-based handlers and graph traversal processing. + +### Webhook Architecture Overview + +Webhooks in jvspatial are designed for: +- **Security**: Path-based authentication tokens, optional HMAC signature verification +- **Reliability**: Idempotency key support to handle duplicate deliveries +- **Flexibility**: JSON/XML/binary payload support with automatic parsing +- **Integration**: Full compatibility with existing authentication and permission systems +- **Processing**: Always return HTTP 200 for proper webhook etiquette + +### Basic Webhook Endpoint Setup + +```python path=null start=null +from fastapi import Request +from jvspatial.api import endpoint +from jvspatial.api.auth.middleware import get_current_user +from typing import Dict, Any +import json + +# Basic webhook handler function +@endpoint("/webhook/{route}/{auth_token}", methods=["POST"], webhook=True) +async def generic_webhook_handler(request: Request) -> Dict[str, Any]: + """Generic webhook handler for multiple services. + + Processes webhooks from various sources using route-based dispatch. + Middleware handles authentication, HMAC verification, and payload parsing. + """ + # Access processed data from middleware + raw_body = request.state.raw_body # Original bytes + content_type = request.state.content_type # Content-Type header + route = getattr(request.state, "webhook_route", "unknown") # Route parameter + current_user = get_current_user(request) # Authenticated user + + # Parse payload based on content type + processed_data = None + if content_type == "application/json": + try: + processed_data = json.loads(raw_body) + except json.JSONDecodeError: + return {"status": "error", "message": "Invalid JSON payload"} + else: + # Handle other content types (XML, form data, binary) + processed_data = {"raw_length": len(raw_body), "type": content_type} + + # Route-based processing + if route == "stripe": + return await process_stripe_webhook(processed_data, current_user) + elif route == "github": + return await process_github_webhook(processed_data, current_user) + elif route == "slack": + return await process_slack_webhook(processed_data, current_user) + else: + # Generic processing for unknown routes + return { + "status": "received", + "route": route, + "payload_type": content_type, + "user_id": current_user.id if current_user else None + } + +# Service-specific webhook handlers +@endpoint("/webhook/stripe/{auth_token}", methods=["POST"], webhook=True) +async def stripe_webhook_handler(request: Request) -> Dict[str, Any]: + """Dedicated Stripe webhook handler with event processing.""" + raw_body = request.state.raw_body + current_user = get_current_user(request) + + try: + event = json.loads(raw_body) + event_type = event.get("type", "unknown") + + # Process different Stripe event types + if event_type == "payment_intent.succeeded": + await handle_successful_payment(event["data"]["object"], current_user) + elif event_type == "customer.subscription.updated": + await handle_subscription_update(event["data"]["object"], current_user) + elif event_type == "invoice.payment_failed": + await handle_payment_failure(event["data"]["object"], current_user) + + return { + "status": "success", + "event_type": event_type, + "processed_at": datetime.now().isoformat() + } + + except Exception as e: + # Always return 200 for webhooks, log errors internally + print(f"Stripe webhook processing error: {e}") + return {"status": "received", "error": "Processing error logged"} + +# Helper functions for webhook processing +async def process_stripe_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process Stripe webhook events.""" + event_type = data.get("type", "unknown") + return { + "status": "success", + "message": f"Processed Stripe {event_type}", + "user_id": user.id if user else None + } + +async def process_github_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process GitHub webhook events.""" + action = data.get("action", "unknown") + repo_name = data.get("repository", {}).get("name", "unknown") + return { + "status": "success", + "message": f"GitHub {action} on {repo_name}", + "user_id": user.id if user else None + } + +async def process_slack_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process Slack webhook events.""" + event_type = data.get("type", "unknown") + return { + "status": "success", + "message": f"Slack {event_type} processed", + "user_id": user.id if user else None + } +``` + +### Webhook Security and Authentication + +Webhook endpoints require authentication tokens in the URL path and support additional security measures: + +```python path=null start=null +# Webhook with permission requirements +@endpoint( + "/webhook/admin/{route}/{auth_token}", + methods=["POST"], + webhook=True, + permissions=["process_webhooks", "admin_access"], + roles=["admin", "webhook_manager"] +) +async def admin_webhook_handler(request: Request) -> Dict[str, Any]: + """Administrative webhook handler with strict permissions.""" + current_user = get_current_user(request) + + # User is guaranteed to have required permissions due to middleware + return { + "status": "success", + "message": "Admin webhook processed", + "admin_user": current_user.username, + "permissions": current_user.permissions + } + +# HMAC signature verification (handled by middleware) +@endpoint("/webhook/secure/{service}/{auth_token}", methods=["POST"], webhook=True) +async def secure_webhook_handler(request: Request) -> Dict[str, Any]: + """Webhook with HMAC signature verification. + + Middleware automatically verifies HMAC signatures when present. + Configure HMAC secrets via environment variables or user settings. + """ + # If this handler executes, HMAC verification passed (if configured) + raw_body = request.state.raw_body + hmac_verified = getattr(request.state, "hmac_verified", False) + + return { + "status": "success", + "message": "Secure webhook processed", + "hmac_verified": hmac_verified, + "payload_size": len(raw_body) + } +``` + +### Graph Traversal Webhook Processing (Future Enhancement) + +The architecture supports webhook processing through graph traversal using Walker classes: + +```python path=null start=null + +# from jvspatial.api.auth.decorators import webhook_endpoint +# from jvspatial.core import Walker, Node +# from jvspatial.core.entities import on_visit + +# @endpoint("/webhook/process/{route}/{auth_token}", methods=["POST"]) +# class WebhookProcessingWalker(Walker): +# """Walker-based webhook processing with graph traversal.""" +# +# def __init__(self): +# super().__init__() +# self.webhook_data = None +# self.processing_results = [] +# +# @on_visit("WebhookEvent") +# async def process_webhook_event(self, here: Node): +# """Process webhook events stored as graph nodes.""" +# # Access webhook data from request.state +# payload = self.webhook_data +# +# # Process event based on node data and webhook payload +# result = await self.analyze_event(here, payload) +# self.processing_results.append(result) +# +# # Continue traversal to related events +# related_events = await here.nodes(node=['WebhookEvent']) +# await self.visit(related_events) +# +# async def analyze_event(self, event_node: Node, payload: dict) -> dict: +# """Analyze webhook event against stored data.""" +# return { +# "event_id": event_node.id, +# "payload_type": payload.get("type"), +# "correlation_score": 0.95 # Example analysis result +# } +``` + +### Idempotency and Duplicate Handling + +Webhooks support idempotency keys to handle duplicate deliveries: + +```python path=null start=null +@endpoint("/webhook/idempotent/{auth_token}", methods=["POST"], webhook=True) +async def idempotent_webhook_handler(request: Request) -> Dict[str, Any]: + """Webhook handler with built-in idempotency support. + + Middleware automatically handles idempotency keys in headers: + - Idempotency-Key header + - X-Idempotency-Key header + - Custom idempotency headers + """ + # Access idempotency information from middleware + idempotency_key = getattr(request.state, "idempotency_key", None) + is_duplicate = getattr(request.state, "is_duplicate_request", False) + + if is_duplicate: + # Return cached response for duplicate requests + cached_response = getattr(request.state, "cached_response", {}) + return { + "status": "success", + "message": "Duplicate request, returning cached response", + "idempotency_key": idempotency_key, + "cached_result": cached_response + } + + # Process new request + raw_body = request.state.raw_body + processed_result = await process_unique_webhook(json.loads(raw_body)) + + return { + "status": "success", + "message": "New webhook processed", + "idempotency_key": idempotency_key, + "result": processed_result + } + +async def process_unique_webhook(payload: dict) -> dict: + """Process a unique webhook payload.""" + # Simulate processing logic + import time + processing_start = time.time() + + # Your actual webhook processing logic here + await asyncio.sleep(0.1) # Simulate work + + return { + "processed_at": processing_start, + "data_processed": True, + "payload_keys": list(payload.keys()) + } +``` + +### Server Integration and Middleware Setup + +Webhook endpoints automatically integrate with the jvspatial server middleware stack: + +```python path=null start=null +from jvspatial.api import Server +from jvspatial.api.auth.middleware import ( + AuthenticationMiddleware, + WebhookMiddleware, + HTTPSRedirectMiddleware +) + +# Server setup with webhook middleware +server = Server( + title="Webhook-Enabled Spatial API", + description="API with secure webhook processing", + version="1.0.0", + host="0.0.0.0", + port=8000 +) + +# Middleware stack (order matters) +server.add_middleware(HTTPSRedirectMiddleware) # Force HTTPS +server.add_middleware(WebhookMiddleware) # Webhook processing +server.add_middleware(AuthenticationMiddleware) # Authentication + +# Webhook endpoints are automatically registered +# Access at: POST https://your-domain.com/webhook/{route}/{auth_token} + +# Environment configuration for webhook security +# Set in .env file: +# WEBHOOK_HMAC_SECRET=your-secret-key +# WEBHOOK_HTTPS_REQUIRED=true +# WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour cache +# WEBHOOK_MAX_PAYLOAD_SIZE=1048576 # 1MB limit + +if __name__ == "__main__": + server.run(port=8000) +``` + +### Webhook Testing and Development + +```python path=null start=null +# Testing webhook handlers +import pytest +from fastapi.testclient import TestClient +from unittest.mock import MagicMock + +@pytest.fixture +def test_webhook_request(): + """Create mock webhook request for testing.""" + request = MagicMock() + request.state.raw_body = b'{"type": "test", "data": {"id": 123}}' + request.state.content_type = "application/json" + request.state.webhook_route = "test" + request.state.current_user = MagicMock(id="user_123") + request.state.hmac_verified = True + request.state.idempotency_key = "test-key-123" + return request + +@pytest.mark.asyncio +async def test_webhook_processing(test_webhook_request): + """Test webhook handler processing.""" + result = await generic_webhook_handler(test_webhook_request) + + assert result["status"] == "received" + assert result["route"] == "test" + assert result["user_id"] == "user_123" + +# Development webhook testing with ngrok or similar +# 1. Start your jvspatial server locally +# 2. Use ngrok to expose: ngrok http 8000 +# 3. Configure webhook URLs: https://abc123.ngrok.io/webhook/test/your-auth-token +# 4. Test with curl: +# curl -X POST https://abc123.ngrok.io/webhook/test/your-token \ +# -H "Content-Type: application/json" \ +# -d '{"test": "data"}' +``` + +### Best Practices for Webhook Implementation + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Always return 200 status for webhooks +@endpoint("/webhook/service/{auth_token}", webhook=True) +async def proper_webhook_handler(request: Request) -> Dict[str, Any]: + try: + # Process webhook + result = await process_webhook_data(request.state.raw_body) + return {"status": "success", "result": result} + except Exception as e: + # Log error but still return 200 + logger.error(f"Webhook processing failed: {e}") + return {"status": "received", "error": "logged"} + +# Good: Use route-based dispatch for multiple services +@endpoint("/webhook/{route}/{auth_token}", webhook=True) +async def multi_service_webhook(request: Request) -> Dict[str, Any]: + route = getattr(request.state, "webhook_route", "unknown") + + handlers = { + "stripe": process_stripe_webhook, + "github": process_github_webhook, + "slack": process_slack_webhook + } + + handler = handlers.get(route, process_generic_webhook) + return await handler(request) + +# Good: Validate authentication token format +@endpoint("/webhook/{service}/{auth_token}", webhook=True) +async def secure_webhook_handler(request: Request) -> Dict[str, Any]: + # Token validation is handled by middleware + current_user = get_current_user(request) + if not current_user: + return {"status": "error", "message": "Invalid authentication"} + + return {"status": "success", "user_verified": True} +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Returning non-200 status codes +@endpoint("/webhook/bad/{auth_token}", webhook=True) +async def bad_webhook_handler(request: Request) -> Dict[str, Any]: + try: + process_webhook(request.state.raw_body) + except Exception: + # Don't do this - breaks webhook retry logic + raise HTTPException(status_code=500, detail="Processing failed") + +# Bad: Not handling authentication properly +@endpoint("/webhook/unsecure/{auth_token}", webhook=True) +async def unsecure_webhook_handler(request: Request) -> Dict[str, Any]: + # Don't bypass authentication checks + # Always use get_current_user() or require auth in decorator + return {"status": "processed"} + +# Bad: Not using middleware-processed data +@endpoint("/webhook/manual/{auth_token}", webhook=True) +async def manual_webhook_handler(request: Request) -> Dict[str, Any]: + # Don't manually read request body - use request.state.raw_body + # raw_body = await request.body() # Wrong - middleware already processed + + # Use middleware-processed data instead + raw_body = request.state.raw_body # Correct + return {"status": "processed"} +``` + +--- + +## 🔗 Webhook System Integration + +JVspatial provides an advanced webhook system for handling external service integrations with enterprise-grade security, reliability, and developer experience. The webhook system supports modern decorators, automatic payload processing, HMAC verification, idempotency handling, and seamless authentication integration. + +### Quick Webhook Setup + +```python path=null start=null +from jvspatial.api import endpoint +from jvspatial.api import Server + +# Simple webhook handler +@endpoint("/webhook/payment", webhook=True) +async def payment_webhook(payload: dict, endpoint): + """Process payment webhooks with automatic JSON parsing.""" + payment_id = payload.get("payment_id") + amount = payload.get("amount") + + # Process payment logic here + print(f"Processing payment {payment_id}: ${amount}") + + return endpoint.response( + content={ + "status": "processed", + "message": f"Payment {payment_id} processed successfully" + } + ) + +# Server automatically detects and configures webhook middleware +server = Server(title="My Webhook API") +server.run() # Webhooks ready at /webhook/* paths +``` + +### Advanced Webhook Features + +```python path=null start=null +# Webhook with full security features +@endpoint( + "/webhook/stripe/{key}", + webhook=True, + path_key_auth=True, # API key in URL path + hmac_secret="stripe-webhook-secret", # HMAC signature verification + idempotency_ttl_hours=48, # Duplicate handling for 48h + permissions=["process_payments"] # RBAC permissions +) +async def secure_stripe_webhook(raw_body: bytes, content_type: str, endpoint): + """Stripe webhook with comprehensive security.""" + import json + + if content_type == "application/json": + payload = json.loads(raw_body.decode('utf-8')) + event_type = payload.get("type", "unknown") + + if event_type == "payment_intent.succeeded": + return endpoint.response( + content={ + "status": "processed", + "event_type": event_type, + "message": "Payment successful" + } + ) + + return endpoint.response(content={"status": "received"}) + +# Multi-service webhook dispatcher +@endpoint("/webhook/{service}", webhook=True) +async def multi_service_webhook(payload: dict, service: str, endpoint): + """Route webhooks based on service parameter.""" + handlers = { + "stripe": process_stripe_event, + "github": process_github_event, + "slack": process_slack_event + } + + handler = handlers.get(service, process_generic_event) + result = await handler(payload) + + return endpoint.response( + content={ + "status": "processed", + "service": service, + "result": result + } + ) + +# Helper functions +async def process_stripe_event(payload: dict) -> dict: + return {"stripe_event": payload.get("type", "unknown")} + +async def process_github_event(payload: dict) -> dict: + return {"github_action": payload.get("action", "unknown")} + +async def process_slack_event(payload: dict) -> dict: + return {"slack_event": payload.get("event", {}).get("type", "unknown")} + +async def process_generic_event(payload: dict) -> dict: + return {"processed": True, "keys": list(payload.keys())} +``` + +### Walker-Based Webhook Processing + +```python path=null start=null +# Future feature - Walker-based webhook processing +# @webhook_walker_endpoint("/webhook/location-update") +# class LocationUpdateWalker(Walker): +# """Process location updates through graph traversal.""" +# +# def __init__(self, payload: dict): +# super().__init__() +# self.payload = payload +# # Use the report() method to collect data during traversal +# +# @on_visit(Node) +# async def update_location_data(self, here: Node): +# locations = self.payload.get("locations", []) +# +# for location_data in locations: +# location_id = location_data.get("id") +# coordinates = location_data.get("coordinates") +# +# if location_id and coordinates: +# here.coordinates = coordinates +# await here.save() +# +# self.report({ +# "updated_location": { +# "id": location_id, +# "coordinates": coordinates +# } +# }) +``` + +### Environment Configuration + +Configure webhook behavior via environment variables: + +```env +# Global webhook settings +JVSPATIAL_WEBHOOK_HMAC_SECRET=your-global-hmac-secret +JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE=5242880 # 5MB +JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour +JVSPATIAL_WEBHOOK_HTTPS_REQUIRED=true + +# Service-specific secrets +JVSPATIAL_WEBHOOK_STRIPE_SECRET=whsec_stripe_secret_key +JVSPATIAL_WEBHOOK_GITHUB_SECRET=github_webhook_secret +``` + +### Testing Webhooks + +```bash +# Basic webhook test +curl -X POST "http://localhost:8000/webhook/payment" \ + -H "Content-Type: application/json" \ + -d '{"payment_id": "pay_123", "amount": 99.99}' + +# Webhook with path-based auth +curl -X POST "http://localhost:8000/webhook/stripe/key123:secret456" \ + -H "Content-Type: application/json" \ + -H "X-Signature: sha256=abc123..." \ + -d '{"type": "payment_intent.succeeded"}' + +# With idempotency key +curl -X POST "http://localhost:8000/webhook/payment" \ + -H "Content-Type: application/json" \ + -H "X-Idempotency-Key: unique-123" \ + -d '{"payment_id": "pay_124"}' +``` + +### Webhook Best Practices + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Always return 200 for webhook endpoints +@endpoint("/webhook/service", webhook=True) +async def proper_webhook(payload: dict, endpoint): + try: + result = await process_webhook_data(payload) + return endpoint.response(content={"status": "success", "result": result}) + except Exception as e: + # Log error but still return 200 + logger.error(f"Webhook processing failed: {e}") + return endpoint.response(content={"status": "received", "error": "logged"}) + +# Good: Use route-based dispatch for multiple services +@endpoint("/webhook/{service}", webhook=True) +async def multi_service_webhook(payload: dict, service: str, endpoint): + handlers = { + "stripe": process_stripe, + "github": process_github + } + + handler = handlers.get(service, process_generic) + return await handler(payload, endpoint) + +# Good: Validate webhook signatures when available +@endpoint("/webhook/secure", webhook=True, hmac_secret="webhook-secret") +async def secure_webhook(raw_body: bytes, endpoint): + # HMAC verification is automatic when secret is provided + return endpoint.response(content={"status": "verified"}) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Returning non-200 status codes +@endpoint("/webhook/bad", webhook=True) +async def bad_webhook(payload: dict, endpoint): + if payload.get("invalid"): + # Don't do this - breaks webhook retry logic + raise HTTPException(status_code=400, detail="Invalid payload") + +# Bad: Not handling errors gracefully +@endpoint("/webhook/risky", webhook=True) +async def risky_webhook(payload: dict, endpoint): + # Unhandled exceptions will return 500 - webhooks will retry + result = dangerous_operation(payload) # Might throw + return endpoint.response(content={"result": result}) + +# Bad: Bypassing security features +@endpoint("/webhook/insecure", webhook=True) +async def insecure_webhook(request: Request, endpoint): + # Don't manually read request body - use automatic payload injection + raw_body = await request.body() # Wrong - middleware already processed + return endpoint.response(content={"status": "received"}) +``` + +> **📖 For complete webhook documentation and advanced patterns:** [Webhook Architecture Guide](docs/md/webhook-architecture.md) | [Webhook Quickstart](docs/md/webhooks-quickstart.md) + +--- + +## 📁 File Storage Quickstart + +jvspatial includes a powerful file storage system with multi-backend support and URL proxy capabilities for secure file sharing. + +### Basic Setup + +```python +from jvspatial.api import Server + +server = Server( + title="File Upload API", + file_storage_enabled=True, + file_storage_provider="local", + file_storage_root=".files", + proxy_enabled=True +) + +if __name__ == "__main__": + server.run() +``` + +### Upload a File + +```bash +curl -X POST -F "file=@document.pdf" \ + http://localhost:8000/storage/upload +``` + +**Response:** +```json +{ + "success": true, + "file_path": "2025/01/05/document-abc123.pdf", + "file_size": 102400, + "content_type": "application/pdf" +} +``` + +### Create a Shareable Link + +```bash +curl -X POST http://localhost:8000/storage/proxy \ + -H "Content-Type: application/json" \ + -d '{ + "file_path": "2025/01/05/document-abc123.pdf", + "expires_in": 3600 + }' +``` + +**Response:** +```json +{ + "success": true, + "proxy_code": "a1b2c3d4", + "proxy_url": "http://localhost:8000/p/a1b2c3d4", + "expires_at": "2025-01-05T23:00:00Z" +} +``` + +### Access via Short URL + +```bash +curl http://localhost:8000/p/a1b2c3d4 +``` + +The file is served directly with appropriate headers. + +### Use in Walkers + +```python +from jvspatial.storage import get_file_interface +from jvspatial.core import Walker, on_visit, Node + +@server.walker("/process-upload") +class ProcessUpload(Walker): + file_path: str + + @on_visit(Node) + async def process(self, here: Node): + # Get file storage interface + storage = get_file_interface( + provider="local", + root_dir=".files" + ) + + # Read file content + content = await storage.get_file(self.file_path) + + # Process file content + self.report({ + "processed_file": { + "path": self.file_path, + "size": len(content), + "status": "success" + } + }) +``` + +### AWS S3 Configuration + +```python +server = Server( + title="S3 File API", + file_storage_enabled=True, + file_storage_provider="s3", + file_storage_s3_bucket="my-bucket", + file_storage_s3_region="us-east-1", + proxy_enabled=True +) +``` + +**Environment Variables:** +```env +JVSPATIAL_FILE_STORAGE_ENABLED=true +JVSPATIAL_FILE_STORAGE_PROVIDER=s3 +JVSPATIAL_FILE_STORAGE_S3_BUCKET=my-bucket +JVSPATIAL_FILE_STORAGE_S3_REGION=us-east-1 +AWS_ACCESS_KEY_ID=your-key-id +AWS_SECRET_ACCESS_KEY=your-secret-key +``` + +### Advanced Usage: Custom Upload Path + +```bash +curl -X POST -F "file=@image.jpg" \ + -F "custom_path=avatars/user123.jpg" \ + http://localhost:8000/storage/upload +``` + +### List Files + +```bash +curl http://localhost:8000/storage/files?prefix=2025/01/ +``` + +**Response:** +```json +{ + "success": true, + "files": [ + { + "path": "2025/01/05/document-abc123.pdf", + "size": 102400, + "modified": "2025-01-05T20:30:00Z" + }, + { + "path": "2025/01/05/image-def456.jpg", + "size": 51200, + "modified": "2025-01-05T21:15:00Z" + } + ] +} +``` + +### Security Features + +```python +server = Server( + title="Secure File API", + file_storage_enabled=True, + file_storage_provider="local", + file_storage_root=".files", + file_storage_max_size=10485760, # 10MB limit + file_storage_allowed_types=["image/jpeg", "image/png", "application/pdf"], + proxy_enabled=True, + proxy_default_ttl=3600 # 1 hour default expiration +) +``` + +### Best Practices + +**✅ Recommended Patterns:** + +```python +# Good: Use environment variables for configuration +from dotenv import load_dotenv +load_dotenv() + +server = Server( + title="Production File API", + file_storage_enabled=True, + # Provider configured via JVSPATIAL_FILE_STORAGE_PROVIDER + # Other settings loaded from environment +) + +# Good: Validate files before processing +@endpoint("/validate-upload") +class ValidateUpload(Walker): + file_path: str + + @on_visit(Node) + async def validate(self, here: Node): + storage = get_file_interface() + + # Check file exists + if not await storage.file_exists(self.file_path): + self.report({"error": "File not found"}) + return + + # Get file metadata + metadata = await storage.get_metadata(self.file_path) + + # Validate size + if metadata.get("size", 0) > 5242880: # 5MB + self.report({"error": "File too large"}) + return + + self.report({"status": "valid", "metadata": metadata}) + +# Good: Use proxy URLs for temporary access +async def create_temp_link(file_path: str, hours: int = 1): + """Create temporary shareable link.""" + response = await storage.create_proxy( + file_path=file_path, + expires_in=hours * 3600 + ) + return response["proxy_url"] +``` + +**❌ Avoided Patterns:** + +```python +# Bad: Hardcoding credentials +server = Server( + file_storage_s3_bucket="my-bucket", + file_storage_s3_access_key="AKIAIOSFODNN7EXAMPLE" # Don't do this! +) + +# Bad: No file validation +@endpoint("/unsafe-upload") +class UnsafeUpload(Walker): + file_path: str + + @on_visit(Node) + async def process(self, here: Node): + # No validation - could process malicious files + content = await storage.get_file(self.file_path) + # Direct processing without checks + +# Bad: Permanent public URLs without expiration +# Always use proxy URLs with expiration for security +``` + +See [File Storage Documentation](docs/md/file-storage-usage.md) for advanced usage and all configuration options. + +--- + +## 🔀 Router Decorators + +jvspatial provides a unified `@endpoint` decorator for all API endpoints: + +1. `@endpoint` - For public endpoints (both functions and Walker classes) +2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) +3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) +4. `@endpoint(..., auth=True, roles=["admin"])` - For admin-only endpoints + +```python +from jvspatial.api import endpoint + +# Function endpoint +@endpoint("/api/users", methods=["GET"]) +async def get_users() -> Dict[str, Any]: + users = await User.all() + return {"users": users} + +# Walker endpoint +@endpoint("/api/graph/traverse", methods=["POST"]) +class GraphTraversal(Walker): + pass + +# Authenticated function endpoint +@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) +async def get_admin_stats() -> Dict[str, Any]: + return {"stats": "admin only"} + +# Authenticated walker endpoint (uses same decorator) +@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) +class SecureProcessor(Walker): + pass + +# Admin-only endpoint +@endpoint("/api/admin/users", auth=True, roles=["admin"], methods=["GET"]) +async def manage_users() -> Dict[str, Any]: + return {"users": "admin access"} +``` + +**❌ DO NOT USE alternative decorators like:** +- `@route` +- `@server.route` +- `@server.walker` +- `@walker_endpoint` (deprecated - use `@endpoint` instead) +- `@auth_endpoint` (deprecated - use `@endpoint(..., auth=True)` instead) +- `@admin_endpoint` (deprecated - use `@endpoint(..., auth=True, roles=["admin"])` instead) + +These are internal or deprecated. + +## 📌 Consolidated Endpoint System + +jvspatial uses a **unified endpoint registration system** where all endpoints (walkers and functions) are registered through a single consolidated mechanism. This ensures clean, maintainable code without backward compatibility cruft. + +### Key Architecture + +All decorators follow the same registration path: + +1. **Decorator** → Attaches metadata to function/walker +2. **Server Detection** → Gets current server from context +3. **Registration** → Registers with `server.endpoint_router` +4. **Tracking** → Tracked by `server._endpoint_registry` + +### Important: Decorator Order + +Always create the server **before** decorating endpoints: + +```python +# ✅ CORRECT +server = Server(title="My API") + +@endpoint("/test") +class TestWalker(Walker): + pass + +# ✗ INCORRECT - endpoint will not be registered +@endpoint("/test") +class TestWalker(Walker): + pass + +server = Server(title="My API") # Created too late +``` + +### Default HTTP Methods + +- **Walkers**: Default to `["POST"]` +- **Functions**: Default to `["GET"]` + +Override with the `methods` parameter: +```python +@endpoint("/data", methods=["GET", "POST"]) +class DataWalker(Walker): + pass +``` + +### Available Response Methods + +Function endpoints can receive an `endpoint` parameter for response formatting: + +```python +@endpoint("/info") +async def get_info(endpoint): + # Use endpoint.success(), endpoint.error(), etc. + return endpoint.success(data={"info": "value"}) +``` + +Walkers automatically have `self.endpoint` available: + +```python +@endpoint("/process") +class ProcessWalker(Walker): + async def process(self): + self.response = self.endpoint.success(data={"result": "done"}) +``` + +**Response Methods:** + +```python +# Success responses +endpoint.success(data=result, message="Success") # 200 OK +endpoint.created(data=new_item, message="Created") # 201 Created +endpoint.no_content() # 204 No Content + +# Error responses +endpoint.bad_request(message="Invalid input") # 400 Bad Request +endpoint.unauthorized(message="Auth required") # 401 Unauthorized +endpoint.forbidden(message="Access denied") # 403 Forbidden +endpoint.not_found(message="Resource not found") # 404 Not Found +endpoint.conflict(message="Resource exists") # 409 Conflict +endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity + +# Flexible custom response +endpoint.response( + content={"custom": "data"}, + status_code=202, + headers={"X-Custom": "value"} +) + +# Generic error with custom status code +endpoint.error( + message="Custom error", + status_code=418, + details={"reason": "custom"} +) +``` + +### Querying Registered Endpoints + +```python +server = Server(title="My API") + +# ... register some endpoints ... + +# List all endpoints +all_endpoints = server.list_all_endpoints() +print(f"Walkers: {len(all_endpoints['walkers'])}") +print(f"Functions: {len(all_endpoints['functions'])}") + +# List just walkers +walkers = server.list_walker_endpoints() + +# List just functions +functions = server.list_function_endpoints() + +# Get registry stats +registry = server._endpoint_registry +counts = registry.count_endpoints() +print(f"Total: {counts['total']}") +``` + +### Unified Registration Benefits + +1. **Single Source of Truth**: All endpoints are registered through `EndpointRouter` +2. **Cleaner Code**: No backward compatibility cruft or deprecated methods +3. **Consistent API**: All decorators follow the same pattern +4. **Better Maintainability**: Future endpoint features only need to be added once +5. **Auto-Detection**: Decorators automatically detect walker vs function + +### Migration from Deprecated Patterns + +If you have code using removed patterns: + +**❌ Old Pattern (Removed)** +```python +# These no longer exist +server._custom_routes.append(...) +server._register_custom_routes(app) +server._setup_webhook_walker_endpoints() +``` + +**✅ New Pattern (Current)** +```python +# Use the decorators - they handle registration automatically +@endpoint("/my-route") +def my_handler(): + pass + +# Or register programmatically +server.register_walker_class(MyWalker, "/my-route", methods=["POST"]) +``` + +## 🌐 API Integration with FastAPI Server + +The **jvspatial API** provides seamless integration with FastAPI to expose your graph operations as REST endpoints. It supports flexible endpoint registration using decorators and automatic parameter model generation from Walker and function properties. + +### Server Setup and Configuration + +```python path=null start=null +from jvspatial.api import Server, ServerConfig, endpoint, walker_endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Node, Walker +from jvspatial.core.entities import on_visit + +# Basic server setup +server = Server( + title="My Spatial API", + description="Graph-based spatial data management API", + version="1.0.0", + host="0.0.0.0", + port=8000, + debug=True # Enable for development +) + +# Advanced server configuration +advanced_config = ServerConfig( + title="Production Spatial API", + description="Enterprise graph data API", + version="2.0.0", + host="0.0.0.0", + port=8080, + # Database configuration + db_type="mongodb", + db_connection_string="mongodb://localhost:27017", + db_database_name="spatial_db", + # CORS settings + cors_enabled=True, + cors_origins=["https://myapp.com", "http://localhost:3000"], + # API documentation + docs_url="/api/docs", + redoc_url="/api/redoc", + log_level="info" +) + +production_server = Server(config=advanced_config) +``` + +### @walker_endpoint Decorator for Walker Classes + +The `@walker_endpoint` decorator automatically exposes Walker classes as API endpoints: + +```python path=null start=null +from jvspatial.api import endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit +from typing import List, Optional + +# Define your node types +class User(Node): + name: str = "" + email: str = "" + department: str = "" + active: bool = True + +class City(Node): + name: str = "" + population: int = 0 + state: str = "" + +# Walker with endpoint configuration using EndpointField +@endpoint("/api/users/process", methods=["POST"]) +class ProcessUser(Walker): + """Process user data with graph traversal.""" + + # Endpoint-exposed fields with configuration + user_name: str = EndpointField( + description="Name of the user to process", + examples=["John Doe", "Jane Smith"], + min_length=2, + max_length=100 + ) + + department: str = EndpointField( + default="general", + description="User department", + examples=["engineering", "marketing", "sales"] + ) + + include_inactive: bool = EndpointField( + default=False, + description="Whether to include inactive users in processing" + ) + + # Field excluded from API endpoint + internal_state: str = EndpointField( + default="processing", + exclude_endpoint=True # Won't appear in API schema + ) + + # Optional configuration field + max_connections: int = EndpointField( + default=10, + description="Maximum number of connections to traverse", + ge=1, + le=100 + ) + + @on_visit("User") + async def process_user(self, here: User): + """Process user nodes during traversal. + + Args: + here: The visited User node + """ + # Check if user matches criteria + if here.name == self.user_name: + self.report({ + "found_user": { + "id": here.id, + "name": here.name, + "email": here.email, + "department": here.department + } + }) + + # Find connected users in same department + colleagues = await here.nodes( + node=['User'], + department=self.department, + active=True if not self.include_inactive else None + ) + + self.report({ + "colleagues": [ + {"name": u.name, "email": u.email} + for u in colleagues[:self.max_connections] + ] + }) + + @on_visit("City") + async def process_city_connection(self, here: City): + """Process city connections if user has location data. + + Args: + here: The visited City node + """ + self.report({ + "city_info": { + "name": here.name, + "population": here.population, + "state": here.state + } + }) + +# Advanced walker with field grouping +@endpoint("/api/analytics/user-report", methods=["POST"]) +class UserAnalytics(Walker): + """Generate user analytics reports.""" + + # Grouped fields for better API organization + report_type: str = EndpointField( + description="Type of report to generate", + examples=["summary", "detailed", "connections"], + endpoint_group="report_config" + ) + + date_range: str = EndpointField( + default="30d", + description="Date range for report", + examples=["7d", "30d", "90d", "1y"], + endpoint_group="report_config" + ) + + include_inactive: bool = EndpointField( + default=False, + description="Include inactive users", + endpoint_group="filters" + ) + + departments: List[str] = EndpointField( + default_factory=list, + description="Departments to include (empty = all)", + examples=[["engineering", "product"], ["marketing"]], + endpoint_group="filters" + ) + + @on_visit("User") + async def analyze_user(self, here: User): + """Analyze user data for report. + + Args: + here: The visited User node + """ + # Filter by department if specified + if self.departments and here.department not in self.departments: + return + + # Filter by active status if needed + if not self.include_inactive and not here.active: + return + + # Report individual user analysis + self.report({ + "user_analyzed": { + "id": here.id, + "name": here.name, + "department": here.department, + "active": here.active + } + }) + } +``` + +### Enhanced Response Handling with endpoint.response() + +The `@walker_endpoint` and `@endpoint` decorators now automatically inject semantic response helpers to make crafting HTTP responses clean and flexible: + +**Walker Endpoints with self.endpoint:** + +```python path=null start=null +@endpoint("/api/users/profile", methods=["POST"]) +class UserProfileWalker(Walker): + """Walker demonstrating semantic response patterns.""" + + user_id: str = EndpointField(description="User ID to retrieve") + include_details: bool = EndpointField( + default=False, + description="Include detailed profile information" + ) + + @on_visit("User") + async def get_user_profile(self, here: User): + """Get user profile with semantic responses.""" + if here.id != self.user_id: + return # Continue traversal + + # User not found scenario + if not here.data: + return self.endpoint.not_found( + message="User profile not found", + details={"user_id": self.user_id} + ) + + # Authorization check + if here.private and not self.include_details: + return self.endpoint.forbidden( + message="Insufficient permissions", + details={"required_permission": "view_details"} + ) + + # Successful response + profile_data = { + "id": here.id, + "name": here.name, + "email": here.email + } + + if self.include_details: + profile_data["department"] = here.department + profile_data["created_at"] = here.created_at + + return self.endpoint.success( + data=profile_data, + message="User profile retrieved successfully" + ) + +@endpoint("/api/users/create", methods=["POST"]) +class CreateUserWalker(Walker): + """Walker for creating users with proper HTTP status codes.""" + + name: str = EndpointField(description="User name") + email: str = EndpointField(description="User email") + + @on_visit("Root") + async def create_user(self, here): + """Create a new user with validation.""" + + # Validation example + if "@" not in self.email: + return self.endpoint.unprocessable_entity( + message="Invalid email format", + details={"email": self.email} + ) + + # Check for conflicts + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await User.find({"context.email": self.email}) + existing_user = users[0] if users else None + if existing_user: + return self.endpoint.conflict( + message="User with this email already exists", + details={"email": self.email} + ) + + # Create user + user = await User.create( + name=self.name, + email=self.email + ) + + # Return 201 Created with location header + return self.endpoint.created( + data={ + "id": user.id, + "name": user.name, + "email": user.email + }, + message="User created successfully", + headers={"Location": f"/api/users/{user.id}"} + ) +``` + +**Function Endpoints with endpoint parameter:** + +```python path=null start=null +@endpoint("/api/health", methods=["GET"]) +async def health_check(endpoint) -> Any: + """Health check with semantic response.""" + return endpoint.success( + data={"status": "healthy", "version": "1.0.0"}, + message="Service is running normally" + ) + +@endpoint("/api/users/{user_id}/status", methods=["PUT"]) +async def update_user_status(user_id: str, status: str, endpoint) -> Any: + """Update user status with validation and error handling.""" + + # Validation + valid_statuses = ["active", "inactive", "suspended"] + if status not in valid_statuses: + return endpoint.bad_request( + message="Invalid status value", + details={"provided": status, "valid_options": valid_statuses} + ) + + # Find user + user = await User.get(user_id) + if not user: + return endpoint.not_found( + message="User not found", + details={"user_id": user_id} + ) + + # Update status + user.status = status + await user.save() + + return endpoint.success( + data={"id": user.id, "status": user.status}, + message=f"User status updated to {status}" + ) + +@endpoint("/api/export", methods=["GET"]) +async def export_data(format: str, endpoint) -> Any: + """Export data with custom response formatting.""" + + if format not in ["json", "csv", "xml"]: + return endpoint.error( + message="Unsupported export format", + status_code=406, # Not Acceptable + details={"format": format, "supported": ["json", "csv", "xml"]} + ) + + # Generate export data + export_data = { + "format": format, + "records": 1500, + "export_id": "exp_20250921", + "download_url": f"/downloads/export.{format}" + } + + # Use flexible response() method for custom headers + return endpoint.response( + content={ + "data": export_data, + "message": f"Export ready in {format} format" + }, + status_code=200, + headers={ + "X-Export-Format": format, + "X-Record-Count": "1500" + } + ) +``` + +**Available Response Methods:** + +```python path=null start=null +# Success responses +endpoint.success(data=result, message="Success") # 200 OK +endpoint.created(data=new_item, message="Created") # 201 Created +endpoint.no_content() # 204 No Content + +# Error responses +endpoint.bad_request(message="Invalid input") # 400 Bad Request +endpoint.unauthorized(message="Auth required") # 401 Unauthorized +endpoint.forbidden(message="Access denied") # 403 Forbidden +endpoint.not_found(message="Resource not found") # 404 Not Found +endpoint.conflict(message="Resource exists") # 409 Conflict +endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity + +# Flexible custom response +endpoint.response( + content={"custom": "data"}, + status_code=202, + headers={"X-Custom": "value"} +) + +# Generic error with custom status code +endpoint.error( + message="Custom error", + status_code=418, # I'm a teapot + details={"reason": "custom"} +) +``` + +### @endpoint Decorator for Regular Functions +The `@endpoint` decorator exposes regular functions as API endpoints: + +```python path=null start=null +from jvspatial.api import endpoint +from fastapi import HTTPException +from typing import Dict, Any, List + +# Simple function endpoint +@endpoint("/api/users/count", methods=["GET"]) +async def get_user_count() -> Dict[str, int]: + """Get total count of users in the system.""" + users = await User.all() + return {"total_users": len(users)} + +# Function endpoint with path parameters +@endpoint("/api/cities/{state}", methods=["GET"]) +async def get_cities_by_state(state: str) -> Dict[str, Any]: + """Get all cities in a specific state.""" + cities = await City.find({"context.state": state}) + + if not cities: + raise HTTPException( + status_code=404, + detail=f"No cities found in state: {state}" + ) + + return { + "state": state, + "cities": [ + { + "name": city.name, + "population": city.population + } for city in cities + ], + "total_count": len(cities) + } + +# Function endpoint with request body +@endpoint("/api/cities/search", methods=["POST"]) +async def search_cities(search_request: Dict[str, Any]) -> Dict[str, Any]: + """Search cities based on criteria.""" + # Extract search parameters + name_pattern = search_request.get("name_pattern") + min_population = search_request.get("min_population", 0) + state = search_request.get("state") + + # Build MongoDB-style query + query = {} + + if name_pattern: + query["context.name"] = {"$regex": name_pattern, "$options": "i"} + + if min_population > 0: + query["context.population"] = {"$gte": min_population} + + if state: + query["context.state"] = state + + # Execute search + cities = await City.find(query) + + return { + "search_criteria": search_request, + "results": [ + { + "id": city.id, + "name": city.name, + "population": city.population, + "state": city.state + } for city in cities + ], + "total_results": len(cities) + } + +# Function endpoint with pagination integration +@endpoint("/api/users/paginated", methods=["GET"]) +async def get_users_paginated( + page: int = 1, + page_size: int = 20, + department: Optional[str] = None, + active_only: bool = True +) -> Dict[str, Any]: + """Get paginated list of users with filtering.""" + from jvspatial.core.pager import ObjectPager + + # Build filters + filters = {} + if department: + filters["context.department"] = department + if active_only: + filters["context.active"] = True + + # Create pager + pager = ObjectPager( + User, + page_size=page_size, + filters=filters, + order_by="name", + order_direction="asc" + ) + + # Get page data + users = await pager.get_page(page) + pagination_info = pager.to_dict() + + return { + "users": [ + { + "id": user.id, + "name": user.name, + "email": user.email, + "department": user.department, + "active": user.active + } for user in users + ], + "pagination": pagination_info + } +``` + +### Server Method Registration + +You can also register endpoints directly on server instances: + +```python path=null start=null +# Using server instance decorators +@server.walker("/process-data", methods=["POST"]) +class DataProcessor(Walker): + """Process data using server instance registration.""" + + data_type: str = EndpointField( + description="Type of data to process", + examples=["user", "city", "connection"] + ) + + batch_size: int = EndpointField( + default=10, + description="Batch size for processing", + ge=1, + le=1000 + ) + + @on_visit("Node") + async def process_any_node(self, here: Node): + """Process any type of node. + + Args: + here: The visited Node + """ + # Use report() to collect processed node information + self.report({ + "processed_node": { + "id": here.id, + "type": here.__class__.__name__, + "processed_at": datetime.now().isoformat() + } + }) + +@server.route("/health-detailed", methods=["GET"]) +async def detailed_health_check() -> Dict[str, Any]: + """Detailed health check endpoint.""" + try: + # Test database connectivity + users_count = await User.count() + cities_count = await City.count() + + return { + "status": "healthy", + "database": "connected", + "statistics": { + "total_users": users_count, + "total_cities": cities_count + }, + "timestamp": datetime.now().isoformat() + } + except Exception as e: + raise HTTPException( + status_code=503, + detail=f"Health check failed: {str(e)}" + ) +``` + +### EndpointField Configuration Options + +The `EndpointField` provides extensive configuration for API parameters: + +```python path=null start=null +from jvspatial.api.decorators import EndpointField +from typing import List, Optional + +@endpoint("/api/advanced-example", methods=["POST"]) +class AdvancedEndpointExample(Walker): + """Demonstrate all EndpointField configuration options.""" + + # Basic field with validation + username: str = EndpointField( + description="User identifier", + examples=["john_doe", "jane_smith"], + min_length=3, + max_length=50, + pattern=r"^[a-zA-Z0-9_]+$" # Alphanumeric with underscores + ) + + # Numeric field with constraints + user_score: float = EndpointField( + description="User score rating", + examples=[85.5, 92.0, 78.3], + ge=0.0, # Greater than or equal to 0 + le=100.0 # Less than or equal to 100 + ) + + # Field with custom endpoint name + internal_id: str = EndpointField( + endpoint_name="user_id", # Will appear as 'user_id' in API + description="Internal user identifier" + ) + + # Optional field made required for endpoint + optional_field: Optional[str] = EndpointField( + default=None, + endpoint_required=True, # Required in API despite being Optional in Walker + description="Field that's optional in Walker but required in API" + ) + + # Required field made optional for endpoint + required_field: str = EndpointField( + endpoint_required=False, # Optional in API despite being required in Walker + description="Field that's required in Walker but optional in API" + ) + + # Grouped fields for organized API schema + config_timeout: int = EndpointField( + default=30, + description="Timeout in seconds", + endpoint_group="configuration", + ge=1, + le=300 + ) + + config_retries: int = EndpointField( + default=3, + description="Number of retries", + endpoint_group="configuration", + ge=0, + le=10 + ) + + # Hidden field (not shown in API docs) + debug_mode: bool = EndpointField( + default=False, + endpoint_hidden=True, # Hidden from OpenAPI documentation + description="Debug mode flag" + ) + + # Deprecated field + legacy_option: Optional[str] = EndpointField( + default=None, + endpoint_deprecated=True, # Marked as deprecated in API docs + description="Legacy option - use new_option instead" + ) + + # Field with additional constraints + email_domain: str = EndpointField( + description="Allowed email domain", + endpoint_constraints={ + "format": "hostname", # Additional OpenAPI constraint + "example": "company.com" + } + ) + + # List field with validation + tags: List[str] = EndpointField( + default_factory=list, + description="User tags", + examples=[["admin", "power-user"], ["guest"]] + ) + + @on_visit("User") + async def process_with_config(self, here: User): + """Process user with advanced configuration. + + Args: + here: The visited User node + """ + self.report({ + "processed_user": { + "username": self.username, + "score": self.user_score, + "config": { + "timeout": self.config_timeout, + "retries": self.config_retries + }, + "tags": self.tags, + "debug_enabled": self.debug_mode + } + }) +``` + +### Running the Server + +```python path=null start=null +# Development server +if __name__ == "__main__": + server.run( + host="0.0.0.0", + port=8000, + reload=True # Auto-reload for development + ) + +# Production server with custom configuration +async def run_production_server(): + """Run server asynchronously for production deployment.""" + await server.run_async( + host="0.0.0.0", + port=8080 + ) + +# Custom startup and shutdown hooks +@server.on_startup +async def startup_tasks(): + """Tasks to run on server startup.""" + print("🚀 Server starting up...") + # Initialize data, warm up caches, etc. + +@server.on_shutdown +async def shutdown_tasks(): + """Tasks to run on server shutdown.""" + print("🛑 Server shutting down...") + # Cleanup resources, save state, etc. + +# Access the underlying FastAPI app if needed +fastapi_app = server.get_app() + +# Custom middleware +@server.middleware("http") +async def log_requests(request, call_next): + """Log all API requests.""" + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print(f"{request.method} {request.url} - {response.status_code} ({process_time:.2f}s)") + return response +``` + +### Router Decorators + +jvspatial provides four standard router decorators for API endpoints. These are the ONLY decorators that should be used for routing: + +1. `@endpoint` - For public endpoints (both functions and Walker classes) +2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) +3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) + +```python +# Function endpoint +@endpoint("/api/users", methods=["GET"]) +async def get_users() -> Dict[str, Any]: + users = await User.all() + return {"users": users} + +# Walker endpoint +@endpoint("/api/graph/traverse", methods=["POST"]) +class GraphTraversal(Walker): + pass + +# Authenticated function endpoint +@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) +async def get_admin_stats() -> Dict[str, Any]: + return {"stats": "admin only"} + +# Authenticated walker endpoint (uses same decorator) +@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) +class SecureProcessor(Walker): + pass +``` + +DO NOT use alternative decorators like `@route`, `@server.route`, or `@server.walker`. These are internal or deprecated. + +### API Usage Examples + +Once your server is running, endpoints are automatically available: + +```bash +# Walker endpoint - POST request with parameters +curl -X POST "http://localhost:8000/api/users/process" \ + -H "Content-Type: application/json" \ + -d '{ + "user_name": "John Doe", + "department": "engineering", + "include_inactive": false, + "max_connections": 5, + "start_node": "n:Root:root" + }' + +# Function endpoint - GET request +curl "http://localhost:8000/api/users/count" + +# Function endpoint with path parameters +curl "http://localhost:8000/api/cities/CA" + +# Function endpoint with query parameters +curl "http://localhost:8000/api/users/paginated?page=1&page_size=10&department=engineering" + +# API documentation is automatically available +# http://localhost:8000/docs (Swagger UI) +# http://localhost:8000/redoc (ReDoc) +``` + +### Best Practices + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Use descriptive endpoint paths +@endpoint("/api/users/analyze-connections", methods=["POST"]) +class AnalyzeUserConnections(Walker): + pass + +# Good: Provide comprehensive field documentation +field_name: str = EndpointField( + description="Clear description of what this field does", + examples=["example1", "example2"], + min_length=1, + max_length=100 +) + +# Good: Use appropriate HTTP methods +@endpoint("/api/users", methods=["GET"]) # Retrieve data +@endpoint("/api/users", methods=["POST"]) # Create data +@endpoint("/api/process", methods=["POST"]) # Process/execute + +# Good: Group related fields +config_field: str = EndpointField( + endpoint_group="configuration", + description="Configuration parameter" +) + +# Good: Handle errors appropriately in functions +@endpoint("/api/data") +async def get_data(): + try: + data = await fetch_data() + return {"data": data} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Vague endpoint paths +@endpoint("/process") # Too generic +@endpoint("/api/thing") # Unclear purpose + +# Bad: Missing field documentation +field_name: str = EndpointField() # No description or examples + +# Bad: Exposing internal fields +internal_state: str = EndpointField() # Should use exclude_endpoint=True + +# Bad: Not handling errors in function endpoints +@endpoint("/api/data") +async def get_data(): + data = await risky_operation() # No error handling + return data # Could fail with 500 error + +# Bad: Using wrong HTTP methods +@endpoint("/api/users/delete", methods=["GET"]) # Should be DELETE +@endpoint("/api/data/get", methods=["POST"]) # Should be GET for retrieval +``` + +--- + +## 🏗️ Library Architecture Concepts + +**jvspatial** is an asynchronous, object-spatial Python library designed for building robust persistence and business logic application layers. Inspired by Jaseci's object-spatial paradigm and leveraging Python's async capabilities, jvspatial empowers developers to model complex relationships, traverse object graphs, and implement agent-based architectures that scale with modern cloud-native concurrency requirements. Key capabilities: + +- Typed node/edge modeling via Pydantic +- Precise control over graph traversal +- Multi-backend persistence (JSON, MongoDB, etc.) +- Integrated REST API endpoints +- Async/await architecture + +### Core Entities + +1. **Object** - Base class for all entities with unified query interface; used to store non-graph data +2. **Node** - Extends Object, represents graph nodes with spatial/contextual data; used only as part of a graph +3. **Edge** - Represents relationships between nodes on a graph +4. **Walker** - Implements graph traversal and pathfinding algorithms +5. **GraphContext** - Low-level database interface (use sparingly) + +Once a graph is established (nodes and edges are connected in a meaningful way), a walker may be spawned on the root node or anywhere on the graph. The visit method enacts the walker's traversal using a starting point or a list of nodes on the walker's visit queue. + +As the walker traverses, it may conditionally trigger methods depending on its position on the graph. This is accomplished by the @on_visit annotation on the walker class. Similarly, as the walker traverses over nodes and edges, these entities may conditionally trigger their methods based on the walker's visitation; also accomplished through the @on_visit annotation on the node/edge class. + +**Execution Order**: When a walker visits a node/edge: +1. **Walker hooks** (methods decorated with `@on_visit` on the walker class) execute first +2. **Node/Edge hooks** (methods decorated with `@on_visit` on the node/edge class) are automatically executed after + +Node/Edge hooks are automatically invoked by the walker - no explicit call is needed. The walker binds the hook to the node/edge instance and calls it with the walker as a parameter. + +### Walker Traversal Pattern + +The **recommended approach** for walker traversal is to use the `nodes()` method to get connected nodes for continued traversal. + +#### Naming Convention for @on_visit Methods + +**IMPORTANT**: When writing `@on_visit` decorated methods, use the following naming convention: + +- **`here`**: Parameter name for the visited node/edge (the current location) +- **`visitor`**: Parameter name for the visiting walker (when accessing walker from node context) + +```python path=null start=null +# ✅ RECOMMENDED: Use 'here' for the visited node +@on_visit("User") +async def process_user(self, here: Node): + """Process user node - 'here' represents the current User being visited. + + Args: + here: The visited User node + """ + print(f"Currently visiting user: {here.name}") + connected_users = await here.nodes(node=['User']) + await self.visit(connected_users) + +# ✅ RECOMMENDED: Use 'visitor' when walker is passed to node methods +def node_method_example(self, visitor: Walker): + """Node method that receives the visiting walker. + + Args: + visitor: The walker currently visiting this node + """ + print(f"Being visited by walker: {visitor.__class__.__name__}") + +# ❌ AVOID: Generic parameter names +@on_visit("User") +async def process_user(self, node: Node): # Less clear + pass + +@on_visit("User") +async def process_user(self, n: Node): # Too abbreviated + pass +``` + +**Why this convention?** +- **`here`** clearly indicates the current location in graph traversal +- **`visitor`** clearly indicates the active walker performing the traversal +- Consistent with spatial/navigational metaphors used throughout jvspatial +- Makes code more readable and self-documenting +- Aligns with the library's entity-centric philosophy + +```python +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit, on_exit + +class DataCollector(Walker): + def __init__(self): + super().__init__() + self.collected_data = [] + self.processed_count = 0 + + @on_visit("User") + async def collect_user_data(self, here: Node): + """Called when walker visits a User node. + + Args: + here: The visited User node + """ + self.collected_data.append(here.name) + + # RECOMMENDED: Use nodes() method to get connected nodes + # Default direction="out" follows outgoing edges (forward traversal) + next_nodes = await here.nodes() + await self.visit(next_nodes) + + # Example with semantic filtering - filter connected Users by department + engineering_users = await here.nodes( + node=['User'], # Only User nodes + department="engineering", # Simple kwargs filtering + active=True # Multiple simple filters + ) + await self.visit(engineering_users) + + @on_visit("City") + async def process_city(self, here: Node): + """Process city nodes with filtering and control flow. + + Args: + here: The visited City node + """ + self.processed_count += 1 + + # Skip processing for certain conditions + if here.population < 10000: + print(f"Skipping small city: {here.name}") + self.skip() # Skip to next node in queue + return # This line won't be reached + + # Disengage if we've processed enough + if self.processed_count >= 10: + print("Processed enough cities, disengaging...") + await self.disengage() # Permanently halt and remove from graph + return + + # Continue with normal processing + large_cities = await here.nodes( + node=[{'City': {"context.population": {"$gte": 500_000}}}], + direction="out" # Explicit direction for clarity + ) + await self.visit(large_cities) + + # Example: Mixed filtering approach - dict filters + kwargs + nearby_cities = await here.nodes( + node=[{'City': {"context.region": here.region}}], # Complex filter + state="NY", # Simple kwargs filter + active=True # Additional simple filter + ) + await self.visit(nearby_cities) + + @on_exit + async def cleanup_and_report(self): + """Called when walker completes or disengages.""" + print(f"Walker finished! Collected {len(self.collected_data)} items") + print(f"Processed {self.processed_count} cities") + # Perform cleanup, save results, send notifications, etc. +``` + +### Walker Control Flow Methods + +#### `skip()` - Skip Current Node Processing +The `skip()` method allows you to immediately halt processing of the current node and proceed to the next node in the queue, similar to `continue` in a loop: + +```python +class ConditionalWalker(Walker): + @on_visit("Product") + async def process_product(self, here: Node): + """Process product nodes with conditional skipping. + + Args: + here: The visited Product node + """ + # Skip discontinued products + if here.status == "discontinued": + self.skip() # Jump to next node in queue + # Code below won't execute + + # Skip products outside price range + if not (10 <= here.price <= 1000): + print(f"Skipping {here.name} - price out of range") + self.skip() + + # Normal processing for valid products + print(f"Processing product: {here.name}") + connected_products = await here.nodes(node=['Product']) + await self.visit(connected_products) +``` + +#### `disengage()` - Permanently Halt and Remove Walker from Graph +The `disengage()` method permanently halts the walker and removes it from the graph. Once disengaged, the walker **cannot be resumed** and is considered finished: + +```python +class CompletionWalker(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.max_items = 100 + self.critical_error = False + + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with completion tracking. + + Args: + here: The visited Document node + """ + try: + # Process document + await self.process_item(here) + self.processed_count += 1 + + # Disengage when reaching target or on critical error + if self.processed_count >= self.max_items: + print(f"Target reached: {self.processed_count} items processed") + await self.disengage() # Permanently finish + return + + if self.critical_error: + print("Critical error encountered, disengaging walker") + await self.disengage() # Permanently halt due to error + return + + # Continue to next documents + next_docs = await here.nodes(node=['Document']) + await self.visit(next_docs) + + except CriticalError as e: + print(f"Critical error: {e}") + self.critical_error = True + await self.disengage() # Permanently halt on critical error + + @on_exit + async def final_cleanup(self): + """Called when walker disengages - perform final cleanup.""" + print(f"Walker disengaged. Final count: {self.processed_count}") + # Perform final cleanup, save state, notify completion + await self.save_final_results() + + async def process_item(self, node): + """Process individual item.""" + # Simulate processing that might fail + if node.status == "corrupted": + raise CriticalError("Corrupted data detected") + await asyncio.sleep(0.01) + + async def save_final_results(self): + """Save final processing results.""" + print("💾 Saving final results...") + +# Usage - disengage() creates permanent termination +walker = CompletionWalker() +root = await Root.get(None) + +# Start and run to completion (or error) +walker = await walker.spawn(root) +print(f"Walker finished. Status: {'disengaged' if walker.paused else 'completed'}") + +# NOTE: Once disengaged, walker cannot be resumed +# walker.resume() would not work - walker is permanently finished + +# For pausable/resumable patterns, use different approaches: +# - Save walker state and create new walker instances +# - Use external queue/state management +# - Implement custom pause/resume logic in @on_visit methods +``` + +#### `pause()` and `resume()` - Temporary Walker Suspension +Walkers can be paused during traversal and resumed later, preserving their queue and state. Unlike `disengage()`, paused walkers can be resumed: + +```python +class BatchProcessor(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.batch_size = 50 + self.total_batches = 0 + + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with batch pausing. + + Args: + here: The visited Document node + """ + # Process document + await self.heavy_processing(here) + self.processed_count += 1 + + # Pause after processing a batch + if self.processed_count % self.batch_size == 0: + self.total_batches += 1 + print(f"Batch {self.total_batches} complete ({self.processed_count} items)") + print("Pausing for rate limiting...") + + # Clean pause using pause() method + self.pause(f"Batch {self.total_batches} processing pause") + + # Continue to next documents + next_docs = await here.nodes(node=['Document'], status="active") + await self.visit(next_docs) + + async def heavy_processing(self, node): + """Simulate expensive processing.""" + await asyncio.sleep(0.1) # Simulate API calls, file I/O, etc. + + @on_exit + async def report_completion(self): + """Called when walker completes or is paused.""" + if self.paused: + print(f"Walker paused after processing {self.processed_count} items") + else: + print(f"Walker completed! Total processed: {self.processed_count}") + +# Usage - pause and resume cycle +walker = BatchProcessor() +root = await Root.get(None) + +# Start processing - will pause after first batch +walker = await walker.spawn(root) +print(f"Walker state: {'paused' if walker.paused else 'completed'}") +print(f"Queue remaining: {len(walker.queue)} items") + +# Simulate processing delay (rate limiting, API cooldown, etc.) +print("\n⏳ Waiting for rate limit cooldown...") +await asyncio.sleep(2.0) + +# Resume processing - will continue from where it left off +print("\n▶️ Resuming processing...") +walker = await walker.resume() +print(f"Walker state after resume: {'paused' if walker.paused else 'completed'}") + +# Can resume multiple times if walker gets paused again +while walker.paused and walker.queue: + print(f"\n🔄 Walker paused again, {len(walker.queue)} items remaining") + await asyncio.sleep(1.0) # Brief pause + walker = await walker.resume() + +print("\n✅ All processing complete!") +``` + +#### Advanced Pause/Resume Patterns + +```python +class SmartProcessor(Walker): + def __init__(self): + super().__init__() + self.api_calls = 0 + self.max_api_calls = 100 + self.error_count = 0 + self.max_errors = 5 + + @on_visit("DataNode") + async def process_data(self, here: Node): + """Process data nodes with smart rate limiting. + + Args: + here: The visited DataNode + """ + try: + # Check rate limits + if self.api_calls >= self.max_api_calls: + print(f"Rate limit reached ({self.api_calls} calls), pausing...") + self.api_calls = 0 # Reset counter + self.pause("Rate limit reached") + + # Check error threshold + if self.error_count >= self.max_errors: + print(f"Too many errors ({self.error_count}), pausing for investigation") + self.pause(f"Error threshold reached: {self.error_count} errors") + + # Process node + result = await self.call_external_api(here) + self.api_calls += 1 + + # Continue traversal based on result + if result.should_continue: + related_nodes = await here.nodes( + node=['DataNode'], + status="pending", + priority={"$gte": result.priority_threshold} + ) + await self.visit(related_nodes) + + except ApiError as e: + self.error_count += 1 + print(f"API error #{self.error_count}: {e}") + # Continue processing - don't pause on single errors + + except CriticalError as e: + print(f"Critical error: {e}") + self.pause(f"Critical error: {e}") + + async def call_external_api(self, node): + """Simulate external API call that might fail.""" + await asyncio.sleep(0.05) # Simulate API latency + # Simulate occasional failures + if random.random() < 0.1: # 10% failure rate + raise ApiError("Temporary API failure") + return APIResult(should_continue=True, priority_threshold=5) + + def pause_for_maintenance(self): + """Manually pause walker for maintenance.""" + print("🔧 Pausing for scheduled maintenance...") + self.paused = True + + @on_exit + async def maintenance_check(self): + """Check if maintenance is needed when paused.""" + if self.paused: + print("Walker paused - performing maintenance checks...") + # Reset error counters, clear caches, etc. + self.error_count = 0 + print("Maintenance complete - ready to resume") + +# Usage with external control +walker = SmartProcessor() +root = await Root.get(None) + +# Start processing +print("🚀 Starting smart processing...") +walker = await walker.spawn(root) + +# External monitoring and control +while walker.paused and walker.queue: + print(f"⏸️ Walker paused - {len(walker.queue)} items remaining") + + # Simulate external decision making + if should_resume_processing(): # Your logic here + print("🔄 Conditions met, resuming...") + walker = await walker.resume() + else: + print("⏳ Waiting for conditions to improve...") + await asyncio.sleep(5.0) + +# Helper classes for example +class ApiError(Exception): pass +class CriticalError(Exception): pass +class APIResult: + def __init__(self, should_continue, priority_threshold): + self.should_continue = should_continue + self.priority_threshold = priority_threshold + +def should_resume_processing(): + """External logic to determine if processing should resume.""" + return True # Simplified for example +``` + +#### `@on_exit` - Cleanup and Finalization +The `@on_exit` decorator marks methods to execute when the walker completes traversal or disengages: + +```python +class AnalyticsWalker(Walker): + def __init__(self): + super().__init__() + self.start_time = time.time() + self.nodes_visited = 0 + self.errors_encountered = 0 + + @on_visit("User") + async def analyze_user(self, here: Node): + """Analyze user behavior and traverse to related users. + + Args: + here: The visited User node + """ + try: + self.nodes_visited += 1 + # Perform analysis + await self.analyze_user_behavior(here) + + # Continue traversal + related_users = await here.nodes(node=['User']) + await self.visit(related_users) + except Exception as e: + self.errors_encountered += 1 + print(f"Error processing user {here.id}: {e}") + + @on_exit + async def generate_report(self): + """Generate analytics report when traversal completes.""" + duration = time.time() - self.start_time + print("\n📊 Analytics Report") + print(f"Duration: {duration:.2f} seconds") + print(f"Nodes visited: {self.nodes_visited}") + print(f"Errors: {self.errors_encountered}") + print(f"Success rate: {(1 - self.errors_encountered/max(self.nodes_visited, 1))*100:.1f}%") + + @on_exit + def save_results(self): + """Save results to database (sync version).""" + # Save analytics data + print("💾 Saving results to database...") + + @on_exit + async def send_notifications(self): + """Send completion notifications (async version).""" + # Send email/slack notifications + print("📧 Sending completion notifications...") + + async def analyze_user_behavior(self, user): + """Simulate user behavior analysis.""" + await asyncio.sleep(0.01) # Simulate work +``` + +### Walker Reporting System + +Walkers feature a simplified reporting system that allows you to collect and aggregate any data during traversal. The reporting system eliminates complex nested structures and provides direct access to collected data. + +#### Basic Reporting + +```python path=null start=null +from jvspatial.core import Walker, on_visit, on_exit + +class DataCollectionWalker(Walker): + """Walker demonstrating the simple reporting system.""" + + def __init__(self): + super().__init__() + self.processed_count = 0 + + @on_visit('User') + async def collect_user_data(self, here: Node): + """Collect user data using the report system.""" + # Report any data - strings, dicts, numbers, lists, etc. + self.report({ + "user_processed": { + "id": here.id, + "name": here.name, + "department": here.department, + "timestamp": time.time() + } + }) + + # Report simple values + self.report(f"Processed user: {here.name}") + + # Report lists + if hasattr(here, 'skills'): + self.report(["skills", here.skills]) + + self.processed_count += 1 + + @on_exit + async def generate_summary(self): + """Generate final summary in the report.""" + report_items = await self.get_report() + + self.report({ + "summary": { + "total_items_collected": len(report_items), + "users_processed": self.processed_count, + "collection_complete": True + } + }) + +# Usage +walker = DataCollectionWalker() +result_walker = await walker.spawn() # spawn() returns the walker instance + +# Access collected data directly as a simple list +report = await result_walker.get_report() +print(f"Total items collected: {len(report)}") + +# Iterate through all collected data +for item in report: + if isinstance(item, dict) and "user_processed" in item: + user_data = item["user_processed"] + print(f"User: {user_data['name']} from {user_data['department']}") + elif isinstance(item, str): + print(f"Log: {item}") +``` + +#### Advanced Reporting Patterns + +```python path=null start=null +class AnalyticsWalker(Walker): + """Walker with advanced reporting for analytics.""" + + def __init__(self): + super().__init__() + self.department_counts = {} + self.error_count = 0 + + @on_visit('User') + async def analyze_user(self, here: Node): + """Analyze user and report findings.""" + try: + # Track department statistics + dept = here.department or "unknown" + self.department_counts[dept] = self.department_counts.get(dept, 0) + 1 + + # Report individual analysis + analysis = await self.perform_user_analysis(here) + self.report({ + "user_analysis": { + "user_id": here.id, + "department": dept, + "performance_score": analysis.get("score", 0), + "risk_level": analysis.get("risk", "low"), + "recommendations": analysis.get("recommendations", []) + } + }) + + except Exception as e: + self.error_count += 1 + self.report({ + "error": { + "user_id": here.id, + "error_message": str(e), + "error_type": type(e).__name__ + } + }) + + @on_exit + async def generate_analytics_report(self): + """Generate comprehensive analytics.""" + all_data = await self.get_report() + + # Analyze collected data + user_analyses = [item for item in all_data + if isinstance(item, dict) and "user_analysis" in item] + errors = [item for item in all_data + if isinstance(item, dict) and "error" in item] + + # Calculate metrics + avg_score = sum(ua["user_analysis"]["performance_score"] + for ua in user_analyses) / len(user_analyses) if user_analyses else 0 + + high_risk_users = [ua for ua in user_analyses + if ua["user_analysis"]["risk_level"] == "high"] + + # Report final analytics + self.report({ + "final_analytics": { + "total_users_analyzed": len(user_analyses), + "average_performance_score": round(avg_score, 2), + "department_breakdown": self.department_counts, + "high_risk_users_count": len(high_risk_users), + "error_rate": self.error_count / max(len(user_analyses), 1), + "processing_summary": { + "success": len(user_analyses), + "errors": self.error_count, + "total_items_in_report": len(all_data) + } + } + }) + + async def perform_user_analysis(self, user): + """Simulate user analysis.""" + import random + return { + "score": random.randint(1, 100), + "risk": random.choice(["low", "medium", "high"]), + "recommendations": ["Update profile", "Complete training"] + } +``` + +### Walker Event System + +Walkers can communicate with each other during traversal using an event system. This enables real-time coordination, data sharing, and complex multi-walker workflows. + +#### Basic Event Communication + +```python path=null start=null +from jvspatial.core.events import on_emit + +class AlertWalker(Walker): + """Walker that emits alerts when finding critical issues.""" + + @on_visit('SystemNode') + async def check_system_health(self, here: Node): + """Check system health and emit alerts.""" + if hasattr(here, 'cpu_usage') and here.cpu_usage > 90: + # Emit event to other walkers + await self.emit("high_cpu_alert", { + "node_id": here.id, + "cpu_usage": here.cpu_usage, + "severity": "critical", + "walker_id": self.id + }) + + self.report({"alert_sent": f"High CPU on {here.id}"}) + +class MonitoringWalker(Walker): + """Walker that receives and processes alerts from other walkers.""" + + def __init__(self): + super().__init__() + self.alerts_received = 0 + + @on_emit("high_cpu_alert") + async def handle_cpu_alert(self, event_data): + """Handle high CPU alerts from AlertWalker.""" + self.alerts_received += 1 + + self.report({ + "alert_processed": { + "from_walker": event_data.get("walker_id"), + "node_id": event_data.get("node_id"), + "cpu_usage": event_data.get("cpu_usage"), + "action_taken": "Notification sent to admin", + "handler_id": self.id + } + }) + + # Take action based on alert + await self.send_notification(event_data) + + @on_visit('SystemNode') + async def log_system_visit(self, here: Node): + """Log system node visits.""" + self.report({"system_visited": here.id}) + + async def send_notification(self, alert_data): + """Send notification to administrators.""" + print(f"🚨 ALERT: High CPU {alert_data['cpu_usage']}% on {alert_data['node_id']}") + +# Run multiple walkers concurrently +import asyncio + +alert_walker = AlertWalker() +monitoring_walker = MonitoringWalker() + +# Start both walkers concurrently +tasks = [ + alert_walker.spawn(), + monitoring_walker.spawn() +] + +walkers = await asyncio.gather(*tasks) + +# Check reports from both walkers +alert_report = await alert_walker.get_report() +monitoring_report = await monitoring_walker.get_report() + +print(f"Alerts sent: {len([r for r in alert_report if 'alert_sent' in str(r)])}") +print(f"Alerts processed: {monitoring_walker.alerts_received}") +``` + +#### Advanced Event Patterns + +```python path=null start=null +class DataProcessorWalker(Walker): + """Walker that processes data and emits completion events.""" + + def __init__(self, batch_id: str): + super().__init__() + self.batch_id = batch_id + self.processed_items = 0 + + @on_visit('DataNode') + async def process_data(self, here: Node): + """Process data nodes.""" + # Simulate processing + await asyncio.sleep(0.01) + self.processed_items += 1 + + self.report({"data_processed": here.id}) + + # Emit progress event every 10 items + if self.processed_items % 10 == 0: + await self.emit("batch_progress", { + "batch_id": self.batch_id, + "processed_count": self.processed_items, + "processor_id": self.id + }) + + @on_exit + async def emit_completion(self): + """Emit batch completion event.""" + await self.emit("batch_complete", { + "batch_id": self.batch_id, + "total_processed": self.processed_items, + "processor_id": self.id + }) + + self.report({"batch_completed": self.batch_id}) + +class BatchCoordinator(Walker): + """Walker that coordinates multiple batch processors.""" + + def __init__(self): + super().__init__() + self.batch_progress = {} + self.completed_batches = [] + + @on_emit("batch_progress") + async def track_progress(self, event_data): + """Track progress from batch processors.""" + batch_id = event_data.get("batch_id") + processed_count = event_data.get("processed_count") + + self.batch_progress[batch_id] = processed_count + + self.report({ + "progress_update": { + "batch_id": batch_id, + "items_processed": processed_count, + "coordinator_id": self.id + } + }) + + @on_emit("batch_complete") + async def handle_completion(self, event_data): + """Handle batch completion events.""" + batch_id = event_data.get("batch_id") + total_processed = event_data.get("total_processed") + + self.completed_batches.append(batch_id) + + self.report({ + "batch_completed": { + "batch_id": batch_id, + "total_items": total_processed, + "completed_batches_count": len(self.completed_batches) + } + }) + + # Check if all batches are complete + if len(self.completed_batches) >= 3: # Expecting 3 batches + await self.emit("all_batches_complete", { + "total_batches": len(self.completed_batches), + "coordinator_id": self.id + }) + + @on_emit("all_batches_complete") + async def finalize_processing(self, event_data): + """Finalize when all processing is complete.""" + self.report({ + "processing_finalized": { + "total_batches_completed": event_data.get("total_batches"), + "finalization_time": time.time() + } + }) + +# Example: Run coordinated batch processing +coordinator = BatchCoordinator() +processors = [ + DataProcessorWalker("batch_1"), + DataProcessorWalker("batch_2"), + DataProcessorWalker("batch_3") +] + +# Start all walkers +all_walkers = [coordinator] + processors +tasks = [walker.spawn() for walker in all_walkers] +results = await asyncio.gather(*tasks) + +# Check final reports +for walker in all_walkers: + report = await walker.get_report() + print(f"Walker {walker.id}: {len(report)} items in report") +``` + +#### Key Reporting & Event Features + +**Reporting System:** +- `walker.report(any_data)` - Add any data to walker's report +- `await walker.get_report()` - Get simple list of all reported items (async) +- No complex nested structures - direct access to your data +- Support for any data type (strings, dicts, lists, numbers, etc.) + +**Event System:** +- `await walker.emit(event_name, payload)` - Send events to other walkers +- `@on_emit(event_name)` - Handle specific events +- Multiple walkers can receive the same event +- Events enable real-time coordination between concurrent walkers +- Both Walkers and Nodes can use `@on_emit` decorators + +**Best Practices:** +- Use `self.report()` to add data, never return values from decorated methods +- Access reports after traversal: `report = await walker.get_report()` +- Use events for walker-to-walker communication during traversal +- Filter reported data by checking item structure/content +- Leverage `@on_exit` hooks for final summaries and cleanup + +### Walker Trail Tracking + +Walkers include built-in **trail tracking** capabilities to monitor and record the complete path taken during graph traversal. This is invaluable for debugging, analytics, audit trails, and optimizing traversal strategies. + +#### Basic Trail Tracking + +```python path=null start=null +from jvspatial.core import Walker, on_visit + +class TrailTrackingWalker(Walker): + def __init__(self): + super().__init__() + # Enable trail tracking with memory management + self.trail_enabled = True + self.max_trail_length = 100 # Keep last 100 steps (0 = unlimited) + + @on_visit('User') + async def process_user_with_trail(self, here: Node): + """Process user nodes while tracking the traversal trail. + + Args: + here: The visited User node + """ + print(f"Processing user: {here.name}") + + # Access current trail information + current_trail = self.get_trail() # List of node IDs + trail_length = self.get_trail_length() # Current trail size + recent_steps = self.get_recent_trail(count=3) # Last 3 steps + + print(f"Trail length: {trail_length}, Recent: {recent_steps}") + + # Avoid revisiting recently visited nodes + if here.id in recent_steps[:-1]: # Exclude current node + print(f"Recently visited {here.name}, skipping deeper traversal") + self.skip() + + # Continue normal traversal + colleagues = await here.nodes( + node=['User'], + department=here.department, + active=True + ) + await self.visit(colleagues) + + @on_exit + async def generate_trail_report(self): + """Generate comprehensive trail analysis report.""" + # Get trail with actual node objects (database lookups) + trail_nodes = await self.get_trail_nodes() + + # Get complete path with connecting edges + trail_path = await self.get_trail_path() + + # Generate detailed report using report() method + trail_report = { + 'summary': { + 'total_steps': self.get_trail_length(), + 'unique_nodes': len(set(self.get_trail())), + 'efficiency_ratio': len(set(self.get_trail())) / max(self.get_trail_length(), 1) + }, + 'visited_nodes': [ + {'step': i+1, 'node_type': node.__class__.__name__, 'node_name': getattr(node, 'name', node.id)} + for i, node in enumerate(trail_nodes) + ], + 'path_analysis': [ + { + 'step': i+1, + 'node': node.name if hasattr(node, 'name') else node.id, + 'via_edge': edge.edge_type if edge else 'start' + } + for i, (node, edge) in enumerate(trail_path) + ] + } + + # Report the trail data + self.report(trail_report) + + print(f"\n📊 Trail Report Generated:") + print(f" - Total steps: {trail_report['summary']['total_steps']}") + print(f" - Unique nodes: {trail_report['summary']['unique_nodes']}") + print(f" - Path efficiency: {trail_report['summary']['efficiency_ratio']:.2%}") + +# Usage example +walker = TrailTrackingWalker() +root = await Root.get(None) +await walker.spawn(root) + +# Access trail data +final_trail = walker.get_trail() +print(f"Final trail: {final_trail}") +# Access the trail report from walker's collected data +report = await walker.get_report() +trail_reports = [item for item in report if isinstance(item, dict) and 'trail_report' in str(item)] +print(f"Trail report: {trail_reports[0] if trail_reports else 'No trail report found'}") +``` + +#### Advanced Trail Use Cases + +```python path=null start=null +class AdvancedTrailWalker(Walker): + def __init__(self): + super().__init__() + self.trail_enabled = True + self.max_trail_length = 0 # Unlimited for comprehensive analysis + self.visited_nodes = set() # For cycle detection + self.performance_metrics = [] + + @on_visit('Document') + async def process_with_cycle_detection(self, here: Node): + """Process documents with cycle detection using trail data. + + Args: + here: The visited Document node + """ + import time + start_time = time.time() + + # Cycle detection using trail + if here.id in self.visited_nodes: + trail = self.get_trail() + first_visit_index = trail.index(here.id) + cycle_length = len(trail) - first_visit_index - 1 + + print(f"🔄 Cycle detected at {here.id}! Length: {cycle_length} steps") + + self.report({ + 'cycle_detected': { + 'node_id': here.id, + 'cycle_length': cycle_length, + 'first_visit_step': first_visit_index + 1, + 'detection_step': len(trail) + } + }) + + # Stop to avoid infinite loop + await self.disengage() + return + + self.visited_nodes.add(here.id) + + # Process document + await self.analyze_document(here) + + # Record performance metrics + processing_time = time.time() - start_time + self.performance_metrics.append({ + 'node_id': here.id, + 'processing_time': processing_time, + 'step_number': self.get_trail_length(), + 'metadata': self.get_trail_metadata() # Get current step metadata + }) + + # Continue traversal with trail-aware filtering + related_docs = await here.nodes( + node=['Document'], + status='active' + ) + + # Filter out recently visited to avoid cycles + recent_trail = self.get_recent_trail(count=10) + unvisited_docs = [doc for doc in related_docs if doc.id not in recent_trail] + + if unvisited_docs: + await self.visit(unvisited_docs) + else: + print("All related documents recently visited, exploring alternatives") + + @on_visit('User') + async def audit_user_access(self, here: Node): + """Create audit trail for user access. + + Args: + here: The visited User node + """ + # Get current trail metadata (automatically includes timestamp, node_type, queue_length) + metadata = self.get_trail_metadata() + + audit_entry = { + 'timestamp': metadata.get('timestamp'), + 'action': 'USER_ACCESS', + 'user_id': here.id, + 'user_name': getattr(here, 'name', 'Unknown'), + 'trail_step': self.get_trail_length(), + 'access_context': { + 'queue_size': metadata.get('queue_length'), + 'node_type': metadata.get('node_type'), + 'previous_steps': self.get_recent_trail(count=3)[:-1] # Exclude current + } + } + + self.report({'audit_entry': audit_entry}) + print(f"📝 Audit: Accessed user {here.id} at step {audit_entry['trail_step']}") + + @on_exit + async def comprehensive_analysis(self): + """Generate comprehensive trail and performance analysis.""" + trail_path = await self.get_trail_path() + + # Performance analysis + avg_processing_time = sum(m['processing_time'] for m in self.performance_metrics) / len(self.performance_metrics) if self.performance_metrics else 0 + + # Path efficiency analysis + total_steps = self.get_trail_length() + unique_nodes = len(set(self.get_trail())) + + # Get report once for all analysis + report = await self.get_report() + + comprehensive_analysis = { + 'trail_summary': { + 'total_steps': total_steps, + 'unique_nodes_visited': unique_nodes, + 'path_efficiency': unique_nodes / total_steps if total_steps > 0 else 0, + 'cycles_detected': len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item]), + 'trail_enabled': self.trail_enabled, + 'trail_limit': self.max_trail_length + }, + 'performance_metrics': { + 'avg_processing_time': avg_processing_time, + 'total_processing_time': sum(m['processing_time'] for m in self.performance_metrics), + 'slowest_step': max(self.performance_metrics, key=lambda x: x['processing_time']) if self.performance_metrics else None + }, + 'audit_summary': { + 'total_audit_entries': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item]), + 'user_accesses': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item and item.get('audit_entry', {}).get('action') == 'USER_ACCESS']) + } + } + + # Report the comprehensive analysis + self.report(comprehensive_analysis) + + print("\n📈 Comprehensive Analysis Complete:") + print(f" - Path efficiency: {comprehensive_analysis['trail_summary']['path_efficiency']:.2%}") + print(f" - Average processing time: {avg_processing_time:.3f}s") + print(f" - Cycles detected: {len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item])}") + + async def analyze_document(self, doc): + """Simulate document analysis.""" + import asyncio + await asyncio.sleep(0.02) # Simulate processing time + +# Usage with trail management +walker = AdvancedTrailWalker() + +# Enable debug mode for detailed trail information +walker.debug_mode = True + +# Spawn and run analysis +root = await Root.get(None) +await walker.spawn(root) + +# Access comprehensive results from walker's report +report = await walker.get_report() +analysis = next((item for item in report if isinstance(item, dict) and 'trail_summary' in item), {}) +cycles = [item for item in report if isinstance(item, dict) and 'cycle_detected' in item] +audit_entries = [item for item in report if isinstance(item, dict) and 'audit_entry' in item] + +print(f"Analysis complete. Efficiency: {analysis.get('trail_summary', {}).get('path_efficiency', 0):.2%}") +print(f"Cycles detected: {len(cycles)}, Audit entries: {len(audit_entries)}") +``` + +#### Trail API Quick Reference + +**Configuration (Read/Write):** +- `self.trail_enabled = True` - Enable trail tracking +- `self.max_trail_length = N` - Limit trail to N steps (0 = unlimited) + +**Trail Data (Read-Only Properties):** +- `self.trail` - List of visited node IDs (returns copy) +- `self.trail_edges` - Edge IDs between nodes (returns copy) +- `self.trail_metadata` - Metadata per step (returns deep copy) + +**Trail Access Methods (O(1) operations):** +- `self.get_trail()` - Get list of visited node IDs +- `self.get_trail_length()` - Get current trail length +- `self.get_recent_trail(count=N)` - Get last N trail steps +- `self.clear_trail()` - Clear entire trail (only way to modify trail) + +**Advanced Access (Database operations):** +- `await self.get_trail_nodes()` - Get actual Node objects from trail +- `await self.get_trail_path()` - Get trail with connecting edges +- `self.get_trail_metadata(step=-1)` - Get metadata for specific step + +**Use Cases:** +- **Debugging**: Track walker path for troubleshooting +- **Cycle Detection**: Avoid infinite loops in graph traversal +- **Performance Analysis**: Measure processing time per step +- **Audit Trails**: Comprehensive access logging for compliance +- **Path Optimization**: Analyze and improve traversal efficiency + +### Walker Queue Manipulation Methods + +Walkers maintain an internal queue (deque) of nodes to visit during traversal. Advanced queue manipulation provides fine-grained control over traversal order, priority handling, and dynamic path planning. These methods allow you to programmatically manage the walker's traversal queue: + +#### Core Queue Methods + +```python path=null start=null +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit +from typing import List, Optional + +class QueueMasterWalker(Walker): + def __init__(self): + super().__init__() + self.priority_nodes = [] + self.deferred_nodes = [] + self.processed_count = 0 + + @on_visit("TaskNode") + async def process_task(self, here: Node): + """Demonstrate queue manipulation methods. + + Args: + here: The visited TaskNode + """ + + # 1. INSPECT QUEUE STATE + current_queue = self.get_queue() # Get queue as list + print(f"Current queue size: {len(current_queue)}") + print(f"Queue contents: {[n.name for n in current_queue]}") + + # Check if specific node is queued + if current_queue: + next_node = current_queue[0] # Peek at next node + print(f"Next node to process: {next_node.name}") + + # 2. ADD NODES TO QUEUE + # Find high-priority nodes to add immediately + urgent_tasks = await here.nodes( + node=['TaskNode'], + priority={"$gte": 9}, + status="pending" + ) + + # Add to front of queue (high priority) + if urgent_tasks: + added = self.prepend(urgent_tasks) # Add to front + print(f"Added {len(added)} urgent tasks to front of queue") + + # Find normal tasks to add to end + normal_tasks = await here.nodes( + node=['TaskNode'], + priority={"$lt": 9, "$gte": 5}, + status="pending" + ) + + # Add to end of queue (normal priority) + if normal_tasks: + added = self.append(normal_tasks) # Add to end + print(f"Added {len(added)} normal tasks to end of queue") + + # Alternative: Use visit() method (equivalent to append) + additional_tasks = await here.nodes(node=['TaskNode'], status="new") + if additional_tasks: + self.visit(additional_tasks) # Same as append() + + # Add nodes right after current processing + immediate_tasks = await here.nodes( + node=['TaskNode'], + priority=10, # Highest priority + status="urgent" + ) + if immediate_tasks: + added = self.add_next(immediate_tasks) # Add next in queue + print(f"Added {len(added)} tasks to process immediately next") + + # 3. CONDITIONAL QUEUE MANIPULATION + # Check if we have too many items in queue + if len(self.get_queue()) > 100: + print("Queue overflow detected, deferring low-priority items") + + # Get current queue and filter it + current_queue = self.get_queue() + low_priority = [] + high_priority = [] + + for item in current_queue: + if hasattr(item, 'priority') and item.priority < 5: + low_priority.append(item) + else: + high_priority.append(item) + + # Clear queue and rebuild with high priority items only + self.clear_queue() + if high_priority: + self.append(high_priority) + + # Store deferred items for later + self.deferred_nodes.extend(low_priority) + print(f"Deferred {len(low_priority)} low-priority items") + + # 4. TARGETED QUEUE MANIPULATION + # Remove specific completed nodes from queue + completed_nodes = await here.nodes( + node=['TaskNode'], + status="completed" + ) + + for completed in completed_nodes: + if self.is_queued(completed): + removed = self.dequeue(completed) + print(f"Removed {len(removed)} completed tasks from queue") + + # 5. PRECISE QUEUE INSERTION + # Find a specific node in queue to insert after + checkpoint_tasks = await here.nodes( + node=['TaskNode'], + task_type="checkpoint" + ) + + followup_tasks = await here.nodes( + node=['TaskNode'], + depends_on=here.id + ) + + for checkpoint in checkpoint_tasks: + if self.is_queued(checkpoint) and followup_tasks: + try: + # Insert followup tasks right after checkpoint + inserted = self.insert_after(checkpoint, followup_tasks) + print(f"Inserted {len(inserted)} followup tasks after checkpoint") + except ValueError as e: + print(f"Could not insert after checkpoint: {e}") + + self.processed_count += 1 + + @on_visit("CompletionNode") + async def handle_completion(self, here: Node): + """Handle task completion and queue cleanup. + + Args: + here: The visited CompletionNode + """ + + # Add back any deferred nodes if queue is manageable + current_queue_size = len(self.get_queue()) + if current_queue_size < 20 and self.deferred_nodes: + reactivated = self.deferred_nodes[:10] # Add up to 10 back + self.deferred_nodes = self.deferred_nodes[10:] # Remove from deferred + + self.append(reactivated) + print(f"Reactivated {len(reactivated)} deferred nodes") + + # Insert critical tasks right at the beginning + critical_tasks = await here.nodes( + node=['TaskNode'], + priority=10, + status="critical" + ) + + if critical_tasks: + # Find first non-critical task and insert before it + queue = self.get_queue() + for i, queued_node in enumerate(queue): + if hasattr(queued_node, 'priority') and queued_node.priority < 10: + try: + inserted = self.insert_before(queued_node, critical_tasks) + print(f"Inserted {len(inserted)} critical tasks before normal task") + break + except ValueError: + # If insertion fails, just prepend + self.prepend(critical_tasks) + break + + @on_exit + async def final_report(self): + """Report final queue statistics.""" + final_queue = self.get_queue() + print(f"\n📊 Queue Processing Complete") + print(f"Total processed: {self.processed_count}") + print(f"Remaining in queue: {len(final_queue)}") + print(f"Deferred nodes: {len(self.deferred_nodes)}") + + if final_queue: + print("Remaining nodes:") + for node in final_queue[:5]: # Show first 5 + print(f" - {node.name}") + if len(final_queue) > 5: + print(f" ... and {len(final_queue) - 5} more") +``` + +#### Queue Method Reference + +Based on the actual Walker implementation in jvspatial: + +**Basic Queue Operations:** +- `self.visit(nodes)` - Add nodes to end of queue (equivalent to append) +- `self.append(nodes)` - Add nodes to end of queue +- `self.prepend(nodes)` - Add nodes to front of queue +- `self.add_next(nodes)` - Add nodes next in queue after current item +- `self.get_queue()` - Return entire queue as a list +- `self.clear_queue()` - Clear all nodes from queue +- `self.is_queued(node)` - Check if specific node is in queue + +**Advanced Queue Operations:** +- `self.dequeue(nodes)` - Remove specific nodes from queue +- `self.insert_after(target_node, nodes)` - Insert nodes after target node +- `self.insert_before(target_node, nodes)` - Insert nodes before target node + +#### Priority-Based Queue Management + +```python path=null start=null +class PriorityQueueWalker(Walker): + def __init__(self): + super().__init__() + self.priority_buckets = { + 'urgent': [], # Priority 9-10 + 'high': [], # Priority 7-8 + 'normal': [], # Priority 4-6 + 'low': [] # Priority 1-3 + } + + @on_visit("WorkItem") + async def process_work_item(self, here: Node): + """Process work items with priority-based queuing. + + Args: + here: The visited WorkItem node + """ + + # Get connected work items + connected_items = await here.nodes( + node=['WorkItem'], + status="pending" + ) + + if connected_items: + # Sort into priority buckets + self._sort_into_priority_buckets(connected_items) + + # Process highest priority items first + self._add_by_priority_order() + + def _sort_into_priority_buckets(self, nodes: List[Node]): + """Sort nodes into priority-based buckets.""" + for node in nodes: + priority = getattr(node, 'priority', 5) + + if priority >= 9: + self.priority_buckets['urgent'].append(node) + elif priority >= 7: + self.priority_buckets['high'].append(node) + elif priority >= 4: + self.priority_buckets['normal'].append(node) + else: + self.priority_buckets['low'].append(node) + + print(f"Sorted into buckets: " + f"urgent={len(self.priority_buckets['urgent'])}, " + f"high={len(self.priority_buckets['high'])}, " + f"normal={len(self.priority_buckets['normal'])}, " + f"low={len(self.priority_buckets['low'])}") + + def _add_by_priority_order(self): + """Add nodes to walker queue in priority order.""" + # Process urgent items first (add to front) + if self.priority_buckets['urgent']: + self.prepend(self.priority_buckets['urgent']) + self.priority_buckets['urgent'].clear() + + # Add high priority items to front (after urgent) + if self.priority_buckets['high']: + # Insert at beginning but after urgent items + current_queue = self.get_queue() + if current_queue: + # Find first non-urgent item and insert before it + high_items = self.priority_buckets['high'] + self.priority_buckets['high'].clear() + + try: + # Try to find insertion point + first_non_urgent = None + for node in current_queue: + if hasattr(node, 'priority') and node.priority < 9: + first_non_urgent = node + break + + if first_non_urgent: + self.insert_before(first_non_urgent, high_items) + else: + self.append(high_items) # All items are urgent + + except ValueError: + # Fallback to prepend if insertion fails + self.prepend(high_items) + else: + self.prepend(self.priority_buckets['high']) + self.priority_buckets['high'].clear() + + # Add normal priority items to end + if self.priority_buckets['normal']: + self.append(self.priority_buckets['normal']) + self.priority_buckets['normal'].clear() + + # Only add low priority if queue is small + current_queue_size = len(self.get_queue()) + if current_queue_size < 50 and self.priority_buckets['low']: + low_items = self.priority_buckets['low'][:10] # Limit low priority + self.priority_buckets['low'] = self.priority_buckets['low'][10:] + self.append(low_items) +``` + +#### Dynamic Queue Filtering and Manipulation + +```python path=null start=null +class AdaptiveQueueWalker(Walker): + def __init__(self): + super().__init__() + self.queue_stats = { + 'added': 0, + 'removed': 0, + 'filtered': 0, + 'reordered': 0 + } + + @on_visit("FilterNode") + async def adaptive_filtering(self, here: Node): + """Demonstrate dynamic queue filtering and manipulation. + + Args: + here: The visited FilterNode + """ + + # Add new nodes based on current context + candidates = await here.nodes( + node=['ProcessNode'], + active=True + ) + + if candidates: + # Filter candidates before adding to queue + filtered_candidates = self._filter_candidates(candidates) + + if filtered_candidates: + # Smart queue insertion based on current load + current_queue_size = len(self.get_queue()) + + if current_queue_size < 20: + # Low load: add all to end + self.append(filtered_candidates) + self.queue_stats['added'] += len(filtered_candidates) + else: + # High load: add only high-priority to front + high_priority = [ + c for c in filtered_candidates + if getattr(c, 'priority', 0) >= 8 + ] + if high_priority: + self.prepend(high_priority) + self.queue_stats['added'] += len(high_priority) + + # Periodic queue maintenance + if hasattr(here, 'name') and here.name.endswith('_maintenance'): + self._perform_queue_maintenance() + + def _filter_candidates(self, candidates: List[Node]) -> List[Node]: + """Filter candidates based on various criteria.""" + filtered = [] + current_queue = self.get_queue() + + for candidate in candidates: + # Check if already in queue (avoid duplicates) + if self.is_queued(candidate): + continue + + # Check resource constraints (mock implementation) + if hasattr(candidate, 'resource_requirement'): + if candidate.resource_requirement > self._get_available_resources(): + continue + + # Check dependencies (mock implementation) + if hasattr(candidate, 'dependencies'): + if not self._dependencies_met(candidate.dependencies): + continue + + filtered.append(candidate) + + self.queue_stats['filtered'] += len(candidates) - len(filtered) + return filtered + + def _perform_queue_maintenance(self): + """Perform queue cleanup and optimization.""" + current_queue = self.get_queue() + if not current_queue: + return + + print("🔧 Performing queue maintenance...") + + # 1. Remove stale items (mock - would check expiration) + non_stale = [] + removed_stale = 0 + + for item in current_queue: + if hasattr(item, 'expires_at'): + # Mock expiration check + if not getattr(item, 'is_expired', False): + non_stale.append(item) + else: + removed_stale += 1 + else: + non_stale.append(item) + + # 2. Deduplicate items by ID + seen_ids = set() + deduplicated = [] + removed_duplicates = 0 + + for item in non_stale: + if item.id not in seen_ids: + seen_ids.add(item.id) + deduplicated.append(item) + else: + removed_duplicates += 1 + + # 3. Reorder by priority + optimized = sorted( + deduplicated, + key=lambda x: getattr(x, 'priority', 0), + reverse=True + ) + + # 4. Rebuild queue with optimized order + self.clear_queue() + if optimized: + self.append(optimized) + + self.queue_stats['removed'] += removed_stale + removed_duplicates + self.queue_stats['reordered'] += 1 + + print(f"Maintenance complete: removed {removed_stale} stale, " + f"{removed_duplicates} duplicates, optimized {len(optimized)} items") + + def _get_available_resources(self) -> int: + """Mock implementation - get available system resources.""" + return 100 + + def _dependencies_met(self, dependencies: List[str]) -> bool: + """Mock implementation - check if dependencies are satisfied.""" + return True + + @on_exit + async def report_queue_stats(self): + """Report queue manipulation statistics.""" + print("\n📈 Queue Statistics:") + for stat, value in self.queue_stats.items(): + print(f" {stat.capitalize()}: {value}") +``` + +#### Best Practices for Queue Manipulation + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Check queue state before manipulation +current_queue = self.get_queue() +if current_queue: + next_item = current_queue[0] # Look at next item + # Make decisions based on next item + +# Good: Use appropriate method for insertion priority +if item.priority >= 8: + self.prepend([item]) # High priority to front +else: + self.append([item]) # Normal priority to end + +# Good: Check if node is queued before operations +if self.is_queued(completed_node): + self.dequeue(completed_node) + +# Good: Batch queue operations for efficiency +new_items = await node.nodes(filters) +if new_items: + self.append(new_items) # Add all at once + +# Good: Safe queue iteration and modification +current_queue = self.get_queue() # Get snapshot +filtered_items = [n for n in current_queue if meets_criteria(n)] +self.clear_queue() +if filtered_items: + self.append(filtered_items) + +# Good: Precise insertion with error handling +try: + self.insert_after(target_node, new_nodes) +except ValueError: + # Target not found, use alternative + self.prepend(new_nodes) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Modifying queue during iteration +for item in self.get_queue(): # Don't iterate over changing queue + if condition: + self.dequeue(item) # Modifying during iteration + +# Bad: Not handling insertion errors +self.insert_after(target_node, nodes) # Could raise ValueError + +# Bad: Inefficient repeated operations +for item in items: + self.append([item]) # Many small operations +# Better: self.append(items) # Single batch operation + +# Bad: Not checking queue state +first_item = self.get_queue()[0] # Could cause IndexError if empty + +# Bad: Assuming nodes are still queued +self.dequeue(node) # Node might not be in queue +# Better: if self.is_queued(node): self.dequeue(node) +``` + +**Key Points:** +- Use `await node.nodes()` to get connected nodes for traversal (NOT `get_edges()`) +- Default `direction="out"` follows outgoing edges (recommended for forward traversal) +- Use `direction="in"` for reverse traversal along incoming edges +- Use `direction="both"` for bidirectional traversal +- **Semantic Filtering Approaches:** + - **Simple filtering**: Use kwargs for connected node properties: `state="NY"` + - **Complex node filtering**: `node=[{'City': {"context.population": {"$gte": 500000}}}]` + - **Complex edge filtering**: `edge=[{'Highway': {"context.condition": {"$ne": "poor"}}}]` + - **Mixed approaches**: Combine dict filters with kwargs for maximum flexibility +- **Database-Level Optimization**: All filtering happens at database level for performance +- **MongoDB-Style Operators**: `$gte`, `$lt`, `$ne`, `$in`, `$nin`, `$regex`, etc. +- **Walker Control Flow:** + - **`skip()`**: Skip current node processing, continue to next (like `continue` in loops) + - **`pause()`/`resume()`**: Temporarily pause walker (use `self.pause()`), can be resumed later + - **`disengage()`**: Permanently halt walker and remove from graph (cannot be resumed) + - **`@on_exit`**: Methods called when walker completes, pauses, or disengages (cleanup) +- The `nodes()` method returns a list that can be directly passed to `walker.visit()` + +### Inheritance Hierarchy + +``` +Object (base class with unified query interface) +├── Node (spatial graph nodes) +├── Edge (relationships) +└── Custom entities (inherit from Node/Object) +``` + +### Database Backends + +- **JSONDatabase** - File-based storage for development/testing +- **MongoDatabase** - MongoDB backend for production +- **Custom databases** - Implement DatabaseInterface + +## 📚 Documentation Maintenance + +### README Updates + +When adding new features: + +1. **Review existing README.md** +2. **Update feature list** if adding major functionality +3. **Update installation/setup** if dependencies change +4. **Update usage examples** to reflect new capabilities +5. **Maintain consistency** with existing documentation style + +### Documentation Directory + +Always review and update `docs/` directory: + +``` +docs/ +├── api/ # API reference documentation +├── guides/ # User guides and tutorials +├── architecture/ # System design documents +└── examples/ # Code examples and tutorials +``` + +**Update procedure:** +1. Check if new feature requires new documentation files +2. Update existing API documentation for modified classes/methods +3. Add user guides for complex features +4. Update architecture docs if design patterns change + +## 💡 Examples Maintenance + +### Example Directory Structure + +``` +examples/ +├── basic/ # Simple usage examples +├── advanced/ # Complex scenarios +├── query_interface_example.py # Comprehensive entity-centric CRUD operations +├── semantic_filtering.py # Advanced semantic filtering with Node.nodes() +└── migration/ # Migration guides (if needed) +``` + +### Example Update Procedure + +1. **Review existing examples** for relevance to new features +2. **Update outdated examples** to use modern entity-centric syntax +3. **Create new examples** for significant new features +4. **Ensure examples are runnable** and well-documented +5. **Remove or archive obsolete examples** +6. **Update key reference examples:** + - `query_interface_example.py` - Showcase latest entity-centric patterns + - `semantic_filtering.py` - Demonstrate advanced Node.nodes() filtering + - Basic examples - Keep simple, focused, and beginner-friendly + - Advanced examples - Show complex real-world scenarios + +### Example Code Standards + +Examples should: +- Use entity-centric syntax exclusively +- Include comprehensive comments +- Demonstrate best practices +- Be self-contained and runnable +- Show error handling patterns + +## 🧪 Testing Requirements + +### Test Organization + +``` +tests/ +├── api/ # FastAPI endpoint tests +├── core/ # Core entity and logic tests +├── db/ # Database backend tests +└── integration/ # End-to-end integration tests +``` + +### Testing Procedure for New Features + +1. **Unit Tests** - Test individual methods and classes +2. **Integration Tests** - Test feature interaction with database +3. **API Tests** - Test HTTP endpoints if applicable +4. **Performance Tests** - For database-heavy features + +### Test Code Standards + +```python +import pytest +from typing import List +from jvspatial.core import Node, Walker, Edge +from jvspatial.core.entities import on_visit +from jvspatial.exceptions import NodeNotFoundError, ValidationError + +class TestUser(Node): + name: str = "" + email: str = "" + department: str = "" + age: int = 0 + active: bool = True + skills: List[str] = [] + +class TestDepartment(Node): + name: str = "" + location: str = "" + budget: int = 0 + +class TestWalker(Walker): + def __init__(self): + super().__init__() + self.visited_users = [] + + @on_visit("TestUser") + async def visit_user(self, node: TestUser): + self.visited_users.append(node.name) + # Test semantic filtering in walker + connected_users = await node.nodes( + node=['TestUser'], + department=node.department, + active=True + ) + await self.visit(connected_users) + +# Entity Creation Tests +@pytest.mark.asyncio +async def test_user_creation(): + """Test entity-centric user creation with full data.""" + user = await TestUser.create( + name="Alice Johnson", + email="alice@company.com", + department="engineering", + age=30, + skills=["python", "javascript"] + ) + assert user.name == "Alice Johnson" + assert user.email == "alice@company.com" + assert user.department == "engineering" + assert user.id is not None + assert "python" in user.skills + +# MongoDB-Style Query Tests +@pytest.mark.asyncio +async def test_mongodb_queries(): + """Test comprehensive MongoDB-style queries.""" + # Setup test data + await TestUser.create(name="Bob Smith", email="bob@test.com", age=25, department="engineering") + await TestUser.create(name="Carol Davis", email="carol@test.com", age=35, department="marketing") + await TestUser.create(name="David Brown", email="david@test.com", age=45, department="engineering") + + # Test regex queries + b_users = await TestUser.find({"context.name": {"$regex": "^B", "$options": "i"}}) + assert len(b_users) >= 1 + assert any(u.name.startswith("B") for u in b_users) + + # Test comparison operators + senior_users = await TestUser.find({"context.age": {"$gte": 35}}) + assert all(u.age >= 35 for u in senior_users) + + # Test logical operators + senior_engineers = await TestUser.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.age": {"$gte": 30}} + ] + }) + assert all(u.department == "engineering" and u.age >= 30 for u in senior_engineers) + + # Test array operations + tech_users = await TestUser.find({"context.skills": {"$in": ["python", "javascript"]}}) + assert all(any(skill in u.skills for skill in ["python", "javascript"]) for u in tech_users) + +# Semantic Filtering Tests +@pytest.mark.asyncio +async def test_semantic_filtering(): + """Test Node.nodes() semantic filtering capabilities.""" + # Create test nodes and edges + dept = await TestDepartment.create(name="Engineering", location="SF") + user1 = await TestUser.create(name="Alice", department="engineering", active=True) + user2 = await TestUser.create(name="Bob", department="engineering", active=False) + user3 = await TestUser.create(name="Carol", department="marketing", active=True) + + # Create connections + await Edge.create(source=dept, target=user1, edge_type="employs") + await Edge.create(source=dept, target=user2, edge_type="employs") + await Edge.create(source=user1, target=user3, edge_type="collaborates") + + # Test simple filtering + active_employees = await dept.nodes( + node=['TestUser'], + active=True + ) + assert len(active_employees) == 1 + assert active_employees[0].name == "Alice" + + # Test complex filtering + engineering_employees = await dept.nodes( + node=[{'TestUser': {"context.department": "engineering"}}], + active=True + ) + assert all(u.department == "engineering" and u.active for u in engineering_employees) + + # Test mixed filtering + collaborators = await user1.nodes( + node=[{'TestUser': {"context.active": True}}], + department="marketing" + ) + assert len(collaborators) >= 1 + +# Walker Tests +@pytest.mark.asyncio +async def test_walker_traversal(): + """Test walker with semantic filtering.""" + # Setup graph + root = await TestUser.create(name="Root", department="engineering") + user1 = await TestUser.create(name="User1", department="engineering", active=True) + user2 = await TestUser.create(name="User2", department="marketing", active=True) + + await Edge.create(source=root, target=user1, edge_type="manages") + await Edge.create(source=root, target=user2, edge_type="manages") + await Edge.create(source=user1, target=user2, edge_type="collaborates") + + # Test walker + walker = TestWalker() + await walker.spawn(root) + + # Verify walker visited users + assert "Root" in walker.visited_users + assert len(walker.visited_users) >= 1 + +# Error Handling Tests +@pytest.mark.asyncio +async def test_error_handling(): + """Test proper error handling patterns.""" + # Test NodeNotFoundError + with pytest.raises(NodeNotFoundError): + await TestUser.get("nonexistent-id") + + # Test safe retrieval + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await TestUser.find({"context.email": "nonexistent@test.com"}) + user = users[0] if users else None + assert user is None + + # Test validation errors (if validation is implemented) + with pytest.raises((ValidationError, ValueError)): + await TestUser.create(name="", email="invalid-email") + +# Performance Tests +@pytest.mark.asyncio +async def test_bulk_operations(): + """Test bulk operations performance.""" + # Create multiple users + users_data = [ + {"name": f"User{i}", "email": f"user{i}@test.com", "department": "engineering"} + for i in range(10) + ] + + # Bulk create + users = [] + for user_data in users_data: + user = await TestUser.create(**user_data) + users.append(user) + + assert len(users) == 10 + + # Bulk query + engineering_users = await TestUser.find({"context.department": "engineering"}) + assert len(engineering_users) >= 10 + + # Bulk update + for user in users[:5]: + user.active = False + await user.save() + + # Verify updates + inactive_users = await TestUser.find({"context.active": False}) + assert len(inactive_users) >= 5 + +# Integration Tests +@pytest.mark.asyncio +async def test_full_workflow(): + """Test complete entity lifecycle with relationships.""" + # Create department + dept = await TestDepartment.create(name="Product", location="NYC", budget=1000000) + + # Create users + manager = await TestUser.create( + name="Jane Manager", + email="jane@company.com", + department="product", + age=40 + ) + + employee = await TestUser.create( + name="John Employee", + email="john@company.com", + department="product", + age=28 + ) + + # Create relationships + await Edge.create(source=dept, target=manager, edge_type="employs") + await Edge.create(source=dept, target=employee, edge_type="employs") + await Edge.create(source=manager, target=employee, edge_type="manages") + + # Test traversal + department_employees = await dept.nodes(node=['TestUser']) + assert len(department_employees) == 2 + + managed_employees = await manager.nodes( + node=['TestUser'], + direction="out", + edge=['manages'] + ) + assert len(managed_employees) == 1 + assert managed_employees[0].name == "John Employee" + + # Test complex query + young_employees = await dept.nodes( + node=[{'TestUser': {"context.age": {"$lt": 30}}}] + ) + assert len(young_employees) == 1 + assert young_employees[0].age < 30 + + # Cleanup + await manager.delete() + await employee.delete() + await dept.delete() + + # Verify deletion + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await TestUser.find({"context.email": "jane@company.com"}) + deleted_manager = users[0] if users else None + assert deleted_manager is None +``` + +## 🗑️ Cleanup and Maintenance + +### Deprecation Procedure + +When evolving the library: + +1. **Identify obsolete code** - Old patterns, unused utilities +2. **Mark as deprecated** - Add deprecation warnings before removal +3. **Update documentation** - Remove references to deprecated features +4. **Update examples** - Remove or update examples using deprecated code +5. **Clean removal** - Remove deprecated code after grace period + +### File Cleanup Checklist + +- [ ] Remove unused import statements +- [ ] Delete empty or obsolete modules +- [ ] Archive outdated examples to `examples/archive/` +- [ ] Update `__all__` exports in `__init__.py` files +- [ ] Remove commented-out code blocks +- [ ] Clean up temporary test files + +### Migration Strategy + +When making breaking changes: + +1. **Create migration guide** in `docs/migration/` +2. **Provide before/after examples** +3. **Update all existing examples** to new patterns +4. **Add runtime warnings** for deprecated usage +5. **Version appropriately** using semantic versioning + +## 🚀 Development Workflow + +### Pre-commit Checklist + +- [ ] Code passes `black --check .` +- [ ] Code passes `flake8 .` +- [ ] Code passes `mypy .` +- [ ] All tests pass: `pytest` +- [ ] Examples are updated and runnable +- [ ] Documentation reflects changes +- [ ] Deprecated code is cleaned up + +### Code Review Focus Areas + +1. **Entity-centric patterns** - Ensure new code uses preferred syntax +2. **Query interface consistency** - MongoDB-style queries throughout +3. **Type safety** - Proper annotations and mypy compliance +4. **Test coverage** - Adequate testing for new features +5. **Documentation completeness** - Examples and guides updated + +## 📋 Quick Reference + +### Preferred Patterns + +```python path=null start=null +# ✅ Entity creation +user = await User.create(name="Alice", email="alice@company.com", department="engineering") + +# ✅ Entity queries with MongoDB-style operators +users = await User.find({"context.active": True}) +# Note: Object.find_one() doesn't exist - use find() and get first result +users_by_email = await User.find({"context.email": email}) +user = users_by_email[0] if users_by_email else None +senior_engineers = await User.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.age": {"$gte": 35}} + ] +}) +tech_users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) + +# ✅ Counting and aggregation +# Note: Object.count() doesn't exist - use len() with find() instead +results = await User.find({"context.department": "engineering"}) +count = len(results) + +# Note: Object.distinct() doesn't exist - query and extract manually +all_users = await User.find({}) +departments = set(u.department for u in all_users if hasattr(u, 'department')) + +# ✅ Entity updates +user.name = "Alice Johnson" +user.active = True +await user.save() + +# ✅ Bulk updates +users = await User.find({"context.department": "old_dept"}) +for user in users: + user.department = "new_dept" + await user.save() + +# ✅ Entity deletion +await user.delete() + +# ✅ Walker traversal with semantic filtering +@on_visit("User") +async def process_user(self, here: Node): + """Process user nodes with semantic filtering. + + Args: + here: The visited User node + """ + # Use nodes() method with semantic filtering + connected_users = await here.nodes( + node=['User'], # Type filtering + department="engineering", # Simple kwargs + active=True # Multiple simple filters + ) + await self.visit(connected_users) + + # Complex filtering with MongoDB operators + senior_connections = await here.nodes( + node=[{'User': {"context.age": {"$gte": 35}}}], + direction="out" + ) + await self.visit(senior_connections) + +# ✅ Walker control flow +class MyWalker(Walker): + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with control flow. + + Args: + here: The visited Document node + """ + if here.status == "archived": + self.skip() # Skip to next node + + if self.processed_count >= 100: + await self.disengage() # Permanently halt + + # Normal processing + next_docs = await here.nodes(node=['Document'], active=True) + await self.visit(next_docs) + + @on_exit + async def cleanup(self): + print(f"Processed {self.processed_count} documents") + +# ✅ Error handling +try: + user = await User.get(user_id) +except NodeNotFoundError: + logger.warning(f"User {user_id} not found") + return None +except DatabaseError as e: + logger.error(f"Database error: {e}") + raise +``` + +### Avoided Patterns + +```python path=null start=null +# ❌ Direct database access (discouraged - use entity methods) +from jvspatial.db import create_database +db = create_database("json") +await db.save("node", data) # Use entity.save() instead +await db.find("node", {"name": "User"}) # Use Entity.find() instead + +# ❌ GraphContext methods for simple operations (use entity methods) +from jvspatial.core import GraphContext +ctx = GraphContext(database=db) +# Prefer entity-centric methods: +# await Node.create(...) instead of ctx.create_node(...) +# await node.get_edges() instead of ctx.get_edges(node_id) + +# ❌ Non-standard query formats +await User.find({"age": 25}) # Missing context. prefix +await User.find({"name": "Alice"}) # Should be context.name + +# ❌ Old traversal patterns (deprecated) +walker.get_edges(node) # Use node.nodes() instead +walker.traverse_edges() # Use semantic filtering + +# ❌ Synchronous operations +user = User.create_sync(**data) # Use async await User.create() +users = User.find_sync(query) # Use async await User.find() + +# ❌ Manual edge management in walkers (show proper naming even in bad examples) +@on_visit("User") +async def visit_user(self, here: Node): + """DEPRECATED: Manual edge management (avoid this pattern). + + Args: + here: The visited User node + """ + # Avoid manual edge retrieval + edges = await here.get_edges() # Deprecated + for edge in edges: + target = await edge.get_target() + await self.visit([target]) + + # Instead, use semantic filtering + connected = await here.nodes() # Preferred + await self.visit(connected) + +# ❌ Missing error handling +user = await User.get(user_id) # Should handle NodeNotFoundError +user.field = value +await user.save() # Should handle ValidationError + +# ❌ Inefficient queries +# Don't fetch all then filter in Python +all_users = await User.find({}) +engineers = [u for u in all_users if u.department == "engineering"] + +# Instead, filter at database level +engineers = await User.find({"context.department": "engineering"}) + +# ❌ Blocking operations in async context +@on_visit("DataNode") +async def process_node(self, here: Node): + """BAD EXAMPLE: Blocking operations in async context. + + Args: + here: The visited DataNode + """ + # Avoid blocking operations + time.sleep(1.0) # Blocks event loop + + # Use async alternatives + await asyncio.sleep(1.0) # Non-blocking +``` + +--- + +## ⏰ Scheduler Integration (Optional) + +jvspatial includes optional scheduler support for background task automation. Install with: + +```bash +pip install jvspatial[scheduler] +``` + +### Basic Scheduled Tasks + +```python path=null start=null +from jvspatial.api import Server +from jvspatial.api.scheduler import on_schedule +from jvspatial.core import Object +from datetime import datetime +from typing import Optional +import logging + +logger = logging.getLogger(__name__) + +# Define entity for job tracking (entity-centric pattern) +class ScheduledJob(Object): + """Entity representing scheduled job execution records.""" + job_name: str = "" + execution_time: datetime = datetime.now() + status: str = "pending" # pending, completed, failed + duration_seconds: Optional[float] = None + error_message: Optional[str] = None + +class SystemMetrics(Object): + """Entity for system metrics collection.""" + timestamp: datetime = datetime.now() + cpu_usage: float = 0.0 + memory_usage: float = 0.0 + active_jobs: int = 0 + +# Scheduled function with proper error handling +@on_schedule("every 30 minutes", description="System cleanup with job tracking") +async def cleanup_system(): + """Automated cleanup with entity-centric job tracking.""" + start_time = datetime.now() + + try: + logger.info("🧹 Starting system cleanup") + + # Perform cleanup work + cleanup_count = await perform_cleanup_work() + + # Create success record + await ScheduledJob.create( + job_name="system_cleanup", + execution_time=start_time, + status="completed", + duration_seconds=(datetime.now() - start_time).total_seconds() + ) + + logger.info(f"✅ Cleanup completed: {cleanup_count} items processed") + + except Exception as e: + # Create error record + await ScheduledJob.create( + job_name="system_cleanup", + execution_time=start_time, + status="failed", + error_message=str(e), + duration_seconds=(datetime.now() - start_time).total_seconds() + ) + logger.error(f"❌ Cleanup failed: {str(e)}") + raise + +# Metrics collection with MongoDB-style queries +@on_schedule("every 5 minutes", retry_count=2, description="Collect system metrics") +async def collect_metrics(): + """Collect system metrics with entity queries.""" + import psutil + + # Get system metrics + cpu_percent = psutil.cpu_percent() + memory = psutil.virtual_memory() + + # Count active jobs using MongoDB-style query + # Note: Object.count() doesn't exist - use len() with find() instead + active_jobs_list = await ScheduledJob.find({ + "context.status": {"$in": ["pending", "running"]} + }) + active_jobs = len(active_jobs_list) + + # Create metrics record + await SystemMetrics.create( + timestamp=datetime.now(), + cpu_usage=cpu_percent, + memory_usage=memory.percent, + active_jobs=active_jobs + ) + + logger.info(f"📊 Metrics: CPU {cpu_percent:.1f}%, Memory {memory.percent:.1f}%") + +async def perform_cleanup_work() -> int: + """Simulate cleanup work.""" + # Query old records using entity-centric approach + cutoff_time = datetime.now().timestamp() - (7 * 24 * 3600) # 7 days ago + + old_jobs = await ScheduledJob.find({ + "context.execution_time": {"$lt": cutoff_time} + }) + + # Delete old records + for job in old_jobs: + await job.delete() + + return len(old_jobs) +``` + +### Server Integration + +```python path=null start=null +from jvspatial.api import Server, endpoint +from jvspatial.api.scheduler import register_scheduled_tasks +from typing import Dict, Any +from dotenv import load_dotenv + +# Load environment configuration (jvspatial pattern) +load_dotenv() + +# Create server with scheduler enabled +server = Server( + title="My Scheduled App", + description="Application with integrated scheduler", + version="1.0.0", + scheduler_enabled=True, # Enable scheduler + scheduler_interval=1, # Check every second + scheduler_timezone="UTC", # Timezone for scheduling +) + +# Register all decorated scheduled tasks +if hasattr(server, 'scheduler_service') and server.scheduler_service: + register_scheduled_tasks(server.scheduler_service) + logger.info("✅ Scheduled tasks registered") + +# Add monitoring endpoint +@endpoint("/api/scheduler/status", methods=["GET"]) +async def get_scheduler_status() -> Dict[str, Any]: + """Get scheduler status with entity-centric job statistics.""" + # Get job statistics using entity queries + # Note: Object.count() doesn't exist - use len() with find() instead + all_jobs = await ScheduledJob.find({}) + total_jobs = len(all_jobs) + + completed_jobs_list = await ScheduledJob.find({"context.status": "completed"}) + completed_jobs = len(completed_jobs_list) + + failed_jobs_list = await ScheduledJob.find({"context.status": "failed"}) + failed_jobs = len(failed_jobs_list) + + return { + "scheduler": "running", + "job_statistics": { + "total_jobs": total_jobs, + "completed_jobs": completed_jobs, + "failed_jobs": failed_jobs, + "success_rate": (completed_jobs / total_jobs * 100) if total_jobs > 0 else 0 + }, + "timestamp": datetime.now().isoformat() + } + +if __name__ == "__main__": + server.run(host="0.0.0.0", port=8000) # Scheduler runs automatically +``` + +**📖 For comprehensive scheduler documentation:** [Scheduler Integration Guide](docs/md/scheduler.md) + +--- + +## 🎯 Key Naming Conventions + +**CRITICAL**: When writing `@on_visit` methods, always use these parameter names: + +- **`here`** - The visited node/edge (current location in traversal) +- **`visitor`** - The walker performing the traversal (when passed to node methods) + +```python path=null start=null +# ✅ CORRECT naming convention +@on_visit("User") +async def process_user(self, here: Node): + """Args: here = visited User node""" + connected = await here.nodes() + await self.visit(connected) + +# ❌ AVOID generic names +@on_visit("User") +async def process_user(self, node: Node): # Less clear + pass +``` + +--- + +**Remember**: This library prioritizes **clean, maintainable code** with **consistent patterns** across all database backends. Always favor the entity-centric approach, MongoDB-style queries, and the **`here`/`visitor`** naming convention for the best developer experience. diff --git a/SPEC.md b/SPEC.md index a80c749..29a2a84 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,5284 +1,785 @@ -# jvspatial Language Model Coding Guide +# jvspatial — Technical Specification + +> **Purpose**: Authoritative technical contract for `jvspatial`. Describes what the library guarantees, what callers must not assume, and where the source of truth for each behavior lives. Cite this document by section number when justifying changes. +> +> **Scope**: Library behavior as of `jvspatial/version.py` → see file. This is a living spec; when behavior changes, this file must change in the same commit. +> +> **Companion documents**: +> - [PRD.md](PRD.md) — *why* the library exists (product context, users, success criteria) +> - [ROADMAP.md](ROADMAP.md) — forward-looking direction and known gaps +> - [CLAUDE.md](CLAUDE.md) — operational guidance for AI agents maintaining this repo +> - [docs/md/README.md](docs/md/README.md) — index of detailed how-to documentation +> - [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md) — usage cookbook (code patterns for callers) -This document provides concise guidance for AI language models to generate code that follows jvspatial library standards and conventions. - -## 🎯 Core Philosophy - -jvspatial emphasizes **entity-centric design** with unified MongoDB-style queries across database backends (JSON, MongoDB). The library distinguishes between: - -- **Objects** - For standalone data entities (users, settings, logs) that don't require graph relationships -- **Nodes** - For graph entities that are interconnected by Edges and traversed by Walkers -- **Edges** - For relationships between Nodes in the graph -- **Walkers** - For traversing and processing graph structures - -**Key Principle**: Use Objects for simple data storage, use Nodes when you need graph traversal and relationships. - -## 🔧 Environment Setup - -### Essential Configuration -```python -from dotenv import load_dotenv -load_dotenv() # Load .env file - -from jvspatial.core import GraphContext -from jvspatial.db import create_database, get_database_manager - -# Option 1: Create database explicitly -db = create_database("json", base_path="./jvdb") -ctx = GraphContext(database=db) - -# Option 2: Use current database from manager (defaults to prime) -manager = get_database_manager() -ctx = GraphContext(database=manager.get_current_database()) -``` - -### Environment Variables -```env -# Choose backend -JVSPATIAL_DB_TYPE=json # or 'mongodb' - -# JSON backend (default) -JVSPATIAL_JSONDB_PATH=./jvdb/dev - -# MongoDB backend -JVSPATIAL_MONGODB_URI=mongodb://localhost:27017 -JVSPATIAL_MONGODB_DB_NAME=jvspatial_dev - -# Caching (optional) -JVSPATIAL_CACHE_BACKEND=memory # 'memory', 'redis', or 'layered' -JVSPATIAL_CACHE_SIZE=1000 # Max cached entities (0 to disable) -``` - -## 📝 Entity-Centric Code Patterns - -### Objects vs Nodes: When to Use Each - -**✅ Objects** - For standalone entities without graph relationships: -```python -from jvspatial.core import Object - -class UserProfile(Object): - name: str = "" - email: str = "" - settings: Dict[str, Any] = {} - -# Use for: user profiles, configuration, logs, simple data -profile = await UserProfile.create(name="Alice", email="alice@company.com") -``` - -**✅ Nodes** - For graph entities with relationships and traversal: -```python -from jvspatial.core import Node - -class User(Node): - name: str = "" - department: str = "" - -class City(Node): - name: str = "" - population: int = 0 - -# Use for: entities that connect to other entities via Edges -user = await User.create(name="Alice", department="engineering") -city = await City.create(name="San Francisco", population=800000) -``` - -### Entity Operations -```python -# Entity creation (no save() needed, automatically cached) -entity = await Entity.create(name="value", field="data") - -# Entity retrieval (uses cache after first access) -entity = await Entity.get(entity_id) # Cached by ID -entities = await Entity.find({"context.active": True}) # Not cached - -# Entity updates (save() only needed after property modification, updates cache) -entity = await Entity.get(entity_id) -entity.name = "Updated Name" # Property modified -await entity.save() # save() required to persist changes + update cache - -# Entity deletion (removes from cache) -await entity.delete() - -# Counting and aggregation (not cached) -# Note: Object.count() doesn't exist - use len() with find() instead -results = await Entity.find({"context.department": "engineering"}) -count = len(results) - -# For distinct values, query and extract manually -all_entities = await Entity.find({}) -departments = set(e.department for e in all_entities if hasattr(e, 'department')) -``` - -**Note**: Caching is automatic and transparent. Individual entity retrievals by ID (`Entity.get(id)`) are cached. Queries (`find()`, `all()`) always hit the database as they can change frequently. For counting, use `Entity.count(query)` for efficient counting without loading all records. - -### save() Operation Rules -**✅ save() is ONLY required when:** -1. You modify entity properties after retrieval: `entity.field = "new_value"` -2. You create entities without using `.create()` method - -**❌ save() is NOT needed when:** -1. Using `.create()` method (automatically persists) -2. Using `.delete()` method (automatically persists deletion) -3. Just reading/querying entities - -**❌ AVOID: Direct database access (use entity methods instead)** -```python -# Don't do this - use entity methods instead -from jvspatial.db import create_database -db = create_database("json") -entities = await db.find("object", {"name": "Entity"}) - -# ✅ Do this instead - use entity-centric methods -entities = await Entity.find({"context.name": "Entity"}) -``` - -## 🗄️ Multi-Database Support - -jvspatial supports managing multiple databases within the same application, with a prime database for core persistence operations (authentication, session management) and additional databases for application-specific data. - -### Basic Multi-Database Usage - -```python -from jvspatial.db import ( - create_database, - get_database_manager, - get_prime_database, - get_current_database, - switch_database, - unregister_database, -) -from jvspatial.core.context import GraphContext - -# Get database manager (singleton) -manager = get_database_manager() - -# Prime database is automatically created for core operations -prime_db = get_prime_database() # Used for auth, sessions, system data - -# Create and register additional database -app_db = create_database( - "json", - base_path="./app_data", - register=True, - name="app" -) - -# Switch to application database -switch_database("app") -current_db = get_current_database() # Now returns app_db - -# Use with GraphContext -app_ctx = GraphContext(database=current_db) - -# Switch back to prime database -switch_database("prime") - -# Unregister non-prime database when no longer needed -unregister_database("app") -``` - -### Prime Database - -The prime database is always used for: -- User authentication -- Session management -- System-level configuration -- Core persistence operations - -It cannot be unregistered and is always available as the default. - -### Database Isolation - -Each database maintains complete isolation: -- Entities in one database are not visible in another -- Switching databases changes the context for all operations -- Prime database ensures core operations always have a stable database - -**📖 For comprehensive multi-database documentation:** [Graph Context Guide](docs/md/graph-context.md) and [Multi-Database Example](examples/database/multi_database_example.py) - -## 🔧 Custom Database Integration - -jvspatial supports seamless extension with custom database backends through a registration system. - -### Registering Custom Database Types - -```python -from jvspatial.db import Database, register_database_type, create_database, list_database_types - -# Define custom database implementation -class CustomDatabase(Database): - async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: - # Implementation - pass - - async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: - # Implementation - pass - - async def delete(self, collection: str, id: str) -> None: - # Implementation - pass - - async def find(self, collection: str, query: Dict[str, Any]) -> List[Dict[str, Any]]: - # Implementation - pass - -# Factory function for creating instances -def create_custom_db(**kwargs: Any) -> CustomDatabase: - return CustomDatabase(**kwargs) - -# Register the custom database type -register_database_type("custom", create_custom_db) - -# Now use it like built-in types -db = create_database("custom", connection_string="custom://...") - -# List all available database types -types = list_database_types() -# Returns: {"json": "JSON file-based database", "mongodb": "MongoDB database", "custom": "Custom database: create_custom_db"} -``` - -### Custom Database Requirements - -Custom databases must: -1. Inherit from `Database` abstract base class -2. Implement all abstract methods: `save()`, `get()`, `delete()`, `find()` -3. Optionally implement `count()` and `find_one()` (default implementations provided) -4. Provide a factory function for creation - -**📖 For comprehensive custom database documentation:** [Custom Database Guide](docs/md/custom-database-guide.md) and [Custom Database Example](examples/database/custom_database_example.py) - -## 🔍 MongoDB-Style Query Patterns - -Always use dot notation for nested fields with `context.` prefix: - -```python -# Comparison operators -users = await User.find({"context.age": {"$gte": 35}}) -users = await User.find({"context.role": {"$ne": "admin"}}) - -# Logical operators -users = await User.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.active": True} - ] -}) - -# Array operations -users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) - -# Regular expressions -users = await User.find({ - "context.name": {"$regex": "Johnson", "$options": "i"} -}) -``` - -## 🔒 Attribute Annotations (Protected & Transient) - -jvspatial provides `@protected` and `@transient` decorators for controlling attribute behavior: - -### Protected Attributes -Protected attributes cannot be modified after initialization (ideal for IDs and immutable config): - -```python -from pydantic import Field -from jvspatial.core.annotations import protected - -class Entity(Node): - # id is already protected in Node - uuid: str = protected("", description="Immutable UUID") - created_at: datetime = protected(Field(default_factory=datetime.now)) - -# ✓ Can set during initialization -entity = await Entity.create(uuid="abc-123") - -# ✗ Cannot modify after creation -entity.uuid = "new-uuid" # Raises AttributeProtectionError -``` - -### Transient Attributes -Transient attributes are excluded from database exports (ideal for runtime caches): - -```python -from jvspatial.core.annotations import transient - -class Entity(Node): - data: str = "" - cache: dict = transient(Field(default_factory=dict)) # Not persisted - temp_count: int = transient(Field(default=0)) # Not persisted - -entity.cache["key"] = "value" # Works at runtime -data = await entity.export() # cache excluded from export -``` +--- -### Compound Decorators -Combine both for internal state that's neither modifiable nor persisted: +## Table of Contents + +1. [Identity Model](#1-identity-model) +2. [Entity Hierarchy](#2-entity-hierarchy) +3. [Async Contract](#3-async-contract) +4. [Persistence Layer](#4-persistence-layer) +5. [Query Interface](#5-query-interface) +6. [Walker Traversal Semantics](#6-walker-traversal-semantics) +7. [GraphContext and Dependency Injection](#7-graphcontext-and-dependency-injection) +8. [API Surface](#8-api-surface) +9. [Authentication and Authorization](#9-authentication-and-authorization) +10. [Configuration and Environment](#10-configuration-and-environment) +11. [Serverless Constraints](#11-serverless-constraints) +12. [File Storage](#12-file-storage) +13. [Caching](#13-caching) +14. [Observability](#14-observability) +15. [Security Boundaries](#15-security-boundaries) +16. [Extension Points](#16-extension-points) +17. [Error Taxonomy](#17-error-taxonomy) +18. [Stability Tiers](#18-stability-tiers) -```python -# Both protected AND transient -_internal: dict = protected(transient(Field(default_factory=dict))) -``` +--- -**Key Points:** -- All `id` fields in `Object`, `Node`, `Edge`, and `Walker` are automatically protected -- Always use `Field(default_factory=dict)` syntax with `@transient` -### Private Attributes -Private attributes are excluded from serialization and database operations (ideal for internal state): +## 1. Identity Model -```python -from jvspatial.core.annotations import private +### 1.1 ID format -class Entity(Node): - _cache: dict = private(default_factory=dict) # Not serialized - _internal_counter: int = private(default=0) # Not serialized +All persistable entities carry an `id` field with the format: -entity._cache["key"] = "value" # Works at runtime -data = await entity.export() # _cache excluded from export -``` - -### Compound Decorators -Combine decorators for complex behaviors: -```python -# Private AND transient -_internal: dict = private(transient(Field(default_factory=dict))) ``` -- See [Attribute Annotations](docs/md/attribute-annotations.md) for full documentation - -## 🏢 Type Annotations & Error Handling - -### Required Typing Pattern -```python -from typing import List, Optional, Dict, Any -from jvspatial.core import Node, Object -from jvspatial.exceptions import NodeNotFoundError, ValidationError - -class User(Node): - name: str = "" - email: str = "" - age: int = 0 - roles: List[str] = [] - active: bool = True - metadata: Dict[str, Any] = {} - -async def get_user_by_id(user_id: str) -> Optional[User]: - """Get user by ID, returning None if not found.""" - try: - return await User.get(user_id) - except NodeNotFoundError: - return None +{type_code}.{entity_name}.{hex_id} ``` -### Error Handling Patterns +Where: -**🎯 Always catch specific exceptions first:** +- `type_code` ∈ `{n, e, w, o}` (Node, Edge, Walker, Object) +- `entity_name` is the persisted discriminator (see 1.2) +- `hex_id` is the first 24 hex chars of `uuid.uuid4().hex` -```python -import logging -from jvspatial.exceptions import ( - JVSpatialError, - ValidationError, - EntityNotFoundError, - NodeNotFoundError, - DatabaseError, - ConnectionError -) +**Source of truth**: `jvspatial/core/utils.py:11-22` (`generate_id`). -logger = logging.getLogger(__name__) - -# Entity operations with error handling -async def safe_user_operation(user_id: str) -> Optional[User]: - try: - user = await User.get(user_id) - return user - except NodeNotFoundError as e: - logger.warning(f"User not found: {e.entity_id}") - return None - except ValidationError as e: - logger.error(f"Validation failed: {e.message}") - if e.field_errors: - for field, error in e.field_errors.items(): - logger.error(f" {field}: {error}") - return None - except DatabaseError as e: - logger.error(f"Database error: {e.message}") - raise # Re-raise for higher-level handling - except JVSpatialError as e: - logger.error(f"jvspatial error: {e.message}") - return None -``` +**Invariants**: +- `id` is `protected=True` — set at construction, cannot be reassigned after `__init__` completes (`jvspatial/core/entities/object.py:46-48`). +- `id` collisions are not detected at insertion; callers depending on uniqueness must coordinate generation. UUID4 collision probability is the only practical guarantee. +- `type_code` is per-class default: `o` for `Object`, `n` for `Node` (`jvspatial/core/entities/node.py:43`), `e` for `Edge` (`jvspatial/core/entities/edge.py:41`), `w` for `Walker` (`jvspatial/core/entities/walker.py:118`). -**🔄 Database operations with fallback:** +### 1.2 Entity name discriminator (`__entity_name__`) -```python -from jvspatial.exceptions import ConnectionError, QueryError - -async def robust_user_search(query: Dict[str, Any]) -> List[User]: - try: - # Try complex query - return await User.find(query) - except QueryError as e: - logger.warning(f"Complex query failed: {e.message}") - # Fallback to simple query - try: - all_users = await User.all() - # Apply filtering in Python - return [u for u in all_users if u.active] - except Exception: - logger.error("All query methods failed") - return [] - except ConnectionError as e: - logger.error(f"Database connection failed: {e.database_type}") - return [] # Graceful degradation -``` - -**⚠️ Walker error handling:** +The `entity_name` segment of the ID, and the persisted `entity` field, default to `cls.__name__`. Subclasses may override by setting `__entity_name__` as a `ClassVar`: ```python -from jvspatial.exceptions import WalkerExecutionError, WalkerTimeoutError - -class SafeUserProcessor(Walker): - @on_visit(User) - async def process_user(self, here: User): - try: - # Potentially risky operation - result = await external_api_call(here) - self.report(result) - except Exception as e: - # Don't let individual errors stop traversal - logger.warning(f"Failed to process user {here.id}: {e}") - self.report({"error": str(e), "user_id": here.id}) - -async def run_safe_walker(): - try: - walker = SafeUserProcessor() - result = await walker.spawn(start_user) - - # Get results and errors - report = await result.get_report() - errors = [r for r in report if isinstance(r, dict) and "error" in r] - logger.info(f"Processed with {len(errors)} errors") - - except WalkerTimeoutError as e: - logger.error(f"Walker timed out after {e.timeout_seconds}s") - # Access partial results if needed - except WalkerExecutionError as e: - logger.error(f"Walker failed: {e.walker_class} - {e.message}") +class App(Node): + __entity_name__: ClassVar[Optional[str]] = "HostApp" ``` -## 📄 ObjectPager for Large Datasets - -### Basic Pagination -```python -from jvspatial.core.pager import paginate_objects, ObjectPager - -# Simple pagination -users = await paginate_objects(User, page=1, page_size=20) +**Source of truth**: `jvspatial/core/entities/object.py:35-44` (`_entity_name` classmethod). -# With filters -active_users = await paginate_objects( - User, - page=1, - page_size=10, - filters={"context.active": True} -) +**Resolution rule**: `cls.__dict__.get("__entity_name__") or cls.__name__`. Not inherited from a parent that set it — each subclass decides for itself. -# Advanced pager -pager = ObjectPager( - User, - page_size=25, - filters={"context.department": "engineering"}, - order_by="name" -) -users = await pager.get_page(1) -``` +**Use case**: disambiguating unrelated `Object` subtrees that share a Python class name (e.g. host-app `App` vs library-internal `App`) so they remain distinguishable at the storage layer. -## ⏰ Scheduler Integration +**Implications**: +- `find_subclass_by_name` (`jvspatial/core/utils.py:58-89`) honors the override and caches positive hits. Negative hits are not cached (avoids bootstrap poisoning when classes are imported later). +- The persisted `entity` field tracks `_entity_name()` at creation time, not class name. Renaming a class without `__entity_name__` will orphan existing records. -### Task Scheduling -```python -from jvspatial.core.scheduler import Scheduler, ScheduledTask -from datetime import datetime, timedelta - -# Create scheduler -scheduler = Scheduler() - -# Schedule recurring tasks -@scheduler.task(interval=timedelta(hours=1)) -async def cleanup_expired_sessions(): - """Clean up expired user sessions hourly.""" - expired = await UserSession.find({ - "context.expires_at": {"$lt": datetime.now()} - }) - for session in expired: - await session.delete() - print(f"Cleaned up {len(expired)} expired sessions") - -# Schedule one-time tasks -@scheduler.task(run_at=datetime.now() + timedelta(minutes=30)) -async def send_reminder_emails(): - """Send reminder emails.""" - users = await User.find({"context.reminder_due": True}) - for user in users: - await send_email(user.email, "Reminder") - user.reminder_due = False - await user.save() - -# Start scheduler -await scheduler.start() -``` +### 1.3 Root singleton -### Walker-Based Scheduled Tasks -```python -from jvspatial.core import Walker -from jvspatial.core.entities import on_visit - -@scheduler.walker_task(interval=timedelta(days=1)) -class DailyMaintenanceWalker(Walker): - """Perform daily maintenance tasks via graph traversal.""" - - @on_visit("User") - async def check_user_activity(self, here: Node): - """Check user activity and update status.""" - if here.last_active < datetime.now() - timedelta(days=30): - here.status = "inactive" - await here.save() - self.report({"deactivated_user": here.id}) - - @on_visit("DataNode") - async def cleanup_old_data(self, here: Node): - """Remove old data nodes.""" - if here.created_at < datetime.now() - timedelta(days=90): - await here.delete() - self.report({"deleted_data_node": here.id}) - -# Start scheduled walker -await scheduler.start_walker_task(DailyMaintenanceWalker) -``` +`Root` is the canonical entry node for a graph. Its ID is fixed: `n.Root.root` (`jvspatial/core/entities/root.py`). Created via an async lock to guarantee single instantiation per database. All graph traversals can originate from `Root` by convention; not enforced. -## 🌐 API Server with Server Class +--- -### Basic Server Setup -```python -from jvspatial.api import Server - -# Create server instance -server = Server( - title="My Spatial API", - description="Graph-based spatial data management API", - version="1.0.0", - host="0.0.0.0", - port=8000 -) +## 2. Entity Hierarchy -# Run server -if __name__ == "__main__": - server.run() ``` - -### Walker Endpoints -```python -from jvspatial.api import endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit - -@endpoint("/api/users/process", methods=["POST"]) -class ProcessUser(Walker): - """Process user data with graph traversal.""" - - user_name: str = EndpointField( - description="Name of user to process", - examples=["John Doe"] - ) - - department: str = EndpointField( - default="general", - description="User department" - ) - - @on_visit("User") - async def process_user(self, here: Node): - """Process user nodes - use 'here' for visited node.""" - if here.name == self.user_name: - self.report({ - "found_user": { - "id": here.id, - "name": here.name, - "department": here.department - } - }) - - # Get connected nodes and continue traversal - colleagues = await here.nodes( - node=['User'], - department=self.department - ) - await self.visit(colleagues) +AttributeMixin + pydantic.BaseModel + │ + Object ─────── (type_code="o") + │ │ + │ ├── Node ─────── (type_code="n") ─── Root (singleton) + │ │ + │ └── Edge ─────── (type_code="e") + │ + Walker (AttributeMixin + BaseModel, not Object) ── (type_code="w") ``` -### Function Endpoints -```python -from jvspatial.api import endpoint - -@endpoint("/api/users/count", methods=["GET"]) -async def get_user_count() -> Dict[str, int]: - """Get total user count.""" - users = await User.all() - return {"total_users": len(users)} - -@endpoint("/api/users/{user_id}", methods=["GET"]) -async def get_user(user_id: str, endpoint) -> Any: - """Get user with semantic response.""" - user = await User.get(user_id) - if not user: - return endpoint.not_found( - message="User not found", - details={"user_id": user_id} - ) - - return endpoint.success( - data={"id": user.id, "name": user.name, "email": user.email} - ) -``` +### 2.1 Object — base persistable entity -### Server with Scheduler Integration -```python -from jvspatial.api import Server -from jvspatial.core.scheduler import Scheduler - -# Create integrated server with scheduler -server = Server(title="Scheduled API", port=8000) -scheduler = Scheduler() - -# Add scheduled tasks -@scheduler.task(interval=timedelta(minutes=5)) -async def periodic_health_check(): - """Check system health every 5 minutes.""" - # Health check logic here - pass - -@server.on_startup -async def startup_tasks(): - """Start scheduler when server starts.""" - await scheduler.start() - print("✅ Server and scheduler started") - -@server.on_shutdown -async def shutdown_tasks(): - """Stop scheduler when server shuts down.""" - await scheduler.stop() - print("🛑 Server and scheduler stopped") - -# Run integrated server -if __name__ == "__main__": - server.run() -``` +`jvspatial/core/entities/object.py`. Provides: +- `id`, `entity`, `type_code` fields +- CRUD methods: `create()`, `get()`, `find()`, `save()`, `delete()`, `count()`, `export()` +- Context lookup: `set_context()`, `get_context()` (default via `get_default_context()`) +- Collection mapping via `get_collection_name()` → `{n: node, e: edge, o: object, w: walker}` -## 🔗 Webhook Integration +**Invariant**: `__setattr__` validates field names against the class hierarchy. Setting an undeclared attribute on an `Object` (post-init) is rejected. Prevents schema injection through attribute assignment. -### Basic Webhook Handler -```python -from jvspatial.api import endpoint -from fastapi import Request - -@endpoint("/webhook/{service}/{auth_token}", methods=["POST"], webhook=True) -async def webhook_handler(request: Request) -> Dict[str, Any]: - """Process webhooks with automatic payload parsing.""" - raw_body = request.state.raw_body - content_type = request.state.content_type - current_user = get_current_user(request) - - # Always return 200 for webhooks - try: - # Process webhook logic here - return {"status": "success", "processed_at": datetime.now().isoformat()} - except Exception as e: - logger.error(f"Webhook error: {e}") - return {"status": "received", "error": "logged"} -``` +### 2.2 Node — graph node -## 🏗️ Core Architecture & Walker Patterns +`jvspatial/core/entities/node.py:34+`. Adds: +- `edge_ids: List[str]` — transient in memory, persisted at the top level as `edges` (`Node._get_top_level_fields` line 55-60) +- `_visitor_ref: weakref` — currently visiting walker, transient +- `_visit_hooks: ClassVar` — populated by `__init_subclass__` from `@on_visit`-decorated methods (line 62+) -### Entity Hierarchy -- **Object** - Base class for all entities with unified query interface -- **Node** - Graph nodes with spatial/contextual data (extends Object) -- **Edge** - Relationships between nodes -- **Walker** - Graph traversal and processing logic -- **GraphContext** - Low-level database interface (use sparingly) +**Hooks**: methods decorated with `@on_visit(WalkerType | "string_name")` are registered per-class at class-creation time. String names allow forward references; resolved when the walker visits. -### Walker Traversal (CRITICAL PATTERNS) +### 2.3 Edge — graph relationship -#### Naming Convention for @on_visit Methods -**ALWAYS** use these parameter names: -- **`here`** - The visited node/edge (current location) -- **`visitor`** - The visiting walker (when accessing from node context) +`jvspatial/core/entities/edge.py:29+`. Top-level persisted fields: `source: str`, `target: str`, `bidirectional: bool = True` (line 42-49). -```python -@on_visit("User") -async def process_user(self, here: Node): - """Use 'here' for visited node.""" - connected_users = await here.nodes(node=['User']) - await self.visit(connected_users) - -@on_visit("City") -async def process_city(self, here: Node): - """Process cities with filtering.""" - # Skip small cities - if here.population < 10000: - self.skip() # Skip to next node - return - - # Get large connected cities - large_cities = await here.nodes( - node=['City'], - population={"$gte": 500000} - ) - await self.visit(large_cities) -``` +**Direction**: `await edge.direction` returns `"both"` if `bidirectional`, else `"out"` (line 57-64). Asymmetric edges flow source → target. -#### Walker Control Flow -```python -class DataWalker(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.max_items = 100 - - @on_visit("Document") - async def process_document(self, here: Node): - """Process with control flow.""" - # Skip invalid documents - if here.status == "invalid": - self.skip() # Continue to next node - - # Stop at limit - if self.processed_count >= self.max_items: - await self.disengage() # Permanently halt walker - return - - # Pause for rate limiting - if self.processed_count % 50 == 0: - self.pause("Rate limit pause") - - # Normal processing - self.processed_count += 1 - next_docs = await here.nodes(node=['Document']) - await self.visit(next_docs) - - @on_exit - async def cleanup(self): - """Called when walker completes/pauses/disengages.""" - print(f"Processed {self.processed_count} documents") -``` +### 2.4 Walker — traversal agent -## 📋 Quick Reference Checklist +`jvspatial/core/entities/walker.py:83+`. Walkers are **not** `Object` subclasses; they share `AttributeMixin` + `BaseModel`. Walkers are **not persisted to the database** by default — they exist for the lifetime of a traversal. -### Entity Operations -- ✅ Use `await Entity.create(**kwargs)` -- ✅ Use `await Entity.find(query_dict)` -- ✅ Use `await entity.save()` only after property modification -- ✅ Use Objects for standalone data, Nodes for graph entities -- ✅ Use `await node1.disconnect(node2)` to remove connections -- ❌ Avoid direct GraphContext database calls +**Components** (composition, in `walker_components/`): +- `WalkerTrail` — visited node IDs, edge IDs, timestamps, optional per-step metadata +- `TraversalProtection` — enforces step/visit/time/queue limits +- `WalkerQueue` — FIFO queue with `max_queue_size` cap +- `WalkerEventSystem` — in-memory event bus for traversal lifecycle -### Disconnecting Nodes -To remove connections between nodes, use the `disconnect()` method. This removes edges between nodes and deletes the edge objects. +**Default protection limits** (Walker class attrs): +- `max_steps`: 10 000 +- `max_visits_per_node`: 100 +- `max_execution_time`: 300 seconds +- `max_queue_size`: 1 000 +- `protection_enabled`: True -```python -# Disconnect two nodes -success = await node1.disconnect(node2) +These are *defaults* — subclasses or callers can override. Disabling protection (`protection_enabled=False`) is allowed but strongly discouraged in untrusted code paths. -# Disconnect with specific edge type -success = await node1.disconnect(node2, edge_type=SpecialEdge) -``` +### 2.5 Attribute system -### Query Patterns -- ✅ Use `"context.field"` dot notation for nested fields -- ✅ Use MongoDB operators: `$gte`, `$in`, `$regex`, `$and`, `$or` -- ✅ Combine dict filters with kwargs: `node=[{'User': {...}}], active=True` - -### Walker Patterns -- ✅ Use `here` parameter for visited nodes -- ✅ Use `await here.nodes()` to get connected nodes -- ✅ Use `await self.visit(nodes)` to continue traversal -- ✅ Use `self.skip()` to skip current node -- ✅ Use `await self.disengage()` to permanently halt -- ✅ Use `self.pause()` for temporary suspension - -### API Patterns -- ✅ Use `@endpoint` for both graph processing and simple functions -- ✅ Use `EndpointField` for parameter configuration -- ✅ Use `endpoint.success()`, `endpoint.not_found()` for responses -- ✅ Always return 200 for webhooks with try/catch - -### Type Safety -- ✅ Always include proper type annotations -- ✅ Import from `typing` and `jvspatial.exceptions` -- ✅ Handle `NodeNotFoundError`, `ValidationError`, `DatabaseError` -- ✅ Use structured error logging - -This guide provides the essential patterns for generating jvspatial-compliant code. Focus on entity-centric operations, proper typing, and following the established naming conventions for walker traversal. - -The **jvspatial webhook system** provides secure, flexible webhook endpoints with built-in authentication, HMAC verification, idempotency keys, and automatic payload processing. Webhooks integrate seamlessly with the FastAPI server and support both function-based handlers and graph traversal processing. - -### Webhook Architecture Overview - -Webhooks in jvspatial are designed for: -- **Security**: Path-based authentication tokens, optional HMAC signature verification -- **Reliability**: Idempotency key support to handle duplicate deliveries -- **Flexibility**: JSON/XML/binary payload support with automatic parsing -- **Integration**: Full compatibility with existing authentication and permission systems -- **Processing**: Always return HTTP 200 for proper webhook etiquette - -### Basic Webhook Endpoint Setup - -```python path=null start=null -from fastapi import Request -from jvspatial.api import endpoint -from jvspatial.api.auth.middleware import get_current_user -from typing import Dict, Any -import json - -# Basic webhook handler function -@endpoint("/webhook/{route}/{auth_token}", methods=["POST"], webhook=True) -async def generic_webhook_handler(request: Request) -> Dict[str, Any]: - """Generic webhook handler for multiple services. - - Processes webhooks from various sources using route-based dispatch. - Middleware handles authentication, HMAC verification, and payload parsing. - """ - # Access processed data from middleware - raw_body = request.state.raw_body # Original bytes - content_type = request.state.content_type # Content-Type header - route = getattr(request.state, "webhook_route", "unknown") # Route parameter - current_user = get_current_user(request) # Authenticated user - - # Parse payload based on content type - processed_data = None - if content_type == "application/json": - try: - processed_data = json.loads(raw_body) - except json.JSONDecodeError: - return {"status": "error", "message": "Invalid JSON payload"} - else: - # Handle other content types (XML, form data, binary) - processed_data = {"raw_length": len(raw_body), "type": content_type} - - # Route-based processing - if route == "stripe": - return await process_stripe_webhook(processed_data, current_user) - elif route == "github": - return await process_github_webhook(processed_data, current_user) - elif route == "slack": - return await process_slack_webhook(processed_data, current_user) - else: - # Generic processing for unknown routes - return { - "status": "received", - "route": route, - "payload_type": content_type, - "user_id": current_user.id if current_user else None - } - -# Service-specific webhook handlers -@endpoint("/webhook/stripe/{auth_token}", methods=["POST"], webhook=True) -async def stripe_webhook_handler(request: Request) -> Dict[str, Any]: - """Dedicated Stripe webhook handler with event processing.""" - raw_body = request.state.raw_body - current_user = get_current_user(request) - - try: - event = json.loads(raw_body) - event_type = event.get("type", "unknown") - - # Process different Stripe event types - if event_type == "payment_intent.succeeded": - await handle_successful_payment(event["data"]["object"], current_user) - elif event_type == "customer.subscription.updated": - await handle_subscription_update(event["data"]["object"], current_user) - elif event_type == "invoice.payment_failed": - await handle_payment_failure(event["data"]["object"], current_user) - - return { - "status": "success", - "event_type": event_type, - "processed_at": datetime.now().isoformat() - } - - except Exception as e: - # Always return 200 for webhooks, log errors internally - print(f"Stripe webhook processing error: {e}") - return {"status": "received", "error": "Processing error logged"} - -# Helper functions for webhook processing -async def process_stripe_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process Stripe webhook events.""" - event_type = data.get("type", "unknown") - return { - "status": "success", - "message": f"Processed Stripe {event_type}", - "user_id": user.id if user else None - } - -async def process_github_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process GitHub webhook events.""" - action = data.get("action", "unknown") - repo_name = data.get("repository", {}).get("name", "unknown") - return { - "status": "success", - "message": f"GitHub {action} on {repo_name}", - "user_id": user.id if user else None - } - -async def process_slack_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process Slack webhook events.""" - event_type = data.get("type", "unknown") - return { - "status": "success", - "message": f"Slack {event_type} processed", - "user_id": user.id if user else None - } -``` +`jvspatial/core/annotations` (re-exported as `jvspatial.core.annotations`): -### Webhook Security and Authentication +| Flag | Effect | +|---|---| +| `protected=True` | Field cannot be reassigned after `__init__` completes | +| `transient=True` | Field is not persisted to the database | +| `private=True` | Field is hidden from `export()` output (and typically transient) | +| `default` / `default_factory` | Pydantic-style defaults | +| `description` | Documentation surfaced in OpenAPI schemas | -Webhook endpoints require authentication tokens in the URL path and support additional security measures: +Indexed fields (`@attribute(indexed=True)`) and compound indexes (`@attribute(compound_index=...)`) inform backend index creation (where supported). -```python path=null start=null -# Webhook with permission requirements -@endpoint( - "/webhook/admin/{route}/{auth_token}", - methods=["POST"], - webhook=True, - permissions=["process_webhooks", "admin_access"], - roles=["admin", "webhook_manager"] -) -async def admin_webhook_handler(request: Request) -> Dict[str, Any]: - """Administrative webhook handler with strict permissions.""" - current_user = get_current_user(request) - - # User is guaranteed to have required permissions due to middleware - return { - "status": "success", - "message": "Admin webhook processed", - "admin_user": current_user.username, - "permissions": current_user.permissions - } - -# HMAC signature verification (handled by middleware) -@endpoint("/webhook/secure/{service}/{auth_token}", methods=["POST"], webhook=True) -async def secure_webhook_handler(request: Request) -> Dict[str, Any]: - """Webhook with HMAC signature verification. - - Middleware automatically verifies HMAC signatures when present. - Configure HMAC secrets via environment variables or user settings. - """ - # If this handler executes, HMAC verification passed (if configured) - raw_body = request.state.raw_body - hmac_verified = getattr(request.state, "hmac_verified", False) - - return { - "status": "success", - "message": "Secure webhook processed", - "hmac_verified": hmac_verified, - "payload_size": len(raw_body) - } -``` +--- -### Graph Traversal Webhook Processing (Future Enhancement) - -The architecture supports webhook processing through graph traversal using Walker classes: - -```python path=null start=null - -# from jvspatial.api.auth.decorators import webhook_endpoint -# from jvspatial.core import Walker, Node -# from jvspatial.core.entities import on_visit - -# @endpoint("/webhook/process/{route}/{auth_token}", methods=["POST"]) -# class WebhookProcessingWalker(Walker): -# """Walker-based webhook processing with graph traversal.""" -# -# def __init__(self): -# super().__init__() -# self.webhook_data = None -# self.processing_results = [] -# -# @on_visit("WebhookEvent") -# async def process_webhook_event(self, here: Node): -# """Process webhook events stored as graph nodes.""" -# # Access webhook data from request.state -# payload = self.webhook_data -# -# # Process event based on node data and webhook payload -# result = await self.analyze_event(here, payload) -# self.processing_results.append(result) -# -# # Continue traversal to related events -# related_events = await here.nodes(node=['WebhookEvent']) -# await self.visit(related_events) -# -# async def analyze_event(self, event_node: Node, payload: dict) -> dict: -# """Analyze webhook event against stored data.""" -# return { -# "event_id": event_node.id, -# "payload_type": payload.get("type"), -# "correlation_score": 0.95 # Example analysis result -# } -``` +## 3. Async Contract -### Idempotency and Duplicate Handling - -Webhooks support idempotency keys to handle duplicate deliveries: - -```python path=null start=null -@endpoint("/webhook/idempotent/{auth_token}", methods=["POST"], webhook=True) -async def idempotent_webhook_handler(request: Request) -> Dict[str, Any]: - """Webhook handler with built-in idempotency support. - - Middleware automatically handles idempotency keys in headers: - - Idempotency-Key header - - X-Idempotency-Key header - - Custom idempotency headers - """ - # Access idempotency information from middleware - idempotency_key = getattr(request.state, "idempotency_key", None) - is_duplicate = getattr(request.state, "is_duplicate_request", False) - - if is_duplicate: - # Return cached response for duplicate requests - cached_response = getattr(request.state, "cached_response", {}) - return { - "status": "success", - "message": "Duplicate request, returning cached response", - "idempotency_key": idempotency_key, - "cached_result": cached_response - } - - # Process new request - raw_body = request.state.raw_body - processed_result = await process_unique_webhook(json.loads(raw_body)) - - return { - "status": "success", - "message": "New webhook processed", - "idempotency_key": idempotency_key, - "result": processed_result - } - -async def process_unique_webhook(payload: dict) -> dict: - """Process a unique webhook payload.""" - # Simulate processing logic - import time - processing_start = time.time() - - # Your actual webhook processing logic here - await asyncio.sleep(0.1) # Simulate work - - return { - "processed_at": processing_start, - "data_processed": True, - "payload_keys": list(payload.keys()) - } -``` +### 3.1 Async-only I/O -### Server Integration and Middleware Setup +Every operation that touches the database, network, or file system is `async`. This includes — without exception: -Webhook endpoints automatically integrate with the jvspatial server middleware stack: +- `Entity.create`, `Entity.get`, `Entity.find`, `Entity.find_one`, `Entity.save`, `Entity.delete`, `Entity.count`, `Entity.export` +- `Database.save`, `Database.get`, `Database.find`, `Database.delete`, `Database.count`, `Database.find_one`, `Database.find_many`, `Database.bulk_save` +- All Walker traversal methods (`spawn`, `step`, `walk`, `enqueue`, `pause`, `resume`) +- Storage backends (`upload`, `download`, `list_versions`, …) +- All API server lifecycle hooks (`on_startup`, `on_shutdown`, …) -```python path=null start=null -from jvspatial.api import Server -from jvspatial.api.auth.middleware import ( - AuthenticationMiddleware, - WebhookMiddleware, - HTTPSRedirectMiddleware -) +### 3.2 Sync operations -# Server setup with webhook middleware -server = Server( - title="Webhook-Enabled Spatial API", - description="API with secure webhook processing", - version="1.0.0", - host="0.0.0.0", - port=8000 -) +Sync functions are reserved for **pure computation only**: -# Middleware stack (order matters) -server.add_middleware(HTTPSRedirectMiddleware) # Force HTTPS -server.add_middleware(WebhookMiddleware) # Webhook processing -server.add_middleware(AuthenticationMiddleware) # Authentication +- `generate_id()`, `find_subclass_by_name()` — `jvspatial/core/utils.py` +- `get_collection_name()` — `jvspatial/core/entities/object.py:78` +- `is_serverless_mode()`, `detect_serverless_provider()` — `jvspatial/runtime/serverless.py` +- Walker trail read-only queries (`get_trail()`, `has_visited()`, `get_trail_summary()`) +- All `@attribute`-related helpers -# Webhook endpoints are automatically registered -# Access at: POST https://your-domain.com/webhook/{route}/{auth_token} +Calling sync code from async is always safe. Calling async code from sync without an event loop is a programming error. -# Environment configuration for webhook security -# Set in .env file: -# WEBHOOK_HMAC_SECRET=your-secret-key -# WEBHOOK_HTTPS_REQUIRED=true -# WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour cache -# WEBHOOK_MAX_PAYLOAD_SIZE=1048576 # 1MB limit +### 3.3 Blocking ops to avoid -if __name__ == "__main__": - server.run(port=8000) -``` +- **Never** do synchronous file or network I/O inside an async handler — it blocks the event loop. +- **Never** spawn a thread to run an async coroutine — use `asyncio.create_task()` or `asyncio.gather()` instead. +- **Never** forget `await` on a database call — at best you receive a coroutine object; at worst you proceed with stale data. -### Webhook Testing and Development - -```python path=null start=null -# Testing webhook handlers -import pytest -from fastapi.testclient import TestClient -from unittest.mock import MagicMock - -@pytest.fixture -def test_webhook_request(): - """Create mock webhook request for testing.""" - request = MagicMock() - request.state.raw_body = b'{"type": "test", "data": {"id": 123}}' - request.state.content_type = "application/json" - request.state.webhook_route = "test" - request.state.current_user = MagicMock(id="user_123") - request.state.hmac_verified = True - request.state.idempotency_key = "test-key-123" - return request - -@pytest.mark.asyncio -async def test_webhook_processing(test_webhook_request): - """Test webhook handler processing.""" - result = await generic_webhook_handler(test_webhook_request) - - assert result["status"] == "received" - assert result["route"] == "test" - assert result["user_id"] == "user_123" - -# Development webhook testing with ngrok or similar -# 1. Start your jvspatial server locally -# 2. Use ngrok to expose: ngrok http 8000 -# 3. Configure webhook URLs: https://abc123.ngrok.io/webhook/test/your-auth-token -# 4. Test with curl: -# curl -X POST https://abc123.ngrok.io/webhook/test/your-token \ -# -H "Content-Type: application/json" \ -# -d '{"test": "data"}' -``` +### 3.4 Deferred saves (optional batching) -### Best Practices for Webhook Implementation - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Always return 200 status for webhooks -@endpoint("/webhook/service/{auth_token}", webhook=True) -async def proper_webhook_handler(request: Request) -> Dict[str, Any]: - try: - # Process webhook - result = await process_webhook_data(request.state.raw_body) - return {"status": "success", "result": result} - except Exception as e: - # Log error but still return 200 - logger.error(f"Webhook processing failed: {e}") - return {"status": "received", "error": "logged"} - -# Good: Use route-based dispatch for multiple services -@endpoint("/webhook/{route}/{auth_token}", webhook=True) -async def multi_service_webhook(request: Request) -> Dict[str, Any]: - route = getattr(request.state, "webhook_route", "unknown") - - handlers = { - "stripe": process_stripe_webhook, - "github": process_github_webhook, - "slack": process_slack_webhook - } - - handler = handlers.get(route, process_generic_webhook) - return await handler(request) - -# Good: Validate authentication token format -@endpoint("/webhook/{service}/{auth_token}", webhook=True) -async def secure_webhook_handler(request: Request) -> Dict[str, Any]: - # Token validation is handled by middleware - current_user = get_current_user(request) - if not current_user: - return {"status": "error", "message": "Invalid authentication"} - - return {"status": "success", "user_verified": True} -``` +When `DeferredSaveMixin` is mixed into an entity *and* `deferred_saves_globally_allowed()` returns `True` (see §11.3), `await entity.save()` marks the entity dirty without writing. Persistence happens on explicit `await entity.flush()` or context exit. -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Returning non-200 status codes -@endpoint("/webhook/bad/{auth_token}", webhook=True) -async def bad_webhook_handler(request: Request) -> Dict[str, Any]: - try: - process_webhook(request.state.raw_body) - except Exception: - # Don't do this - breaks webhook retry logic - raise HTTPException(status_code=500, detail="Processing failed") - -# Bad: Not handling authentication properly -@endpoint("/webhook/unsecure/{auth_token}", webhook=True) -async def unsecure_webhook_handler(request: Request) -> Dict[str, Any]: - # Don't bypass authentication checks - # Always use get_current_user() or require auth in decorator - return {"status": "processed"} - -# Bad: Not using middleware-processed data -@endpoint("/webhook/manual/{auth_token}", webhook=True) -async def manual_webhook_handler(request: Request) -> Dict[str, Any]: - # Don't manually read request body - use request.state.raw_body - # raw_body = await request.body() # Wrong - middleware already processed - - # Use middleware-processed data instead - raw_body = request.state.raw_body # Correct - return {"status": "processed"} -``` +**MRO requirement**: the mixin **must precede** the base class: `class MyEntity(DeferredSaveMixin, Node)` — not the reverse. Wrong MRO silently disables batching. --- -## 🔗 Webhook System Integration +## 4. Persistence Layer -JVspatial provides an advanced webhook system for handling external service integrations with enterprise-grade security, reliability, and developer experience. The webhook system supports modern decorators, automatic payload processing, HMAC verification, idempotency handling, and seamless authentication integration. +### 4.1 Database abstraction -### Quick Webhook Setup +`jvspatial/db/database.py:48+` — `Database` ABC. All adapters must implement: -```python path=null start=null -from jvspatial.api import endpoint -from jvspatial.api import Server +| Method | Required | Description | +|---|---|---| +| `save(collection, data)` | Yes | Insert-or-replace by ID; returns saved record | +| `get(collection, id)` | Yes | Fetch by ID or `None` | +| `delete(collection, id)` | Yes | Idempotent delete by ID | +| `find(collection, query, *, limit, sort)` | Yes | Mongo-style query; returns list | +| `count(collection, query=None)` | Default impl | Default counts the result of `find`; adapters should override for efficiency | +| `find_one(collection, query)` | Default impl | First match or `None` | +| `find_many(collection, ids)` | Default impl | Bulk-fetch by ID; default is N sequential `get`s — adapters override for round-trip efficiency | +| `find_one_and_update` | Default impl | Read-modify-write; **not atomic** except where overridden (MongoDB) | +| `find_one_and_delete` | Default impl | Read-then-delete; **not atomic** except where overridden (MongoDB) | +| `bulk_save` | Default impl | Multi-record save; partial-success semantics vary by adapter | +| `begin_transaction` | Optional | Returns a transaction context manager if `supports_transactions=True` | -# Simple webhook handler -@endpoint("/webhook/payment", webhook=True) -async def payment_webhook(payload: dict, endpoint): - """Process payment webhooks with automatic JSON parsing.""" - payment_id = payload.get("payment_id") - amount = payload.get("amount") +### 4.2 Capability flags - # Process payment logic here - print(f"Processing payment {payment_id}: ${amount}") +Adapters declare capabilities as class attributes: - return endpoint.response( - content={ - "status": "processed", - "message": f"Payment {payment_id} processed successfully" - } - ) +- `supports_transactions: bool` — `True` for MongoDB (replica set); `False` for SQLite (best-effort), JSON, DynamoDB. -# Server automatically detects and configures webhook middleware -server = Server(title="My Webhook API") -server.run() # Webhooks ready at /webhook/* paths -``` +Callers branching on capabilities should test the flag, not the adapter class. -### Advanced Webhook Features - -```python path=null start=null -# Webhook with full security features -@endpoint( - "/webhook/stripe/{key}", - webhook=True, - path_key_auth=True, # API key in URL path - hmac_secret="stripe-webhook-secret", # HMAC signature verification - idempotency_ttl_hours=48, # Duplicate handling for 48h - permissions=["process_payments"] # RBAC permissions -) -async def secure_stripe_webhook(raw_body: bytes, content_type: str, endpoint): - """Stripe webhook with comprehensive security.""" - import json - - if content_type == "application/json": - payload = json.loads(raw_body.decode('utf-8')) - event_type = payload.get("type", "unknown") - - if event_type == "payment_intent.succeeded": - return endpoint.response( - content={ - "status": "processed", - "event_type": event_type, - "message": "Payment successful" - } - ) - - return endpoint.response(content={"status": "received"}) - -# Multi-service webhook dispatcher -@endpoint("/webhook/{service}", webhook=True) -async def multi_service_webhook(payload: dict, service: str, endpoint): - """Route webhooks based on service parameter.""" - handlers = { - "stripe": process_stripe_event, - "github": process_github_event, - "slack": process_slack_event - } - - handler = handlers.get(service, process_generic_event) - result = await handler(payload) - - return endpoint.response( - content={ - "status": "processed", - "service": service, - "result": result - } - ) - -# Helper functions -async def process_stripe_event(payload: dict) -> dict: - return {"stripe_event": payload.get("type", "unknown")} - -async def process_github_event(payload: dict) -> dict: - return {"github_action": payload.get("action", "unknown")} - -async def process_slack_event(payload: dict) -> dict: - return {"slack_event": payload.get("event", {}).get("type", "unknown")} - -async def process_generic_event(payload: dict) -> dict: - return {"processed": True, "keys": list(payload.keys())} -``` - -### Walker-Based Webhook Processing - -```python path=null start=null -# Future feature - Walker-based webhook processing -# @webhook_walker_endpoint("/webhook/location-update") -# class LocationUpdateWalker(Walker): -# """Process location updates through graph traversal.""" -# -# def __init__(self, payload: dict): -# super().__init__() -# self.payload = payload -# # Use the report() method to collect data during traversal -# -# @on_visit(Node) -# async def update_location_data(self, here: Node): -# locations = self.payload.get("locations", []) -# -# for location_data in locations: -# location_id = location_data.get("id") -# coordinates = location_data.get("coordinates") -# -# if location_id and coordinates: -# here.coordinates = coordinates -# await here.save() -# -# self.report({ -# "updated_location": { -# "id": location_id, -# "coordinates": coordinates -# } -# }) -``` +### 4.3 Built-in adapters -### Environment Configuration +| Adapter | File | Transactions | Notes | +|---|---|---|---| +| JSON | `jvspatial/db/jsondb.py` | No | File-per-record, atomic writes (`_atomic.py`), per-file path locks (`_path_locks.py`) | +| SQLite | `jvspatial/db/sqlite.py` | No (single-conn fsync) | `aiosqlite`; Mongo→SQL via `SQLiteTranslator` | +| MongoDB | `jvspatial/db/mongodb.py` | Yes | `motor`; native bulk writes; native compound ops | +| DynamoDB | `jvspatial/db/dynamodb.py` | No | `aioboto3`; throttle-retry; `BatchGetItem` chunks of 100 | -Configure webhook behavior via environment variables: +### 4.4 Atomic IO (JSON) -```env -# Global webhook settings -JVSPATIAL_WEBHOOK_HMAC_SECRET=your-global-hmac-secret -JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE=5242880 # 5MB -JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour -JVSPATIAL_WEBHOOK_HTTPS_REQUIRED=true - -# Service-specific secrets -JVSPATIAL_WEBHOOK_STRIPE_SECRET=whsec_stripe_secret_key -JVSPATIAL_WEBHOOK_GITHUB_SECRET=github_webhook_secret -``` +`jvspatial/db/_atomic.py` provides crash-safe writes: temp file → `fsync` → `rename` → `fsync(directory)`. Per-file mutex via `PathLockManager` (`_path_locks.py`) serializes concurrent writes to the same record. Bounded LRU prevents lock-table growth. -### Testing Webhooks - -```bash -# Basic webhook test -curl -X POST "http://localhost:8000/webhook/payment" \ - -H "Content-Type: application/json" \ - -d '{"payment_id": "pay_123", "amount": 99.99}' - -# Webhook with path-based auth -curl -X POST "http://localhost:8000/webhook/stripe/key123:secret456" \ - -H "Content-Type: application/json" \ - -H "X-Signature: sha256=abc123..." \ - -d '{"type": "payment_intent.succeeded"}' - -# With idempotency key -curl -X POST "http://localhost:8000/webhook/payment" \ - -H "Content-Type: application/json" \ - -H "X-Idempotency-Key: unique-123" \ - -d '{"payment_id": "pay_124"}' -``` +### 4.5 Multi-database -### Webhook Best Practices - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Always return 200 for webhook endpoints -@endpoint("/webhook/service", webhook=True) -async def proper_webhook(payload: dict, endpoint): - try: - result = await process_webhook_data(payload) - return endpoint.response(content={"status": "success", "result": result}) - except Exception as e: - # Log error but still return 200 - logger.error(f"Webhook processing failed: {e}") - return endpoint.response(content={"status": "received", "error": "logged"}) - -# Good: Use route-based dispatch for multiple services -@endpoint("/webhook/{service}", webhook=True) -async def multi_service_webhook(payload: dict, service: str, endpoint): - handlers = { - "stripe": process_stripe, - "github": process_github - } - - handler = handlers.get(service, process_generic) - return await handler(payload, endpoint) - -# Good: Validate webhook signatures when available -@endpoint("/webhook/secure", webhook=True, hmac_secret="webhook-secret") -async def secure_webhook(raw_body: bytes, endpoint): - # HMAC verification is automatic when secret is provided - return endpoint.response(content={"status": "verified"}) -``` +`jvspatial/db/manager.py` — one "prime" database for core ops (auth, sessions, API keys), plus additional databases for specialized use. Auth state is **always** on the prime database; this cannot be relocated. -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Returning non-200 status codes -@endpoint("/webhook/bad", webhook=True) -async def bad_webhook(payload: dict, endpoint): - if payload.get("invalid"): - # Don't do this - breaks webhook retry logic - raise HTTPException(status_code=400, detail="Invalid payload") - -# Bad: Not handling errors gracefully -@endpoint("/webhook/risky", webhook=True) -async def risky_webhook(payload: dict, endpoint): - # Unhandled exceptions will return 500 - webhooks will retry - result = dangerous_operation(payload) # Might throw - return endpoint.response(content={"result": result}) - -# Bad: Bypassing security features -@endpoint("/webhook/insecure", webhook=True) -async def insecure_webhook(request: Request, endpoint): - # Don't manually read request body - use automatic payload injection - raw_body = await request.body() # Wrong - middleware already processed - return endpoint.response(content={"status": "received"}) -``` +### 4.6 Schema migration -> **📖 For complete webhook documentation and advanced patterns:** [Webhook Architecture Guide](docs/md/webhook-architecture.md) | [Webhook Quickstart](docs/md/webhooks-quickstart.md) +No built-in migration framework. Adapters do not enforce schemas. Adding optional fields with defaults is backward-compatible on read. Removing fields breaks existing records that still contain them. Application owners are responsible for migration scripts. --- -## 📁 File Storage Quickstart +## 5. Query Interface -jvspatial includes a powerful file storage system with multi-backend support and URL proxy capabilities for secure file sharing. +### 5.1 Mongo-style operators -### Basic Setup +`jvspatial/db/query.py` — unified query DSL across adapters. Supported operators: -```python -from jvspatial.api import Server - -server = Server( - title="File Upload API", - file_storage_enabled=True, - file_storage_provider="local", - file_storage_root=".files", - proxy_enabled=True -) +| Operator | Meaning | +|---|---| +| `$eq`, `$ne` | Equality / inequality | +| `$gt`, `$gte`, `$lt`, `$lte` | Comparison | +| `$in`, `$nin` | Membership | +| `$exists` | Field presence | +| `$and`, `$or` | Logical combinators | +| `$regex` | Regex match (string fields) | -if __name__ == "__main__": - server.run() -``` +### 5.2 Pushdown vs in-memory -### Upload a File +- **MongoDB**: native pushdown; queries run server-side. +- **SQLite**: translated to SQL via `SQLiteTranslator` (subset; complex `$or` chains may fall back). +- **DynamoDB**: limited pushdown via `Select=COUNT` and key conditions; remainder filtered client-side. +- **JSON**: full in-memory evaluation after loading matching collection. -```bash -curl -X POST -F "file=@document.pdf" \ - http://localhost:8000/storage/upload -``` +### 5.3 Compiled query cache -**Response:** -```json -{ - "success": true, - "file_path": "2025/01/05/document-abc123.pdf", - "file_size": 102400, - "content_type": "application/pdf" -} -``` +`QueryEngine` caches compiled queries (default LRU size 1024, configurable per database via `query_cache_size`). Cache bounds prevent unbounded growth from dynamic query construction. -### Create a Shareable Link +### 5.4 `Entity.find` field paths -```bash -curl -X POST http://localhost:8000/storage/proxy \ - -H "Content-Type: application/json" \ - -d '{ - "file_path": "2025/01/05/document-abc123.pdf", - "expires_in": 3600 - }' -``` - -**Response:** -```json -{ - "success": true, - "proxy_code": "a1b2c3d4", - "proxy_url": "http://localhost:8000/p/a1b2c3d4", - "expires_at": "2025-01-05T23:00:00Z" -} -``` +Caller-facing queries on `Entity.find({...})` operate on the persisted document shape. Entity attributes (other than top-level fields like `id`, `entity`, `edges`, `source`, `target`, `bidirectional`) live under a `context` sub-object. Use `"context.field_name"` as the query key for attribute fields. -### Access via Short URL - -```bash -curl http://localhost:8000/p/a1b2c3d4 -``` - -The file is served directly with appropriate headers. - -### Use in Walkers - -```python -from jvspatial.storage import get_file_interface -from jvspatial.core import Walker, on_visit, Node - -@server.walker("/process-upload") -class ProcessUpload(Walker): - file_path: str - - @on_visit(Node) - async def process(self, here: Node): - # Get file storage interface - storage = get_file_interface( - provider="local", - root_dir=".files" - ) - - # Read file content - content = await storage.get_file(self.file_path) - - # Process file content - self.report({ - "processed_file": { - "path": self.file_path, - "size": len(content), - "status": "success" - } - }) -``` - -### AWS S3 Configuration - -```python -server = Server( - title="S3 File API", - file_storage_enabled=True, - file_storage_provider="s3", - file_storage_s3_bucket="my-bucket", - file_storage_s3_region="us-east-1", - proxy_enabled=True -) -``` - -**Environment Variables:** -```env -JVSPATIAL_FILE_STORAGE_ENABLED=true -JVSPATIAL_FILE_STORAGE_PROVIDER=s3 -JVSPATIAL_FILE_STORAGE_S3_BUCKET=my-bucket -JVSPATIAL_FILE_STORAGE_S3_REGION=us-east-1 -AWS_ACCESS_KEY_ID=your-key-id -AWS_SECRET_ACCESS_KEY=your-secret-key -``` - -### Advanced Usage: Custom Upload Path - -```bash -curl -X POST -F "file=@image.jpg" \ - -F "custom_path=avatars/user123.jpg" \ - http://localhost:8000/storage/upload -``` +--- -### List Files +## 6. Walker Traversal Semantics -```bash -curl http://localhost:8000/storage/files?prefix=2025/01/ -``` +### 6.1 Lifecycle -**Response:** -```json -{ - "success": true, - "files": [ - { - "path": "2025/01/05/document-abc123.pdf", - "size": 102400, - "modified": "2025-01-05T20:30:00Z" - }, - { - "path": "2025/01/05/image-def456.jpg", - "size": 51200, - "modified": "2025-01-05T21:15:00Z" - } - ] -} -``` +1. Instantiate walker: `walker = MyWalker(...)`. +2. Begin: `await walker.spawn(start_node)` or `await walker.walk(start_node)`. +3. Per step: dequeue node, set visitor, run matching `@on_visit` hooks, optionally enqueue more nodes. +4. End: queue empty, `pause()` called, protection limit hit, or `WalkerTimeoutError` raised. -### Security Features - -```python -server = Server( - title="Secure File API", - file_storage_enabled=True, - file_storage_provider="local", - file_storage_root=".files", - file_storage_max_size=10485760, # 10MB limit - file_storage_allowed_types=["image/jpeg", "image/png", "application/pdf"], - proxy_enabled=True, - proxy_default_ttl=3600 # 1 hour default expiration -) -``` +### 6.2 Visit hooks -### Best Practices +- `@on_visit(NodeType)` on a Walker method — fires when the walker visits `NodeType` (or its subclass). +- `@on_visit(WalkerType)` on a Node method — fires when `WalkerType` visits *this* node. +- `@on_visit` with no target — fires for every visit. -**✅ Recommended Patterns:** +Hooks are registered at class-creation time in `__init_subclass__` (`node.py:62`, `edge.py:66`). -```python -# Good: Use environment variables for configuration -from dotenv import load_dotenv -load_dotenv() - -server = Server( - title="Production File API", - file_storage_enabled=True, - # Provider configured via JVSPATIAL_FILE_STORAGE_PROVIDER - # Other settings loaded from environment -) +### 6.3 Protection invariants -# Good: Validate files before processing -@endpoint("/validate-upload") -class ValidateUpload(Walker): - file_path: str - - @on_visit(Node) - async def validate(self, here: Node): - storage = get_file_interface() - - # Check file exists - if not await storage.file_exists(self.file_path): - self.report({"error": "File not found"}) - return - - # Get file metadata - metadata = await storage.get_metadata(self.file_path) - - # Validate size - if metadata.get("size", 0) > 5242880: # 5MB - self.report({"error": "File too large"}) - return - - self.report({"status": "valid", "metadata": metadata}) - -# Good: Use proxy URLs for temporary access -async def create_temp_link(file_path: str, hours: int = 1): - """Create temporary shareable link.""" - response = await storage.create_proxy( - file_path=file_path, - expires_in=hours * 3600 - ) - return response["proxy_url"] -``` +When `protection_enabled=True` (default): -**❌ Avoided Patterns:** +- Step counter increments on every visit; exceeding `max_steps` raises `WalkerExecutionError`. +- Per-node visit counter increments on every visit; exceeding `max_visits_per_node` raises `InfiniteLoopError`. +- Wall-clock timer started at `spawn`; exceeding `max_execution_time` raises `WalkerTimeoutError`. +- Enqueue refuses when queue length would exceed `max_queue_size` (silent drop, logged). -```python -# Bad: Hardcoding credentials -server = Server( - file_storage_s3_bucket="my-bucket", - file_storage_s3_access_key="AKIAIOSFODNN7EXAMPLE" # Don't do this! -) +Protection is *advisory* for safety, not for security — untrusted user input should not influence walker construction. -# Bad: No file validation -@endpoint("/unsafe-upload") -class UnsafeUpload(Walker): - file_path: str +### 6.4 Trail tracking - @on_visit(Node) - async def process(self, here: Node): - # No validation - could process malicious files - content = await storage.get_file(self.file_path) - # Direct processing without checks +`WalkerTrail` records `(node_id, edge_id, timestamp, node_type, queue_length, metadata)` for every visit (when `trail_enabled=True`). `max_trail_length=0` is unlimited; bounded by memory. In serverless deployments, trails do not persist across invocations. -# Bad: Permanent public URLs without expiration -# Always use proxy URLs with expiration for security -``` +### 6.5 Control flow -See [File Storage Documentation](docs/md/file-storage-usage.md) for advanced usage and all configuration options. +- `walker.pause()` — raises `TraversalPaused`; caller catches to suspend. +- `walker.skip()` — raises `TraversalSkipped`; advances to next queued node. +- `walker.resume()` — re-enters traversal from saved state. --- -## 🔀 Router Decorators - -jvspatial provides a unified `@endpoint` decorator for all API endpoints: - -1. `@endpoint` - For public endpoints (both functions and Walker classes) -2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) -3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) -4. `@endpoint(..., auth=True, roles=["admin"])` - For admin-only endpoints - -```python -from jvspatial.api import endpoint - -# Function endpoint -@endpoint("/api/users", methods=["GET"]) -async def get_users() -> Dict[str, Any]: - users = await User.all() - return {"users": users} - -# Walker endpoint -@endpoint("/api/graph/traverse", methods=["POST"]) -class GraphTraversal(Walker): - pass - -# Authenticated function endpoint -@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) -async def get_admin_stats() -> Dict[str, Any]: - return {"stats": "admin only"} - -# Authenticated walker endpoint (uses same decorator) -@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) -class SecureProcessor(Walker): - pass - -# Admin-only endpoint -@endpoint("/api/admin/users", auth=True, roles=["admin"], methods=["GET"]) -async def manage_users() -> Dict[str, Any]: - return {"users": "admin access"} -``` - -**❌ DO NOT USE alternative decorators like:** -- `@route` -- `@server.route` -- `@server.walker` -- `@walker_endpoint` (deprecated - use `@endpoint` instead) -- `@auth_endpoint` (deprecated - use `@endpoint(..., auth=True)` instead) -- `@admin_endpoint` (deprecated - use `@endpoint(..., auth=True, roles=["admin"])` instead) - -These are internal or deprecated. - -## 📌 Consolidated Endpoint System - -jvspatial uses a **unified endpoint registration system** where all endpoints (walkers and functions) are registered through a single consolidated mechanism. This ensures clean, maintainable code without backward compatibility cruft. +## 7. GraphContext and Dependency Injection -### Key Architecture +`jvspatial/core/context.py` — `GraphContext` binds a `Database` + optional `Cache` + `PerformanceMonitor` to a scope. -All decorators follow the same registration path: +### 7.1 Resolution order -1. **Decorator** → Attaches metadata to function/walker -2. **Server Detection** → Gets current server from context -3. **Registration** → Registers with `server.endpoint_router` -4. **Tracking** → Tracked by `server._endpoint_registry` +When an entity needs a context, it resolves in this order: +1. Explicitly set via `await entity.set_context(ctx)`. +2. Default context: `get_default_context()` (singleton, lazily initialized from environment). -### Important: Decorator Order +### 7.2 Scoping -Always create the server **before** decorating endpoints: +`GraphContext` is request-scoped by convention. The API server installs a per-request context via middleware (`jvspatial/api/components/auth_middleware.py` and lifecycle), so endpoint handlers reach the correct database without manual injection. -```python -# ✅ CORRECT -server = Server(title="My API") - -@endpoint("/test") -class TestWalker(Walker): - pass - -# ✗ INCORRECT - endpoint will not be registered -@endpoint("/test") -class TestWalker(Walker): - pass - -server = Server(title="My API") # Created too late -``` +### 7.3 Performance monitoring -### Default HTTP Methods +`PerformanceMonitor` (within `GraphContext`) records: +- DB operation counts and latencies +- Hook execution counts and latencies +- Cache hits and misses -- **Walkers**: Default to `["POST"]` -- **Functions**: Default to `["GET"]` +Surfaced via `observability/` for export to metrics sinks. -Override with the `methods` parameter: -```python -@endpoint("/data", methods=["GET", "POST"]) -class DataWalker(Walker): - pass -``` +--- -### Available Response Methods +## 8. API Surface -Function endpoints can receive an `endpoint` parameter for response formatting: +### 8.1 Server class -```python -@endpoint("/info") -async def get_info(endpoint): - # Use endpoint.success(), endpoint.error(), etc. - return endpoint.success(data={"info": "value"}) -``` +`jvspatial/api/server.py` — composition of four mixins: -Walkers automatically have `self.endpoint` available: +| Mixin | File | Concern | +|---|---|---| +| `AppFactoryMixin` | `api/server_app_factory.py` | Build FastAPI app, init DB, set up CORS | +| `RegistrationMixin` | `api/server_registration.py` | Register routes from `@endpoint`-decorated targets | +| `LifecycleMixin` | `api/server_lifecycle.py` | Startup/shutdown hooks, context wiring | +| `RunMixin` | `api/server_run.py` | Uvicorn invocation, host/port resolution | -```python -@endpoint("/process") -class ProcessWalker(Walker): - async def process(self): - self.response = self.endpoint.success(data={"result": "done"}) -``` +### 8.2 `@endpoint` decorator -**Response Methods:** +`jvspatial/api/decorators/route.py` — single decorator for both async functions and `Walker` subclasses: ```python -# Success responses -endpoint.success(data=result, message="Success") # 200 OK -endpoint.created(data=new_item, message="Created") # 201 Created -endpoint.no_content() # 204 No Content - -# Error responses -endpoint.bad_request(message="Invalid input") # 400 Bad Request -endpoint.unauthorized(message="Auth required") # 401 Unauthorized -endpoint.forbidden(message="Access denied") # 403 Forbidden -endpoint.not_found(message="Resource not found") # 404 Not Found -endpoint.conflict(message="Resource exists") # 409 Conflict -endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity - -# Flexible custom response -endpoint.response( - content={"custom": "data"}, - status_code=202, - headers={"X-Custom": "value"} -) +@endpoint("/users/{user_id}", methods=["GET"], auth=True, roles=["admin"]) +async def get_user(user_id: str): ... -# Generic error with custom status code -endpoint.error( - message="Custom error", - status_code=418, - details={"reason": "custom"} -) +@endpoint("/walk", methods=["POST"]) +class MyWalker(Walker): + ... ``` -### Querying Registered Endpoints - -```python -server = Server(title="My API") +**Parameters**: +- `path` — FastAPI path with parameters +- `methods` — HTTP methods list +- `auth: bool` — require authentication (default `False`) +- `roles: list[str]` — RBAC roles +- `webhook: bool` / `webhook_model` — webhook handler mode with signature verification +- `response` — response schema using `ResponseField` + `success_response()` -# ... register some endpoints ... +### 8.3 Deferred registry -# List all endpoints -all_endpoints = server.list_all_endpoints() -print(f"Walkers: {len(all_endpoints['walkers'])}") -print(f"Functions: {len(all_endpoints['functions'])}") +Endpoints decorated at import time are collected in a deferred registry (`api/endpoints/registry.py`). The server resolves them at app-build time, after the database context is available. This allows entity-bound endpoints to be declared at module scope. -# List just walkers -walkers = server.list_walker_endpoints() +### 8.4 Built-in endpoints -# List just functions -functions = server.list_function_endpoints() +When `auth_enabled=True`, the server registers `/auth/register`, `/auth/login`, `/auth/logout`, plus token refresh and password reset endpoints. With `auth_enabled=False`, no auth endpoints are registered. -# Get registry stats -registry = server._endpoint_registry -counts = registry.count_endpoints() -print(f"Total: {counts['total']}") -``` +OpenAPI docs at `/docs` and `/redoc` unless `JVSPATIAL_DOCS_DISABLED` is truthy (see §10.5). -### Unified Registration Benefits +--- -1. **Single Source of Truth**: All endpoints are registered through `EndpointRouter` -2. **Cleaner Code**: No backward compatibility cruft or deprecated methods -3. **Consistent API**: All decorators follow the same pattern -4. **Better Maintainability**: Future endpoint features only need to be added once -5. **Auto-Detection**: Decorators automatically detect walker vs function +## 9. Authentication and Authorization -### Migration from Deprecated Patterns +### 9.1 Authentication mechanisms -If you have code using removed patterns: +- **JWT**: `PyJWT`-based. Secret required at startup (`jwt_secret` config or `JVSPATIAL_JWT_SECRET_KEY` env). Server **fails fast** on missing secret. +- **API keys**: SHA-256 hashed at rest, plaintext returned **only once** on creation. Verification uses `hmac.compare_digest` (constant-time). +- **Refresh tokens**: rotate on use; previous tokens invalidated. +- **Password reset**: `token_lookup` field provides O(1) lookup; constant-time comparison. -**❌ Old Pattern (Removed)** -```python -# These no longer exist -server._custom_routes.append(...) -server._register_custom_routes(app) -server._setup_webhook_walker_endpoints() -``` +### 9.2 Authorization (RBAC) -**✅ New Pattern (Current)** -```python -# Use the decorators - they handle registration automatically -@endpoint("/my-route") -def my_handler(): - pass +`jvspatial/api/auth/rbac.py` — roles map to permission unions. Wildcard support (e.g. `users:*`). Admin-only routes enforced on `/status`, `/logs`, and `/graph` subtrees by default. -# Or register programmatically -server.register_walker_class(MyWalker, "/my-route", methods=["POST"]) -``` +### 9.3 Session management -## 🌐 API Integration with FastAPI Server +`SessionManager` tracks active sessions per user. **Per-process, in-memory** — true limits in multi-worker deployments are `limit × workers`. Document this when configuring session caps. -The **jvspatial API** provides seamless integration with FastAPI to expose your graph operations as REST endpoints. It supports flexible endpoint registration using decorators and automatic parameter model generation from Walker and function properties. +### 9.4 Logout blacklist -### Server Setup and Configuration +JWT tokens are blacklisted on logout. Blacklist storage is per-worker in default config. Cross-worker invalidation requires a shared blacklist store (caller-provided). -```python path=null start=null -from jvspatial.api import Server, ServerConfig, endpoint, walker_endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Node, Walker -from jvspatial.core.entities import on_visit +### 9.5 Webhook authentication -# Basic server setup -server = Server( - title="My Spatial API", - description="Graph-based spatial data management API", - version="1.0.0", - host="0.0.0.0", - port=8000, - debug=True # Enable for development -) +`jvspatial/api/integrations/webhooks/` — HMAC signature verification with constant-time comparison. Per-source secret rotation supported. Replay protection via timestamp window (configurable). -# Advanced server configuration -advanced_config = ServerConfig( - title="Production Spatial API", - description="Enterprise graph data API", - version="2.0.0", - host="0.0.0.0", - port=8080, - # Database configuration - db_type="mongodb", - db_connection_string="mongodb://localhost:27017", - db_database_name="spatial_db", - # CORS settings - cors_enabled=True, - cors_origins=["https://myapp.com", "http://localhost:3000"], - # API documentation - docs_url="/api/docs", - redoc_url="/api/redoc", - log_level="info" -) +--- -production_server = Server(config=advanced_config) -``` +## 10. Configuration and Environment -### @walker_endpoint Decorator for Walker Classes - -The `@walker_endpoint` decorator automatically exposes Walker classes as API endpoints: - -```python path=null start=null -from jvspatial.api import endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit -from typing import List, Optional - -# Define your node types -class User(Node): - name: str = "" - email: str = "" - department: str = "" - active: bool = True - -class City(Node): - name: str = "" - population: int = 0 - state: str = "" - -# Walker with endpoint configuration using EndpointField -@endpoint("/api/users/process", methods=["POST"]) -class ProcessUser(Walker): - """Process user data with graph traversal.""" - - # Endpoint-exposed fields with configuration - user_name: str = EndpointField( - description="Name of the user to process", - examples=["John Doe", "Jane Smith"], - min_length=2, - max_length=100 - ) - - department: str = EndpointField( - default="general", - description="User department", - examples=["engineering", "marketing", "sales"] - ) - - include_inactive: bool = EndpointField( - default=False, - description="Whether to include inactive users in processing" - ) - - # Field excluded from API endpoint - internal_state: str = EndpointField( - default="processing", - exclude_endpoint=True # Won't appear in API schema - ) - - # Optional configuration field - max_connections: int = EndpointField( - default=10, - description="Maximum number of connections to traverse", - ge=1, - le=100 - ) - - @on_visit("User") - async def process_user(self, here: User): - """Process user nodes during traversal. - - Args: - here: The visited User node - """ - # Check if user matches criteria - if here.name == self.user_name: - self.report({ - "found_user": { - "id": here.id, - "name": here.name, - "email": here.email, - "department": here.department - } - }) - - # Find connected users in same department - colleagues = await here.nodes( - node=['User'], - department=self.department, - active=True if not self.include_inactive else None - ) - - self.report({ - "colleagues": [ - {"name": u.name, "email": u.email} - for u in colleagues[:self.max_connections] - ] - }) - - @on_visit("City") - async def process_city_connection(self, here: City): - """Process city connections if user has location data. - - Args: - here: The visited City node - """ - self.report({ - "city_info": { - "name": here.name, - "population": here.population, - "state": here.state - } - }) - -# Advanced walker with field grouping -@endpoint("/api/analytics/user-report", methods=["POST"]) -class UserAnalytics(Walker): - """Generate user analytics reports.""" - - # Grouped fields for better API organization - report_type: str = EndpointField( - description="Type of report to generate", - examples=["summary", "detailed", "connections"], - endpoint_group="report_config" - ) - - date_range: str = EndpointField( - default="30d", - description="Date range for report", - examples=["7d", "30d", "90d", "1y"], - endpoint_group="report_config" - ) - - include_inactive: bool = EndpointField( - default=False, - description="Include inactive users", - endpoint_group="filters" - ) - - departments: List[str] = EndpointField( - default_factory=list, - description="Departments to include (empty = all)", - examples=[["engineering", "product"], ["marketing"]], - endpoint_group="filters" - ) - - @on_visit("User") - async def analyze_user(self, here: User): - """Analyze user data for report. - - Args: - here: The visited User node - """ - # Filter by department if specified - if self.departments and here.department not in self.departments: - return - - # Filter by active status if needed - if not self.include_inactive and not here.active: - return - - # Report individual user analysis - self.report({ - "user_analyzed": { - "id": here.id, - "name": here.name, - "department": here.department, - "active": here.active - } - }) - } -``` +### 10.1 ServerConfig -### Enhanced Response Handling with endpoint.response() - -The `@walker_endpoint` and `@endpoint` decorators now automatically inject semantic response helpers to make crafting HTTP responses clean and flexible: - -**Walker Endpoints with self.endpoint:** - -```python path=null start=null -@endpoint("/api/users/profile", methods=["POST"]) -class UserProfileWalker(Walker): - """Walker demonstrating semantic response patterns.""" - - user_id: str = EndpointField(description="User ID to retrieve") - include_details: bool = EndpointField( - default=False, - description="Include detailed profile information" - ) - - @on_visit("User") - async def get_user_profile(self, here: User): - """Get user profile with semantic responses.""" - if here.id != self.user_id: - return # Continue traversal - - # User not found scenario - if not here.data: - return self.endpoint.not_found( - message="User profile not found", - details={"user_id": self.user_id} - ) - - # Authorization check - if here.private and not self.include_details: - return self.endpoint.forbidden( - message="Insufficient permissions", - details={"required_permission": "view_details"} - ) - - # Successful response - profile_data = { - "id": here.id, - "name": here.name, - "email": here.email - } - - if self.include_details: - profile_data["department"] = here.department - profile_data["created_at"] = here.created_at - - return self.endpoint.success( - data=profile_data, - message="User profile retrieved successfully" - ) - -@endpoint("/api/users/create", methods=["POST"]) -class CreateUserWalker(Walker): - """Walker for creating users with proper HTTP status codes.""" - - name: str = EndpointField(description="User name") - email: str = EndpointField(description="User email") - - @on_visit("Root") - async def create_user(self, here): - """Create a new user with validation.""" - - # Validation example - if "@" not in self.email: - return self.endpoint.unprocessable_entity( - message="Invalid email format", - details={"email": self.email} - ) - - # Check for conflicts - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await User.find({"context.email": self.email}) - existing_user = users[0] if users else None - if existing_user: - return self.endpoint.conflict( - message="User with this email already exists", - details={"email": self.email} - ) - - # Create user - user = await User.create( - name=self.name, - email=self.email - ) - - # Return 201 Created with location header - return self.endpoint.created( - data={ - "id": user.id, - "name": user.name, - "email": user.email - }, - message="User created successfully", - headers={"Location": f"/api/users/{user.id}"} - ) -``` +`jvspatial/api/config.py` — Pydantic model. Hierarchical groups: -**Function Endpoints with endpoint parameter:** - -```python path=null start=null -@endpoint("/api/health", methods=["GET"]) -async def health_check(endpoint) -> Any: - """Health check with semantic response.""" - return endpoint.success( - data={"status": "healthy", "version": "1.0.0"}, - message="Service is running normally" - ) - -@endpoint("/api/users/{user_id}/status", methods=["PUT"]) -async def update_user_status(user_id: str, status: str, endpoint) -> Any: - """Update user status with validation and error handling.""" - - # Validation - valid_statuses = ["active", "inactive", "suspended"] - if status not in valid_statuses: - return endpoint.bad_request( - message="Invalid status value", - details={"provided": status, "valid_options": valid_statuses} - ) - - # Find user - user = await User.get(user_id) - if not user: - return endpoint.not_found( - message="User not found", - details={"user_id": user_id} - ) - - # Update status - user.status = status - await user.save() - - return endpoint.success( - data={"id": user.id, "status": user.status}, - message=f"User status updated to {status}" - ) - -@endpoint("/api/export", methods=["GET"]) -async def export_data(format: str, endpoint) -> Any: - """Export data with custom response formatting.""" - - if format not in ["json", "csv", "xml"]: - return endpoint.error( - message="Unsupported export format", - status_code=406, # Not Acceptable - details={"format": format, "supported": ["json", "csv", "xml"]} - ) - - # Generate export data - export_data = { - "format": format, - "records": 1500, - "export_id": "exp_20250921", - "download_url": f"/downloads/export.{format}" - } - - # Use flexible response() method for custom headers - return endpoint.response( - content={ - "data": export_data, - "message": f"Export ready in {format} format" - }, - status_code=200, - headers={ - "X-Export-Format": format, - "X-Record-Count": "1500" - } - ) ``` - -**Available Response Methods:** - -```python path=null start=null -# Success responses -endpoint.success(data=result, message="Success") # 200 OK -endpoint.created(data=new_item, message="Created") # 201 Created -endpoint.no_content() # 204 No Content - -# Error responses -endpoint.bad_request(message="Invalid input") # 400 Bad Request -endpoint.unauthorized(message="Auth required") # 401 Unauthorized -endpoint.forbidden(message="Access denied") # 403 Forbidden -endpoint.not_found(message="Resource not found") # 404 Not Found -endpoint.conflict(message="Resource exists") # 409 Conflict -endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity - -# Flexible custom response -endpoint.response( - content={"custom": "data"}, - status_code=202, - headers={"X-Custom": "value"} +ServerConfig( + title, description, version, debug, docs_url, redoc_url, + host, port, serverless_mode, deferred_task_provider, + scheduler_enabled, scheduler_interval, + database=DatabaseConfig(...), + security=SecurityConfig(...), + cors=CORSConfig(...), + auth=AuthConfig(...), + rate_limit=RateLimitConfig(...), + file_storage=FileStorageConfig(...), + webhook=WebhookConfig(...), + proxy=ProxyConfig(...), + log_level, startup_hooks, shutdown_hooks, + graph_endpoint_enabled=True, ) - -# Generic error with custom status code -endpoint.error( - message="Custom error", - status_code=418, # I'm a teapot - details={"reason": "custom"} -) -``` - -### @endpoint Decorator for Regular Functions -The `@endpoint` decorator exposes regular functions as API endpoints: - -```python path=null start=null -from jvspatial.api import endpoint -from fastapi import HTTPException -from typing import Dict, Any, List - -# Simple function endpoint -@endpoint("/api/users/count", methods=["GET"]) -async def get_user_count() -> Dict[str, int]: - """Get total count of users in the system.""" - users = await User.all() - return {"total_users": len(users)} - -# Function endpoint with path parameters -@endpoint("/api/cities/{state}", methods=["GET"]) -async def get_cities_by_state(state: str) -> Dict[str, Any]: - """Get all cities in a specific state.""" - cities = await City.find({"context.state": state}) - - if not cities: - raise HTTPException( - status_code=404, - detail=f"No cities found in state: {state}" - ) - - return { - "state": state, - "cities": [ - { - "name": city.name, - "population": city.population - } for city in cities - ], - "total_count": len(cities) - } - -# Function endpoint with request body -@endpoint("/api/cities/search", methods=["POST"]) -async def search_cities(search_request: Dict[str, Any]) -> Dict[str, Any]: - """Search cities based on criteria.""" - # Extract search parameters - name_pattern = search_request.get("name_pattern") - min_population = search_request.get("min_population", 0) - state = search_request.get("state") - - # Build MongoDB-style query - query = {} - - if name_pattern: - query["context.name"] = {"$regex": name_pattern, "$options": "i"} - - if min_population > 0: - query["context.population"] = {"$gte": min_population} - - if state: - query["context.state"] = state - - # Execute search - cities = await City.find(query) - - return { - "search_criteria": search_request, - "results": [ - { - "id": city.id, - "name": city.name, - "population": city.population, - "state": city.state - } for city in cities - ], - "total_results": len(cities) - } - -# Function endpoint with pagination integration -@endpoint("/api/users/paginated", methods=["GET"]) -async def get_users_paginated( - page: int = 1, - page_size: int = 20, - department: Optional[str] = None, - active_only: bool = True -) -> Dict[str, Any]: - """Get paginated list of users with filtering.""" - from jvspatial.core.pager import ObjectPager - - # Build filters - filters = {} - if department: - filters["context.department"] = department - if active_only: - filters["context.active"] = True - - # Create pager - pager = ObjectPager( - User, - page_size=page_size, - filters=filters, - order_by="name", - order_direction="asc" - ) - - # Get page data - users = await pager.get_page(page) - pagination_info = pager.to_dict() - - return { - "users": [ - { - "id": user.id, - "name": user.name, - "email": user.email, - "department": user.department, - "active": user.active - } for user in users - ], - "pagination": pagination_info - } -``` - -### Server Method Registration - -You can also register endpoints directly on server instances: - -```python path=null start=null -# Using server instance decorators -@server.walker("/process-data", methods=["POST"]) -class DataProcessor(Walker): - """Process data using server instance registration.""" - - data_type: str = EndpointField( - description="Type of data to process", - examples=["user", "city", "connection"] - ) - - batch_size: int = EndpointField( - default=10, - description="Batch size for processing", - ge=1, - le=1000 - ) - - @on_visit("Node") - async def process_any_node(self, here: Node): - """Process any type of node. - - Args: - here: The visited Node - """ - # Use report() to collect processed node information - self.report({ - "processed_node": { - "id": here.id, - "type": here.__class__.__name__, - "processed_at": datetime.now().isoformat() - } - }) - -@server.route("/health-detailed", methods=["GET"]) -async def detailed_health_check() -> Dict[str, Any]: - """Detailed health check endpoint.""" - try: - # Test database connectivity - users_count = await User.count() - cities_count = await City.count() - - return { - "status": "healthy", - "database": "connected", - "statistics": { - "total_users": users_count, - "total_cities": cities_count - }, - "timestamp": datetime.now().isoformat() - } - except Exception as e: - raise HTTPException( - status_code=503, - detail=f"Health check failed: {str(e)}" - ) -``` - -### EndpointField Configuration Options - -The `EndpointField` provides extensive configuration for API parameters: - -```python path=null start=null -from jvspatial.api.decorators import EndpointField -from typing import List, Optional - -@endpoint("/api/advanced-example", methods=["POST"]) -class AdvancedEndpointExample(Walker): - """Demonstrate all EndpointField configuration options.""" - - # Basic field with validation - username: str = EndpointField( - description="User identifier", - examples=["john_doe", "jane_smith"], - min_length=3, - max_length=50, - pattern=r"^[a-zA-Z0-9_]+$" # Alphanumeric with underscores - ) - - # Numeric field with constraints - user_score: float = EndpointField( - description="User score rating", - examples=[85.5, 92.0, 78.3], - ge=0.0, # Greater than or equal to 0 - le=100.0 # Less than or equal to 100 - ) - - # Field with custom endpoint name - internal_id: str = EndpointField( - endpoint_name="user_id", # Will appear as 'user_id' in API - description="Internal user identifier" - ) - - # Optional field made required for endpoint - optional_field: Optional[str] = EndpointField( - default=None, - endpoint_required=True, # Required in API despite being Optional in Walker - description="Field that's optional in Walker but required in API" - ) - - # Required field made optional for endpoint - required_field: str = EndpointField( - endpoint_required=False, # Optional in API despite being required in Walker - description="Field that's required in Walker but optional in API" - ) - - # Grouped fields for organized API schema - config_timeout: int = EndpointField( - default=30, - description="Timeout in seconds", - endpoint_group="configuration", - ge=1, - le=300 - ) - - config_retries: int = EndpointField( - default=3, - description="Number of retries", - endpoint_group="configuration", - ge=0, - le=10 - ) - - # Hidden field (not shown in API docs) - debug_mode: bool = EndpointField( - default=False, - endpoint_hidden=True, # Hidden from OpenAPI documentation - description="Debug mode flag" - ) - - # Deprecated field - legacy_option: Optional[str] = EndpointField( - default=None, - endpoint_deprecated=True, # Marked as deprecated in API docs - description="Legacy option - use new_option instead" - ) - - # Field with additional constraints - email_domain: str = EndpointField( - description="Allowed email domain", - endpoint_constraints={ - "format": "hostname", # Additional OpenAPI constraint - "example": "company.com" - } - ) - - # List field with validation - tags: List[str] = EndpointField( - default_factory=list, - description="User tags", - examples=[["admin", "power-user"], ["guest"]] - ) - - @on_visit("User") - async def process_with_config(self, here: User): - """Process user with advanced configuration. - - Args: - here: The visited User node - """ - self.report({ - "processed_user": { - "username": self.username, - "score": self.user_score, - "config": { - "timeout": self.config_timeout, - "retries": self.config_retries - }, - "tags": self.tags, - "debug_enabled": self.debug_mode - } - }) ``` -### Running the Server - -```python path=null start=null -# Development server -if __name__ == "__main__": - server.run( - host="0.0.0.0", - port=8000, - reload=True # Auto-reload for development - ) - -# Production server with custom configuration -async def run_production_server(): - """Run server asynchronously for production deployment.""" - await server.run_async( - host="0.0.0.0", - port=8080 - ) - -# Custom startup and shutdown hooks -@server.on_startup -async def startup_tasks(): - """Tasks to run on server startup.""" - print("🚀 Server starting up...") - # Initialize data, warm up caches, etc. - -@server.on_shutdown -async def shutdown_tasks(): - """Tasks to run on server shutdown.""" - print("🛑 Server shutting down...") - # Cleanup resources, save state, etc. - -# Access the underlying FastAPI app if needed -fastapi_app = server.get_app() - -# Custom middleware -@server.middleware("http") -async def log_requests(request, call_next): - """Log all API requests.""" - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - print(f"{request.method} {request.url} - {response.status_code} ({process_time:.2f}s)") - return response -``` - -### Router Decorators - -jvspatial provides four standard router decorators for API endpoints. These are the ONLY decorators that should be used for routing: +Flat keyword arguments (e.g. `Server(db_type=..., jwt_secret=...)`) are mapped to the appropriate group by a model validator (lines 103-128). The flat form is convenient; the hierarchical form is canonical. -1. `@endpoint` - For public endpoints (both functions and Walker classes) -2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) -3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) +### 10.2 Merge order -```python -# Function endpoint -@endpoint("/api/users", methods=["GET"]) -async def get_users() -> Dict[str, Any]: - users = await User.all() - return {"users": users} - -# Walker endpoint -@endpoint("/api/graph/traverse", methods=["POST"]) -class GraphTraversal(Walker): - pass - -# Authenticated function endpoint -@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) -async def get_admin_stats() -> Dict[str, Any]: - return {"stats": "admin only"} - -# Authenticated walker endpoint (uses same decorator) -@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) -class SecureProcessor(Walker): - pass -``` +`Server(...)` arguments merge in this order (later wins): -DO NOT use alternative decorators like `@route`, `@server.route`, or `@server.walker`. These are internal or deprecated. +1. `ServerConfig` defaults +2. Allowlisted `JVSPATIAL_*` environment variables (see `jvspatial/env_adapter.py`) +3. `config=` dict or keyword arguments -### API Usage Examples +Unknown `JVSPATIAL_*` keys are rejected at startup to catch typos and removed settings. -Once your server is running, endpoints are automatically available: +### 10.3 Environment variable allowlist -```bash -# Walker endpoint - POST request with parameters -curl -X POST "http://localhost:8000/api/users/process" \ - -H "Content-Type: application/json" \ - -d '{ - "user_name": "John Doe", - "department": "engineering", - "include_inactive": false, - "max_connections": 5, - "start_node": "n:Root:root" - }' +`jvspatial/env_adapter.py` lists every accepted `JVSPATIAL_*` key with type coercion rules. The canonical reference is [docs/md/environment-keys-reference.md](docs/md/environment-keys-reference.md). -# Function endpoint - GET request -curl "http://localhost:8000/api/users/count" +### 10.4 Database environment -# Function endpoint with path parameters -curl "http://localhost:8000/api/cities/CA" +`jvspatial/env.py` provides `resolve_db_paths()`, `parse_bool()`, and friends. Database type defaults: +- `JVSPATIAL_DB_TYPE` — backend selector +- `JVSPATIAL_DB_PATH` — backend-specific path or URI -# Function endpoint with query parameters -curl "http://localhost:8000/api/users/paginated?page=1&page_size=10&department=engineering" +### 10.5 Docs gating -# API documentation is automatically available -# http://localhost:8000/docs (Swagger UI) -# http://localhost:8000/redoc (ReDoc) -``` - -### Best Practices - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Use descriptive endpoint paths -@endpoint("/api/users/analyze-connections", methods=["POST"]) -class AnalyzeUserConnections(Walker): - pass - -# Good: Provide comprehensive field documentation -field_name: str = EndpointField( - description="Clear description of what this field does", - examples=["example1", "example2"], - min_length=1, - max_length=100 -) +`JVSPATIAL_DOCS_DISABLED` (truthy values: `1`, `true`, `yes`, `on`) disables `/docs`, `/redoc`, `/openapi.json`, and `/docs/oauth2-redirect` at app build time. CSP headers are relaxed only on docs paths to allow the Swagger UI CDN; app routes retain strict CSP. -# Good: Use appropriate HTTP methods -@endpoint("/api/users", methods=["GET"]) # Retrieve data -@endpoint("/api/users", methods=["POST"]) # Create data -@endpoint("/api/process", methods=["POST"]) # Process/execute - -# Good: Group related fields -config_field: str = EndpointField( - endpoint_group="configuration", - description="Configuration parameter" -) - -# Good: Handle errors appropriately in functions -@endpoint("/api/data") -async def get_data(): - try: - data = await fetch_data() - return {"data": data} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) -``` - -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Vague endpoint paths -@endpoint("/process") # Too generic -@endpoint("/api/thing") # Unclear purpose - -# Bad: Missing field documentation -field_name: str = EndpointField() # No description or examples - -# Bad: Exposing internal fields -internal_state: str = EndpointField() # Should use exclude_endpoint=True - -# Bad: Not handling errors in function endpoints -@endpoint("/api/data") -async def get_data(): - data = await risky_operation() # No error handling - return data # Could fail with 500 error - -# Bad: Using wrong HTTP methods -@endpoint("/api/users/delete", methods=["GET"]) # Should be DELETE -@endpoint("/api/data/get", methods=["POST"]) # Should be GET for retrieval -``` +**Source of truth**: `jvspatial/api/components/app_builder.py`. --- -## 🏗️ Library Architecture Concepts - -**jvspatial** is an asynchronous, object-spatial Python library designed for building robust persistence and business logic application layers. Inspired by Jaseci's object-spatial paradigm and leveraging Python's async capabilities, jvspatial empowers developers to model complex relationships, traverse object graphs, and implement agent-based architectures that scale with modern cloud-native concurrency requirements. Key capabilities: - -- Typed node/edge modeling via Pydantic -- Precise control over graph traversal -- Multi-backend persistence (JSON, MongoDB, etc.) -- Integrated REST API endpoints -- Async/await architecture - -### Core Entities - -1. **Object** - Base class for all entities with unified query interface; used to store non-graph data -2. **Node** - Extends Object, represents graph nodes with spatial/contextual data; used only as part of a graph -3. **Edge** - Represents relationships between nodes on a graph -4. **Walker** - Implements graph traversal and pathfinding algorithms -5. **GraphContext** - Low-level database interface (use sparingly) - -Once a graph is established (nodes and edges are connected in a meaningful way), a walker may be spawned on the root node or anywhere on the graph. The visit method enacts the walker's traversal using a starting point or a list of nodes on the walker's visit queue. - -As the walker traverses, it may conditionally trigger methods depending on its position on the graph. This is accomplished by the @on_visit annotation on the walker class. Similarly, as the walker traverses over nodes and edges, these entities may conditionally trigger their methods based on the walker's visitation; also accomplished through the @on_visit annotation on the node/edge class. - -**Execution Order**: When a walker visits a node/edge: -1. **Walker hooks** (methods decorated with `@on_visit` on the walker class) execute first -2. **Node/Edge hooks** (methods decorated with `@on_visit` on the node/edge class) are automatically executed after - -Node/Edge hooks are automatically invoked by the walker - no explicit call is needed. The walker binds the hook to the node/edge instance and calls it with the walker as a parameter. - -### Walker Traversal Pattern - -The **recommended approach** for walker traversal is to use the `nodes()` method to get connected nodes for continued traversal. - -#### Naming Convention for @on_visit Methods +## 11. Serverless Constraints -**IMPORTANT**: When writing `@on_visit` decorated methods, use the following naming convention: +### 11.1 Detection precedence -- **`here`**: Parameter name for the visited node/edge (the current location) -- **`visitor`**: Parameter name for the visiting walker (when accessing walker from node context) +`is_serverless_mode(config=None)` (`jvspatial/runtime/serverless.py:66-87`) resolves in this order: -```python path=null start=null -# ✅ RECOMMENDED: Use 'here' for the visited node -@on_visit("User") -async def process_user(self, here: Node): - """Process user node - 'here' represents the current User being visited. +1. Explicit `config.serverless_mode` if set (not `None`) +2. Same via `get_current_server()` when `config` is omitted +3. `SERVERLESS_MODE` env var (`true`/`1`/`yes`/`enabled`) +4. Auto-detection from platform env vars (Lambda, Azure Functions, Cloud Run, Vercel) - Args: - here: The visited User node - """ - print(f"Currently visiting user: {here.name}") - connected_users = await here.nodes(node=['User']) - await self.visit(connected_users) +Detection results are memoized via `lru_cache`; tests call `reset_serverless_mode_cache()` between cases. -# ✅ RECOMMENDED: Use 'visitor' when walker is passed to node methods -def node_method_example(self, visitor: Walker): - """Node method that receives the visiting walker. +### 11.2 Mode-dependent behavior - Args: - visitor: The walker currently visiting this node - """ - print(f"Being visited by walker: {visitor.__class__.__name__}") +| Behavior | Standard | Serverless | +|---|---|---| +| Deferred saves enabled by env | Yes (`JVSPATIAL_ENABLE_DEFERRED_SAVES`) | **No (forced off)** | +| Default JSON DB path | Working directory | `/tmp` (Lambda-writable) | +| bcrypt rounds | 12 | 10 (faster cold start) | +| Auto-create DB indexes | Yes | No (CloudWatch log cost) | +| JsonDB orphan cleanup | Yes | Skipped | +| Walker trail persistence | In-memory only | In-memory only (lost on cold start) | -# ❌ AVOID: Generic parameter names -@on_visit("User") -async def process_user(self, node: Node): # Less clear - pass +### 11.3 Deferred task dispatch -@on_visit("User") -async def process_user(self, n: Node): # Too abbreviated - pass -``` - -**Why this convention?** -- **`here`** clearly indicates the current location in graph traversal -- **`visitor`** clearly indicates the active walker performing the traversal -- Consistent with spatial/navigational metaphors used throughout jvspatial -- Makes code more readable and self-documenting -- Aligns with the library's entity-centric philosophy - -```python -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit, on_exit - -class DataCollector(Walker): - def __init__(self): - super().__init__() - self.collected_data = [] - self.processed_count = 0 - - @on_visit("User") - async def collect_user_data(self, here: Node): - """Called when walker visits a User node. - - Args: - here: The visited User node - """ - self.collected_data.append(here.name) - - # RECOMMENDED: Use nodes() method to get connected nodes - # Default direction="out" follows outgoing edges (forward traversal) - next_nodes = await here.nodes() - await self.visit(next_nodes) - - # Example with semantic filtering - filter connected Users by department - engineering_users = await here.nodes( - node=['User'], # Only User nodes - department="engineering", # Simple kwargs filtering - active=True # Multiple simple filters - ) - await self.visit(engineering_users) - - @on_visit("City") - async def process_city(self, here: Node): - """Process city nodes with filtering and control flow. - - Args: - here: The visited City node - """ - self.processed_count += 1 - - # Skip processing for certain conditions - if here.population < 10000: - print(f"Skipping small city: {here.name}") - self.skip() # Skip to next node in queue - return # This line won't be reached - - # Disengage if we've processed enough - if self.processed_count >= 10: - print("Processed enough cities, disengaging...") - await self.disengage() # Permanently halt and remove from graph - return - - # Continue with normal processing - large_cities = await here.nodes( - node=[{'City': {"context.population": {"$gte": 500_000}}}], - direction="out" # Explicit direction for clarity - ) - await self.visit(large_cities) - - # Example: Mixed filtering approach - dict filters + kwargs - nearby_cities = await here.nodes( - node=[{'City': {"context.region": here.region}}], # Complex filter - state="NY", # Simple kwargs filter - active=True # Additional simple filter - ) - await self.visit(nearby_cities) - - @on_exit - async def cleanup_and_report(self): - """Called when walker completes or disengages.""" - print(f"Walker finished! Collected {len(self.collected_data)} items") - print(f"Processed {self.processed_count} cities") - # Perform cleanup, save results, send notifications, etc. -``` - -### Walker Control Flow Methods - -#### `skip()` - Skip Current Node Processing -The `skip()` method allows you to immediately halt processing of the current node and proceed to the next node in the queue, similar to `continue` in a loop: - -```python -class ConditionalWalker(Walker): - @on_visit("Product") - async def process_product(self, here: Node): - """Process product nodes with conditional skipping. - - Args: - here: The visited Product node - """ - # Skip discontinued products - if here.status == "discontinued": - self.skip() # Jump to next node in queue - # Code below won't execute - - # Skip products outside price range - if not (10 <= here.price <= 1000): - print(f"Skipping {here.name} - price out of range") - self.skip() - - # Normal processing for valid products - print(f"Processing product: {here.name}") - connected_products = await here.nodes(node=['Product']) - await self.visit(connected_products) -``` - -#### `disengage()` - Permanently Halt and Remove Walker from Graph -The `disengage()` method permanently halts the walker and removes it from the graph. Once disengaged, the walker **cannot be resumed** and is considered finished: +`jvspatial/serverless/` — `dispatch_deferred_task(task_type, payload)` invokes a registered async handler out-of-band. On AWS, transport is Lambda async-invoke or EventBridge (configurable). -```python -class CompletionWalker(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.max_items = 100 - self.critical_error = False - - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with completion tracking. - - Args: - here: The visited Document node - """ - try: - # Process document - await self.process_item(here) - self.processed_count += 1 - - # Disengage when reaching target or on critical error - if self.processed_count >= self.max_items: - print(f"Target reached: {self.processed_count} items processed") - await self.disengage() # Permanently finish - return - - if self.critical_error: - print("Critical error encountered, disengaging walker") - await self.disengage() # Permanently halt due to error - return - - # Continue to next documents - next_docs = await here.nodes(node=['Document']) - await self.visit(next_docs) - - except CriticalError as e: - print(f"Critical error: {e}") - self.critical_error = True - await self.disengage() # Permanently halt on critical error - - @on_exit - async def final_cleanup(self): - """Called when walker disengages - perform final cleanup.""" - print(f"Walker disengaged. Final count: {self.processed_count}") - # Perform final cleanup, save state, notify completion - await self.save_final_results() - - async def process_item(self, node): - """Process individual item.""" - # Simulate processing that might fail - if node.status == "corrupted": - raise CriticalError("Corrupted data detected") - await asyncio.sleep(0.01) - - async def save_final_results(self): - """Save final processing results.""" - print("💾 Saving final results...") - -# Usage - disengage() creates permanent termination -walker = CompletionWalker() -root = await Root.get(None) - -# Start and run to completion (or error) -walker = await walker.spawn(root) -print(f"Walker finished. Status: {'disengaged' if walker.paused else 'completed'}") - -# NOTE: Once disengaged, walker cannot be resumed -# walker.resume() would not work - walker is permanently finished - -# For pausable/resumable patterns, use different approaches: -# - Save walker state and create new walker instances -# - Use external queue/state management -# - Implement custom pause/resume logic in @on_visit methods -``` - -#### `pause()` and `resume()` - Temporary Walker Suspension -Walkers can be paused during traversal and resumed later, preserving their queue and state. Unlike `disengage()`, paused walkers can be resumed: +Register handlers with `@deferred_invoke_handler("task.name")`. Handlers **must be idempotent**; the framework provides no exactly-once guarantee. -```python -class BatchProcessor(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.batch_size = 50 - self.total_batches = 0 - - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with batch pausing. - - Args: - here: The visited Document node - """ - # Process document - await self.heavy_processing(here) - self.processed_count += 1 - - # Pause after processing a batch - if self.processed_count % self.batch_size == 0: - self.total_batches += 1 - print(f"Batch {self.total_batches} complete ({self.processed_count} items)") - print("Pausing for rate limiting...") - - # Clean pause using pause() method - self.pause(f"Batch {self.total_batches} processing pause") - - # Continue to next documents - next_docs = await here.nodes(node=['Document'], status="active") - await self.visit(next_docs) - - async def heavy_processing(self, node): - """Simulate expensive processing.""" - await asyncio.sleep(0.1) # Simulate API calls, file I/O, etc. - - @on_exit - async def report_completion(self): - """Called when walker completes or is paused.""" - if self.paused: - print(f"Walker paused after processing {self.processed_count} items") - else: - print(f"Walker completed! Total processed: {self.processed_count}") - -# Usage - pause and resume cycle -walker = BatchProcessor() -root = await Root.get(None) - -# Start processing - will pause after first batch -walker = await walker.spawn(root) -print(f"Walker state: {'paused' if walker.paused else 'completed'}") -print(f"Queue remaining: {len(walker.queue)} items") - -# Simulate processing delay (rate limiting, API cooldown, etc.) -print("\n⏳ Waiting for rate limit cooldown...") -await asyncio.sleep(2.0) - -# Resume processing - will continue from where it left off -print("\n▶️ Resuming processing...") -walker = await walker.resume() -print(f"Walker state after resume: {'paused' if walker.paused else 'completed'}") - -# Can resume multiple times if walker gets paused again -while walker.paused and walker.queue: - print(f"\n🔄 Walker paused again, {len(walker.queue)} items remaining") - await asyncio.sleep(1.0) # Brief pause - walker = await walker.resume() - -print("\n✅ All processing complete!") -``` +### 11.4 Lambda Web Adapter -#### Advanced Pause/Resume Patterns +When LWA is detected, `Server` applies best-effort defaults for `AWS_LWA_PASS_THROUGH_PATH` and `AWS_LWA_INVOKE_MODE`. The LWA extension reads these *before* Python starts, so IaC should still set them explicitly for reliability. -```python -class SmartProcessor(Walker): - def __init__(self): - super().__init__() - self.api_calls = 0 - self.max_api_calls = 100 - self.error_count = 0 - self.max_errors = 5 - - @on_visit("DataNode") - async def process_data(self, here: Node): - """Process data nodes with smart rate limiting. - - Args: - here: The visited DataNode - """ - try: - # Check rate limits - if self.api_calls >= self.max_api_calls: - print(f"Rate limit reached ({self.api_calls} calls), pausing...") - self.api_calls = 0 # Reset counter - self.pause("Rate limit reached") - - # Check error threshold - if self.error_count >= self.max_errors: - print(f"Too many errors ({self.error_count}), pausing for investigation") - self.pause(f"Error threshold reached: {self.error_count} errors") - - # Process node - result = await self.call_external_api(here) - self.api_calls += 1 - - # Continue traversal based on result - if result.should_continue: - related_nodes = await here.nodes( - node=['DataNode'], - status="pending", - priority={"$gte": result.priority_threshold} - ) - await self.visit(related_nodes) - - except ApiError as e: - self.error_count += 1 - print(f"API error #{self.error_count}: {e}") - # Continue processing - don't pause on single errors - - except CriticalError as e: - print(f"Critical error: {e}") - self.pause(f"Critical error: {e}") - - async def call_external_api(self, node): - """Simulate external API call that might fail.""" - await asyncio.sleep(0.05) # Simulate API latency - # Simulate occasional failures - if random.random() < 0.1: # 10% failure rate - raise ApiError("Temporary API failure") - return APIResult(should_continue=True, priority_threshold=5) - - def pause_for_maintenance(self): - """Manually pause walker for maintenance.""" - print("🔧 Pausing for scheduled maintenance...") - self.paused = True - - @on_exit - async def maintenance_check(self): - """Check if maintenance is needed when paused.""" - if self.paused: - print("Walker paused - performing maintenance checks...") - # Reset error counters, clear caches, etc. - self.error_count = 0 - print("Maintenance complete - ready to resume") - -# Usage with external control -walker = SmartProcessor() -root = await Root.get(None) - -# Start processing -print("🚀 Starting smart processing...") -walker = await walker.spawn(root) - -# External monitoring and control -while walker.paused and walker.queue: - print(f"⏸️ Walker paused - {len(walker.queue)} items remaining") - - # Simulate external decision making - if should_resume_processing(): # Your logic here - print("🔄 Conditions met, resuming...") - walker = await walker.resume() - else: - print("⏳ Waiting for conditions to improve...") - await asyncio.sleep(5.0) - -# Helper classes for example -class ApiError(Exception): pass -class CriticalError(Exception): pass -class APIResult: - def __init__(self, should_continue, priority_threshold): - self.should_continue = should_continue - self.priority_threshold = priority_threshold - -def should_resume_processing(): - """External logic to determine if processing should resume.""" - return True # Simplified for example -``` - -#### `@on_exit` - Cleanup and Finalization -The `@on_exit` decorator marks methods to execute when the walker completes traversal or disengages: - -```python -class AnalyticsWalker(Walker): - def __init__(self): - super().__init__() - self.start_time = time.time() - self.nodes_visited = 0 - self.errors_encountered = 0 - - @on_visit("User") - async def analyze_user(self, here: Node): - """Analyze user behavior and traverse to related users. - - Args: - here: The visited User node - """ - try: - self.nodes_visited += 1 - # Perform analysis - await self.analyze_user_behavior(here) - - # Continue traversal - related_users = await here.nodes(node=['User']) - await self.visit(related_users) - except Exception as e: - self.errors_encountered += 1 - print(f"Error processing user {here.id}: {e}") - - @on_exit - async def generate_report(self): - """Generate analytics report when traversal completes.""" - duration = time.time() - self.start_time - print("\n📊 Analytics Report") - print(f"Duration: {duration:.2f} seconds") - print(f"Nodes visited: {self.nodes_visited}") - print(f"Errors: {self.errors_encountered}") - print(f"Success rate: {(1 - self.errors_encountered/max(self.nodes_visited, 1))*100:.1f}%") - - @on_exit - def save_results(self): - """Save results to database (sync version).""" - # Save analytics data - print("💾 Saving results to database...") - - @on_exit - async def send_notifications(self): - """Send completion notifications (async version).""" - # Send email/slack notifications - print("📧 Sending completion notifications...") - - async def analyze_user_behavior(self, user): - """Simulate user behavior analysis.""" - await asyncio.sleep(0.01) # Simulate work -``` - -### Walker Reporting System - -Walkers feature a simplified reporting system that allows you to collect and aggregate any data during traversal. The reporting system eliminates complex nested structures and provides direct access to collected data. - -#### Basic Reporting - -```python path=null start=null -from jvspatial.core import Walker, on_visit, on_exit - -class DataCollectionWalker(Walker): - """Walker demonstrating the simple reporting system.""" - - def __init__(self): - super().__init__() - self.processed_count = 0 - - @on_visit('User') - async def collect_user_data(self, here: Node): - """Collect user data using the report system.""" - # Report any data - strings, dicts, numbers, lists, etc. - self.report({ - "user_processed": { - "id": here.id, - "name": here.name, - "department": here.department, - "timestamp": time.time() - } - }) - - # Report simple values - self.report(f"Processed user: {here.name}") - - # Report lists - if hasattr(here, 'skills'): - self.report(["skills", here.skills]) - - self.processed_count += 1 - - @on_exit - async def generate_summary(self): - """Generate final summary in the report.""" - report_items = await self.get_report() - - self.report({ - "summary": { - "total_items_collected": len(report_items), - "users_processed": self.processed_count, - "collection_complete": True - } - }) - -# Usage -walker = DataCollectionWalker() -result_walker = await walker.spawn() # spawn() returns the walker instance - -# Access collected data directly as a simple list -report = await result_walker.get_report() -print(f"Total items collected: {len(report)}") - -# Iterate through all collected data -for item in report: - if isinstance(item, dict) and "user_processed" in item: - user_data = item["user_processed"] - print(f"User: {user_data['name']} from {user_data['department']}") - elif isinstance(item, str): - print(f"Log: {item}") -``` - -#### Advanced Reporting Patterns - -```python path=null start=null -class AnalyticsWalker(Walker): - """Walker with advanced reporting for analytics.""" - - def __init__(self): - super().__init__() - self.department_counts = {} - self.error_count = 0 - - @on_visit('User') - async def analyze_user(self, here: Node): - """Analyze user and report findings.""" - try: - # Track department statistics - dept = here.department or "unknown" - self.department_counts[dept] = self.department_counts.get(dept, 0) + 1 - - # Report individual analysis - analysis = await self.perform_user_analysis(here) - self.report({ - "user_analysis": { - "user_id": here.id, - "department": dept, - "performance_score": analysis.get("score", 0), - "risk_level": analysis.get("risk", "low"), - "recommendations": analysis.get("recommendations", []) - } - }) - - except Exception as e: - self.error_count += 1 - self.report({ - "error": { - "user_id": here.id, - "error_message": str(e), - "error_type": type(e).__name__ - } - }) - - @on_exit - async def generate_analytics_report(self): - """Generate comprehensive analytics.""" - all_data = await self.get_report() - - # Analyze collected data - user_analyses = [item for item in all_data - if isinstance(item, dict) and "user_analysis" in item] - errors = [item for item in all_data - if isinstance(item, dict) and "error" in item] - - # Calculate metrics - avg_score = sum(ua["user_analysis"]["performance_score"] - for ua in user_analyses) / len(user_analyses) if user_analyses else 0 - - high_risk_users = [ua for ua in user_analyses - if ua["user_analysis"]["risk_level"] == "high"] - - # Report final analytics - self.report({ - "final_analytics": { - "total_users_analyzed": len(user_analyses), - "average_performance_score": round(avg_score, 2), - "department_breakdown": self.department_counts, - "high_risk_users_count": len(high_risk_users), - "error_rate": self.error_count / max(len(user_analyses), 1), - "processing_summary": { - "success": len(user_analyses), - "errors": self.error_count, - "total_items_in_report": len(all_data) - } - } - }) - - async def perform_user_analysis(self, user): - """Simulate user analysis.""" - import random - return { - "score": random.randint(1, 100), - "risk": random.choice(["low", "medium", "high"]), - "recommendations": ["Update profile", "Complete training"] - } -``` - -### Walker Event System - -Walkers can communicate with each other during traversal using an event system. This enables real-time coordination, data sharing, and complex multi-walker workflows. +--- -#### Basic Event Communication +## 12. File Storage -```python path=null start=null -from jvspatial.core.events import on_emit +`jvspatial/storage/` — abstract `FileStorageInterface` (`interfaces/base.py`) with two built-in implementations: -class AlertWalker(Walker): - """Walker that emits alerts when finding critical issues.""" +| Adapter | File | Backing | Versioning | +|---|---|---|---| +| Local | `interfaces/local.py` | Filesystem | Version directory per file | +| S3 | `interfaces/s3.py` | AWS S3 | Native S3 versioning + multipart ≥8 MiB | - @on_visit('SystemNode') - async def check_system_health(self, here: Node): - """Check system health and emit alerts.""" - if hasattr(here, 'cpu_usage') and here.cpu_usage > 90: - # Emit event to other walkers - await self.emit("high_cpu_alert", { - "node_id": here.id, - "cpu_usage": here.cpu_usage, - "severity": "critical", - "walker_id": self.id - }) +### 12.1 Security layer - self.report({"alert_sent": f"High CPU on {here.id}"}) +- `storage/security/path_sanitizer.py` — five-stage validation: regex blocklist (11 patterns), normalization with re-check, hidden-file allowlist, symlink resolution, base-directory confinement. +- `storage/security/validator.py` — content-based MIME via `python-magic`. ~25 allowed types, 14 blocked types, 19 blocked extensions. Internal markers bypassed via metadata validation only (never user-supplied). -class MonitoringWalker(Walker): - """Walker that receives and processes alerts from other walkers.""" +### 12.2 Upload contract - def __init__(self): - super().__init__() - self.alerts_received = 0 +All uploads pass through `path_sanitizer` then `validator`. Failures raise `ValidationError` with `field_errors` populated. Successful uploads return a versioned descriptor. - @on_emit("high_cpu_alert") - async def handle_cpu_alert(self, event_data): - """Handle high CPU alerts from AlertWalker.""" - self.alerts_received += 1 +### 12.3 S3 specifics - self.report({ - "alert_processed": { - "from_walker": event_data.get("walker_id"), - "node_id": event_data.get("node_id"), - "cpu_usage": event_data.get("cpu_usage"), - "action_taken": "Notification sent to admin", - "handler_id": self.id - } - }) +- Multipart upload threshold: 8 MiB (chunks of 8 MiB). +- Throttle retry on `ThrottlingException` with exponential backoff. +- Credentials required if `file_storage_enabled=True` and backend is `s3`. - # Take action based on alert - await self.send_notification(event_data) +--- - @on_visit('SystemNode') - async def log_system_visit(self, here: Node): - """Log system node visits.""" - self.report({"system_visited": here.id}) +## 13. Caching - async def send_notification(self, alert_data): - """Send notification to administrators.""" - print(f"🚨 ALERT: High CPU {alert_data['cpu_usage']}% on {alert_data['node_id']}") +`jvspatial/cache/` — pluggable cache layer. -# Run multiple walkers concurrently -import asyncio +### 13.1 Backends -alert_walker = AlertWalker() -monitoring_walker = MonitoringWalker() +| Backend | Use case | +|---|---| +| `memory` | Per-process LRU; default for single-instance deployments | +| `redis` | Shared cache across workers/instances | +| `layered` | Memory L1 + Redis L2; promotes hot keys to L1 | -# Start both walkers concurrently -tasks = [ - alert_walker.spawn(), - monitoring_walker.spawn() -] +### 13.2 Wrapping pattern -walkers = await asyncio.gather(*tasks) +`jvspatial/db/_cache.py` provides a read-through wrapper for `Database`. Enabled per database via `cache_get_size` parameter in `create_database(...)`. Auto-invalidates on `save` and `delete`. -# Check reports from both walkers -alert_report = await alert_walker.get_report() -monitoring_report = await monitoring_walker.get_report() +### 13.3 TTL and invalidation -print(f"Alerts sent: {len([r for r in alert_report if 'alert_sent' in str(r)])}") -print(f"Alerts processed: {monitoring_walker.alerts_received}") -``` +TTL is per-cache-instance; default `None` (cache until eviction). Invalidation is event-driven: write operations on the wrapped database publish invalidation events. Cross-instance invalidation requires the Redis or layered backend. -#### Advanced Event Patterns - -```python path=null start=null -class DataProcessorWalker(Walker): - """Walker that processes data and emits completion events.""" - - def __init__(self, batch_id: str): - super().__init__() - self.batch_id = batch_id - self.processed_items = 0 - - @on_visit('DataNode') - async def process_data(self, here: Node): - """Process data nodes.""" - # Simulate processing - await asyncio.sleep(0.01) - self.processed_items += 1 - - self.report({"data_processed": here.id}) - - # Emit progress event every 10 items - if self.processed_items % 10 == 0: - await self.emit("batch_progress", { - "batch_id": self.batch_id, - "processed_count": self.processed_items, - "processor_id": self.id - }) - - @on_exit - async def emit_completion(self): - """Emit batch completion event.""" - await self.emit("batch_complete", { - "batch_id": self.batch_id, - "total_processed": self.processed_items, - "processor_id": self.id - }) - - self.report({"batch_completed": self.batch_id}) - -class BatchCoordinator(Walker): - """Walker that coordinates multiple batch processors.""" - - def __init__(self): - super().__init__() - self.batch_progress = {} - self.completed_batches = [] - - @on_emit("batch_progress") - async def track_progress(self, event_data): - """Track progress from batch processors.""" - batch_id = event_data.get("batch_id") - processed_count = event_data.get("processed_count") - - self.batch_progress[batch_id] = processed_count - - self.report({ - "progress_update": { - "batch_id": batch_id, - "items_processed": processed_count, - "coordinator_id": self.id - } - }) - - @on_emit("batch_complete") - async def handle_completion(self, event_data): - """Handle batch completion events.""" - batch_id = event_data.get("batch_id") - total_processed = event_data.get("total_processed") - - self.completed_batches.append(batch_id) - - self.report({ - "batch_completed": { - "batch_id": batch_id, - "total_items": total_processed, - "completed_batches_count": len(self.completed_batches) - } - }) - - # Check if all batches are complete - if len(self.completed_batches) >= 3: # Expecting 3 batches - await self.emit("all_batches_complete", { - "total_batches": len(self.completed_batches), - "coordinator_id": self.id - }) - - @on_emit("all_batches_complete") - async def finalize_processing(self, event_data): - """Finalize when all processing is complete.""" - self.report({ - "processing_finalized": { - "total_batches_completed": event_data.get("total_batches"), - "finalization_time": time.time() - } - }) - -# Example: Run coordinated batch processing -coordinator = BatchCoordinator() -processors = [ - DataProcessorWalker("batch_1"), - DataProcessorWalker("batch_2"), - DataProcessorWalker("batch_3") -] - -# Start all walkers -all_walkers = [coordinator] + processors -tasks = [walker.spawn() for walker in all_walkers] -results = await asyncio.gather(*tasks) - -# Check final reports -for walker in all_walkers: - report = await walker.get_report() - print(f"Walker {walker.id}: {len(report)} items in report") -``` +--- -#### Key Reporting & Event Features - -**Reporting System:** -- `walker.report(any_data)` - Add any data to walker's report -- `await walker.get_report()` - Get simple list of all reported items (async) -- No complex nested structures - direct access to your data -- Support for any data type (strings, dicts, lists, numbers, etc.) - -**Event System:** -- `await walker.emit(event_name, payload)` - Send events to other walkers -- `@on_emit(event_name)` - Handle specific events -- Multiple walkers can receive the same event -- Events enable real-time coordination between concurrent walkers -- Both Walkers and Nodes can use `@on_emit` decorators - -**Best Practices:** -- Use `self.report()` to add data, never return values from decorated methods -- Access reports after traversal: `report = await walker.get_report()` -- Use events for walker-to-walker communication during traversal -- Filter reported data by checking item structure/content -- Leverage `@on_exit` hooks for final summaries and cleanup - -### Walker Trail Tracking - -Walkers include built-in **trail tracking** capabilities to monitor and record the complete path taken during graph traversal. This is invaluable for debugging, analytics, audit trails, and optimizing traversal strategies. - -#### Basic Trail Tracking - -```python path=null start=null -from jvspatial.core import Walker, on_visit - -class TrailTrackingWalker(Walker): - def __init__(self): - super().__init__() - # Enable trail tracking with memory management - self.trail_enabled = True - self.max_trail_length = 100 # Keep last 100 steps (0 = unlimited) - - @on_visit('User') - async def process_user_with_trail(self, here: Node): - """Process user nodes while tracking the traversal trail. - - Args: - here: The visited User node - """ - print(f"Processing user: {here.name}") - - # Access current trail information - current_trail = self.get_trail() # List of node IDs - trail_length = self.get_trail_length() # Current trail size - recent_steps = self.get_recent_trail(count=3) # Last 3 steps - - print(f"Trail length: {trail_length}, Recent: {recent_steps}") - - # Avoid revisiting recently visited nodes - if here.id in recent_steps[:-1]: # Exclude current node - print(f"Recently visited {here.name}, skipping deeper traversal") - self.skip() - - # Continue normal traversal - colleagues = await here.nodes( - node=['User'], - department=here.department, - active=True - ) - await self.visit(colleagues) - - @on_exit - async def generate_trail_report(self): - """Generate comprehensive trail analysis report.""" - # Get trail with actual node objects (database lookups) - trail_nodes = await self.get_trail_nodes() - - # Get complete path with connecting edges - trail_path = await self.get_trail_path() - - # Generate detailed report using report() method - trail_report = { - 'summary': { - 'total_steps': self.get_trail_length(), - 'unique_nodes': len(set(self.get_trail())), - 'efficiency_ratio': len(set(self.get_trail())) / max(self.get_trail_length(), 1) - }, - 'visited_nodes': [ - {'step': i+1, 'node_type': node.__class__.__name__, 'node_name': getattr(node, 'name', node.id)} - for i, node in enumerate(trail_nodes) - ], - 'path_analysis': [ - { - 'step': i+1, - 'node': node.name if hasattr(node, 'name') else node.id, - 'via_edge': edge.edge_type if edge else 'start' - } - for i, (node, edge) in enumerate(trail_path) - ] - } - - # Report the trail data - self.report(trail_report) - - print(f"\n📊 Trail Report Generated:") - print(f" - Total steps: {trail_report['summary']['total_steps']}") - print(f" - Unique nodes: {trail_report['summary']['unique_nodes']}") - print(f" - Path efficiency: {trail_report['summary']['efficiency_ratio']:.2%}") - -# Usage example -walker = TrailTrackingWalker() -root = await Root.get(None) -await walker.spawn(root) - -# Access trail data -final_trail = walker.get_trail() -print(f"Final trail: {final_trail}") -# Access the trail report from walker's collected data -report = await walker.get_report() -trail_reports = [item for item in report if isinstance(item, dict) and 'trail_report' in str(item)] -print(f"Trail report: {trail_reports[0] if trail_reports else 'No trail report found'}") -``` +## 14. Observability -#### Advanced Trail Use Cases - -```python path=null start=null -class AdvancedTrailWalker(Walker): - def __init__(self): - super().__init__() - self.trail_enabled = True - self.max_trail_length = 0 # Unlimited for comprehensive analysis - self.visited_nodes = set() # For cycle detection - self.performance_metrics = [] - - @on_visit('Document') - async def process_with_cycle_detection(self, here: Node): - """Process documents with cycle detection using trail data. - - Args: - here: The visited Document node - """ - import time - start_time = time.time() - - # Cycle detection using trail - if here.id in self.visited_nodes: - trail = self.get_trail() - first_visit_index = trail.index(here.id) - cycle_length = len(trail) - first_visit_index - 1 - - print(f"🔄 Cycle detected at {here.id}! Length: {cycle_length} steps") - - self.report({ - 'cycle_detected': { - 'node_id': here.id, - 'cycle_length': cycle_length, - 'first_visit_step': first_visit_index + 1, - 'detection_step': len(trail) - } - }) - - # Stop to avoid infinite loop - await self.disengage() - return - - self.visited_nodes.add(here.id) - - # Process document - await self.analyze_document(here) - - # Record performance metrics - processing_time = time.time() - start_time - self.performance_metrics.append({ - 'node_id': here.id, - 'processing_time': processing_time, - 'step_number': self.get_trail_length(), - 'metadata': self.get_trail_metadata() # Get current step metadata - }) - - # Continue traversal with trail-aware filtering - related_docs = await here.nodes( - node=['Document'], - status='active' - ) - - # Filter out recently visited to avoid cycles - recent_trail = self.get_recent_trail(count=10) - unvisited_docs = [doc for doc in related_docs if doc.id not in recent_trail] - - if unvisited_docs: - await self.visit(unvisited_docs) - else: - print("All related documents recently visited, exploring alternatives") - - @on_visit('User') - async def audit_user_access(self, here: Node): - """Create audit trail for user access. - - Args: - here: The visited User node - """ - # Get current trail metadata (automatically includes timestamp, node_type, queue_length) - metadata = self.get_trail_metadata() - - audit_entry = { - 'timestamp': metadata.get('timestamp'), - 'action': 'USER_ACCESS', - 'user_id': here.id, - 'user_name': getattr(here, 'name', 'Unknown'), - 'trail_step': self.get_trail_length(), - 'access_context': { - 'queue_size': metadata.get('queue_length'), - 'node_type': metadata.get('node_type'), - 'previous_steps': self.get_recent_trail(count=3)[:-1] # Exclude current - } - } - - self.report({'audit_entry': audit_entry}) - print(f"📝 Audit: Accessed user {here.id} at step {audit_entry['trail_step']}") - - @on_exit - async def comprehensive_analysis(self): - """Generate comprehensive trail and performance analysis.""" - trail_path = await self.get_trail_path() - - # Performance analysis - avg_processing_time = sum(m['processing_time'] for m in self.performance_metrics) / len(self.performance_metrics) if self.performance_metrics else 0 - - # Path efficiency analysis - total_steps = self.get_trail_length() - unique_nodes = len(set(self.get_trail())) - - # Get report once for all analysis - report = await self.get_report() - - comprehensive_analysis = { - 'trail_summary': { - 'total_steps': total_steps, - 'unique_nodes_visited': unique_nodes, - 'path_efficiency': unique_nodes / total_steps if total_steps > 0 else 0, - 'cycles_detected': len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item]), - 'trail_enabled': self.trail_enabled, - 'trail_limit': self.max_trail_length - }, - 'performance_metrics': { - 'avg_processing_time': avg_processing_time, - 'total_processing_time': sum(m['processing_time'] for m in self.performance_metrics), - 'slowest_step': max(self.performance_metrics, key=lambda x: x['processing_time']) if self.performance_metrics else None - }, - 'audit_summary': { - 'total_audit_entries': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item]), - 'user_accesses': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item and item.get('audit_entry', {}).get('action') == 'USER_ACCESS']) - } - } - - # Report the comprehensive analysis - self.report(comprehensive_analysis) - - print("\n📈 Comprehensive Analysis Complete:") - print(f" - Path efficiency: {comprehensive_analysis['trail_summary']['path_efficiency']:.2%}") - print(f" - Average processing time: {avg_processing_time:.3f}s") - print(f" - Cycles detected: {len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item])}") - - async def analyze_document(self, doc): - """Simulate document analysis.""" - import asyncio - await asyncio.sleep(0.02) # Simulate processing time - -# Usage with trail management -walker = AdvancedTrailWalker() - -# Enable debug mode for detailed trail information -walker.debug_mode = True - -# Spawn and run analysis -root = await Root.get(None) -await walker.spawn(root) - -# Access comprehensive results from walker's report -report = await walker.get_report() -analysis = next((item for item in report if isinstance(item, dict) and 'trail_summary' in item), {}) -cycles = [item for item in report if isinstance(item, dict) and 'cycle_detected' in item] -audit_entries = [item for item in report if isinstance(item, dict) and 'audit_entry' in item] - -print(f"Analysis complete. Efficiency: {analysis.get('trail_summary', {}).get('path_efficiency', 0):.2%}") -print(f"Cycles detected: {len(cycles)}, Audit entries: {len(audit_entries)}") -``` +`jvspatial/observability/` — metrics and tracing surface. -#### Trail API Quick Reference - -**Configuration (Read/Write):** -- `self.trail_enabled = True` - Enable trail tracking -- `self.max_trail_length = N` - Limit trail to N steps (0 = unlimited) - -**Trail Data (Read-Only Properties):** -- `self.trail` - List of visited node IDs (returns copy) -- `self.trail_edges` - Edge IDs between nodes (returns copy) -- `self.trail_metadata` - Metadata per step (returns deep copy) - -**Trail Access Methods (O(1) operations):** -- `self.get_trail()` - Get list of visited node IDs -- `self.get_trail_length()` - Get current trail length -- `self.get_recent_trail(count=N)` - Get last N trail steps -- `self.clear_trail()` - Clear entire trail (only way to modify trail) - -**Advanced Access (Database operations):** -- `await self.get_trail_nodes()` - Get actual Node objects from trail -- `await self.get_trail_path()` - Get trail with connecting edges -- `self.get_trail_metadata(step=-1)` - Get metadata for specific step - -**Use Cases:** -- **Debugging**: Track walker path for troubleshooting -- **Cycle Detection**: Avoid infinite loops in graph traversal -- **Performance Analysis**: Measure processing time per step -- **Audit Trails**: Comprehensive access logging for compliance -- **Path Optimization**: Analyze and improve traversal efficiency - -### Walker Queue Manipulation Methods - -Walkers maintain an internal queue (deque) of nodes to visit during traversal. Advanced queue manipulation provides fine-grained control over traversal order, priority handling, and dynamic path planning. These methods allow you to programmatically manage the walker's traversal queue: - -#### Core Queue Methods - -```python path=null start=null -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit -from typing import List, Optional - -class QueueMasterWalker(Walker): - def __init__(self): - super().__init__() - self.priority_nodes = [] - self.deferred_nodes = [] - self.processed_count = 0 - - @on_visit("TaskNode") - async def process_task(self, here: Node): - """Demonstrate queue manipulation methods. - - Args: - here: The visited TaskNode - """ - - # 1. INSPECT QUEUE STATE - current_queue = self.get_queue() # Get queue as list - print(f"Current queue size: {len(current_queue)}") - print(f"Queue contents: {[n.name for n in current_queue]}") - - # Check if specific node is queued - if current_queue: - next_node = current_queue[0] # Peek at next node - print(f"Next node to process: {next_node.name}") - - # 2. ADD NODES TO QUEUE - # Find high-priority nodes to add immediately - urgent_tasks = await here.nodes( - node=['TaskNode'], - priority={"$gte": 9}, - status="pending" - ) - - # Add to front of queue (high priority) - if urgent_tasks: - added = self.prepend(urgent_tasks) # Add to front - print(f"Added {len(added)} urgent tasks to front of queue") - - # Find normal tasks to add to end - normal_tasks = await here.nodes( - node=['TaskNode'], - priority={"$lt": 9, "$gte": 5}, - status="pending" - ) - - # Add to end of queue (normal priority) - if normal_tasks: - added = self.append(normal_tasks) # Add to end - print(f"Added {len(added)} normal tasks to end of queue") - - # Alternative: Use visit() method (equivalent to append) - additional_tasks = await here.nodes(node=['TaskNode'], status="new") - if additional_tasks: - self.visit(additional_tasks) # Same as append() - - # Add nodes right after current processing - immediate_tasks = await here.nodes( - node=['TaskNode'], - priority=10, # Highest priority - status="urgent" - ) - if immediate_tasks: - added = self.add_next(immediate_tasks) # Add next in queue - print(f"Added {len(added)} tasks to process immediately next") - - # 3. CONDITIONAL QUEUE MANIPULATION - # Check if we have too many items in queue - if len(self.get_queue()) > 100: - print("Queue overflow detected, deferring low-priority items") - - # Get current queue and filter it - current_queue = self.get_queue() - low_priority = [] - high_priority = [] - - for item in current_queue: - if hasattr(item, 'priority') and item.priority < 5: - low_priority.append(item) - else: - high_priority.append(item) - - # Clear queue and rebuild with high priority items only - self.clear_queue() - if high_priority: - self.append(high_priority) - - # Store deferred items for later - self.deferred_nodes.extend(low_priority) - print(f"Deferred {len(low_priority)} low-priority items") - - # 4. TARGETED QUEUE MANIPULATION - # Remove specific completed nodes from queue - completed_nodes = await here.nodes( - node=['TaskNode'], - status="completed" - ) - - for completed in completed_nodes: - if self.is_queued(completed): - removed = self.dequeue(completed) - print(f"Removed {len(removed)} completed tasks from queue") - - # 5. PRECISE QUEUE INSERTION - # Find a specific node in queue to insert after - checkpoint_tasks = await here.nodes( - node=['TaskNode'], - task_type="checkpoint" - ) - - followup_tasks = await here.nodes( - node=['TaskNode'], - depends_on=here.id - ) - - for checkpoint in checkpoint_tasks: - if self.is_queued(checkpoint) and followup_tasks: - try: - # Insert followup tasks right after checkpoint - inserted = self.insert_after(checkpoint, followup_tasks) - print(f"Inserted {len(inserted)} followup tasks after checkpoint") - except ValueError as e: - print(f"Could not insert after checkpoint: {e}") - - self.processed_count += 1 - - @on_visit("CompletionNode") - async def handle_completion(self, here: Node): - """Handle task completion and queue cleanup. - - Args: - here: The visited CompletionNode - """ - - # Add back any deferred nodes if queue is manageable - current_queue_size = len(self.get_queue()) - if current_queue_size < 20 and self.deferred_nodes: - reactivated = self.deferred_nodes[:10] # Add up to 10 back - self.deferred_nodes = self.deferred_nodes[10:] # Remove from deferred - - self.append(reactivated) - print(f"Reactivated {len(reactivated)} deferred nodes") - - # Insert critical tasks right at the beginning - critical_tasks = await here.nodes( - node=['TaskNode'], - priority=10, - status="critical" - ) - - if critical_tasks: - # Find first non-critical task and insert before it - queue = self.get_queue() - for i, queued_node in enumerate(queue): - if hasattr(queued_node, 'priority') and queued_node.priority < 10: - try: - inserted = self.insert_before(queued_node, critical_tasks) - print(f"Inserted {len(inserted)} critical tasks before normal task") - break - except ValueError: - # If insertion fails, just prepend - self.prepend(critical_tasks) - break - - @on_exit - async def final_report(self): - """Report final queue statistics.""" - final_queue = self.get_queue() - print(f"\n📊 Queue Processing Complete") - print(f"Total processed: {self.processed_count}") - print(f"Remaining in queue: {len(final_queue)}") - print(f"Deferred nodes: {len(self.deferred_nodes)}") - - if final_queue: - print("Remaining nodes:") - for node in final_queue[:5]: # Show first 5 - print(f" - {node.name}") - if len(final_queue) > 5: - print(f" ... and {len(final_queue) - 5} more") -``` +### 14.1 MetricsRecorder protocol -#### Queue Method Reference - -Based on the actual Walker implementation in jvspatial: - -**Basic Queue Operations:** -- `self.visit(nodes)` - Add nodes to end of queue (equivalent to append) -- `self.append(nodes)` - Add nodes to end of queue -- `self.prepend(nodes)` - Add nodes to front of queue -- `self.add_next(nodes)` - Add nodes next in queue after current item -- `self.get_queue()` - Return entire queue as a list -- `self.clear_queue()` - Clear all nodes from queue -- `self.is_queued(node)` - Check if specific node is in queue - -**Advanced Queue Operations:** -- `self.dequeue(nodes)` - Remove specific nodes from queue -- `self.insert_after(target_node, nodes)` - Insert nodes after target node -- `self.insert_before(target_node, nodes)` - Insert nodes before target node - -#### Priority-Based Queue Management - -```python path=null start=null -class PriorityQueueWalker(Walker): - def __init__(self): - super().__init__() - self.priority_buckets = { - 'urgent': [], # Priority 9-10 - 'high': [], # Priority 7-8 - 'normal': [], # Priority 4-6 - 'low': [] # Priority 1-3 - } - - @on_visit("WorkItem") - async def process_work_item(self, here: Node): - """Process work items with priority-based queuing. - - Args: - here: The visited WorkItem node - """ - - # Get connected work items - connected_items = await here.nodes( - node=['WorkItem'], - status="pending" - ) - - if connected_items: - # Sort into priority buckets - self._sort_into_priority_buckets(connected_items) - - # Process highest priority items first - self._add_by_priority_order() - - def _sort_into_priority_buckets(self, nodes: List[Node]): - """Sort nodes into priority-based buckets.""" - for node in nodes: - priority = getattr(node, 'priority', 5) - - if priority >= 9: - self.priority_buckets['urgent'].append(node) - elif priority >= 7: - self.priority_buckets['high'].append(node) - elif priority >= 4: - self.priority_buckets['normal'].append(node) - else: - self.priority_buckets['low'].append(node) - - print(f"Sorted into buckets: " - f"urgent={len(self.priority_buckets['urgent'])}, " - f"high={len(self.priority_buckets['high'])}, " - f"normal={len(self.priority_buckets['normal'])}, " - f"low={len(self.priority_buckets['low'])}") - - def _add_by_priority_order(self): - """Add nodes to walker queue in priority order.""" - # Process urgent items first (add to front) - if self.priority_buckets['urgent']: - self.prepend(self.priority_buckets['urgent']) - self.priority_buckets['urgent'].clear() - - # Add high priority items to front (after urgent) - if self.priority_buckets['high']: - # Insert at beginning but after urgent items - current_queue = self.get_queue() - if current_queue: - # Find first non-urgent item and insert before it - high_items = self.priority_buckets['high'] - self.priority_buckets['high'].clear() - - try: - # Try to find insertion point - first_non_urgent = None - for node in current_queue: - if hasattr(node, 'priority') and node.priority < 9: - first_non_urgent = node - break - - if first_non_urgent: - self.insert_before(first_non_urgent, high_items) - else: - self.append(high_items) # All items are urgent - - except ValueError: - # Fallback to prepend if insertion fails - self.prepend(high_items) - else: - self.prepend(self.priority_buckets['high']) - self.priority_buckets['high'].clear() - - # Add normal priority items to end - if self.priority_buckets['normal']: - self.append(self.priority_buckets['normal']) - self.priority_buckets['normal'].clear() - - # Only add low priority if queue is small - current_queue_size = len(self.get_queue()) - if current_queue_size < 50 and self.priority_buckets['low']: - low_items = self.priority_buckets['low'][:10] # Limit low priority - self.priority_buckets['low'] = self.priority_buckets['low'][10:] - self.append(low_items) -``` +`MetricsRecorder` defines `record_db_op`, `record_hook`, `record_cache_event`. Implementations may push to Prometheus, StatsD, or any sink. The library provides: -#### Dynamic Queue Filtering and Manipulation - -```python path=null start=null -class AdaptiveQueueWalker(Walker): - def __init__(self): - super().__init__() - self.queue_stats = { - 'added': 0, - 'removed': 0, - 'filtered': 0, - 'reordered': 0 - } - - @on_visit("FilterNode") - async def adaptive_filtering(self, here: Node): - """Demonstrate dynamic queue filtering and manipulation. - - Args: - here: The visited FilterNode - """ - - # Add new nodes based on current context - candidates = await here.nodes( - node=['ProcessNode'], - active=True - ) - - if candidates: - # Filter candidates before adding to queue - filtered_candidates = self._filter_candidates(candidates) - - if filtered_candidates: - # Smart queue insertion based on current load - current_queue_size = len(self.get_queue()) - - if current_queue_size < 20: - # Low load: add all to end - self.append(filtered_candidates) - self.queue_stats['added'] += len(filtered_candidates) - else: - # High load: add only high-priority to front - high_priority = [ - c for c in filtered_candidates - if getattr(c, 'priority', 0) >= 8 - ] - if high_priority: - self.prepend(high_priority) - self.queue_stats['added'] += len(high_priority) - - # Periodic queue maintenance - if hasattr(here, 'name') and here.name.endswith('_maintenance'): - self._perform_queue_maintenance() - - def _filter_candidates(self, candidates: List[Node]) -> List[Node]: - """Filter candidates based on various criteria.""" - filtered = [] - current_queue = self.get_queue() - - for candidate in candidates: - # Check if already in queue (avoid duplicates) - if self.is_queued(candidate): - continue - - # Check resource constraints (mock implementation) - if hasattr(candidate, 'resource_requirement'): - if candidate.resource_requirement > self._get_available_resources(): - continue - - # Check dependencies (mock implementation) - if hasattr(candidate, 'dependencies'): - if not self._dependencies_met(candidate.dependencies): - continue - - filtered.append(candidate) - - self.queue_stats['filtered'] += len(candidates) - len(filtered) - return filtered - - def _perform_queue_maintenance(self): - """Perform queue cleanup and optimization.""" - current_queue = self.get_queue() - if not current_queue: - return - - print("🔧 Performing queue maintenance...") - - # 1. Remove stale items (mock - would check expiration) - non_stale = [] - removed_stale = 0 - - for item in current_queue: - if hasattr(item, 'expires_at'): - # Mock expiration check - if not getattr(item, 'is_expired', False): - non_stale.append(item) - else: - removed_stale += 1 - else: - non_stale.append(item) - - # 2. Deduplicate items by ID - seen_ids = set() - deduplicated = [] - removed_duplicates = 0 - - for item in non_stale: - if item.id not in seen_ids: - seen_ids.add(item.id) - deduplicated.append(item) - else: - removed_duplicates += 1 - - # 3. Reorder by priority - optimized = sorted( - deduplicated, - key=lambda x: getattr(x, 'priority', 0), - reverse=True - ) - - # 4. Rebuild queue with optimized order - self.clear_queue() - if optimized: - self.append(optimized) - - self.queue_stats['removed'] += removed_stale + removed_duplicates - self.queue_stats['reordered'] += 1 - - print(f"Maintenance complete: removed {removed_stale} stale, " - f"{removed_duplicates} duplicates, optimized {len(optimized)} items") - - def _get_available_resources(self) -> int: - """Mock implementation - get available system resources.""" - return 100 - - def _dependencies_met(self, dependencies: List[str]) -> bool: - """Mock implementation - check if dependencies are satisfied.""" - return True - - @on_exit - async def report_queue_stats(self): - """Report queue manipulation statistics.""" - print("\n📈 Queue Statistics:") - for stat, value in self.queue_stats.items(): - print(f" {stat.capitalize()}: {value}") -``` +- `NullRecorder` — default; no-op. +- OpenTelemetry adapter (`observability/otel.py`) — optional, requires `[otel]` extra. -#### Best Practices for Queue Manipulation - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Check queue state before manipulation -current_queue = self.get_queue() -if current_queue: - next_item = current_queue[0] # Look at next item - # Make decisions based on next item - -# Good: Use appropriate method for insertion priority -if item.priority >= 8: - self.prepend([item]) # High priority to front -else: - self.append([item]) # Normal priority to end - -# Good: Check if node is queued before operations -if self.is_queued(completed_node): - self.dequeue(completed_node) - -# Good: Batch queue operations for efficiency -new_items = await node.nodes(filters) -if new_items: - self.append(new_items) # Add all at once - -# Good: Safe queue iteration and modification -current_queue = self.get_queue() # Get snapshot -filtered_items = [n for n in current_queue if meets_criteria(n)] -self.clear_queue() -if filtered_items: - self.append(filtered_items) - -# Good: Precise insertion with error handling -try: - self.insert_after(target_node, new_nodes) -except ValueError: - # Target not found, use alternative - self.prepend(new_nodes) -``` +### 14.2 Structured DB logging -**❌ Avoided Patterns:** +`jvspatial/db/_observable.py` — wraps a `Database` to emit structured log records per op: `{op, collection, duration_ms, query_size, result_size}`. Configurable slow-query threshold (`slow_query_ms`) emits warnings. -```python path=null start=null -# Bad: Modifying queue during iteration -for item in self.get_queue(): # Don't iterate over changing queue - if condition: - self.dequeue(item) # Modifying during iteration +### 14.3 GraphContext PerformanceMonitor -# Bad: Not handling insertion errors -self.insert_after(target_node, nodes) # Could raise ValueError +In-context counters for DB ops, hook execs, cache stats. Read at end-of-request to surface aggregate metrics. -# Bad: Inefficient repeated operations -for item in items: - self.append([item]) # Many small operations -# Better: self.append(items) # Single batch operation +--- -# Bad: Not checking queue state -first_item = self.get_queue()[0] # Could cause IndexError if empty +## 15. Security Boundaries -# Bad: Assuming nodes are still queued -self.dequeue(node) # Node might not be in queue -# Better: if self.is_queued(node): self.dequeue(node) -``` +### 15.1 User-input boundary -**Key Points:** -- Use `await node.nodes()` to get connected nodes for traversal (NOT `get_edges()`) -- Default `direction="out"` follows outgoing edges (recommended for forward traversal) -- Use `direction="in"` for reverse traversal along incoming edges -- Use `direction="both"` for bidirectional traversal -- **Semantic Filtering Approaches:** - - **Simple filtering**: Use kwargs for connected node properties: `state="NY"` - - **Complex node filtering**: `node=[{'City': {"context.population": {"$gte": 500000}}}]` - - **Complex edge filtering**: `edge=[{'Highway': {"context.condition": {"$ne": "poor"}}}]` - - **Mixed approaches**: Combine dict filters with kwargs for maximum flexibility -- **Database-Level Optimization**: All filtering happens at database level for performance -- **MongoDB-Style Operators**: `$gte`, `$lt`, `$ne`, `$in`, `$nin`, `$regex`, etc. -- **Walker Control Flow:** - - **`skip()`**: Skip current node processing, continue to next (like `continue` in loops) - - **`pause()`/`resume()`**: Temporarily pause walker (use `self.pause()`), can be resumed later - - **`disengage()`**: Permanently halt walker and remove from graph (cannot be resumed) - - **`@on_exit`**: Methods called when walker completes, pauses, or disengages (cleanup) -- The `nodes()` method returns a list that can be directly passed to `walker.visit()` - -### Inheritance Hierarchy +User input crosses the trust boundary at: +1. **HTTP request handlers** — validated by Pydantic at the FastAPI layer. +2. **`Entity.update()`** — validates all field names against the class hierarchy; rejects undeclared attributes (`object.py` setter). +3. **File uploads** — content-based MIME validation, not extension-based. +4. **Path inputs** — `path_sanitizer.py` before any filesystem access. -``` -Object (base class with unified query interface) -├── Node (spatial graph nodes) -├── Edge (relationships) -└── Custom entities (inherit from Node/Object) -``` +### 15.2 Constant-time comparisons -### Database Backends +All secret/key/token/hash comparisons use `hmac.compare_digest`. Affected paths: +- API key verification +- Refresh token comparison +- Password reset token comparison +- Webhook HMAC signature verification +- Deferred-invoke secret comparison +- Legacy bcrypt fallback hash verification -- **JSONDatabase** - File-based storage for development/testing -- **MongoDatabase** - MongoDB backend for production -- **Custom databases** - Implement DatabaseInterface +Removing constant-time comparisons in favor of `==` is a security regression. -## 📚 Documentation Maintenance +### 15.3 Security headers -### README Updates +`jvspatial/api/components/security_headers_middleware.py` injects: +- `X-Content-Type-Options: nosniff` +- `X-Frame-Options: DENY` +- `X-XSS-Protection: 1; mode=block` +- `Content-Security-Policy` — strict on app routes; relaxed only on `/docs`, `/redoc`, `/openapi.json` (CDN allowed). +- `Strict-Transport-Security` (HSTS) when HTTPS termination is configured. -When adding new features: +### 15.4 CORS defaults -1. **Review existing README.md** -2. **Update feature list** if adding major functionality -3. **Update installation/setup** if dependencies change -4. **Update usage examples** to reflect new capabilities -5. **Maintain consistency** with existing documentation style +Default CORS is restrictive (no wildcards). Wildcard origins must be set explicitly and trigger a startup warning. -### Documentation Directory +### 15.5 Secret handling -Always review and update `docs/` directory: +- JWT secret required at startup; empty/weak values rejected. +- S3 credentials required if file storage is enabled. +- API key secrets shown plaintext **only** on creation response; stored hashed. +- 500 error responses sanitize secret-bearing fields. -``` -docs/ -├── api/ # API reference documentation -├── guides/ # User guides and tutorials -├── architecture/ # System design documents -└── examples/ # Code examples and tutorials -``` +--- -**Update procedure:** -1. Check if new feature requires new documentation files -2. Update existing API documentation for modified classes/methods -3. Add user guides for complex features -4. Update architecture docs if design patterns change +## 16. Extension Points -## 💡 Examples Maintenance +### 16.1 Custom databases -### Example Directory Structure +`register_database_type("name", factory_fn)` (`jvspatial/db/factory.py`) registers a custom adapter. The factory must return a `Database` subclass instance. After registration, callers use `create_database("name", ...)` like a built-in. -``` -examples/ -├── basic/ # Simple usage examples -├── advanced/ # Complex scenarios -├── query_interface_example.py # Comprehensive entity-centric CRUD operations -├── semantic_filtering.py # Advanced semantic filtering with Node.nodes() -└── migration/ # Migration guides (if needed) -``` +### 16.2 Endpoint registration -### Example Update Procedure +`@endpoint(...)` collects targets at import time. Either decorate functions or `Walker` subclasses. The framework registers routes at server build. -1. **Review existing examples** for relevance to new features -2. **Update outdated examples** to use modern entity-centric syntax -3. **Create new examples** for significant new features -4. **Ensure examples are runnable** and well-documented -5. **Remove or archive obsolete examples** -6. **Update key reference examples:** - - `query_interface_example.py` - Showcase latest entity-centric patterns - - `semantic_filtering.py` - Demonstrate advanced Node.nodes() filtering - - Basic examples - Keep simple, focused, and beginner-friendly - - Advanced examples - Show complex real-world scenarios +### 16.3 Hooks -### Example Code Standards +`Server(...)` accepts lifecycle and event hooks: -Examples should: -- Use entity-centric syntax exclusively -- Include comprehensive comments -- Demonstrate best practices -- Be self-contained and runnable -- Show error handling patterns +| Hook | Fires on | +|---|---| +| `on_startup` | App boot, after DB ready | +| `on_shutdown` | App teardown | +| `on_admin_bootstrapped` | First admin user created | +| `on_user_registered` | New user creation | +| `on_enrich_current_user` | Per-request user enrichment | +| `on_password_reset_requested` | Password reset flow | -## 🧪 Testing Requirements +### 16.4 Deferred task handlers -### Test Organization +`@deferred_invoke_handler("task.type")` registers an async handler. Handlers must be idempotent. -``` -tests/ -├── api/ # FastAPI endpoint tests -├── core/ # Core entity and logic tests -├── db/ # Database backend tests -└── integration/ # End-to-end integration tests -``` +### 16.5 Custom storage backends -### Testing Procedure for New Features +Implement `FileStorageInterface` (`storage/interfaces/base.py`) and register via the storage manager (`storage/managers/proxy.py`). -1. **Unit Tests** - Test individual methods and classes -2. **Integration Tests** - Test feature interaction with database -3. **API Tests** - Test HTTP endpoints if applicable -4. **Performance Tests** - For database-heavy features +### 16.6 Middleware -### Test Code Standards +`server.middleware_manager.add(...)` accepts any Starlette-compatible middleware. -```python -import pytest -from typing import List -from jvspatial.core import Node, Walker, Edge -from jvspatial.core.entities import on_visit -from jvspatial.exceptions import NodeNotFoundError, ValidationError - -class TestUser(Node): - name: str = "" - email: str = "" - department: str = "" - age: int = 0 - active: bool = True - skills: List[str] = [] - -class TestDepartment(Node): - name: str = "" - location: str = "" - budget: int = 0 - -class TestWalker(Walker): - def __init__(self): - super().__init__() - self.visited_users = [] - - @on_visit("TestUser") - async def visit_user(self, node: TestUser): - self.visited_users.append(node.name) - # Test semantic filtering in walker - connected_users = await node.nodes( - node=['TestUser'], - department=node.department, - active=True - ) - await self.visit(connected_users) - -# Entity Creation Tests -@pytest.mark.asyncio -async def test_user_creation(): - """Test entity-centric user creation with full data.""" - user = await TestUser.create( - name="Alice Johnson", - email="alice@company.com", - department="engineering", - age=30, - skills=["python", "javascript"] - ) - assert user.name == "Alice Johnson" - assert user.email == "alice@company.com" - assert user.department == "engineering" - assert user.id is not None - assert "python" in user.skills - -# MongoDB-Style Query Tests -@pytest.mark.asyncio -async def test_mongodb_queries(): - """Test comprehensive MongoDB-style queries.""" - # Setup test data - await TestUser.create(name="Bob Smith", email="bob@test.com", age=25, department="engineering") - await TestUser.create(name="Carol Davis", email="carol@test.com", age=35, department="marketing") - await TestUser.create(name="David Brown", email="david@test.com", age=45, department="engineering") - - # Test regex queries - b_users = await TestUser.find({"context.name": {"$regex": "^B", "$options": "i"}}) - assert len(b_users) >= 1 - assert any(u.name.startswith("B") for u in b_users) - - # Test comparison operators - senior_users = await TestUser.find({"context.age": {"$gte": 35}}) - assert all(u.age >= 35 for u in senior_users) - - # Test logical operators - senior_engineers = await TestUser.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.age": {"$gte": 30}} - ] - }) - assert all(u.department == "engineering" and u.age >= 30 for u in senior_engineers) - - # Test array operations - tech_users = await TestUser.find({"context.skills": {"$in": ["python", "javascript"]}}) - assert all(any(skill in u.skills for skill in ["python", "javascript"]) for u in tech_users) - -# Semantic Filtering Tests -@pytest.mark.asyncio -async def test_semantic_filtering(): - """Test Node.nodes() semantic filtering capabilities.""" - # Create test nodes and edges - dept = await TestDepartment.create(name="Engineering", location="SF") - user1 = await TestUser.create(name="Alice", department="engineering", active=True) - user2 = await TestUser.create(name="Bob", department="engineering", active=False) - user3 = await TestUser.create(name="Carol", department="marketing", active=True) - - # Create connections - await Edge.create(source=dept, target=user1, edge_type="employs") - await Edge.create(source=dept, target=user2, edge_type="employs") - await Edge.create(source=user1, target=user3, edge_type="collaborates") - - # Test simple filtering - active_employees = await dept.nodes( - node=['TestUser'], - active=True - ) - assert len(active_employees) == 1 - assert active_employees[0].name == "Alice" - - # Test complex filtering - engineering_employees = await dept.nodes( - node=[{'TestUser': {"context.department": "engineering"}}], - active=True - ) - assert all(u.department == "engineering" and u.active for u in engineering_employees) - - # Test mixed filtering - collaborators = await user1.nodes( - node=[{'TestUser': {"context.active": True}}], - department="marketing" - ) - assert len(collaborators) >= 1 - -# Walker Tests -@pytest.mark.asyncio -async def test_walker_traversal(): - """Test walker with semantic filtering.""" - # Setup graph - root = await TestUser.create(name="Root", department="engineering") - user1 = await TestUser.create(name="User1", department="engineering", active=True) - user2 = await TestUser.create(name="User2", department="marketing", active=True) - - await Edge.create(source=root, target=user1, edge_type="manages") - await Edge.create(source=root, target=user2, edge_type="manages") - await Edge.create(source=user1, target=user2, edge_type="collaborates") - - # Test walker - walker = TestWalker() - await walker.spawn(root) - - # Verify walker visited users - assert "Root" in walker.visited_users - assert len(walker.visited_users) >= 1 - -# Error Handling Tests -@pytest.mark.asyncio -async def test_error_handling(): - """Test proper error handling patterns.""" - # Test NodeNotFoundError - with pytest.raises(NodeNotFoundError): - await TestUser.get("nonexistent-id") - - # Test safe retrieval - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await TestUser.find({"context.email": "nonexistent@test.com"}) - user = users[0] if users else None - assert user is None - - # Test validation errors (if validation is implemented) - with pytest.raises((ValidationError, ValueError)): - await TestUser.create(name="", email="invalid-email") - -# Performance Tests -@pytest.mark.asyncio -async def test_bulk_operations(): - """Test bulk operations performance.""" - # Create multiple users - users_data = [ - {"name": f"User{i}", "email": f"user{i}@test.com", "department": "engineering"} - for i in range(10) - ] - - # Bulk create - users = [] - for user_data in users_data: - user = await TestUser.create(**user_data) - users.append(user) - - assert len(users) == 10 - - # Bulk query - engineering_users = await TestUser.find({"context.department": "engineering"}) - assert len(engineering_users) >= 10 - - # Bulk update - for user in users[:5]: - user.active = False - await user.save() - - # Verify updates - inactive_users = await TestUser.find({"context.active": False}) - assert len(inactive_users) >= 5 - -# Integration Tests -@pytest.mark.asyncio -async def test_full_workflow(): - """Test complete entity lifecycle with relationships.""" - # Create department - dept = await TestDepartment.create(name="Product", location="NYC", budget=1000000) - - # Create users - manager = await TestUser.create( - name="Jane Manager", - email="jane@company.com", - department="product", - age=40 - ) - - employee = await TestUser.create( - name="John Employee", - email="john@company.com", - department="product", - age=28 - ) - - # Create relationships - await Edge.create(source=dept, target=manager, edge_type="employs") - await Edge.create(source=dept, target=employee, edge_type="employs") - await Edge.create(source=manager, target=employee, edge_type="manages") - - # Test traversal - department_employees = await dept.nodes(node=['TestUser']) - assert len(department_employees) == 2 - - managed_employees = await manager.nodes( - node=['TestUser'], - direction="out", - edge=['manages'] - ) - assert len(managed_employees) == 1 - assert managed_employees[0].name == "John Employee" - - # Test complex query - young_employees = await dept.nodes( - node=[{'TestUser': {"context.age": {"$lt": 30}}}] - ) - assert len(young_employees) == 1 - assert young_employees[0].age < 30 - - # Cleanup - await manager.delete() - await employee.delete() - await dept.delete() - - # Verify deletion - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await TestUser.find({"context.email": "jane@company.com"}) - deleted_manager = users[0] if users else None - assert deleted_manager is None -``` +### 16.7 Custom log levels -## 🗑️ Cleanup and Maintenance +`jvspatial/logging/` permits adding domain levels (e.g. `AUDIT`, `SECURITY`, `TRACE`) above standard levels. See `docs/md/custom-log-levels.md`. -### Deprecation Procedure - -When evolving the library: - -1. **Identify obsolete code** - Old patterns, unused utilities -2. **Mark as deprecated** - Add deprecation warnings before removal -3. **Update documentation** - Remove references to deprecated features -4. **Update examples** - Remove or update examples using deprecated code -5. **Clean removal** - Remove deprecated code after grace period - -### File Cleanup Checklist - -- [ ] Remove unused import statements -- [ ] Delete empty or obsolete modules -- [ ] Archive outdated examples to `examples/archive/` -- [ ] Update `__all__` exports in `__init__.py` files -- [ ] Remove commented-out code blocks -- [ ] Clean up temporary test files - -### Migration Strategy - -When making breaking changes: - -1. **Create migration guide** in `docs/migration/` -2. **Provide before/after examples** -3. **Update all existing examples** to new patterns -4. **Add runtime warnings** for deprecated usage -5. **Version appropriately** using semantic versioning - -## 🚀 Development Workflow - -### Pre-commit Checklist - -- [ ] Code passes `black --check .` -- [ ] Code passes `flake8 .` -- [ ] Code passes `mypy .` -- [ ] All tests pass: `pytest` -- [ ] Examples are updated and runnable -- [ ] Documentation reflects changes -- [ ] Deprecated code is cleaned up - -### Code Review Focus Areas - -1. **Entity-centric patterns** - Ensure new code uses preferred syntax -2. **Query interface consistency** - MongoDB-style queries throughout -3. **Type safety** - Proper annotations and mypy compliance -4. **Test coverage** - Adequate testing for new features -5. **Documentation completeness** - Examples and guides updated - -## 📋 Quick Reference - -### Preferred Patterns - -```python path=null start=null -# ✅ Entity creation -user = await User.create(name="Alice", email="alice@company.com", department="engineering") - -# ✅ Entity queries with MongoDB-style operators -users = await User.find({"context.active": True}) -# Note: Object.find_one() doesn't exist - use find() and get first result -users_by_email = await User.find({"context.email": email}) -user = users_by_email[0] if users_by_email else None -senior_engineers = await User.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.age": {"$gte": 35}} - ] -}) -tech_users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) - -# ✅ Counting and aggregation -# Note: Object.count() doesn't exist - use len() with find() instead -results = await User.find({"context.department": "engineering"}) -count = len(results) - -# Note: Object.distinct() doesn't exist - query and extract manually -all_users = await User.find({}) -departments = set(u.department for u in all_users if hasattr(u, 'department')) - -# ✅ Entity updates -user.name = "Alice Johnson" -user.active = True -await user.save() - -# ✅ Bulk updates -users = await User.find({"context.department": "old_dept"}) -for user in users: - user.department = "new_dept" - await user.save() - -# ✅ Entity deletion -await user.delete() - -# ✅ Walker traversal with semantic filtering -@on_visit("User") -async def process_user(self, here: Node): - """Process user nodes with semantic filtering. - - Args: - here: The visited User node - """ - # Use nodes() method with semantic filtering - connected_users = await here.nodes( - node=['User'], # Type filtering - department="engineering", # Simple kwargs - active=True # Multiple simple filters - ) - await self.visit(connected_users) - - # Complex filtering with MongoDB operators - senior_connections = await here.nodes( - node=[{'User': {"context.age": {"$gte": 35}}}], - direction="out" - ) - await self.visit(senior_connections) - -# ✅ Walker control flow -class MyWalker(Walker): - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with control flow. - - Args: - here: The visited Document node - """ - if here.status == "archived": - self.skip() # Skip to next node - - if self.processed_count >= 100: - await self.disengage() # Permanently halt - - # Normal processing - next_docs = await here.nodes(node=['Document'], active=True) - await self.visit(next_docs) - - @on_exit - async def cleanup(self): - print(f"Processed {self.processed_count} documents") - -# ✅ Error handling -try: - user = await User.get(user_id) -except NodeNotFoundError: - logger.warning(f"User {user_id} not found") - return None -except DatabaseError as e: - logger.error(f"Database error: {e}") - raise -``` +--- -### Avoided Patterns - -```python path=null start=null -# ❌ Direct database access (discouraged - use entity methods) -from jvspatial.db import create_database -db = create_database("json") -await db.save("node", data) # Use entity.save() instead -await db.find("node", {"name": "User"}) # Use Entity.find() instead - -# ❌ GraphContext methods for simple operations (use entity methods) -from jvspatial.core import GraphContext -ctx = GraphContext(database=db) -# Prefer entity-centric methods: -# await Node.create(...) instead of ctx.create_node(...) -# await node.get_edges() instead of ctx.get_edges(node_id) - -# ❌ Non-standard query formats -await User.find({"age": 25}) # Missing context. prefix -await User.find({"name": "Alice"}) # Should be context.name - -# ❌ Old traversal patterns (deprecated) -walker.get_edges(node) # Use node.nodes() instead -walker.traverse_edges() # Use semantic filtering - -# ❌ Synchronous operations -user = User.create_sync(**data) # Use async await User.create() -users = User.find_sync(query) # Use async await User.find() - -# ❌ Manual edge management in walkers (show proper naming even in bad examples) -@on_visit("User") -async def visit_user(self, here: Node): - """DEPRECATED: Manual edge management (avoid this pattern). - - Args: - here: The visited User node - """ - # Avoid manual edge retrieval - edges = await here.get_edges() # Deprecated - for edge in edges: - target = await edge.get_target() - await self.visit([target]) - - # Instead, use semantic filtering - connected = await here.nodes() # Preferred - await self.visit(connected) - -# ❌ Missing error handling -user = await User.get(user_id) # Should handle NodeNotFoundError -user.field = value -await user.save() # Should handle ValidationError - -# ❌ Inefficient queries -# Don't fetch all then filter in Python -all_users = await User.find({}) -engineers = [u for u in all_users if u.department == "engineering"] - -# Instead, filter at database level -engineers = await User.find({"context.department": "engineering"}) - -# ❌ Blocking operations in async context -@on_visit("DataNode") -async def process_node(self, here: Node): - """BAD EXAMPLE: Blocking operations in async context. - - Args: - here: The visited DataNode - """ - # Avoid blocking operations - time.sleep(1.0) # Blocks event loop - - # Use async alternatives - await asyncio.sleep(1.0) # Non-blocking -``` +## 17. Error Taxonomy + +`jvspatial/exceptions.py` centralizes all exceptions. Hierarchy: + +``` +JVSpatialError +├── EntityError +│ ├── EntityNotFoundError +│ │ ├── NodeNotFoundError +│ │ ├── EdgeNotFoundError +│ │ └── ObjectNotFoundError +│ └── DuplicateEntityError +├── ValidationError +│ └── FieldValidationError +├── ConfigurationError +│ ├── InvalidConfigurationError +│ └── MissingConfigurationError +├── DatabaseError +│ ├── ConnectionError +│ ├── QueryError +│ ├── TransactionError +│ └── VersionConflictError +├── GraphError +│ ├── InvalidGraphStructureError +│ ├── CircularReferenceError +│ └── EdgeConnectionError +├── WalkerError +│ ├── WalkerExecutionError +│ ├── WalkerTimeoutError +│ └── InfiniteLoopError +├── APIError (via jvspatial.api.exceptions) +│ ├── JVSpatialAPIException +│ ├── EndpointError +│ ├── ParameterError +│ ├── AuthenticationError +│ ├── AuthorizationError +│ ├── RateLimitError +│ ├── InvalidCredentialsError +│ └── RegistrationDisabledError +└── SecurityError + └── PermissionDeniedError +``` + +Control-flow exceptions for walkers live in `jvspatial.core.entities`: +- `TraversalPaused` — caller catches to suspend traversal +- `TraversalSkipped` — caller catches to skip current node + +### 17.1 Propagation rules + +- Database errors propagate to the caller; entity methods may wrap with context. +- Validation errors raise `ValidationError` with `field_errors`. The API layer maps to HTTP 400. +- Authentication errors map to HTTP 401; authorization errors to HTTP 403. +- Walker errors are logged and surface in `walker.response["errors"]`; traversal halts. --- -## ⏰ Scheduler Integration (Optional) +## 18. Stability Tiers -jvspatial includes optional scheduler support for background task automation. Install with: +`docs/md/stability.md` is the canonical tier reference. Summary: -```bash -pip install jvspatial[scheduler] -``` +| Tier | Guarantee | +|---|---| +| **Stable** | Public API. Breaking changes require a major version bump and two-version deprecation grace. | +| **Internal** | May change between minor versions. Documented for callers who need the detail, but not contracted. | +| **Experimental** | May change in any release. Opt-in only. Examples: `JsonDBTransaction(best_effort=True)`, some deferred-save edge cases. | -### Basic Scheduled Tasks - -```python path=null start=null -from jvspatial.api import Server -from jvspatial.api.scheduler import on_schedule -from jvspatial.core import Object -from datetime import datetime -from typing import Optional -import logging - -logger = logging.getLogger(__name__) - -# Define entity for job tracking (entity-centric pattern) -class ScheduledJob(Object): - """Entity representing scheduled job execution records.""" - job_name: str = "" - execution_time: datetime = datetime.now() - status: str = "pending" # pending, completed, failed - duration_seconds: Optional[float] = None - error_message: Optional[str] = None - -class SystemMetrics(Object): - """Entity for system metrics collection.""" - timestamp: datetime = datetime.now() - cpu_usage: float = 0.0 - memory_usage: float = 0.0 - active_jobs: int = 0 - -# Scheduled function with proper error handling -@on_schedule("every 30 minutes", description="System cleanup with job tracking") -async def cleanup_system(): - """Automated cleanup with entity-centric job tracking.""" - start_time = datetime.now() - - try: - logger.info("🧹 Starting system cleanup") - - # Perform cleanup work - cleanup_count = await perform_cleanup_work() - - # Create success record - await ScheduledJob.create( - job_name="system_cleanup", - execution_time=start_time, - status="completed", - duration_seconds=(datetime.now() - start_time).total_seconds() - ) - - logger.info(f"✅ Cleanup completed: {cleanup_count} items processed") - - except Exception as e: - # Create error record - await ScheduledJob.create( - job_name="system_cleanup", - execution_time=start_time, - status="failed", - error_message=str(e), - duration_seconds=(datetime.now() - start_time).total_seconds() - ) - logger.error(f"❌ Cleanup failed: {str(e)}") - raise - -# Metrics collection with MongoDB-style queries -@on_schedule("every 5 minutes", retry_count=2, description="Collect system metrics") -async def collect_metrics(): - """Collect system metrics with entity queries.""" - import psutil - - # Get system metrics - cpu_percent = psutil.cpu_percent() - memory = psutil.virtual_memory() - - # Count active jobs using MongoDB-style query - # Note: Object.count() doesn't exist - use len() with find() instead - active_jobs_list = await ScheduledJob.find({ - "context.status": {"$in": ["pending", "running"]} - }) - active_jobs = len(active_jobs_list) - - # Create metrics record - await SystemMetrics.create( - timestamp=datetime.now(), - cpu_usage=cpu_percent, - memory_usage=memory.percent, - active_jobs=active_jobs - ) - - logger.info(f"📊 Metrics: CPU {cpu_percent:.1f}%, Memory {memory.percent:.1f}%") - -async def perform_cleanup_work() -> int: - """Simulate cleanup work.""" - # Query old records using entity-centric approach - cutoff_time = datetime.now().timestamp() - (7 * 24 * 3600) # 7 days ago - - old_jobs = await ScheduledJob.find({ - "context.execution_time": {"$lt": cutoff_time} - }) - - # Delete old records - for job in old_jobs: - await job.delete() - - return len(old_jobs) -``` +**Stable** includes (non-exhaustive): `Object`/`Node`/`Edge`/`Walker` public methods, `@endpoint` and `@attribute` decorators, `Database` ABC contract, `Server` constructor. -### Server Integration - -```python path=null start=null -from jvspatial.api import Server, endpoint -from jvspatial.api.scheduler import register_scheduled_tasks -from typing import Dict, Any -from dotenv import load_dotenv - -# Load environment configuration (jvspatial pattern) -load_dotenv() - -# Create server with scheduler enabled -server = Server( - title="My Scheduled App", - description="Application with integrated scheduler", - version="1.0.0", - scheduler_enabled=True, # Enable scheduler - scheduler_interval=1, # Check every second - scheduler_timezone="UTC", # Timezone for scheduling -) +**Internal** includes: contents of `jvspatial/db/_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`; `jvspatial/api/components/*` implementation details; `jvspatial/storage/managers/*` internals. -# Register all decorated scheduled tasks -if hasattr(server, 'scheduler_service') and server.scheduler_service: - register_scheduled_tasks(server.scheduler_service) - logger.info("✅ Scheduled tasks registered") - -# Add monitoring endpoint -@endpoint("/api/scheduler/status", methods=["GET"]) -async def get_scheduler_status() -> Dict[str, Any]: - """Get scheduler status with entity-centric job statistics.""" - # Get job statistics using entity queries - # Note: Object.count() doesn't exist - use len() with find() instead - all_jobs = await ScheduledJob.find({}) - total_jobs = len(all_jobs) - - completed_jobs_list = await ScheduledJob.find({"context.status": "completed"}) - completed_jobs = len(completed_jobs_list) - - failed_jobs_list = await ScheduledJob.find({"context.status": "failed"}) - failed_jobs = len(failed_jobs_list) - - return { - "scheduler": "running", - "job_statistics": { - "total_jobs": total_jobs, - "completed_jobs": completed_jobs, - "failed_jobs": failed_jobs, - "success_rate": (completed_jobs / total_jobs * 100) if total_jobs > 0 else 0 - }, - "timestamp": datetime.now().isoformat() - } - -if __name__ == "__main__": - server.run(host="0.0.0.0", port=8000) # Scheduler runs automatically -``` +**Experimental** includes: anything explicitly flagged in `docs/md/stability.md`. -**📖 For comprehensive scheduler documentation:** [Scheduler Integration Guide](docs/md/scheduler.md) +Deprecations emit warnings via `jvspatial/utils/deprecation.py` (see file). Removal happens no sooner than two minor versions after the deprecation warning lands. --- -## 🎯 Key Naming Conventions +## Appendix A — Critical File Map + +| Concern | Primary file | +|---|---| +| Identity / `__entity_name__` | `jvspatial/core/entities/object.py:35-44`, `jvspatial/core/utils.py:11-89` | +| Node model | `jvspatial/core/entities/node.py` | +| Edge model | `jvspatial/core/entities/edge.py` | +| Walker model | `jvspatial/core/entities/walker.py`, `walker_components/*` | +| Root singleton | `jvspatial/core/entities/root.py` | +| GraphContext | `jvspatial/core/context.py` | +| Database ABC | `jvspatial/db/database.py:48+` | +| Built-in DB adapters | `jvspatial/db/{jsondb,sqlite,mongodb,dynamodb}.py` | +| Query engine | `jvspatial/db/query.py` | +| Atomic IO | `jvspatial/db/_atomic.py` | +| Path locks | `jvspatial/db/_path_locks.py` | +| DB cache wrapper | `jvspatial/db/_cache.py` | +| DB observability wrapper | `jvspatial/db/_observable.py` | +| Server class | `jvspatial/api/server.py` | +| App builder + CSP/docs gating | `jvspatial/api/components/app_builder.py` | +| `@endpoint` decorator | `jvspatial/api/decorators/route.py` | +| Endpoint registry | `jvspatial/api/endpoints/registry.py` | +| Auth service | `jvspatial/api/auth/service.py` | +| RBAC | `jvspatial/api/auth/rbac.py` | +| API key service | `jvspatial/api/auth/api_key_service.py` | +| Config + env merge | `jvspatial/api/config.py`, `jvspatial/env.py`, `jvspatial/env_adapter.py` | +| Serverless detection | `jvspatial/runtime/serverless.py` | +| Deferred task dispatch | `jvspatial/serverless/deferred_invoke.py`, `serverless/tasks/*` | +| Storage interfaces | `jvspatial/storage/interfaces/{base,local,s3}.py` | +| Storage security | `jvspatial/storage/security/{path_sanitizer,validator}.py` | +| Cache backends | `jvspatial/cache/{base,memory,redis,layered}.py` | +| Observability | `jvspatial/observability/{metrics,otel}.py` | +| Logging service | `jvspatial/logging/{config,service,models}.py` | +| Exceptions | `jvspatial/exceptions.py`, `jvspatial/api/exceptions.py` | +| Version | `jvspatial/version.py` | -**CRITICAL**: When writing `@on_visit` methods, always use these parameter names: - -- **`here`** - The visited node/edge (current location in traversal) -- **`visitor`** - The walker performing the traversal (when passed to node methods) +--- -```python path=null start=null -# ✅ CORRECT naming convention -@on_visit("User") -async def process_user(self, here: Node): - """Args: here = visited User node""" - connected = await here.nodes() - await self.visit(connected) +## Appendix B — Maintenance Checklist for Spec Changes -# ❌ AVOID generic names -@on_visit("User") -async def process_user(self, node: Node): # Less clear - pass -``` - ---- +When code changes alter behavior described in this document: -**Remember**: This library prioritizes **clean, maintainable code** with **consistent patterns** across all database backends. Always favor the entity-centric approach, MongoDB-style queries, and the **`here`/`visitor`** naming convention for the best developer experience. +1. Update the affected section, citing the new `file:line` if structure changed. +2. If the change is breaking, also update [ROADMAP.md](ROADMAP.md) (release notes) and [CHANGELOG.md](CHANGELOG.md). +3. If the change affects stability tier, also update [docs/md/stability.md](docs/md/stability.md). +4. If the change affects a security invariant (§15), also update [docs/md/security-review.md](docs/md/security-review.md). +5. Add a test that exercises the new behavior. From cac1e895270f1206ef76b96e888ca180a5ebe0c1 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:14:35 -0400 Subject: [PATCH 04/22] docs: add PRD capturing product context and decision boundaries Adds PRD.md to complement SPEC.md (technical contract) with the product layer: positioning, target users (described by role, not by named product), core value propositions, explicit non-goals, constraints, success criteria, stability-tier ratification, and decision boundaries for evaluating proposed changes. Establishes the documentation triangle: PRD answers "why", SPEC answers "what", and the docs/md/ tree answers "how". Co-Authored-By: Claude Opus 4.7 (1M context) --- PRD.md | 195 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 PRD.md diff --git a/PRD.md b/PRD.md new file mode 100644 index 0000000..c820f62 --- /dev/null +++ b/PRD.md @@ -0,0 +1,195 @@ +# jvspatial — Product Requirements Document + +> **Purpose**: Capture *why* jvspatial exists, who it serves, and what success looks like. This is the product context for the technical contract in [SPEC.md](SPEC.md). +> +> **Audience**: Maintainers (human and AI), reviewers, future contributors evaluating whether a proposed change is in scope. +> +> **Status**: Living document. Update in the same commit as any change that alters scope, target users, or success criteria. + +--- + +## 1. Positioning + +jvspatial is an **async-first, serverless-compatible, object-spatial Python library** for building persistence and business-logic application layers on top of a graph data model. + +It is *not*: + +- A graph database engine. jvspatial layers a graph model on top of existing backends (MongoDB, SQLite, JSON files, DynamoDB) and unifies query semantics across them. +- A web framework. It builds on FastAPI; it does not replace it. +- A workflow engine. Walkers traverse data; they are not a generic task orchestrator. +- A UI library. Visualization helpers exist (DOT, Mermaid), but rendering is out of scope. + +It *is*: + +- A substrate for graph-native business logic, with an entity-centric API that hides storage details from domain code. +- A FastAPI integration layer that turns decorated entities and walkers into HTTP endpoints with consistent auth, validation, and observability. +- A serverless-aware library that adapts its defaults (deferred saves, index creation, hashing cost) to the deployment environment. + +The library is inspired by Jaseci's object-spatial paradigm. The implementation is pure Python, async-throughout, and uses Pydantic v2 for schemas. + +--- + +## 2. Target Users + +Described by role, not by named product. jvspatial serves callers in three roles: + +### 2.1 Backend engineers building graph-shaped applications + +People modeling domains where relationships are first-class: knowledge graphs, social structures, agent memory, organizational hierarchies, recommendation systems, geographic networks. + +**What they need**: an entity model with first-class edges, traversal primitives that respect graph semantics, async I/O so a request handler can fan out across the graph without blocking. + +### 2.2 Platform teams building serverless APIs over graph data + +People deploying graph-backed services to AWS Lambda, Google Cloud Run, Azure Functions, Vercel — environments that need stateless request handling, fast cold starts, and explicit boundaries between in-request work and deferred work. + +**What they need**: serverless mode detection that flips opinionated defaults (deferred saves off, indexes off, lower bcrypt cost, `/tmp` DB paths), a deferred-task dispatcher that survives invocation boundaries, idempotency-friendly handler registration. + +### 2.3 Library authors needing a persistence + traversal substrate + +People building higher-level frameworks that ship a graph model to their own users. They want a foundation they can extend without forking — custom database backends, custom storage backends, custom auth flows, custom log levels. + +**What they need**: stable extension points (`register_database_type`, `FileStorageInterface`, lifecycle hooks, deferred-task handlers), a documented stability contract so their downstream depends only on stable surface, and a clean separation between public API and internal helpers. + +--- + +## 3. Core Value Propositions + +| # | Value | What it means | Where it lives | +|---|---|---|---| +| 1 | **Unified query DSL across backends** | One Mongo-style query syntax works against JSON, SQLite, MongoDB, and DynamoDB. Switch backends without rewriting queries. | `jvspatial/db/query.py` | +| 2 | **Entity-centric API** | `Node.create()`, `Node.get()`, `node.save()` — domain code calls entity methods, never raw DB clients. | `jvspatial/core/entities/object.py` | +| 3 | **First-class walkers** | Graph traversal is a built-in primitive with infinite-loop protection, trail tracking, and event hooks. Not a hand-rolled BFS in every handler. | `jvspatial/core/entities/walker.py` | +| 4 | **Identity model with explicit discriminators** | Every entity carries `type_code.entity_name.uuid`. `__entity_name__` lets disjoint hierarchies coexist. | SPEC §1 | +| 5 | **Serverless ergonomics by default** | One env var (`SERVERLESS_MODE=true`) flips a coherent set of defaults — no scattered serverless special cases in application code. | `jvspatial/runtime/serverless.py` | +| 6 | **Security defaults that fail closed** | JWT secret required, constant-time secret comparisons, content-based file MIME validation, restrictive CORS, strict CSP on app routes, optional `/docs` gating. | SPEC §15 | +| 7 | **FastAPI integration without lock-in** | `@endpoint` exposes functions and walkers as routes. The underlying FastAPI app is accessible for middleware, custom routers, and lifecycle hooks. | `jvspatial/api/server.py` | +| 8 | **Observable by construction** | Structured DB op logs, `MetricsRecorder` protocol, GraphContext `PerformanceMonitor`, OpenTelemetry adapter. | `jvspatial/observability/`, `jvspatial/db/_observable.py` | + +--- + +## 4. Non-Goals + +Explicit out-of-scope statements. These represent decisions, not gaps. + +| Non-goal | Why | +|---|---| +| Replace a real graph database (Neo4j, ArangoDB, Neptune) | jvspatial layers on document stores; it does not implement graph-native storage, indexes, or query planning. Use a graph DB when the workload demands it. | +| Provide a graph traversal query language | Walkers are imperative async Python. There is no Cypher or Gremlin equivalent and no plan to add one. | +| Synchronous API surface | The library is async-only by design. A sync wrapper would force `asyncio.run` per call and degrade performance. Callers stuck in sync contexts should run jvspatial in a worker. | +| Distributed transactions across backends | Multi-database operations are best-effort. Only single-backend transactions are supported (and only where the backend natively supports them — MongoDB replica sets). | +| Built-in caching strategy beyond the wrappers provided | We provide memory, redis, layered. Beyond that, callers compose their own. | +| Schema migrations | No migration framework. Adding optional fields with defaults is forward-safe; everything else is the caller's responsibility. | +| UI / dashboard | Graph export to DOT/Mermaid is provided. Rendering, visualization tooling, and a Web UI are downstream. | +| First-party multi-tenancy primitives | Multi-tenancy is doable via context scoping but is not codified. Tenant isolation is a caller responsibility. | + +--- + +## 5. Constraints + +### 5.1 Language and runtime + +- Python 3.9+ supported (3.8 declared in `pyproject.toml` classifiers but should be considered legacy). +- Pydantic v2 is required — Pydantic v1 compatibility is not maintained. +- FastAPI / Starlette pinned to versions that retain the `Router(on_startup, on_shutdown)` API (Starlette `<1.0.0`). + +### 5.2 Concurrency + +- Async-only I/O. Every database call, network call, and file-system call is a coroutine. +- No global mutable state that survives a request, except the database manager and module-level registries (decorator-collected endpoints, deferred task handlers). All such state is initialized at app build, not per-request. +- Sessions, rate-limit counters, and JWT blacklists are **per-process / per-worker** by default. Cross-worker sharing requires Redis or a similar shared store. + +### 5.3 Serverless + +- Cold-start cost is a budget, not a goal. Reduced bcrypt rounds and skipped index creation in serverless mode reflect this. +- No assumption that any in-memory state survives across invocations. Walker trails, sessions, and caches reset. +- `/tmp` is the only writable filesystem on Lambda; JSON DB defaults route there. + +### 5.4 Security + +- Secrets are never logged. 500 error responses are sanitized. +- Comparison of any secret, key, token, or hash uses `hmac.compare_digest`. This is non-negotiable. +- New auth flows, key types, or token formats require a security review (see [SECURITY.md](SECURITY.md) and `docs/md/security-review.md`). + +### 5.5 Compatibility + +- The `jvspatial/__init__.py` `__all__` export list is the public contract. Submodule import paths are *not* stable; promote symbols through the top-level package. +- Breaking changes to public API require a major version bump (post-1.0) or a minor bump (pre-1.0) with a `**BREAKING**` callout in [CHANGELOG.md](CHANGELOG.md). +- Deprecations follow the policy in [docs/md/stability.md](docs/md/stability.md): mark with `@deprecated`, warn for at least one minor cycle, then remove. + +--- + +## 6. Success Criteria + +### 6.1 Qualitative + +- **A caller building a new graph-backed service can get to a working CRUD endpoint in under 30 minutes**, following [docs/md/quick-start-guide.md](docs/md/quick-start-guide.md) and one of the [examples/api/](examples/api/) reference implementations, without reading any source code. +- **A caller can switch backends (JSON → SQLite → MongoDB → DynamoDB) without rewriting queries or entities** — only configuration changes. +- **A caller deploying to Lambda does not need to write Lambda-specific code**, beyond the entry adapter for their function. Defaults adapt; deferred work has a single registration point. +- **An AI agent maintaining the library can locate the right code path from a single CLAUDE.md read**, without having to crawl the docs tree. +- **A security reviewer can audit the trust boundary by reading SPEC §15** and confirm every claim against `file:line` citations. + +### 6.2 Quantitative + +Quantitative budgets are deferred to the upcoming foundation audit. Placeholders that will be filled in during that pass: + +- p50 / p99 latency budgets for `Entity.get`, `Entity.find`, `Walker.walk` against each backend. +- Cold-start envelope on Lambda (target: well under the platform's 1s default timeout for HTTP API + Lambda). +- Memory footprint per active walker. +- Test-coverage gate (current floor: 60%; CHANGELOG implies a tighter floor is desired but not yet enforced). +- Benchmark regression detection: the existing pytest-benchmark suite must remain green in CI. + +--- + +## 7. Stability Tiers (Ratification) + +The library distinguishes three tiers; [docs/md/stability.md](docs/md/stability.md) is the canonical reference. + +| Tier | Contract | Examples | +|---|---|---| +| **Stable / Public** | Names in `jvspatial.__all__`. Breaking changes require a major version bump (post-1.0) or a minor bump with a `**BREAKING**` callout (pre-1.0). At least one deprecation cycle before removal. | `Object`, `Node`, `Edge`, `Walker`, `Root`, `GraphContext`, `Server`, `ServerConfig`, `Database`, `@endpoint`, `@attribute`, `DeferredSaveMixin`, `is_serverless_mode`, `dispatch_deferred_task`, `MetricsRecorder`. | +| **Internal** | Module path begins with `_` or is documented internal. Can change in any release without notice. Callers should not import directly. | `jvspatial/db/_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`; `jvspatial/api/server_*.py`; `jvspatial/runtime/lwa.py`. | +| **Experimental** | Public but opt-in. May change or be removed in any minor release. Marked via `@experimental` decorator, emits a once-per-process warning. | `JsonDBTransaction(best_effort=True)`. | + +The PRD ratifies this tiering. Promotion of a feature from experimental to stable, or demotion of an internal helper to public, is a product decision and must be reflected here as well as in `docs/md/stability.md`. + +--- + +## 8. Decision Boundaries + +When evaluating proposed changes, apply these decision rules: + +| Question | Answer | +|---|---| +| Does this change introduce a new sync API on a code path that touches I/O? | **Reject.** Async-only is a core constraint (§5.2). | +| Does this change broaden the CORS default to a wildcard, or remove a constant-time comparison? | **Reject.** Security defaults must fail closed (§5.4). | +| Does this change add a feature that duplicates an existing public API? | **Reject** unless the existing API is being deprecated in the same change. | +| Does this change add a new env var? | **Allowed**, but must be added to the `JVSPATIAL_*` allowlist (`jvspatial/env_adapter.py`) and documented in [docs/md/environment-keys-reference.md](docs/md/environment-keys-reference.md). | +| Does this change add a new public name? | **Allowed**, but must be added to `__all__` and [docs/md/stability.md](docs/md/stability.md). | +| Does this change extend the database ABC contract? | **Allowed** only if all four built-in adapters implement it or fall through a sensible default. | +| Does this change couple jvspatial to a specific upstream consumer? | **Reject.** jvspatial is a foundation; consumer-specific logic lives in the consumer. | + +--- + +## 9. Open Questions + +Items the PRD does not yet decide. These are inputs to the upcoming foundation audit. + +- Quantitative success criteria (§6.2 placeholders). +- Whether session state, rate-limit counters, and JWT blacklists should have first-party shared-storage adapters or remain per-worker by default. +- Whether the multi-database manager should grow first-class tenant scoping primitives or remain a thin registry. +- Whether `count()` semantics should guarantee consistency vs. allow eventual count from DynamoDB. +- Whether the experimental `JsonDBTransaction(best_effort=True)` should be promoted, demoted, or removed. + +--- + +## 10. Living-Document Discipline + +This PRD is authored to remain accurate over time. Update rules: + +1. Adding a public API → update §3 (value props) and §7 (stability tiers). +2. Adding a non-goal → update §4 with one-line justification. +3. Changing a constraint → update §5 and reflect in [SPEC.md](SPEC.md) where contract changes. +4. Closing an open question → move from §9 into the appropriate section. + +Any code change that contradicts a current statement here must update this file in the same commit, or document an exception in [ROADMAP.md](ROADMAP.md). From 2993e8a994dd4947b81839526874c42c9b06c360 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:16:48 -0400 Subject: [PATCH 05/22] docs: refresh docs index and add ROADMAP Updates docs/md/README.md to: - Reflect current 0.0.8+ state (was stale at 0.0.6 / 2025-03-16) - Index the 8 docs that existed but were not listed (benchmarks, observability, production-deployment, rate-limiting, serverless-mode, stability, security-review, security-operational-notes, api-keys) - Cross-link to PRD/SPEC/CLAUDE/ROADMAP at repo root - Establish authoritative source order when docs disagree Adds ROADMAP.md capturing current focus areas (security hardening, performance, observability, stability contract), known gaps with file:line citations (migration story, multi-DB transactions, per-worker state, count consistency, walker trail in serverless, quantitative budgets, Python 3.8 classifier, experimental JsonDBTransaction status), and out-of-scope reaffirmations. Co-Authored-By: Claude Opus 4.7 (1M context) --- ROADMAP.md | 154 +++++++++++++++ docs/md/README.md | 468 +++++++++++++--------------------------------- 2 files changed, 284 insertions(+), 338 deletions(-) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..39e9107 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,154 @@ +# jvspatial — Roadmap + +> **Purpose**: Forward-looking maintenance direction. Captures current focus, known gaps and tech debt, areas under active hardening, and the release process. Not a feature wishlist — every entry has a citation or a justification. +> +> **Audience**: Maintainers planning the next pass. AI agents looking for "what should I work on next" should start here. +> +> **Status**: Living document. Update when scope shifts, when a known gap is closed, or when a new area enters active hardening. + +--- + +## 1. Current Focus + +Derived from recent commit trajectory ([CHANGELOG.md](CHANGELOG.md), `git log`). + +### 1.1 Security hardening (active) + +Recent waves have systematically eliminated constant-time-comparison gaps, fail-open auth defaults, and CSP holes. Continuing work: + +- Maintain the audit cadence from `docs/md/security-review.md`. Each new auth flow or secret-touching path requires a review entry. +- Keep `JVSPATIAL_DOCS_DISABLED` as the standard production posture and ensure new app routes inherit strict CSP. +- Per-path CSP / docs gating: monitor for new routes added under `/docs` or `/openapi.json` namespaces. + +### 1.2 Performance and bulk APIs (active) + +- `find_many`, `bulk_save`, native `count()` pushdown landed in 0.0.7. Continue: +- Native bulk-update equivalents on backends that support them (Mongo `bulk_write`, SQLite `executemany UPDATE`). +- Continue surfacing slow-query telemetry via the observability layer; tune `slow_query_ms` defaults per backend. + +### 1.3 Observability (active) + +- `MetricsRecorder` protocol and OTEL adapter shipped in 0.0.7. Continue: +- Expand the per-op metric set (cache hits, hook latencies, walker step counts) beyond DB ops. +- Validate OTEL adapter against a real OTLP collector and add an integration test. + +### 1.4 Stability contract enforcement (active) + +- The public / internal / experimental tiers in `docs/md/stability.md` are now defined. Continue: +- Audit every public name in `jvspatial/__init__.py` for tier annotation. +- Wire `@experimental` warnings to any opt-in surface still missing them. + +--- + +## 2. Known Gaps and Tech Debt + +Items called out by SPEC audit (this docs pass) and prior work. Each carries a code-path citation. + +### 2.1 Schema migration story is implicit + +No migration framework. Field removals or renames break existing records silently. Adapters do not enforce schemas. + +- **Decision needed**: ship a thin migration helper or document the per-application pattern. +- **Location**: cross-cutting; would touch `jvspatial/core/entities/object.py` field handling and `jvspatial/db/*` adapters. + +### 2.2 Multi-database transactions are best-effort only + +Only MongoDB supports ACID transactions. Cross-backend operations have no built-in coordination. + +- **Decision needed**: document the limit explicitly in handler patterns or build an opt-in saga primitive. +- **Location**: `jvspatial/db/transaction.py`, `jvspatial/db/database.py:48+`. + +### 2.3 Session state, rate-limit counters, JWT blacklist are per-worker + +In multi-worker deployments, configured limits multiply by worker count. Cross-worker invalidation requires a shared store the library does not ship. + +- **Decision needed**: provide first-party Redis-backed adapters for these stores or remain caller-owned. +- **Location**: `jvspatial/api/auth/enhanced.py` (SessionManager), `jvspatial/api/middleware/rate_limit*.py`. + +### 2.4 `count()` consistency varies by backend + +DynamoDB `count()` is eventually consistent; MongoDB and SQLite are strongly consistent; JsonDB is read-at-call-time. + +- **Decision needed**: document the divergence per adapter, or add a `consistency=` argument. +- **Location**: `jvspatial/db/database.py:144`, per-backend overrides. + +### 2.5 Walker trail does not persist across serverless invocations + +By design (in-memory only). Long-running traversals across cold-starts cannot resume. + +- **Decision needed**: add an optional persistent-trail adapter or accept the limit explicitly. +- **Location**: `jvspatial/core/entities/walker_components/walker_trail.py`. + +### 2.6 Quantitative success criteria are unset + +[PRD §6.2](PRD.md#62-quantitative) lists placeholders for latency, cold-start, and memory budgets. The upcoming foundation audit must fill these in. + +### 2.7 Legacy `LLM-CODING-GUIDE.md` overlap with `docs/md/` + +The legacy file contains code patterns that duplicate parts of `docs/md/quick-start-guide.md`, `entity-reference.md`, and others. Consider deprecating sections that are fully covered elsewhere. + +- **Location**: [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md). + +### 2.8 Python 3.8 declared but not really supported + +`pyproject.toml` classifiers include Python 3.8, but several features (notably `typing.Literal` and `ParamSpec` usage) prefer 3.9+. Either confirm 3.8 support with CI coverage or drop the classifier. + +- **Location**: `pyproject.toml`. + +### 2.9 Experimental status of `JsonDBTransaction(best_effort=True)` unresolved + +[PRD §9](PRD.md#9-open-questions) flags this. Promote, demote, or remove. + +- **Location**: `jvspatial/db/transaction.py`. + +--- + +## 3. Areas Under Active Hardening + +Code paths receiving sustained attention. Changes here warrant elevated review. + +| Area | Why | Recent changes | +|---|---|---| +| `jvspatial/api/auth/*` | Auth touches every request; security regressions are high-blast-radius. | 0.0.7 hardening wave; ongoing per-flow reviews. | +| `jvspatial/storage/security/*` | File-upload trust boundary; path traversal is a known threat surface. | Internal-marker handling, validator extension to markdown files. | +| `jvspatial/db/_atomic.py`, `_path_locks.py` | Crash safety and concurrent-write correctness. | 0.0.7 atomic-write introduction; benchmark coverage added. | +| `jvspatial/runtime/serverless.py` | Detection precedence + mode-dependent defaults; subtle changes have wide effects. | 0.0.7 LWA env defaults, bcrypt rounds adjustment. | +| `jvspatial/db/_observable.py`, `jvspatial/observability/*` | Public log-field schema is part of the stability contract. | 0.0.7 introduction; field-set changes require deprecation. | +| `jvspatial/core/entities/walker.py` and `walker_components/*` | Protection limits prevent DOS; subclass extension surface. | Ongoing; trail-tracking refinements. | + +--- + +## 4. Versioning and Release Policy + +Authoritative process lives in [RELEASING.md](RELEASING.md). Summary: + +- Version is read from `jvspatial/version.py`; the release workflow auto-creates the tag. +- Breaking changes pre-1.0 require a `**BREAKING**` callout in [CHANGELOG.md](CHANGELOG.md) and a one-cycle deprecation where reasonable. +- Deprecation pattern: `@deprecated(replacement=..., remove_in=...)` from `jvspatial/utils/deprecation.py`. Emits a once-per-process `DeprecationWarning`. +- Removal happens at the earliest in the second minor release after deprecation. + +Stability tier reference: [docs/md/stability.md](docs/md/stability.md). + +--- + +## 5. Out-of-Scope (Reaffirmed) + +These items appeared in past planning discussions and remain out of scope. Listed so they are not reopened without a deliberate decision. + +| Item | Rejected because | +|---|---| +| Built-in graph query language (Cypher / Gremlin equivalent) | Walkers are imperative async Python by design (PRD §4). | +| Synchronous API surface | Library is async-only (PRD §5.2). | +| Built-in dashboard / web UI | Visualization helpers (DOT, Mermaid) are sufficient; rendering is downstream. | +| First-party multi-tenancy primitives | Tenant isolation is caller-owned through context scoping. | + +--- + +## 6. How to Use This Roadmap + +- **Adding a new gap**: include the code-path citation and the decision needed. Do not list aspirations without a justification. +- **Closing a gap**: remove the entry from §2 in the same commit that lands the fix. Add a corresponding line to [CHANGELOG.md](CHANGELOG.md). +- **Promoting a focus area**: move from §2 (Gaps) into §1 (Current Focus) when sustained work begins. +- **Reaffirming out-of-scope**: append to §5 with the rejecting rationale. + +The roadmap is for orientation. It is not a backlog and not a contract. Commitments live in CHANGELOG and milestones. diff --git a/docs/md/README.md b/docs/md/README.md index 44088c5..f826efc 100644 --- a/docs/md/README.md +++ b/docs/md/README.md @@ -1,353 +1,145 @@ -# JVspatial Documentation +# jvspatial Documentation Index -**Version**: 0.0.6 -**Last Updated**: 2025-03-16 +This directory holds detailed how-to and reference documentation. For higher-level orientation start at the repo root: -Welcome to the jvspatial documentation! This guide will help you understand and use the jvspatial library effectively. +- [README.md](../../README.md) — project overview and installation +- [PRD.md](../../PRD.md) — *why* the library exists, target users, non-goals +- [SPEC.md](../../SPEC.md) — *what* the library guarantees (technical contract) +- [ROADMAP.md](../../ROADMAP.md) — forward direction and known gaps +- [CLAUDE.md](../../CLAUDE.md) — agent maintenance guide +- [CHANGELOG.md](../../CHANGELOG.md) — release history -> **Developer Quickstart**: For minimal setup (Server with database + auth, import API modules, run), see [Auth Quickstart](auth-quickstart.md) and [Endpoint Registration Guide](endpoint-registration-guide.md). +The documents below answer *how* to use each subsystem. Every link resolves; entries marked **NEW** were added since the previous index revision. --- -## 📚 **Documentation Index** - -### **Getting Started** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Quick Start Guide](quick-start-guide.md) | Get started in 5 minutes | Beginners | -| [Examples](examples.md) | Code examples and tutorials | All levels | -| [Installation Guide](../README.md) | Installation and setup | Beginners | - -### **Core Concepts** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Graph Traversal](graph-traversal.md) | Walker pattern and graph operations | Intermediate | -| [Graph Visualization](graph-visualization.md) | Export graphs in DOT/Mermaid formats | All levels | -| [Entity Reference](entity-reference.md) | Node, Edge, Walker classes | All levels | -| [Context Management](context-management-guide.md) | GraphContext, ServerContext usage | Intermediate | -| [Node Operations](node-operations.md) | Working with nodes | All levels | - -### **API & Server** - -| Document | Description | Audience | -|----------|-------------|----------| -| [REST API](rest-api.md) | API design and endpoints | All levels | -| [API Architecture](api-architecture.md) | Server architecture | Advanced | -| [Server API](server-api.md) | Server configuration | Intermediate | -| [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended entrypoint and auto-registration | Intermediate | -| [Examples](examples.md) | **Standard implementation examples** | ⭐ **Start Here** | -| [Decorator Reference](decorator-reference.md) | All decorators explained | All levels | - -### **Authentication & Security** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Authentication](authentication.md) | Auth system overview | Intermediate | -| [Auth Quickstart](auth-quickstart.md) | Get auth working fast | Beginners | -| [Testing Guide](testing-guide.md) | Test auth mode and isolated databases | Intermediate | -| [Password Migration Guide](password-migration-guide.md) | bcrypt upgrade and transparent migration | Intermediate | - -### **Integrations** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Webhooks Architecture](webhook-architecture.md) | Webhook system design | Advanced | -| [Webhooks Quickstart](webhooks-quickstart.md) | Using webhooks | Intermediate | -| [Scheduler](scheduler.md) | Background job scheduling | Intermediate | -| [File Storage](file-storage-architecture.md) | File storage system | Intermediate | -| [File Storage Usage](file-storage-usage.md) | Using file storage | All levels | - -### **Database & Caching** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Graph Context](graph-context.md) | Database management and multi-database support (JSON, SQLite, MongoDB, DynamoDB) | Intermediate | -| [DynamoDB Guide](dynamodb-guide.md) | DynamoDB setup and configuration | Intermediate | -| [MongoDB Query Interface](mongodb-query-interface.md) | Database queries | Intermediate | -| [Custom Database Guide](custom-database-guide.md) | Implementing custom database backends | Advanced | -| [Caching](caching.md) | Cache strategies | Intermediate | -| [Text Normalization](text-normalization.md) | Unicode to ASCII text normalization | All levels | - -### **Logging** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Logging Service](logging-service.md) | Database logging with automatic persistence | All levels | -| [Custom Log Levels](custom-log-levels.md) | Domain-specific log levels (AUDIT, SECURITY, etc.) | Intermediate | - -### **Advanced Topics** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Architectural Decisions](architectural-decisions.md) | ADRs and design rationale | Advanced | -| [Module Responsibility Matrix](module-responsibility-matrix.md) | Module organization | Advanced | -| [Import Patterns](import-patterns.md) | Best practices for imports | Intermediate | -| [Design Decisions](design-decisions.md) | Design philosophy | Advanced | -| [Optimization](optimization.md) | Performance tuning | Advanced | -| [Error Handling](error-handling.md) | Error patterns | Intermediate | - -### **Development** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Contributing](contributing.md) | Contribution guide | Developers | -| [Custom Database Guide](custom-database-guide.md) | Extending with custom databases | Advanced | -| [Troubleshooting](troubleshooting.md) | Common issues | All levels | -| [Migration Guide](migration.md) | Adopting jvspatial | Users | - -### **Reference** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Attribute Annotations](attribute-annotations.md) | @attribute | All levels | -| [Walker Events](walker-reporting-events.md) | Walker event system | Intermediate | -| [Walker Queue](walker-queue-operations.md) | Queue management | Advanced | -| [Walker Trail](walker-trail-tracking.md) | Trail tracking | Advanced | -| [Pagination](pagination.md) | Paginating results | Intermediate | -| [Environment Config](environment-configuration.md) | Configuration options | All levels | -| [Environment Keys Reference](environment-keys-reference.md) | Canonical valid env key inventory | All levels | -| [Infinite Walk Protection](infinite-walk-protection.md) | Preventing infinite loops | Advanced | - ---- - -## 🎯 **Learning Paths** - -### **Path 1: Beginner → Intermediate** - -1. ✅ [Quick Start Guide](quick-start-guide.md) -2. ✅ [Examples](examples.md) -3. ✅ [Entity Reference](entity-reference.md) -4. ✅ [Graph Traversal](graph-traversal.md) -5. ✅ [REST API](rest-api.md) -6. ✅ [Decorator Reference](decorator-reference.md) - -### **Path 2: API Development** - -1. ✅ [Quick Start Guide](quick-start-guide.md) -2. ✅ [REST API](rest-api.md) -3. ✅ [Decorator Reference](decorator-reference.md) -4. ✅ [Endpoint Registration Guide](endpoint-registration-guide.md) -5. ✅ [Authentication](authentication.md) -6. ✅ [Server API](server-api.md) -7. ✅ [Webhooks Quickstart](webhooks-quickstart.md) - -### **Path 3: Advanced Architecture** - -1. ✅ [Module Responsibility Matrix](module-responsibility-matrix.md) -2. ✅ [Import Patterns](import-patterns.md) -3. ✅ [API Architecture](api-architecture.md) -4. ✅ [Architectural Decisions](architectural-decisions.md) -5. ✅ [Design Decisions](design-decisions.md) -6. ✅ [Optimization](optimization.md) - ---- - -## 🔍 **Quick Reference** - -### **Common Tasks** - -| Task | Document | Section | -|------|----------|---------| -| Create a node | [Quick Start](quick-start-guide.md) | Pattern 1 | -| Define an endpoint | [Quick Start](quick-start-guide.md) | Step 3 | -| Setup endpoint registration | [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended Entrypoint | -| Build a walker | [Graph Traversal](graph-traversal.md) | Basic Walker | -| Visualize graph | [Graph Visualization](graph-visualization.md) | Quick Start | -| Query database | [MongoDB Query](mongodb-query-interface.md) | Query Builder | -| Add authentication | [Auth Quickstart](auth-quickstart.md) | Setup | -| Setup caching | [Caching](caching.md) | Configuration | -| Handle files | [File Storage Usage](file-storage-usage.md) | Basic Usage | -| Schedule jobs | [Scheduler](scheduler.md) | Basic Tasks | -| Setup logging | [Logging Service](logging-service.md) | Quick Start | -| Use custom log levels | [Custom Log Levels](custom-log-levels.md) | Quick Start | - -### **Common Issues** - -| Issue | Document | Solution | -|-------|----------|----------| -| Import errors | [Troubleshooting](troubleshooting.md) | Import Issues | -| Context errors | [Context Management](context-management-guide.md) | Usage Patterns | -| Import patterns | [Import Patterns](import-patterns.md) | Best Practices | -| Authentication fails | [Troubleshooting](troubleshooting.md) | Auth Issues | -| 401 with valid token | [Troubleshooting](troubleshooting.md) | Authentication: 401 with valid token | -| Database path wrong | [Troubleshooting](troubleshooting.md) | Database path wrong | - ---- - -## 📖 **Document Categories** - -### **By Complexity** - -**Beginner** (⭐): -- Quick Start Guide -- Examples -- Entity Reference -- Node Operations -- REST API -- Auth Quickstart -- Webhooks Quickstart -- File Storage Usage - -**Intermediate** (⭐⭐): -- Graph Traversal -- Context Management -- Authentication -- Server API -- Scheduler -- MongoDB Query Interface -- Caching -- Import Patterns -- Error Handling - -**Advanced** (⭐⭐⭐): -- API Architecture -- Module Responsibility Matrix -- Architectural Decisions -- Design Decisions -- Optimization -- Webhook Architecture -- File Storage Architecture -- Walker Queue Operations -- Walker Trail Tracking -- Infinite Walk Protection - -### **By Topic** - -**Core Graph**: -- Graph Traversal -- Graph Visualization -- Entity Reference -- Node Operations -- Walker Events -- Walker Queue Operations -- Walker Trail Tracking - -**API Development**: -- REST API -- API Architecture -- Server API -- Endpoint Registration Guide -- Decorator Reference -- Error Handling - -**Authentication & Security**: -- Authentication -- Auth Quickstart -- Attribute Annotations - -**Integrations**: -- Webhooks Architecture -- Webhooks Quickstart -- Scheduler -- File Storage Architecture -- File Storage Usage - -**Data Management**: -- MongoDB Query Interface -- Caching -- Pagination -- Text Normalization - -**Logging**: -- Logging Service -- Custom Log Levels - -**Architecture**: -- Module Responsibility Matrix -- Import Patterns -- Architectural Decisions -- Design Decisions - ---- - -## 🎓 **Glossary** - -| Term | Definition | -|------|------------| -| **Node** | A data point in the graph | -| **Edge** | A relationship between nodes | -| **Walker** | A pattern for traversing the graph | -| **Context** | Manages database and configuration | -| **Root** | Entry point to the graph | -| **Endpoint** | An API route | -| **Query Builder** | Fluent interface for database queries | -| **Cache Backend** | Storage for cached data | -| **Storage Interface** | File storage abstraction | -| **Decorator** | Function/class modifier | +## Getting Started + +| Document | What's in it | +|---|---| +| [Quick Start Guide](quick-start-guide.md) | Minimal end-to-end setup: install, create a Server, define a Node, expose an endpoint. | +| [Examples](examples.md) | Index of runnable scripts in [examples/](../../examples/). | +| [Auth Quickstart](auth-quickstart.md) | Fast path to authenticated endpoints. | +| [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended entrypoint pattern and auto-registration semantics. | +| [Migration Guide](migration.md) | Adopting jvspatial in an existing project. | + +## Core Concepts + +| Document | What's in it | +|---|---| +| [Entity Reference](entity-reference.md) | `Object`, `Node`, `Edge`, `Walker`, `Root` — fields, lifecycle, persistence shape. | +| [Attribute Annotations](attribute-annotations.md) | `@attribute(protected, transient, private, indexed, …)` semantics. | +| [Graph Context](graph-context.md) | Database + cache + monitor binding; multi-database setup. | +| [Context Management Guide](context-management-guide.md) | When and how to scope `GraphContext` / `ServerContext`. | +| [Graph Traversal](graph-traversal.md) | Walker pattern, queue semantics, visit hooks. | +| [Graph Visualization](graph-visualization.md) | DOT / Mermaid export. | +| [Node Operations](node-operations.md) | Connect / disconnect / neighbor queries. | +| [Walker Events](walker-reporting-events.md) | Walker event bus and reporting. | +| [Walker Queue Operations](walker-queue-operations.md) | Queue manipulation patterns. | +| [Walker Trail Tracking](walker-trail-tracking.md) | Trail capture, metadata, summary. | +| [Infinite Walk Protection](infinite-walk-protection.md) | `max_steps` / `max_visits_per_node` / `max_execution_time`. | + +## API and Server + +| Document | What's in it | +|---|---| +| [REST API](rest-api.md) | Endpoint patterns and conventions. | +| [Server API](server-api.md) | `Server` configuration surface. | +| [API Architecture](api-architecture.md) | Mixin composition, request lifecycle, middleware stack. | +| [Decorator Reference](decorator-reference.md) | Every decorator the library ships: `@endpoint`, `@attribute`, `@on_visit`, etc. | +| [Pagination](pagination.md) | `ObjectPager` usage. | +| [Rate Limiting](rate-limiting.md) | **NEW** Token-bucket rate limit configuration. | +| [Error Handling](error-handling.md) | Exception taxonomy and propagation. | + +## Authentication and Security + +| Document | What's in it | +|---|---| +| [Authentication](authentication.md) | JWT, API key, refresh-token flows. | +| [API Keys](api-keys.md) | **NEW** Key management endpoints and hashing model. | +| [Password Migration Guide](password-migration-guide.md) | bcrypt upgrade path. | +| [Security Review](security-review.md) | **NEW** Audit findings and resolved fixes. | +| [Security Operational Notes](security-operational-notes.md) | **NEW** Runtime security guidance. | +| [Production Deployment](production-deployment.md) | **NEW** Hardening checklist for production. | + +## Database and Storage + +| Document | What's in it | +|---|---| +| [MongoDB Query Interface](mongodb-query-interface.md) | Mongo-style operators across all backends. | +| [Custom Database Guide](custom-database-guide.md) | Implementing and registering a new `Database` adapter. | +| [DynamoDB Guide](dynamodb-guide.md) | DynamoDB-specific setup and limits. | +| [File Storage Architecture](file-storage-architecture.md) | Storage interface, security layer, version model. | +| [File Storage Usage](file-storage-usage.md) | Upload, list, download, versioning. | +| [Caching](caching.md) | Memory / Redis / layered caches; read-through DB wrapper. | +| [Text Normalization](text-normalization.md) | Unicode → ASCII normalization for query stability. | + +## Observability and Operations + +| Document | What's in it | +|---|---| +| [Observability](observability.md) | **NEW** Structured DB op logs, `MetricsRecorder`, slow-query threshold, OTEL adapter. | +| [Logging Service](logging-service.md) | Persistent application logging. | +| [Custom Log Levels](custom-log-levels.md) | Adding domain levels (`AUDIT`, `SECURITY`, …). | +| [Benchmarks](benchmarks.md) | **NEW** Regression-detection bench suite. | +| [Serverless Mode](serverless-mode.md) | **NEW** Detection precedence, mode-dependent defaults, LWA env. | +| [Scheduler](scheduler.md) | In-process scheduler and decorators. | +| [Webhook Architecture](webhook-architecture.md) | Webhook routing, signature verification. | +| [Webhooks Quickstart](webhooks-quickstart.md) | Common webhook patterns. | +| [Troubleshooting](troubleshooting.md) | Symptoms → root causes for common issues. | + +## Architecture and Conventions + +| Document | What's in it | +|---|---| +| [Architectural Decisions](architectural-decisions.md) | ADR-style design rationale. | +| [Design Decisions](design-decisions.md) | Philosophy and tradeoffs. | +| [Module Responsibility Matrix](module-responsibility-matrix.md) | Which package owns which concern. | +| [Import Patterns](import-patterns.md) | Stable vs internal imports. | +| [Stability](stability.md) | **NEW** Public / internal / experimental tiers + deprecation policy. | +| [Optimization](optimization.md) | Performance tuning playbook. | + +## Configuration + +| Document | What's in it | +|---|---| +| [Environment Configuration](environment-configuration.md) | `JVSPATIAL_*` allowlist behavior and merge order. | +| [Environment Keys Reference](environment-keys-reference.md) | Canonical inventory of every valid env key. | + +## Contributing + +| Document | What's in it | +|---|---| +| [Contributing](contributing.md) | Dev loop, conventions, label glossary. | +| [Testing Guide](testing-guide.md) | Async test patterns, fixtures, auth-test isolation. | +| [License](license.md) | MIT license reference. | --- -## 🔧 **API Reference** - -### **Core Modules** - -```python -# Core entities -from jvspatial import Object, Node, Edge, Walker, Root - -# Graph operations -from jvspatial.core import GraphContext, on_visit, on_exit - -# Graph visualization -from jvspatial.core.graph import generate_graph_dot, generate_graph_mermaid, export_graph +## Quick Reference -# API -from jvspatial.api import Server, ServerConfig, endpoint, get_auth_service +### Common Tasks -# Database -from jvspatial.db import create_database, Database +| Task | Start here | +|---|---| +| Create your first node + endpoint | [Quick Start Guide](quick-start-guide.md) | +| Switch from JSON to MongoDB | [Graph Context](graph-context.md) | +| Add authentication | [Auth Quickstart](auth-quickstart.md) | +| Implement a custom database backend | [Custom Database Guide](custom-database-guide.md) | +| Deploy to AWS Lambda | [Serverless Mode](serverless-mode.md), [Production Deployment](production-deployment.md) | +| Build a walker | [Graph Traversal](graph-traversal.md), [Walker Events](walker-reporting-events.md) | +| Surface metrics | [Observability](observability.md) | +| Verify SLA / catch regressions | [Benchmarks](benchmarks.md) | +| Audit security posture | [Security Review](security-review.md), [Security Operational Notes](security-operational-notes.md) | -# Cache -from jvspatial.cache import create_cache +### Authoritative Sources -# Storage -from jvspatial.storage.interfaces import LocalFileInterface +When documents disagree, this is the order of trust: -# Utils -from jvspatial.utils import memoize, retry, NodeId -``` - ---- - -## 📊 **Version History** - -| Version | Date | Changes | -|---------|------|---------| -| **0.0.6** | 2025-03-16 | Auth streamlining: get_auth_service(), bootstrap_admin, on_user_registered/on_admin_bootstrapped/on_enrich_current_user callbacks, built-in /auth/me, user_id guarantee, exempt path auto-discovery, argon2 fix | -| **0.0.5** | 2025-03-15 | JWT auth uses prime DB (fixes 401 with valid token), db_path_resolve, docs updates | -| **0.0.4** | 2025-03-14 | Auto-registration for endpoints, Endpoint Registration Guide | -| **0.0.3** | 2025-02-22 | Documentation updates, DynamoDB support | -| **0.0.2** | 2025-xx-xx | Bug fixes and improvements | -| **0.0.1** | 2025-xx-xx | Initial release | - ---- - -## 🤝 **Contributing** - -Want to improve the documentation? - -1. Read the [Contributing Guide](contributing.md) -2. Check for open documentation issues -3. Submit a pull request - ---- - -## 📧 **Support** - -- **Documentation Issues**: Open a GitHub issue -- **Questions**: GitHub Discussions -- **Email**: support@jvspatial.com - ---- - -## 📝 **License** - -See [License](license.md) for details. - ---- +1. Source code (cited in [SPEC.md](../../SPEC.md)) +2. [SPEC.md](../../SPEC.md) — contract +3. [PRD.md](../../PRD.md) — product context +4. [docs/md/](.) — how-to documentation +5. [LLM-CODING-GUIDE.md](../../LLM-CODING-GUIDE.md) — legacy code-pattern cookbook (no longer authoritative on contracts) -**Last Updated**: 2025-03-16 -**Version**: 0.0.6 -**Maintainer**: JVspatial Team +If you find a doc in this directory that contradicts SPEC, the doc is wrong — open an issue. From b93f16536bc9fde6d6660d1a9b1cfc735305c310 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:18:34 -0400 Subject: [PATCH 06/22] docs: add agent entry points (CLAUDE.md + AGENTS.md) CLAUDE.md is the canonical agent maintenance guide loaded automatically by Claude Code. It captures: - What jvspatial is, in one paragraph - Order of trust for resolving context (PRD -> SPEC -> docs -> code) - Nine non-negotiable invariants with file:line citations - Common gotchas table mapping symptom -> cause -> fix - Dev loop commands - Safe-change playbooks for features, fixes, security paths, serverless paths, and new database backends - Explicit boundaries (sync APIs, CORS, secret comparison, internal imports, etc.) - Repo geography overview AGENTS.md is a thin shim for the multi-agent convention (Codex, Cursor, Aider, Continue, OpenHands). It points at CLAUDE.md as canonical and reserves space only for editor-specific quirks. Co-Authored-By: Claude Opus 4.7 (1M context) --- AGENTS.md | 57 +++++++++++++ CLAUDE.md | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8b5648c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,57 @@ +# AGENTS.md — Multi-agent compatibility shim + +This file exists so AI coding assistants that follow the `AGENTS.md` convention (Codex, Cursor, Aider, Continue, OpenHands, and others) find the same guidance as Claude Code. + +**The canonical agent guide is [CLAUDE.md](CLAUDE.md).** Treat it as authoritative. + +This file does not duplicate that content; agents that read this file should read [CLAUDE.md](CLAUDE.md) instead. + +--- + +## Quick orientation + +If you cannot follow the link to `CLAUDE.md`, here is the minimum: + +- **What this repo is**: `jvspatial`, an async-first Python library for graph-based persistence with FastAPI integration. See [README.md](README.md) and [PRD.md](PRD.md). +- **Where the contract lives**: [SPEC.md](SPEC.md). Every claim cites a `file:line`. If your edit changes a contract, update SPEC in the same commit. +- **How docs are organized**: [docs/md/README.md](docs/md/README.md) is the index. PRD / SPEC / ROADMAP / CLAUDE live at the repo root. +- **Forward direction**: [ROADMAP.md](ROADMAP.md) — current focus areas, known gaps, out-of-scope. +- **What changed recently**: [CHANGELOG.md](CHANGELOG.md). + +--- + +## Non-negotiable invariants (summary) + +Full list in [CLAUDE.md § Non-negotiable invariants](CLAUDE.md#non-negotiable-invariants). Top items: + +1. Async-only I/O. No sync wrappers. +2. `hmac.compare_digest` for every secret comparison. +3. `__entity_name__` honors per-subclass override; do not assemble IDs from `cls.__name__`. +4. Serverless detection precedence: explicit config → current Server config → `SERVERLESS_MODE` env → auto-detect. +5. Stability tiers in [docs/md/stability.md](docs/md/stability.md) are binding. Public names live in `jvspatial.__all__`; underscore modules are internal. +6. CORS/CSP/docs defaults fail closed; do not weaken without a security review. + +--- + +## Run the dev loop + +```bash +pip install -e '.[dev,test]' +pre-commit install +pytest -q +pre-commit run --all-files +``` + +Tests are async (`pytest-asyncio` auto mode). Benchmarks are skipped by default — run with `pytest tests/benchmarks --benchmark-only`. + +--- + +## Editor / agent-tool specific notes + +This section is the only place where agent-tool-specific guidance lives. Keep it short; cross-link to CLAUDE.md for everything else. + +- **Cursor / Continue**: `.cursor/` directory is gitignored except for committed rules. There are no committed `.cursorrules`; this file plus CLAUDE.md is the full agent contract. +- **Aider**: `--read CLAUDE.md` to load the canonical guide on session start. +- **Codex / OpenHands**: this file is loaded as agent instructions; read CLAUDE.md for full details. + +Any agent-specific quirk worth recording goes here, with a one-line rationale. If a quirk grows beyond a line, promote it into CLAUDE.md and reference back. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..f07d75f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,233 @@ +# CLAUDE.md — Agent Guide for jvspatial + +> This file is read automatically by Claude Code at the start of every session in this repo. It is the **canonical entry point** for any AI agent maintaining `jvspatial`. Keep it short, current, and actionable. + +--- + +## What jvspatial is + +`jvspatial` is an **async-first, serverless-compatible, object-spatial Python library** for graph-based persistence and business-logic layers. It layers an entity-centric graph model (Object → Node / Edge / Walker / Root) over four database backends (JSON, SQLite, MongoDB, DynamoDB), and ships a FastAPI integration with auth, file storage, observability, and serverless ergonomics. + +For full positioning and non-goals, read [PRD.md](PRD.md). +For the technical contract, read [SPEC.md](SPEC.md). + +--- + +## Where to look first + +When given a task, resolve context in this order: + +1. **[PRD.md](PRD.md)** — what's the product context? Is this in scope? +2. **[SPEC.md](SPEC.md)** — what does the library currently guarantee about the area you're touching? Every claim cites a `file:line`. +3. **[docs/md/README.md](docs/md/README.md)** — index of how-to docs. Find the relevant one before reading source. +4. **[ROADMAP.md](ROADMAP.md)** — is this area under active hardening? Known gap? Out of scope? +5. **[CHANGELOG.md](CHANGELOG.md)** — recent changes that may affect your work. +6. **Source code** — last, not first. The map of subpackage READMEs (`jvspatial/*/README.md`) shortens this hop. + +If a doc disagrees with code, the doc is wrong. File an issue and trust the code — but cite the discrepancy. + +--- + +## Non-negotiable invariants + +These are properties the library guarantees. Breaking them is a regression even if tests pass. + +### 1. Async-only I/O + +Every database call, network call, and file-system call is `async`. There are **no sync wrappers** in the library. Sync is for pure computation only. + +- ❌ `entity.save()` (missing `await`) +- ✅ `await entity.save()` + +If you find yourself wanting a sync wrapper, you are solving the wrong problem. + +### 2. Constant-time secret comparison + +Any comparison of a secret, key, token, or password hash uses `hmac.compare_digest`. Affected paths (SPEC §15.2): +- API key verification, refresh token comparison, password reset token, webhook HMAC, deferred-invoke secret, bcrypt legacy fallback. + +Never replace `hmac.compare_digest` with `==` for performance or readability. + +### 3. Entity name override (`__entity_name__`) + +Persisted entity discriminator is `cls.__dict__.get("__entity_name__") or cls.__name__` (SPEC §1.2). Resolution is **per-subclass**, not inherited. Code that constructs IDs or looks up subclasses must go through `generate_id()` and `find_subclass_by_name()` — never assemble IDs from `cls.__name__` directly. + +### 4. Serverless detection precedence + +`is_serverless_mode(config=None)` resolves: explicit config → current Server config → `SERVERLESS_MODE` env → auto-detection (SPEC §11.1). Do not bypass with custom env reads. Do not memoize results across tests — call `reset_serverless_mode_cache()` between cases. + +### 5. Protected attribute validation + +`Object.__setattr__` validates field names against the class hierarchy. Setting an undeclared attribute on an `Object` is rejected. Never use `object.__setattr__(self, ...)` to bypass; if you need a new field, declare it. + +### 6. Deferred-save MRO + +`class MyEntity(DeferredSaveMixin, Node)` — mixin must come **before** the base. Wrong order silently disables batching. Tests should assert MRO-sensitive behavior. + +### 7. Stability tier discipline + +Symbols in `jvspatial.__all__` are public — breaking changes require a deprecation cycle (`docs/md/stability.md`). Underscore-prefixed modules are internal — callers should not import directly. Promoting an internal helper to public is a product decision; update PRD §7 and docs/md/stability.md together. + +### 8. CORS / CSP / docs defaults + +CORS does **not** default to wildcard. CSP is strict on app routes, relaxed only on `/docs`, `/redoc`, `/openapi.json`. `JVSPATIAL_DOCS_DISABLED` is the production posture. Do not weaken these defaults without a security review entry. + +### 9. Walker protection + +`max_steps=10000`, `max_visits_per_node=100`, `max_execution_time=300s`, `max_queue_size=1000` are defaults that prevent DOS. Disabling protection is allowed locally; never disable globally or in code that touches user input. + +--- + +## Common gotchas + +| Symptom | Cause | Fix | +|---|---|---| +| `TypeError: object NoneType can't be used in 'await' expression` | Forgot `await` on async DB call | Add `await` | +| Custom mixin doesn't batch saves | `DeferredSaveMixin` placed after base in MRO | Mixin first: `class X(DeferredSaveMixin, Node)` | +| Subclass lookup returns wrong class | Two unrelated classes share `__name__` | Set `__entity_name__: ClassVar[Optional[str]] = "Distinct"` on one | +| Tests pass locally, fail under serverless mode | Forgot to test both modes | Use `reset_serverless_mode_cache()` between tests | +| 401 with valid token | Auth state on wrong database | Auth is **always** on prime DB; do not relocate | +| Slow query never logged | `slow_query_ms` not configured | Set in `create_database(observe=True, slow_query_ms=N)` | +| Walker visits the same node forever | Protection disabled | Re-enable `protection_enabled=True` | +| `JVSPATIAL_FOO` env var ignored | Not in allowlist | Add to `jvspatial/env_adapter.py` allowlist | +| SQLite query falls back to in-memory filter | Operator not yet pushed down by `SQLiteTranslator` | Check `jvspatial/db/_sqlite_translate.py` for supported operators | + +--- + +## Run the dev loop + +```bash +# install +pip install -e '.[dev,test]' +pre-commit install + +# fast feedback +pytest -q # unit + integration (skips benchmarks) +pre-commit run --all-files # lint / format / type-check + +# full quality bar (run before opening PR) +pytest --cov=jvspatial --cov-report=term-missing + +# benchmarks (regression detection) +pytest tests/benchmarks --benchmark-only +``` + +Async tests use `pytest-asyncio` in auto mode (`pyproject.toml`). No manual marker needed for `async def test_*`. + +--- + +## How to make changes safely + +### Add a feature + +1. Read [PRD §8](PRD.md#8-decision-boundaries) — does the change pass the decision rules? +2. Read the relevant SPEC section. Does the change alter the contract? If yes, plan a SPEC update in the same commit. +3. Test-first on the **JSON backend** — fastest iteration, no external dependencies. +4. Check stability tier of the file you're touching ([docs/md/stability.md](docs/md/stability.md)). +5. Update [CHANGELOG.md](CHANGELOG.md) under `## [Unreleased]`. +6. If touching a public name, also update `jvspatial.__all__`. + +### Fix a bug + +1. Reproduce in a failing test. +2. Resolve root cause; do not paper over. +3. If the bug reveals a SPEC inaccuracy, update SPEC. +4. CHANGELOG entry under `### Fixed`. + +### Touch auth, secrets, or security + +1. Read [docs/md/security-review.md](docs/md/security-review.md) and [docs/md/security-operational-notes.md](docs/md/security-operational-notes.md). +2. Preserve constant-time comparisons. +3. Add a corresponding security-review entry for the change. + +### Touch a serverless code path + +1. Check `is_serverless_mode()` precedence (SPEC §11.1) before adding new mode-sensitive defaults. +2. Test both modes; reset the detection cache between tests. +3. If adding a Lambda-specific behavior, document the LWA interaction in [docs/md/serverless-mode.md](docs/md/serverless-mode.md). + +### Add a database backend + +1. Subclass `Database` (`jvspatial/db/database.py:48`). +2. Register via `register_database_type("name", factory_fn)` (`jvspatial/db/factory.py`). +3. Implement bulk overrides (`find_many`, `bulk_save`); the defaults are correct but slow. +4. Set `supports_transactions` capability flag. +5. Add backend-specific tests in `tests/db/`. + +--- + +## Boundaries — what NOT to do + +- Do **not** add a sync version of any I/O API. +- Do **not** broaden CORS defaults to wildcards. +- Do **not** replace `hmac.compare_digest` with `==`. +- Do **not** commit `.env` or any secrets. Use `.env.example` for templates. +- Do **not** import from underscore-prefixed modules (`_atomic`, `_path_locks`, `_cache`, `_observable`) — go through the public factory. +- Do **not** modify SPEC.md without modifying the corresponding code in the same commit (or vice versa). +- Do **not** edit `LLM-CODING-GUIDE.md` — it is preserved as a legacy reference. New documentation belongs in `docs/md/` or in PRD/SPEC/ROADMAP. +- Do **not** add features mentioning specific downstream consumers in the library or its docs. jvspatial is a foundation; consumer-specific logic lives downstream. + +--- + +## Repo geography + +``` +jvspatial/ +├── core/ # Entity hierarchy, GraphContext, events +│ ├── entities/ # Object, Node, Edge, Walker, Root +│ ├── annotations/ # @attribute system +│ └── walker_components/ # Trail, protection, queue, events +├── api/ # FastAPI integration +│ ├── auth/ # JWT, API keys, RBAC, sessions +│ ├── components/ # AppBuilder, AuthConfigurator, middleware +│ ├── decorators/ # @endpoint +│ ├── endpoints/ # Registry, factory, router +│ └── integrations/ # Webhooks, scheduler, storage service +├── db/ # Database backends + abstraction +│ ├── jsondb.py, sqlite.py, mongodb.py, dynamodb.py +│ ├── _atomic.py, _path_locks.py, _cache.py, _observable.py # internal +│ └── query.py, factory.py, manager.py +├── storage/ # File storage +│ ├── interfaces/ # Local, S3 +│ └── security/ # path_sanitizer, validator +├── cache/ # Memory, Redis, layered +├── serverless/ # Deferred task dispatch +├── runtime/ # Serverless detection, LWA helpers +├── observability/ # Metrics, OTEL adapter +├── logging/ # Custom levels, persisted logging +├── exceptions.py # Central exception hierarchy +├── env.py # Env helpers +├── env_adapter.py # JVSPATIAL_* allowlist + merge +└── version.py # Single source of version truth + +tests/ # Mirrors jvspatial/ layout +docs/md/ # How-to and reference docs (see docs/md/README.md) +examples/ # Runnable example scripts +``` + +Each top-level subpackage has its own `README.md` (added in this docs pass) — read that before diving into source. + +--- + +## Slash commands and skills relevant in this repo + +- `pytest` and `pytest --benchmark-only` are the canonical test entry points. +- `pre-commit run --all-files` runs the full linter/formatter suite (black, isort, flake8, mypy, detect-secrets). +- For larger refactors, follow the [docs/md/contributing.md](docs/md/contributing.md) workflow; for security-sensitive changes, the workflow in [docs/md/security-review.md](docs/md/security-review.md). + +--- + +## Pointers to authoritative sources + +When in doubt, the source-of-truth order is: + +1. Code (always) +2. [SPEC.md](SPEC.md) (technical contract; should match code) +3. [PRD.md](PRD.md) (product context) +4. [docs/md/](docs/md/) (how-to) +5. [CHANGELOG.md](CHANGELOG.md) (release notes) +6. [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md) (legacy; informational only, not contractual) + +--- + +**Last updated**: Same commit as the PRD/SPEC/ROADMAP docs pass. Update this file in the same commit as any change that alters an invariant or boundary above. From 9cbad1306f3367a2035cd9540d0ed00f2b938974 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:21:53 -0400 Subject: [PATCH 07/22] docs: add per-subpackage READMEs for in-context discovery Adds README.md to each top-level subpackage that lacked one: core, api, db, storage, cache, serverless, observability, logging, runtime. Each README is navigational, not encyclopedic. Sections cover: purpose, layout, public API (with what each name does), invariants specific to the subpackage, common modification patterns, related docs in docs/md/, and stability tier. When an agent opens a subdirectory, the README is the first hop and tells it what lives here and where to look next. File paths and citations cross-reference SPEC.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/README.md | 83 +++++++++++++++++++++++++++++ jvspatial/cache/README.md | 56 ++++++++++++++++++++ jvspatial/core/README.md | 71 +++++++++++++++++++++++++ jvspatial/db/README.md | 87 +++++++++++++++++++++++++++++++ jvspatial/logging/README.md | 87 +++++++++++++++++++++++++++++++ jvspatial/observability/README.md | 55 +++++++++++++++++++ jvspatial/runtime/README.md | 54 +++++++++++++++++++ jvspatial/serverless/README.md | 59 +++++++++++++++++++++ jvspatial/storage/README.md | 71 +++++++++++++++++++++++++ 9 files changed, 623 insertions(+) create mode 100644 jvspatial/api/README.md create mode 100644 jvspatial/cache/README.md create mode 100644 jvspatial/core/README.md create mode 100644 jvspatial/db/README.md create mode 100644 jvspatial/logging/README.md create mode 100644 jvspatial/observability/README.md create mode 100644 jvspatial/runtime/README.md create mode 100644 jvspatial/serverless/README.md create mode 100644 jvspatial/storage/README.md diff --git a/jvspatial/api/README.md b/jvspatial/api/README.md new file mode 100644 index 0000000..86bd83f --- /dev/null +++ b/jvspatial/api/README.md @@ -0,0 +1,83 @@ +# jvspatial/api + +FastAPI integration: `Server` class, endpoint decorator, auth, middleware, and lifecycle. + +> **Read first**: [SPEC §8-10](../../SPEC.md), [docs/md/api-architecture.md](../../docs/md/api-architecture.md) + +--- + +## Purpose + +`api/` adapts the entity layer to FastAPI. The `Server` class composes app construction, route registration, lifecycle, and uvicorn invocation. `@endpoint` turns functions or Walker classes into HTTP routes with consistent auth, validation, and response shaping. + +## Layout + +``` +api/ +├── server.py # Server (composition of 4 mixins) +├── server_app_factory.py # AppFactoryMixin (internal) +├── server_registration.py # RegistrationMixin (internal) +├── server_lifecycle.py # LifecycleMixin (internal) +├── server_run.py # RunMixin (internal) +├── config.py # ServerConfig (Pydantic) +├── config_groups.py # DatabaseConfig, AuthConfig, etc. +├── context.py # ServerContext + per-request helpers +├── exceptions.py # APIError hierarchy +├── auth/ # JWT, API keys, RBAC, sessions +├── components/ # AppBuilder, AuthConfigurator, middleware +├── decorators/ # @endpoint, deferred registry, endpoint fields +├── endpoints/ # Registry, factory, router, response +├── integrations/ # Webhooks, scheduler, storage service +├── middleware/ # Rate limit, manager +├── services/ # Endpoint discovery +└── utils/ # Misc helpers +``` + +## Public API (from `jvspatial.api`) + +| Name | What it does | +|---|---| +| `Server` | Main server class (SPEC §8.1) | +| `create_server` | Functional constructor | +| `ServerConfig` | Pydantic config (SPEC §10.1) | +| `ServerContext`, `get_current_server`, `set_current_server` | Per-request server binding | +| `get_auth_service` | Singleton accessor for `AuthenticationService` | +| `@endpoint` | Unified function / Walker route decorator (SPEC §8.2) | +| `BaseRouter`, `EndpointRouter` | Mountable routers | +| `endpoint_field`, `EndpointField`, `EndpointFieldInfo` | Endpoint parameter helpers | +| `format_response`, `ResponseHelper` | Response shaping | +| `register_deferred_endpoint`, `flush_deferred_endpoints`, `get_deferred_endpoint_count`, `clear_deferred_endpoints`, `sync_endpoint_modules` | Deferred-registry utilities (SPEC §8.3) | + +## Invariants + +- **Auth state lives on the prime database only.** Users, sessions, API keys, refresh tokens, password-reset tokens — all on prime DB. Not relocatable. (`auth/service.py`) +- **JWT secret is required when auth is enabled.** Server fails fast with a clear error if `JVSPATIAL_JWT_SECRET_KEY` is empty or a placeholder. (`auth/service.py`, [CHANGELOG 0.0.7](../../CHANGELOG.md)) +- **All secret comparisons use `hmac.compare_digest`.** No `==` for tokens, keys, or hashes. +- **CORS defaults are restrictive.** Wildcards must be explicit and trigger a startup warning. (`components/cors_configurator.py`) +- **CSP is strict on app routes, relaxed only on `/docs`, `/redoc`, `/openapi.json`.** (`components/app_builder.py`) +- **`JVSPATIAL_DOCS_DISABLED` removes the entire docs surface.** No spec leak when truthy. (`components/app_builder.py`) +- **Sessions and rate-limit counters are per-process.** Multi-worker deployments multiply configured limits by worker count. +- **Endpoint registration is deferred.** `@endpoint` collects targets at import; `Server` resolves them at app build time. + +## Modification patterns + +- Adding a new endpoint kind: extend `decorators/route.py` and update `endpoints/factory.py`. New decorator forms must be auth/role/webhook-aware. +- Adding new middleware: register via `Server.middleware_manager.add(...)`. Built-ins live in `middleware/` and `components/`. +- Adding a new auth flow: extend `auth/service.py` and `auth/rbac.py`. Add a security-review entry per [docs/md/security-review.md](../../docs/md/security-review.md). +- Adding a new lifecycle hook: declare in `Server` constructor signature, wire through `LifecycleMixin`, document under [docs/md/server-api.md](../../docs/md/server-api.md). + +## Related docs + +- [docs/md/api-architecture.md](../../docs/md/api-architecture.md) +- [docs/md/server-api.md](../../docs/md/server-api.md) +- [docs/md/endpoint-registration-guide.md](../../docs/md/endpoint-registration-guide.md) +- [docs/md/decorator-reference.md](../../docs/md/decorator-reference.md) +- [docs/md/authentication.md](../../docs/md/authentication.md) +- [docs/md/api-keys.md](../../docs/md/api-keys.md) +- [docs/md/auth-quickstart.md](../../docs/md/auth-quickstart.md) +- [docs/md/rate-limiting.md](../../docs/md/rate-limiting.md) +- [docs/md/webhook-architecture.md](../../docs/md/webhook-architecture.md) + +## Stability + +Public names listed above are stable. The `server_*` mixin modules are internal — assemble through `Server` only. `components/`, `middleware/`, `services/`, and `integrations/` internals can change between minor versions; cross them only through the public surface. diff --git a/jvspatial/cache/README.md b/jvspatial/cache/README.md new file mode 100644 index 0000000..98461c4 --- /dev/null +++ b/jvspatial/cache/README.md @@ -0,0 +1,56 @@ +# jvspatial/cache + +Cache backends: memory, Redis, layered. Plus a factory and the abstract base. + +> **Read first**: [SPEC §13](../../SPEC.md), [docs/md/caching.md](../../docs/md/caching.md) + +--- + +## Purpose + +`cache/` provides pluggable cache backends. The most common use is the read-through DB wrapper enabled via `create_database(cache_get_size=N)` — this package exposes the underlying backends and a factory so callers can build their own cache strategies. + +## Layout + +``` +cache/ +├── base.py # CacheBackend ABC +├── factory.py # create_cache, create_default_cache +├── memory.py # MemoryCache (LRU) +├── redis.py # RedisCache (optional) +└── layered.py # LayeredCache (memory L1 + redis L2) +``` + +## Public API (from `jvspatial.cache`) + +| Name | What it does | +|---|---| +| `CacheBackend` | ABC for cache backends | +| `create_cache(backend, **kwargs)` | Factory entry point | +| `create_default_cache()` | Env-driven default | +| `MemoryCache` | Per-process LRU | +| `RedisCache` | Shared cross-process cache (requires `redis-py`) | +| `LayeredCache` | Memory L1 + Redis L2, promotes hot keys to L1 | + +## Invariants + +- **Memory cache is per-process.** Cross-worker invalidation requires Redis or layered backend. +- **TTL is per-cache-instance, default `None`.** Cache until eviction. +- **DB-wrapped caches invalidate on `save` / `delete`.** No stale read after a write through the wrapped database. (`jvspatial/db/_cache.py`, internal) +- **`bulk_save` and `find_one_and_update` refresh the cache** for affected keys; other writes invalidate. +- **Cache is skipped in serverless mode** when wrapped via `create_database(cache_get_size=N)`. Cold-start memory does not survive invocations anyway. + +## Modification patterns + +- **Adding a backend**: implement `CacheBackend`. Register in `create_cache` factory. Decide eviction semantics up front. +- **Changing invalidation rules**: edit `jvspatial/db/_cache.py` (internal). Add tests that cover save / delete / bulk_save / find_one_and_update. +- **Tuning the read-through wrapper**: pass `cache_get_size=`, `cache_get_ttl=` to `create_database`. Do not import `CachingDatabase` directly. + +## Related docs + +- [docs/md/caching.md](../../docs/md/caching.md) +- [docs/md/optimization.md](../../docs/md/optimization.md) + +## Stability + +`CacheBackend`, `create_cache`, `MemoryCache` are stable. `RedisCache` and `LayeredCache` are stable when their optional dependency is installed. Internal wrappers (`jvspatial/db/_cache.py`) are not part of the public surface. diff --git a/jvspatial/core/README.md b/jvspatial/core/README.md new file mode 100644 index 0000000..b7818bf --- /dev/null +++ b/jvspatial/core/README.md @@ -0,0 +1,71 @@ +# jvspatial/core + +Entity hierarchy, graph context, traversal primitives, and decorators. + +> **Read first**: [SPEC §1-7](../../SPEC.md), [docs/md/entity-reference.md](../../docs/md/entity-reference.md) + +--- + +## Purpose + +`core/` defines the graph data model. Everything jvspatial persists is one of the entity types here: `Object`, `Node`, `Edge`, `Walker`, or `Root`. Traversal is performed by walkers; database I/O is brokered through `GraphContext`. + +## Layout + +``` +core/ +├── entities/ # Object, Node, Edge, Walker, Root + walker_components +├── annotations/ # @attribute system + index helpers +├── decorators/ # @on_visit, @on_exit +├── mixins/ # DeferredSaveMixin and globals +├── walker_components/ # Trail, protection, queue, event system (under entities/) +├── context.py # GraphContext + scoping helpers +├── events.py # Global event bus + @on_emit +├── graph.py # DOT / Mermaid export +├── graph_expansion.py # BFS / subgraph utilities +├── pager.py # ObjectPager +└── utils.py # generate_id, find_subclass_by_name, datetime helpers +``` + +## Public API (from `jvspatial.core`) + +| Name | What it does | +|---|---| +| `Object`, `Node`, `Edge`, `Walker`, `Root` | Entity classes (SPEC §2) | +| `NodeQuery` | Typed node-query helper | +| `GraphContext` | Database + cache + monitor binding (SPEC §7) | +| `get_default_context` / `set_default_context` / `scoped_default_context` | Context lifecycle | +| `graph_context` / `async_graph_context` | Sync / async context managers | +| `@on_visit`, `@on_exit`, `@on_emit` | Hook decorators | +| `DeferredSaveMixin`, `deferred_saves_globally_allowed`, `flush_deferred_entities` | Save-batching opt-in | +| `ObjectPager`, `paginate_objects`, `paginate_by_field` | Pagination | +| `generate_id`, `find_subclass_by_name`, `serialize_datetime` | Utilities | +| `export_graph`, `generate_graph_dot`, `generate_graph_mermaid`, `expand_node`, `subgraph_bfs` | Visualization / expansion | + +## Invariants + +- **`__entity_name__` is per-subclass.** Resolution: `cls.__dict__.get("__entity_name__") or cls.__name__`. Not inherited. (`entities/object.py:35-44`) +- **`id` is protected.** Set in `__init__`, cannot be reassigned. (`entities/object.py:46-48`) +- **Walker protection is on by default.** `max_steps=10000`, `max_visits_per_node=100`, `max_execution_time=300s`, `max_queue_size=1000`. Disabling globally is forbidden. (`entities/walker.py:106-115`) +- **Subclass lookup honors entity-name override and caches positive hits only.** Negative caching would break later imports. (`utils.py:58-89`) +- **Root is a singleton with fixed ID `n.Root.root`.** Created under async lock. (`entities/root.py`) +- **MRO matters for mixins.** `DeferredSaveMixin` must precede the base class. (`mixins/`) + +## Modification patterns + +- Adding a new entity field: declare with `@attribute(...)`. Top-level persisted fields are added via `_get_top_level_fields()`. Update tests under `tests/core/`. +- Adding a new walker hook: decorate with `@on_visit(NodeType | "string_name")`. Hooks are registered at `__init_subclass__` time. +- Adding a new graph utility: prefer `core/graph.py` (export) or `core/graph_expansion.py` (traversal). Both have sibling tests. + +## Related docs + +- [docs/md/entity-reference.md](../../docs/md/entity-reference.md) +- [docs/md/graph-traversal.md](../../docs/md/graph-traversal.md) +- [docs/md/graph-context.md](../../docs/md/graph-context.md) +- [docs/md/walker-trail-tracking.md](../../docs/md/walker-trail-tracking.md) +- [docs/md/infinite-walk-protection.md](../../docs/md/infinite-walk-protection.md) +- [docs/md/attribute-annotations.md](../../docs/md/attribute-annotations.md) + +## Stability + +All names listed in "Public API" are part of `jvspatial.__all__` and follow the stable-tier contract (see [docs/md/stability.md](../../docs/md/stability.md)). The contents of `walker_components/`, `mixins/_internal/`, and any underscore-prefixed module are internal — do not import directly. diff --git a/jvspatial/db/README.md b/jvspatial/db/README.md new file mode 100644 index 0000000..5fa4ee8 --- /dev/null +++ b/jvspatial/db/README.md @@ -0,0 +1,87 @@ +# jvspatial/db + +Database abstraction and backends — JSON, SQLite, MongoDB, DynamoDB. Plus query engine, transactions, atomic IO, path locks, cache wrapper, and observability wrapper. + +> **Read first**: [SPEC §4-5](../../SPEC.md), [docs/md/mongodb-query-interface.md](../../docs/md/mongodb-query-interface.md) + +--- + +## Purpose + +`db/` defines the `Database` ABC and ships four production backends, all sharing a Mongo-style query DSL. Internal wrappers (`_cache`, `_observable`) add cross-cutting concerns to any backend through factory flags. + +## Layout + +``` +db/ +├── database.py # Database ABC + finalize_find_results +├── factory.py # create_database, register_database_type, switch_database +├── manager.py # DatabaseManager + prime DB convention +├── query.py # QueryEngine + Mongo-style operators +├── transaction.py # Transaction context + JsonDBTransaction +├── jsondb.py # JSON file-per-record backend +├── sqlite.py # aiosqlite backend + Mongo→SQL translator +├── mongodb.py # motor backend +├── dynamodb.py # aioboto3 backend +├── _atomic.py # internal: crash-safe write helper +├── _path_locks.py # internal: bounded-LRU per-path locks +├── _cache.py # internal: read-through cache wrapper +├── _observable.py # internal: structured log + metrics wrapper +└── _sqlite_translate.py # internal: Mongo → SQL translator +``` + +## Backend matrix + +| Backend | Transactions | Bulk APIs | Native count | Notes | +|---|---|---|---|---| +| JSON | No (best-effort opt-in only) | Parallel reads/writes | Dirent fast path | Atomic writes, per-file locks | +| SQLite | No (single-conn fsync) | `executemany` + `IN` | Mongo→SQL pushdown | Translator covers `$eq/$ne/$gt/$gte/$lt/$lte/$in/$nin/$exists`, AND, `$and/$or` | +| MongoDB | **Yes** (replica set required) | `bulk_write`, `$in` | `count_documents` / `estimated_document_count` | Native compound ops; shared retry helper | +| DynamoDB | No | `BatchGetItem`/`BatchWriteItem` (100/batch) | `Select="COUNT"` | Throttle retry with backoff | + +## Public API (from `jvspatial.db`) + +| Name | What it does | +|---|---| +| `Database` | ABC every adapter implements (SPEC §4.1) | +| `create_database(type, ...)` | Factory entry point | +| `create_default_database()` | Environment-driven default | +| `register_database_type(name, factory)` | Register a custom adapter | +| `unregister_database`, `unregister_database_type` | Cleanup | +| `list_database_types()` | Inventory | +| `get_prime_database`, `get_current_database`, `switch_database` | Multi-DB lookup | +| `DatabaseManager`, `get_database_manager`, `set_database_manager` | Manager surface | +| `DatabaseError`, `VersionConflictError` | DB-specific exceptions | +| `JsonDB`, `SQLiteDB`, `MongoDB`, `DynamoDB` | Concrete adapters (optional extras for non-JSON) | + +## Invariants + +- **`Database.supports_transactions` is a capability flag.** Branch on it; do not sniff adapter class. (`database.py:84`) +- **`find_many` and `bulk_save` are public and benefit from native overrides.** Defaults exist but are slow. (`database.py:176+`) +- **`find_one_and_update` / `find_one_and_delete` are NOT atomic by default.** Only MongoDB overrides with native atomic versions. +- **Atomic JSON writes use `temp + fsync + rename + fsync(dir)`.** No partial records survive a crash. (`_atomic.py`) +- **Per-file locks serialize concurrent writes to the same record only.** Different files run in parallel. (`_path_locks.py`) +- **`QueryEngine` LRU is bounded.** Default 1024; configurable. Unbounded query construction will not leak memory. (`query.py`) +- **Prime database is unique.** Auth state, sessions, API keys live there. Cannot be switched. + +## Modification patterns + +- **Adding a custom backend**: subclass `Database`, implement abstract methods, override `find_many` / `bulk_save` if you can, set `supports_transactions`, register via `register_database_type`. Add tests under `tests/db/`. +- **Adding a query operator**: extend `QueryEngine.evaluate_*`. If pushdown is feasible, update `SQLiteTranslator` and the DynamoDB query path. +- **Adding a backend capability flag**: declare as class attribute on `Database`, default to `False` (or the safe value). Document in SPEC §4.2. +- **Touching internal wrappers** (`_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`): they are not part of the public API but **the observability log-field schema IS public** (see [docs/md/stability.md](../../docs/md/stability.md)). + +## Related docs + +- [docs/md/mongodb-query-interface.md](../../docs/md/mongodb-query-interface.md) +- [docs/md/custom-database-guide.md](../../docs/md/custom-database-guide.md) +- [docs/md/dynamodb-guide.md](../../docs/md/dynamodb-guide.md) +- [docs/md/graph-context.md](../../docs/md/graph-context.md) +- [docs/md/caching.md](../../docs/md/caching.md) +- [docs/md/observability.md](../../docs/md/observability.md) +- [docs/md/optimization.md](../../docs/md/optimization.md) +- [docs/md/benchmarks.md](../../docs/md/benchmarks.md) + +## Stability + +Public surface (above) is stable. Underscore-prefixed modules are internal. The structured log field set from `ObservableDatabase` is part of the public contract — schema changes require a deprecation cycle. diff --git a/jvspatial/logging/README.md b/jvspatial/logging/README.md new file mode 100644 index 0000000..1bbf7af --- /dev/null +++ b/jvspatial/logging/README.md @@ -0,0 +1,87 @@ +# jvspatial/logging + +Structured console logging, persistent database logging, custom log levels. + +> **Read first**: [docs/md/logging-service.md](../../docs/md/logging-service.md), [docs/md/custom-log-levels.md](../../docs/md/custom-log-levels.md) + +--- + +## Purpose + +`logging/` covers two surfaces: + +1. **Structured console logging** — `JVSpatialLogger`, optionally backed by `structlog`. Falls back to stdlib `logging` if `structlog` is not installed. +2. **Database logging service** — `BaseLoggingService` persists log records to a backing database via `DBLogHandler`. Useful for audit trails and post-hoc query. + +Custom log levels (`AUDIT`, `SECURITY`, etc.) are supported and integrate with both surfaces. + +## Layout + +``` +logging/ +├── __init__.py # Public surface (this file documents) +├── config.py # Initialization helpers +├── service.py # BaseLoggingService +├── handler.py # DBLogHandler, install_db_log_handler, exception hook +├── models.py # DBLog entity +├── custom_levels.py # add_custom_log_level, CUSTOM_LEVEL_NUMBER +└── endpoints.py # /logs API endpoints (admin) +``` + +## Public API (from `jvspatial.logging`) + +### Database logging + +| Name | What it does | +|---|---| +| `BaseLoggingService` | Service that persists log records | +| `get_logging_service()` | Singleton accessor | +| `DBLog` | Pydantic model for persisted records | +| `get_logging_config`, `initialize_logging_database` | Initialization helpers | +| `DBLogHandler`, `install_db_log_handler`, `install_exception_hook` | Wiring | + +### Custom log levels + +| Name | What it does | +|---|---| +| `add_custom_log_level(name, level_number)` | Register a custom level | +| `get_custom_levels()` | Inventory | +| `is_custom_level(name)` | Predicate | +| `CUSTOM_LEVEL_NUMBER` | Default integer for ad-hoc custom level | + +### Structured console logging + +| Name | What it does | +|---|---| +| `JVSpatialLogger` | Logger with optional structlog backing | +| `StructuredLoggingConfig` | JSON / coloring configuration | +| `PerformanceLogger` | Convenience for op duration logging | +| `SecurityLogger` | Convenience for auth / rate-limit / brute-force events | +| `get_logger(name)` | Factory | +| `configure_logging(...)` | Configure structlog (JSON / colors) | +| `configure_standard_logging(...)` | Configure stdlib handler (level / colors / preserved handlers) | +| `performance_logger`, `security_logger` | Module-global instances | + +## Invariants + +- **Custom levels are global state.** Registering twice is idempotent; registering with a conflicting number is rejected. +- **`SecurityLogger` events feed downstream auth analysis.** Auth attempts, rate-limit hits, and brute-force detections must call into `SecurityLogger` or the equivalent, not silently log elsewhere. +- **`DBLogHandler` persists asynchronously.** Synchronous failures (DB down) must not block the request handler — they downgrade to local logging. +- **`structlog` is optional.** Code must not assume it; use the fallback path when `STRUCTLOG_AVAILABLE` is `False`. +- **Secrets must never reach the log pipeline.** Sanitize before logging — see the 500-response sanitizer pattern. + +## Modification patterns + +- **Adding a custom log level**: call `add_custom_log_level("FOO", 25)` once at startup. Reuse by `logger.log(25, ...)` or extend `JVSpatialLogger`. +- **Persisting a new event type**: extend `DBLog` model or add a sibling model. Update the handler wiring in `handler.py`. +- **Adding a security-relevant log call**: prefer `SecurityLogger` over ad-hoc info logging. Helps downstream audit tooling find events consistently. + +## Related docs + +- [docs/md/logging-service.md](../../docs/md/logging-service.md) +- [docs/md/custom-log-levels.md](../../docs/md/custom-log-levels.md) +- [docs/md/security-operational-notes.md](../../docs/md/security-operational-notes.md) + +## Stability + +All names in the public API tables above are stable. `endpoints.py` (admin `/logs` routes) is wired via the API layer and follows the same stability tier as the rest of `jvspatial/api`. diff --git a/jvspatial/observability/README.md b/jvspatial/observability/README.md new file mode 100644 index 0000000..63b0664 --- /dev/null +++ b/jvspatial/observability/README.md @@ -0,0 +1,55 @@ +# jvspatial/observability + +Metrics protocol, default no-op recorder, OpenTelemetry adapter. + +> **Read first**: [SPEC §14](../../SPEC.md), [docs/md/observability.md](../../docs/md/observability.md) + +--- + +## Purpose + +`observability/` defines a tiny protocol (`MetricsRecorder`) that any metrics backend can implement. The library wires `ObservableDatabase` (internal, in `jvspatial/db/_observable.py`) to emit a structured log line and one metric per DB operation, with WARNING-level elevation past a configurable slow-query threshold. + +## Layout + +``` +observability/ +├── metrics.py # MetricsRecorder Protocol + NullMetricsRecorder +└── otel.py # OpenTelemetryMetricsRecorder (optional, [otel] extra) +``` + +## Public API (from `jvspatial.observability`) + +| Name | What it does | +|---|---| +| `MetricsRecorder` | Protocol implemented by metrics backends | +| `NullMetricsRecorder` | Default no-op | + +From `jvspatial.observability.otel` (requires `pip install jvspatial[otel]`): + +| Name | What it does | +|---|---| +| `OpenTelemetryMetricsRecorder` | OTEL implementation of `MetricsRecorder` | + +## Invariants + +- **The structured log field set from `ObservableDatabase` is public.** Schema: `backend`, `op`, `collection`, `duration_ms`, `success`, `result_count`. Breaking changes require a deprecation cycle. (See [docs/md/stability.md](../../docs/md/stability.md).) +- **`NullMetricsRecorder` is zero-overhead.** Calls dispatch to no-op methods; do not add allocations or branching here. +- **Slow-query threshold elevates the log to WARNING.** Configurable per database via `create_database(..., slow_query_ms=N)`. +- **Four metric names are emitted per DB op**: `jvspatial.db.op.duration_seconds`, `jvspatial.db.op.count`, `jvspatial.db.op.slow_count`, `jvspatial.db.op.result_count`. + +## Modification patterns + +- **Adding a metrics backend**: implement the `MetricsRecorder` protocol. No subclassing required; structural typing applies. Pass via `create_database(..., metrics=YourRecorder())`. +- **Adding a new metric**: extend the protocol with a new method, default-implement on `NullMetricsRecorder` so existing recorders do not break. Document in `docs/md/observability.md` and add to the stability tier table. +- **Changing emitted log fields**: requires a deprecation cycle. Add the new field alongside the old, deprecate the old, remove after one minor cycle. + +## Related docs + +- [docs/md/observability.md](../../docs/md/observability.md) +- [docs/md/benchmarks.md](../../docs/md/benchmarks.md) +- [docs/md/stability.md](../../docs/md/stability.md) + +## Stability + +`MetricsRecorder`, `NullMetricsRecorder`, and the `ObservableDatabase` log field set are all stable. `OpenTelemetryMetricsRecorder` is stable when the `[otel]` extra is installed. diff --git a/jvspatial/runtime/README.md b/jvspatial/runtime/README.md new file mode 100644 index 0000000..0846bff --- /dev/null +++ b/jvspatial/runtime/README.md @@ -0,0 +1,54 @@ +# jvspatial/runtime + +Runtime capability detection — primarily serverless mode and Lambda Web Adapter glue. + +> **Read first**: [SPEC §11](../../SPEC.md), [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) + +--- + +## Purpose + +`runtime/` answers questions about the deployment environment: "are we in a serverless runtime?", "which provider?", "are we behind Lambda Web Adapter?". Detection results inform mode-dependent defaults (deferred saves, index creation, bcrypt cost) without leaking environment-checking code throughout the library. + +## Layout + +``` +runtime/ +├── serverless.py # is_serverless_mode, detect_serverless_provider, cache reset +├── lwa.py # internal: Lambda Web Adapter glue +└── eventbridge.py # internal: EventBridge wiring helpers +``` + +## Public API (from `jvspatial.runtime`) + +| Name | What it does | +|---|---| +| `is_serverless_mode(config=None)` | Effective mode using precedence (SPEC §11.1) | +| `reset_serverless_mode_cache()` | Clear memoization (tests only) | + +From `jvspatial.runtime.serverless` (re-exported through `jvspatial`): + +| Name | What it does | +|---|---| +| `detect_serverless_provider()` | Returns one of `"aws"`, `"azure"`, `"gcp"`, `"vercel"`, `"unknown"` | + +## Invariants + +- **Detection precedence is fixed**: explicit `config.serverless_mode` → `get_current_server().config.serverless_mode` → `SERVERLESS_MODE` env → auto-detect from platform env vars. Do not bypass with custom env reads. +- **Auto-detection is memoized via `lru_cache`.** Tests must call `reset_serverless_mode_cache()` between cases that toggle env. +- **Provider detection is best-effort.** Falls back to `"unknown"`; do not rely on provider-specific behavior outside of well-known providers. +- **`SERVERLESS_MODE` env values**: `true`, `1`, `yes`, `enabled` (case-insensitive) → True. Everything else → False. + +## Modification patterns + +- **Adding a new serverless provider**: extend `_detect_serverless_mode` and `_detect_serverless_provider` with the relevant env var checks. Add a `ServerlessProvider` literal entry. Update tests in `tests/runtime/`. +- **Adding a new mode-dependent default**: read `is_serverless_mode()` at the point of decision, not at import time. Make the default explicit in the relevant SPEC table (§11.2). + +## Related docs + +- [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) +- [docs/md/production-deployment.md](../../docs/md/production-deployment.md) + +## Stability + +`is_serverless_mode`, `reset_serverless_mode_cache`, and `detect_serverless_provider` are stable. `lwa.py` and `eventbridge.py` are internal — go through the public serverless helpers in `jvspatial.serverless`. diff --git a/jvspatial/serverless/README.md b/jvspatial/serverless/README.md new file mode 100644 index 0000000..c1a847e --- /dev/null +++ b/jvspatial/serverless/README.md @@ -0,0 +1,59 @@ +# jvspatial/serverless + +Deferred task dispatch for serverless environments. Sister to `jvspatial/runtime` (detection). + +> **Read first**: [SPEC §11](../../SPEC.md), [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) + +--- + +## Purpose + +`serverless/` provides a dispatcher for tasks that must run outside the current HTTP invocation: long-running work, retryable steps, fan-out workflows. On AWS this typically routes through Lambda async-invoke or EventBridge; the API is provider-agnostic. + +## Layout + +``` +serverless/ +├── factory.py # dispatch_deferred_task, get_task_scheduler +├── deferred_invoke.py # register_deferred_invoke_handler, dispatch_deferred_invoke, normalize_deferred_envelope +└── tasks/ # Provider-specific scheduler implementations + └── __init__.py +``` + +## Public API (from `jvspatial.serverless`) + +| Name | What it does | +|---|---| +| `dispatch_deferred_task(task_type, payload)` | Enqueue a task for out-of-band execution | +| `get_task_scheduler()` | Return the configured scheduler instance | + +Plus from `jvspatial.serverless.deferred_invoke`: + +| Name | What it does | +|---|---| +| `register_deferred_invoke_handler(task_type, handler)` | Register an async handler for a task type | +| `dispatch_deferred_invoke(task_type, payload)` | Dispatch and immediately invoke registered handler (in-process testing path) | +| `normalize_deferred_envelope(event)` | Flatten provider-specific event envelopes (SQS, EventBridge) | + +## Invariants + +- **Handlers must be idempotent.** The framework provides no exactly-once guarantee. Retries may deliver the same payload multiple times. +- **Handler registration is process-global.** Late registration after the first dispatch is allowed but discouraged. +- **Envelope normalization is provider-aware.** New providers should extend `normalize_deferred_envelope`, not bypass it. +- **Secrets in deferred-invoke headers use constant-time comparison.** (See `deferred_invoke.py`.) +- **Lambda Web Adapter env defaults are set best-effort by `Server`** when LWA is detected. IaC should set them explicitly. (SPEC §11.4) + +## Modification patterns + +- **Adding a provider**: implement a scheduler in `tasks/`, register it via configuration. Extend `normalize_deferred_envelope` if the new provider has a distinct event envelope. +- **Adding a handler**: use `@deferred_invoke_handler("task.name")` or `register_deferred_invoke_handler(...)`. Document idempotency requirements at the call site. +- **Adding a new envelope flattener**: keep flattening in `normalize_deferred_envelope` rather than per-handler. + +## Related docs + +- [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) +- [docs/md/production-deployment.md](../../docs/md/production-deployment.md) + +## Stability + +`dispatch_deferred_task`, `get_task_scheduler`, `register_deferred_invoke_handler`, `dispatch_deferred_invoke`, `normalize_deferred_envelope` are stable. `tasks/` internals can change. LWA-specific env wiring in `jvspatial/runtime/lwa.py` is internal. diff --git a/jvspatial/storage/README.md b/jvspatial/storage/README.md new file mode 100644 index 0000000..0cbed4a --- /dev/null +++ b/jvspatial/storage/README.md @@ -0,0 +1,71 @@ +# jvspatial/storage + +File storage system: interfaces, security layer, version model, URL proxy manager. + +> **Read first**: [SPEC §12](../../SPEC.md), [docs/md/file-storage-architecture.md](../../docs/md/file-storage-architecture.md) + +--- + +## Purpose + +`storage/` provides a secure abstraction over file backends. Every upload passes a five-stage path sanitizer and a content-based MIME validator before reaching the backend. Local-filesystem and S3 backends ship today; the interface is open for additional providers. + +## Layout + +``` +storage/ +├── interfaces/ +│ ├── base.py # FileStorageInterface ABC +│ ├── local.py # LocalFileInterface +│ └── s3.py # S3FileInterface (optional extra) +├── security/ +│ ├── path_sanitizer.py # Five-stage validation +│ └── validator.py # Content-based MIME validation +├── managers/ +│ ├── proxy.py # URL proxy lifecycle +│ └── *.py # Other management helpers +├── models.py # Pydantic file/version/proxy models +└── exceptions.py # Storage-specific exceptions +``` + +## Public API (from `jvspatial.storage`) + +| Name | What it does | +|---|---| +| `create_storage(provider, **kwargs)` | Factory entry point (provider ∈ `local`, `s3`) | +| `create_default_storage()` | Env-driven default (`JVSPATIAL_FILE_INTERFACE`) | +| `FileStorageInterface` | ABC for storage backends | +| `LocalFileInterface` | Filesystem backend | +| `S3FileInterface` | S3 backend (requires boto3) | +| `PathSanitizer` | Path validation | +| `FileValidator` | MIME / size validation | +| `URLProxy`, `URLProxyManager`, `get_proxy_manager` | Time-limited / one-time download URLs | +| `StorageError`, `PathTraversalError`, `InvalidPathError`, `ValidationError`, `FileNotFoundError`, `FileSizeLimitError`, `InvalidMimeTypeError`, `StorageProviderError`, `AccessDeniedError` | Exception types | + +## Invariants + +- **Every upload passes `PathSanitizer` then `FileValidator`.** No bypass for "trusted" callers. (`security/path_sanitizer.py`, `security/validator.py`) +- **MIME validation is content-based, not extension-based.** Uses `python-magic`. Renaming `.exe` → `.txt` does not bypass blocking. +- **Path sanitization is five stages**: regex blocklist → normalization with re-check → hidden-file allowlist → symlink resolution → base-dir confinement. +- **Internal directory markers bypass user-input checks via metadata only.** Not via filenames. (`storage/security/path_sanitizer.py`) +- **Atomic writes for local backend.** Same `temp + fsync + rename + fsync(dir)` helper as JsonDB. +- **S3 multipart at ≥8 MiB.** Configurable via constructor or `JVSPATIAL_S3_MULTIPART_THRESHOLD`. +- **S3 throttle retry** uses the shared retry helper with exponential backoff + jitter. + +## Modification patterns + +- **Adding a backend**: implement `FileStorageInterface`. Wire into `create_storage(provider="...")` and the storage manager. Add tests under `tests/storage/`. +- **Changing path sanitization**: review all five stages; add tests to `tests/storage/test_path_sanitizer.py`. Each stage has its own test surface. +- **Extending allowed MIME types**: update `FileValidator.ALLOWED_MIME_TYPES` and add an integration test that round-trips the new type. +- **Adding a security rule**: prefer adding to the sanitizer (paths) or validator (content) rather than ad-hoc in the backend. + +## Related docs + +- [docs/md/file-storage-architecture.md](../../docs/md/file-storage-architecture.md) +- [docs/md/file-storage-usage.md](../../docs/md/file-storage-usage.md) +- [docs/md/security-review.md](../../docs/md/security-review.md) +- [docs/md/security-operational-notes.md](../../docs/md/security-operational-notes.md) + +## Stability + +`create_storage`, `FileStorageInterface`, `LocalFileInterface`, `S3FileInterface`, the security classes, and the exception hierarchy are public. `managers/` internals can change between minor versions; cross them through `get_proxy_manager` and the documented API. From 8b0b12014f33ec12edcb9edbf427be8107667d31 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 08:54:15 -0400 Subject: [PATCH 08/22] docs: add foundation audit (150 findings across 5 dimensions) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First retrospective audit of jvspatial against the contracts now codified in SPEC/PRD/CLAUDE. Five parallel reviewers covered: async correctness, security boundaries, database adapter parity, walker/identity invariants, and serverless/config/stability. Headline findings (16 CRIT, ~42 HIGH, ~58 MED, ~34 LOW): 1. __entity_name__ override is silently rewritten by GraphContext on every save -- the 0.0.8 headline feature is data-corrupting for classes that use it (5 CRIT sites in walker.py + context.py). 2. Walker protection partially fictional: max_trail_length unwired, queue insert paths bypass max_size, resume() resets timer + counters, ProtectionViolation swallowed into report rather than raised as InfiniteLoopError/WalkerTimeoutError. 3. Five async-contract CRITs: async def __init__ on webhook walker, unawaited delete_file in storage service, two Path.write_text inside async graph exporters, unawaited webhook endpoint_func. 4. LocalFileInterface versioning methods (create_version, etc.) bypass PathSanitizer -- user-controlled file_path escapes the storage root. 5. Webhook HMAC verifier compares signature against a 7-char-too- short slice -- always returns False. Webhook signature auth is currently broken-by-default. 6. SHA-256 fallback in refresh-token/password-reset compares with `==` not hmac.compare_digest (CLAUDE.md non-negotiable). 7. DynamoDB throttle retry covers only save/get/delete; find/count/ batch ops surface ProvisionedThroughputExceededException directly to callers despite the documented contract. 8. SPEC §10.2 promises env allowlist enforcement; env_adapter only reads enumerated keys, never scans for stray JVSPATIAL_* keys to reject typos. Plus three divergent parse_bool implementations across env.py / env_adapter.py / runtime/serverless.py producing different truth values for the same input. The audit is non-destructive: zero code changes. Findings are the backlog for the next milestone. AUDIT.md includes a four-wave remediation sequence ordered by blast radius x cost. Co-Authored-By: Claude Opus 4.7 (1M context) --- AUDIT.md | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 AUDIT.md diff --git a/AUDIT.md b/AUDIT.md new file mode 100644 index 0000000..5b79a98 --- /dev/null +++ b/AUDIT.md @@ -0,0 +1,279 @@ +# jvspatial — Foundation Audit + +> **Purpose**: First retrospective audit of the jvspatial foundation against the contracts now codified in [SPEC.md](SPEC.md), [PRD.md](PRD.md), and [CLAUDE.md](CLAUDE.md). Findings are the backlog for the next phase of hardening. +> +> **Method**: Five parallel reviewers, one per dimension: async contract, security boundaries, database adapter parity, walker/identity invariants, serverless/config/stability. Each returned a severity-tagged finding table with file:line citations. +> +> **Date**: 2026-05-17. +> **Library version**: 0.0.8. +> **Tests collected at audit time**: 1845. + +--- + +## Executive Summary + +**150 distinct findings**: 16 CRIT, ~42 HIGH, ~58 MED, ~34 LOW. + +Top themes: + +1. **Identity model is partially fictional.** `__entity_name__` override (the 0.0.8 headline feature, commit `a3964ab`) is ignored in 7+ code paths — Walker ID construction, GraphContext ID validation/regeneration, Node neighbor queries, Edge filtering, ObjectPager metadata. Today, any class using `__entity_name__` to disambiguate from a same-`__name__` peer will have its IDs **silently rewritten back to `cls.__name__`** by `GraphContext.save_object`. **This is a data-integrity bug for the feature shipped two commits ago.** +2. **Walker protection is partially fictional.** SPEC §6.3 promises `max_steps`, `max_visits_per_node`, `max_execution_time`, `max_queue_size`. In practice: `WalkerTrail` ignores `max_trail_length` entirely; queue insert paths (`prepend`, `add_next`, `insert_after`, `insert_before`) bypass `max_size`; queue drops are silent (no log); `resume()` resets step/visit counters and restarts the wall-clock timer on every call; and `ProtectionViolation` is swallowed into `walker.report` rather than raised as `InfiniteLoopError` / `WalkerTimeoutError`. +3. **Async contract has real holes, mostly in JsonDB.** Five CRITs: webhook walker `enhanced_init` is `async def __init__` (Python ignores; coroutine leaks every construction), webhook `endpoint_func(**kwargs)` is unawaited, storage `delete_file` unawaited, two `Path.write_text` calls inside `async def` graph exporters. Plus a string of `path.exists()` / `path.glob()` sync stats inside JsonDB `async` methods that block the event loop. +4. **Storage path traversal in `LocalFileInterface` versioning methods.** Primary save/read/delete go through `PathSanitizer`; the versioning subpaths (`create_version`, `get_version`, `list_versions`, `delete_version`, `get_latest_version`) compute `root_dir / f"{file_path}.versions"` without sanitization — user-controlled `file_path` like `../../etc/passwd` escapes the storage root. +5. **Webhook HMAC verification is broken.** A 7-character slice bug in `webhook_auth.utils.verify_signature` makes `hmac.compare_digest` always compare a 64-char hex digest to a 57-char prefix → always False. Webhook signature auth currently rejects every request. +6. **SHA-256 fallback uses `==`, not `compare_digest`.** `AuthenticationService._verify_refresh_token` falls back to `hashlib.sha256(token) == hashed` when bcrypt/argon2 fail to import; this path covers refresh tokens AND password-reset tokens. CLAUDE.md §2 non-negotiable. +7. **DynamoDB throttle-retry is partial.** Only `save`/`get`/`delete` are wrapped. `find` / `count` / `batch_get` / `batch_write` surface `ProvisionedThroughputExceededException` directly to callers, even though the adapter claims throttle retry (SPEC §4.3). +8. **Env-var allowlist is not enforced.** SPEC §10.2 promises "Unknown `JVSPATIAL_*` keys are rejected at startup to catch typos." `env_adapter.py` only *reads* enumerated keys; it never *scans* the environment for stray `JVSPATIAL_*` and rejects. Plus three divergent `parse_bool` implementations across `env.py` / `env_adapter.py` / `runtime/serverless.py`. + +These are listed below with file:line citations and one-line fixes. The audit was non-destructive — no fixes applied. The findings inform the next milestone. + +--- + +## How to read this + +- **Sections** are by dimension (async, security, database, walker, serverless). +- **Within a section**, findings sorted by severity then file. +- **`Cite`** column references SPEC §, CLAUDE.md §, or ROADMAP §. +- **`Fix`** is the smallest change that closes the gap, not necessarily the best long-term fix. + +When the same root cause shows up in multiple dimensions, it is listed once (under the most specific dimension) and cross-referenced. + +--- + +## 1. Identity Model — `__entity_name__` Coverage Gap + +The most important class of findings. SPEC §1.2: `__entity_name__` is per-subclass; ID construction and queries must go through `_entity_name()`. Commit `a3964ab` added the override but did not update every consumer. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 1.1 | CRIT | `core/entities/walker.py:308` | `generate_id(type_code, self.__class__.__name__)` — walker IDs ignore `__entity_name__`. | `self.__class__._entity_name()` (must add classmethod to Walker; Walker does not subclass Object). | SPEC §1.2 | +| 1.2 | CRIT | `core/entities/walker.py:311` | `kwargs["entity"] = self.__class__.__name__` — persisted `entity` field ignores override. | Same as 1.1. | SPEC §1.2 | +| 1.3 | CRIT | `core/context.py:753` | ID-format check `id_parts[1] != entity.__class__.__name__` triggers regeneration for every save of any entity using `__entity_name__` override — clobbers correct IDs. | Compare against `entity.__class__._entity_name()`. | SPEC §1.2 | +| 1.4 | CRIT | `core/context.py:759` | Regeneration uses `entity.__class__.__name__` → rewrites IDs to wrong discriminator on save. | Use `_entity_name()`. | SPEC §1.2 | +| 1.5 | CRIT | `core/context.py:1403` | `find_edges_between`: `query["entity"] = edge_class.__name__` — won't find edges whose stored entity uses override. | Use `edge_class._entity_name()`. | SPEC §1.2 | +| 1.6 | HIGH | `core/entities/node.py:561, 593` | `_node_query` filters edges by `edge_filter.__name__` against persisted `entity` field → silent no-op for override classes. | Use `"entity": edge_filter._entity_name()`. | SPEC §1.2, §2.3 | +| 1.7 | HIGH | `core/entities/node.py:426, 434` | `count_neighbors` fast-path uses `node.__name__` and `re.escape(entity_name)` against `n..` ID pattern. | Use `node._entity_name()`. | SPEC §1.2 | +| 1.8 | HIGH | `core/entities/node.py:668, 678, 688` | `_matches_node_filter` compares `node_obj.__class__.__name__` to string filter; `nodes(node="HostApp")` won't match. | Compare against `node_obj.__class__._entity_name()`. | SPEC §1.2 | +| 1.9 | MED | `core/entities/walker.py:308-311` | Walker has no `_entity_name()` method at all (doesn't subclass Object). | Add classmethod to Walker (or extract to shared mixin). | SPEC §1.2 | +| 1.10 | MED | `core/utils.py:40-89` | `_subclass_cache` is process-global, never cleared; stale identity after module reload (tests). | Add `clear_subclass_cache()` callable from test fixtures. | SPEC §1.2 | +| 1.11 | LOW | `core/pager.py:296` | `to_dict()` reports `object_type: self.object_class.__name__` instead of `_entity_name()`. | Resolve via `getattr(cls, "_entity_name", lambda: cls.__name__)()`. | SPEC §1.2 | + +--- + +## 2. Walker Protection Gaps + +SPEC §6.3 / CLAUDE.md §9. Promised limits are partially unenforced. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 2.1 | CRIT | `core/entities/walker.py:684-687` | `run()` catch-all swallows `ProtectionViolation` into `self.report(...)` instead of raising `InfiniteLoopError` / `WalkerTimeoutError` as SPEC promises. | Re-raise protection violations (or map to documented exception types) instead of stuffing in report. | SPEC §6.3 | +| 2.2 | CRIT | `core/entities/walker.py:886, 651` | `resume()` → `run()` → `_protection.reset()` resets step/visit counters AND restarts the wall-clock timer on every resume — protection trivially bypassed by repeated pause/resume. | Split `reset()` from `start_timer_if_needed()`; only reset on `spawn()` / explicit `reset_protection()`. | SPEC §6.3 | +| 2.3 | HIGH | `core/entities/walker_components/walker_trail.py:19-30` | `WalkerTrail` has no length bound — SPEC §6.4 promises `max_trail_length` "0 = unlimited, configurable" but it's not wired. Unbounded memory growth. | Pass `max_trail_length` from Walker into `WalkerTrail.__init__`; drop oldest on overflow. | SPEC §6.4 | +| 2.4 | HIGH | `core/entities/walker_components/walker_queue.py:67-94, 135-195` | `prepend`, `add_next`, `insert_after`, `insert_before` ignore `max_size` — protection bypass via these enqueue paths. | Apply same `max_size` guard (and warning log) used by `append`/`visit`. | SPEC §6.3 | +| 2.5 | HIGH | `core/entities/walker_components/walker_queue.py:33-36, 76-85` | `visit`/`append` silently drop on `max_size` hit — SPEC says "silent drop, logged" but no log is emitted. | Emit a one-shot WARNING when first drop occurs. | SPEC §6.3 | +| 2.6 | MED | `core/entities/walker.py:74-80` | `WalkerVisitingContext.__exit__` does not wrap `set_visitor(None)` in `try/finally`; if `current_node = None` raises, `_visitor_ref` never cleared. | Restructure with explicit `try/finally`. | SPEC §6.1 | +| 2.7 | MED | `core/entities/walker.py:63-71` | `WalkerVisitingContext.__enter__` calls `record_step` + `record_visit` — but `Walker.run()` (line 671, 674) calls the same methods directly without `visiting()`. Double-counting risk for callers who use `visiting()`. | Make `run()` use `WalkerVisitingContext`; one entry point. | SPEC §6.1, §6.3 | +| 2.8 | MED | `core/entities/walker.py:117-121` | Walker uses `extra="allow"` and does NOT declare `entity` as a model field — `entity` is set as dynamic attr, `protected=True` invariant is not enforced. | Declare `entity: str = attribute(protected=True, transient=True)` on Walker. | SPEC §1.2 | +| 2.9 | LOW | `core/entities/walker.py:734-748, 797-810` | Hook errors detected by `"Node skipped" in str(e)` substring match — fragile. | Define `SkipNode` exception class, catch directly. | SPEC §6.5 | +| 2.10 | LOW | `core/entities/walker.py:308-311` | Walker doesn't enforce `type_code="w"`; kwargs can corrupt SPEC §1.1 invariant. | Force `type_code="w"`. | SPEC §1.1 | + +--- + +## 3. Async Contract Violations + +CLAUDE.md §1 / SPEC §3. Five real bugs, plus a long tail of `path.glob`/`path.exists` sync stats blocking the event loop in JsonDB. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 3.1 | CRIT | `api/integrations/webhooks/helpers.py:135` | `enhanced_init` defined as `async def __init__`; Python ignores async-ness on `__init__`; each Walker construction returns a coroutine that's never awaited and leaks. | Remove `async` keyword. | SPEC §3 | +| 3.2 | CRIT | `api/integrations/webhooks/helpers.py:217` | `iscoroutinefunction(endpoint_func)` branch calls `endpoint_func(**kwargs)` without `await`; both `if/else` arms identical. | `result = await endpoint_func(**kwargs)` in the async branch. | SPEC §3.1 | +| 3.3 | CRIT | `api/integrations/storage/service.py:159` | `self.file_interface.delete_file(file_path)` not awaited; deletion never executes. | Add `await`. | SPEC §3.1 | +| 3.4 | CRIT | `core/graph.py:174` | `Path(output_file).write_text(...)` inside `async def generate_graph_dot` blocks the event loop. | `await asyncio.to_thread(...)`. | SPEC §3.3 | +| 3.5 | CRIT | `core/graph.py:332` | Same blocking write inside `async def generate_graph_mermaid`. | Same fix. | SPEC §3.3 | +| 3.6 | HIGH | `db/jsondb.py:168, 250, 254, 286, 355, 362` | Six sync `path.exists()` / `glob()` calls inside `async` DB methods — block event loop on disk I/O. | Wrap in `asyncio.to_thread`. | SPEC §3.3 | +| 3.7 | HIGH | `storage/interfaces/local.py:561` | `list(_read_chunks())` materializes entire file inside `to_thread` — defeats streaming purpose of `stream_file`. | Queue / `run_in_executor` iterator pattern, or use `aiofiles`. | SPEC §3 | +| 3.8 | HIGH | `db/dynamodb.py:675, 775` | `asyncio.gather(*[process_batch(...)])` without `return_exceptions=True` — one batch failure cancels siblings; partial-success state lost. | Add `return_exceptions=True`, aggregate per-batch failures. | SPEC §4.1 | +| 3.9 | HIGH | `api/integrations/webhooks/middleware.py:245, 514` | `asyncio.create_task(...)` fire-and-forget without strong reference or `add_done_callback`; tasks GC'd mid-flight, exceptions dropped. | Track tasks in a set; attach error sink. | SPEC §3 | +| 3.10 | HIGH | `testing/__init__.py:610`, `async_utils/__init__.py:37` | `asyncio.gather` without `return_exceptions=True` in test harness / general helper. | Add the flag. | SPEC §3 | +| 3.11 | MED | (multiple) `core/entities/walker_components/{walker_queue,protection,walker_trail}.py` and `core/utils.py:25`, `core/context.py:1424`, `core/entities/{object,node,edge,walker}.py` (~30 sites) | Many methods declared `async def` with no `await` — false-async. Wasted context switches; not bugs, but degrade the contract. | Convert each to sync, or genuinely await something. Roll into a single PR after the CRITs/HIGHs land. | SPEC §3.2 | + +(Full async list: see reviewer report. Includes WalkerQueue mutators, TraversalProtection state methods, JsonDBTransaction buffered ops, several auth and rate-limit dict mutators.) + +--- + +## 4. Security Boundaries + +SPEC §15 / CLAUDE.md §2. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 4.1 | CRIT | `api/auth/service.py:367` | `hashlib.sha256(token).hexdigest() == hashed` — refresh-token / password-reset-token hash compared with `==`. | `hmac.compare_digest(...)`. | SPEC §15.2, CLAUDE §2 | +| 4.2 | CRIT | `storage/interfaces/local.py:207, 220, 252, 260, 288, 327, 332, 333, 356` | All file-versioning methods compute `root_dir / f"{file_path}.versions"` without `PathSanitizer`; user-controlled `file_path` escapes the storage root. | Sanitize `file_path` through `_get_full_path()` before building versioned paths; resolve and `.relative_to(self.root_dir)` the result. | SPEC §15.1 | +| 4.3 | CRIT | `api/integrations/webhooks/utils.py:176` | `hmac.compare_digest(signature, expected_signature[len(prefix):])` — `expected_signature` is the bare hex digest; slicing 7 chars produces 57-char string vs 64-char signature → always False. Webhook HMAC always rejects. | Drop the slice: `compare_digest(signature, expected_signature)`. | SPEC §15.2 | +| 4.4 | HIGH | `api/auth/api_key_service.py:30` | `self.context = context or get_default_context()` — auth state can land on non-prime DB when a caller forgets to pass `context`. | `context or GraphContext(database=get_prime_database())`. | SPEC §9, CLAUDE §1 | +| 4.5 | HIGH | `api/auth/api_key_service.py:213-246` + `api/integrations/webhooks/webhook_auth.py:19-21, 180-183` | `revoke_key` flips `is_active=False` in DB but does not invalidate the 300s in-memory `_API_KEY_CACHE`. Revoked key authenticates for up to 5 minutes after revocation. | Add `webhook_auth.invalidate_cache(...)` hook called from `revoke_key`; or re-check `is_active` on cache hit. | SPEC §15.2 | +| 4.6 | HIGH | `api/components/auth_configurator.py:175-391` | `/auth/register`, `/auth/login`, `/auth/forgot-password`, `/auth/reset-password`, `/auth/change-password` registered without endpoint rate-limit configs; fallback is global `default_limit=60/60s` if rate limiting is enabled at all. | Hard-code rate-limit configs during `_register_auth_endpoints`; document rate-limit middleware as required when auth enabled. | SPEC §15 | +| 4.7 | HIGH | `api/integrations/webhooks/webhook_auth.py:19, 147-183` | `_API_KEY_CACHE` mutated across `await` without lock; eviction at size cap races with reads (`KeyError`). | Wrap in `asyncio.Lock`, or use bounded LRU with single-lock guard. | CLAUDE.md "race conditions in auth state" | +| 4.8 | HIGH | `api/auth/enhanced.py:235-389` | `SessionManager._sessions` / `_user_sessions` mutated across `await` without lock; concurrent logout-vs-login → `RuntimeError: dictionary changed size during iteration`; `max_sessions_per_user` enforcement is racy. | `asyncio.Lock` around session create/invalidate/cleanup. | CLAUDE.md races | +| 4.9 | HIGH | `storage/interfaces/local.py:657-659` | `get_metadata` MIME detection passes `content=b""` — falls back to extension-based `mimetypes.guess_type`; mislabels served `Content-Type`. | Read first 4 KiB of file, pass as content; or store validated MIME in sidecar at save. | SPEC §15.1 | +| 4.10 | HIGH | `api/components/error_handler.py:763, 769-770` | If operator sets `JVSPATIAL_EXPOSE_ERROR_DETAILS=true` in production, raw exception messages leak in 500 responses; safe default exists but no guard. | Refuse to honor the flag when `get_environment_mode() == "production"`. | SPEC §15.5 | +| 4.11 | MED | `api/auth/service.py:427` | Debug log discloses JWT secret length. | Log `secret_configured=bool(secret)` only. | SPEC §15.5 | +| 4.12 | MED | `api/middleware/manager.py:194-197` + `api/config_groups.py:66-75` | `CORSConfig` accepts wildcard origins; no startup validator. SPEC says wildcards must trigger startup warning. | Add validator on `CORSConfig` to warn on `"*"`; fail loudly if `allow_credentials=True` + wildcard. | SPEC §15.4 | +| 4.13 | MED | `api/auth/service.py:1052-1059` | `validate_token` warning logs the DB `base_path` — discloses internal filesystem path. | Log `db_type=type(database).__name__`. | SPEC §15.5 | +| 4.14 | MED | `api/auth/service.py:1038-1049` | DB-error fallback in `validate_token` trusts JWT-payload roles when DB lookup fails — fails open on DB outage, within JWT-expiry window. | Fail closed: return `None` on DB error. | SPEC §9 | +| 4.15 | MED | `api/auth/service.py:469-531` | `_blacklist_cache` dict mutated across `await` without lock; unbounded growth (per-entry TTL only). | Add `asyncio.Lock` + size cap with LRU eviction. | CLAUDE.md races | +| 4.16 | LOW | `api/deferred_invoke_route.py:28-37` | Empty `JVSPATIAL_DEFERRED_INVOKE_SECRET` returns `True` from `_deferred_invoke_secret_ok` — misconfigured deployment exposes internal endpoint. | Treat empty secret as "deny all". | SPEC §15.2 | +| 4.17 | LOW | `api/auth/service.py:331, 243-244` | `JVSPATIAL_AUTH_STRICT_HASHING` defaults True for passwords, False for tokens — asymmetry surprising. | Apply strict to both; document rationale. | SPEC §15.5 | +| 4.18 | LOW | `storage/interfaces/local.py:194-196` | Windows reserved names (CON, AUX, NUL) pass `SAFE_FILENAME_PATTERN`. | Add Windows-reserved-name check. | SPEC §15.1 | + +--- + +## 5. Database Adapter Parity + +SPEC §4-5 / ROADMAP §2.2, §2.4. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 5.1 | CRIT | `db/dynamodb.py:1052-1064, 1194-1265, 668-684, 770-778` | DynamoDB throttle retry (`_run_with_throttle_retry`) covers only `save`/`get`/`delete`. `find` / `count` / `batch_get` / `batch_write` surface `ProvisionedThroughputExceededException` immediately. | Wrap all CRUD paths in throttle retry; escalate `batch_write` inner-retry to the shared envelope. | SPEC §4.3 | +| 5.2 | HIGH | `db/query.py:425-426, 502-520, 559-564, 663-679` | `QueryBuilder` exposes `nor_`, `mod`, `all_`, `regex`, `type_`, `size`, `$nor`, `$mod`, `$type`, `$all`, `$where`, `$text` — but `QueryEngine._match_value` returns `False` for any unknown operator; `match` never branches on `$nor`. Queries built via the builder silently match nothing. | Implement the operators in `_match_value`/`match`, OR have builder + engine raise `QueryError("unsupported operator")`. | SPEC §5.1 | +| 5.3 | HIGH | `db/database.py:279-322`, `db/work_claim.py:53-62` | Default `find_one_and_update` / `find_one_and_delete` query on `_id`, but JsonDB/SQLite/DynamoDB persist only `id`; silent miss on non-Mongo backends. | Normalize `_id` ↔ `id` in default impls, or override on each backend. | SPEC §4.1 | +| 5.4 | HIGH | `db/database.py:144-159` | `Database.count` default materializes every record via `find`. Third-party adapters registered via `register_database_type` inherit this — silent OOM on large collections. | Either mark `count` abstract, or warn once-per-class on hit. | SPEC §4.1 | +| 5.5 | HIGH | `db/mongodb.py:332-351, 380-401` | `find_many` and `bulk_save` reconnect-retry once but escaping raw `PyMongoError` is not wrapped in `DatabaseError` (CRUD paths are wrapped). | Route through `_run_with_reconnect`. | SPEC §4.1, §4.3 | +| 5.6 | HIGH | `db/database.py:212-250` | Default `bulk_save` does sequential `save()` without per-record exception handling — partial-success leaves DB in unobservable state. Doc claims "all-or-throw" but JsonDB override logs and continues. | Return `(saved, failed_ids)` or document loudly + reconcile JsonDB override. | SPEC §4.1 | +| 5.7 | MED | `db/dynamodb.py:686-778` | `batch_write` inner retry — at most 3 attempts then logs warning and returns success. Lost writes silent. | Return actual saved count, propagate through `bulk_save`, raise on persistent unprocessed items. | SPEC §4.1 | +| 5.8 | MED | `db/query.py:170-196` | `_add_indexing_hints` mutates the cached optimized query in-place; subsequent callers receive same dict. Also `$hint` falls through in non-Mongo backends → match returns False. | Deep-copy on cache hit; whitelist-skip `$hint`/`$select` in `_match_value`. | SPEC §5.3 | +| 5.9 | MED | `db/transaction.py:122-194`, `db/mongodb.py:67, 699-733` | `MongoDB.supports_transactions = True` unconditionally; `begin_transaction` returns `None` when not on a replica set. Flag is dishonest. | Probe replica-set support lazily; cache; or rename flag and add `is_transactional()`. | SPEC §4.2 | +| 5.10 | MED | `db/sqlite.py:34-64, 117-162` | One persistent connection per `SQLiteDB` instance; no cross-loop detection (MongoDB has one). Use across multiple event loops fails silently. | Track creating loop in `__init__`; rebind on detection or raise `DatabaseError`. | SPEC §4.3 | +| 5.11 | MED | `db/_sqlite_translate.py:86-98` | `_safe_field_path` regex rejects digit-only segments (e.g. `arr.0.value`); query silently falls back to full scan + Python match. | Widen segment regex, OR log debug on fallback so slow queries are diagnosable. | SPEC §5.2 | +| 5.12 | MED | `db/dynamodb.py:1066-1265` | `find()` fallback to `Scan` does not push `$or`/`$and`/`$gt` into FilterExpression; full client-side scan + match. | Push to FilterExpression where supported. | SPEC §5.2 | +| 5.13 | LOW | `db/database.py:212-250` and adapter overrides | `bulk_save` returns `int` everywhere; callers cannot distinguish SQLite all-or-nothing vs DynamoDB silent drop. | Expose `BulkSaveResult(attempted, saved, failed_ids)` (or sibling `bulk_save_detailed`). | SPEC §4.1 | +| 5.14 | LOW | `db/transaction.py:363-413` | `JSONTransaction` is dead code; `JsonDBTransaction` replaces it; still in `__all__`. | Remove or `@deprecated`. | ROADMAP §2.9 | +| 5.15 | LOW | `db/sqlite.py:260-285` | `save()` and `bulk_save` differ in `id` coercion (`record.setdefault("id", uuid)` vs `str(r["id"])`). Inconsistent get-by-id. | Force `record["id"] = str(record["id"])` after setdefault. | SPEC §4.1 | + +(Plus minor LOWs on log hygiene, dead glob filters in JsonDB, double cache writes in `CachingDatabase.find_one_and_update`.) + +--- + +## 6. Inheritance, Subclass Init, and Library Self-Discipline + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 6.1 | HIGH | `core/entities/node.py:62-118` | `Node.__init_subclass__` does NOT call `super().__init_subclass__()` — `AttributeMixin.__init_subclass__` never runs for Node subclasses; `_PROTECTED_ATTRS` registration skipped. | Add `super().__init_subclass__(**kwargs)` at top. | SPEC §2.5 | +| 6.2 | HIGH | `core/entities/edge.py:66-123` | Same: `Edge.__init_subclass__` skips `super()`. | Same fix. | SPEC §2.5 | +| 6.3 | HIGH | `core/entities/walker.py:366-396` | Same: `Walker.__init_subclass__` skips `super()`. | Same fix. | SPEC §2.5 | +| 6.4 | HIGH | `core/entities/root.py:19` | `_lock = asyncio.Lock()` is `ClassVar` — single global lock across all GraphContexts and event loops; "different loop" errors under per-test loop fixtures; SPEC §1.3 implies per-context. | Lock per `GraphContext` (e.g. lazy-init dict keyed by `id(context)`). | SPEC §1.3, §7.2 | +| 6.5 | HIGH | `core/entities/root.py:100` | `object.__setattr__(self, "id", "n.Root.root")` — library bypasses its own `protected=True`. | `_unsafe_set_id` helper, or check/clear `_initializing` and route through normal setter. | CLAUDE §5 | +| 6.6 | HIGH | `core/context.py:761, 812, 818, 1229` | Six `object.__setattr__` call sites bypass protected-attribute enforcement (id, edge_ids, atomic_increment). | Route through `AttributeMixin.__setattr__` with explicit override flag. | CLAUDE §5 | +| 6.7 | MED | `core/mixins/deferred_save.py:119+` | No runtime check of MRO order — wrong order silently disables batching (CLAUDE.md warns but library doesn't detect). | In `__init_subclass__`, assert mixin precedes persistable base; warn or raise. | CLAUDE §6 | +| 6.8 | MED | `core/entities/object.py:112-140` | `__setattr__` allows ANY `name.startswith("_")` — callers can attach arbitrary `_foo` attributes, bypassing schema validation (SPEC promises rejection). | Restrict private bypass to declared `__private_attributes__` + `_initializing`. | SPEC §2.1 | +| 6.9 | MED | `core/events.py:36, 119`, `core/context.py:1818-1843` | `EventBus._lock` and `event_bus` module global bound to import-time loop; tests with new loops fail. Also `get_default_context()` has check-and-set race. | Lazy-init locks per first-async-use; use `ContextVar.get` + token for default context init. | SPEC §6.x, §7.1 | +| 6.10 | MED | `core/events.py:38-45` | `register_entity` on `walker.spawn()` has no symmetric `unregister_entity` — events keep firing to done walkers; weakref GC only. | Call `event_bus.unregister_entity(self.id)` in `disengage()` and end-of-`spawn`. | SPEC §6.x | +| 6.11 | MED | `core/entities/walker_components/protection.py:108-115` | `_check_timeout` reads `_start_time` then computes elapsed; concurrent `reset()` corrupts. | Snapshot to local before subtraction. | SPEC §6.3 | + +--- + +## 7. Serverless / Config / Stability Discipline + +SPEC §10-11, §18 / CLAUDE.md §4, §7, §8. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 7.1 | HIGH | `env_adapter.py:52-193` | Allowlist not enforced. SPEC §10.2 promises "Unknown `JVSPATIAL_*` keys are rejected at startup." Adapter only *reads* enumerated keys; never *scans* for stray `JVSPATIAL_*`. | Add startup validator: scan `os.environ` for `JVSPATIAL_*` prefix; reject unknowns. | SPEC §10.2 | +| 7.2 | HIGH | `env_adapter.py:11` vs `env.py:36-43` vs `runtime/serverless.py:11` | Three divergent `parse_bool` implementations. `JVSPATIAL_DEBUG=on` parses different ways in different code paths. | Consolidate on `env.parse_bool`; have others import. | SPEC §10.2 | +| 7.3 | MED | `env_adapter.py:11`, `runtime/serverless.py:11` | `_parse_bool` non-strict; `JVSPATIAL_DEBUG=garbage` silently maps to False with no validation error. Hides typos. | Raise `ValueError` on unrecognized non-empty. | SPEC §10.2 | +| 7.4 | MED | `serverless/deferred_invoke.py:52-73` | `normalize_deferred_envelope` covers flat Lambda invoke + SQS scheduler shapes only. Direct Lambda SQS-batch trigger (`{"Records": [...]}`) raises. | Accept `Records[...]` and dispatch per-record, OR document the unwrap requirement. | SPEC §11.3 | +| 7.5 | MED | `.env.example:94-106` vs `api/config_groups.py:66-81` | `.env.example` documents `JVSPATIAL_CORS_ORIGINS=*` with "Default: *" — actual default is localhost whitelist. Following docs degrades security. | Update `.env.example`; add production-only note. | CLAUDE §8 | +| 7.6 | MED | `runtime/lwa.py:107-110` (called from `server.py:137`) | LWA env defaults applied inside `Server.__init__`; LWA reads them before Python starts. The docstring acknowledges this; the practical effect is zero for the actual LWA bootstrap. | Downgrade to operator warning, or remove and document IaC-only. | SPEC §11.4 | +| 7.7 | MED | `db/transaction.py:243-249` | `JsonDBTransaction(best_effort=True)` imports private `_emit_once` from `jvspatial.utils.stability` — internal symbol crossed module boundary. | Expose public `emit_experimental_once(...)`. | SPEC §18 | +| 7.8 | MED | `serverless/deferred_invoke.py:1-88` | Handlers registered post-import → race: request arriving before user modules import returns 404 `UnknownDeferredTaskError`. Not silently dropped (good); but no startup-readiness log. | Log registered handlers on startup, or log debug on empty-registry first-dispatch. | SPEC §11.3 | +| 7.9 | LOW | `env_adapter.py:39-49` | `deep_merge` silently skips `None` in override; `Server(host=None, ...)` does NOT override env. Surprising. | Document; or sentinel-based override. | SPEC §10.2 | +| 7.10 | LOW | `api/auth/service.py:164-170` | `AuthenticationService` captures `is_serverless_mode()` and `_bcrypt_rounds` at construction; per-test mode flips require service rebuild + `reset_serverless_mode_cache()`. | Note in CLAUDE.md §4 (test guidance). | CLAUDE §4 | +| 7.11 | LOW | `runtime/lwa.py:18-23` | `_deferred_invoke_pass_through_path` reads `JVSPATIAL_API_PREFIX` directly, bypassing `resolve_api_prefix()`. Two divergent reads. | Use `resolve_api_prefix()`. | SPEC §10.4 | +| 7.12 | LOW | `api/components/app_builder.py:58-65` | `JVSPATIAL_DOCS_DISABLED` parses with ad-hoc inline set `{"1","true","yes","on"}` — fourth divergent bool parser. | Use `env.parse_bool`. | SPEC §10.5 | +| 7.13 | LOW | `api/middleware/manager.py:54-59` | `_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json")` hardcoded — customizing `docs_url` breaks Swagger UI via strict CSP. | Derive from `config.docs_url`/`redoc_url`/`openapi_url`. | SPEC §10.5 | +| 7.14 | LOW | `serverless/tasks/stub.py:27` | `LoggingNoopTaskScheduler.schedule()` logs warning per call. CloudWatch cost in misconfigured serverless deploys. | Downgrade per-call to debug; rely on once-per-process error. | SPEC §11.2 | +| 7.15 | LOW | `serverless/factory.py:73-80` | Fallback to `LoggingNoopTaskScheduler` when AWS deferred-task config missing — warning string only; no startup error in non-`strict` mode. | Once-per-process startup error log when serverless + provider=aws and config missing. | SPEC §11.3 | + +--- + +## 8. ObjectPager / Pagination + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 8.1 | HIGH | `core/pager.py:118-144` | Keyset pagination filters `id > after_id` but sorts by `context.` — cursor doesn't track sort key; wrong/missing rows on writes between pages. | Disallow `order_by` with `after_id`, OR include order field in cursor. | SPEC §5 | +| 8.2 | HIGH | `core/pager.py:73, 142, 214` | `_cache` keyed by `(page, hash(filters))` and never invalidated — stale results after writes. | Drop the cache or expose `invalidate()`. | SPEC §5 | + +--- + +## 9. Verified Correct (no findings) + +These are explicitly checked — the audit found no gap. Listed so future reviewers know the area was inspected: + +- **Detection precedence in `is_serverless_mode`** (SPEC §11.1) — exact match. +- **`reset_serverless_mode_cache`** clears both lru_caches. +- **Mode-dependent defaults read mode at call time**, not at import. +- **`JVSPATIAL_DOCS_DISABLED` coverage**: all four URLs (`docs_url`, `redoc_url`, `openapi_url`, `swagger_ui_oauth2_redirect_url`) set to None. +- **CSP per-path match** uses exact-or-segment matching; `/docsfoo` correctly stays on strict CSP. +- **bcrypt rounds**: serverless=10, standard=12, env-overridable. +- **JsonDB `/tmp` default**: only applied under serverless mode. +- **`__init__.py` `__all__`**: every imported public symbol is in `__all__`; no underscore-module imports leak to public surface. +- **`Server.run()` blocking**: `uvicorn.run()` is the only blocking call; `run_async()` uses async `serve()`. +- **JWT secret validation**: rejected if empty or known-insecure default. +- **CSP defaults**: strict on app routes; relaxed only on docs paths. +- **JsonDB atomic writes**: every write path goes through `_atomic.atomic_write_bytes`; no direct `open().write()` bypasses. +- **PathLock coverage in JsonDB**: every record write/delete acquires the per-file lock. +- **SQLite Mongo→SQL translator escape hygiene**: values bound, field paths regex-validated, unknown ops trigger fallback (parameterized, no injection). +- **Auth state on prime DB** in `AuthenticationService` (one drift in `APIKeyService` flagged separately at 4.4). +- **Primary file save/read/delete in `LocalFileInterface`** correctly use `PathSanitizer` + `resolve()` + `relative_to(root_dir)`. The escape vector is exclusively in versioning methods (4.2). +- **`hmac.compare_digest`** used everywhere SPEC §15.2 lists, **except** the SHA-256 fallback (4.1) and the broken-by-slice webhook signature path (4.3). + +--- + +## 10. Recommended Remediation Sequence + +Roughly ordered by blast radius × cost. CRITs first, then HIGHs that block CRIT fixes. + +### Wave 1 — restore the contracts the library claims to provide (1-2 weeks) + +1. **Identity model** (§1.1-1.5): patch all 5 CRIT call sites to use `_entity_name()`. Add classmethod to Walker. Add regression test that subclasses with `__entity_name__` round-trip through save/load. +2. **Walker protection** (§2.1-2.5): make `ProtectionViolation` raise documented exception types; separate timer-start from counter-reset; wire `max_trail_length`; cap insert paths; emit drop warnings. +3. **Async CRITs** (§3.1-3.5): five surgical fixes. Add tests that assert coroutine results are not leaked. +4. **Security CRITs** (§4.1-4.3): one `compare_digest` fix; three sanitizer calls in versioning paths; one slice removal in webhook verifier. +5. **DynamoDB throttle retry** (§5.1): wrap the four uncovered code paths in `_run_with_throttle_retry`. + +### Wave 2 — close the HIGHs that hide bugs (1-2 weeks) + +6. **JsonDB sync stats** (§3.6): six `to_thread` wraps. Bench to confirm regression budget. +7. **`__init_subclass__` super-chains** (§6.1-6.3): three one-line additions. Add regression test that protected attrs are registered on Node/Edge/Walker subclasses. +8. **Auth races + cache invalidation** (§4.5-4.8): four `asyncio.Lock`s. Add concurrent-stress test. +9. **Pager + cache** (§8.1-8.2): drop the stale cache; constrain or fix keyset pagination. +10. **Default `find_one_and_update` `_id` vs `id`** (§5.3): normalize in default impls. + +### Wave 3 — close the latent timebombs (2-4 weeks) + +11. **Env allowlist enforcement + parse_bool consolidation** (§7.1-7.2): both small, both gate a class of future config-typo bugs. +12. **`bulk_save` partial-success semantics** (§5.6, 5.7): unify return type across adapters. +13. **MongoDB `supports_transactions` honesty** (§5.9): probe replica set; rename or runtime-check. +14. **QueryBuilder ↔ QueryEngine operator parity** (§5.2): implement or reject. +15. **CORS wildcard startup warning** (§4.12), **`JVSPATIAL_EXPOSE_ERROR_DETAILS` production guard** (§4.10). + +### Wave 4 — cleanup, parity, polish (ongoing) + +16. False-async cleanup (§3.11): roll-up PR converting 30+ `async def` with no `await` to `def`. Or accept and document. +17. SQLite cross-loop detection (§5.10). +18. Stability-tier cleanup (§7.7 expose `emit_experimental_once`; §7.13 docs-prefix derivation from config). +19. Dead code (§5.14 `JSONTransaction`). +20. LOW hygiene findings (log content sanitization, Windows-reserved-name filename check, `.env.example` accuracy). + +--- + +## Appendix — Reviewer Reports + +Five reviewers produced this audit: +1. **Async contract**: agent `a85870fd5ef8e1790` +2. **Security boundaries**: agent `a27d75965d59e33a0` +3. **DB adapter parity**: agent `ac27d884ad66bfd59` +4. **Walker / identity / invariants**: agent `a716803cafeda1acb` +5. **Serverless / config / stability**: agent `a7842dedd5dfe62d4` + +Full reports are in the agent transcripts. This file synthesizes and de-duplicates their findings; for the raw lists with extended rationale, query the agents above via SendMessage. From c7b412b603f3111295684a29578ce1534cc63104 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 10:30:53 -0400 Subject: [PATCH 09/22] =?UTF-8?q?fix:=20Wave=201=20audit=20remediation=20?= =?UTF-8?q?=E2=80=94=20identity,=20walker=20protection,=20async,=20securit?= =?UTF-8?q?y,=20dynamodb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses 16 CRIT-class findings from AUDIT.md §1-§5 (Wave 1 of the post-audit hardening plan). All changes are surgical and SPEC-aligned; no API removals. --- Group A: identity model (audit §1) --- The 0.0.8 ``__entity_name__`` override (commit a3964ab) was ignored in the walker constructor, in GraphContext save/edge-query paths, and in several Node lookup helpers. The result: any subclass using the override had its IDs silently rewritten back to ``cls.__name__`` on every save (data corruption), and edge/neighbor queries by override class returned nothing. * core/entities/walker.py — declare ``__entity_name__`` and add ``_entity_name()`` classmethod parallel to Object's; thread through ID generation and persisted ``entity`` (§1.1, §1.2, §1.9). * core/context.py — save_object ID validation + regeneration uses ``_entity_name()`` (§1.3, §1.4). find_edges_between uses ``_entity_name()`` (§1.5). * core/entities/node.py — _node_query uses persisted ``entity`` field (not the wrong ``name`` key) and honors ``_entity_name()`` (§1.6). count_neighbors fast path resolves via ``_entity_name()`` (§1.7). _matches_node_filter compares against ``_entity_name()`` (§1.8). --- Group B: security CRITs (audit §4) --- * api/auth/service.py — replace ``==`` with ``hmac.compare_digest`` for SHA-256 refresh-token / password-reset-token fallback verification (§4.1 / SPEC §15.2). CLAUDE.md §2 non-negotiable. * storage/interfaces/local.py — add ``_sanitized_version_base`` helper and route every versioning method (create_version, get_version, list_versions, delete_version, get_latest_version) through it. Previously these computed ``self.root_dir / f"{file_path}.versions"`` without sanitization — ``../../etc/passwd`` escaped the storage root (§4.2 / SPEC §15.1). * api/integrations/webhooks/utils.py — drop the trailing ``[len(prefix):]`` slice in verify_hmac_signature. The earlier slice truncated 7 chars off a 64-char SHA-256 hex digest, making ``hmac.compare_digest`` always return False — webhook HMAC verification rejected every request (§4.3 / SPEC §15.2). --- Group C: async CRITs (audit §3) --- * api/integrations/webhooks/helpers.py — convert ``async def enhanced_init`` to sync ``def``. Python ignores async-ness on ``__init__``; the coroutine returned from the async form was never awaited and leaked on every webhook walker construction (§3.1). * api/integrations/webhooks/helpers.py — add missing ``await`` in the async-endpoint branch of webhook_wrapper (§3.2). Both arms of the if/else were identical, so coroutines leaked unawaited. * api/integrations/storage/service.py — add missing ``await`` on ``self.file_interface.delete_file(...)``; the unawaited call left ``success`` as the coroutine object and skipped the delete (§3.3). * core/graph.py — wrap Path.write_text calls inside ``generate_graph_dot`` and ``generate_graph_mermaid`` in ``asyncio.to_thread`` so blocking disk I/O does not stall the event loop inside async functions (§3.4-§3.5). --- Group D: walker protection (audit §2) --- SPEC §6.3 promised that exceeding ``max_steps`` / ``max_visits_per_node`` / ``max_execution_time`` raises documented exception types. The implementation silently swallowed ProtectionViolation into ``walker.report`` and continued. ``run()`` also called ``_protection.reset()`` unconditionally — ``resume()`` re-entering ``run()`` cleared step / visit / timer counters, defeating limits via pause/resume cycling. ``max_trail_length`` was unimplemented despite docstrings. Multiple WalkerQueue insert paths bypassed ``max_size``. * core/entities/walker_components/protection.py — add ``_started`` flag and ``start_if_needed()`` idempotent initializer; preserve explicit ``reset()`` for callers wanting a clean restart (§2.2). * core/entities/walker.py — Walker.run() now calls ``start_if_needed`` (so resume cannot reset counters) and catches ``ProtectionViolation`` to raise the documented ``InfiniteLoopError`` / ``WalkerTimeoutError`` / ``WalkerExecutionError`` types (§2.1, SPEC §6.3 / §17). Pop ``max_trail_length`` from kwargs/env and rebuild the trail tracker with the configured bound (§2.3). * core/entities/walker_components/walker_trail.py — back ``_trail`` with a bounded ``deque(maxlen=max_length)``; ``0`` means unlimited (SPEC §6.4 / §2.3). * core/entities/walker_components/walker_queue.py — every insert path (prepend, append, add_next, insert_after, insert_before) now respects ``max_size`` and emits a one-shot WARNING on first drop (§2.4, §2.5). insert_after / insert_before return the list of actually-inserted items rather than the requested list. --- Group E: DynamoDB throttle retry coverage (audit §5) --- * db/dynamodb.py — wrap every aioboto3 wire call (batch_get_item, batch_write_item, scan, query) in ``_run_with_throttle_retry`` so transient ProvisionedThroughputExceededException / ThrottlingException trigger the documented exponential backoff (§5.1 / SPEC §4.3). Was previously applied only to save/get/delete; find / count / batch ops surfaced throttles to callers as immediate failures. --- Tests --- Existing tests in tests/core/test_walker_protection.py and test_walker_trail_new.py cemented the swallow-everything behavior; updated to assert the documented exception types are raised (SPEC §6.3). New regression coverage (28 new test cases, all passing): * tests/core/test_entity_name_walker_and_save.py — Walker ``_entity_name`` classmethod; ID stability through save; find_edges_between with override. * tests/core/test_walker_protection_audit_fixes.py — ``start_if_needed`` idempotency vs ``reset``; run() raises InfiniteLoopError / WalkerExecutionError; max_trail_length wired; WalkerQueue insert paths respect max_size. * tests/storage/test_versioning_path_sanitizer_audit.py — path- traversal coverage on all five versioning methods. * tests/api/test_webhook_hmac_audit_fix.py — HMAC verify round-trip + tamper rejection. Verification: pytest tests/core tests/storage tests/db tests/api → 1537 passed; tests/integration → 47 passed; tests/cache + observability + runtime + serverless + logging + utils + testing → 192 passed, 1 otel-import skip. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/auth/service.py | 7 +- jvspatial/api/integrations/storage/service.py | 5 +- .../api/integrations/webhooks/helpers.py | 13 +- jvspatial/api/integrations/webhooks/utils.py | 9 +- jvspatial/core/context.py | 32 +++- jvspatial/core/entities/node.py | 37 +++- jvspatial/core/entities/walker.py | 80 +++++++- .../entities/walker_components/protection.py | 25 ++- .../walker_components/walker_queue.py | 100 ++++++++-- .../walker_components/walker_trail.py | 30 ++- jvspatial/core/graph.py | 10 +- jvspatial/db/dynamodb.py | 63 +++++-- jvspatial/storage/interfaces/local.py | 46 ++++- tests/api/test_webhook_hmac_audit_fix.py | 47 +++++ .../core/test_entity_name_walker_and_save.py | 114 ++++++++++++ tests/core/test_walker_protection.py | 57 ++++-- .../test_walker_protection_audit_fixes.py | 173 ++++++++++++++++++ tests/core/test_walker_trail_new.py | 15 +- .../test_versioning_path_sanitizer_audit.py | 62 +++++++ 19 files changed, 825 insertions(+), 100 deletions(-) create mode 100644 tests/api/test_webhook_hmac_audit_fix.py create mode 100644 tests/core/test_entity_name_walker_and_save.py create mode 100644 tests/core/test_walker_protection_audit_fixes.py create mode 100644 tests/storage/test_versioning_path_sanitizer_audit.py diff --git a/jvspatial/api/auth/service.py b/jvspatial/api/auth/service.py index e25793c..0892e66 100644 --- a/jvspatial/api/auth/service.py +++ b/jvspatial/api/auth/service.py @@ -364,7 +364,12 @@ def _verify_refresh_token(self, token: str, hashed: str) -> bool: except Exception: return False else: - return hashlib.sha256(token.encode()).hexdigest() == hashed + # Constant-time comparison is non-negotiable for any secret + # comparison (SPEC §15.2, CLAUDE.md invariant §2). The earlier + # ``==`` form was timing-leakable on partial hex prefix matches. + return hmac.compare_digest( + hashlib.sha256(token.encode()).hexdigest(), hashed + ) def _generate_jwt_token( self, diff --git a/jvspatial/api/integrations/storage/service.py b/jvspatial/api/integrations/storage/service.py index a085cf9..bea61b1 100644 --- a/jvspatial/api/integrations/storage/service.py +++ b/jvspatial/api/integrations/storage/service.py @@ -156,7 +156,10 @@ async def handle_delete(self, file_path: str) -> Dict[str, Any]: HTTPException: On storage error """ try: - success = self.file_interface.delete_file(file_path) + # ``delete_file`` is async on every backend; missing ``await`` + # silently skipped the delete and assigned the coroutine to + # ``success`` (audit §3.3). + success = await self.file_interface.delete_file(file_path) return {"success": success, "file_path": file_path} except StorageError as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/jvspatial/api/integrations/webhooks/helpers.py b/jvspatial/api/integrations/webhooks/helpers.py index d14c9f0..43b442d 100644 --- a/jvspatial/api/integrations/webhooks/helpers.py +++ b/jvspatial/api/integrations/webhooks/helpers.py @@ -132,7 +132,11 @@ def inject_walker_webhook_payload(walker_class: type) -> type: """ original_init = walker_class.__init__ # type: ignore - async def enhanced_init(self, *args: Any, **kwargs: Any) -> None: + # ``__init__`` is a sync protocol — Python ignores ``async def __init__`` + # and returns the coroutine to the constructor, which then never gets + # awaited. The earlier ``async def`` here leaked a coroutine on every + # webhook walker construction (audit §3.1 / SPEC §3.1). + def enhanced_init(self, *args: Any, **kwargs: Any) -> None: # Extract webhook data from kwargs if present webhook_data = kwargs.pop("webhook_data", {}) @@ -212,9 +216,12 @@ async def webhook_wrapper(request: Request) -> Any: # For any other required parameters, we'll let the function handle it # and raise an error if needed - # Call original function with only the parameters it expects + # Call original function with only the parameters it expects. + # The async branch must ``await``; previously both arms were + # identical, so coroutines leaked unawaited (audit §3.2 / + # SPEC §3.1). if asyncio.iscoroutinefunction(endpoint_func): - result = endpoint_func(**kwargs) + result = await endpoint_func(**kwargs) else: result = endpoint_func(**kwargs) diff --git a/jvspatial/api/integrations/webhooks/utils.py b/jvspatial/api/integrations/webhooks/utils.py index a889a28..336e014 100644 --- a/jvspatial/api/integrations/webhooks/utils.py +++ b/jvspatial/api/integrations/webhooks/utils.py @@ -172,8 +172,13 @@ def verify_hmac_signature( payload, secret, algorithm, "" ) # No prefix for comparison - # Use constant-time comparison to prevent timing attacks - return hmac.compare_digest(signature, expected_signature[len(prefix) :]) + # ``signature`` is the incoming digest after prefix stripping above, + # and ``expected_signature`` is generated with ``prefix=""`` so it is + # already the bare hex digest. The earlier ``[len(prefix):]`` slice + # truncated 7 chars off a 64-char SHA-256 digest, making + # ``compare_digest`` always return False — webhook HMAC verification + # was effectively broken (audit §4.3 / SPEC §15.2). + return hmac.compare_digest(signature, expected_signature) except Exception: return False diff --git a/jvspatial/core/context.py b/jvspatial/core/context.py index 11b1eb8..4a3a72c 100644 --- a/jvspatial/core/context.py +++ b/jvspatial/core/context.py @@ -741,23 +741,30 @@ async def save( else: # For Objects, export returns nested format (id, entity, context) record = await entity.export() - # Ensure ID follows proper format: type_code.ClassName.hex_id - # Check entity's ID directly (it's transient so not in export) + # Ensure ID follows proper format: type_code.entity_name.hex_id + # entity_name honors ``__entity_name__`` override (SPEC §1.2) + # so classes that disambiguate from a same-``__name__`` peer + # round-trip through save/load without ID rewriting. entity_id = getattr(entity, "id", None) if entity_id: id_parts = entity_id.split(".") - # Check if ID matches expected format: type_code.ClassName.hex_id + entity_name_resolver = getattr( + entity.__class__, "_entity_name", None + ) + expected_entity_name = ( + entity_name_resolver() + if callable(entity_name_resolver) + else entity.__class__.__name__ + ) if ( len(id_parts) != 3 or id_parts[0] != entity.type_code - or id_parts[1] != entity.__class__.__name__ + or id_parts[1] != expected_entity_name ): # ID doesn't match expected format, regenerate it from jvspatial.core.utils import generate_id - new_id = generate_id( - entity.type_code, entity.__class__.__name__ - ) + new_id = generate_id(entity.type_code, expected_entity_name) object.__setattr__(entity, "id", new_id) record["id"] = new_id # ID is in record from export() @@ -1398,9 +1405,16 @@ async def find_edges_between( if target_id is not None: query["target"] = target_id - # Filter by edge class if specified + # Filter by edge class if specified — honor ``__entity_name__`` + # override so subclasses persisted under a custom discriminator are + # found (SPEC §1.2). if edge_class: - query["entity"] = edge_class.__name__ + entity_name_resolver = getattr(edge_class, "_entity_name", None) + query["entity"] = ( + entity_name_resolver() + if callable(entity_name_resolver) + else edge_class.__name__ + ) # Add additional property filters for key, value in kwargs.items(): diff --git a/jvspatial/core/entities/node.py b/jvspatial/core/entities/node.py index cad6351..429fd76 100644 --- a/jvspatial/core/entities/node.py +++ b/jvspatial/core/entities/node.py @@ -423,7 +423,10 @@ async def count_neighbors( if isinstance(node, str): entity_name = node elif isinstance(node, type): - entity_name = node.__name__ + # Honor ``__entity_name__`` so subclasses with custom + # discriminators match their persisted ID prefix. + resolver = getattr(node, "_entity_name", None) + entity_name = resolver() if callable(resolver) else node.__name__ if entity_name is not None: try: from ..context import get_default_context @@ -558,7 +561,12 @@ async def _node_query( "$or": [{"source": self.id}, {"target": self.id}] } if isinstance(edge_filter, type): - edge_query["name"] = edge_filter.__name__ + # Persisted discriminator field is ``entity``; honor + # ``__entity_name__`` override (SPEC §1.2). + resolver = getattr(edge_filter, "_entity_name", None) + edge_query["entity"] = ( + resolver() if callable(resolver) else edge_filter.__name__ + ) # Cap edge fan-out so hub nodes don't accidentally load unbounded sets. edge_results = await context.database.find( @@ -590,7 +598,10 @@ async def _node_query( # Find incoming edges (where this node is the target) query: Dict[str, Any] = {"target": self.id} if isinstance(edge_filter, type): - query["name"] = edge_filter.__name__ + resolver = getattr(edge_filter, "_entity_name", None) + query["entity"] = ( + resolver() if callable(resolver) else edge_filter.__name__ + ) edge_results = await context.database.find("edge", query) for edge_data in edge_results: @@ -663,9 +674,17 @@ def _matches_node_filter( Returns: True if node matches the filter """ + # ``_entity_name()`` honors the ``__entity_name__`` override + # (SPEC §1.2). Falls back to ``__name__`` if absent so this helper + # works on non-Object types passed by mistake. + resolver = getattr(node_obj.__class__, "_entity_name", None) + obj_entity_name = ( + resolver() if callable(resolver) else node_obj.__class__.__name__ + ) + if isinstance(node_filter, str): - # Simple string filter - match by class name - return node_obj.__class__.__name__ == node_filter + # Simple string filter - match by entity_name + return obj_entity_name == node_filter elif isinstance(node_filter, type): # Class type filter - match by class or inheritance @@ -674,18 +693,18 @@ def _matches_node_filter( elif isinstance(node_filter, list): for filter_item in node_filter: if isinstance(filter_item, str): - # String in list - match by class name - if node_obj.__class__.__name__ == filter_item: + # String in list - match by entity_name + if obj_entity_name == filter_item: return True elif isinstance(filter_item, type): # Class type in list - match by class or inheritance if isinstance(node_obj, filter_item): return True elif isinstance(filter_item, dict): - # Dict filter - match by class name and criteria + # Dict filter - match by entity_name and criteria for class_name, criteria in filter_item.items(): if ( - node_obj.__class__.__name__ == class_name + obj_entity_name == class_name and self._matches_property_filter(node_obj, criteria) ): return True diff --git a/jvspatial/core/entities/walker.py b/jvspatial/core/entities/walker.py index ffe3584..2ef0732 100644 --- a/jvspatial/core/entities/walker.py +++ b/jvspatial/core/entities/walker.py @@ -115,6 +115,21 @@ class Walker(AttributeMixin, BaseModel): """ model_config = ConfigDict(extra="allow") + + # Per-class override of the persisted ``entity`` discriminator. Mirrors + # ``Object.__entity_name__`` so Walker subclasses can disambiguate from + # same-``__name__`` peers without subclassing Object. + __entity_name__: ClassVar[Optional[str]] = None + + @classmethod + def _entity_name(cls) -> str: + """Return the persisted entity discriminator for this walker class. + + Honors ``__entity_name__`` when set directly on the class (not + inherited), otherwise falls back to ``cls.__name__``. + """ + return cls.__dict__.get("__entity_name__") or cls.__name__ + type_code: str = attribute(transient=True, default="w") id: str = attribute( protected=True, transient=True, description="Unique identifier for the walker" @@ -137,7 +152,9 @@ class Walker(AttributeMixin, BaseModel): ] = {} _paused: bool = attribute(private=True, default=False) - # Trail tracking + # Trail tracking. Default factory creates an unbounded trail; Walker + # ``__init__`` rebuilds with ``max_trail_length`` from kwargs/env so + # SPEC §6.4 ``max_trail_length`` is actually wired (audit §2.3). _trail_tracker: WalkerTrail = attribute(private=True, default_factory=WalkerTrail) # Trail-related methods (sync for pure computations, async for I/O) @@ -302,13 +319,16 @@ def _get_edge_class(cls) -> Type["Edge"]: def __init__(self: "Walker", **kwargs: Any) -> None: """Initialize a walker with auto-generated ID if not provided.""" + entity_name = self.__class__._entity_name() if "id" not in kwargs: # Use class-level type_code or default from Field type_code = kwargs.get("type_code", "w") - kwargs["id"] = generate_id(type_code, self.__class__.__name__) - # Set entity to class name if not provided (protected attribute) + kwargs["id"] = generate_id(type_code, entity_name) + # Set entity to class entity_name if not provided (protected attribute). + # Honors ``__entity_name__`` override so subclasses with the same + # Python ``__name__`` as a sibling persist a distinct discriminator. if "entity" not in kwargs: - kwargs["entity"] = self.__class__.__name__ + kwargs["entity"] = entity_name # Extract component configuration from kwargs (before super().__init__) from jvspatial.env import env, parse_bool_basic @@ -328,6 +348,12 @@ def __init__(self: "Walker", **kwargs: Any) -> None: "max_queue_size", int(env("JVSPATIAL_WALKER_MAX_QUEUE_SIZE", default="1000")), ) + # 0 = unlimited (SPEC §6.4). Bounds the in-memory trail so long + # traversals cannot blow memory (audit §2.3). + max_trail_length = kwargs.pop( + "max_trail_length", + int(env("JVSPATIAL_WALKER_MAX_TRAIL_LENGTH", default="0")), + ) paused = kwargs.pop("paused", False) super().__init__(**kwargs) @@ -347,7 +373,9 @@ def __init__(self: "Walker", **kwargs: Any) -> None: # Initialize composition components self._queue: deque[Any] = deque() # Create new deque for queue manager self.queue = WalkerQueue(backing_deque=self._queue, max_size=max_queue_size) - # Trail is initialized as _trail_tracker class attribute + # Replace default unbounded trail with a (possibly bounded) one so + # ``max_trail_length`` from kwargs/env is honored (SPEC §6.4). + self._trail_tracker = WalkerTrail(max_length=max_trail_length) self._protection = TraversalProtection( max_steps=max_steps, max_visits_per_node=max_visits_per_node, @@ -647,8 +675,19 @@ async def run(self: "Walker") -> List[Any]: Returns: List of all reported items """ - # Reset protection state - await self._protection.reset() + # Initialize protection state once per traversal session. + # ``start_if_needed`` is idempotent so ``resume()`` does not reset + # the step / visit / timer counters (audit §2.2 / SPEC §6.3). + await self._protection.start_if_needed() + + # Late import to avoid circular dependency at module load. + from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, + ) + + from .walker_components.protection import ProtectionViolation # Process queue until empty or paused while self.queue and not self._paused: @@ -681,8 +720,33 @@ async def run(self: "Walker") -> List[Any]: # Execute visit hooks await self._execute_visit_hooks(current) + except ProtectionViolation as pv: + # SPEC §6.3 / §17: surface protection violations as the + # documented exception types instead of swallowing them + # into the walker report (audit §2.1). + walker_class = type(self).__name__ + ptype = pv.protection_type + if ptype == "max_visits_per_node": + raise InfiniteLoopError( + walker_class=walker_class, + node_id=pv.details.get("node_id", ""), + visit_count=pv.details.get("visit_count", 0), + ) from pv + if ptype == "timeout": + raise WalkerTimeoutError( + walker_class=walker_class, + timeout_seconds=pv.details.get( + "max_execution_time", self._max_execution_time + ), + ) from pv + # ``max_steps`` and any future protection types. + raise WalkerExecutionError( + walker_class=walker_class, + reason=f"protection_triggered: {ptype}", + details=pv.details, + ) from pv except Exception as e: - # Handle errors gracefully + # Handle other (non-protection) errors gracefully. await self.report(f"Error during traversal: {e}") break diff --git a/jvspatial/core/entities/walker_components/protection.py b/jvspatial/core/entities/walker_components/protection.py index 6b230ac..147d549 100644 --- a/jvspatial/core/entities/walker_components/protection.py +++ b/jvspatial/core/entities/walker_components/protection.py @@ -53,6 +53,10 @@ def __init__( # Track visit violations for O(1) check_limits() - set when any node # reaches max_visits_per_node (before raising exception) self._visit_violation_detected: bool = False + # Whether the traversal session has begun. ``start_if_needed`` flips + # this once per session so ``resume()`` cannot reset step / visit / + # timer counters (audit §2.2 / SPEC §6.3). + self._started: bool = False async def increment_step(self) -> None: """Increment the step counter and check protection limits.""" @@ -116,11 +120,30 @@ def _check_timeout(self) -> None: ) async def reset(self) -> None: - """Reset protection state and start timing.""" + """Reset protection state and start timing. + + Explicit reset for callers that want to restart a traversal from + scratch. ``Walker.run()`` should call :meth:`start_if_needed` + instead — calling ``reset`` on every ``run`` (including from + ``resume()``) lets pause/resume cycle around the configured limits + (audit §2.2 / SPEC §6.3). + """ self._steps = 0 self._visit_counts.clear() self._visit_violation_detected = False self._start_time = time.time() + self._started = True + + async def start_if_needed(self) -> None: + """Initialize the timer / counters on first call, then no-op. + + Idempotent across ``Walker.run()`` invocations so ``resume()`` does + not wipe accumulated step / visit / timer state. Callers wanting a + fresh session must call :meth:`reset` explicitly. + """ + if self._started: + return + await self.reset() @property def step_count(self) -> int: diff --git a/jvspatial/core/entities/walker_components/walker_queue.py b/jvspatial/core/entities/walker_components/walker_queue.py index e3e8cad..917609a 100644 --- a/jvspatial/core/entities/walker_components/walker_queue.py +++ b/jvspatial/core/entities/walker_components/walker_queue.py @@ -6,9 +6,12 @@ from __future__ import annotations +import logging from collections import deque from typing import Deque, Iterable, List, Optional +logger = logging.getLogger(__name__) + class WalkerQueue: """Manages walker traversal queue operations.""" @@ -27,13 +30,35 @@ def __init__( backing_deque if backing_deque is not None else deque() ) self._max_size = max_size + # One-shot warning when ``max_size`` is hit — repeated drops are + # rate-limited so a runaway producer cannot flood logs. + self._drop_warned: bool = False + + def _has_capacity(self) -> bool: + return self._max_size <= 0 or len(self._backing) < self._max_size + + def _warn_drop(self, op: str, count: int) -> None: + """Warn once per queue when ``max_size`` causes drops (SPEC §6.3).""" + if not self._drop_warned: + logger.warning( + "WalkerQueue.%s: queue at max_size=%d, dropped %d node(s); " + "subsequent drops will be silent.", + op, + self._max_size, + count, + ) + self._drop_warned = True async def visit(self, nodes: Iterable[object]) -> None: """Add nodes to queue, respecting max_size limit.""" + dropped = 0 for n in nodes: - # Only add if we haven't hit the limit - if self._max_size <= 0 or len(self._backing) < self._max_size: + if self._has_capacity(): self._backing.append(n) + else: + dropped += 1 + if dropped: + self._warn_drop("visit", dropped) async def dequeue(self, nodes: object) -> List[object]: """Remove specified node(s) from the queue. @@ -67,11 +92,21 @@ async def dequeue(self, nodes: object) -> List[object]: async def prepend(self, nodes: Iterable[object]) -> None: """Add nodes to the front of the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3) — earlier versions + of this method bypassed the cap, providing a silent protection + bypass via front-of-queue inserts. + Args: nodes: Nodes to prepend to the queue """ + dropped = 0 for n in reversed(list(nodes)): - self._backing.appendleft(n) + if self._has_capacity(): + self._backing.appendleft(n) + else: + dropped += 1 + if dropped: + self._warn_drop("prepend", dropped) async def append(self, nodes: Iterable[object]) -> None: """Add nodes to the end of the queue. @@ -79,19 +114,31 @@ async def append(self, nodes: Iterable[object]) -> None: Args: nodes: Nodes to append to the queue """ + dropped = 0 for n in nodes: - # Only add if we haven't hit the limit - if self._max_size <= 0 or len(self._backing) < self._max_size: + if self._has_capacity(): self._backing.append(n) + else: + dropped += 1 + if dropped: + self._warn_drop("append", dropped) async def add_next(self, nodes: Iterable[object]) -> None: """Add nodes to the front of the queue in the order provided. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). + Args: nodes: Nodes to add to the front of the queue """ + dropped = 0 for n in reversed(list(nodes)): - self._backing.appendleft(n) + if self._has_capacity(): + self._backing.appendleft(n) + else: + dropped += 1 + if dropped: + self._warn_drop("add_next", dropped) async def clear(self) -> None: """Clear all nodes from the queue.""" @@ -137,12 +184,15 @@ async def insert_after( ) -> List[object]: """Insert nodes after a target node in the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). Excess nodes are + dropped from the tail of the input rather than inserted. + Args: target_node: The node to insert after nodes: Nodes to insert Returns: - List of inserted nodes + List of nodes that were actually inserted Raises: ValueError: If target node is not found in the queue @@ -157,23 +207,32 @@ async def insert_after( except ValueError: raise ValueError(f"Target node {target_node} not found in queue") - # Insert after the target - for i, node in enumerate(nodes_list): - self._backing.insert(target_index + 1 + i, node) + inserted: List[object] = [] + dropped = 0 + for node in nodes_list: + if self._has_capacity(): + self._backing.insert(target_index + 1 + len(inserted), node) + inserted.append(node) + else: + dropped += 1 + if dropped: + self._warn_drop("insert_after", dropped) - return nodes_list + return inserted async def insert_before( self, target_node: object, nodes: Iterable[object] ) -> List[object]: """Insert nodes before a target node in the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). + Args: target_node: The node to insert before nodes: Nodes to insert Returns: - List of inserted nodes + List of nodes that were actually inserted Raises: ValueError: If target node is not found in the queue @@ -188,8 +247,15 @@ async def insert_before( except ValueError: raise ValueError(f"Target node {target_node} not found in queue") - # Insert before the target - for i, node in enumerate(nodes_list): - self._backing.insert(target_index + i, node) - - return nodes_list + inserted: List[object] = [] + dropped = 0 + for node in nodes_list: + if self._has_capacity(): + self._backing.insert(target_index + len(inserted), node) + inserted.append(node) + else: + dropped += 1 + if dropped: + self._warn_drop("insert_before", dropped) + + return inserted diff --git a/jvspatial/core/entities/walker_components/walker_trail.py b/jvspatial/core/entities/walker_components/walker_trail.py index fb11d1c..3fdda51 100644 --- a/jvspatial/core/entities/walker_components/walker_trail.py +++ b/jvspatial/core/entities/walker_components/walker_trail.py @@ -6,15 +6,31 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from collections import deque +from typing import Any, Deque, Dict, List, Optional class WalkerTrail: - """Tracks traversal steps and metadata.""" + """Tracks traversal steps and metadata. - def __init__(self) -> None: - """Initialize the trail tracker.""" - self._trail: List[Dict[str, Any]] = [] + ``max_length`` bounds the trail so long traversals cannot blow memory + (SPEC §6.4 — ``0`` means unlimited, the documented Walker contract). + Older steps are dropped from the front when the bound is hit. + """ + + def __init__(self, max_length: int = 0) -> None: + """Initialize the trail tracker. + + Args: + max_length: Maximum number of steps retained. ``0`` (default) + means unlimited. Use a positive integer to cap memory on + long-running traversals. + """ + self._max_length = max(0, int(max_length)) + # ``maxlen=None`` makes the deque unbounded. Annotating in one place + # so mypy knows the element type regardless of which branch ran. + bound: Optional[int] = self._max_length if self._max_length > 0 else None + self._trail: Deque[Dict[str, Any]] = deque(maxlen=bound) def record_step( self, node_id: Any, edge_id: Optional[Any] = None, **metadata: Any @@ -41,7 +57,9 @@ async def get_recent(self, count: int = 5) -> List[str]: """Get most recent node IDs from trail.""" if count <= 0: return [] - return [step["node"] for step in self._trail[-count:]] + # ``deque`` does not support slice access; materialize once. + trail_list = list(self._trail) + return [step["node"] for step in trail_list[-count:]] def get_length(self) -> int: """Get trail length.""" diff --git a/jvspatial/core/graph.py b/jvspatial/core/graph.py index f035e59..5f75f30 100644 --- a/jvspatial/core/graph.py +++ b/jvspatial/core/graph.py @@ -169,9 +169,12 @@ async def generate_graph_dot( # Optionally save to file if output_file: + import asyncio from pathlib import Path - Path(output_file).write_text(result, encoding="utf-8") + # ``write_text`` is sync I/O and blocks the event loop in an async + # function (audit §3.4 / SPEC §3.3). + await asyncio.to_thread(Path(output_file).write_text, result, encoding="utf-8") return result @@ -327,9 +330,12 @@ async def generate_graph_mermaid( # Optionally save to file if output_file: + import asyncio from pathlib import Path - Path(output_file).write_text(result, encoding="utf-8") + # Same blocking-write issue as generate_graph_dot + # (audit §3.5 / SPEC §3.3). + await asyncio.to_thread(Path(output_file).write_text, result, encoding="utf-8") return result diff --git a/jvspatial/db/dynamodb.py b/jvspatial/db/dynamodb.py index fbff8d7..2e1b0d0 100644 --- a/jvspatial/db/dynamodb.py +++ b/jvspatial/db/dynamodb.py @@ -16,6 +16,7 @@ import asyncio import json import logging +from functools import partial from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union try: @@ -637,8 +638,13 @@ async def process_batch(batch_ids: List[str]) -> List[Dict[str, Any]]: } try: - # Execute batch get - response = await client.batch_get_item(RequestItems=request_items) + # Throttle-retry wraps the wire call so transient + # ProvisionedThroughputExceeded / ThrottlingException are + # backed off (audit §5.1 / SPEC §4.3). + response = await self._run_with_throttle_retry( + "batch_get_item", + lambda: client.batch_get_item(RequestItems=request_items), + ) # Process responses items = response.get("Responses", {}).get(table_name, []) @@ -653,10 +659,11 @@ async def process_batch(batch_ids: List[str]) -> List[Dict[str, Any]]: logger.debug( f"Unprocessed keys in batch_get for collection '{collection}': {len(unprocessed.get(table_name, {}).get('Keys', []))}" ) - # Retry unprocessed keys once + # Retry unprocessed keys once (also throttle-aware). retry_items = {table_name: unprocessed[table_name]} - retry_response = await client.batch_get_item( - RequestItems=retry_items + retry_response = await self._run_with_throttle_retry( + "batch_get_item_retry", + lambda: client.batch_get_item(RequestItems=retry_items), ) retry_items_list = retry_response.get("Responses", {}).get( table_name, [] @@ -739,8 +746,11 @@ async def process_batch(batch_items: List[Dict[str, Any]]) -> None: request_items = {table_name: write_requests} try: - # Execute batch write - response = await client.batch_write_item(RequestItems=request_items) + # Throttle-retry wraps the wire call (audit §5.1 / SPEC §4.3). + response = await self._run_with_throttle_retry( + "batch_write_item", + lambda: client.batch_write_item(RequestItems=request_items), + ) # Handle unprocessed items with retry logic unprocessed = response.get("UnprocessedItems", {}) @@ -756,8 +766,13 @@ async def process_batch(batch_items: List[Dict[str, Any]]) -> None: # Wait before retry (exponential backoff) await asyncio.sleep(0.1 * (2 ** (retry_count - 1))) - retry_response = await client.batch_write_item( - RequestItems=unprocessed + # ``partial`` snapshots kwargs at construction time, so + # the closure does not refer to the loop variable + # ``unprocessed`` (flake8-bugbear B023). The retry + # helper awaits the resulting coroutine inline. + retry_response = await self._run_with_throttle_retry( + "batch_write_item_retry", + partial(client.batch_write_item, RequestItems=unprocessed), ) unprocessed = retry_response.get("UnprocessedItems", {}) @@ -993,7 +1008,10 @@ async def count( total = 0 while True: - resp = await client.query(**params) + resp = await self._run_with_throttle_retry( + "count_query", + lambda: client.query(**params), + ) total += int(resp.get("Count", 0)) if "LastEvaluatedKey" not in resp: break @@ -1042,7 +1060,10 @@ async def count( } total = 0 while True: - resp = await client.scan(**scan_params) + resp = await self._run_with_throttle_retry( + "count_scan", + lambda: client.scan(**scan_params), + ) total += int(resp.get("Count", 0)) if "LastEvaluatedKey" not in resp: break @@ -1146,7 +1167,10 @@ async def find( if filter_expr: query_params["FilterExpression"] = filter_expr - response = await client.query(**query_params) + response = await self._run_with_throttle_retry( + "find_query", + lambda: client.query(**query_params), + ) results = [] for item in response.get("Items", []): @@ -1170,7 +1194,10 @@ async def find( query_params["ExclusiveStartKey"] = response["LastEvaluatedKey"] if fetch_limit: query_params["Limit"] = fetch_limit - len(results) - response = await client.query(**query_params) + response = await self._run_with_throttle_retry( + "find_query", + lambda: client.query(**query_params), + ) for item in response.get("Items", []): data = json.loads(item["data"]["S"]) @@ -1226,7 +1253,10 @@ async def find( if fetch_limit: scan_params["Limit"] = fetch_limit - response = await client.scan(**scan_params) + response = await self._run_with_throttle_retry( + "find_scan", + lambda: client.scan(**scan_params), + ) results = [] for item in response.get("Items", []): @@ -1247,7 +1277,10 @@ async def find( scan_params["ExclusiveStartKey"] = response["LastEvaluatedKey"] if fetch_limit: scan_params["Limit"] = fetch_limit - len(results) - response = await client.scan(**scan_params) + response = await self._run_with_throttle_retry( + "find_scan", + lambda: client.scan(**scan_params), + ) for item in response.get("Items", []): data = json.loads(item["data"]["S"]) diff --git a/jvspatial/storage/interfaces/local.py b/jvspatial/storage/interfaces/local.py index 6bbc9b6..b0e4d28 100644 --- a/jvspatial/storage/interfaces/local.py +++ b/jvspatial/storage/interfaces/local.py @@ -159,6 +159,30 @@ def _get_full_path(self, file_path: str) -> Path: return full_path + def _sanitized_version_base(self, file_path: str) -> Path: + """Return a sandboxed base path for versioning helpers. + + ``create_version`` / ``get_version`` / ``list_versions`` / + ``delete_version`` / ``get_latest_version`` all derive ``.versions`` + and ``.latest`` paths from ``file_path``. Without sanitization here, + a caller-supplied ``file_path`` like ``../../etc/passwd`` escapes + ``self.root_dir`` (audit §4.2 / SPEC §15.1). Centralizing the check + in one helper guarantees uniform coverage across every versioning + entry point. + """ + sanitized = PathSanitizer.sanitize_path(file_path, base_dir=str(self.root_dir)) + base = self.root_dir / sanitized + try: + # ``resolve(strict=False)`` traverses any existing symlinks but + # tolerates non-existent leaves (versioned bases often don't + # exist yet). The ``relative_to`` check then enforces the root. + resolved = base.resolve() + resolved.relative_to(self.root_dir) + except (ValueError, RuntimeError): + logger.error(f"Version path escape attempt blocked: {file_path}") + raise PathTraversalError("Path escapes root directory", path=file_path) + return base + def _http_file_url(self, file_path: str) -> str: """Full URL for HTTP GET ``{FILES_ROOT}/{file_path}`` (FileStorageService).""" return f"{self.base_url.rstrip('/')}{APIRoutes.FILES_ROOT}/{file_path}" @@ -203,8 +227,9 @@ async def create_version( if version is None: version = f"v{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" - # Create version directory (use .versions suffix to avoid conflicts) - version_dir = self.root_dir / f"{file_path}.versions" + # Sanitize ``file_path`` and derive sandboxed version paths. + version_base = self._sanitized_version_base(file_path) + version_dir = version_base.with_name(version_base.name + ".versions") await to_thread(version_dir.mkdir, parents=True, exist_ok=True) version_metadata = { @@ -217,7 +242,7 @@ async def create_version( version_file = version_dir / f"{version}.bin" metadata_file = version_dir / f"{version}.meta.json" - latest_file = self.root_dir / f"{file_path}.latest" + latest_file = version_base.with_name(version_base.name + ".latest") # 1) Write content atomically (fully durable). await to_thread(atomic_write_bytes, version_file, content) @@ -249,7 +274,9 @@ async def get_version(self, file_path: str, version: str) -> Dict[str, Any]: Returns: Dictionary with version information and content """ - version_file = self.root_dir / f"{file_path}.versions" / f"{version}.bin" + version_base = self._sanitized_version_base(file_path) + version_dir = version_base.with_name(version_base.name + ".versions") + version_file = version_dir / f"{version}.bin" if not await to_thread(version_file.exists): return None @@ -257,7 +284,7 @@ async def get_version(self, file_path: str, version: str) -> Dict[str, Any]: content = await to_thread(version_file.read_bytes) # Try to get metadata - metadata_file = self.root_dir / f"{file_path}.versions" / f"{version}.meta.json" + metadata_file = version_dir / f"{version}.meta.json" metadata = {} if await to_thread(metadata_file.exists): try: @@ -285,7 +312,8 @@ async def list_versions(self, file_path: str) -> List[Dict[str, Any]]: Returns: List of version information dictionaries """ - versions_dir = self.root_dir / f"{file_path}.versions" + version_base = self._sanitized_version_base(file_path) + versions_dir = version_base.with_name(version_base.name + ".versions") if not await to_thread(versions_dir.exists): return [] @@ -324,7 +352,8 @@ async def delete_version(self, file_path: str, version: str) -> bool: Returns: True if version was deleted, False otherwise """ - versions_dir = self.root_dir / f"{file_path}.versions" + version_base = self._sanitized_version_base(file_path) + versions_dir = version_base.with_name(version_base.name + ".versions") if not await to_thread(versions_dir.exists): return False @@ -353,7 +382,8 @@ async def get_latest_version(self, file_path: str) -> Optional[Dict[str, Any]]: Returns: Latest version information dictionary, or None if no versions exist """ - latest_file = self.root_dir / f"{file_path}.latest" + version_base = self._sanitized_version_base(file_path) + latest_file = version_base.with_name(version_base.name + ".latest") if not await to_thread(latest_file.exists): return None diff --git a/tests/api/test_webhook_hmac_audit_fix.py b/tests/api/test_webhook_hmac_audit_fix.py new file mode 100644 index 0000000..ebbe117 --- /dev/null +++ b/tests/api/test_webhook_hmac_audit_fix.py @@ -0,0 +1,47 @@ +"""HMAC verification regression coverage (audit §4.3 / SPEC §15.2). + +``verify_hmac_signature`` previously sliced 7 chars off +``expected_signature`` before comparison, while the incoming signature +had already had its prefix stripped. The mismatched lengths fed into +``hmac.compare_digest`` always returned False — webhook signature auth +rejected every request. +""" + +import pytest + +from jvspatial.api.integrations.webhooks.utils import ( + generate_hmac_signature, + verify_hmac_signature, +) + + +def test_valid_hmac_with_default_prefix_verifies(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + signature_with_prefix = generate_hmac_signature(payload, secret) + assert signature_with_prefix.startswith("sha256=") + + assert verify_hmac_signature(payload, signature_with_prefix, secret) is True + + +def test_valid_hmac_without_prefix_verifies(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + bare = generate_hmac_signature(payload, secret, prefix="") + + assert verify_hmac_signature(payload, bare, secret, prefix="") is True + + +def test_invalid_hmac_rejects(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + # Tampered signature + bad = "sha256=" + ("0" * 64) + assert verify_hmac_signature(payload, bad, secret) is False + + +def test_empty_signature_or_secret_rejects(): + assert ( + verify_hmac_signature(b"x", "", "secret") is False + ) # pragma: allowlist secret + assert verify_hmac_signature(b"x", "sha256=abc", "") is False diff --git a/tests/core/test_entity_name_walker_and_save.py b/tests/core/test_entity_name_walker_and_save.py new file mode 100644 index 0000000..e0d9f63 --- /dev/null +++ b/tests/core/test_entity_name_walker_and_save.py @@ -0,0 +1,114 @@ +"""Walker and save-cycle coverage for ``__entity_name__`` override (audit §1). + +Walker now ships its own ``__entity_name__`` classmethod parallel to +``Object._entity_name`` so walker IDs and the persisted ``entity`` field +honor the per-subclass discriminator. The previous behavior used +``cls.__name__`` unconditionally, which broke the override for walkers +entirely. + +``GraphContext.save_object`` previously regenerated entity IDs when +``id_parts[1] != cls.__name__`` — that check used ``__name__`` instead of +``_entity_name()``, so every save of an override-using class rewrote the +ID with the wrong discriminator. The fix uses ``_entity_name()`` so saves +are stable. + +``GraphContext.find_edges_between`` previously queried +``entity == edge_class.__name__``; an edge subclass with +``__entity_name__`` would never match its own rows. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.core.context import GraphContext +from jvspatial.core.entities import Edge, Node, Walker +from jvspatial.db import create_database + + +class CustomNamedWalker(Walker): + __entity_name__ = "FleetTraversalWalker" + + +class CustomNamedEdge(Edge): + __entity_name__ = "FleetLink" + weight: float = 1.0 + + +class FleetNode(Node): + label: str = "" + + +@pytest.fixture +def json_context(): + with tempfile.TemporaryDirectory() as tmpdir: + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + yield GraphContext(database=database) + + +def test_walker_id_honors_entity_name_override(): + walker = CustomNamedWalker() + # ID prefix and persisted entity must reflect the override, not the + # Python class name. + assert walker.id.startswith("w.FleetTraversalWalker.") + assert walker.entity == "FleetTraversalWalker" + + +def test_walker_entity_name_classmethod(): + assert CustomNamedWalker._entity_name() == "FleetTraversalWalker" + + # Per-subclass — not inherited from a parent that happens to override. + class GrandchildWalker(CustomNamedWalker): + pass + + assert GrandchildWalker._entity_name() == "GrandchildWalker" + + +@pytest.mark.asyncio +async def test_save_does_not_rewrite_override_ids(json_context): + """``GraphContext.save_object`` must not regenerate IDs that use the + override. + + Before the §1 fix, the ID-validation check at context.py:753 compared + against ``__name__``, so any class with ``__entity_name__`` had its + ID rewritten on every save — silent data corruption. + """ + + class WidgetNode(Node): + __entity_name__ = "MarketplaceWidget" + name: str = "" + + n = WidgetNode(name="alpha") + original_id = n.id + assert original_id.startswith("n.MarketplaceWidget.") + + await json_context.save(n) + # ID is unchanged after save. + assert n.id == original_id + + # And on a second save (covers the regeneration branch). + await json_context.save(n) + assert n.id == original_id + + +@pytest.mark.asyncio +async def test_find_edges_between_honors_override(json_context): + """``find_edges_between`` builds an ``entity == edge_class._entity_name()`` + query rather than ``__name__`` so override edges are findable.""" + a = FleetNode(label="A") + b = FleetNode(label="B") + await json_context.save(a) + await json_context.save(b) + + edge = CustomNamedEdge(source=a.id, target=b.id, weight=2.0) + await json_context.save(edge) + assert edge.entity == "FleetLink" + assert edge.id.startswith("e.FleetLink.") + + found = await json_context.find_edges_between( + source_id=a.id, target_id=b.id, edge_class=CustomNamedEdge + ) + assert len(found) == 1 + assert found[0].id == edge.id diff --git a/tests/core/test_walker_protection.py b/tests/core/test_walker_protection.py index 8ed3dd4..2a941c0 100644 --- a/tests/core/test_walker_protection.py +++ b/tests/core/test_walker_protection.py @@ -27,6 +27,16 @@ Root, Walker, ) +from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, +) + +# Per SPEC §6.3 / audit §2.1: protection limits now raise the documented +# exception types instead of being swallowed into the walker report. Tests +# that intentionally trip a limit must wrap ``spawn`` in this tuple. +PROTECTION_EXCS = (WalkerExecutionError, InfiniteLoopError, WalkerTimeoutError) class ProtectionTestNode(Node): @@ -230,8 +240,9 @@ async def test_max_steps_protection_triggers(self): walker = StepCountTestWalker(max_steps=5) start_node = ProtectionTestNode(name="start", value=0) - # Run walker and expect protection to trigger - await walker.spawn(start_node) + # SPEC §6.3: exceeding max_steps raises WalkerExecutionError. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check protection was triggered by checking step count and limits # The protection system stops traversal when limits are exceeded @@ -250,8 +261,10 @@ async def test_max_steps_protection_disabled(self): walker = StepCountTestWalker(max_steps=50) start_node = ProtectionTestNode(name="start", value=0) - # Run walker - it will stop at 50 steps due to the walker's internal limit - await walker.spawn(start_node) + # Walker still hits the cap, but at 50 it is "effectively disabled" + # from the test's perspective; SPEC §6.3 says it still raises. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check that walker ran more than the original 5 steps # but still stopped due to protection at 50 @@ -270,7 +283,9 @@ async def test_step_counting_accuracy(self): walker = StepCountTestWalker(max_steps=10) start_node = ProtectionTestNode(name="start", value=0) - await walker.spawn(start_node) + # StepCountTestWalker is designed to trip max_steps. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check step counting matches expected # The Walker should have taken some steps and stopped @@ -291,7 +306,9 @@ async def test_max_visits_per_node_protection(self): walker = NodeRevisitWalker(max_visits_per_node=3) start_node = ProtectionTestNode(name="revisit_test", value=0) - await walker.spawn(start_node) + # SPEC §6.3: max_visits_per_node raises InfiniteLoopError. + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check protection was triggered by checking visit counts # The protection system stops traversal when limits are exceeded @@ -309,7 +326,8 @@ async def test_node_visit_counting(self): walker = NodeRevisitWalker(max_visits_per_node=5) start_node = ProtectionTestNode(name="count_test", value=0) - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check visit counting through protection component visit_counts = walker._protection.visit_counts @@ -334,7 +352,8 @@ async def visit_multiple_nodes(self, here): walker = MultiNodeWalker(max_steps=20) start_node = ProtectionTestNode(name="multi_start") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check that multiple nodes were tracked visit_counts = walker._protection.visit_counts @@ -397,7 +416,8 @@ async def test_queue_size_protection(self): ) start_node = ProtectionTestNode(name="queue_test") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check that protection triggered (should be max_steps since queue limiting doesn't stop traversal) # The protection system stops traversal when limits are exceeded @@ -426,7 +446,8 @@ async def test_queue_size_protection_disabled(self): start_node = ProtectionTestNode(name="queue_unlimited_test") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # With high queue limit, queue can grow larger # But step protection should still stop the walker @@ -493,8 +514,9 @@ async def test_protection_status_during_execution(self): initial_step_count = walker.step_count initial_queue_size = len(walker.queue) - # Run walker - await walker.spawn(start_node) + # Run walker (expected to trip max_steps). + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check final status final_step_count = walker.step_count @@ -534,7 +556,8 @@ async def test_protection_with_trail_tracking(self): ) start_node = ProtectionTestNode(name="trail_protection_test") - await walker.spawn(start_node) + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check both protection and trail worked # The protection system stops traversal when limits are exceeded @@ -570,7 +593,8 @@ async def set_response_data(self, here): walker = ResponseTestWalker(max_steps=5) start_node = ProtectionTestNode(name="response_test") - await walker.spawn(start_node) + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check protection data was added without overwriting custom data # The protection system stops traversal when limits are exceeded @@ -597,7 +621,10 @@ async def test_multiple_protection_triggers(self): ) start_node = ProtectionTestNode(name="multi_protection_test") - await walker.spawn(start_node) + # Any of the three limits can fire first — accept all three exception + # types (SPEC §6.3). + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # One of the protections should have triggered # The protection system stops traversal when limits are exceeded diff --git a/tests/core/test_walker_protection_audit_fixes.py b/tests/core/test_walker_protection_audit_fixes.py new file mode 100644 index 0000000..e89064c --- /dev/null +++ b/tests/core/test_walker_protection_audit_fixes.py @@ -0,0 +1,173 @@ +"""Regression coverage for Wave 1 walker-protection fixes (audit §2). + +The earlier implementation: + +* Swallowed every ``ProtectionViolation`` into ``walker.report``, so callers + never saw ``InfiniteLoopError`` / ``WalkerTimeoutError`` / ``WalkerExecutionError`` + even though SPEC §6.3 promised them. +* Called ``self._protection.reset()`` at the top of ``Walker.run()``. ``resume()`` + re-enters ``run()`` → resetting step / visit counters and the wall-clock timer. + Repeated pause/resume cycles cleared protection state, trivially defeating the + cap. +* Ignored ``max_trail_length`` — the docstring promised it, no code wired it. +* Allowed ``WalkerQueue.prepend`` / ``add_next`` / ``insert_after`` / ``insert_before`` + to grow past ``max_size``, providing a silent protection bypass. + +These tests pin the corrected behavior so future regressions are loud. +""" + +import asyncio + +import pytest + +from jvspatial.core.entities import Node, Walker +from jvspatial.core.entities.walker_components.protection import ( + ProtectionViolation, + TraversalProtection, +) +from jvspatial.core.entities.walker_components.walker_queue import WalkerQueue +from jvspatial.core.entities.walker_components.walker_trail import WalkerTrail +from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, +) + +# ---------- TraversalProtection ---------- + + +@pytest.mark.asyncio +async def test_start_if_needed_is_idempotent(): + prot = TraversalProtection(max_steps=10) + await prot.start_if_needed() + first_start = prot._start_time + await prot.increment_step() + await prot.increment_step() + assert prot.step_count == 2 + + # Calling start_if_needed again must NOT reset counters or restart the + # wall-clock timer. Previously ``run()`` called ``reset()`` which did. + await prot.start_if_needed() + assert prot.step_count == 2 + assert prot._start_time == first_start + + +@pytest.mark.asyncio +async def test_reset_is_still_available_for_explicit_restart(): + prot = TraversalProtection(max_steps=10) + await prot.start_if_needed() + await prot.increment_step() + await prot.reset() + # Explicit reset starts a fresh session — start_if_needed becomes a no-op + # after reset because reset sets _started = True. + assert prot.step_count == 0 + + +# ---------- Walker.run() raises documented exceptions ---------- + + +class AlwaysReenqueueNode(Node): + name: str = "" + + +class TightLoopWalker(Walker): + """Visits the same node repeatedly so max_visits_per_node fires.""" + + async def visit(self, target): + await self.queue.append([target]) + + +@pytest.mark.asyncio +async def test_walker_run_raises_infinite_loop_error_on_visit_cap(): + node = AlwaysReenqueueNode(name="cycle") + walker = TightLoopWalker(max_visits_per_node=3) + # Seed the queue with the same node many times so record_visit hits the cap. + await walker.queue.append([node] * 20) + + with pytest.raises(InfiniteLoopError) as exc_info: + await walker.run() + err = exc_info.value + assert err.node_id == node.id + assert err.walker_class == "TightLoopWalker" + + +@pytest.mark.asyncio +async def test_walker_run_raises_walker_execution_error_on_step_cap(): + walker = TightLoopWalker(max_steps=3, max_visits_per_node=10_000) + # Distinct nodes so visit-cap does not fire first. + nodes = [AlwaysReenqueueNode(name=f"n{i}") for i in range(20)] + await walker.queue.append(nodes) + + with pytest.raises(WalkerExecutionError) as exc_info: + await walker.run() + assert "max_steps" in exc_info.value.reason + + +# ---------- max_trail_length wired ---------- + + +def test_walker_trail_unbounded_by_default(): + trail = WalkerTrail() + for i in range(50): + trail.record_step(f"n.X.{i}") + assert trail.get_length() == 50 + + +def test_walker_trail_respects_max_length(): + trail = WalkerTrail(max_length=10) + for i in range(50): + trail.record_step(f"n.X.{i}") + # Only the most recent 10 retained. + assert trail.get_length() == 10 + most_recent_ids = [step["node"] for step in trail.get_trail()] + assert most_recent_ids[0] == "n.X.40" + assert most_recent_ids[-1] == "n.X.49" + + +def test_walker_propagates_max_trail_length(): + """Walker.__init__ pops ``max_trail_length`` and rebuilds the trail tracker.""" + walker = TightLoopWalker(max_trail_length=3) + for i in range(10): + walker._trail_tracker.record_step(f"n.X.{i}") + assert walker._trail_tracker.get_length() == 3 + + +# ---------- WalkerQueue inserts respect max_size ---------- + + +@pytest.mark.asyncio +async def test_prepend_respects_max_size(): + q = WalkerQueue(max_size=3) + await q.visit(["a", "b", "c"]) + # Past cap — prepend must drop, not silently grow. + await q.prepend(["x", "y"]) + assert len(q) == 3 + # Existing items preserved at the head's downstream side. + assert list(q.to_list()) == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_add_next_respects_max_size(): + q = WalkerQueue(max_size=2) + await q.visit(["a", "b"]) + await q.add_next(["x"]) + assert len(q) == 2 + + +@pytest.mark.asyncio +async def test_insert_after_respects_max_size_and_returns_inserted(): + q = WalkerQueue(max_size=3) + await q.visit(["a", "b", "c"]) + inserted = await q.insert_after("a", ["x", "y"]) + assert inserted == [] # No room. + assert len(q) == 3 + + +@pytest.mark.asyncio +async def test_insert_before_respects_max_size_and_returns_inserted(): + q = WalkerQueue(max_size=4) + await q.visit(["a", "b"]) + inserted = await q.insert_before("b", ["x", "y", "z"]) + # First two fit (filling the 4-slot cap), third is dropped. + assert inserted == ["x", "y"] + assert len(q) == 4 diff --git a/tests/core/test_walker_trail_new.py b/tests/core/test_walker_trail_new.py index ad4e23f..34402ab 100644 --- a/tests/core/test_walker_trail_new.py +++ b/tests/core/test_walker_trail_new.py @@ -368,15 +368,24 @@ async def test_trail_during_resume(self): @pytest.mark.asyncio async def test_trail_with_protection(self): - """Test trail recording with protection limits.""" + """Test trail recording with protection limits. + + SPEC §6.3 / audit §2.1: tripping ``max_steps`` now raises + ``WalkerExecutionError`` instead of being silently absorbed into the + walker report. The trail captured up to the violation is still + readable after the exception unwinds. + """ + from jvspatial.exceptions import WalkerExecutionError + walker = TrailTrackingWalker(max_steps=2) nodes = [TrailTestNode(name=f"node{i}") for i in range(5)] # Add nodes to queue await walker.visit(nodes) - # Spawn with protection should record trail steps - await walker.spawn(nodes[0]) + # Spawn with protection should record trail steps and then raise. + with pytest.raises(WalkerExecutionError): + await walker.spawn(nodes[0]) # Check that trail was recorded trail = walker.get_trail() diff --git a/tests/storage/test_versioning_path_sanitizer_audit.py b/tests/storage/test_versioning_path_sanitizer_audit.py new file mode 100644 index 0000000..a686721 --- /dev/null +++ b/tests/storage/test_versioning_path_sanitizer_audit.py @@ -0,0 +1,62 @@ +"""Path-traversal coverage for ``LocalFileInterface`` versioning methods. + +Audit §4.2 / SPEC §15.1: ``create_version`` / ``get_version`` / +``list_versions`` / ``delete_version`` / ``get_latest_version`` previously +computed ``self.root_dir / f"{file_path}.versions"`` without sanitizing +``file_path`` — a caller-supplied ``../../etc/passwd`` escaped the storage +root entirely. +""" + +import tempfile + +import pytest + +from jvspatial.storage import create_storage +from jvspatial.storage.exceptions import PathTraversalError + + +@pytest.fixture +def local_storage(): + with tempfile.TemporaryDirectory() as tmpdir: + storage = create_storage("local", root_dir=tmpdir) + yield storage + + +@pytest.mark.asyncio +async def test_create_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.create_version("../../etc/passwd", b"payload") + + +@pytest.mark.asyncio +async def test_get_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.get_version("../../etc/passwd", "v1") + + +@pytest.mark.asyncio +async def test_list_versions_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.list_versions("../../etc/passwd") + + +@pytest.mark.asyncio +async def test_delete_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.delete_version("../../etc/passwd", "v1") + + +@pytest.mark.asyncio +async def test_get_latest_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.get_latest_version("../../etc/passwd") + + +@pytest.mark.asyncio +async def test_create_version_then_get_round_trips_for_safe_path(local_storage): + """Sanitizer must not break the happy path — legitimate paths still work.""" + res = await local_storage.create_version("safe/file.txt", b"hello") + assert res["path"] == "safe/file.txt" + fetched = await local_storage.get_version("safe/file.txt", res["version_id"]) + assert fetched is not None + assert fetched["content"] == b"hello" From 495ccdd0ba830aadc55fd33cc35ab0e600bcbd18 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 10:42:19 -0400 Subject: [PATCH 10/22] docs(changelog): document Wave 1 audit remediation Adds detailed [Unreleased] entries for the 16 CRIT findings closed in the previous commit, including the breaking behavioral change in Walker.run() (protection limits now raise documented exception types instead of being silently swallowed) and the new public surface (Walker.__entity_name__, TraversalProtection.start_if_needed, WalkerTrail max_length, JVSPATIAL_WALKER_MAX_TRAIL_LENGTH env var). Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eadac4..b359197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `JVSPATIAL_DOCS_DISABLED` env var (truthy `1`/`true`/`yes`/`on`) — when set, `AppBuilder.create_app` constructs FastAPI with `docs_url=None`, `redoc_url=None`, `openapi_url=None`, and `swagger_ui_oauth2_redirect_url=None` so the documentation surface is fully unpublished (404 with no spec leak). Recommended for production. +- `Walker.__entity_name__` / `Walker._entity_name()` — parallel to `Object._entity_name()` so walker IDs and the persisted `entity` discriminator honor the per-subclass override. (Audit §1.1, §1.2, §1.9.) +- `TraversalProtection.start_if_needed()` — idempotent initializer for `Walker.run()`. Pause/resume cycles no longer reset step / visit / wall-clock counters. (Audit §2.2.) +- `WalkerTrail(max_length=N)` — wires the previously-undocumented bound. `0` (default) means unlimited; positive integers cap the in-memory trail. Threaded through `Walker(max_trail_length=...)` and `JVSPATIAL_WALKER_MAX_TRAIL_LENGTH`. (Audit §2.3 / SPEC §6.4.) +- `tests/core/test_entity_name_walker_and_save.py`, `tests/core/test_walker_protection_audit_fixes.py`, `tests/storage/test_versioning_path_sanitizer_audit.py`, `tests/api/test_webhook_hmac_audit_fix.py` — 28 new regression cases pinning Wave 1 audit fixes. ### Fixed +- **BREAKING (behavioral):** `Walker.run()` now raises `InfiniteLoopError` / `WalkerTimeoutError` / `WalkerExecutionError` when protection limits trip, as SPEC §6.3 has always promised. Earlier behavior silently swallowed `ProtectionViolation` into `walker.report` and returned. Callers that relied on the swallow contract must wrap `spawn()` / `run()` in `try`/`except`. (Audit §2.1.) +- `GraphContext.save_object` no longer rewrites entity IDs when `__entity_name__` differs from `cls.__name__`. Earlier the ID-validation check compared against `cls.__name__` and regenerated through `cls.__name__` — any entity using the override had its ID silently corrupted on every save. (Audit §1.3, §1.4.) +- `GraphContext.find_edges_between` honors `__entity_name__` so override-using edges are findable. Earlier filtered `entity == edge_class.__name__`. (Audit §1.5.) +- `Node._node_query` keys edge lookups by the persisted `entity` field (not the non-existent `name` column) and honors `__entity_name__`; `Node.count_neighbors` fast-path regex uses `_entity_name()`; `Node._matches_node_filter` compares against `_entity_name()`. (Audit §1.6-§1.8.) +- `AuthenticationService._verify_refresh_token` SHA-256 fallback now uses `hmac.compare_digest` instead of `==`. Removes a timing oracle on refresh-token and password-reset-token hash comparison. (Audit §4.1 / SPEC §15.2.) +- `LocalFileInterface.{create_version, get_version, list_versions, delete_version, get_latest_version}` route `file_path` through a new `_sanitized_version_base` helper. Earlier these computed `self.root_dir / f"{file_path}.versions"` without sanitization — a caller-supplied `../../etc/passwd` escaped the storage root entirely. (Audit §4.2 / SPEC §15.1.) +- `verify_hmac_signature` no longer slices `expected_signature[len(prefix):]`. The earlier slice truncated 7 chars off a 64-char SHA-256 digest so `hmac.compare_digest` always returned False — webhook HMAC verification rejected every request. (Audit §4.3 / SPEC §15.2.) +- Webhook walker `inject_walker_webhook_payload.enhanced_init` is now sync. Python ignores `async def __init__`; the earlier form returned a coroutine that was never awaited and leaked on every webhook walker construction. (Audit §3.1.) +- `webhook_wrapper` now awaits `endpoint_func(**kwargs)` in the async branch. Both arms of the `if/else` were identical, so coroutines from async endpoints leaked unawaited. (Audit §3.2.) +- `FileStorageService.handle_delete_file` now awaits `self.file_interface.delete_file(file_path)`. The missing `await` left `success` as the coroutine object and skipped the delete. (Audit §3.3.) +- `generate_graph_dot` and `generate_graph_mermaid` now wrap `Path.write_text` in `asyncio.to_thread` so disk I/O does not block the event loop inside their async bodies. (Audit §3.4-§3.5.) +- `WalkerQueue.prepend` / `append` / `add_next` / `insert_after` / `insert_before` now respect `max_size` and emit a one-shot WARNING on first drop. Earlier the front-of-queue and middle-insert paths bypassed the cap, providing a silent protection bypass. (Audit §2.4-§2.5.) +- `DynamoDB.{find, count, batch_get, batch_write}` now route every `aioboto3` wire call through `_run_with_throttle_retry`. Earlier the helper was only applied to `save`/`get`/`delete`; `ProvisionedThroughputExceededException` and `ThrottlingException` from scan / query / batch ops surfaced to callers as immediate failures despite the documented backoff. (Audit §5.1 / SPEC §4.3.) - Security headers middleware now emits a relaxed Content-Security-Policy on `/docs`, `/redoc`, `/openapi.json` (and sub-paths) that permits `cdn.jsdelivr.net` so FastAPI's bundled Swagger UI / ReDoc pages render. The previous strict default blocked the CDN-hosted JS/CSS and the docs loaded blank. Application routes keep the strict default policy. ## [0.0.7] - 2026-05-08 From 8eaa9930ae96dbcbe27d23b7f3a040eaa9f79fd7 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 11:43:19 -0400 Subject: [PATCH 11/22] =?UTF-8?q?fix:=20Wave=202=20audit=20remediation=20?= =?UTF-8?q?=E2=80=94=20async=20I/O,=20subclass=20chains,=20auth=20races,?= =?UTF-8?q?=20pager,=20=5Fid=20queries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses 11 HIGH-class findings from AUDIT.md (Wave 2). All changes SPEC-aligned; behavior matches the documented contract. --- Group A: JsonDB sync stats (audit §3.6) --- Six ``Path.exists()`` / ``Path.glob()`` calls inside ``async`` methods of JsonDB performed blocking syscalls and stalled the event loop. * db/jsondb.py — wrap stat/glob in ``asyncio.to_thread`` for ``_async_read_json``, ``count``, ``find_many``, ``find``. Extracted the glob-and-filter logic into a static ``_list_collection_json_files`` helper so the to_thread call sites stay readable. --- Group B: __init_subclass__ super() chains (audit §6.1-6.3) --- ``Node`` / ``Edge`` / ``Walker`` ``__init_subclass__`` did not call ``super().__init_subclass__``, so ``AttributeMixin.__init_subclass__`` never ran for their subclasses. ``protected`` / ``transient`` / ``private`` attribute registration was silently skipped. * core/entities/{node,edge,walker}.py — add ``super().__init_subclass__(**kwargs)`` at the top of each. --- Group C: auth races + API-key cache invalidation (audit §4.4-4.8) --- ``SessionManager._sessions`` / ``_user_sessions`` were mutated across ``await`` points without a lock — concurrent logout-vs-login raised ``RuntimeError: dictionary changed size during iteration`` and ``max_sessions_per_user`` enforcement was racy. The webhook-layer ``_API_KEY_CACHE`` had the same issue (``KeyError`` when a size-cap eviction raced a reader). ``APIKeyService`` defaulted to ``get_default_context()`` when constructed without a context, putting auth state on the wrong DB. ``APIKeyService.revoke_key`` did not invalidate the cache so revoked keys authenticated for up to the 300-second TTL. * api/auth/enhanced.py — single ``asyncio.Lock`` guards every ``SessionManager`` mutation. Cleanup helpers split into locked vs. unlocked variants. Per-user cap now enforced under the lock and evicts the oldest session by ``last_accessed`` when over the cap. * api/integrations/webhooks/webhook_auth.py — single ``_API_KEY_CACHE_LOCK`` guards reads, eviction, and miss-population. Introduces public ``invalidate_api_key_cache`` / ``invalidate_api_key_cache_hash`` hooks. * api/auth/api_key_service.py — default to the prime database (never ``get_default_context()``). ``revoke_key`` invokes the cache invalidation hook so revocation is immediate. --- Group D: ObjectPager (audit §8.1-8.2) --- The in-memory ``_cache`` was never invalidated on writes — callers got stale rows after any save/delete. Keyset pagination via ``after_id`` combined with ``order_by`` returned wrong/missing rows because the cursor only tracks ``id``. * core/pager.py — drop ``_cache`` entirely; every ``get_page`` hits the database. Reject ``after_id`` combined with ``order_by`` with a clear ``ValueError`` rather than silently breaking cursor semantics. * tests/core/test_pagination.py — existing tests asserted the cache behavior; updated to assert the new (cache-free) behavior and absent ``_cache`` attribute. --- Group E: default find_one_and_update _id-vs-id normalization (audit §5.3) --- Default ``find_one_and_update`` / ``find_one_and_delete`` impls fed the caller's query into ``QueryEngine.match`` against records stored by non-Mongo backends (JsonDB / SQLite / DynamoDB) — but those backends only persist ``id``. Callers following the Mongo convention of ``{"_id": ...}`` got silent misses. * db/database.py — add ``_normalize_id_query`` helper; default impls rewrite ``_id`` → ``id`` when only ``_id`` is present. MongoDB override is unaffected. Callers passing both keys preserve verbatim. --- New regression tests (11 cases) --- * tests/db/test_default_compound_ops_id_normalization.py — verifies ``_id`` and ``id`` query keys are equivalent on default compound ops. * tests/core/test_pager_audit_fixes.py — verifies the pager has no cache attribute, rejects after_id + order_by, and never returns stale rows after writes. Verification: pytest across all subdirs → 1783 passed, 1 skipped (otel), 2 warnings. No regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/auth/api_key_service.py | 21 ++- jvspatial/api/auth/enhanced.py | 148 ++++++++++-------- .../api/integrations/webhooks/webhook_auth.py | 74 +++++++-- jvspatial/core/entities/edge.py | 9 +- jvspatial/core/entities/node.py | 11 +- jvspatial/core/entities/walker.py | 11 +- jvspatial/core/pager.py | 32 ++-- jvspatial/db/database.py | 29 +++- jvspatial/db/jsondb.py | 39 +++-- tests/core/test_pager_audit_fixes.py | 69 ++++++++ tests/core/test_pagination.py | 28 ++-- ...t_default_compound_ops_id_normalization.py | 72 +++++++++ 12 files changed, 412 insertions(+), 131 deletions(-) create mode 100644 tests/core/test_pager_audit_fixes.py create mode 100644 tests/db/test_default_compound_ops_id_normalization.py diff --git a/jvspatial/api/auth/api_key_service.py b/jvspatial/api/auth/api_key_service.py index b24d1bf..8d10653 100644 --- a/jvspatial/api/auth/api_key_service.py +++ b/jvspatial/api/auth/api_key_service.py @@ -23,11 +23,15 @@ def __init__(self, context: Optional[GraphContext] = None): Args: context: GraphContext instance for database operations. - If None, uses the default context. + If None, defaults to the prime database — never the + application default context. Auth state must live on + the prime DB (SPEC §9 / CLAUDE.md §1; audit §4.4). """ - from jvspatial.core.context import get_default_context + if context is None: + from jvspatial.db import get_prime_database - self.context = context or get_default_context() + context = GraphContext(database=get_prime_database()) + self.context = context self._logger = logging.getLogger(__name__) self.key_prefix = "sk_" # Default prefix, can be configured self.key_length = 32 # Length of random part after prefix @@ -242,6 +246,17 @@ async def revoke_key(self, key_id: str, user_id: str) -> bool: api_key._graph_context = self.context await self.context.save(api_key) + # Drop the webhook-layer cache entry so revocation is effective + # immediately rather than after the 5-minute TTL (audit §4.5). + try: + from jvspatial.api.integrations.webhooks.webhook_auth import ( + invalidate_api_key_cache_hash, + ) + + invalidate_api_key_cache_hash(api_key.key_hash) + except Exception: # pragma: no cover — webhook module is optional + pass + self._logger.info(f"Revoked API key {key_id} for user {user_id}") return True diff --git a/jvspatial/api/auth/enhanced.py b/jvspatial/api/auth/enhanced.py index ddc3c87..dae2001 100644 --- a/jvspatial/api/auth/enhanced.py +++ b/jvspatial/api/auth/enhanced.py @@ -4,6 +4,7 @@ rate limiting, brute force protection, and session management. """ +import asyncio import hashlib import logging import time @@ -244,6 +245,11 @@ def __init__(self, session_timeout: int = 3600, max_sessions_per_user: int = 5): self._sessions: Dict[str, Dict[str, Any]] = {} self._user_sessions: Dict[str, Set[str]] = {} self._logger = logging.getLogger(__name__) + # Single lock guards both ``_sessions`` and ``_user_sessions`` so + # concurrent create/invalidate/cleanup cannot raise + # ``RuntimeError: dictionary changed size during iteration`` and + # ``max_sessions_per_user`` enforcement is not racy (audit §4.8). + self._lock = asyncio.Lock() async def create_session(self, user_id: str, user_data: Dict[str, Any]) -> str: """Create a new session for a user. @@ -260,22 +266,33 @@ async def create_session(self, user_id: str, user_data: Dict[str, Any]) -> str: session_id = str(uuid.uuid4()) now = time.time() - # Clean up old sessions for this user - await self._cleanup_user_sessions(user_id) - - # Create new session - self._sessions[session_id] = { - "user_id": user_id, - "created_at": now, - "last_accessed": now, - "user_data": user_data, - "is_active": True, - } - - # Track user sessions - if user_id not in self._user_sessions: - self._user_sessions[user_id] = set() - self._user_sessions[user_id].add(session_id) + async with self._lock: + # Inline expired-session sweep under the lock so the cap + # check below sees a coherent set. + self._sweep_user_sessions_locked(user_id, now) + + # Enforce per-user cap. + existing = self._user_sessions.get(user_id, set()) + while len(existing) >= self.max_sessions_per_user: + # Drop the oldest by last_accessed. + oldest = min( + (s for s in existing if s in self._sessions), + key=lambda s: self._sessions[s].get("last_accessed", 0), + default=None, + ) + if oldest is None: + break + self._invalidate_session_locked(oldest) + existing = self._user_sessions.get(user_id, set()) + + self._sessions[session_id] = { + "user_id": user_id, + "created_at": now, + "last_accessed": now, + "user_data": user_data, + "is_active": True, + } + self._user_sessions.setdefault(user_id, set()).add(session_id) self._logger.info(f"Created session {session_id} for user {user_id}") return session_id @@ -289,21 +306,22 @@ async def validate_session(self, session_id: str) -> Optional[Dict[str, Any]]: Returns: User data if session is valid, None otherwise """ - if session_id not in self._sessions: - return None + async with self._lock: + if session_id not in self._sessions: + return None - session = self._sessions[session_id] - now = time.time() + session = self._sessions[session_id] + now = time.time() - # Check if session is expired - if now - session["last_accessed"] > self.session_timeout: - await self.invalidate_session(session_id) - return None + # Check if session is expired + if now - session["last_accessed"] > self.session_timeout: + self._invalidate_session_locked(session_id) + return None - # Update last accessed time - session["last_accessed"] = now + # Update last accessed time + session["last_accessed"] = now - return session["user_data"] + return session["user_data"] async def invalidate_session(self, session_id: str) -> bool: """Invalidate a session. @@ -314,6 +332,11 @@ async def invalidate_session(self, session_id: str) -> bool: Returns: True if session was invalidated, False otherwise """ + async with self._lock: + return self._invalidate_session_locked(session_id) + + def _invalidate_session_locked(self, session_id: str) -> bool: + """Remove ``session_id`` from internal maps. Caller holds ``_lock``.""" if session_id not in self._sessions: return False @@ -341,38 +364,31 @@ async def invalidate_user_sessions(self, user_id: str) -> int: Returns: Number of sessions invalidated """ - if user_id not in self._user_sessions: - return 0 - - session_ids = list(self._user_sessions[user_id]) - invalidated_count = 0 - - for session_id in session_ids: - if await self.invalidate_session(session_id): - invalidated_count += 1 - - return invalidated_count - - async def _cleanup_user_sessions(self, user_id: str) -> None: - """Clean up old sessions for a user. - - Args: - user_id: User ID to clean up sessions for - """ + async with self._lock: + session_ids = list(self._user_sessions.get(user_id, set())) + count = 0 + for session_id in session_ids: + if self._invalidate_session_locked(session_id): + count += 1 + return count + + def _sweep_user_sessions_locked(self, user_id: str, now: float) -> None: + """Drop expired sessions for ``user_id``. Caller holds ``_lock``.""" if user_id not in self._user_sessions: return - now = time.time() - sessions_to_remove = [] - - for session_id in self._user_sessions[user_id]: - if session_id in self._sessions: - session = self._sessions[session_id] - if now - session["last_accessed"] > self.session_timeout: - sessions_to_remove.append(session_id) + # Iterate a snapshot — _invalidate_session_locked mutates the set. + for session_id in list(self._user_sessions[user_id]): + session = self._sessions.get(session_id) + if session is None: + continue + if now - session["last_accessed"] > self.session_timeout: + self._invalidate_session_locked(session_id) - for session_id in sessions_to_remove: - await self.invalidate_session(session_id) + async def _cleanup_user_sessions(self, user_id: str) -> None: + """Clean up old sessions for a user (acquires the lock).""" + async with self._lock: + self._sweep_user_sessions_locked(user_id, time.time()) async def cleanup_expired_sessions(self) -> int: """Clean up all expired sessions. @@ -380,17 +396,17 @@ async def cleanup_expired_sessions(self) -> int: Returns: Number of sessions cleaned up """ - now = time.time() - expired_sessions = [] - - for session_id, session in self._sessions.items(): - if now - session["last_accessed"] > self.session_timeout: - expired_sessions.append(session_id) - - for session_id in expired_sessions: - await self.invalidate_session(session_id) - - return len(expired_sessions) + async with self._lock: + now = time.time() + # Snapshot keys so the locked invalidate helper can mutate. + expired = [ + sid + for sid, s in self._sessions.items() + if now - s["last_accessed"] > self.session_timeout + ] + for sid in expired: + self._invalidate_session_locked(sid) + return len(expired) class AuthenticationEnhancer: diff --git a/jvspatial/api/integrations/webhooks/webhook_auth.py b/jvspatial/api/integrations/webhooks/webhook_auth.py index dacb59d..2f53da1 100644 --- a/jvspatial/api/integrations/webhooks/webhook_auth.py +++ b/jvspatial/api/integrations/webhooks/webhook_auth.py @@ -20,16 +20,48 @@ _API_KEY_CACHE_TTL = 300.0 _API_KEY_CACHE_MAX_SIZE = 500 _API_KEY_VALIDATE_TIMEOUT = 10.0 +# Single lock guards every read/write/eviction of ``_API_KEY_CACHE`` so +# concurrent webhook requests cannot trip a ``KeyError`` when the size-cap +# eviction races a cache read (audit §4.7). +_API_KEY_CACHE_LOCK = asyncio.Lock() -def _api_key_cache_cleanup() -> None: - """Remove expired entries from API key cache.""" +def _api_key_cache_cleanup_locked() -> None: + """Remove expired entries from API key cache. Caller holds the lock.""" now = time.time() expired = [k for k, (exp, _) in _API_KEY_CACHE.items() if exp <= now] for k in expired: del _API_KEY_CACHE[k] +async def invalidate_api_key_cache(api_key: Optional[str] = None) -> None: + """Drop a single key (or all keys) from the webhook API-key cache. + + ``APIKeyService.revoke_key`` calls this so revoking a key is reflected + immediately, not after the up-to-5-minute TTL (audit §4.5). + """ + async with _API_KEY_CACHE_LOCK: + if api_key is None: + _API_KEY_CACHE.clear() + return + cache_key = hashlib.sha256(api_key.encode()).hexdigest() + _API_KEY_CACHE.pop(cache_key, None) + + +def invalidate_api_key_cache_hash(cache_key: str) -> None: + """Drop a single hashed cache key (sync variant). + + Callers that already hold the cache hash (e.g. ``APIKeyService`` + working with the persisted hash, not the plaintext key) can invoke + this from sync code paths such as ``APIKeyService.revoke_key``. + + The unlocked ``pop`` accepts a one-shot race where a concurrent + reader sees the stale entry once — preferable to failing in callers + that have no bound event loop. + """ + _API_KEY_CACHE.pop(cache_key, None) + + async def authenticate_webhook_api_key( request: Request, auth_mode: str, @@ -144,17 +176,24 @@ async def authenticate_webhook_api_key( cache_key = hashlib.sha256(api_key.encode()).hexdigest() now = time.time() cache_hit = False - if cache_key in _API_KEY_CACHE: - cached_expiry, cached_entity = _API_KEY_CACHE[cache_key] - if cached_expiry > now and cached_entity is not None: - api_key_entity = cached_entity - cache_hit = True - logger.debug(f"Webhook API key cache hit: key_id={cached_entity.id}") + # Single lock around read + eviction + miss-population so a + # concurrent size-cap cleanup cannot ``KeyError`` a reader and + # the size cap is not racy (audit §4.7). + async with _API_KEY_CACHE_LOCK: + cached = _API_KEY_CACHE.get(cache_key) + if cached is not None: + cached_expiry, cached_entity = cached + if cached_expiry > now and cached_entity is not None: + api_key_entity = cached_entity + cache_hit = True + logger.debug( + f"Webhook API key cache hit: key_id={cached_entity.id}" + ) + else: + api_key_entity = None + _API_KEY_CACHE.pop(cache_key, None) else: api_key_entity = None - del _API_KEY_CACHE[cache_key] - else: - api_key_entity = None prime_ctx = None service = None @@ -175,12 +214,13 @@ async def authenticate_webhook_api_key( detail="Service temporarily unavailable", ) if api_key_entity: - if len(_API_KEY_CACHE) >= _API_KEY_CACHE_MAX_SIZE: - _api_key_cache_cleanup() - _API_KEY_CACHE[cache_key] = ( - now + _API_KEY_CACHE_TTL, - api_key_entity, - ) + async with _API_KEY_CACHE_LOCK: + if len(_API_KEY_CACHE) >= _API_KEY_CACHE_MAX_SIZE: + _api_key_cache_cleanup_locked() + _API_KEY_CACHE[cache_key] = ( + now + _API_KEY_CACHE_TTL, + api_key_entity, + ) if not api_key_entity: raise HTTPException(status_code=401, detail="Invalid or expired API key") diff --git a/jvspatial/core/entities/edge.py b/jvspatial/core/entities/edge.py index 6502472..23a58f1 100644 --- a/jvspatial/core/entities/edge.py +++ b/jvspatial/core/entities/edge.py @@ -63,8 +63,13 @@ async def direction(self: "Edge") -> str: """ return "both" if self.bidirectional else "out" - def __init_subclass__(cls: Type["Edge"]) -> None: - """Initialize subclass by registering visit hooks.""" + def __init_subclass__(cls: Type["Edge"], **kwargs: Any) -> None: + """Initialize subclass by registering visit hooks. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs (audit §6.2). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} cls._is_visit_hook = {} diff --git a/jvspatial/core/entities/node.py b/jvspatial/core/entities/node.py index 429fd76..cbd0ace 100644 --- a/jvspatial/core/entities/node.py +++ b/jvspatial/core/entities/node.py @@ -59,8 +59,15 @@ def _get_top_level_fields(cls: Type["Node"]) -> set: "id", } # edge_ids is stored as "edges" at top level; id is top-level on node docs - def __init_subclass__(cls: Type["Node"]) -> None: - """Initialize subclass by registering visit hooks.""" + def __init_subclass__(cls: Type["Node"], **kwargs: Any) -> None: + """Initialize subclass by registering visit hooks. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs and registers + ``protected`` / ``transient`` / ``private`` attribute metadata + for Node subclasses (audit §6.1). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} for _name, method in inspect.getmembers(cls, inspect.isfunction): diff --git a/jvspatial/core/entities/walker.py b/jvspatial/core/entities/walker.py index 2ef0732..9d3757b 100644 --- a/jvspatial/core/entities/walker.py +++ b/jvspatial/core/entities/walker.py @@ -391,8 +391,15 @@ def __init__(self: "Walker", **kwargs: Any) -> None: # Register with global event bus # Note: We need to register asynchronously, so we'll do it in spawn() - def __init_subclass__(cls: Type["Walker"]) -> None: - """Handle subclass initialization.""" + def __init_subclass__(cls: Type["Walker"], **kwargs: Any) -> None: + """Handle subclass initialization. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs and registers + ``protected`` / ``transient`` / ``private`` attribute metadata + for Walker subclasses (audit §6.3). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} for _name, method in inspect.getmembers(cls, inspect.isfunction): diff --git a/jvspatial/core/pager.py b/jvspatial/core/pager.py index fb5881e..a411a53 100644 --- a/jvspatial/core/pager.py +++ b/jvspatial/core/pager.py @@ -67,11 +67,12 @@ def __init__( self.total_pages = 1 self.has_previous = False self.has_next = False + # Result caching removed (audit §8.2). The previous ``_cache`` + # was never invalidated on writes so callers got stale rows after + # any save/delete on the underlying collection. Backend-level + # caches (``create_database(cache_get_size=...)``) remain. self.is_cached = False - # Internal cache for results (includes filter hash in key) - self._cache: Dict[str, List[T]] = {} - async def get_page( self, page: int = 1, @@ -117,16 +118,20 @@ async def get_page( # --- Keyset (cursor) pagination --- if after_id is not None: + # Cursor semantics only hold when the sort key matches the + # cursor field. ``after_id`` filters by ``id > after_id`` so + # sorting by anything else would skip rows / return duplicates + # on writes between pages. Reject the combo loudly rather + # than silently return broken pages (audit §8.1). + if self.order_by: + raise ValueError( + "ObjectPager: ``after_id`` (keyset pagination) cannot be " + "combined with ``order_by``; the cursor only tracks id. " + "Use offset pagination if you need a custom sort key." + ) keyset_filter = dict(db_filter) keyset_filter["id"] = {"$gt": after_id} sort: Optional[List[Any]] = [("id", 1)] - if self.order_by: - sort = [ - ( - f"context.{self.order_by}", - 1 if self.order_direction.lower() == "asc" else -1, - ) - ] raw_items = await db.find( collection, keyset_filter, limit=self.page_size + 1, sort=sort ) @@ -138,17 +143,11 @@ async def get_page( obj = await context._deserialize_entity(self.object_class, item_data) if obj: objects.append(obj) - cache_key = f"keyset_{after_id}_{hash(str(additional_filters))}" - self._cache[cache_key] = objects # type: ignore[assignment] self.is_cached = False return objects # --- Page-number (offset) pagination --- self.current_page = max(1, page) - cache_key = f"{self.current_page}_{hash(str(additional_filters))}" - if cache_key in self._cache: - self.is_cached = True - return self._cache[cache_key] # type: ignore[return-value] self.is_cached = False self.total_items = await db.count(collection, db_filter) @@ -211,7 +210,6 @@ async def get_page( if obj: page_objects.append(obj) - self._cache[cache_key] = page_objects # type: ignore[assignment] return page_objects async def next_page( diff --git a/jvspatial/db/database.py b/jvspatial/db/database.py index eb444a5..37c5288 100644 --- a/jvspatial/db/database.py +++ b/jvspatial/db/database.py @@ -20,6 +20,24 @@ def _find_sort_key(record: Dict[str, Any], field: str) -> Tuple[bool, Any]: return (value is None, value) +def _normalize_id_query(query: Dict[str, Any]) -> Dict[str, Any]: + """Map ``_id`` → ``id`` when only ``_id`` is present. + + The default ``find_one_and_update`` / ``find_one_and_delete`` impls + feed the query into ``QueryEngine.match`` against records stored by + non-Mongo backends (JsonDB / SQLite / DynamoDB) which only persist + ``id``. Callers that follow the Mongo-style convention of querying + by ``_id`` would otherwise silently miss every row on those backends + (audit §5.3). When both keys are present the caller's intent is + preserved verbatim. + """ + if "_id" in query and "id" not in query: + normalized = {k: v for k, v in query.items() if k != "_id"} + normalized["id"] = query["_id"] + return normalized + return query + + def finalize_find_results( records: List[Dict[str, Any]], *, @@ -268,7 +286,10 @@ async def find_one_and_delete( Returns: Deleted document if found, ``None`` otherwise """ - doc = await self.find_one(collection, query) + # Non-Mongo backends store records keyed by ``id`` only. Normalize + # the Mongo-style ``_id`` filter so default matching works + # uniformly (audit §5.3 / SPEC §4.1). + doc = await self.find_one(collection, _normalize_id_query(query)) if doc is None: return None record_id = doc.get("_id", doc.get("id")) @@ -301,7 +322,11 @@ async def find_one_and_update( Returns: Updated document, or ``None`` if no match and ``upsert`` is ``False`` """ - doc = await self.find_one(collection, query) + # Non-Mongo backends store records keyed by ``id`` only. Normalize + # the Mongo-style ``_id`` filter so default matching works + # uniformly (audit §5.3 / SPEC §4.1). + normalized = _normalize_id_query(query) + doc = await self.find_one(collection, normalized) is_new = doc is None if is_new: if not upsert: diff --git a/jvspatial/db/jsondb.py b/jvspatial/db/jsondb.py index ddb5de3..37e8931 100644 --- a/jvspatial/db/jsondb.py +++ b/jvspatial/db/jsondb.py @@ -149,6 +149,17 @@ def _get_record_path(self, collection: str, record_id: str) -> Path: collection_dir = self._get_collection_dir(collection) return collection_dir / f"{record_id}.json" + @staticmethod + def _list_collection_json_files(collection_dir: Path) -> List[Path]: + """Enumerate persisted ``*.json`` records, skipping in-flight tmps. + + Called from inside ``asyncio.to_thread`` by ``count`` and ``find`` + so the directory scan does not block the event loop. + """ + return [ + p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") + ] + async def _async_write_json(self, path: Path, data: Dict[str, Any]) -> None: """Write JSON data to file asynchronously. @@ -165,7 +176,10 @@ async def _async_read_json(self, path: Path) -> Optional[Dict[str, Any]]: Uses aiofiles if available, otherwise falls back to asyncio.to_thread. Returns None if file doesn't exist or is invalid. """ - if not path.exists(): + # ``Path.exists`` performs a stat() syscall — blocking I/O inside + # an async function (audit §3.6 / SPEC §3.3). Offload to the + # default executor instead. + if not await asyncio.to_thread(path.exists): return None try: @@ -247,12 +261,14 @@ async def count( """ q = query or {} collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync ``exists``/``glob`` block the event loop — offload both + # (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return 0 - json_files = [ - p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") - ] + json_files = await asyncio.to_thread( + self._list_collection_json_files, collection_dir + ) if not q: return len(json_files) @@ -283,7 +299,8 @@ async def find_many( # Build (id, path) pairs first so we can short-circuit when # the collection dir doesn't exist. collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync stat blocks the event loop (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return {} paths = [ (rec_id, self._get_record_path(collection, rec_id)) for rec_id in unique_ids @@ -352,15 +369,17 @@ async def find( """ collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync ``exists``/``glob`` block the event loop — offload both + # (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return [] # Get all JSON files in the collection directory. # Skip ``*.jvtmp`` files left behind by an in-flight write -- they # are not yet part of the published dataset. - json_files = [ - p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") - ] + json_files = await asyncio.to_thread( + self._list_collection_json_files, collection_dir + ) if not json_files: return [] diff --git a/tests/core/test_pager_audit_fixes.py b/tests/core/test_pager_audit_fixes.py new file mode 100644 index 0000000..9dcff91 --- /dev/null +++ b/tests/core/test_pager_audit_fixes.py @@ -0,0 +1,69 @@ +"""ObjectPager fixes (audit §8.1, §8.2). + +Wave 2 changes: + +* Drop the in-memory ``_cache`` entirely — entries were never invalidated + on writes, so callers got stale rows after any save/delete on the + underlying collection. +* Reject ``after_id`` combined with ``order_by`` — the cursor only tracks + ``id`` so a non-id sort key would skip or duplicate rows on writes + between pages. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.core.context import GraphContext, scoped_default_context_async +from jvspatial.core.entities import Node +from jvspatial.core.pager import ObjectPager +from jvspatial.db import create_database + + +class PageNode(Node): + name: str = "" + value: int = 0 + + +@pytest.fixture +async def context(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + ctx = GraphContext(database=db) + async with scoped_default_context_async(ctx): + yield ctx + + +def test_object_pager_has_no_cache_attribute(): + """The previously-stale ``_cache`` attribute is gone.""" + pager = ObjectPager(PageNode, page_size=3) + assert not hasattr(pager, "_cache") + + +@pytest.mark.asyncio +async def test_after_id_with_order_by_rejected(): + pager = ObjectPager(PageNode, page_size=3, order_by="value") + with pytest.raises(ValueError, match="after_id"): + await pager.get_page(after_id="n.PageNode.abc") + + +@pytest.mark.asyncio +async def test_get_page_no_stale_cache_after_write(context): + # Seed: 4 nodes. + nodes = [PageNode(name=f"n{i}", value=i) for i in range(4)] + for n in nodes: + await context.save(n) + + pager = ObjectPager(PageNode, page_size=10) + first = await pager.get_page(page=1) + assert len(first) == 4 + + # Add a node and re-page — must see the new node, not a cached + # 4-item snapshot. + extra = PageNode(name="n_extra", value=99) + await context.save(extra) + + refreshed = await pager.get_page(page=1) + assert len(refreshed) == 5 + assert any(n.name == "n_extra" for n in refreshed) diff --git a/tests/core/test_pagination.py b/tests/core/test_pagination.py index 677ffe7..319c3b0 100644 --- a/tests/core/test_pagination.py +++ b/tests/core/test_pagination.py @@ -281,8 +281,9 @@ async def mock_deserialize(cls, data): assert pager.has_next_page() assert len(page1) == 2 - # Clear cache to ensure next call hits the database - pager._cache.clear() + # Audit §8.2: pager no longer has an in-memory ``_cache`` — + # every ``get_page`` hits the database. No explicit clear + # needed. # Get next page page2 = await pager.next_page() @@ -340,11 +341,17 @@ async def test_has_previous_page(self): class TestObjectPagerCaching: - """Test ObjectPager caching behavior.""" + """Pager no longer caches results (audit §8.2).""" @pytest.mark.asyncio - async def test_page_caching(self, mock_context, sample_data): - """Test that pages are cached properly.""" + async def test_get_page_does_not_cache(self, mock_context, sample_data): + """Every ``get_page`` call must hit the database. + + The previous in-memory ``_cache`` was never invalidated on writes, + so callers got stale rows after any save/delete on the underlying + collection. The cache has been removed; ``is_cached`` is always + False. + """ with patch( "jvspatial.core.context.get_default_context", return_value=mock_context ): @@ -357,16 +364,17 @@ async def mock_deserialize(cls, data): mock_context._deserialize_entity.side_effect = mock_deserialize pager = ObjectPager(PaginationTestObject, page_size=2) + assert not hasattr(pager, "_cache") - # First call should hit the database results1 = await pager.get_page(1) assert not pager.is_cached - assert mock_context.database.find.call_count == 1 + first_call_count = mock_context.database.find.call_count + assert first_call_count >= 1 - # Second call to same page should use cache + # Second call to same page MUST re-fetch. results2 = await pager.get_page(1) - assert pager.is_cached - assert mock_context.database.find.call_count == 1 # No additional calls + assert not pager.is_cached + assert mock_context.database.find.call_count > first_call_count assert len(results1) == len(results2) diff --git a/tests/db/test_default_compound_ops_id_normalization.py b/tests/db/test_default_compound_ops_id_normalization.py new file mode 100644 index 0000000..91d14c3 --- /dev/null +++ b/tests/db/test_default_compound_ops_id_normalization.py @@ -0,0 +1,72 @@ +"""Default ``find_one_and_update`` / ``find_one_and_delete`` must honor +both ``_id`` and ``id`` query keys. + +Audit §5.3 / SPEC §4.1: non-Mongo backends (JsonDB, SQLite, DynamoDB) +persist records keyed by ``id`` only. Callers that follow the +Mongo-style convention of querying by ``_id`` were silently missed by +the default compound-op implementation, which fed the query into +``QueryEngine.match`` against records that have no ``_id`` field. + +The default impls now normalize ``_id`` → ``id`` before matching. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.db import create_database + + +@pytest.fixture +async def jsondb(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + yield db + + +@pytest.mark.asyncio +async def test_find_one_and_update_matches_by_underscored_id(jsondb): + await jsondb.save("widgets", {"id": "w1", "qty": 1}) + out = await jsondb.find_one_and_update( + "widgets", {"_id": "w1"}, {"$set": {"qty": 2}} + ) + assert out is not None + assert out["qty"] == 2 + # Side-effect on disk reflects the update. + fresh = await jsondb.get("widgets", "w1") + assert fresh["qty"] == 2 + + +@pytest.mark.asyncio +async def test_find_one_and_update_matches_by_id_unchanged(jsondb): + await jsondb.save("widgets", {"id": "w2", "qty": 5}) + out = await jsondb.find_one_and_update( + "widgets", {"id": "w2"}, {"$set": {"qty": 7}} + ) + assert out is not None + assert out["qty"] == 7 + + +@pytest.mark.asyncio +async def test_find_one_and_delete_matches_by_underscored_id(jsondb): + await jsondb.save("widgets", {"id": "w3", "qty": 9}) + deleted = await jsondb.find_one_and_delete("widgets", {"_id": "w3"}) + assert deleted is not None + assert deleted["id"] == "w3" + assert await jsondb.get("widgets", "w3") is None + + +@pytest.mark.asyncio +async def test_find_one_and_update_upsert_with_underscored_id(jsondb): + out = await jsondb.find_one_and_update( + "widgets", + {"_id": "w_new"}, + {"$set": {"qty": 42}}, + upsert=True, + ) + assert out is not None + assert out["id"] == "w_new" + assert out["qty"] == 42 + persisted = await jsondb.get("widgets", "w_new") + assert persisted is not None From a6486146416de66e57834de3b0d4527bbde1525e Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 11:44:04 -0400 Subject: [PATCH 12/22] docs(changelog): document Wave 2 audit remediation Adds [Unreleased] entries for the 11 HIGH findings closed in the previous commit. Two breaking behavioral changes flagged: ObjectPager no longer caches, and ObjectPager rejects after_id + order_by combo. Two new public surface entries documented: invalidate_api_key_cache and invalidate_api_key_cache_hash. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b359197..e2ab24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `TraversalProtection.start_if_needed()` — idempotent initializer for `Walker.run()`. Pause/resume cycles no longer reset step / visit / wall-clock counters. (Audit §2.2.) - `WalkerTrail(max_length=N)` — wires the previously-undocumented bound. `0` (default) means unlimited; positive integers cap the in-memory trail. Threaded through `Walker(max_trail_length=...)` and `JVSPATIAL_WALKER_MAX_TRAIL_LENGTH`. (Audit §2.3 / SPEC §6.4.) - `tests/core/test_entity_name_walker_and_save.py`, `tests/core/test_walker_protection_audit_fixes.py`, `tests/storage/test_versioning_path_sanitizer_audit.py`, `tests/api/test_webhook_hmac_audit_fix.py` — 28 new regression cases pinning Wave 1 audit fixes. +- Public `invalidate_api_key_cache(api_key)` and `invalidate_api_key_cache_hash(cache_key)` helpers in `jvspatial.api.integrations.webhooks.webhook_auth`. `APIKeyService.revoke_key` now invokes the latter so revocations are effective immediately rather than after the 5-minute TTL. (Audit §4.5.) +- `tests/db/test_default_compound_ops_id_normalization.py`, `tests/core/test_pager_audit_fixes.py` — 11 new regression cases pinning Wave 2 audit fixes. ### Fixed @@ -31,6 +33,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `WalkerQueue.prepend` / `append` / `add_next` / `insert_after` / `insert_before` now respect `max_size` and emit a one-shot WARNING on first drop. Earlier the front-of-queue and middle-insert paths bypassed the cap, providing a silent protection bypass. (Audit §2.4-§2.5.) - `DynamoDB.{find, count, batch_get, batch_write}` now route every `aioboto3` wire call through `_run_with_throttle_retry`. Earlier the helper was only applied to `save`/`get`/`delete`; `ProvisionedThroughputExceededException` and `ThrottlingException` from scan / query / batch ops surfaced to callers as immediate failures despite the documented backoff. (Audit §5.1 / SPEC §4.3.) - Security headers middleware now emits a relaxed Content-Security-Policy on `/docs`, `/redoc`, `/openapi.json` (and sub-paths) that permits `cdn.jsdelivr.net` so FastAPI's bundled Swagger UI / ReDoc pages render. The previous strict default blocked the CDN-hosted JS/CSS and the docs loaded blank. Application routes keep the strict default policy. +- **BREAKING (behavioral):** `ObjectPager.get_page` no longer returns cached results. The in-memory `_cache` attribute is removed entirely; every call hits the database. Stale-after-write semantics are eliminated. Callers that relied on caching should use the backend-level read-through cache via `create_database(cache_get_size=...)`. (Audit §8.2.) +- `ObjectPager.get_page(after_id=..., order_by=...)` now raises `ValueError`. Keyset pagination via `after_id` only tracks `id`; combining it with a custom sort key silently skipped or duplicated rows on writes between pages. Use offset pagination if you need a custom sort. (Audit §8.1.) +- Default `Database.find_one_and_update` and `Database.find_one_and_delete` now normalize `{"_id": x}` queries to `{"id": x}` when only `_id` is present, so Mongo-style queries no longer silent-miss on JsonDB / SQLite / DynamoDB (which persist records keyed by `id` only). MongoDB native override is unaffected. (Audit §5.3 / SPEC §4.1.) +- JsonDB no longer blocks the event loop. Every `Path.exists()` / `Path.glob()` call inside `async` methods (`_async_read_json`, `count`, `find_many`, `find`) is now wrapped in `asyncio.to_thread`. (Audit §3.6 / SPEC §3.3.) +- `Node.__init_subclass__`, `Edge.__init_subclass__`, and `Walker.__init_subclass__` now call `super().__init_subclass__(**kwargs)` so `AttributeMixin.__init_subclass__` runs and `protected` / `transient` / `private` attribute registration completes for their subclasses. (Audit §6.1-§6.3 / SPEC §2.5.) +- `SessionManager` mutations now hold an `asyncio.Lock` so concurrent create/invalidate/cleanup cannot raise `RuntimeError: dictionary changed size during iteration`. `max_sessions_per_user` enforcement is no longer racy — over-cap creation evicts the oldest session by `last_accessed`. (Audit §4.8.) +- `_API_KEY_CACHE` (webhook layer) now holds a lock around reads, eviction, and miss-population. Removes a `KeyError` window when a size-cap cleanup races a reader. (Audit §4.7.) +- `APIKeyService(context=None)` now defaults to the **prime** database instead of `get_default_context()`. Auth state is required to live on the prime DB (SPEC §9 / CLAUDE.md §1). (Audit §4.4.) +- `APIKeyService.revoke_key` now invokes the new webhook cache-invalidation hook so a revoked key stops authenticating immediately rather than after the 5-minute TTL. (Audit §4.5.) ## [0.0.7] - 2026-05-08 From 809fe2eab025e4750d02c2583709156435a19e0e Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 13:12:39 -0400 Subject: [PATCH 13/22] =?UTF-8?q?fix:=20Wave=203=20audit=20remediation=20?= =?UTF-8?q?=E2=80=94=20env=20allowlist,=20CORS=20warn,=20txn=20honesty,=20?= =?UTF-8?q?query=20parity,=20bulk=5Fsave?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses 12 MED-class findings from AUDIT.md (Wave 3). Closes the latent-timebomb tier of the post-audit hardening plan. --- Group A: env allowlist + parse_bool consolidation (audit §7.1-§7.3) --- SPEC §10.2 promised unknown JVSPATIAL_* keys would be rejected at startup. ``env_adapter`` only READ enumerated keys; never scanned for strays. Three divergent ``parse_bool`` implementations (env_adapter, runtime/serverless, app_builder inline) produced different truth values for the same input. * env_adapter.py — declare ``ALLOWED_ENV_KEYS`` (frozenset of every JVSPATIAL_* key the library reads), add ``enforce_env_allowlist`` hooked into ``validate_server_config_requirements``. Default: warn once per unknown key. Strict mode via ``JVSPATIAL_STRICT_ENV_ALLOWLIST=true`` raises ``ValueError`` so typos fail-fast in production. * env_adapter._parse_bool — delegates to ``env.parse_bool`` and logs a warning on unrecognized values rather than silently mapping to False. * runtime/serverless._parse_bool — delegates to ``env.parse_bool``; preserves the historical ``enabled``/``disabled`` aliases for ``SERVERLESS_MODE``. * api/components/app_builder.py — JVSPATIAL_DOCS_DISABLED now uses ``env.parse_bool`` instead of an inline 4-element truthy set. --- Group B: CORS wildcard warning + EXPOSE_ERROR_DETAILS prod guard --- * api/config_groups.py — ``CORSConfig`` gains a model_validator that emits a WARNING when ``cors_origins`` contains a wildcard. Add ``cors_allow_wildcard=True`` opt-out for callers who genuinely need it (audit §4.12 / SPEC §15.4). * api/components/error_handler.py — ``_expose_error_details_to_clients`` consults ``JVSPATIAL_ENVIRONMENT`` (or ``ENVIRONMENT``); when the runtime is signalled as production, the flag is ignored and a one-shot warning is logged. Generic 500 message is returned instead (audit §4.10 / SPEC §15.5). --- Group C: MongoDB supports_transactions honesty (audit §5.9) --- * db/mongodb.py — add ``MongoDB.is_transactional()`` async probe. Uses the ``hello`` admin command to detect replica set / sharded topology and caches the result. ``begin_transaction`` short-circuits to ``None`` on non-transactional topologies instead of attempting start_session/start_transaction every time. Class attribute ``supports_transactions = True`` is preserved as the documented API-surface contract. --- Group D: QueryBuilder ↔ QueryEngine operator parity (audit §5.2) --- ``QueryBuilder`` advertised ``$nor``, ``$mod``, ``$all``, ``$type``, ``$not`` (field-level), but ``QueryEngine._match_value`` returned False for any unknown operator and ``match`` ignored ``$nor``. Queries built via the public builder silently matched nothing. * db/query.py — implement ``$nor`` at top level; ``$mod``, ``$all``, ``$type``, ``$not`` at field level. Optimizer markers (``$hint``, ``$select``) injected by ``optimize_query`` are skipped explicitly rather than silently no-matching. Unknown operators (top-level or field-level) now raise ``QueryError`` so callers see the bug rather than mysteriously empty result sets. --- Group E: bulk_save partial-success semantics (audit §5.6, §5.7) --- * db/database.py — add ``BulkSaveResult`` dataclass and ``bulk_save_detailed`` API returning ``attempted``/``saved``/ ``failed_ids``. ``bulk_save`` is preserved as a thin wrapper that returns ``result.saved`` for back-compat. Default impl catches per-record exceptions, accumulates ``failed_ids``, and reports the breakdown. The ``ValueError`` on missing-id includes the offending index (audit §5.15). --- Tests (22 new cases, all passing) --- * tests/api/test_env_allowlist_audit.py — unknown-key warn vs strict raise, ALLOWED_ENV_KEYS spot checks. * tests/api/test_cors_wildcard_and_error_detail_audit.py — wildcard warning, opt-out, prod-guard for EXPOSE_ERROR_DETAILS. * tests/db/test_query_operator_parity_audit.py — $nor/$mod/$all/$type/$not semantics, unknown-op raises, optimizer-marker skip. * tests/db/test_bulk_save_detailed_audit.py — BulkSaveResult, back-compat int return, missing-id index in error. Verification: pytest across all subdirs → 1805 passed, 1 skipped (otel), 2 warnings. No regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/components/app_builder.py | 17 +- jvspatial/api/components/error_handler.py | 41 +++- jvspatial/api/config_groups.py | 30 +++ jvspatial/db/database.py | 95 ++++++-- jvspatial/db/mongodb.py | 54 ++++- jvspatial/db/query.py | 103 ++++++++- jvspatial/env_adapter.py | 211 +++++++++++++++++- jvspatial/runtime/serverless.py | 19 +- ...st_cors_wildcard_and_error_detail_audit.py | 63 ++++++ tests/api/test_env_allowlist_audit.py | 73 ++++++ tests/db/test_bulk_save_detailed_audit.py | 62 +++++ tests/db/test_query_operator_parity_audit.py | 103 +++++++++ 12 files changed, 836 insertions(+), 35 deletions(-) create mode 100644 tests/api/test_cors_wildcard_and_error_detail_audit.py create mode 100644 tests/api/test_env_allowlist_audit.py create mode 100644 tests/db/test_bulk_save_detailed_audit.py create mode 100644 tests/db/test_query_operator_parity_audit.py diff --git a/jvspatial/api/components/app_builder.py b/jvspatial/api/components/app_builder.py index 94c2ac7..b9fa6a4 100644 --- a/jvspatial/api/components/app_builder.py +++ b/jvspatial/api/components/app_builder.py @@ -5,7 +5,6 @@ """ import logging -import os from pathlib import Path from typing import Any, Dict, List, Optional @@ -55,14 +54,14 @@ def create_app(self, lifespan: Optional[Any] = None) -> FastAPI: Swagger UI HTML is emitted. Use this in production when the API surface should not be self-documenting. """ - docs_disabled = ( - os.getenv("JVSPATIAL_DOCS_DISABLED") or "" - ).strip().lower() in { - "1", - "true", - "yes", - "on", - } + # Use the canonical boolean parser instead of an ad-hoc inline + # set so behavior matches the rest of the codebase + # (audit §7.2 / §7.12). + from jvspatial.env import env, parse_bool + + docs_disabled = bool( + env("JVSPATIAL_DOCS_DISABLED", default=False, parse=parse_bool) + ) app_kwargs: Dict[str, Any] = { "title": self.config.title, diff --git a/jvspatial/api/components/error_handler.py b/jvspatial/api/components/error_handler.py index c68b745..a3f5925 100644 --- a/jvspatial/api/components/error_handler.py +++ b/jvspatial/api/components/error_handler.py @@ -22,9 +22,46 @@ ) +def _is_production_environment() -> bool: + """Return True when the runtime is signalled as production. + + Looks at ``JVSPATIAL_ENVIRONMENT`` (canonical) and ``ENVIRONMENT`` + (fallback) for values ``prod`` / ``production``. Used to fail-closed + on the ``JVSPATIAL_EXPOSE_ERROR_DETAILS`` toggle (audit §4.10 / + SPEC §15.5). + """ + import os + + for var in ("JVSPATIAL_ENVIRONMENT", "ENVIRONMENT"): + raw = (os.environ.get(var) or "").strip().lower() + if raw in ("prod", "production"): + return True + return False + + def _expose_error_details_to_clients() -> bool: - """When true, unhandled 500 responses may include exception text (dev only).""" - return bool(env("JVSPATIAL_EXPOSE_ERROR_DETAILS", default=False, parse=parse_bool)) + """When true, unhandled 500 responses may include exception text. + + Refuses to honor the flag in production environments — leaking raw + exception text (which can include DB query fragments, file paths, + and secret-bearing config values) is a footgun (audit §4.10). + Emits a one-shot warning when the flag is requested but suppressed. + """ + requested = bool( + env("JVSPATIAL_EXPOSE_ERROR_DETAILS", default=False, parse=parse_bool) + ) + if requested and _is_production_environment(): + if not getattr(_expose_error_details_to_clients, "_warned", False): + logging.getLogger(__name__).warning( + "JVSPATIAL_EXPOSE_ERROR_DETAILS is set but the runtime " + "appears to be production (JVSPATIAL_ENVIRONMENT). " + "Ignoring the flag — 500 responses will use the generic " + "message. Unset JVSPATIAL_ENVIRONMENT or change the value " + "to enable detailed errors." + ) + _expose_error_details_to_clients._warned = True # type: ignore[attr-defined] + return False + return requested # Context variable to track exceptions that have been logged by our handler diff --git a/jvspatial/api/config_groups.py b/jvspatial/api/config_groups.py index a6c5cfe..b519d44 100644 --- a/jvspatial/api/config_groups.py +++ b/jvspatial/api/config_groups.py @@ -79,6 +79,36 @@ class CORSConfig(BaseModel): cors_headers: List[str] = Field( default_factory=lambda: ["Content-Type", "Authorization", "X-API-Key"] ) + # Set to ``True`` to silence the startup warning that fires when + # ``cors_origins`` contains a wildcard. Default opt-in is loud so + # accidental wildcards in production surface as logs (audit §4.12 / + # SPEC §15.4). + cors_allow_wildcard: bool = False + + @model_validator(mode="after") + def _warn_on_wildcard_origin(self) -> "CORSConfig": + """Emit a startup warning when CORS origins include a wildcard. + + SPEC §15.4: "Wildcard origins must be set explicitly and trigger + a startup warning." Browsers reject ``allow_origins=["*"]`` when + ``allow_credentials=True`` at runtime, but the surprise factor + of a permissive wildcard left in by mistake warrants a loud + warning here too (audit §4.12). + """ + import logging + + if not self.cors_enabled: + return self + wildcards = [o for o in self.cors_origins if o.strip() == "*" or "*" in o] + if wildcards and not self.cors_allow_wildcard: + logging.getLogger(__name__).warning( + "CORSConfig: wildcard origins detected: %s. This permits " + "any origin and is rarely correct in production. Set " + "cors_allow_wildcard=True on CORSConfig to silence this " + "warning.", + wildcards, + ) + return self class AuthConfig(BaseModel): diff --git a/jvspatial/db/database.py b/jvspatial/db/database.py index 37c5288..5ab5716 100644 --- a/jvspatial/db/database.py +++ b/jvspatial/db/database.py @@ -6,11 +6,34 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass, field from functools import partial from typing import Any, Dict, List, Optional, Tuple, Union from jvspatial.db.query import QueryEngine + +@dataclass(frozen=True) +class BulkSaveResult: + """Structured outcome of a :meth:`Database.bulk_save_detailed` call. + + Adapters with partial-success semantics (JsonDB, DynamoDB, MongoDB + ``ordered=False``) report the per-record breakdown so callers can + distinguish "all saved" from "some lost" — previously every adapter + returned the same ``int`` and silent drops were indistinguishable + from success (audit §5.7 / §5.13). + """ + + attempted: int + saved: int + failed_ids: List[str] = field(default_factory=list) + + @property + def all_saved(self) -> bool: + """Return True when every attempted record persisted.""" + return self.saved == self.attempted and not self.failed_ids + + logger = logging.getLogger(__name__) @@ -53,9 +76,9 @@ def finalize_find_results( out = records if sort: out = list(records) - for field, direction in reversed(sort): + for sort_field, direction in reversed(sort): out.sort( - key=partial(_find_sort_key, field=field), + key=partial(_find_sort_key, field=sort_field), reverse=(direction == -1), ) if limit is not None: @@ -238,34 +261,68 @@ async def bulk_save(self, collection: str, records: List[Dict[str, Any]]) -> int Returns: Number of records successfully saved. + For partial-success visibility (which IDs failed) use + :meth:`bulk_save_detailed`. Adapters with all-or-nothing + semantics (SQLite) raise rather than return a partial count. + """ + result = await self.bulk_save_detailed(collection, records) + return result.saved + + async def bulk_save_detailed( + self, collection: str, records: List[Dict[str, Any]] + ) -> BulkSaveResult: + """Save many records, returning a structured per-record outcome. + + Args: + collection: Collection name. + records: Iterable of record dicts. Each must have an ``id`` + field. Records without ``id`` raise ``ValueError`` (the + check is per-batch, before any save attempt). + + Returns: + :class:`BulkSaveResult` with ``attempted`` / ``saved`` / + ``failed_ids``. + Atomicity (per backend): - * MongoDB: ``bulk_write`` with ``ordered=False`` -- partial + * MongoDB: ``bulk_write`` with ``ordered=False`` — partial successes are reported; failures don't block other writes. * SQLite: single transaction with ``executemany``; **all records or none** land. A constraint violation rolls back - the whole batch. + the whole batch and raises (``failed_ids`` is unset). * DynamoDB: ``BatchWriteItem`` with unprocessed-item retry; - partial successes possible. - * JsonDB: parallel atomic per-file writes; partial successes - possible. + unprocessed items after retries land in ``failed_ids``. + * JsonDB: parallel atomic per-file writes; per-record + exceptions land in ``failed_ids``. The default implementation in this base class is a serial - loop of ``save()`` calls (partial success on failure), which - is correct but slow -- adapters should override. + loop of ``save()`` calls catching per-record exceptions. + Adapters should override for round-trip efficiency. """ if not records: - return 0 - for r in records: + return BulkSaveResult(attempted=0, saved=0, failed_ids=[]) + for idx, r in enumerate(records): if "id" not in r: - raise ValueError( - "bulk_save requires every record to have an 'id' field" - ) - # Sequential save() calls; on any failure the exception - # propagates and the caller sees no return value, so reaching - # the ``return`` always means every record persisted. + # Caller error — surface the offending index for fast + # debugging (audit §5.15). + raise ValueError(f"bulk_save: record at index {idx} has no 'id' field") + + saved = 0 + failed_ids: List[str] = [] for r in records: - await self.save(collection, dict(r)) - return len(records) + try: + await self.save(collection, dict(r)) + saved += 1 + except Exception as e: # pragma: no cover — defensive + logger.warning( + "%s.bulk_save: record id=%r failed: %s", + type(self).__name__, + r.get("id"), + e, + ) + failed_ids.append(str(r.get("id", ""))) + return BulkSaveResult( + attempted=len(records), saved=saved, failed_ids=failed_ids + ) async def find_one_and_delete( self, collection: str, query: Dict[str, Any] diff --git a/jvspatial/db/mongodb.py b/jvspatial/db/mongodb.py index 2fda38a..8350f6f 100644 --- a/jvspatial/db/mongodb.py +++ b/jvspatial/db/mongodb.py @@ -62,8 +62,12 @@ class MongoDB(Database): """Simplified MongoDB-based database implementation.""" # Advertised capability. The adapter implements the transactional API; - # the deployment must be a replica set (even single-node) for the - # actual ``begin_transaction()`` call to succeed at runtime. + # actual transactional semantics also require a replica-set (or + # ``sharded`` 4.2+) deployment. The class attribute stays ``True`` + # so callers branching on the static flag (the documented form) still + # see "MongoDB supports transactions". Use :meth:`is_transactional` + # for a runtime probe that honors the deployment topology + # (audit §5.9 / SPEC §4.2). supports_transactions: bool = True def __init__( @@ -100,6 +104,10 @@ def __init__( self._created_indexes: Dict[str, Set[str]] = ( {} ) # collection -> set of index names + # Memoized result of the transactional-topology probe — ``None`` + # until first checked, ``True`` if the deployment supports + # transactions, ``False`` otherwise. See :meth:`is_transactional`. + self._transactional_probe: Optional[bool] = None async def _ensure_connected(self) -> None: """Ensure database connection is established. @@ -696,6 +704,41 @@ async def create_index( except PyMongoError as e: raise DatabaseError(f"MongoDB index creation error: {e}") from e + async def is_transactional(self) -> bool: + """Return True only when the deployment supports MongoDB transactions. + + Probes the cluster topology on first call (``hello`` admin command) + and caches the result on the instance. Standalone deployments and + DocumentDB instances without transaction support return ``False``; + replica sets (single- or multi-node) and sharded clusters at + version 4.2+ return ``True``. + + Use this instead of branching on the static + :attr:`supports_transactions` class attribute when the caller + intends to actually open a transaction (audit §5.9 / SPEC §4.2). + """ + if self._transactional_probe is not None: + return self._transactional_probe + await self._ensure_connected() + if self._client is None: + self._transactional_probe = False + return False + try: + info = await self._client.admin.command("hello") + # Replica sets expose ``setName``; sharded clusters expose + # ``msg: "isdbgrid"``. Standalone deployments have neither. + is_replica_set = bool(info.get("setName")) + is_mongos = info.get("msg") == "isdbgrid" + self._transactional_probe = is_replica_set or is_mongos + except Exception: + logger.debug( + "MongoDB transaction-support probe failed; treating as " + "non-transactional", + exc_info=True, + ) + self._transactional_probe = False + return self._transactional_probe + async def begin_transaction(self): """Start a MongoDB transaction (requires replica set). @@ -710,6 +753,11 @@ async def begin_transaction(self): await self._ensure_connected() if self._client is None or self._db is None: return None + # Refuse to attempt a transaction on a topology that does not + # support it — saves an exception round-trip on every call after + # the first (audit §5.9). + if not await self.is_transactional(): + return None try: import uuid @@ -718,6 +766,8 @@ async def begin_transaction(self): return MongoDBTransaction(str(uuid.uuid4()), session, self._db) except Exception: logger.debug("Transactions not available on this deployment", exc_info=True) + # Cache the negative probe so subsequent calls short-circuit. + self._transactional_probe = False return None async def commit_transaction(self, txn) -> None: diff --git a/jvspatial/db/query.py b/jvspatial/db/query.py index 23b5cc0..93f7d73 100644 --- a/jvspatial/db/query.py +++ b/jvspatial/db/query.py @@ -10,6 +10,8 @@ from collections import OrderedDict from typing import Any, Callable, Dict, List, Optional, Union +from jvspatial.exceptions import QueryError + # Default upper bound for the per-instance ``optimize_query`` cache. The # previous implementation grew unboundedly which is fine for short-lived # processes but a slow leak in long-lived servers. 1024 entries holds @@ -17,6 +19,28 @@ DEFAULT_QUERY_CACHE_SIZE = 1024 +# Mapping from MongoDB ``$type`` operand strings to Python types used by +# the in-memory matcher. BSON type codes are not carried through this +# layer; Mongo callers using the native driver's ``$type`` get native +# behavior. (Audit §5.2.) +_TYPE_NAME_MAP = { + "string": str, + "str": str, + "int": int, + "long": int, + "double": float, + "float": float, + "bool": bool, + "boolean": bool, + "list": list, + "array": list, + "dict": dict, + "object": dict, + "null": type(None), + "none": type(None), +} + + # Unified evaluation and builder in a single module @@ -315,6 +339,13 @@ def unset_field_value(document: Dict[str, Any], field: str) -> None: elif isinstance(current, dict): current.pop(last, None) + # Operators recognized at the top-level of a query document. Anything + # else is rejected to surface typos rather than silently match nothing + # (audit §5.2). Field-name keys are checked separately below. + _TOP_LEVEL_LOGICAL_OPERATORS = frozenset({"$and", "$or", "$nor", "$not"}) + # Markers the query optimizer may inject; ignored by the matcher. + _IGNORED_TOP_LEVEL_MARKERS = frozenset({"$hint", "$select"}) + @staticmethod def match(document: Dict[str, Any], query: Optional[Dict[str, Any]]) -> bool: """Check if a document matches a query. @@ -335,9 +366,31 @@ def match(document: Dict[str, Any], query: Optional[Dict[str, Any]]) -> bool: elif key == "$or": if not any(QueryEngine.match(document, sub) for sub in condition or []): return False + elif key == "$nor": + # Audit §5.2 / SPEC §5.1: QueryBuilder.nor_() existed + # but match() ignored ``$nor`` entirely. Now: ``$nor`` + # matches when none of the sub-conditions match. + if any(QueryEngine.match(document, sub) for sub in condition or []): + return False elif key == "$not": if QueryEngine.match(document, condition): return False + elif key in QueryEngine._IGNORED_TOP_LEVEL_MARKERS: + # Optimizer hints — irrelevant to in-memory matching. + continue + elif key.startswith("$"): + # Unknown top-level operator. Refuse silently-matching + # nothing — raise so callers see the bug immediately + # (audit §5.2). + raise QueryError( + query=str(query), + reason=( + f"unsupported top-level query operator: {key!r}. " + "Supported: $and, $or, $nor, $not. Field-level " + "operators (e.g. $regex, $mod, $type, $size) live " + "inside a field condition dict." + ), + ) else: value = QueryEngine.get_field_value(document, key) if not QueryEngine._match_value(value, condition): @@ -422,8 +475,56 @@ def _match_value(value: Any, condition: Any) -> bool: for elem in value ): return False + elif op == "$mod": + # MongoDB ``$mod`` — operand is ``[divisor, remainder]``. + # Previously the QueryBuilder.mod() helper produced this + # but the engine returned False for it (audit §5.2). + try: + divisor, remainder = operand + if not isinstance(value, (int, float)): + return False + if value % divisor != remainder: + return False + except (TypeError, ValueError, ZeroDivisionError): + return False + elif op == "$all": + # ``$all`` — value must be a list containing every operand + # element. Previously silently no-matched (audit §5.2). + if not isinstance(value, list): + return False + try: + operand_iter = list(operand) + except TypeError: + return False + if not all(item in value for item in operand_iter): + return False + elif op == "$type": + # MongoDB ``$type`` — accept a Python type name string + # (``"int"``, ``"string"``, ``"list"`` …) since we do not + # carry BSON type codes through. + expected = _TYPE_NAME_MAP.get( + operand.lower() if isinstance(operand, str) else None + ) + if expected is None: + return False + if not isinstance(value, expected): + return False + elif op == "$not": + # Field-level negation — operand is another condition dict. + if QueryEngine._match_value(value, operand): + return False else: - return False + # Unknown field-level operator. Refuse silent no-match — + # raise so callers see the bug (audit §5.2). + raise QueryError( + query=str(condition), + reason=( + f"unsupported field-level query operator: {op!r}. " + "Supported: $eq, $ne, $gt, $gte, $lt, $lte, $in, " + "$nin, $exists, $regex, $size, $elemMatch, $mod, " + "$all, $type, $not." + ), + ) return True @staticmethod diff --git a/jvspatial/env_adapter.py b/jvspatial/env_adapter.py index 74b9794..a89f492 100644 --- a/jvspatial/env_adapter.py +++ b/jvspatial/env_adapter.py @@ -2,14 +2,33 @@ from __future__ import annotations +import logging import os from typing import Any, Dict, List, Optional +from jvspatial.env import parse_bool from jvspatial.runtime.eventbridge_readiness import resolve_eventbridge_lambda_arn +logger = logging.getLogger(__name__) + def _parse_bool(val: str) -> bool: - return str(val).strip().lower() in ("true", "1", "yes") + """Permissive boolean parser; consolidated with :func:`jvspatial.env.parse_bool`. + + Accepts ``true/false``, ``1/0``, ``yes/no``, ``on/off`` (case + insensitive). Falls back to ``False`` for unrecognized values rather + than raising — preserves prior behavior for misconfigured env values + (audit §7.2-§7.3). + """ + try: + return parse_bool(val) + except ValueError: + logger.warning( + "env_adapter: unrecognized boolean env value %r; treating as False. " + "Use one of: true/false, 1/0, yes/no, on/off.", + val, + ) + return False def _split_csv_list(raw: Optional[str]) -> Optional[List[str]]: @@ -193,8 +212,198 @@ def server_config_overrides_from_env() -> Dict[str, Any]: return o +# Canonical allowlist of every ``JVSPATIAL_*`` environment variable the +# library reads. Anything outside this set is rejected at startup so +# typos (``JVSPATIAL_JWT_SECRET`` for ``JVSPATIAL_JWT_SECRET_KEY`` and +# similar) surface immediately rather than silently no-op'ing. +# SPEC §10.2: "Unknown JVSPATIAL_* keys are rejected at startup to catch +# typos and removed settings." Audit §7.1 closed. +ALLOWED_ENV_KEYS: frozenset[str] = frozenset( + { + # API metadata / server runtime + "JVSPATIAL_TITLE", + "JVSPATIAL_API_TITLE", + "JVSPATIAL_DESCRIPTION", + "JVSPATIAL_API_DESCRIPTION", + "JVSPATIAL_VERSION", + "JVSPATIAL_API_VERSION", + "JVSPATIAL_API_PREFIX", + "JVSPATIAL_API_HEALTH", + "JVSPATIAL_API_ROOT", + "JVSPATIAL_HOST", + "JVSPATIAL_PORT", + "JVSPATIAL_DEBUG", + "JVSPATIAL_LOG_LEVEL", + "JVSPATIAL_GRAPH_ENDPOINT_ENABLED", + "JVSPATIAL_ENVIRONMENT", + "JVSPATIAL_DOCS_DISABLED", + # Database + "JVSPATIAL_DB_TYPE", + "JVSPATIAL_DB_PATH", + "JVSPATIAL_MONGODB_URI", + "JVSPATIAL_MONGODB_DB_NAME", + "JVSPATIAL_MONGODB_MAX_POOL_SIZE", + "JVSPATIAL_MONGODB_MIN_POOL_SIZE", + "JVSPATIAL_DYNAMODB_TABLE_NAME", + "JVSPATIAL_DYNAMODB_REGION", + "JVSPATIAL_DYNAMODB_ENDPOINT_URL", + "JVSPATIAL_DYNAMODB_WAIT_FOR_INDEX", + "JVSPATIAL_AUTO_CREATE_INDEXES", + # Auth + "JVSPATIAL_AUTH_ENABLED", + "JVSPATIAL_AUTH_STRICT_HASHING", + "JVSPATIAL_AUTH_BLACKLIST_FAIL_CLOSED", + "JVSPATIAL_JWT_SECRET_KEY", + "JVSPATIAL_JWT_ALGORITHM", + "JVSPATIAL_JWT_EXPIRE_MINUTES", + "JVSPATIAL_JWT_REFRESH_EXPIRE_DAYS", + "JVSPATIAL_BCRYPT_ROUNDS", + "JVSPATIAL_BCRYPT_ROUNDS_SERVERLESS", + # CORS + "JVSPATIAL_CORS_ENABLED", + "JVSPATIAL_CORS_ORIGINS", + "JVSPATIAL_CORS_METHODS", + "JVSPATIAL_CORS_HEADERS", + # Rate limiting + "JVSPATIAL_RATE_LIMIT_ENABLED", + "JVSPATIAL_RATE_LIMIT_DEFAULT_REQUESTS", + "JVSPATIAL_RATE_LIMIT_DEFAULT_WINDOW", + # File storage + "JVSPATIAL_FILE_STORAGE_ENABLED", + "JVSPATIAL_FILE_STORAGE_PROVIDER", + "JVSPATIAL_FILE_STORAGE_BASE_URL", + "JVSPATIAL_FILE_STORAGE_MAX_SIZE", + "JVSPATIAL_FILE_STORAGE_SERVERLESS_SHARED", + "JVSPATIAL_FILES_ROOT_PATH", + "JVSPATIAL_FILES_PUBLIC_READ", + "JVSPATIAL_FILE_INTERFACE", + "JVSPATIAL_S3_BUCKET_NAME", + "JVSPATIAL_S3_REGION", + "JVSPATIAL_S3_ACCESS_KEY", + "JVSPATIAL_S3_SECRET_KEY", + "JVSPATIAL_S3_ENDPOINT_URL", + "JVSPATIAL_S3_MULTIPART_THRESHOLD", + # URL proxy + "JVSPATIAL_PROXY_ENABLED", + "JVSPATIAL_PROXY_DEFAULT_EXPIRATION", + "JVSPATIAL_PROXY_MAX_EXPIRATION", + "JVSPATIAL_PROXY_PREFIX", + # Cache + "JVSPATIAL_CACHE_BACKEND", + "JVSPATIAL_CACHE_SIZE", + "JVSPATIAL_REDIS_URL", + "JVSPATIAL_REDIS_TTL", + "JVSPATIAL_REDIS_SERIALIZATION", + # Scheduler / deferred / serverless + "JVSPATIAL_SCHEDULER_ENABLED", + "JVSPATIAL_SCHEDULER_INTERVAL", + "JVSPATIAL_DEFERRED_TASK_PROVIDER", + "JVSPATIAL_DEFERRED_INVOKE_DISABLED", + "JVSPATIAL_DEFERRED_INVOKE_SECRET", + "JVSPATIAL_ENABLE_DEFERRED_SAVES", + "JVSPATIAL_AWS_DEFERRED_TRANSPORT", + "JVSPATIAL_AWS_SQS_QUEUE_URL", + "JVSPATIAL_EVENTBRIDGE_LAMBDA_ARN", + "JVSPATIAL_EVENTBRIDGE_ROLE_ARN", + "JVSPATIAL_EVENTBRIDGE_SCHEDULER_ENABLED", + "JVSPATIAL_EVENTBRIDGE_SCHEDULER_GROUP", + "JVSPATIAL_LWA_ENV_DEFAULTS", + # Webhooks + "JVSPATIAL_WEBHOOK_HMAC_ALGORITHM", + "JVSPATIAL_WEBHOOK_HMAC_SECRET", + "JVSPATIAL_WEBHOOK_HTTPS_REQUIRED", + "JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL", + "JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE", + # Walkers + "JVSPATIAL_WALKER_MAX_STEPS", + "JVSPATIAL_WALKER_MAX_VISITS_PER_NODE", + "JVSPATIAL_WALKER_MAX_EXECUTION_TIME", + "JVSPATIAL_WALKER_MAX_QUEUE_SIZE", + "JVSPATIAL_WALKER_MAX_TRAIL_LENGTH", + "JVSPATIAL_WALKER_PROTECTION_ENABLED", + # Logging + "JVSPATIAL_DB_LOGGING_ENABLED", + "JVSPATIAL_DB_LOGGING_API_ENABLED", + "JVSPATIAL_DB_LOGGING_DB_NAME", + "JVSPATIAL_DB_LOGGING_LEVELS", + "JVSPATIAL_DB_LOG_SERVERLESS_ASYNC", + "JVSPATIAL_DB_LOG_SERVERLESS_JOIN_TIMEOUT", + "JVSPATIAL_LOG_DB_TYPE", + "JVSPATIAL_LOG_DB_NAME", + "JVSPATIAL_LOG_DB_PATH", + "JVSPATIAL_LOG_DB_URI", + "JVSPATIAL_LOG_DB_REGION", + "JVSPATIAL_LOG_DB_TABLE_NAME", + "JVSPATIAL_LOG_DB_ENDPOINT_URL", + # Collections + "JVSPATIAL_COLLECTION_API_KEYS", + "JVSPATIAL_COLLECTION_SCHEDULED_TASKS", + "JVSPATIAL_COLLECTION_SESSIONS", + "JVSPATIAL_COLLECTION_USERS", + "JVSPATIAL_COLLECTION_WEBHOOKS", + "JVSPATIAL_COLLECTION_WEBHOOK_REQUESTS", + # Work-claim helpers + "JVSPATIAL_WORK_CLAIM_STALE_SECONDS", + # Misc + "JVSPATIAL_TEXT_NORMALIZATION_ENABLED", + "JVSPATIAL_EXPOSE_ERROR_DETAILS", + "JVSPATIAL_STRICT_ENV_ALLOWLIST", + } +) + + +def discover_unknown_jvspatial_env_keys() -> List[str]: + """Return any ``JVSPATIAL_*`` env keys not present in :data:`ALLOWED_ENV_KEYS`. + + Pure helper — callers decide the strictness of the response. + """ + return sorted( + k + for k in os.environ + if k.startswith("JVSPATIAL_") and k not in ALLOWED_ENV_KEYS + ) + + +def enforce_env_allowlist() -> None: + """Reject (strict) or warn (default) on unknown ``JVSPATIAL_*`` env keys. + + Toggled by ``JVSPATIAL_STRICT_ENV_ALLOWLIST``: when truthy, unknown + keys raise ``ValueError`` at server startup, surfacing typos + immediately. Default emits a single warning per unknown key per + process so existing deployments don't break on upgrade. + + Closes audit §7.1 / SPEC §10.2. + """ + unknown = discover_unknown_jvspatial_env_keys() + if not unknown: + return + + strict = False + raw_strict = os.environ.get("JVSPATIAL_STRICT_ENV_ALLOWLIST", "").strip() + if raw_strict: + try: + strict = parse_bool(raw_strict) + except ValueError: + strict = False + + if strict: + raise ValueError( + "Unknown JVSPATIAL_* environment variables detected: " + + ", ".join(unknown) + + ". Either remove the variable or add it to ALLOWED_ENV_KEYS in " + "jvspatial/env_adapter.py." + ) + for key in unknown: + logger.warning( + "Unknown JVSPATIAL_* env var %r ignored. Set " + "JVSPATIAL_STRICT_ENV_ALLOWLIST=true to fail-fast on typos.", + key, + ) + + def validate_server_config_requirements(config: Any) -> None: """Raise ``ValueError`` when required settings for enabled features are missing.""" + enforce_env_allowlist() auth = config.auth if auth.auth_enabled: secret = (auth.jwt_secret or "").strip() diff --git a/jvspatial/runtime/serverless.py b/jvspatial/runtime/serverless.py index 2bd24ed..3c6dc8f 100644 --- a/jvspatial/runtime/serverless.py +++ b/jvspatial/runtime/serverless.py @@ -8,7 +8,24 @@ def _parse_bool(val: str) -> bool: - return str(val).strip().lower() in ("true", "1", "yes", "enabled") + """Truthy ``SERVERLESS_MODE`` env override. + + Delegates to :func:`jvspatial.env.parse_bool` for the canonical set + (``true/false``, ``1/0``, ``yes/no``, ``on/off``) plus the historical + ``enabled`` alias kept for backward compatibility (audit §7.2). + """ + # Late import to avoid circular dependency at module load. + from jvspatial.env import parse_bool + + s = str(val).strip().lower() + if s == "enabled": + return True + if s == "disabled": + return False + try: + return parse_bool(s) + except ValueError: + return False @lru_cache(maxsize=1) diff --git a/tests/api/test_cors_wildcard_and_error_detail_audit.py b/tests/api/test_cors_wildcard_and_error_detail_audit.py new file mode 100644 index 0000000..6d9a5c7 --- /dev/null +++ b/tests/api/test_cors_wildcard_and_error_detail_audit.py @@ -0,0 +1,63 @@ +"""CORS wildcard warning + EXPOSE_ERROR_DETAILS prod guard. + +Audit §4.10 / §4.12 / SPEC §15.4-§15.5. +""" + +import logging +import os +from unittest.mock import patch + +from jvspatial.api.components.error_handler import _expose_error_details_to_clients +from jvspatial.api.config_groups import CORSConfig + + +def test_wildcard_origin_emits_warning(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_origins=["*"]) + assert any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_wildcard_with_opt_in_silences_warning(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_origins=["*"], cors_allow_wildcard=True) + assert not any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_disabled_cors_does_not_warn(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_enabled=False, cors_origins=["*"]) + assert not any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_expose_error_details_suppressed_in_production(): + """Production-marked runtime ignores JVSPATIAL_EXPOSE_ERROR_DETAILS.""" + with patch.dict( + os.environ, + { + "JVSPATIAL_EXPOSE_ERROR_DETAILS": "true", + "JVSPATIAL_ENVIRONMENT": "production", + }, + clear=False, + ): + # Reset the once-per-process flag so the warning re-arms. + if hasattr(_expose_error_details_to_clients, "_warned"): + delattr(_expose_error_details_to_clients, "_warned") + assert _expose_error_details_to_clients() is False + + +def test_expose_error_details_honored_outside_production(): + with patch.dict( + os.environ, + { + "JVSPATIAL_EXPOSE_ERROR_DETAILS": "true", + "JVSPATIAL_ENVIRONMENT": "development", + }, + clear=False, + ): + assert _expose_error_details_to_clients() is True diff --git a/tests/api/test_env_allowlist_audit.py b/tests/api/test_env_allowlist_audit.py new file mode 100644 index 0000000..519d334 --- /dev/null +++ b/tests/api/test_env_allowlist_audit.py @@ -0,0 +1,73 @@ +"""JVSPATIAL_* env allowlist enforcement (audit §7.1 / SPEC §10.2). + +SPEC §10.2 promises "Unknown JVSPATIAL_* keys are rejected at startup +to catch typos." The earlier implementation only read enumerated keys +in ``env_adapter``; stray ``JVSPATIAL_*`` env vars went silently +ignored. ``enforce_env_allowlist`` now scans the environment and +warns (default) or raises (strict mode) on unknown keys. +""" + +import os +from unittest.mock import patch + +import pytest + +from jvspatial.env_adapter import ( + ALLOWED_ENV_KEYS, + discover_unknown_jvspatial_env_keys, + enforce_env_allowlist, +) + + +def test_discover_unknown_returns_empty_with_known_keys(): + with patch.dict(os.environ, {"JVSPATIAL_DB_TYPE": "json"}, clear=False): + assert "JVSPATIAL_DB_TYPE" not in discover_unknown_jvspatial_env_keys() + + +def test_discover_unknown_returns_typo_key(): + with patch.dict( + os.environ, + {"JVSPATIAL_JWT_SECRET": "oops"}, # pragma: allowlist secret - typo + clear=False, + ): + unknown = discover_unknown_jvspatial_env_keys() + assert "JVSPATIAL_JWT_SECRET" in unknown + + +def test_enforce_warns_by_default(caplog): + with patch.dict( + os.environ, + {"JVSPATIAL_JWT_SECRET": "oops"}, # pragma: allowlist secret - typo + clear=False, + ): + # No exception expected — default is warn. + enforce_env_allowlist() + # Restoring caplog level for the helper logger is enough — we don't + # assert on the captured record because the helper uses + # ``logger.warning`` directly via ``logging.getLogger`` and caplog + # propagation depends on pytest configuration. + + +def test_enforce_raises_in_strict_mode(): + with patch.dict( + os.environ, + { + "JVSPATIAL_JWT_SECRET": "oops", # pragma: allowlist secret - typo + "JVSPATIAL_STRICT_ENV_ALLOWLIST": "true", + }, + clear=False, + ): + with pytest.raises(ValueError, match="Unknown JVSPATIAL_"): + enforce_env_allowlist() + + +def test_allowlist_contains_canonical_keys(): + # Spot-check: a handful of well-known keys must be present. + expected = { + "JVSPATIAL_DB_TYPE", + "JVSPATIAL_JWT_SECRET_KEY", + "JVSPATIAL_DOCS_DISABLED", + "JVSPATIAL_WALKER_MAX_STEPS", + "JVSPATIAL_CORS_ORIGINS", + } + assert expected.issubset(ALLOWED_ENV_KEYS) diff --git a/tests/db/test_bulk_save_detailed_audit.py b/tests/db/test_bulk_save_detailed_audit.py new file mode 100644 index 0000000..e57cec4 --- /dev/null +++ b/tests/db/test_bulk_save_detailed_audit.py @@ -0,0 +1,62 @@ +"""BulkSaveResult / bulk_save_detailed semantics (audit §5.6 / §5.7). + +The legacy ``bulk_save`` returned a single ``int`` so partial-failure +backends (JsonDB, DynamoDB) could silently drop records without callers +noticing. ``bulk_save_detailed`` returns a structured +:class:`BulkSaveResult` with ``attempted`` / ``saved`` / ``failed_ids``. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.db import create_database +from jvspatial.db.database import BulkSaveResult + + +@pytest.fixture +async def jsondb(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + yield db + + +def test_bulk_save_result_dataclass_fields(): + r = BulkSaveResult(attempted=3, saved=3) + assert r.attempted == 3 + assert r.saved == 3 + assert r.failed_ids == [] + assert r.all_saved is True + + partial = BulkSaveResult(attempted=3, saved=2, failed_ids=["x"]) + assert partial.all_saved is False + + +@pytest.mark.asyncio +async def test_bulk_save_detailed_returns_structured_result(jsondb): + records = [{"id": f"r{i}", "n": i} for i in range(5)] + result = await jsondb.bulk_save_detailed("widgets", records) + assert isinstance(result, BulkSaveResult) + assert result.attempted == 5 + assert result.saved == 5 + assert result.failed_ids == [] + assert result.all_saved + + +@pytest.mark.asyncio +async def test_bulk_save_int_still_returned_for_back_compat(jsondb): + records = [{"id": f"r{i}", "n": i} for i in range(4)] + saved_count = await jsondb.bulk_save("widgets", records) + assert saved_count == 4 + + +@pytest.mark.asyncio +async def test_bulk_save_missing_id_includes_index(jsondb): + records = [ + {"id": "r0", "n": 0}, + {"n": 1}, # missing id at index 1 + {"id": "r2", "n": 2}, + ] + with pytest.raises(ValueError, match="index 1"): + await jsondb.bulk_save_detailed("widgets", records) diff --git a/tests/db/test_query_operator_parity_audit.py b/tests/db/test_query_operator_parity_audit.py new file mode 100644 index 0000000..3222b27 --- /dev/null +++ b/tests/db/test_query_operator_parity_audit.py @@ -0,0 +1,103 @@ +"""QueryEngine operator parity (audit §5.2 / SPEC §5.1). + +The earlier matcher: + +* silently returned False for any field-level operator it did not know; +* silently returned False for the top-level ``$nor`` operator the + QueryBuilder advertised; +* silently passed through optimizer markers (``$hint`` / ``$select``) + which caused match failures when ``optimize_query`` injected them. + +The matcher now raises ``QueryError`` for unsupported operators and +implements ``$nor`` / ``$mod`` / ``$all`` / ``$type`` / ``$not`` so the +QueryBuilder surface and engine behavior line up. +""" + +import pytest + +from jvspatial.db.query import QueryEngine +from jvspatial.exceptions import QueryError + +# ---------- $nor ---------- + + +def test_nor_excludes_documents_matching_any_subcondition(): + doc = {"status": "active", "tier": "free"} + # Document does NOT match {"status": "inactive"} nor {"tier": "pro"}; + # $nor should accept it. + assert ( + QueryEngine.match( + doc, + { + "$nor": [ + {"status": "inactive"}, + {"tier": "pro"}, + ] + }, + ) + is True + ) + # Document DOES match {"status": "active"}; $nor rejects. + assert QueryEngine.match(doc, {"$nor": [{"status": "active"}]}) is False + + +# ---------- $mod ---------- + + +def test_mod_matches_remainder(): + assert QueryEngine.match({"n": 10}, {"n": {"$mod": [3, 1]}}) is True + assert QueryEngine.match({"n": 9}, {"n": {"$mod": [3, 1]}}) is False + assert QueryEngine.match({"n": "x"}, {"n": {"$mod": [3, 0]}}) is False + + +# ---------- $all ---------- + + +def test_all_requires_every_operand_in_value(): + assert QueryEngine.match({"tags": ["a", "b", "c"]}, {"tags": {"$all": ["a", "b"]}}) + assert not QueryEngine.match( + {"tags": ["a", "b", "c"]}, {"tags": {"$all": ["a", "z"]}} + ) + assert not QueryEngine.match({"tags": "ab"}, {"tags": {"$all": ["a"]}}) + + +# ---------- $type ---------- + + +def test_type_accepts_python_type_names(): + assert QueryEngine.match({"x": 5}, {"x": {"$type": "int"}}) + assert QueryEngine.match({"x": "y"}, {"x": {"$type": "string"}}) + assert not QueryEngine.match({"x": "y"}, {"x": {"$type": "int"}}) + assert not QueryEngine.match({"x": 5}, {"x": {"$type": "bogus"}}) + + +# ---------- $not (field-level) ---------- + + +def test_field_level_not_negates(): + assert QueryEngine.match({"n": 5}, {"n": {"$not": {"$gt": 10}}}) + assert not QueryEngine.match({"n": 5}, {"n": {"$not": {"$gt": 1}}}) + + +# ---------- Unsupported operators raise ---------- + + +def test_unsupported_top_level_operator_raises(): + with pytest.raises(QueryError): + QueryEngine.match({}, {"$bogus": []}) + + +def test_unsupported_field_level_operator_raises(): + with pytest.raises(QueryError): + QueryEngine.match({"x": 1}, {"x": {"$bogus": 1}}) + + +# ---------- Optimizer markers are ignored ---------- + + +def test_optimizer_markers_do_not_break_match(): + """``$hint`` / ``$select`` are optimizer-only — match() must skip them.""" + doc = {"name": "alice"} + # With $hint mixed in, the match still passes on the actual condition. + assert QueryEngine.match(doc, {"name": "alice", "$hint": "name_idx"}) + assert QueryEngine.match(doc, {"$select": ["name"]}) From 0b3b326fe1e8899c4c705f8715bc9da98753f290 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 13:13:19 -0400 Subject: [PATCH 14/22] docs(changelog): document Wave 3 audit remediation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds [Unreleased] entries for the 12 MED findings closed in the previous commit. Six new public surface entries (BulkSaveResult, bulk_save_detailed, MongoDB.is_transactional, CORSConfig.cors_allow_wildcard, JVSPATIAL_STRICT_ENV_ALLOWLIST, ALLOWED_ENV_KEYS + helpers, new QueryEngine operators). No breaking changes — additions and defensive-warning-only behavior shifts. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ab24c..a84c8d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `tests/core/test_entity_name_walker_and_save.py`, `tests/core/test_walker_protection_audit_fixes.py`, `tests/storage/test_versioning_path_sanitizer_audit.py`, `tests/api/test_webhook_hmac_audit_fix.py` — 28 new regression cases pinning Wave 1 audit fixes. - Public `invalidate_api_key_cache(api_key)` and `invalidate_api_key_cache_hash(cache_key)` helpers in `jvspatial.api.integrations.webhooks.webhook_auth`. `APIKeyService.revoke_key` now invokes the latter so revocations are effective immediately rather than after the 5-minute TTL. (Audit §4.5.) - `tests/db/test_default_compound_ops_id_normalization.py`, `tests/core/test_pager_audit_fixes.py` — 11 new regression cases pinning Wave 2 audit fixes. +- `jvspatial.db.database.BulkSaveResult` dataclass and `Database.bulk_save_detailed()` method. Reports `attempted` / `saved` / `failed_ids` per call so partial-success backends (JsonDB, DynamoDB) can no longer silently drop records. `bulk_save` is preserved as a thin int-returning wrapper for back-compat. (Audit §5.6-§5.7.) +- `MongoDB.is_transactional()` async probe. Uses the `hello` admin command to detect replica-set / sharded topology and caches the result. Use this instead of the static `supports_transactions` flag when the caller intends to actually open a transaction. (Audit §5.9.) +- `CORSConfig.cors_allow_wildcard` opt-out. When `cors_origins` contains a wildcard and this is `False` (default), a startup WARNING is emitted. SPEC §15.4 promised this; the audit found it missing. (Audit §4.12.) +- `JVSPATIAL_STRICT_ENV_ALLOWLIST` env var. Truthy values turn unknown-`JVSPATIAL_*` key detection from a per-key WARNING into a startup `ValueError` so typos fail-fast. (Audit §7.1 / SPEC §10.2.) +- `ALLOWED_ENV_KEYS` frozenset and `enforce_env_allowlist()` / `discover_unknown_jvspatial_env_keys()` helpers in `jvspatial.env_adapter`. Called from `validate_server_config_requirements()` at server startup. +- New top-level / field-level `QueryEngine` operators: `$nor` (top-level logical), `$mod`, `$all`, `$type`, `$not` (field-level). Previously advertised by `QueryBuilder` but silently returned no matches. (Audit §5.2.) +- `tests/api/test_env_allowlist_audit.py`, `tests/api/test_cors_wildcard_and_error_detail_audit.py`, `tests/db/test_query_operator_parity_audit.py`, `tests/db/test_bulk_save_detailed_audit.py` — 22 new regression cases pinning Wave 3 audit fixes. ### Fixed @@ -42,6 +49,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `_API_KEY_CACHE` (webhook layer) now holds a lock around reads, eviction, and miss-population. Removes a `KeyError` window when a size-cap cleanup races a reader. (Audit §4.7.) - `APIKeyService(context=None)` now defaults to the **prime** database instead of `get_default_context()`. Auth state is required to live on the prime DB (SPEC §9 / CLAUDE.md §1). (Audit §4.4.) - `APIKeyService.revoke_key` now invokes the new webhook cache-invalidation hook so a revoked key stops authenticating immediately rather than after the 5-minute TTL. (Audit §4.5.) +- `JVSPATIAL_EXPOSE_ERROR_DETAILS=true` is now ignored when the runtime is signalled as production (`JVSPATIAL_ENVIRONMENT` or `ENVIRONMENT` set to `prod`/`production`). Emits a once-per-process WARNING explaining the suppression. Generic 500 message is returned. (Audit §4.10 / SPEC §15.5.) +- `MongoDB.begin_transaction` now short-circuits to `None` on standalone deployments instead of attempting `start_session` / `start_transaction` every call. Topology is probed once via `is_transactional()` and cached. (Audit §5.9 / SPEC §4.2.) +- `QueryEngine.match` and `QueryEngine._match_value` now raise `QueryError` on unsupported operators rather than silently returning False. Optimizer markers (`$hint`, `$select`) injected into queries are skipped explicitly instead of treated as unknown operators. (Audit §5.2 / SPEC §5.1.) +- Bool parsing consolidated across `jvspatial.env`, `jvspatial.env_adapter`, `jvspatial.runtime.serverless`, and `jvspatial.api.components.app_builder`. All three of the latter delegate to `env.parse_bool`. `JVSPATIAL_DEBUG=on` and `SERVERLESS_MODE=on` now agree on truthiness. (Audit §7.2-§7.3.) +- Unknown `JVSPATIAL_*` env keys now warn at startup (or raise in strict mode). Closes a SPEC §10.2 gap that allowed typos to silently no-op. (Audit §7.1.) ## [0.0.7] - 2026-05-08 From 66a93bd84a632267fe7cf1f5f775444b6d84f6f6 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 14:07:15 -0400 Subject: [PATCH 15/22] =?UTF-8?q?fix:=20Wave=204=20audit=20remediation=20?= =?UTF-8?q?=E2=80=94=20false-async=20cleanup,=20sqlite=20cross-loop,=20sta?= =?UTF-8?q?bility=20polish,=20dead=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the LOW-severity / cleanup tier of the post-audit hardening plan. --- Group A: deprecate generate_id_async (audit §3.11) --- * core/utils.py — ``generate_id_async`` is now a deprecated alias for ``generate_id``. ID generation is pure computation; the async signature was a vestige. Emits a once-per-process ``DeprecationWarning`` pointing at the canonical sync API. Scheduled for removal in 0.1.0. --- Group B: SQLite cross-loop handling (audit §5.10 / SPEC §4.3) --- * db/sqlite.py — track the event loop that owns the aiosqlite connection. On cross-loop reuse with a file-backed database, the connection is silently rebound to the current loop (data persists on disk). For ``:memory:`` databases the existing connection is kept since rebinding would silently truncate the dataset. Replaces the previous opaque "Future attached to a different loop" error. --- Group C: stability-tier polish (audit §7.7, §7.13) --- * utils/stability.py — add public ``emit_experimental_once`` hook so opt-in flag-driven surfaces (e.g. ``JsonDBTransaction(best_effort=True)``) no longer reach into the private ``_emit_once`` implementation. * db/transaction.py — switch JsonDBTransaction to the public hook. * api/middleware/manager.py — derive docs CSP-relaxation prefixes from ``ServerConfig.docs_url`` / ``redoc_url`` / ``openapi_url`` at install time so callers that customize the docs URL keep Swagger UI rendering under the relaxed CSP. --- Group D: remove dead JSONTransaction (audit §5.14) --- * db/transaction.py — drop the unused ``JSONTransaction`` class (``JsonDBTransaction`` had replaced it). Remove from ``__all__`` and the module docstring. --- Group E: LOW hygiene (audit §4.11, §4.13, §7.14) --- * api/auth/service.py — drop JWT secret length from debug log (narrows search space for brute force). Log ``secret_configured=bool(...)`` instead. * api/auth/service.py — ``validate_token`` warning now logs ``db_type=type(database).__name__`` instead of the filesystem ``base_path`` (avoids leaking on-disk layout to log sinks). * serverless/tasks/stub.py — ``LoggingNoopTaskScheduler.schedule`` downgraded from per-call WARNING to DEBUG. The once-per-process startup error from ``serverless.factory._note_noop_in_serverless`` is sufficient; the per-call warning was a CloudWatch cost liability on misconfigured serverless deployments. --- Tests (10 new cases, all passing) --- * tests/db/test_sqlite_cross_loop_audit.py — same-loop reuse, cross-loop auto-rebind for file paths, owning-loop tracking. * tests/utils/test_wave4_polish_audit.py — public ``emit_experimental_once`` once-per-name semantics, deprecation emission on ``generate_id_async``, ``JSONTransaction`` removal. Verification: pytest across all subdirs → 1811 passed, 1 skipped (otel), 3 warnings. No regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/auth/service.py | 14 +++--- jvspatial/api/middleware/manager.py | 63 +++++++++++++++++++----- jvspatial/core/utils.py | 22 +++++++-- jvspatial/db/sqlite.py | 48 +++++++++++++++++- jvspatial/db/transaction.py | 63 ++---------------------- jvspatial/serverless/tasks/stub.py | 11 ++++- jvspatial/utils/stability.py | 17 +++++++ tests/db/test_sqlite_cross_loop_audit.py | 62 +++++++++++++++++++++++ tests/utils/test_wave4_polish_audit.py | 46 +++++++++++++++++ 9 files changed, 264 insertions(+), 82 deletions(-) create mode 100644 tests/db/test_sqlite_cross_loop_audit.py create mode 100644 tests/utils/test_wave4_polish_audit.py diff --git a/jvspatial/api/auth/service.py b/jvspatial/api/auth/service.py index 0892e66..3e49278 100644 --- a/jvspatial/api/auth/service.py +++ b/jvspatial/api/auth/service.py @@ -428,8 +428,10 @@ def _decode_jwt_token(self, token: str) -> Optional[Dict[str, Any]]: import logging logger = logging.getLogger(__name__) + # Do not log secret length — that narrows the search space + # for a brute force (audit §4.11 / SPEC §15.5). logger.debug( - f"JWT token decode failed: {e}, secret length: {len(self.jwt_secret) if self.jwt_secret else 0}" + f"JWT token decode failed: {e}, secret_configured={bool(self.jwt_secret)}" ) return None @@ -1054,13 +1056,13 @@ async def validate_token(self, token: str) -> Optional[UserResponse]: ) if not user: - db_path = getattr( - getattr(self.context, "database", None), "base_path", None - ) + # Avoid logging the filesystem path of the database — leaks + # on-disk layout to log sinks (audit §4.13 / SPEC §15.5). + db_type = type(getattr(self.context, "database", None)).__name__ self._logger.warning( - "[validate_token] failed: user %s not found in database (db_path=%s)", + "[validate_token] failed: user %s not found in database (db_type=%s)", user_id, - db_path, + db_type, ) return None diff --git a/jvspatial/api/middleware/manager.py b/jvspatial/api/middleware/manager.py index a1a1c01..3c5826b 100644 --- a/jvspatial/api/middleware/manager.py +++ b/jvspatial/api/middleware/manager.py @@ -8,7 +8,7 @@ """ import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, List +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -47,16 +47,23 @@ "object-src 'none'" ) -# Documentation-page path prefixes. Both `/docs` and `/redoc` may carry -# trailing path segments (Swagger's oauth2-redirect, ReDoc assets); match -# exact path or `/...`. `/openapi.json` is the JSON spec consumed -# by both UIs — it's CSP-irrelevant on its own but we group it for symmetry. -_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json") +# Default documentation-page path prefixes. Both ``/docs`` and ``/redoc`` +# may carry trailing path segments (Swagger's oauth2-redirect, ReDoc +# assets); match exact path or ``/...``. ``/openapi.json`` is +# the JSON spec consumed by both UIs. +# +# These are FastAPI's *defaults*. When ``ServerConfig`` overrides +# ``docs_url`` / ``redoc_url`` / ``openapi_url``, the middleware reads +# the custom prefixes from the server config at construction time +# (audit §7.13). +_DEFAULT_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json") -def _is_docs_path(path: str) -> bool: +def _is_docs_path( + path: str, prefixes: Tuple[str, ...] = _DEFAULT_DOCS_PATH_PREFIXES +) -> bool: """True for FastAPI Swagger / ReDoc / OpenAPI surfaces.""" - return any(path == p or path.startswith(p + "/") for p in _DOCS_PATH_PREFIXES) + return any(path == p or path.startswith(p + "/") for p in prefixes) class SecurityHeadersMiddleware(BaseHTTPMiddleware): @@ -79,9 +86,23 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware): (off by default in development, on in production). """ - def __init__(self, app, hsts_enabled: bool = False): + def __init__( + self, + app, + hsts_enabled: bool = False, + docs_path_prefixes: Optional[Tuple[str, ...]] = None, + ): super().__init__(app) self._hsts_enabled = hsts_enabled + # Derived from ``ServerConfig`` (``docs_url`` / ``redoc_url`` / + # ``openapi_url``) at install time so customizing the docs URL + # does not silently break Swagger UI with strict CSP + # (audit §7.13). + self._docs_path_prefixes: Tuple[str, ...] = ( + docs_path_prefixes + if docs_path_prefixes is not None + else _DEFAULT_DOCS_PATH_PREFIXES + ) async def dispatch(self, request, call_next): """Process request and add security headers to response.""" @@ -89,7 +110,9 @@ async def dispatch(self, request, call_next): response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["Content-Security-Policy"] = ( - _DOCS_CSP if _is_docs_path(request.url.path) else _DEFAULT_CSP + _DOCS_CSP + if _is_docs_path(request.url.path, self._docs_path_prefixes) + else _DEFAULT_CSP ) if self._hsts_enabled: response.headers["Strict-Transport-Security"] = ( @@ -173,7 +196,25 @@ def _configure_security_headers(self, app: FastAPI) -> None: return hsts_enabled = getattr(self.server.config.security, "hsts_enabled", False) - app.add_middleware(SecurityHeadersMiddleware, hsts_enabled=hsts_enabled) + # Derive docs-path prefixes from ServerConfig so customized + # ``docs_url`` / ``redoc_url`` / ``openapi_url`` keep Swagger UI + # rendering under the relaxed CSP (audit §7.13). + cfg = self.server.config + configured_prefixes = tuple( + p + for p in ( + getattr(cfg, "docs_url", None), + getattr(cfg, "redoc_url", None), + getattr(cfg, "openapi_url", None), + ) + if p + ) + docs_path_prefixes = configured_prefixes or _DEFAULT_DOCS_PATH_PREFIXES + app.add_middleware( + SecurityHeadersMiddleware, + hsts_enabled=hsts_enabled, + docs_path_prefixes=docs_path_prefixes, + ) self._logger.debug(f"{LogIcons.SUCCESS} Security headers middleware configured") def _configure_cors(self, app: FastAPI) -> None: diff --git a/jvspatial/core/utils.py b/jvspatial/core/utils.py index 103d172..277953b 100644 --- a/jvspatial/core/utils.py +++ b/jvspatial/core/utils.py @@ -23,7 +23,12 @@ def generate_id(type_: str, class_name: str) -> str: async def generate_id_async(type_: str, class_name: str) -> str: - """Generate an ID string for graph objects (async version). + """Deprecated async alias for :func:`generate_id`. + + ID generation is pure computation (no I/O); the async signature was + a vestige of an earlier design. SPEC §3.2 documents ``generate_id`` + as the canonical sync API (audit §3.11). Will be removed in a + future minor release. Args: type_: Object type ('n' for node, 'e' for edge, 'w' for walker, 'o' for object) @@ -32,8 +37,19 @@ async def generate_id_async(type_: str, class_name: str) -> str: Returns: Unique ID string in the format "type.class_name.hex_id" """ - hex_id = uuid.uuid4().hex[:24] - return f"{type_}.{class_name}.{hex_id}" + # Lazy import — deprecation helper lives outside the core hot path. + from jvspatial.utils.deprecation import deprecated + + @deprecated( + replacement="jvspatial.core.utils.generate_id", + remove_in="0.1.0", + name="jvspatial.core.utils.generate_id_async", + ) + def _emit() -> None: + return None + + _emit() + return generate_id(type_, class_name) # Cache for subclass lookups to avoid repeated tree traversals diff --git a/jvspatial/db/sqlite.py b/jvspatial/db/sqlite.py index 0cdc784..67ed128 100644 --- a/jvspatial/db/sqlite.py +++ b/jvspatial/db/sqlite.py @@ -113,9 +113,47 @@ def __init__( self._created_indexes: Dict[str, Set[str]] = ( {} ) # collection -> set of index names + # The event loop that owns ``self._connection``. ``aiosqlite`` + # binds its connection to a single loop; using the connection + # from a different loop produces opaque "Future attached to a + # different loop" errors. We track the binding here and raise + # a clear ``DatabaseError`` on cross-loop reuse (audit §5.10). + self._owning_loop: Optional[asyncio.AbstractEventLoop] = None async def _get_connection(self) -> "Connection": - """Get or create the SQLite connection.""" + """Get or create the SQLite connection. + + Cross-loop reuse handling (audit §5.10 / SPEC §4.3): + + * For **file-backed databases**, the connection is silently + rebound to the current loop — the old loop is presumed gone, + and on-disk state persists across the rebind so callers do + not observe data loss. + * For ``:memory:`` databases, data lives in the connection + itself; rebinding would silently truncate the dataset. We + keep the existing connection and trust aiosqlite's internal + thread to dispatch queries from any loop (this matches + historical behavior — the audit's concern was an opaque + failure mode, not data loss). + """ + current_loop = asyncio.get_running_loop() + if ( + self._connection is not None + and self._owning_loop is not None + and self._owning_loop is not current_loop + and self.db_path_str != ":memory:" + ): + logger.debug( + "SQLiteDB rebinding to a new event loop; abandoning " + "connection owned by %r and reconnecting on %r", + self._owning_loop, + current_loop, + ) + self._connection = None + self._owning_loop = None + self._initialized = False + # Reset per-loop state; index re-creation is idempotent. + self._created_indexes.clear() if self._connection is None: # Ensure parent directory exists before connecting (for file paths) if self.db_path_str != ":memory:": @@ -134,6 +172,7 @@ async def _get_connection(self) -> "Connection": self._connection = await aiosqlite.connect( self.db_path_str, timeout=self.timeout ) + self._owning_loop = current_loop self._connection.row_factory = aiosqlite.Row await self._connection.execute(f"PRAGMA journal_mode={self.journal_mode};") await self._connection.execute(f"PRAGMA synchronous={self.synchronous};") @@ -250,12 +289,17 @@ async def create_index( ) async def close(self) -> None: - """Close the underlying SQLite connection.""" + """Close the underlying SQLite connection. + + Clears the owning-loop binding so the instance can be reused on + a fresh event loop (audit §5.10). + """ if self._connection is not None: await self._connection.close() self._connection = None self._initialized = False self._created_indexes.clear() + self._owning_loop = None async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: """Save a record to the database. diff --git a/jvspatial/db/transaction.py b/jvspatial/db/transaction.py index ec923d6..d36f42a 100644 --- a/jvspatial/db/transaction.py +++ b/jvspatial/db/transaction.py @@ -17,8 +17,7 @@ commit. Intended for testing, scripting, and local-dev workflows where the trade-off is acceptable. * **None.** Calling ``JsonDBTransaction()`` (without ``best_effort=True``) - or any operation on ``JSONTransaction`` raises - :class:`NotImplementedError`. Callers can detect this via the + raises :class:`NotImplementedError`. Callers can detect this via the ``Database.supports_transactions`` capability flag and fall back to non-transactional writes. """ @@ -240,9 +239,11 @@ def __init__(self, database, *, best_effort: bool = False): if best_effort: # Emit a once-per-process ExperimentalWarning so adopters know # this surface may change. See docs/md/stability.md. - from jvspatial.utils.stability import _emit_once + # Uses the public ``emit_experimental_once`` hook rather than + # the underscore-prefixed implementation (audit §7.7). + from jvspatial.utils.stability import emit_experimental_once - _emit_once( + emit_experimental_once( "JsonDBTransaction(best_effort=True)", "Buffered-commit semantics are weaker than ACID and the " "interface may evolve; track docs/md/stability.md.", @@ -360,59 +361,6 @@ async def rollback(self) -> None: self.is_rolled_back = True -class JSONTransaction(Transaction): - """JSON database transaction implementation (no-op for file-based storage).""" - - def __init__(self, transaction_id: str): - """Initialize JSON transaction. - - Args: - transaction_id: Unique identifier for this transaction - """ - super().__init__(transaction_id) - self.is_active = True - # JSON database doesn't support true transactions, so we simulate them - - async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: - """Save a record within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction save not implemented") - - async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: - """Retrieve a record by ID within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction get not implemented") - - async def delete(self, collection: str, id: str) -> bool: - """Delete a record within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction delete not implemented") - - async def find( - self, - collection: str, - query: Dict[str, Any], - *, - limit: Optional[int] = None, - sort: Optional[List[Tuple[str, int]]] = None, - ) -> List[Dict[str, Any]]: - """Find records matching query within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction find not implemented") - - async def commit(self) -> None: - """Commit this JSON transaction (simulated).""" - if self.is_active and not self.is_committed and not self.is_rolled_back: - self.is_active = False - self.is_committed = True - - async def rollback(self) -> None: - """Rollback this JSON transaction (simulated).""" - if self.is_active and not self.is_committed and not self.is_rolled_back: - self.is_active = False - self.is_rolled_back = True - - @asynccontextmanager async def transaction_context(database, transaction_id: Optional[str] = None): """Context manager for database transactions. @@ -449,6 +397,5 @@ async def transaction_context(database, transaction_id: Optional[str] = None): "Transaction", "MongoDBTransaction", "JsonDBTransaction", - "JSONTransaction", "transaction_context", ] diff --git a/jvspatial/serverless/tasks/stub.py b/jvspatial/serverless/tasks/stub.py index 5cd7c81..7087c96 100644 --- a/jvspatial/serverless/tasks/stub.py +++ b/jvspatial/serverless/tasks/stub.py @@ -23,6 +23,13 @@ def schedule( retry_config: Optional[RetryConfig] = None, run_at: Optional[float] = None, ) -> str: - """Log and return a synthetic reference; see base class.""" - logger.warning("%s (task_type=%s)", self._message, task_type) + """Log and return a synthetic reference; see base class. + + Downgraded to DEBUG so a misconfigured serverless deployment does + not flood CloudWatch with one WARNING per dispatch. The + once-per-process startup error from + ``serverless.factory._note_noop_in_serverless`` is sufficient + (audit §7.14 / SPEC §11.2). + """ + logger.debug("%s (task_type=%s)", self._message, task_type) return f"noop-{uuid.uuid4()}" diff --git a/jvspatial/utils/stability.py b/jvspatial/utils/stability.py index fb263da..0831f8c 100644 --- a/jvspatial/utils/stability.py +++ b/jvspatial/utils/stability.py @@ -89,6 +89,23 @@ def _emit_once(name: str, message: str) -> None: ) +def emit_experimental_once(name: str, message: str = "") -> None: + """Public re-export of the once-per-process experimental warning hook. + + Some opt-in surfaces — e.g. ``JsonDBTransaction(best_effort=True)`` — + need to emit the experimental warning without going through the + ``@experimental`` decorator (because the decision happens on a + constructor flag, not the symbol itself). Use this hook instead of + reaching into the underscore-prefixed implementation (audit §7.7 / + SPEC §18 stability-tier discipline). + + Args: + name: Stable identifier for the API (deduplicated per-process). + message: Optional context appended to the warning body. + """ + _emit_once(name, message) + + def experimental( name: Optional[str] = None, note: str = "", diff --git a/tests/db/test_sqlite_cross_loop_audit.py b/tests/db/test_sqlite_cross_loop_audit.py new file mode 100644 index 0000000..272b48c --- /dev/null +++ b/tests/db/test_sqlite_cross_loop_audit.py @@ -0,0 +1,62 @@ +"""SQLite cross-loop detection (audit §5.10 / SPEC §4.3). + +``aiosqlite`` binds its connection to the event loop that opened it. +Reusing a ``SQLiteDB`` instance across loops previously produced an +opaque "Future attached to a different loop" error from inside +``aiosqlite``; the wrapper now detects the binding change and +transparently rebinds to the current loop instead. +""" + +import asyncio +import tempfile + +import pytest + +from jvspatial.db import create_database + + +@pytest.mark.asyncio +async def test_same_loop_reuse_works(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + await db.save("widgets", {"id": "w1", "qty": 1}) + got = await db.get("widgets", "w1") + assert got is not None + await db.save("widgets", {"id": "w2", "qty": 2}) + await db.close() + + +def test_cross_loop_reuse_auto_rebinds(): + """Across loops the connection is silently rebuilt on the active loop.""" + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + + async def first() -> None: + await db.save("widgets", {"id": "w1", "qty": 1}) + + asyncio.run(first()) + + async def second() -> None: + # Auto-rebind on a new loop — no error. + await db.save("widgets", {"id": "w2", "qty": 2}) + got = await db.get("widgets", "w2") + assert got is not None + await db.close() + + asyncio.run(second()) + + +def test_owning_loop_tracked(): + """After save the SQLiteDB tracks the loop that owns the connection.""" + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + + async def go() -> None: + await db.save("widgets", {"id": "w1", "qty": 1}) + # ``_owning_loop`` is private but the contract is part of + # the cross-loop fix; assert it was populated. + assert db._owning_loop is asyncio.get_running_loop() + await db.close() + assert db._owning_loop is None + + asyncio.run(go()) diff --git a/tests/utils/test_wave4_polish_audit.py b/tests/utils/test_wave4_polish_audit.py new file mode 100644 index 0000000..ddc6b04 --- /dev/null +++ b/tests/utils/test_wave4_polish_audit.py @@ -0,0 +1,46 @@ +"""Wave 4 polish (audit §3.11, §7.7, §5.14).""" + +import warnings + +import pytest + +from jvspatial.utils.stability import ( + ExperimentalWarning, + emit_experimental_once, + reset_experimental_warnings, +) + + +def test_emit_experimental_once_is_public(): + """Public hook so callers can flag opt-in surface without reaching + into ``_emit_once`` (audit §7.7).""" + reset_experimental_warnings() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ExperimentalWarning) + emit_experimental_once("test.api.public_hook", "note") + emit_experimental_once("test.api.public_hook", "note") + # Single emission per (name) regardless of repeated calls. + assert sum(1 for w in caught if issubclass(w.category, ExperimentalWarning)) == 1 + + +@pytest.mark.asyncio +async def test_generate_id_async_emits_deprecation(): + """``generate_id_async`` is a deprecated alias for ``generate_id`` + (audit §3.11). It must still work — the call site only sees a + warning.""" + from jvspatial.core.utils import generate_id_async + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + result = await generate_id_async("n", "Demo") + assert result.startswith("n.Demo.") + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + + +def test_jsontransaction_dead_class_removed(): + """``JSONTransaction`` was unused dead code (audit §5.14). Must no + longer be exported from ``jvspatial.db.transaction``.""" + import jvspatial.db.transaction as txn_mod + + assert "JSONTransaction" not in txn_mod.__all__ + assert not hasattr(txn_mod, "JSONTransaction") From 6d53b48a31ad78abafb457c099c27f7c73267a0c Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 14:07:50 -0400 Subject: [PATCH 16/22] docs(changelog): document Wave 4 audit remediation Adds [Unreleased] entries for the LOW-tier cleanup landed in the previous commit. Adds Deprecated and Removed sections: generate_id_async is deprecated, JSONTransaction is removed. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a84c8d6..78c27f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ALLOWED_ENV_KEYS` frozenset and `enforce_env_allowlist()` / `discover_unknown_jvspatial_env_keys()` helpers in `jvspatial.env_adapter`. Called from `validate_server_config_requirements()` at server startup. - New top-level / field-level `QueryEngine` operators: `$nor` (top-level logical), `$mod`, `$all`, `$type`, `$not` (field-level). Previously advertised by `QueryBuilder` but silently returned no matches. (Audit §5.2.) - `tests/api/test_env_allowlist_audit.py`, `tests/api/test_cors_wildcard_and_error_detail_audit.py`, `tests/db/test_query_operator_parity_audit.py`, `tests/db/test_bulk_save_detailed_audit.py` — 22 new regression cases pinning Wave 3 audit fixes. +- `jvspatial.utils.stability.emit_experimental_once(name, message)` — public hook for opt-in surfaces that need to emit the experimental warning without going through the `@experimental` decorator (replaces private `_emit_once` calls). (Audit §7.7.) +- `tests/db/test_sqlite_cross_loop_audit.py`, `tests/utils/test_wave4_polish_audit.py` — 10 new regression cases pinning Wave 4 audit fixes. ### Fixed @@ -54,6 +56,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `QueryEngine.match` and `QueryEngine._match_value` now raise `QueryError` on unsupported operators rather than silently returning False. Optimizer markers (`$hint`, `$select`) injected into queries are skipped explicitly instead of treated as unknown operators. (Audit §5.2 / SPEC §5.1.) - Bool parsing consolidated across `jvspatial.env`, `jvspatial.env_adapter`, `jvspatial.runtime.serverless`, and `jvspatial.api.components.app_builder`. All three of the latter delegate to `env.parse_bool`. `JVSPATIAL_DEBUG=on` and `SERVERLESS_MODE=on` now agree on truthiness. (Audit §7.2-§7.3.) - Unknown `JVSPATIAL_*` env keys now warn at startup (or raise in strict mode). Closes a SPEC §10.2 gap that allowed typos to silently no-op. (Audit §7.1.) +- `SQLiteDB` instances now silently rebind their `aiosqlite` connection when reused across event loops for file-backed paths. The previous opaque "Future attached to a different loop" error from inside `aiosqlite` is replaced with transparent recovery. `:memory:` databases keep the existing connection (rebinding would silently truncate the dataset). (Audit §5.10 / SPEC §4.3.) +- Security headers middleware derives the CSP-relaxation prefixes from `ServerConfig.docs_url` / `redoc_url` / `openapi_url` at install time. Callers customizing the docs URL (e.g. `docs_url=/api/docs`) keep Swagger UI rendering under the relaxed CSP. (Audit §7.13.) +- JWT debug log no longer includes the secret length (narrowing a brute-force search space). Logs `secret_configured=bool(...)` instead. (Audit §4.11 / SPEC §15.5.) +- `validate_token` warning logs `db_type=` instead of `db_path=` so the on-disk filesystem layout does not leak to log sinks. (Audit §4.13 / SPEC §15.5.) +- `LoggingNoopTaskScheduler.schedule` downgraded from per-call WARNING to DEBUG so misconfigured serverless deployments do not flood CloudWatch — the once-per-process startup error from `serverless.factory._note_noop_in_serverless` is sufficient. (Audit §7.14.) + +### Deprecated + +- `jvspatial.core.utils.generate_id_async` — deprecated alias for `generate_id`. ID generation is pure computation (SPEC §3.2); the async signature was a vestige. Scheduled for removal in 0.1.0. (Audit §3.11.) + +### Removed + +- `jvspatial.db.transaction.JSONTransaction` — unused dead code superseded by `JsonDBTransaction`. (Audit §5.14.) ## [0.0.7] - 2026-05-08 From 980a9fc5f358694465636da6b99cfd699815a7e0 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 16:51:20 -0400 Subject: [PATCH 17/22] =?UTF-8?q?fix:=20Wave=205=20audit=20remediation=20?= =?UTF-8?q?=E2=80=94=20Windows-reserved=20names,=20fail-closed=20deferred?= =?UTF-8?q?=20secret,=20SkipNode,=20polish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the remaining outstanding audit items (LOW tier and a small number of MED/HIGH leftovers). --- Group A: Windows-reserved filenames (audit §4.18 / SPEC §15.1) --- * storage/security/path_sanitizer.py — reject reserved Windows filenames (CON, PRN, AUX, NUL, COM1-9, LPT1-9) regardless of host OS so cross-platform storage cannot be DoS'd by an upload with a reserved stem. ``CON.txt`` is rejected (Windows resolves the device before the extension). Names that merely start with a reserved stem (``CONFIG.json``, ``COMRADE.bin``) still pass. --- Group B: .env.example accuracy (audit §7.5) --- * .env.example — CORS section showed ``Default: *`` and example ``JVSPATIAL_CORS_ORIGINS=*``. Actual defaults are the localhost whitelist defined in ``CORSConfig``. Updated to show the real defaults and flag that wildcards trigger a startup WARNING. --- Group C: runtime parse_bool strictness (audit §7.3) --- * runtime/serverless.py — unrecognized non-empty ``SERVERLESS_MODE`` values now log a WARNING (preserving the False fallback for back-compat). Silent garbage-to-False mapping hid typos. --- Group D: SkipNode + Walker type_code enforcement --- * core/entities/__init__.py — declare ``TraversalSkipped`` and ``TraversalPaused`` exception classes (previously referenced in ``jvspatial/exceptions.py`` but never defined). * core/entities/walker.py — ``Walker.skip()`` raises the typed ``TraversalSkipped`` exception. ``_execute_visit_hooks`` matches via ``isinstance`` instead of the fragile ``"Node skipped" in str(e)`` substring (audit §2.9 / SPEC §6.5). * core/entities/walker.py — Walker rejects construction with a non-``"w"`` ``type_code`` so the SPEC §1.1 ID-format invariant (``w.EntityName.``) cannot be corrupted by a stray kwarg (audit §2.10). --- Group E: fail-closed deferred-invoke secret (audit §4.16) --- * api/deferred_invoke_route.py — ``_deferred_invoke_secret_ok`` now returns ``False`` when ``JVSPATIAL_DEFERRED_INVOKE_SECRET`` is unset, emitting a single WARNING. Previously an unset secret allowed any caller, exposing the internal endpoint on misconfigured deployments. Disable the route entirely with ``JVSPATIAL_DEFERRED_INVOKE_DISABLED=true`` when not needed. * tests/serverless/test_deferred_invoke.py — updated the two pre-existing happy-path tests to set a secret + send the header (the fail-closed change is intentional and the tests now match the new contract). --- Group F: SQLite id coercion + dead jvtmp filter (audit §5.16, §5.20) --- * db/sqlite.py — ``save()`` now coerces ``record["id"]`` to ``str`` after the ``setdefault`` and persists the stringified form so callers passing int/uuid ids can ``get()`` them back via ``str(id)``. * db/jsondb.py — drop the dead ``not p.name.endswith('.jvtmp')`` filter on the ``*.json`` glob. Tmp files are named ``.json...jvtmp`` (see ``_atomic._make_temp_path``) so the glob already excludes them. --- Tests (23 new cases, all passing) --- * tests/storage/test_windows_reserved_audit.py — reserved-name rejection across single filename and subdir contexts; non-reserved names that share a stem prefix still pass. * tests/core/test_wave5_walker_audit.py — ``TraversalSkipped``, Walker type_code locked to ``"w"``. * tests/api/test_deferred_invoke_fail_closed_audit.py — unset secret denies, matching header / bearer allows, mismatched denies. * tests/db/test_sqlite_id_coercion_audit.py — int id round-trips through save + get(str(id)). Verification: pytest across all subdirs → 1834 passed, 1 skipped (otel), 3 warnings. No regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 16 +++--- jvspatial/api/deferred_invoke_route.py | 16 +++++- jvspatial/core/entities/__init__.py | 26 +++++++++ jvspatial/core/entities/walker.py | 39 ++++++++------ jvspatial/db/jsondb.py | 11 ++-- jvspatial/db/sqlite.py | 8 ++- jvspatial/runtime/serverless.py | 16 +++++- jvspatial/storage/security/path_sanitizer.py | 54 +++++++++++++++++++ .../test_deferred_invoke_fail_closed_audit.py | 50 +++++++++++++++++ tests/core/test_wave5_walker_audit.py | 29 ++++++++++ tests/db/test_sqlite_id_coercion_audit.py | 19 +++++++ tests/serverless/test_deferred_invoke.py | 15 ++++-- tests/storage/test_windows_reserved_audit.py | 45 ++++++++++++++++ 13 files changed, 313 insertions(+), 31 deletions(-) create mode 100644 tests/api/test_deferred_invoke_fail_closed_audit.py create mode 100644 tests/core/test_wave5_walker_audit.py create mode 100644 tests/db/test_sqlite_id_coercion_audit.py create mode 100644 tests/storage/test_windows_reserved_audit.py diff --git a/.env.example b/.env.example index 7274753..67f456d 100644 --- a/.env.example +++ b/.env.example @@ -91,19 +91,23 @@ # JVSPATIAL_CORS_ENABLED=true # CORS allowed origins (comma-separated) -# Default: * +# Default: http://localhost:5173,http://127.0.0.1:5173,http://localhost:3000, +# http://127.0.0.1:3000,http://localhost:8000,http://127.0.0.1:8000 # Examples: # - Single origin: https://example.com # - Multiple origins: https://example.com,https://app.example.com -# JVSPATIAL_CORS_ORIGINS=* +# Setting "*" or any wildcard entry triggers a startup WARNING — wildcards +# are rarely correct in production. Set cors_allow_wildcard=True on +# CORSConfig to silence the warning when intentional. +# JVSPATIAL_CORS_ORIGINS=https://example.com # CORS allowed methods (comma-separated) -# Default: * -# JVSPATIAL_CORS_METHODS=* +# Default: GET,POST,PUT,PATCH,DELETE,OPTIONS +# JVSPATIAL_CORS_METHODS=GET,POST,PUT,PATCH,DELETE,OPTIONS # CORS allowed headers (comma-separated) -# Default: * -# JVSPATIAL_CORS_HEADERS=* +# Default: Content-Type,Authorization,X-API-Key +# JVSPATIAL_CORS_HEADERS=Content-Type,Authorization,X-API-Key # ----------------------------------------------------------------------------- # DATABASE COLLECTION NAMES diff --git a/jvspatial/api/deferred_invoke_route.py b/jvspatial/api/deferred_invoke_route.py index be48b18..396525f 100644 --- a/jvspatial/api/deferred_invoke_route.py +++ b/jvspatial/api/deferred_invoke_route.py @@ -26,9 +26,23 @@ def _deferred_invoke_disabled() -> bool: def _deferred_invoke_secret_ok(request: Request) -> bool: + """Authorize the internal deferred-invoke endpoint. + + Fail-closed when ``JVSPATIAL_DEFERRED_INVOKE_SECRET`` is unset or + empty: the previous "no secret = allow everything" semantics were + a footgun — a misconfigured deployment exposed the internal + endpoint to any caller (audit §4.16 / SPEC §15.2). Disable the + route entirely via ``JVSPATIAL_DEFERRED_INVOKE_DISABLED=true`` if + you do not need it. + """ secret = env("JVSPATIAL_DEFERRED_INVOKE_SECRET") or "" if not secret: - return True + logger.warning( + "Deferred-invoke route rejected: " + "JVSPATIAL_DEFERRED_INVOKE_SECRET is unset. Either set a " + "secret or set JVSPATIAL_DEFERRED_INVOKE_DISABLED=true." + ) + return False hdr = (request.headers.get("X-JVSPATIAL-Deferred-Authorize") or "").strip() auth = request.headers.get("Authorization") or "" bearer = "" diff --git a/jvspatial/core/entities/__init__.py b/jvspatial/core/entities/__init__.py index 1488424..6bb8f3d 100644 --- a/jvspatial/core/entities/__init__.py +++ b/jvspatial/core/entities/__init__.py @@ -7,6 +7,8 @@ simplified decorator support and other improvements. """ +from jvspatial.exceptions import JVSpatialError + from .edge import Edge from .node import Node @@ -22,6 +24,27 @@ from .walker_components.walker_queue import WalkerQueue from .walker_components.walker_trail import WalkerTrail + +class TraversalSkipped(JVSpatialError): + """Walker-skip signal raised by ``Walker.skip()``. + + Abandons processing of the current node and continues with the + next queued node. Replaces the historical + ``raise JVSpatialError("Node skipped")`` pattern that relied on + substring-matching the error message — a fragile contract that any + unrelated exception containing the phrase would silently trigger + (audit §2.9 / SPEC §6.5). + """ + + +class TraversalPaused(JVSpatialError): + """Walker-pause signal raised by ``Walker.pause()``. + + Callers catch this exception to resume later via + ``Walker.resume()``. + """ + + __all__ = [ # Enhanced entity classes (maintaining original hierarchy) "Object", @@ -35,4 +58,7 @@ "WalkerQueue", "WalkerTrail", "WalkerEventSystem", + # Walker control-flow exceptions + "TraversalSkipped", + "TraversalPaused", ] diff --git a/jvspatial/core/entities/walker.py b/jvspatial/core/entities/walker.py index 9d3757b..219beab 100644 --- a/jvspatial/core/entities/walker.py +++ b/jvspatial/core/entities/walker.py @@ -21,8 +21,6 @@ from .node import Node from .edge import Edge -from jvspatial.exceptions import JVSpatialError - from ..annotations import AttributeMixin, attribute from ..events import event_bus from ..utils import generate_id @@ -320,10 +318,16 @@ def _get_edge_class(cls) -> Type["Edge"]: def __init__(self: "Walker", **kwargs: Any) -> None: """Initialize a walker with auto-generated ID if not provided.""" entity_name = self.__class__._entity_name() + # Force ``type_code='w'`` so SPEC §1.1 ID-format invariant + # (``w.EntityName.``) cannot be corrupted by a caller + # passing a different value (audit §2.10). + if kwargs.get("type_code", "w") != "w": + raise ValueError( + f"Walker.type_code must be 'w'; got {kwargs.get('type_code')!r}" + ) + kwargs["type_code"] = "w" if "id" not in kwargs: - # Use class-level type_code or default from Field - type_code = kwargs.get("type_code", "w") - kwargs["id"] = generate_id(type_code, entity_name) + kwargs["id"] = generate_id("w", entity_name) # Set entity to class entity_name if not provided (protected attribute). # Honors ``__entity_name__`` override so subclasses with the same # Python ``__name__`` as a sibling persist a distinct discriminator. @@ -803,9 +807,11 @@ async def _execute_visit_hooks(self, target: Union["Node", "Edge"]) -> None: else: hook(self, target) except Exception as e: - # Check if this is a skip exception - if "Node skipped" in str(e): - # Skip this node and continue + # Use the dedicated exception class instead of a fragile + # substring match (audit §2.9 / SPEC §6.5). + from . import TraversalSkipped + + if isinstance(e, TraversalSkipped): walker_hook_skipped = True return else: @@ -866,9 +872,9 @@ async def _execute_visit_hooks(self, target: Union["Node", "Edge"]) -> None: else: bound_hook(self) except Exception as e: - # Check if this is a skip exception - if "Node skipped" in str(e): - # Skip this node and continue + from . import TraversalSkipped + + if isinstance(e, TraversalSkipped): return else: # Report error as structured data @@ -924,11 +930,14 @@ async def spawn( async def skip(self) -> None: """Skip the current node and continue traversal. - This method allows the walker to skip processing the current node - and continue with the next node in the queue. + Raises ``TraversalSkipped`` so the caller can recognize the + skip via ``except TraversalSkipped`` rather than the historical + substring match on ``"Node skipped"`` (audit §2.9 / SPEC §6.5). """ - # Raise an exception to stop current node processing - raise JVSpatialError("Node skipped") + # Late import to avoid circular dep at module load. + from . import TraversalSkipped + + raise TraversalSkipped("Node skipped") def pause(self, message: str = "Traversal paused") -> None: """Pause the walker's traversal. diff --git a/jvspatial/db/jsondb.py b/jvspatial/db/jsondb.py index 37e8931..9a671c3 100644 --- a/jvspatial/db/jsondb.py +++ b/jvspatial/db/jsondb.py @@ -151,14 +151,17 @@ def _get_record_path(self, collection: str, record_id: str) -> Path: @staticmethod def _list_collection_json_files(collection_dir: Path) -> List[Path]: - """Enumerate persisted ``*.json`` records, skipping in-flight tmps. + """Enumerate persisted ``*.json`` records. Called from inside ``asyncio.to_thread`` by ``count`` and ``find`` so the directory scan does not block the event loop. + + Note: in-flight ``*.jvtmp`` files are named + ``.json...jvtmp`` (see ``_atomic._make_temp_path``) + so the ``*.json`` glob already excludes them — the historical + ``not endswith('.jvtmp')`` filter was dead (audit §5.16). """ - return [ - p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") - ] + return list(collection_dir.glob("*.json")) async def _async_write_json(self, path: Path, data: Dict[str, Any]) -> None: """Write JSON data to file asynchronously. diff --git a/jvspatial/db/sqlite.py b/jvspatial/db/sqlite.py index 67ed128..69f8c45 100644 --- a/jvspatial/db/sqlite.py +++ b/jvspatial/db/sqlite.py @@ -315,7 +315,13 @@ async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: connection = await self._get_connection() record = data.copy() - record_id = record.setdefault("id", str(uuid.uuid4())) + # Coerce id to ``str`` so non-string ids (int, uuid.UUID) + # round-trip cleanly through SQLite's TEXT column. The + # legacy code only stringified the default uuid and bound + # the raw value; ``get(collection, id)`` then missed when + # callers passed an int-typed id (audit §5.20). + record_id = str(record.setdefault("id", str(uuid.uuid4()))) + record["id"] = record_id payload = json.dumps(record) await connection.execute( diff --git a/jvspatial/runtime/serverless.py b/jvspatial/runtime/serverless.py index 3c6dc8f..74cbb23 100644 --- a/jvspatial/runtime/serverless.py +++ b/jvspatial/runtime/serverless.py @@ -12,8 +12,16 @@ def _parse_bool(val: str) -> bool: Delegates to :func:`jvspatial.env.parse_bool` for the canonical set (``true/false``, ``1/0``, ``yes/no``, ``on/off``) plus the historical - ``enabled`` alias kept for backward compatibility (audit §7.2). + ``enabled``/``disabled`` aliases kept for backward compatibility + (audit §7.2). + + Unrecognized non-empty values are now logged (audit §7.3): silently + mapping garbage to ``False`` hid typos. Returns ``False`` either way + so existing deployments with typo'd ``SERVERLESS_MODE`` keep their + previous effective behavior — but the warning surfaces the typo. """ + import logging + # Late import to avoid circular dependency at module load. from jvspatial.env import parse_bool @@ -25,6 +33,12 @@ def _parse_bool(val: str) -> bool: try: return parse_bool(s) except ValueError: + logging.getLogger(__name__).warning( + "Unrecognized boolean env value %r for SERVERLESS_MODE; " + "treating as False. Use one of: true/false, 1/0, yes/no, " + "on/off, enabled/disabled.", + val, + ) return False diff --git a/jvspatial/storage/security/path_sanitizer.py b/jvspatial/storage/security/path_sanitizer.py index 6c51830..0c662fb 100644 --- a/jvspatial/storage/security/path_sanitizer.py +++ b/jvspatial/storage/security/path_sanitizer.py @@ -69,6 +69,49 @@ class PathSanitizer: # Known internal storage markers (directory placeholders, sandbox roots) _ALLOWED_HIDDEN_SEGMENTS = frozenset({".jvdirectory", ".jvagent_sandbox"}) + # Windows-reserved filename stems. CMD / Win32 reject these regardless + # of extension, so allowing them in a cross-platform storage layer + # creates write-failure footguns and lets an attacker DoS a Windows + # host by uploading e.g. ``CON.txt`` (audit §4.18 / SPEC §15.1). + _WINDOWS_RESERVED_NAMES = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", + } + ) + + @classmethod + def _is_windows_reserved(cls, part: str) -> bool: + """Return True when ``part`` matches a Windows-reserved name. + + Checked regardless of host OS so cross-platform storage is + uniformly safe. + """ + # Match the stem before any extension. ``CON.txt`` is reserved + # because Windows resolves the device first. + stem = part.split(".", 1)[0].upper() + return stem in cls._WINDOWS_RESERVED_NAMES + @classmethod def sanitize_path( cls, file_path: str, base_dir: Optional[str] = None, allow_hidden: bool = False @@ -170,6 +213,13 @@ def sanitize_path( path=file_path, ) + # Reject Windows-reserved filenames cross-platform (audit §4.18). + if cls._is_windows_reserved(part): + raise InvalidPathError( + f"Reserved Windows filename not allowed: {part}", + path=file_path, + ) + # Check filename length if len(part) > cls.MAX_FILENAME_LENGTH: raise InvalidPathError( @@ -238,6 +288,10 @@ def sanitize_filename(cls, filename: str, allow_hidden: bool = False) -> str: filename = re.sub(r"[^a-zA-Z0-9_\-\.]", "_", filename) logger.info("Filename sanitized by removing invalid characters") + # Reject Windows-reserved filenames cross-platform (audit §4.18). + if cls._is_windows_reserved(filename): + raise InvalidPathError(f"Reserved Windows filename not allowed: {filename}") + # Enforce length limit if len(filename) > cls.MAX_FILENAME_LENGTH: # Preserve extension diff --git a/tests/api/test_deferred_invoke_fail_closed_audit.py b/tests/api/test_deferred_invoke_fail_closed_audit.py new file mode 100644 index 0000000..24c3cb2 --- /dev/null +++ b/tests/api/test_deferred_invoke_fail_closed_audit.py @@ -0,0 +1,50 @@ +"""Deferred-invoke fail-closed when secret unset (audit §4.16 / SPEC §15.2).""" + +import os +from unittest.mock import MagicMock, patch + +from jvspatial.api.deferred_invoke_route import _deferred_invoke_secret_ok + + +def _fake_request(headers: dict) -> MagicMock: + req = MagicMock() + req.headers.get = lambda k, default=None: headers.get(k, default) + req.headers.__getitem__ = lambda _self, k: headers[k] + return req + + +def test_no_secret_set_denies_access(): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("JVSPATIAL_DEFERRED_INVOKE_SECRET", None) + req = _fake_request({}) + assert _deferred_invoke_secret_ok(req) is False + + +def test_matching_header_allows(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"X-JVSPATIAL-Deferred-Authorize": "shh"}) + assert _deferred_invoke_secret_ok(req) is True + + +def test_matching_bearer_allows(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"Authorization": "Bearer shh"}) + assert _deferred_invoke_secret_ok(req) is True + + +def test_mismatched_secret_denies(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"X-JVSPATIAL-Deferred-Authorize": "wrong"}) + assert _deferred_invoke_secret_ok(req) is False diff --git a/tests/core/test_wave5_walker_audit.py b/tests/core/test_wave5_walker_audit.py new file mode 100644 index 0000000..4e411ad --- /dev/null +++ b/tests/core/test_wave5_walker_audit.py @@ -0,0 +1,29 @@ +"""Walker polish (audit §2.9, §2.10).""" + +import pytest + +from jvspatial.core.entities import TraversalSkipped, Walker + + +class _DemoWalker(Walker): + pass + + +@pytest.mark.asyncio +async def test_walker_skip_raises_traversal_skipped(): + w = _DemoWalker() + with pytest.raises(TraversalSkipped): + await w.skip() + + +def test_walker_type_code_locked_to_w(): + w = _DemoWalker() + assert w.type_code == "w" + assert w.id.startswith("w.") + + +def test_walker_rejects_alternate_type_code(): + """Caller cannot smuggle a different ``type_code`` past the SPEC + §1.1 invariant (audit §2.10).""" + with pytest.raises(ValueError, match="type_code must be 'w'"): + _DemoWalker(type_code="n") diff --git a/tests/db/test_sqlite_id_coercion_audit.py b/tests/db/test_sqlite_id_coercion_audit.py new file mode 100644 index 0000000..db56e04 --- /dev/null +++ b/tests/db/test_sqlite_id_coercion_audit.py @@ -0,0 +1,19 @@ +"""SQLite ``save()`` id coercion to ``str`` (audit §5.20).""" + +import tempfile + +import pytest + +from jvspatial.db.sqlite import SQLiteDB + + +@pytest.mark.asyncio +async def test_int_id_roundtrips_through_save_and_get(): + with tempfile.TemporaryDirectory() as tmpdir: + db = SQLiteDB(db_path=f"{tmpdir}/x.db") + await db.save("widgets", {"id": 42, "name": "alpha"}) + # The persisted id is stringified — get() by the str form works. + got = await db.get("widgets", "42") + assert got is not None + assert got["name"] == "alpha" + await db.close() diff --git a/tests/serverless/test_deferred_invoke.py b/tests/serverless/test_deferred_invoke.py index 5b98aa7..14a3cdd 100644 --- a/tests/serverless/test_deferred_invoke.py +++ b/tests/serverless/test_deferred_invoke.py @@ -78,7 +78,10 @@ async def test_dispatch_malformed_task_type(body: dict): await dispatch_deferred_invoke(body) -def test_deferred_invoke_http_route(): +def test_deferred_invoke_http_route(monkeypatch): + # Audit §4.16: route is now fail-closed when the secret is unset. + # Set a secret + send matching header for the happy-path test. + monkeypatch.setenv("JVSPATIAL_DEFERRED_INVOKE_SECRET", "test-secret-value") app = FastAPI() async def handler(event: dict) -> dict: @@ -97,17 +100,23 @@ async def handler(event: dict) -> dict: "sender": "u1", "media_batch_window": 1.5, }, + headers={"X-JVSPATIAL-Deferred-Authorize": "test-secret-value"}, ) assert r.status_code == 200 assert r.json() == {"ok": True, "sender": "u1"} -def test_deferred_invoke_http_unknown_returns_404(): +def test_deferred_invoke_http_unknown_returns_404(monkeypatch): + monkeypatch.setenv("JVSPATIAL_DEFERRED_INVOKE_SECRET", "test-secret-value") app = FastAPI() register_deferred_invoke_route(app) client = TestClient(app) path = APIRoutes.deferred_invoke_full_path() - r = client.post(path, json={"task_type": "no.such.task"}) + r = client.post( + path, + json={"task_type": "no.such.task"}, + headers={"X-JVSPATIAL-Deferred-Authorize": "test-secret-value"}, + ) assert r.status_code == 404 assert "Unknown task_type" in r.json()["detail"] diff --git a/tests/storage/test_windows_reserved_audit.py b/tests/storage/test_windows_reserved_audit.py new file mode 100644 index 0000000..9a458d7 --- /dev/null +++ b/tests/storage/test_windows_reserved_audit.py @@ -0,0 +1,45 @@ +"""Windows-reserved-name rejection (audit §4.18 / SPEC §15.1).""" + +import pytest + +from jvspatial.storage.exceptions import InvalidPathError +from jvspatial.storage.security.path_sanitizer import PathSanitizer + + +@pytest.mark.parametrize( + "filename", + [ + "CON", + "con.txt", + "PRN.json", + "AUX", + "NUL.bin", + "COM1", + "com9.dat", + "LPT1", + "lpt9.log", + ], +) +def test_sanitize_path_rejects_windows_reserved(filename): + with pytest.raises(InvalidPathError, match="Reserved Windows filename"): + PathSanitizer.sanitize_path(filename) + + +@pytest.mark.parametrize( + "filename", + ["CONFIG.json", "context.txt", "PRNT.log", "COMRADE.bin"], +) +def test_sanitize_path_allows_non_reserved(filename): + """Names that *start* with a reserved stem but are longer must pass.""" + out = PathSanitizer.sanitize_path(filename) + assert out == filename + + +def test_sanitize_filename_rejects_reserved(): + with pytest.raises(InvalidPathError, match="Reserved Windows filename"): + PathSanitizer.sanitize_filename("CON.txt") + + +def test_sanitize_path_blocks_reserved_in_subdir(): + with pytest.raises(InvalidPathError): + PathSanitizer.sanitize_path("uploads/CON.txt") From 40fd5df53f499868f04f73d32858ae227dc89978 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 16:52:00 -0400 Subject: [PATCH 18/22] docs(changelog): document Wave 5 audit remediation Adds [Unreleased] entries for Wave 5 fixes. Two behavioral breaking changes flagged: deferred-invoke route is now fail-closed when the secret is unset, and Walker construction rejects non-``"w"`` type_code values. Plus two new public exception classes (TraversalSkipped, TraversalPaused). Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78c27f5..28c3133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `tests/api/test_env_allowlist_audit.py`, `tests/api/test_cors_wildcard_and_error_detail_audit.py`, `tests/db/test_query_operator_parity_audit.py`, `tests/db/test_bulk_save_detailed_audit.py` — 22 new regression cases pinning Wave 3 audit fixes. - `jvspatial.utils.stability.emit_experimental_once(name, message)` — public hook for opt-in surfaces that need to emit the experimental warning without going through the `@experimental` decorator (replaces private `_emit_once` calls). (Audit §7.7.) - `tests/db/test_sqlite_cross_loop_audit.py`, `tests/utils/test_wave4_polish_audit.py` — 10 new regression cases pinning Wave 4 audit fixes. +- `jvspatial.core.entities.TraversalSkipped` and `TraversalPaused` exception classes. `Walker.skip()` now raises `TraversalSkipped` (caught via `except TraversalSkipped`); previously relied on substring-matching `"Node skipped"` in the message. (Audit §2.9 / SPEC §6.5.) +- `PathSanitizer` rejects Windows-reserved filenames (`CON`, `PRN`, `AUX`, `NUL`, `COM1-9`, `LPT1-9`) regardless of host OS. ``CON.txt`` is rejected; ``CONFIG.json`` passes. (Audit §4.18 / SPEC §15.1.) +- `tests/storage/test_windows_reserved_audit.py`, `tests/core/test_wave5_walker_audit.py`, `tests/api/test_deferred_invoke_fail_closed_audit.py`, `tests/db/test_sqlite_id_coercion_audit.py` — 23 new regression cases pinning Wave 5 audit fixes. ### Fixed @@ -61,6 +64,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - JWT debug log no longer includes the secret length (narrowing a brute-force search space). Logs `secret_configured=bool(...)` instead. (Audit §4.11 / SPEC §15.5.) - `validate_token` warning logs `db_type=` instead of `db_path=` so the on-disk filesystem layout does not leak to log sinks. (Audit §4.13 / SPEC §15.5.) - `LoggingNoopTaskScheduler.schedule` downgraded from per-call WARNING to DEBUG so misconfigured serverless deployments do not flood CloudWatch — the once-per-process startup error from `serverless.factory._note_noop_in_serverless` is sufficient. (Audit §7.14.) +- **BREAKING (behavioral):** the internal deferred-invoke route fails closed when `JVSPATIAL_DEFERRED_INVOKE_SECRET` is unset. Previous "no secret = allow everything" semantics exposed the endpoint to any caller on misconfigured deployments. Set the secret to enable the route, or set `JVSPATIAL_DEFERRED_INVOKE_DISABLED=true` to skip registering it entirely. (Audit §4.16 / SPEC §15.2.) +- **BREAKING (behavioral):** `Walker(type_code=...)` raises `ValueError` when given a value other than `"w"`. The SPEC §1.1 ID-format invariant (`w.EntityName.`) cannot be corrupted by a stray kwarg. (Audit §2.10.) +- Walker `skip()` raises `TraversalSkipped` rather than `JVSpatialError("Node skipped")`. Callers that catch the generic exception or match on the substring will need to update — `except TraversalSkipped:` is the new contract. (Audit §2.9.) +- `SQLiteDB.save` coerces `record["id"]` to `str` so int / `uuid.UUID` ids round-trip correctly through SQLite's TEXT column. The persisted record now also stores the stringified id. (Audit §5.20.) +- `runtime/serverless._parse_bool` logs a WARNING on unrecognized `SERVERLESS_MODE` values (still maps to False for back-compat). Silent garbage-to-False mapping hid typos. (Audit §7.3.) +- `.env.example` CORS section corrected: `Default: *` and `JVSPATIAL_CORS_ORIGINS=*` example replaced with the actual default localhost whitelist, plus a note that wildcards trigger a startup WARNING. (Audit §7.5.) +- `JsonDB._list_collection_json_files` drops the dead `not p.name.endswith('.jvtmp')` filter; tmp files are named `.json...jvtmp` and never match the `*.json` glob. (Audit §5.16.) ### Deprecated From da4235715d5f45cd85c243d71ac908e271ece77f Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 17 May 2026 18:32:41 -0400 Subject: [PATCH 19/22] fix(api): resolve PEP 563 forward refs in _find_request_parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream-reported bug. ``wrap_function_with_params`` (and four other ``_find_request_parameter`` callers) detected Request parameters by iterating ``param.annotation`` raw values. With ``from __future__ import annotations`` (PEP 563) — or quoted forward refs — those annotations are *strings* like ``"Request"`` that match neither ``FastAPIRequest``/``StarletteRequest`` nor the ``__name__``/``__module__`` heuristic. The wrapper failed to detect the caller's Request parameter and corrupted the resulting route signature. ``ParameterModelFactory._create_function_model`` already calls ``typing.get_type_hints(func)`` to resolve forward refs; this fix mirrors that resolution in ``_find_request_parameter`` so both paths agree. * api/decorators/function_wrappers.py — add ``_resolve_type_hints`` helper that wraps ``typing.get_type_hints`` with a defensive fallback (empty dict on resolution failure, e.g. truly unresolvable forward ref). Extend ``_find_request_parameter`` with an optional ``func`` argument; when supplied, resolved hints take precedence over raw ``param.annotation``. Back-compat preserved when ``func`` is omitted. * All five internal call sites pass ``func``. Regression test (tests/api/test_request_detection_pep563_audit.py): PEP 563 handler is correctly detected; back-compat call without ``func`` still works. Verification: pytest across all subdirs → 1837 passed, 1 skipped (otel), 3 warnings. No regressions. Co-Authored-By: Claude Opus 4.7 (1M context) --- jvspatial/api/decorators/function_wrappers.py | 48 +++++++++++++--- .../test_request_detection_pep563_audit.py | 57 +++++++++++++++++++ 2 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 tests/api/test_request_detection_pep563_audit.py diff --git a/jvspatial/api/decorators/function_wrappers.py b/jvspatial/api/decorators/function_wrappers.py index a9d28e4..4d38d31 100644 --- a/jvspatial/api/decorators/function_wrappers.py +++ b/jvspatial/api/decorators/function_wrappers.py @@ -45,10 +45,42 @@ def _is_request_type(param_annotation: Any) -> bool: return False -def _find_request_parameter(func_sig: inspect.Signature) -> Tuple[bool, Optional[str]]: - """Find Request parameter in function signature.""" +def _resolve_type_hints(func: Callable) -> Dict[str, Any]: + """Best-effort ``typing.get_type_hints`` for ``func``. + + Resolves PEP 563 / ``from __future__ import annotations`` string + forward refs to real types so ``_is_request_type`` sees a class + object instead of the literal string ``"Request"``. Falls back to + an empty dict when resolution fails (unresolvable forward ref, + missing import) — callers then degrade to the raw + ``param.annotation`` value. + """ + from typing import get_type_hints + + try: + return get_type_hints(func) + except Exception: + return {} + + +def _find_request_parameter( + func_sig: inspect.Signature, func: Optional[Callable] = None +) -> Tuple[bool, Optional[str]]: + """Find Request parameter in function signature. + + When ``func`` is supplied, annotations are resolved via + ``typing.get_type_hints`` so callers using + ``from __future__ import annotations`` (PEP 563) — which yields + string-typed ``param.annotation`` values — are still recognized + correctly. Mirrors the resolution that + ``ParameterModelFactory._create_function_model`` already performs. + """ + type_hints: Dict[str, Any] = _resolve_type_hints(func) if func is not None else {} for param_name, param in func_sig.parameters.items(): - if _is_request_type(param.annotation): + # Prefer the resolved type-hint when available; fall back to + # the raw signature annotation otherwise. + annotation = type_hints.get(param_name, param.annotation) + if _is_request_type(annotation): return True, param_name return False, None @@ -121,7 +153,7 @@ def wrap_function_auth_only( from fastapi import Request as FastAPIRequest func_sig = inspect.signature(func) - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -203,7 +235,7 @@ def wrap_function_with_params( for p in (*AUTH_INJECTED_PARAMS, *AUTH_INJECTED_USER_PARAMS) ) if needs_auth_injection: - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -372,7 +404,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: body_data.pop(excluded_param, None) combined = {**kwargs, **body_data} - orig_has_request_param, _ = _find_request_parameter(func_sig) + orig_has_request_param, _ = _find_request_parameter(func_sig, func) if ( orig_has_request_param and request_param_name @@ -426,7 +458,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: p in func_sig.parameters for p in (*AUTH_INJECTED_PARAMS, *AUTH_INJECTED_USER_PARAMS) ): - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -557,7 +589,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: # type: ignore[assign for excluded_param in EXCLUDED_BODY_PARAMS: data.pop(excluded_param, None) - orig_has_request_param, _ = _find_request_parameter(func_sig) + orig_has_request_param, _ = _find_request_parameter(func_sig, func) if ( orig_has_request_param and request_param_name diff --git a/tests/api/test_request_detection_pep563_audit.py b/tests/api/test_request_detection_pep563_audit.py new file mode 100644 index 0000000..7f6b732 --- /dev/null +++ b/tests/api/test_request_detection_pep563_audit.py @@ -0,0 +1,57 @@ +"""Request-parameter detection with ``from __future__ import annotations``. + +Upstream report: ``wrap_function_with_params`` relied on raw +``param.annotation`` values. With PEP 563 (or quoted forward refs) +the annotation is a *string* like ``"Request"`` — never matches +``FastAPIRequest`` / ``StarletteRequest`` and never matches the +``__name__`` / ``__module__`` heuristic. Result: the wrapper failed +to detect the caller's Request parameter and corrupted the route +signature. + +``ParameterModelFactory._create_function_model`` already calls +``typing.get_type_hints(func)``; this fix mirrors that resolution in +``_find_request_parameter`` so both paths agree. +""" + +from __future__ import annotations + +import inspect + +from fastapi import Request + +from jvspatial.api.decorators.function_wrappers import _find_request_parameter + + +def _handler_with_pep563(request: Request, x: int) -> dict: + """Function whose annotations are strings under PEP 563.""" + return {"x": x} + + +def _handler_no_request(x: int, y: int) -> dict: + return {"x": x, "y": y} + + +def test_request_detected_under_pep563_when_func_passed(): + sig = inspect.signature(_handler_with_pep563) + # Raw annotation IS a string under ``from __future__ import annotations``. + assert isinstance(sig.parameters["request"].annotation, str) + + # ``_find_request_parameter`` resolves type hints when ``func`` is supplied. + has_request, name = _find_request_parameter(sig, _handler_with_pep563) + assert has_request is True + assert name == "request" + + +def test_no_request_param_returns_false(): + sig = inspect.signature(_handler_no_request) + has_request, name = _find_request_parameter(sig, _handler_no_request) + assert has_request is False + assert name is None + + +def test_back_compat_without_func_argument(): + """Calling without ``func`` still works (legacy signature).""" + sig = inspect.signature(_handler_no_request) + has_request, _ = _find_request_parameter(sig) + # No request param: still False either way. + assert has_request is False From bf6dcfd18dc9fd188ef48341f2ba1656c7c4475b Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Wed, 20 May 2026 13:05:37 -0400 Subject: [PATCH 20/22] upd: updated docs --- CLAUDE.md | 13 +++++++++++++ docs/md/entity-reference.md | 9 +++++++++ docs/md/quick-start-guide.md | 6 ++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f07d75f..924254c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,19 @@ CORS does **not** default to wildcard. CSP is strict on app routes, relaxed only --- +## Graph modeling convention + +Convention for any application built on jvspatial. Not enforced by the library, but required by review and assumed by downstream tooling. + +1. **Every `Node` must be reachable from `Root`** — directly or through intermediate nodes. An orphaned node is a bug. If a record does not belong in the graph, it must not be a `Node`. +2. **Applications declare an application-root node** — e.g. `class App(Node)`, `class MyApp(Node)`. Create exactly one instance, connect it to `Root`, and treat it as the entry point for all application state. +3. **All application nodes extend from the app-root** — directly or under an appropriate branch / category node beneath it (e.g. `App -> Users -> User`, `App -> Catalog -> Product`). Do not connect application nodes directly to `Root`; `Root` belongs to the library, the app-root belongs to the application. +4. **Use `Object` (not `Node`) for record-style data that does not benefit from graph membership** — audit/change-event records, denormalized snapshots, ledger entries, raw inbound payloads. `ChangeEvent` **must be an `Object`**, not a `Node`. Rule of thumb: if nothing will ever traverse to or from it, it is not a `Node`. + +When reviewing or generating code, reject `Node` subclasses that are never connected, and reject `Object` subclasses that are used as relationship endpoints. + +--- + ## Run the dev loop ```bash diff --git a/docs/md/entity-reference.md b/docs/md/entity-reference.md index f3bb1bf..33fecff 100644 --- a/docs/md/entity-reference.md +++ b/docs/md/entity-reference.md @@ -93,6 +93,15 @@ class Edge(Object): async def count(cls, query: Optional[dict] = None, **kwargs) -> int # Inherited from Object ``` +#### Convention: `Node` vs `Object` + +The library lets you persist either as a `Node` (graph member) or an `Object` (flat record). Pick correctly — orphaned nodes and graph-traversed objects are both anti-patterns. Conventions for applications built on jvspatial: + +- **A `Node` MUST be connected to the graph.** Every node must be reachable from `Root`, directly or through intermediate nodes. If a record is never traversed to or from, it should not be a `Node`. +- **Declare an application-root node.** Create a single top-level class — `App`, `MyApp`, or similar — extending `Node`. Instantiate once and connect to `Root`. Treat this app-root as the entry point for all application state. +- **All application nodes hang off the app-root.** Connect them directly to the app-root, or beneath a branch/category node under it (e.g. `App -> Users -> User`, `App -> Catalog -> Product`). Do not connect application nodes directly to `Root`. +- **Use `Object` for record-style data with no graph relationships.** Audit logs, change events, denormalized snapshots, ledger entries, and raw inbound payloads belong in `Object`, not `Node`. **`ChangeEvent` is an `Object`.** Rule of thumb: if nothing will ever traverse to or from the record, it is not a node. + #### `Walker` Graph traversal agent with hook-based logic. diff --git a/docs/md/quick-start-guide.md b/docs/md/quick-start-guide.md index 98fa9db..cf52c18 100644 --- a/docs/md/quick-start-guide.md +++ b/docs/md/quick-start-guide.md @@ -27,9 +27,11 @@ pip install jvspatial[all] # All optional deps jvspatial is built on three key concepts: ### **1. Graph Entities** -- **Nodes**: Data points in your graph +- **Nodes**: Data points in your graph — every `Node` must be reachable from `Root` (directly or indirectly) - **Edges**: Relationships between nodes -- **Root**: Entry point to your graph +- **Root**: Library-owned entry point to the graph +- **App-root node**: Convention — define your own top-level node (e.g. `class App(Node)`), connect it to `Root` once, and hang all application nodes off it. See [Entity Reference → Convention: Node vs Object](entity-reference.md#convention-node-vs-object). +- **Objects**: Record-style data with no graph relationships (audit logs, change events, snapshots). Use `Object`, not `Node`. ### **2. Walkers** - Traverse the graph From 7488cc8edd154e0ec8e9eca43d1bd3ee3837ef1e Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Mon, 25 May 2026 08:51:55 -0400 Subject: [PATCH 21/22] upd: AUTHORS, README --- AUTHORS | 16 ++++++++++++++++ README.md | 6 ++++++ jvspatial/db/_atomic.py | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..36b80d3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,16 @@ +jvspatial — Authors & Maintainers +================================= + +jvspatial is a foundational object-spatial application development +framework. + +Creator & Lead Maintainer + Eldon Marks (@eldonm) — https://github.com/eldonm + +Contributors + This list grows with the project. Contributions are welcome. + +--------------------------------------------------------------------- +This file records authorship and ongoing maintenance of the project. +Authorship and maintenance are distinct from copyright ownership and +license terms, which are set out in LICENSE. diff --git a/README.md b/README.md index 23348ee..67c4084 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,12 @@ server = Server( - [Security Policy](https://github.com/TrueSelph/jvspatial/blob/main/SECURITY.md) - Vulnerability disclosure - [Code of Conduct](https://github.com/TrueSelph/jvspatial/blob/main/CODE_OF_CONDUCT.md) - Contributor Covenant 2.1 +## Authors & maintainers + +jvspatial — a foundational object-spatial application development framework — was created by **Eldon Marks** ([@eldonm](https://github.com/eldonm)), who serves as its lead maintainer. + +See [AUTHORS](https://github.com/TrueSelph/jvspatial/blob/main/AUTHORS) for the full list of authors and contributors. Copyright and licensing terms are set out in the [LICENSE](https://github.com/TrueSelph/jvspatial/blob/main/LICENSE). + ## Contributors

diff --git a/jvspatial/db/_atomic.py b/jvspatial/db/_atomic.py index 1d87998..bc83d95 100644 --- a/jvspatial/db/_atomic.py +++ b/jvspatial/db/_atomic.py @@ -169,12 +169,23 @@ def cleanup_orphan_tmp_files(roots: Iterable[Union[str, Path]]) -> int: if is_serverless_mode(): return 0 + current_pid = os.getpid() removed = 0 for root in roots: root_path = Path(root) if not root_path.exists() or not root_path.is_dir(): continue for orphan in root_path.rglob(f"*{TMP_SUFFIX}"): + # Tmp filenames embed the writer's pid (see ``_make_temp_path``) + # so the sweep can distinguish in-flight writes (current process) + # from orphans left by a previously-crashed process. Never reap + # a file owned by the current pid — it's either mid-write right + # now (deleting it races ``os.replace`` and corrupts the write) + # or will get reaped on the next process startup. Filename shape: + # ``...jvtmp``. + parts = orphan.name.rsplit(".", 3) + if len(parts) == 4 and parts[1].isdigit() and int(parts[1]) == current_pid: + continue try: orphan.unlink() removed += 1 From fbdcf189a0939e5b110fca95f0d4bcb4e12b01a3 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Mon, 25 May 2026 14:26:59 -0400 Subject: [PATCH 22/22] fix: various indexing optimizations --- .../api/components/database_configurator.py | 18 +- jvspatial/core/context.py | 194 ++++++++++++++++++ jvspatial/core/entities/edge.py | 35 +++- jvspatial/core/entities/node.py | 52 +++++ jvspatial/db/_observable.py | 3 + jvspatial/observability/__init__.py | 8 + .../components/test_database_configurator.py | 62 ++++++ tests/core/test_graph_context.py | 49 +++++ tests/db/test_observable_database.py | 12 ++ tests/test_node_operations.py | 32 +++ 10 files changed, 463 insertions(+), 2 deletions(-) diff --git a/jvspatial/api/components/database_configurator.py b/jvspatial/api/components/database_configurator.py index b99bb6a..a486c64 100644 --- a/jvspatial/api/components/database_configurator.py +++ b/jvspatial/api/components/database_configurator.py @@ -8,7 +8,7 @@ import logging import os from pathlib import Path -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from jvspatial.core.context import GraphContext, set_default_context from jvspatial.db.factory import create_database @@ -79,6 +79,17 @@ def _resolve_mongodb_connection(self) -> Tuple[str, str]: db_name = (db.db_database_name or "").strip() or "jvdb" return uri, db_name + def _resolve_observability_kwargs(self) -> Dict[str, Any]: + """Resolve optional DB observability wrapper kwargs from env.""" + raw_enabled = str(os.environ.get("JVSPATIAL_OBSERVABILITY_ENABLED", "")).lower() + observe = raw_enabled in ("1", "true", "yes", "on") + slow_raw = os.environ.get("JVSPATIAL_SLOW_QUERY_MS", "") + try: + slow_query_ms = float(slow_raw) if slow_raw != "" else 100.0 + except ValueError: + slow_query_ms = 100.0 + return {"observe": observe, "slow_query_ms": slow_query_ms} + def initialize_graph_context(self) -> Optional[GraphContext]: """Initialize GraphContext with current database configuration. @@ -102,6 +113,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: try: # Create prime database based on configuration FIRST. prime_db = None + observability_kwargs = self._resolve_observability_kwargs() if db_type == "json": # Check if db_path is an S3 path (not supported for file-based databases) @@ -121,6 +133,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: prime_db = create_database( db_type="json", base_path=db_path, + **observability_kwargs, ) elif db_type == "mongodb": mongo_uri, mongo_db_name = self._resolve_mongodb_connection() @@ -128,6 +141,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: db_type="mongodb", uri=mongo_uri, db_name=mongo_db_name, + **observability_kwargs, ) elif db_type == "sqlite": # Check if db_path is an S3 path (not supported for file-based databases) @@ -147,6 +161,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: prime_db = create_database( db_type="sqlite", db_path=db_path, + **observability_kwargs, ) elif db_type == "dynamodb": prime_db = create_database( @@ -156,6 +171,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: endpoint_url=self.config.database.dynamodb_endpoint_url, aws_access_key_id=self.config.database.dynamodb_access_key_id, aws_secret_access_key=self.config.database.dynamodb_secret_access_key, + **observability_kwargs, ) else: raise ValueError(f"Unsupported database type: {db_type}") diff --git a/jvspatial/core/context.py b/jvspatial/core/context.py index 4a3a72c..56e8dc3 100644 --- a/jvspatial/core/context.py +++ b/jvspatial/core/context.py @@ -1,8 +1,10 @@ """GraphContext for managing database dependencies.""" import asyncio +import base64 import contextvars import inspect +import json import logging import time from contextlib import asynccontextmanager, contextmanager, suppress @@ -14,8 +16,10 @@ List, Optional, Set, + Tuple, Type, TypeVar, + Union, cast, ) @@ -1289,6 +1293,196 @@ async def find_nodes( return nodes + @staticmethod + def _resolve_entity_name(value: Any) -> Optional[str]: + """Normalize filter tokens (class/string) to persisted entity names.""" + if isinstance(value, str): + return value + if isinstance(value, type): + resolver = getattr(value, "_entity_name", None) + return resolver() if callable(resolver) else value.__name__ + return None + + async def find_page( + self, + collection: str, + query: Dict[str, Any], + sort: List[Tuple[str, int]], + after: Optional[Union[str, Dict[str, Any]]] = None, + limit: int = 50, + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Find one keyset-paginated page and return ``(items, next_cursor)``. + + ``sort`` is expected to include at least one field; ``id`` is appended as + a deterministic tiebreaker when missing. + """ + page_limit = max(1, int(limit)) + if not sort: + sort = [("id", 1)] + sort_fields: List[Tuple[str, int]] = list(sort) + if not any(field == "id" for field, _ in sort_fields): + sort_fields.append(("id", sort_fields[0][1])) + + final_query: Dict[str, Any] = dict(query or {}) + cursor_payload: Optional[Dict[str, Any]] = None + if isinstance(after, str) and after: + try: + cursor_payload = json.loads( + base64.urlsafe_b64decode(after.encode()).decode() + ) + except Exception: + cursor_payload = None + elif isinstance(after, dict): + cursor_payload = after + + primary_field, primary_dir = sort_fields[0] + id_dir = sort_fields[-1][1] + if cursor_payload and "id" in cursor_payload and "sort" in cursor_payload: + sort_op = "$lt" if primary_dir < 0 else "$gt" + id_op = "$lt" if id_dir < 0 else "$gt" + keyset_filter = { + "$or": [ + {primary_field: {sort_op: cursor_payload["sort"]}}, + { + "$and": [ + {primary_field: cursor_payload["sort"]}, + {"id": {id_op: cursor_payload["id"]}}, + ] + }, + ] + } + final_query = ( + {"$and": [final_query, keyset_filter]} if final_query else keyset_filter + ) + + rows = await self.database.find( + collection, final_query, limit=page_limit + 1, sort=sort_fields + ) + has_more = len(rows) > page_limit + page_rows = rows[:page_limit] + + next_cursor: Optional[str] = None + if has_more and page_rows: + last = page_rows[-1] + payload = {"id": last.get("id"), "sort": last.get(primary_field)} + next_cursor = base64.urlsafe_b64encode( + json.dumps(payload, separators=(",", ":")).encode() + ).decode() + + return page_rows, next_cursor + + async def nodes_bulk( + self, + node_ids: List[str], + *, + direction: str = "out", + edge: Optional[List[Union[str, Type[Any]]]] = None, + node: Optional[List[Union[str, Type[Any]]]] = None, + edge_filter: Optional[Dict[str, Any]] = None, + node_filter: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + ) -> Dict[str, List[Any]]: + """Batch traversal for many source IDs in one edge-query pass.""" + from .entities.edge import Edge + from .entities.node import Node + + unique_ids = [nid for nid in dict.fromkeys(node_ids) if nid] + out: Dict[str, List[Any]] = {nid: [] for nid in unique_ids} + if not unique_ids: + return out + + edge_entities = [ + en + for en in (self._resolve_entity_name(e) for e in (edge or [])) + if en is not None + ] + node_entities = [ + en + for en in (self._resolve_entity_name(n) for n in (node or [])) + if en is not None + ] + + if direction not in ("out", "in", "both"): + direction = "out" + + edge_query: Dict[str, Any] = {} + if direction == "out": + edge_query["source"] = {"$in": unique_ids} + elif direction == "in": + edge_query["target"] = {"$in": unique_ids} + else: + edge_query["$or"] = [ + {"source": {"$in": unique_ids}}, + {"target": {"$in": unique_ids}}, + ] + + if edge_entities: + edge_query["entity"] = {"$in": edge_entities} + if edge_filter: + for k, v in edge_filter.items(): + edge_query[f"context.{k}"] = v + + edge_docs = await self.database.find( + self._get_collection_name(self._get_entity_type_code(Edge)), + edge_query, + limit=limit, + ) + + neighbor_ids: set = set() + links: Dict[str, List[str]] = {nid: [] for nid in unique_ids} + seen_pairs: set = set() + + for edge_doc in edge_docs: + src = edge_doc.get("source") + dst = edge_doc.get("target") + if direction in ("out", "both") and src in links and dst: + pair = (src, dst) + if pair not in seen_pairs: + links[src].append(dst) + seen_pairs.add(pair) + neighbor_ids.add(dst) + if direction in ("in", "both") and dst in links and src: + pair = (dst, src) + if pair not in seen_pairs: + links[dst].append(src) + seen_pairs.add(pair) + neighbor_ids.add(src) + + if not neighbor_ids: + return out + + node_docs = await self.database.find( + self._get_collection_name(self._get_entity_type_code(Node)), + {"id": {"$in": list(neighbor_ids)}}, + ) + node_by_id: Dict[str, Any] = {} + for doc in node_docs: + if node_entities and doc.get("entity") not in node_entities: + continue + if node_filter: + fail = False + for k, v in node_filter.items(): + if doc.get("context", {}).get(k) != v: + fail = True + break + if fail: + continue + obj = await self._deserialize_entity(Node, doc) + if obj is not None: + node_by_id[doc["id"]] = obj + + for src_id, target_ids in links.items(): + matched: List[Any] = [] + for tid in target_ids: + obj = node_by_id.get(tid) + if obj is None: + continue + matched.append(obj) + if limit is not None and len(matched) >= max(1, limit): + break + out[src_id] = matched + return out + async def ensure_indexes(self, entity_class: Type[T]) -> None: """Ensure indexes are created for the given entity class. diff --git a/jvspatial/core/entities/edge.py b/jvspatial/core/entities/edge.py index 23a58f1..71299d0 100644 --- a/jvspatial/core/entities/edge.py +++ b/jvspatial/core/entities/edge.py @@ -240,7 +240,19 @@ async def export( @classmethod def get_indexes(cls: Type["Edge"]) -> List[Dict[str, Any]]: - """Get index definitions including unique compound index on (source, target, entity).""" + """Default indexes for every edge collection. + + ``idx_source_target_entity_unique`` enforces edge uniqueness and serves + outgoing traversal via leftmost-prefix on ``source``. The remaining + indexes cover traversal shapes the unique index cannot: + + - ``idx_target_entity`` — incoming traversal (``direction="in"``). + - ``idx_entity_source`` / ``idx_entity_target`` — typed-edge sweeps + that filter by ``entity`` without pinning a node. + + All four are domain-agnostic — they only reference fields every Edge + document carries (``entity``, ``source``, ``target``). + """ indexes = super().get_indexes() indexes.append( { @@ -249,6 +261,27 @@ def get_indexes(cls: Type["Edge"]) -> List[Dict[str, Any]]: "name": "idx_source_target_entity_unique", } ) + indexes.append( + { + "fields": [("target", 1), ("entity", 1)], + "unique": False, + "name": "idx_target_entity", + } + ) + indexes.append( + { + "fields": [("entity", 1), ("source", 1)], + "unique": False, + "name": "idx_entity_source", + } + ) + indexes.append( + { + "fields": [("entity", 1), ("target", 1)], + "unique": False, + "name": "idx_entity_target", + } + ) return indexes @classmethod diff --git a/jvspatial/core/entities/node.py b/jvspatial/core/entities/node.py index cbd0ace..6e59b0d 100644 --- a/jvspatial/core/entities/node.py +++ b/jvspatial/core/entities/node.py @@ -59,6 +59,28 @@ def _get_top_level_fields(cls: Type["Node"]) -> set: "id", } # edge_ids is stored as "edges" at top level; id is top-level on node docs + @classmethod + def get_indexes(cls: Type["Node"]) -> List[Dict[str, Any]]: + """Default node indexes. + + Adds an index on the top-level ``entity`` discriminator. Every typed + find (``ClassName.find(...)`` / ``GraphContext.find_by_class(...)``) + is rewritten to ``{"entity": "", ...}``, so this index is + the universal first cut for any multi-type node collection. Generic + — it only touches fields jvspatial itself attaches to every node + document. + """ + indexes = super().get_indexes() + indexes.append( + { + "field": "entity", + "unique": False, + "direction": 1, + "name": "idx_node_entity", + } + ) + return indexes + def __init_subclass__(cls: Type["Node"], **kwargs: Any) -> None: """Initialize subclass by registering visit hooks. @@ -390,6 +412,36 @@ async def nodes( **kwargs, ) + @classmethod + async def nodes_bulk( + cls, + node_ids: List[str], + *, + direction: str = "out", + edge: Optional[List[Union[str, Type["Edge"]]]] = None, + node: Optional[List[Union[str, Type["Node"]]]] = None, + edge_filter: Optional[Dict[str, Any]] = None, + node_filter: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + ) -> Dict[str, List["Node"]]: + """Batch version of ``nodes()`` for many source IDs. + + Returns a mapping of ``source_id -> connected nodes`` using a single + edge-query pass in the active GraphContext. + """ + from ..context import get_default_context + + context = get_default_context() + return await context.nodes_bulk( + node_ids, + direction=direction, + edge=edge, + node=node, + edge_filter=edge_filter, + node_filter=node_filter, + limit=limit, + ) + async def count_neighbors( self, direction: str = "out", diff --git a/jvspatial/db/_observable.py b/jvspatial/db/_observable.py index b2cbc9a..ceee1b2 100644 --- a/jvspatial/db/_observable.py +++ b/jvspatial/db/_observable.py @@ -38,6 +38,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union from jvspatial.db.database import Database +from jvspatial.observability import db_op_counter from jvspatial.observability.metrics import ( MetricsRecorder, NullMetricsRecorder, @@ -118,6 +119,8 @@ async def _instrument( success = False raise finally: + # Increment per-flow DB operation count even on failures. + db_op_counter.set(db_op_counter.get() + 1) duration_s = time.monotonic() - start duration_ms = duration_s * 1000.0 result_count: Optional[int] = None diff --git a/jvspatial/observability/__init__.py b/jvspatial/observability/__init__.py index 4561b4a..203e1eb 100644 --- a/jvspatial/observability/__init__.py +++ b/jvspatial/observability/__init__.py @@ -15,12 +15,20 @@ contract and ``docs/md/stability.md`` for the stability tier. """ +from contextvars import ContextVar + from jvspatial.observability.metrics import ( MetricsRecorder, NullMetricsRecorder, ) +# Per-request (or per-logical execution flow) database operation counter. +# ObservableDatabase increments this on every wrapped operation so callers +# can cheaply surface round-trip count in headers/logs. +db_op_counter: ContextVar[int] = ContextVar("jvspatial_db_op_counter", default=0) + __all__ = [ "MetricsRecorder", "NullMetricsRecorder", + "db_op_counter", ] diff --git a/tests/api/components/test_database_configurator.py b/tests/api/components/test_database_configurator.py index 3fd89c6..05b72c7 100644 --- a/tests/api/components/test_database_configurator.py +++ b/tests/api/components/test_database_configurator.py @@ -11,6 +11,12 @@ class TestDatabaseConfigurator: """Test DatabaseConfigurator functionality.""" + @pytest.fixture(autouse=True) + def _isolate_observability_env(self, monkeypatch): + """create_database observability kwargs must not leak from host .env.""" + monkeypatch.delenv("JVSPATIAL_OBSERVABILITY_ENABLED", raising=False) + monkeypatch.delenv("JVSPATIAL_SLOW_QUERY_MS", raising=False) + @pytest.fixture def config(self): """Create test server config.""" @@ -62,6 +68,8 @@ async def test_initialize_graph_context_json(self, configurator): mock_create.assert_called_once_with( db_type="json", base_path="./test_db", + observe=False, + slow_query_ms=100.0, ) @pytest.mark.asyncio @@ -107,6 +115,8 @@ async def test_initialize_graph_context_mongodb(self, monkeypatch): db_type="mongodb", uri="mongodb://localhost:27017", db_name="testdb", + observe=False, + slow_query_ms=100.0, ) def test_resolve_mongodb_uses_config_uri_only(self, monkeypatch): @@ -134,6 +144,54 @@ def test_resolve_mongodb_config_only_when_env_absent(self, monkeypatch): assert uri == "mongodb://cfg:27017" assert db_name == "cfgdb" + def test_resolve_observability_kwargs_defaults(self, configurator): + assert configurator._resolve_observability_kwargs() == { + "observe": False, + "slow_query_ms": 100.0, + } + + def test_resolve_observability_kwargs_from_env(self, configurator, monkeypatch): + monkeypatch.setenv("JVSPATIAL_OBSERVABILITY_ENABLED", "true") + monkeypatch.setenv("JVSPATIAL_SLOW_QUERY_MS", "250") + assert configurator._resolve_observability_kwargs() == { + "observe": True, + "slow_query_ms": 250.0, + } + + @pytest.mark.asyncio + async def test_initialize_graph_context_passes_observability_from_env( + self, configurator, monkeypatch + ): + monkeypatch.setenv("JVSPATIAL_OBSERVABILITY_ENABLED", "1") + monkeypatch.setenv("JVSPATIAL_SLOW_QUERY_MS", "42") + + with patch( + "jvspatial.api.components.database_configurator.create_database" + ) as mock_create: + mock_create.return_value = MagicMock() + with patch( + "jvspatial.api.components.database_configurator.get_database_manager", + side_effect=RuntimeError(), + ): + with patch( + "jvspatial.api.components.database_configurator.set_database_manager" + ): + with patch( + "jvspatial.api.components.database_configurator.GraphContext", + return_value=MagicMock(), + ): + with patch( + "jvspatial.api.components.database_configurator.set_default_context" + ): + configurator.initialize_graph_context() + + mock_create.assert_called_once_with( + db_type="json", + base_path="./test_db", + observe=True, + slow_query_ms=42.0, + ) + def test_resolve_mongodb_empty_config_uses_localhost_jvdb(self, monkeypatch): monkeypatch.delenv("JVSPATIAL_MONGODB_URI", raising=False) monkeypatch.delenv("JVSPATIAL_MONGODB_DB_NAME", raising=False) @@ -187,6 +245,8 @@ async def test_initialize_graph_context_dynamodb(self): endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, + observe=False, + slow_query_ms=100.0, ) @pytest.mark.asyncio @@ -250,6 +310,8 @@ async def test_db_path_resolve_app_resolves_relative_path(self, configurator): mock_create.assert_called_once_with( db_type="json", base_path="/opt/backend/track75_db", + observe=False, + slow_query_ms=100.0, ) def test_resolve_db_path_absolute_unchanged(self, configurator): diff --git a/tests/core/test_graph_context.py b/tests/core/test_graph_context.py index 77158e9..e123ed5 100644 --- a/tests/core/test_graph_context.py +++ b/tests/core/test_graph_context.py @@ -855,3 +855,52 @@ async def test_performance_stats_structure(self, temp_context): # Check structure assert "total_operations" in stats assert isinstance(stats["total_operations"], int) + + +class TestGraphContextFindPage: + @pytest.fixture + def temp_context(self): + with tempfile.TemporaryDirectory() as tmpdir: + import uuid + + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + context = GraphContext(database=database) + set_default_context(context) + yield context + + @pytest.mark.asyncio + async def test_find_page_returns_next_cursor(self, temp_context): + await temp_context.database.save("node", {"id": "a", "ts": 3}) + await temp_context.database.save("node", {"id": "b", "ts": 2}) + await temp_context.database.save("node", {"id": "c", "ts": 1}) + + page1, cursor = await temp_context.find_page( + "node", {}, sort=[("ts", -1)], limit=2 + ) + assert [row["id"] for row in page1] == ["a", "b"] + assert cursor + + page2, next2 = await temp_context.find_page( + "node", {}, sort=[("ts", -1)], after=cursor, limit=2 + ) + assert [row["id"] for row in page2] == ["c"] + assert next2 is None + + @pytest.mark.asyncio + async def test_find_page_accepts_dict_cursor_payload(self, temp_context): + await temp_context.database.save("node", {"id": "a", "ts": 2}) + await temp_context.database.save("node", {"id": "b", "ts": 2}) + await temp_context.database.save("node", {"id": "c", "ts": 1}) + + page1, _ = await temp_context.find_page("node", {}, sort=[("ts", -1)], limit=1) + assert page1[0]["id"] in ("a", "b") + + page2, _ = await temp_context.find_page( + "node", + {}, + sort=[("ts", -1)], + after={"sort": page1[0]["ts"], "id": page1[0]["id"]}, + limit=2, + ) + assert len(page2) >= 1 diff --git a/tests/db/test_observable_database.py b/tests/db/test_observable_database.py index 09ee943..b23103b 100644 --- a/tests/db/test_observable_database.py +++ b/tests/db/test_observable_database.py @@ -19,6 +19,7 @@ from jvspatial.db._observable import ObservableDatabase from jvspatial.db.factory import create_database from jvspatial.db.jsondb import JsonDB +from jvspatial.observability import db_op_counter class _CapturingRecorder: @@ -161,6 +162,17 @@ async def test_slow_op_increments_slow_count_metric(self, jsondb): ] assert slow_counters, "slow op should emit slow_count counter" + async def test_db_op_counter_increments_per_operation(self, jsondb): + wrapped = ObservableDatabase(jsondb) + tok = db_op_counter.set(0) + try: + await wrapped.save("node", {"id": "x", "v": 1}) + await wrapped.get("node", "x") + await wrapped.find("node", {}) + assert db_op_counter.get() == 3 + finally: + db_op_counter.reset(tok) + # ---------------------- error path ------------------------------------ diff --git a/tests/test_node_operations.py b/tests/test_node_operations.py index 3d49dbd..b6da782 100644 --- a/tests/test_node_operations.py +++ b/tests/test_node_operations.py @@ -262,6 +262,38 @@ async def test_node_method_optimizes_with_limit(context): assert isinstance(result, Memory) +@pytest.mark.asyncio +async def test_nodes_bulk_returns_per_source_neighbors(context): + city1 = await City.create(name="NY") + city2 = await City.create(name="SF") + city3 = await City.create(name="LA") + + await city1.connect(city2, edge=LocatedIn) + await city1.connect(city3, edge=LocatedIn) + await city2.connect(city3, edge=LocatedIn) + + result = await Node.nodes_bulk( + [city1.id, city2.id], direction="out", edge=["LocatedIn"], node=["City"] + ) + + assert set(result.keys()) == {city1.id, city2.id} + assert {n.id for n in result[city1.id]} == {city2.id, city3.id} + assert {n.id for n in result[city2.id]} == {city3.id} + + +@pytest.mark.asyncio +async def test_nodes_bulk_supports_incoming_direction(context): + city1 = await City.create(name="One") + city2 = await City.create(name="Two") + city3 = await City.create(name="Three") + await city1.connect(city3, edge=LocatedIn) + await city2.connect(city3, edge=LocatedIn) + + result = await Node.nodes_bulk([city3.id], direction="in", edge=["LocatedIn"]) + assert city3.id in result + assert {n.id for n in result[city3.id]} == {city1.id, city2.id} + + @pytest.mark.asyncio async def test_node_method_use_case_example(context): """Test the real-world use case that motivated this method."""