diff --git a/.env.example b/.env.example index b140634..67f456d 100644 --- a/.env.example +++ b/.env.example @@ -75,6 +75,12 @@ # Default: API built with jvspatial framework # JVSPATIAL_API_DESCRIPTION=API built with jvspatial framework +# Unpublish documentation surface in production. +# When truthy, /docs, /redoc, /openapi.json, and /docs/oauth2-redirect +# are not registered (return 404 — no Swagger HTML, no OpenAPI JSON). +# Truthy values: 1, true, yes, on (case-insensitive). Default: published. +# JVSPATIAL_DOCS_DISABLED=1 + # ----------------------------------------------------------------------------- # CORS CONFIGURATION # ----------------------------------------------------------------------------- @@ -85,19 +91,23 @@ # JVSPATIAL_CORS_ENABLED=true # CORS allowed origins (comma-separated) -# Default: * +# Default: http://localhost:5173,http://127.0.0.1:5173,http://localhost:3000, +# http://127.0.0.1:3000,http://localhost:8000,http://127.0.0.1:8000 # Examples: # - Single origin: https://example.com # - Multiple origins: https://example.com,https://app.example.com -# JVSPATIAL_CORS_ORIGINS=* +# Setting "*" or any wildcard entry triggers a startup WARNING — wildcards +# are rarely correct in production. Set cors_allow_wildcard=True on +# CORSConfig to silence the warning when intentional. +# JVSPATIAL_CORS_ORIGINS=https://example.com # CORS allowed methods (comma-separated) -# Default: * -# JVSPATIAL_CORS_METHODS=* +# Default: GET,POST,PUT,PATCH,DELETE,OPTIONS +# JVSPATIAL_CORS_METHODS=GET,POST,PUT,PATCH,DELETE,OPTIONS # CORS allowed headers (comma-separated) -# Default: * -# JVSPATIAL_CORS_HEADERS=* +# Default: Content-Type,Authorization,X-API-Key +# JVSPATIAL_CORS_HEADERS=Content-Type,Authorization,X-API-Key # ----------------------------------------------------------------------------- # DATABASE COLLECTION NAMES diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8b5648c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,57 @@ +# AGENTS.md — Multi-agent compatibility shim + +This file exists so AI coding assistants that follow the `AGENTS.md` convention (Codex, Cursor, Aider, Continue, OpenHands, and others) find the same guidance as Claude Code. + +**The canonical agent guide is [CLAUDE.md](CLAUDE.md).** Treat it as authoritative. + +This file does not duplicate that content; agents that read this file should read [CLAUDE.md](CLAUDE.md) instead. + +--- + +## Quick orientation + +If you cannot follow the link to `CLAUDE.md`, here is the minimum: + +- **What this repo is**: `jvspatial`, an async-first Python library for graph-based persistence with FastAPI integration. See [README.md](README.md) and [PRD.md](PRD.md). +- **Where the contract lives**: [SPEC.md](SPEC.md). Every claim cites a `file:line`. If your edit changes a contract, update SPEC in the same commit. +- **How docs are organized**: [docs/md/README.md](docs/md/README.md) is the index. PRD / SPEC / ROADMAP / CLAUDE live at the repo root. +- **Forward direction**: [ROADMAP.md](ROADMAP.md) — current focus areas, known gaps, out-of-scope. +- **What changed recently**: [CHANGELOG.md](CHANGELOG.md). + +--- + +## Non-negotiable invariants (summary) + +Full list in [CLAUDE.md § Non-negotiable invariants](CLAUDE.md#non-negotiable-invariants). Top items: + +1. Async-only I/O. No sync wrappers. +2. `hmac.compare_digest` for every secret comparison. +3. `__entity_name__` honors per-subclass override; do not assemble IDs from `cls.__name__`. +4. Serverless detection precedence: explicit config → current Server config → `SERVERLESS_MODE` env → auto-detect. +5. Stability tiers in [docs/md/stability.md](docs/md/stability.md) are binding. Public names live in `jvspatial.__all__`; underscore modules are internal. +6. CORS/CSP/docs defaults fail closed; do not weaken without a security review. + +--- + +## Run the dev loop + +```bash +pip install -e '.[dev,test]' +pre-commit install +pytest -q +pre-commit run --all-files +``` + +Tests are async (`pytest-asyncio` auto mode). Benchmarks are skipped by default — run with `pytest tests/benchmarks --benchmark-only`. + +--- + +## Editor / agent-tool specific notes + +This section is the only place where agent-tool-specific guidance lives. Keep it short; cross-link to CLAUDE.md for everything else. + +- **Cursor / Continue**: `.cursor/` directory is gitignored except for committed rules. There are no committed `.cursorrules`; this file plus CLAUDE.md is the full agent contract. +- **Aider**: `--read CLAUDE.md` to load the canonical guide on session start. +- **Codex / OpenHands**: this file is loaded as agent instructions; read CLAUDE.md for full details. + +Any agent-specific quirk worth recording goes here, with a one-line rationale. If a quirk grows beyond a line, promote it into CLAUDE.md and reference back. diff --git a/AUDIT.md b/AUDIT.md new file mode 100644 index 0000000..5b79a98 --- /dev/null +++ b/AUDIT.md @@ -0,0 +1,279 @@ +# jvspatial — Foundation Audit + +> **Purpose**: First retrospective audit of the jvspatial foundation against the contracts now codified in [SPEC.md](SPEC.md), [PRD.md](PRD.md), and [CLAUDE.md](CLAUDE.md). Findings are the backlog for the next phase of hardening. +> +> **Method**: Five parallel reviewers, one per dimension: async contract, security boundaries, database adapter parity, walker/identity invariants, serverless/config/stability. Each returned a severity-tagged finding table with file:line citations. +> +> **Date**: 2026-05-17. +> **Library version**: 0.0.8. +> **Tests collected at audit time**: 1845. + +--- + +## Executive Summary + +**150 distinct findings**: 16 CRIT, ~42 HIGH, ~58 MED, ~34 LOW. + +Top themes: + +1. **Identity model is partially fictional.** `__entity_name__` override (the 0.0.8 headline feature, commit `a3964ab`) is ignored in 7+ code paths — Walker ID construction, GraphContext ID validation/regeneration, Node neighbor queries, Edge filtering, ObjectPager metadata. Today, any class using `__entity_name__` to disambiguate from a same-`__name__` peer will have its IDs **silently rewritten back to `cls.__name__`** by `GraphContext.save_object`. **This is a data-integrity bug for the feature shipped two commits ago.** +2. **Walker protection is partially fictional.** SPEC §6.3 promises `max_steps`, `max_visits_per_node`, `max_execution_time`, `max_queue_size`. In practice: `WalkerTrail` ignores `max_trail_length` entirely; queue insert paths (`prepend`, `add_next`, `insert_after`, `insert_before`) bypass `max_size`; queue drops are silent (no log); `resume()` resets step/visit counters and restarts the wall-clock timer on every call; and `ProtectionViolation` is swallowed into `walker.report` rather than raised as `InfiniteLoopError` / `WalkerTimeoutError`. +3. **Async contract has real holes, mostly in JsonDB.** Five CRITs: webhook walker `enhanced_init` is `async def __init__` (Python ignores; coroutine leaks every construction), webhook `endpoint_func(**kwargs)` is unawaited, storage `delete_file` unawaited, two `Path.write_text` calls inside `async def` graph exporters. Plus a string of `path.exists()` / `path.glob()` sync stats inside JsonDB `async` methods that block the event loop. +4. **Storage path traversal in `LocalFileInterface` versioning methods.** Primary save/read/delete go through `PathSanitizer`; the versioning subpaths (`create_version`, `get_version`, `list_versions`, `delete_version`, `get_latest_version`) compute `root_dir / f"{file_path}.versions"` without sanitization — user-controlled `file_path` like `../../etc/passwd` escapes the storage root. +5. **Webhook HMAC verification is broken.** A 7-character slice bug in `webhook_auth.utils.verify_signature` makes `hmac.compare_digest` always compare a 64-char hex digest to a 57-char prefix → always False. Webhook signature auth currently rejects every request. +6. **SHA-256 fallback uses `==`, not `compare_digest`.** `AuthenticationService._verify_refresh_token` falls back to `hashlib.sha256(token) == hashed` when bcrypt/argon2 fail to import; this path covers refresh tokens AND password-reset tokens. CLAUDE.md §2 non-negotiable. +7. **DynamoDB throttle-retry is partial.** Only `save`/`get`/`delete` are wrapped. `find` / `count` / `batch_get` / `batch_write` surface `ProvisionedThroughputExceededException` directly to callers, even though the adapter claims throttle retry (SPEC §4.3). +8. **Env-var allowlist is not enforced.** SPEC §10.2 promises "Unknown `JVSPATIAL_*` keys are rejected at startup to catch typos." `env_adapter.py` only *reads* enumerated keys; it never *scans* the environment for stray `JVSPATIAL_*` and rejects. Plus three divergent `parse_bool` implementations across `env.py` / `env_adapter.py` / `runtime/serverless.py`. + +These are listed below with file:line citations and one-line fixes. The audit was non-destructive — no fixes applied. The findings inform the next milestone. + +--- + +## How to read this + +- **Sections** are by dimension (async, security, database, walker, serverless). +- **Within a section**, findings sorted by severity then file. +- **`Cite`** column references SPEC §, CLAUDE.md §, or ROADMAP §. +- **`Fix`** is the smallest change that closes the gap, not necessarily the best long-term fix. + +When the same root cause shows up in multiple dimensions, it is listed once (under the most specific dimension) and cross-referenced. + +--- + +## 1. Identity Model — `__entity_name__` Coverage Gap + +The most important class of findings. SPEC §1.2: `__entity_name__` is per-subclass; ID construction and queries must go through `_entity_name()`. Commit `a3964ab` added the override but did not update every consumer. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 1.1 | CRIT | `core/entities/walker.py:308` | `generate_id(type_code, self.__class__.__name__)` — walker IDs ignore `__entity_name__`. | `self.__class__._entity_name()` (must add classmethod to Walker; Walker does not subclass Object). | SPEC §1.2 | +| 1.2 | CRIT | `core/entities/walker.py:311` | `kwargs["entity"] = self.__class__.__name__` — persisted `entity` field ignores override. | Same as 1.1. | SPEC §1.2 | +| 1.3 | CRIT | `core/context.py:753` | ID-format check `id_parts[1] != entity.__class__.__name__` triggers regeneration for every save of any entity using `__entity_name__` override — clobbers correct IDs. | Compare against `entity.__class__._entity_name()`. | SPEC §1.2 | +| 1.4 | CRIT | `core/context.py:759` | Regeneration uses `entity.__class__.__name__` → rewrites IDs to wrong discriminator on save. | Use `_entity_name()`. | SPEC §1.2 | +| 1.5 | CRIT | `core/context.py:1403` | `find_edges_between`: `query["entity"] = edge_class.__name__` — won't find edges whose stored entity uses override. | Use `edge_class._entity_name()`. | SPEC §1.2 | +| 1.6 | HIGH | `core/entities/node.py:561, 593` | `_node_query` filters edges by `edge_filter.__name__` against persisted `entity` field → silent no-op for override classes. | Use `"entity": edge_filter._entity_name()`. | SPEC §1.2, §2.3 | +| 1.7 | HIGH | `core/entities/node.py:426, 434` | `count_neighbors` fast-path uses `node.__name__` and `re.escape(entity_name)` against `n..` ID pattern. | Use `node._entity_name()`. | SPEC §1.2 | +| 1.8 | HIGH | `core/entities/node.py:668, 678, 688` | `_matches_node_filter` compares `node_obj.__class__.__name__` to string filter; `nodes(node="HostApp")` won't match. | Compare against `node_obj.__class__._entity_name()`. | SPEC §1.2 | +| 1.9 | MED | `core/entities/walker.py:308-311` | Walker has no `_entity_name()` method at all (doesn't subclass Object). | Add classmethod to Walker (or extract to shared mixin). | SPEC §1.2 | +| 1.10 | MED | `core/utils.py:40-89` | `_subclass_cache` is process-global, never cleared; stale identity after module reload (tests). | Add `clear_subclass_cache()` callable from test fixtures. | SPEC §1.2 | +| 1.11 | LOW | `core/pager.py:296` | `to_dict()` reports `object_type: self.object_class.__name__` instead of `_entity_name()`. | Resolve via `getattr(cls, "_entity_name", lambda: cls.__name__)()`. | SPEC §1.2 | + +--- + +## 2. Walker Protection Gaps + +SPEC §6.3 / CLAUDE.md §9. Promised limits are partially unenforced. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 2.1 | CRIT | `core/entities/walker.py:684-687` | `run()` catch-all swallows `ProtectionViolation` into `self.report(...)` instead of raising `InfiniteLoopError` / `WalkerTimeoutError` as SPEC promises. | Re-raise protection violations (or map to documented exception types) instead of stuffing in report. | SPEC §6.3 | +| 2.2 | CRIT | `core/entities/walker.py:886, 651` | `resume()` → `run()` → `_protection.reset()` resets step/visit counters AND restarts the wall-clock timer on every resume — protection trivially bypassed by repeated pause/resume. | Split `reset()` from `start_timer_if_needed()`; only reset on `spawn()` / explicit `reset_protection()`. | SPEC §6.3 | +| 2.3 | HIGH | `core/entities/walker_components/walker_trail.py:19-30` | `WalkerTrail` has no length bound — SPEC §6.4 promises `max_trail_length` "0 = unlimited, configurable" but it's not wired. Unbounded memory growth. | Pass `max_trail_length` from Walker into `WalkerTrail.__init__`; drop oldest on overflow. | SPEC §6.4 | +| 2.4 | HIGH | `core/entities/walker_components/walker_queue.py:67-94, 135-195` | `prepend`, `add_next`, `insert_after`, `insert_before` ignore `max_size` — protection bypass via these enqueue paths. | Apply same `max_size` guard (and warning log) used by `append`/`visit`. | SPEC §6.3 | +| 2.5 | HIGH | `core/entities/walker_components/walker_queue.py:33-36, 76-85` | `visit`/`append` silently drop on `max_size` hit — SPEC says "silent drop, logged" but no log is emitted. | Emit a one-shot WARNING when first drop occurs. | SPEC §6.3 | +| 2.6 | MED | `core/entities/walker.py:74-80` | `WalkerVisitingContext.__exit__` does not wrap `set_visitor(None)` in `try/finally`; if `current_node = None` raises, `_visitor_ref` never cleared. | Restructure with explicit `try/finally`. | SPEC §6.1 | +| 2.7 | MED | `core/entities/walker.py:63-71` | `WalkerVisitingContext.__enter__` calls `record_step` + `record_visit` — but `Walker.run()` (line 671, 674) calls the same methods directly without `visiting()`. Double-counting risk for callers who use `visiting()`. | Make `run()` use `WalkerVisitingContext`; one entry point. | SPEC §6.1, §6.3 | +| 2.8 | MED | `core/entities/walker.py:117-121` | Walker uses `extra="allow"` and does NOT declare `entity` as a model field — `entity` is set as dynamic attr, `protected=True` invariant is not enforced. | Declare `entity: str = attribute(protected=True, transient=True)` on Walker. | SPEC §1.2 | +| 2.9 | LOW | `core/entities/walker.py:734-748, 797-810` | Hook errors detected by `"Node skipped" in str(e)` substring match — fragile. | Define `SkipNode` exception class, catch directly. | SPEC §6.5 | +| 2.10 | LOW | `core/entities/walker.py:308-311` | Walker doesn't enforce `type_code="w"`; kwargs can corrupt SPEC §1.1 invariant. | Force `type_code="w"`. | SPEC §1.1 | + +--- + +## 3. Async Contract Violations + +CLAUDE.md §1 / SPEC §3. Five real bugs, plus a long tail of `path.glob`/`path.exists` sync stats blocking the event loop in JsonDB. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 3.1 | CRIT | `api/integrations/webhooks/helpers.py:135` | `enhanced_init` defined as `async def __init__`; Python ignores async-ness on `__init__`; each Walker construction returns a coroutine that's never awaited and leaks. | Remove `async` keyword. | SPEC §3 | +| 3.2 | CRIT | `api/integrations/webhooks/helpers.py:217` | `iscoroutinefunction(endpoint_func)` branch calls `endpoint_func(**kwargs)` without `await`; both `if/else` arms identical. | `result = await endpoint_func(**kwargs)` in the async branch. | SPEC §3.1 | +| 3.3 | CRIT | `api/integrations/storage/service.py:159` | `self.file_interface.delete_file(file_path)` not awaited; deletion never executes. | Add `await`. | SPEC §3.1 | +| 3.4 | CRIT | `core/graph.py:174` | `Path(output_file).write_text(...)` inside `async def generate_graph_dot` blocks the event loop. | `await asyncio.to_thread(...)`. | SPEC §3.3 | +| 3.5 | CRIT | `core/graph.py:332` | Same blocking write inside `async def generate_graph_mermaid`. | Same fix. | SPEC §3.3 | +| 3.6 | HIGH | `db/jsondb.py:168, 250, 254, 286, 355, 362` | Six sync `path.exists()` / `glob()` calls inside `async` DB methods — block event loop on disk I/O. | Wrap in `asyncio.to_thread`. | SPEC §3.3 | +| 3.7 | HIGH | `storage/interfaces/local.py:561` | `list(_read_chunks())` materializes entire file inside `to_thread` — defeats streaming purpose of `stream_file`. | Queue / `run_in_executor` iterator pattern, or use `aiofiles`. | SPEC §3 | +| 3.8 | HIGH | `db/dynamodb.py:675, 775` | `asyncio.gather(*[process_batch(...)])` without `return_exceptions=True` — one batch failure cancels siblings; partial-success state lost. | Add `return_exceptions=True`, aggregate per-batch failures. | SPEC §4.1 | +| 3.9 | HIGH | `api/integrations/webhooks/middleware.py:245, 514` | `asyncio.create_task(...)` fire-and-forget without strong reference or `add_done_callback`; tasks GC'd mid-flight, exceptions dropped. | Track tasks in a set; attach error sink. | SPEC §3 | +| 3.10 | HIGH | `testing/__init__.py:610`, `async_utils/__init__.py:37` | `asyncio.gather` without `return_exceptions=True` in test harness / general helper. | Add the flag. | SPEC §3 | +| 3.11 | MED | (multiple) `core/entities/walker_components/{walker_queue,protection,walker_trail}.py` and `core/utils.py:25`, `core/context.py:1424`, `core/entities/{object,node,edge,walker}.py` (~30 sites) | Many methods declared `async def` with no `await` — false-async. Wasted context switches; not bugs, but degrade the contract. | Convert each to sync, or genuinely await something. Roll into a single PR after the CRITs/HIGHs land. | SPEC §3.2 | + +(Full async list: see reviewer report. Includes WalkerQueue mutators, TraversalProtection state methods, JsonDBTransaction buffered ops, several auth and rate-limit dict mutators.) + +--- + +## 4. Security Boundaries + +SPEC §15 / CLAUDE.md §2. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 4.1 | CRIT | `api/auth/service.py:367` | `hashlib.sha256(token).hexdigest() == hashed` — refresh-token / password-reset-token hash compared with `==`. | `hmac.compare_digest(...)`. | SPEC §15.2, CLAUDE §2 | +| 4.2 | CRIT | `storage/interfaces/local.py:207, 220, 252, 260, 288, 327, 332, 333, 356` | All file-versioning methods compute `root_dir / f"{file_path}.versions"` without `PathSanitizer`; user-controlled `file_path` escapes the storage root. | Sanitize `file_path` through `_get_full_path()` before building versioned paths; resolve and `.relative_to(self.root_dir)` the result. | SPEC §15.1 | +| 4.3 | CRIT | `api/integrations/webhooks/utils.py:176` | `hmac.compare_digest(signature, expected_signature[len(prefix):])` — `expected_signature` is the bare hex digest; slicing 7 chars produces 57-char string vs 64-char signature → always False. Webhook HMAC always rejects. | Drop the slice: `compare_digest(signature, expected_signature)`. | SPEC §15.2 | +| 4.4 | HIGH | `api/auth/api_key_service.py:30` | `self.context = context or get_default_context()` — auth state can land on non-prime DB when a caller forgets to pass `context`. | `context or GraphContext(database=get_prime_database())`. | SPEC §9, CLAUDE §1 | +| 4.5 | HIGH | `api/auth/api_key_service.py:213-246` + `api/integrations/webhooks/webhook_auth.py:19-21, 180-183` | `revoke_key` flips `is_active=False` in DB but does not invalidate the 300s in-memory `_API_KEY_CACHE`. Revoked key authenticates for up to 5 minutes after revocation. | Add `webhook_auth.invalidate_cache(...)` hook called from `revoke_key`; or re-check `is_active` on cache hit. | SPEC §15.2 | +| 4.6 | HIGH | `api/components/auth_configurator.py:175-391` | `/auth/register`, `/auth/login`, `/auth/forgot-password`, `/auth/reset-password`, `/auth/change-password` registered without endpoint rate-limit configs; fallback is global `default_limit=60/60s` if rate limiting is enabled at all. | Hard-code rate-limit configs during `_register_auth_endpoints`; document rate-limit middleware as required when auth enabled. | SPEC §15 | +| 4.7 | HIGH | `api/integrations/webhooks/webhook_auth.py:19, 147-183` | `_API_KEY_CACHE` mutated across `await` without lock; eviction at size cap races with reads (`KeyError`). | Wrap in `asyncio.Lock`, or use bounded LRU with single-lock guard. | CLAUDE.md "race conditions in auth state" | +| 4.8 | HIGH | `api/auth/enhanced.py:235-389` | `SessionManager._sessions` / `_user_sessions` mutated across `await` without lock; concurrent logout-vs-login → `RuntimeError: dictionary changed size during iteration`; `max_sessions_per_user` enforcement is racy. | `asyncio.Lock` around session create/invalidate/cleanup. | CLAUDE.md races | +| 4.9 | HIGH | `storage/interfaces/local.py:657-659` | `get_metadata` MIME detection passes `content=b""` — falls back to extension-based `mimetypes.guess_type`; mislabels served `Content-Type`. | Read first 4 KiB of file, pass as content; or store validated MIME in sidecar at save. | SPEC §15.1 | +| 4.10 | HIGH | `api/components/error_handler.py:763, 769-770` | If operator sets `JVSPATIAL_EXPOSE_ERROR_DETAILS=true` in production, raw exception messages leak in 500 responses; safe default exists but no guard. | Refuse to honor the flag when `get_environment_mode() == "production"`. | SPEC §15.5 | +| 4.11 | MED | `api/auth/service.py:427` | Debug log discloses JWT secret length. | Log `secret_configured=bool(secret)` only. | SPEC §15.5 | +| 4.12 | MED | `api/middleware/manager.py:194-197` + `api/config_groups.py:66-75` | `CORSConfig` accepts wildcard origins; no startup validator. SPEC says wildcards must trigger startup warning. | Add validator on `CORSConfig` to warn on `"*"`; fail loudly if `allow_credentials=True` + wildcard. | SPEC §15.4 | +| 4.13 | MED | `api/auth/service.py:1052-1059` | `validate_token` warning logs the DB `base_path` — discloses internal filesystem path. | Log `db_type=type(database).__name__`. | SPEC §15.5 | +| 4.14 | MED | `api/auth/service.py:1038-1049` | DB-error fallback in `validate_token` trusts JWT-payload roles when DB lookup fails — fails open on DB outage, within JWT-expiry window. | Fail closed: return `None` on DB error. | SPEC §9 | +| 4.15 | MED | `api/auth/service.py:469-531` | `_blacklist_cache` dict mutated across `await` without lock; unbounded growth (per-entry TTL only). | Add `asyncio.Lock` + size cap with LRU eviction. | CLAUDE.md races | +| 4.16 | LOW | `api/deferred_invoke_route.py:28-37` | Empty `JVSPATIAL_DEFERRED_INVOKE_SECRET` returns `True` from `_deferred_invoke_secret_ok` — misconfigured deployment exposes internal endpoint. | Treat empty secret as "deny all". | SPEC §15.2 | +| 4.17 | LOW | `api/auth/service.py:331, 243-244` | `JVSPATIAL_AUTH_STRICT_HASHING` defaults True for passwords, False for tokens — asymmetry surprising. | Apply strict to both; document rationale. | SPEC §15.5 | +| 4.18 | LOW | `storage/interfaces/local.py:194-196` | Windows reserved names (CON, AUX, NUL) pass `SAFE_FILENAME_PATTERN`. | Add Windows-reserved-name check. | SPEC §15.1 | + +--- + +## 5. Database Adapter Parity + +SPEC §4-5 / ROADMAP §2.2, §2.4. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 5.1 | CRIT | `db/dynamodb.py:1052-1064, 1194-1265, 668-684, 770-778` | DynamoDB throttle retry (`_run_with_throttle_retry`) covers only `save`/`get`/`delete`. `find` / `count` / `batch_get` / `batch_write` surface `ProvisionedThroughputExceededException` immediately. | Wrap all CRUD paths in throttle retry; escalate `batch_write` inner-retry to the shared envelope. | SPEC §4.3 | +| 5.2 | HIGH | `db/query.py:425-426, 502-520, 559-564, 663-679` | `QueryBuilder` exposes `nor_`, `mod`, `all_`, `regex`, `type_`, `size`, `$nor`, `$mod`, `$type`, `$all`, `$where`, `$text` — but `QueryEngine._match_value` returns `False` for any unknown operator; `match` never branches on `$nor`. Queries built via the builder silently match nothing. | Implement the operators in `_match_value`/`match`, OR have builder + engine raise `QueryError("unsupported operator")`. | SPEC §5.1 | +| 5.3 | HIGH | `db/database.py:279-322`, `db/work_claim.py:53-62` | Default `find_one_and_update` / `find_one_and_delete` query on `_id`, but JsonDB/SQLite/DynamoDB persist only `id`; silent miss on non-Mongo backends. | Normalize `_id` ↔ `id` in default impls, or override on each backend. | SPEC §4.1 | +| 5.4 | HIGH | `db/database.py:144-159` | `Database.count` default materializes every record via `find`. Third-party adapters registered via `register_database_type` inherit this — silent OOM on large collections. | Either mark `count` abstract, or warn once-per-class on hit. | SPEC §4.1 | +| 5.5 | HIGH | `db/mongodb.py:332-351, 380-401` | `find_many` and `bulk_save` reconnect-retry once but escaping raw `PyMongoError` is not wrapped in `DatabaseError` (CRUD paths are wrapped). | Route through `_run_with_reconnect`. | SPEC §4.1, §4.3 | +| 5.6 | HIGH | `db/database.py:212-250` | Default `bulk_save` does sequential `save()` without per-record exception handling — partial-success leaves DB in unobservable state. Doc claims "all-or-throw" but JsonDB override logs and continues. | Return `(saved, failed_ids)` or document loudly + reconcile JsonDB override. | SPEC §4.1 | +| 5.7 | MED | `db/dynamodb.py:686-778` | `batch_write` inner retry — at most 3 attempts then logs warning and returns success. Lost writes silent. | Return actual saved count, propagate through `bulk_save`, raise on persistent unprocessed items. | SPEC §4.1 | +| 5.8 | MED | `db/query.py:170-196` | `_add_indexing_hints` mutates the cached optimized query in-place; subsequent callers receive same dict. Also `$hint` falls through in non-Mongo backends → match returns False. | Deep-copy on cache hit; whitelist-skip `$hint`/`$select` in `_match_value`. | SPEC §5.3 | +| 5.9 | MED | `db/transaction.py:122-194`, `db/mongodb.py:67, 699-733` | `MongoDB.supports_transactions = True` unconditionally; `begin_transaction` returns `None` when not on a replica set. Flag is dishonest. | Probe replica-set support lazily; cache; or rename flag and add `is_transactional()`. | SPEC §4.2 | +| 5.10 | MED | `db/sqlite.py:34-64, 117-162` | One persistent connection per `SQLiteDB` instance; no cross-loop detection (MongoDB has one). Use across multiple event loops fails silently. | Track creating loop in `__init__`; rebind on detection or raise `DatabaseError`. | SPEC §4.3 | +| 5.11 | MED | `db/_sqlite_translate.py:86-98` | `_safe_field_path` regex rejects digit-only segments (e.g. `arr.0.value`); query silently falls back to full scan + Python match. | Widen segment regex, OR log debug on fallback so slow queries are diagnosable. | SPEC §5.2 | +| 5.12 | MED | `db/dynamodb.py:1066-1265` | `find()` fallback to `Scan` does not push `$or`/`$and`/`$gt` into FilterExpression; full client-side scan + match. | Push to FilterExpression where supported. | SPEC §5.2 | +| 5.13 | LOW | `db/database.py:212-250` and adapter overrides | `bulk_save` returns `int` everywhere; callers cannot distinguish SQLite all-or-nothing vs DynamoDB silent drop. | Expose `BulkSaveResult(attempted, saved, failed_ids)` (or sibling `bulk_save_detailed`). | SPEC §4.1 | +| 5.14 | LOW | `db/transaction.py:363-413` | `JSONTransaction` is dead code; `JsonDBTransaction` replaces it; still in `__all__`. | Remove or `@deprecated`. | ROADMAP §2.9 | +| 5.15 | LOW | `db/sqlite.py:260-285` | `save()` and `bulk_save` differ in `id` coercion (`record.setdefault("id", uuid)` vs `str(r["id"])`). Inconsistent get-by-id. | Force `record["id"] = str(record["id"])` after setdefault. | SPEC §4.1 | + +(Plus minor LOWs on log hygiene, dead glob filters in JsonDB, double cache writes in `CachingDatabase.find_one_and_update`.) + +--- + +## 6. Inheritance, Subclass Init, and Library Self-Discipline + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 6.1 | HIGH | `core/entities/node.py:62-118` | `Node.__init_subclass__` does NOT call `super().__init_subclass__()` — `AttributeMixin.__init_subclass__` never runs for Node subclasses; `_PROTECTED_ATTRS` registration skipped. | Add `super().__init_subclass__(**kwargs)` at top. | SPEC §2.5 | +| 6.2 | HIGH | `core/entities/edge.py:66-123` | Same: `Edge.__init_subclass__` skips `super()`. | Same fix. | SPEC §2.5 | +| 6.3 | HIGH | `core/entities/walker.py:366-396` | Same: `Walker.__init_subclass__` skips `super()`. | Same fix. | SPEC §2.5 | +| 6.4 | HIGH | `core/entities/root.py:19` | `_lock = asyncio.Lock()` is `ClassVar` — single global lock across all GraphContexts and event loops; "different loop" errors under per-test loop fixtures; SPEC §1.3 implies per-context. | Lock per `GraphContext` (e.g. lazy-init dict keyed by `id(context)`). | SPEC §1.3, §7.2 | +| 6.5 | HIGH | `core/entities/root.py:100` | `object.__setattr__(self, "id", "n.Root.root")` — library bypasses its own `protected=True`. | `_unsafe_set_id` helper, or check/clear `_initializing` and route through normal setter. | CLAUDE §5 | +| 6.6 | HIGH | `core/context.py:761, 812, 818, 1229` | Six `object.__setattr__` call sites bypass protected-attribute enforcement (id, edge_ids, atomic_increment). | Route through `AttributeMixin.__setattr__` with explicit override flag. | CLAUDE §5 | +| 6.7 | MED | `core/mixins/deferred_save.py:119+` | No runtime check of MRO order — wrong order silently disables batching (CLAUDE.md warns but library doesn't detect). | In `__init_subclass__`, assert mixin precedes persistable base; warn or raise. | CLAUDE §6 | +| 6.8 | MED | `core/entities/object.py:112-140` | `__setattr__` allows ANY `name.startswith("_")` — callers can attach arbitrary `_foo` attributes, bypassing schema validation (SPEC promises rejection). | Restrict private bypass to declared `__private_attributes__` + `_initializing`. | SPEC §2.1 | +| 6.9 | MED | `core/events.py:36, 119`, `core/context.py:1818-1843` | `EventBus._lock` and `event_bus` module global bound to import-time loop; tests with new loops fail. Also `get_default_context()` has check-and-set race. | Lazy-init locks per first-async-use; use `ContextVar.get` + token for default context init. | SPEC §6.x, §7.1 | +| 6.10 | MED | `core/events.py:38-45` | `register_entity` on `walker.spawn()` has no symmetric `unregister_entity` — events keep firing to done walkers; weakref GC only. | Call `event_bus.unregister_entity(self.id)` in `disengage()` and end-of-`spawn`. | SPEC §6.x | +| 6.11 | MED | `core/entities/walker_components/protection.py:108-115` | `_check_timeout` reads `_start_time` then computes elapsed; concurrent `reset()` corrupts. | Snapshot to local before subtraction. | SPEC §6.3 | + +--- + +## 7. Serverless / Config / Stability Discipline + +SPEC §10-11, §18 / CLAUDE.md §4, §7, §8. + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 7.1 | HIGH | `env_adapter.py:52-193` | Allowlist not enforced. SPEC §10.2 promises "Unknown `JVSPATIAL_*` keys are rejected at startup." Adapter only *reads* enumerated keys; never *scans* for stray `JVSPATIAL_*`. | Add startup validator: scan `os.environ` for `JVSPATIAL_*` prefix; reject unknowns. | SPEC §10.2 | +| 7.2 | HIGH | `env_adapter.py:11` vs `env.py:36-43` vs `runtime/serverless.py:11` | Three divergent `parse_bool` implementations. `JVSPATIAL_DEBUG=on` parses different ways in different code paths. | Consolidate on `env.parse_bool`; have others import. | SPEC §10.2 | +| 7.3 | MED | `env_adapter.py:11`, `runtime/serverless.py:11` | `_parse_bool` non-strict; `JVSPATIAL_DEBUG=garbage` silently maps to False with no validation error. Hides typos. | Raise `ValueError` on unrecognized non-empty. | SPEC §10.2 | +| 7.4 | MED | `serverless/deferred_invoke.py:52-73` | `normalize_deferred_envelope` covers flat Lambda invoke + SQS scheduler shapes only. Direct Lambda SQS-batch trigger (`{"Records": [...]}`) raises. | Accept `Records[...]` and dispatch per-record, OR document the unwrap requirement. | SPEC §11.3 | +| 7.5 | MED | `.env.example:94-106` vs `api/config_groups.py:66-81` | `.env.example` documents `JVSPATIAL_CORS_ORIGINS=*` with "Default: *" — actual default is localhost whitelist. Following docs degrades security. | Update `.env.example`; add production-only note. | CLAUDE §8 | +| 7.6 | MED | `runtime/lwa.py:107-110` (called from `server.py:137`) | LWA env defaults applied inside `Server.__init__`; LWA reads them before Python starts. The docstring acknowledges this; the practical effect is zero for the actual LWA bootstrap. | Downgrade to operator warning, or remove and document IaC-only. | SPEC §11.4 | +| 7.7 | MED | `db/transaction.py:243-249` | `JsonDBTransaction(best_effort=True)` imports private `_emit_once` from `jvspatial.utils.stability` — internal symbol crossed module boundary. | Expose public `emit_experimental_once(...)`. | SPEC §18 | +| 7.8 | MED | `serverless/deferred_invoke.py:1-88` | Handlers registered post-import → race: request arriving before user modules import returns 404 `UnknownDeferredTaskError`. Not silently dropped (good); but no startup-readiness log. | Log registered handlers on startup, or log debug on empty-registry first-dispatch. | SPEC §11.3 | +| 7.9 | LOW | `env_adapter.py:39-49` | `deep_merge` silently skips `None` in override; `Server(host=None, ...)` does NOT override env. Surprising. | Document; or sentinel-based override. | SPEC §10.2 | +| 7.10 | LOW | `api/auth/service.py:164-170` | `AuthenticationService` captures `is_serverless_mode()` and `_bcrypt_rounds` at construction; per-test mode flips require service rebuild + `reset_serverless_mode_cache()`. | Note in CLAUDE.md §4 (test guidance). | CLAUDE §4 | +| 7.11 | LOW | `runtime/lwa.py:18-23` | `_deferred_invoke_pass_through_path` reads `JVSPATIAL_API_PREFIX` directly, bypassing `resolve_api_prefix()`. Two divergent reads. | Use `resolve_api_prefix()`. | SPEC §10.4 | +| 7.12 | LOW | `api/components/app_builder.py:58-65` | `JVSPATIAL_DOCS_DISABLED` parses with ad-hoc inline set `{"1","true","yes","on"}` — fourth divergent bool parser. | Use `env.parse_bool`. | SPEC §10.5 | +| 7.13 | LOW | `api/middleware/manager.py:54-59` | `_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json")` hardcoded — customizing `docs_url` breaks Swagger UI via strict CSP. | Derive from `config.docs_url`/`redoc_url`/`openapi_url`. | SPEC §10.5 | +| 7.14 | LOW | `serverless/tasks/stub.py:27` | `LoggingNoopTaskScheduler.schedule()` logs warning per call. CloudWatch cost in misconfigured serverless deploys. | Downgrade per-call to debug; rely on once-per-process error. | SPEC §11.2 | +| 7.15 | LOW | `serverless/factory.py:73-80` | Fallback to `LoggingNoopTaskScheduler` when AWS deferred-task config missing — warning string only; no startup error in non-`strict` mode. | Once-per-process startup error log when serverless + provider=aws and config missing. | SPEC §11.3 | + +--- + +## 8. ObjectPager / Pagination + +| # | Sev | File:Line | Problem | Fix | Cite | +|---|---|---|---|---|---| +| 8.1 | HIGH | `core/pager.py:118-144` | Keyset pagination filters `id > after_id` but sorts by `context.` — cursor doesn't track sort key; wrong/missing rows on writes between pages. | Disallow `order_by` with `after_id`, OR include order field in cursor. | SPEC §5 | +| 8.2 | HIGH | `core/pager.py:73, 142, 214` | `_cache` keyed by `(page, hash(filters))` and never invalidated — stale results after writes. | Drop the cache or expose `invalidate()`. | SPEC §5 | + +--- + +## 9. Verified Correct (no findings) + +These are explicitly checked — the audit found no gap. Listed so future reviewers know the area was inspected: + +- **Detection precedence in `is_serverless_mode`** (SPEC §11.1) — exact match. +- **`reset_serverless_mode_cache`** clears both lru_caches. +- **Mode-dependent defaults read mode at call time**, not at import. +- **`JVSPATIAL_DOCS_DISABLED` coverage**: all four URLs (`docs_url`, `redoc_url`, `openapi_url`, `swagger_ui_oauth2_redirect_url`) set to None. +- **CSP per-path match** uses exact-or-segment matching; `/docsfoo` correctly stays on strict CSP. +- **bcrypt rounds**: serverless=10, standard=12, env-overridable. +- **JsonDB `/tmp` default**: only applied under serverless mode. +- **`__init__.py` `__all__`**: every imported public symbol is in `__all__`; no underscore-module imports leak to public surface. +- **`Server.run()` blocking**: `uvicorn.run()` is the only blocking call; `run_async()` uses async `serve()`. +- **JWT secret validation**: rejected if empty or known-insecure default. +- **CSP defaults**: strict on app routes; relaxed only on docs paths. +- **JsonDB atomic writes**: every write path goes through `_atomic.atomic_write_bytes`; no direct `open().write()` bypasses. +- **PathLock coverage in JsonDB**: every record write/delete acquires the per-file lock. +- **SQLite Mongo→SQL translator escape hygiene**: values bound, field paths regex-validated, unknown ops trigger fallback (parameterized, no injection). +- **Auth state on prime DB** in `AuthenticationService` (one drift in `APIKeyService` flagged separately at 4.4). +- **Primary file save/read/delete in `LocalFileInterface`** correctly use `PathSanitizer` + `resolve()` + `relative_to(root_dir)`. The escape vector is exclusively in versioning methods (4.2). +- **`hmac.compare_digest`** used everywhere SPEC §15.2 lists, **except** the SHA-256 fallback (4.1) and the broken-by-slice webhook signature path (4.3). + +--- + +## 10. Recommended Remediation Sequence + +Roughly ordered by blast radius × cost. CRITs first, then HIGHs that block CRIT fixes. + +### Wave 1 — restore the contracts the library claims to provide (1-2 weeks) + +1. **Identity model** (§1.1-1.5): patch all 5 CRIT call sites to use `_entity_name()`. Add classmethod to Walker. Add regression test that subclasses with `__entity_name__` round-trip through save/load. +2. **Walker protection** (§2.1-2.5): make `ProtectionViolation` raise documented exception types; separate timer-start from counter-reset; wire `max_trail_length`; cap insert paths; emit drop warnings. +3. **Async CRITs** (§3.1-3.5): five surgical fixes. Add tests that assert coroutine results are not leaked. +4. **Security CRITs** (§4.1-4.3): one `compare_digest` fix; three sanitizer calls in versioning paths; one slice removal in webhook verifier. +5. **DynamoDB throttle retry** (§5.1): wrap the four uncovered code paths in `_run_with_throttle_retry`. + +### Wave 2 — close the HIGHs that hide bugs (1-2 weeks) + +6. **JsonDB sync stats** (§3.6): six `to_thread` wraps. Bench to confirm regression budget. +7. **`__init_subclass__` super-chains** (§6.1-6.3): three one-line additions. Add regression test that protected attrs are registered on Node/Edge/Walker subclasses. +8. **Auth races + cache invalidation** (§4.5-4.8): four `asyncio.Lock`s. Add concurrent-stress test. +9. **Pager + cache** (§8.1-8.2): drop the stale cache; constrain or fix keyset pagination. +10. **Default `find_one_and_update` `_id` vs `id`** (§5.3): normalize in default impls. + +### Wave 3 — close the latent timebombs (2-4 weeks) + +11. **Env allowlist enforcement + parse_bool consolidation** (§7.1-7.2): both small, both gate a class of future config-typo bugs. +12. **`bulk_save` partial-success semantics** (§5.6, 5.7): unify return type across adapters. +13. **MongoDB `supports_transactions` honesty** (§5.9): probe replica set; rename or runtime-check. +14. **QueryBuilder ↔ QueryEngine operator parity** (§5.2): implement or reject. +15. **CORS wildcard startup warning** (§4.12), **`JVSPATIAL_EXPOSE_ERROR_DETAILS` production guard** (§4.10). + +### Wave 4 — cleanup, parity, polish (ongoing) + +16. False-async cleanup (§3.11): roll-up PR converting 30+ `async def` with no `await` to `def`. Or accept and document. +17. SQLite cross-loop detection (§5.10). +18. Stability-tier cleanup (§7.7 expose `emit_experimental_once`; §7.13 docs-prefix derivation from config). +19. Dead code (§5.14 `JSONTransaction`). +20. LOW hygiene findings (log content sanitization, Windows-reserved-name filename check, `.env.example` accuracy). + +--- + +## Appendix — Reviewer Reports + +Five reviewers produced this audit: +1. **Async contract**: agent `a85870fd5ef8e1790` +2. **Security boundaries**: agent `a27d75965d59e33a0` +3. **DB adapter parity**: agent `ac27d884ad66bfd59` +4. **Walker / identity / invariants**: agent `a716803cafeda1acb` +5. **Serverless / config / stability**: agent `a7842dedd5dfe62d4` + +Full reports are in the agent transcripts. This file synthesizes and de-duplicates their findings; for the raw lists with extended rationale, query the agents above via SendMessage. diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..36b80d3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,16 @@ +jvspatial — Authors & Maintainers +================================= + +jvspatial is a foundational object-spatial application development +framework. + +Creator & Lead Maintainer + Eldon Marks (@eldonm) — https://github.com/eldonm + +Contributors + This list grows with the project. Contributions are welcome. + +--------------------------------------------------------------------- +This file records authorship and ongoing maintenance of the project. +Authorship and maintenance are distinct from copyright ownership and +license terms, which are set out in LICENSE. diff --git a/CHANGELOG.md b/CHANGELOG.md index 647b46a..28c3133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,79 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `JVSPATIAL_DOCS_DISABLED` env var (truthy `1`/`true`/`yes`/`on`) — when set, `AppBuilder.create_app` constructs FastAPI with `docs_url=None`, `redoc_url=None`, `openapi_url=None`, and `swagger_ui_oauth2_redirect_url=None` so the documentation surface is fully unpublished (404 with no spec leak). Recommended for production. +- `Walker.__entity_name__` / `Walker._entity_name()` — parallel to `Object._entity_name()` so walker IDs and the persisted `entity` discriminator honor the per-subclass override. (Audit §1.1, §1.2, §1.9.) +- `TraversalProtection.start_if_needed()` — idempotent initializer for `Walker.run()`. Pause/resume cycles no longer reset step / visit / wall-clock counters. (Audit §2.2.) +- `WalkerTrail(max_length=N)` — wires the previously-undocumented bound. `0` (default) means unlimited; positive integers cap the in-memory trail. Threaded through `Walker(max_trail_length=...)` and `JVSPATIAL_WALKER_MAX_TRAIL_LENGTH`. (Audit §2.3 / SPEC §6.4.) +- `tests/core/test_entity_name_walker_and_save.py`, `tests/core/test_walker_protection_audit_fixes.py`, `tests/storage/test_versioning_path_sanitizer_audit.py`, `tests/api/test_webhook_hmac_audit_fix.py` — 28 new regression cases pinning Wave 1 audit fixes. +- Public `invalidate_api_key_cache(api_key)` and `invalidate_api_key_cache_hash(cache_key)` helpers in `jvspatial.api.integrations.webhooks.webhook_auth`. `APIKeyService.revoke_key` now invokes the latter so revocations are effective immediately rather than after the 5-minute TTL. (Audit §4.5.) +- `tests/db/test_default_compound_ops_id_normalization.py`, `tests/core/test_pager_audit_fixes.py` — 11 new regression cases pinning Wave 2 audit fixes. +- `jvspatial.db.database.BulkSaveResult` dataclass and `Database.bulk_save_detailed()` method. Reports `attempted` / `saved` / `failed_ids` per call so partial-success backends (JsonDB, DynamoDB) can no longer silently drop records. `bulk_save` is preserved as a thin int-returning wrapper for back-compat. (Audit §5.6-§5.7.) +- `MongoDB.is_transactional()` async probe. Uses the `hello` admin command to detect replica-set / sharded topology and caches the result. Use this instead of the static `supports_transactions` flag when the caller intends to actually open a transaction. (Audit §5.9.) +- `CORSConfig.cors_allow_wildcard` opt-out. When `cors_origins` contains a wildcard and this is `False` (default), a startup WARNING is emitted. SPEC §15.4 promised this; the audit found it missing. (Audit §4.12.) +- `JVSPATIAL_STRICT_ENV_ALLOWLIST` env var. Truthy values turn unknown-`JVSPATIAL_*` key detection from a per-key WARNING into a startup `ValueError` so typos fail-fast. (Audit §7.1 / SPEC §10.2.) +- `ALLOWED_ENV_KEYS` frozenset and `enforce_env_allowlist()` / `discover_unknown_jvspatial_env_keys()` helpers in `jvspatial.env_adapter`. Called from `validate_server_config_requirements()` at server startup. +- New top-level / field-level `QueryEngine` operators: `$nor` (top-level logical), `$mod`, `$all`, `$type`, `$not` (field-level). Previously advertised by `QueryBuilder` but silently returned no matches. (Audit §5.2.) +- `tests/api/test_env_allowlist_audit.py`, `tests/api/test_cors_wildcard_and_error_detail_audit.py`, `tests/db/test_query_operator_parity_audit.py`, `tests/db/test_bulk_save_detailed_audit.py` — 22 new regression cases pinning Wave 3 audit fixes. +- `jvspatial.utils.stability.emit_experimental_once(name, message)` — public hook for opt-in surfaces that need to emit the experimental warning without going through the `@experimental` decorator (replaces private `_emit_once` calls). (Audit §7.7.) +- `tests/db/test_sqlite_cross_loop_audit.py`, `tests/utils/test_wave4_polish_audit.py` — 10 new regression cases pinning Wave 4 audit fixes. +- `jvspatial.core.entities.TraversalSkipped` and `TraversalPaused` exception classes. `Walker.skip()` now raises `TraversalSkipped` (caught via `except TraversalSkipped`); previously relied on substring-matching `"Node skipped"` in the message. (Audit §2.9 / SPEC §6.5.) +- `PathSanitizer` rejects Windows-reserved filenames (`CON`, `PRN`, `AUX`, `NUL`, `COM1-9`, `LPT1-9`) regardless of host OS. ``CON.txt`` is rejected; ``CONFIG.json`` passes. (Audit §4.18 / SPEC §15.1.) +- `tests/storage/test_windows_reserved_audit.py`, `tests/core/test_wave5_walker_audit.py`, `tests/api/test_deferred_invoke_fail_closed_audit.py`, `tests/db/test_sqlite_id_coercion_audit.py` — 23 new regression cases pinning Wave 5 audit fixes. + +### Fixed + +- **BREAKING (behavioral):** `Walker.run()` now raises `InfiniteLoopError` / `WalkerTimeoutError` / `WalkerExecutionError` when protection limits trip, as SPEC §6.3 has always promised. Earlier behavior silently swallowed `ProtectionViolation` into `walker.report` and returned. Callers that relied on the swallow contract must wrap `spawn()` / `run()` in `try`/`except`. (Audit §2.1.) +- `GraphContext.save_object` no longer rewrites entity IDs when `__entity_name__` differs from `cls.__name__`. Earlier the ID-validation check compared against `cls.__name__` and regenerated through `cls.__name__` — any entity using the override had its ID silently corrupted on every save. (Audit §1.3, §1.4.) +- `GraphContext.find_edges_between` honors `__entity_name__` so override-using edges are findable. Earlier filtered `entity == edge_class.__name__`. (Audit §1.5.) +- `Node._node_query` keys edge lookups by the persisted `entity` field (not the non-existent `name` column) and honors `__entity_name__`; `Node.count_neighbors` fast-path regex uses `_entity_name()`; `Node._matches_node_filter` compares against `_entity_name()`. (Audit §1.6-§1.8.) +- `AuthenticationService._verify_refresh_token` SHA-256 fallback now uses `hmac.compare_digest` instead of `==`. Removes a timing oracle on refresh-token and password-reset-token hash comparison. (Audit §4.1 / SPEC §15.2.) +- `LocalFileInterface.{create_version, get_version, list_versions, delete_version, get_latest_version}` route `file_path` through a new `_sanitized_version_base` helper. Earlier these computed `self.root_dir / f"{file_path}.versions"` without sanitization — a caller-supplied `../../etc/passwd` escaped the storage root entirely. (Audit §4.2 / SPEC §15.1.) +- `verify_hmac_signature` no longer slices `expected_signature[len(prefix):]`. The earlier slice truncated 7 chars off a 64-char SHA-256 digest so `hmac.compare_digest` always returned False — webhook HMAC verification rejected every request. (Audit §4.3 / SPEC §15.2.) +- Webhook walker `inject_walker_webhook_payload.enhanced_init` is now sync. Python ignores `async def __init__`; the earlier form returned a coroutine that was never awaited and leaked on every webhook walker construction. (Audit §3.1.) +- `webhook_wrapper` now awaits `endpoint_func(**kwargs)` in the async branch. Both arms of the `if/else` were identical, so coroutines from async endpoints leaked unawaited. (Audit §3.2.) +- `FileStorageService.handle_delete_file` now awaits `self.file_interface.delete_file(file_path)`. The missing `await` left `success` as the coroutine object and skipped the delete. (Audit §3.3.) +- `generate_graph_dot` and `generate_graph_mermaid` now wrap `Path.write_text` in `asyncio.to_thread` so disk I/O does not block the event loop inside their async bodies. (Audit §3.4-§3.5.) +- `WalkerQueue.prepend` / `append` / `add_next` / `insert_after` / `insert_before` now respect `max_size` and emit a one-shot WARNING on first drop. Earlier the front-of-queue and middle-insert paths bypassed the cap, providing a silent protection bypass. (Audit §2.4-§2.5.) +- `DynamoDB.{find, count, batch_get, batch_write}` now route every `aioboto3` wire call through `_run_with_throttle_retry`. Earlier the helper was only applied to `save`/`get`/`delete`; `ProvisionedThroughputExceededException` and `ThrottlingException` from scan / query / batch ops surfaced to callers as immediate failures despite the documented backoff. (Audit §5.1 / SPEC §4.3.) +- Security headers middleware now emits a relaxed Content-Security-Policy on `/docs`, `/redoc`, `/openapi.json` (and sub-paths) that permits `cdn.jsdelivr.net` so FastAPI's bundled Swagger UI / ReDoc pages render. The previous strict default blocked the CDN-hosted JS/CSS and the docs loaded blank. Application routes keep the strict default policy. +- **BREAKING (behavioral):** `ObjectPager.get_page` no longer returns cached results. The in-memory `_cache` attribute is removed entirely; every call hits the database. Stale-after-write semantics are eliminated. Callers that relied on caching should use the backend-level read-through cache via `create_database(cache_get_size=...)`. (Audit §8.2.) +- `ObjectPager.get_page(after_id=..., order_by=...)` now raises `ValueError`. Keyset pagination via `after_id` only tracks `id`; combining it with a custom sort key silently skipped or duplicated rows on writes between pages. Use offset pagination if you need a custom sort. (Audit §8.1.) +- Default `Database.find_one_and_update` and `Database.find_one_and_delete` now normalize `{"_id": x}` queries to `{"id": x}` when only `_id` is present, so Mongo-style queries no longer silent-miss on JsonDB / SQLite / DynamoDB (which persist records keyed by `id` only). MongoDB native override is unaffected. (Audit §5.3 / SPEC §4.1.) +- JsonDB no longer blocks the event loop. Every `Path.exists()` / `Path.glob()` call inside `async` methods (`_async_read_json`, `count`, `find_many`, `find`) is now wrapped in `asyncio.to_thread`. (Audit §3.6 / SPEC §3.3.) +- `Node.__init_subclass__`, `Edge.__init_subclass__`, and `Walker.__init_subclass__` now call `super().__init_subclass__(**kwargs)` so `AttributeMixin.__init_subclass__` runs and `protected` / `transient` / `private` attribute registration completes for their subclasses. (Audit §6.1-§6.3 / SPEC §2.5.) +- `SessionManager` mutations now hold an `asyncio.Lock` so concurrent create/invalidate/cleanup cannot raise `RuntimeError: dictionary changed size during iteration`. `max_sessions_per_user` enforcement is no longer racy — over-cap creation evicts the oldest session by `last_accessed`. (Audit §4.8.) +- `_API_KEY_CACHE` (webhook layer) now holds a lock around reads, eviction, and miss-population. Removes a `KeyError` window when a size-cap cleanup races a reader. (Audit §4.7.) +- `APIKeyService(context=None)` now defaults to the **prime** database instead of `get_default_context()`. Auth state is required to live on the prime DB (SPEC §9 / CLAUDE.md §1). (Audit §4.4.) +- `APIKeyService.revoke_key` now invokes the new webhook cache-invalidation hook so a revoked key stops authenticating immediately rather than after the 5-minute TTL. (Audit §4.5.) +- `JVSPATIAL_EXPOSE_ERROR_DETAILS=true` is now ignored when the runtime is signalled as production (`JVSPATIAL_ENVIRONMENT` or `ENVIRONMENT` set to `prod`/`production`). Emits a once-per-process WARNING explaining the suppression. Generic 500 message is returned. (Audit §4.10 / SPEC §15.5.) +- `MongoDB.begin_transaction` now short-circuits to `None` on standalone deployments instead of attempting `start_session` / `start_transaction` every call. Topology is probed once via `is_transactional()` and cached. (Audit §5.9 / SPEC §4.2.) +- `QueryEngine.match` and `QueryEngine._match_value` now raise `QueryError` on unsupported operators rather than silently returning False. Optimizer markers (`$hint`, `$select`) injected into queries are skipped explicitly instead of treated as unknown operators. (Audit §5.2 / SPEC §5.1.) +- Bool parsing consolidated across `jvspatial.env`, `jvspatial.env_adapter`, `jvspatial.runtime.serverless`, and `jvspatial.api.components.app_builder`. All three of the latter delegate to `env.parse_bool`. `JVSPATIAL_DEBUG=on` and `SERVERLESS_MODE=on` now agree on truthiness. (Audit §7.2-§7.3.) +- Unknown `JVSPATIAL_*` env keys now warn at startup (or raise in strict mode). Closes a SPEC §10.2 gap that allowed typos to silently no-op. (Audit §7.1.) +- `SQLiteDB` instances now silently rebind their `aiosqlite` connection when reused across event loops for file-backed paths. The previous opaque "Future attached to a different loop" error from inside `aiosqlite` is replaced with transparent recovery. `:memory:` databases keep the existing connection (rebinding would silently truncate the dataset). (Audit §5.10 / SPEC §4.3.) +- Security headers middleware derives the CSP-relaxation prefixes from `ServerConfig.docs_url` / `redoc_url` / `openapi_url` at install time. Callers customizing the docs URL (e.g. `docs_url=/api/docs`) keep Swagger UI rendering under the relaxed CSP. (Audit §7.13.) +- JWT debug log no longer includes the secret length (narrowing a brute-force search space). Logs `secret_configured=bool(...)` instead. (Audit §4.11 / SPEC §15.5.) +- `validate_token` warning logs `db_type=` instead of `db_path=` so the on-disk filesystem layout does not leak to log sinks. (Audit §4.13 / SPEC §15.5.) +- `LoggingNoopTaskScheduler.schedule` downgraded from per-call WARNING to DEBUG so misconfigured serverless deployments do not flood CloudWatch — the once-per-process startup error from `serverless.factory._note_noop_in_serverless` is sufficient. (Audit §7.14.) +- **BREAKING (behavioral):** the internal deferred-invoke route fails closed when `JVSPATIAL_DEFERRED_INVOKE_SECRET` is unset. Previous "no secret = allow everything" semantics exposed the endpoint to any caller on misconfigured deployments. Set the secret to enable the route, or set `JVSPATIAL_DEFERRED_INVOKE_DISABLED=true` to skip registering it entirely. (Audit §4.16 / SPEC §15.2.) +- **BREAKING (behavioral):** `Walker(type_code=...)` raises `ValueError` when given a value other than `"w"`. The SPEC §1.1 ID-format invariant (`w.EntityName.`) cannot be corrupted by a stray kwarg. (Audit §2.10.) +- Walker `skip()` raises `TraversalSkipped` rather than `JVSpatialError("Node skipped")`. Callers that catch the generic exception or match on the substring will need to update — `except TraversalSkipped:` is the new contract. (Audit §2.9.) +- `SQLiteDB.save` coerces `record["id"]` to `str` so int / `uuid.UUID` ids round-trip correctly through SQLite's TEXT column. The persisted record now also stores the stringified id. (Audit §5.20.) +- `runtime/serverless._parse_bool` logs a WARNING on unrecognized `SERVERLESS_MODE` values (still maps to False for back-compat). Silent garbage-to-False mapping hid typos. (Audit §7.3.) +- `.env.example` CORS section corrected: `Default: *` and `JVSPATIAL_CORS_ORIGINS=*` example replaced with the actual default localhost whitelist, plus a note that wildcards trigger a startup WARNING. (Audit §7.5.) +- `JsonDB._list_collection_json_files` drops the dead `not p.name.endswith('.jvtmp')` filter; tmp files are named `.json...jvtmp` and never match the `*.json` glob. (Audit §5.16.) + +### Deprecated + +- `jvspatial.core.utils.generate_id_async` — deprecated alias for `generate_id`. ID generation is pure computation (SPEC §3.2); the async signature was a vestige. Scheduled for removal in 0.1.0. (Audit §3.11.) + +### Removed + +- `jvspatial.db.transaction.JSONTransaction` — unused dead code superseded by `JsonDBTransaction`. (Audit §5.14.) + ## [0.0.7] - 2026-05-08 ### Security diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..924254c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,246 @@ +# CLAUDE.md — Agent Guide for jvspatial + +> This file is read automatically by Claude Code at the start of every session in this repo. It is the **canonical entry point** for any AI agent maintaining `jvspatial`. Keep it short, current, and actionable. + +--- + +## What jvspatial is + +`jvspatial` is an **async-first, serverless-compatible, object-spatial Python library** for graph-based persistence and business-logic layers. It layers an entity-centric graph model (Object → Node / Edge / Walker / Root) over four database backends (JSON, SQLite, MongoDB, DynamoDB), and ships a FastAPI integration with auth, file storage, observability, and serverless ergonomics. + +For full positioning and non-goals, read [PRD.md](PRD.md). +For the technical contract, read [SPEC.md](SPEC.md). + +--- + +## Where to look first + +When given a task, resolve context in this order: + +1. **[PRD.md](PRD.md)** — what's the product context? Is this in scope? +2. **[SPEC.md](SPEC.md)** — what does the library currently guarantee about the area you're touching? Every claim cites a `file:line`. +3. **[docs/md/README.md](docs/md/README.md)** — index of how-to docs. Find the relevant one before reading source. +4. **[ROADMAP.md](ROADMAP.md)** — is this area under active hardening? Known gap? Out of scope? +5. **[CHANGELOG.md](CHANGELOG.md)** — recent changes that may affect your work. +6. **Source code** — last, not first. The map of subpackage READMEs (`jvspatial/*/README.md`) shortens this hop. + +If a doc disagrees with code, the doc is wrong. File an issue and trust the code — but cite the discrepancy. + +--- + +## Non-negotiable invariants + +These are properties the library guarantees. Breaking them is a regression even if tests pass. + +### 1. Async-only I/O + +Every database call, network call, and file-system call is `async`. There are **no sync wrappers** in the library. Sync is for pure computation only. + +- ❌ `entity.save()` (missing `await`) +- ✅ `await entity.save()` + +If you find yourself wanting a sync wrapper, you are solving the wrong problem. + +### 2. Constant-time secret comparison + +Any comparison of a secret, key, token, or password hash uses `hmac.compare_digest`. Affected paths (SPEC §15.2): +- API key verification, refresh token comparison, password reset token, webhook HMAC, deferred-invoke secret, bcrypt legacy fallback. + +Never replace `hmac.compare_digest` with `==` for performance or readability. + +### 3. Entity name override (`__entity_name__`) + +Persisted entity discriminator is `cls.__dict__.get("__entity_name__") or cls.__name__` (SPEC §1.2). Resolution is **per-subclass**, not inherited. Code that constructs IDs or looks up subclasses must go through `generate_id()` and `find_subclass_by_name()` — never assemble IDs from `cls.__name__` directly. + +### 4. Serverless detection precedence + +`is_serverless_mode(config=None)` resolves: explicit config → current Server config → `SERVERLESS_MODE` env → auto-detection (SPEC §11.1). Do not bypass with custom env reads. Do not memoize results across tests — call `reset_serverless_mode_cache()` between cases. + +### 5. Protected attribute validation + +`Object.__setattr__` validates field names against the class hierarchy. Setting an undeclared attribute on an `Object` is rejected. Never use `object.__setattr__(self, ...)` to bypass; if you need a new field, declare it. + +### 6. Deferred-save MRO + +`class MyEntity(DeferredSaveMixin, Node)` — mixin must come **before** the base. Wrong order silently disables batching. Tests should assert MRO-sensitive behavior. + +### 7. Stability tier discipline + +Symbols in `jvspatial.__all__` are public — breaking changes require a deprecation cycle (`docs/md/stability.md`). Underscore-prefixed modules are internal — callers should not import directly. Promoting an internal helper to public is a product decision; update PRD §7 and docs/md/stability.md together. + +### 8. CORS / CSP / docs defaults + +CORS does **not** default to wildcard. CSP is strict on app routes, relaxed only on `/docs`, `/redoc`, `/openapi.json`. `JVSPATIAL_DOCS_DISABLED` is the production posture. Do not weaken these defaults without a security review entry. + +### 9. Walker protection + +`max_steps=10000`, `max_visits_per_node=100`, `max_execution_time=300s`, `max_queue_size=1000` are defaults that prevent DOS. Disabling protection is allowed locally; never disable globally or in code that touches user input. + +--- + +## Common gotchas + +| Symptom | Cause | Fix | +|---|---|---| +| `TypeError: object NoneType can't be used in 'await' expression` | Forgot `await` on async DB call | Add `await` | +| Custom mixin doesn't batch saves | `DeferredSaveMixin` placed after base in MRO | Mixin first: `class X(DeferredSaveMixin, Node)` | +| Subclass lookup returns wrong class | Two unrelated classes share `__name__` | Set `__entity_name__: ClassVar[Optional[str]] = "Distinct"` on one | +| Tests pass locally, fail under serverless mode | Forgot to test both modes | Use `reset_serverless_mode_cache()` between tests | +| 401 with valid token | Auth state on wrong database | Auth is **always** on prime DB; do not relocate | +| Slow query never logged | `slow_query_ms` not configured | Set in `create_database(observe=True, slow_query_ms=N)` | +| Walker visits the same node forever | Protection disabled | Re-enable `protection_enabled=True` | +| `JVSPATIAL_FOO` env var ignored | Not in allowlist | Add to `jvspatial/env_adapter.py` allowlist | +| SQLite query falls back to in-memory filter | Operator not yet pushed down by `SQLiteTranslator` | Check `jvspatial/db/_sqlite_translate.py` for supported operators | + +--- + +## Graph modeling convention + +Convention for any application built on jvspatial. Not enforced by the library, but required by review and assumed by downstream tooling. + +1. **Every `Node` must be reachable from `Root`** — directly or through intermediate nodes. An orphaned node is a bug. If a record does not belong in the graph, it must not be a `Node`. +2. **Applications declare an application-root node** — e.g. `class App(Node)`, `class MyApp(Node)`. Create exactly one instance, connect it to `Root`, and treat it as the entry point for all application state. +3. **All application nodes extend from the app-root** — directly or under an appropriate branch / category node beneath it (e.g. `App -> Users -> User`, `App -> Catalog -> Product`). Do not connect application nodes directly to `Root`; `Root` belongs to the library, the app-root belongs to the application. +4. **Use `Object` (not `Node`) for record-style data that does not benefit from graph membership** — audit/change-event records, denormalized snapshots, ledger entries, raw inbound payloads. `ChangeEvent` **must be an `Object`**, not a `Node`. Rule of thumb: if nothing will ever traverse to or from it, it is not a `Node`. + +When reviewing or generating code, reject `Node` subclasses that are never connected, and reject `Object` subclasses that are used as relationship endpoints. + +--- + +## Run the dev loop + +```bash +# install +pip install -e '.[dev,test]' +pre-commit install + +# fast feedback +pytest -q # unit + integration (skips benchmarks) +pre-commit run --all-files # lint / format / type-check + +# full quality bar (run before opening PR) +pytest --cov=jvspatial --cov-report=term-missing + +# benchmarks (regression detection) +pytest tests/benchmarks --benchmark-only +``` + +Async tests use `pytest-asyncio` in auto mode (`pyproject.toml`). No manual marker needed for `async def test_*`. + +--- + +## How to make changes safely + +### Add a feature + +1. Read [PRD §8](PRD.md#8-decision-boundaries) — does the change pass the decision rules? +2. Read the relevant SPEC section. Does the change alter the contract? If yes, plan a SPEC update in the same commit. +3. Test-first on the **JSON backend** — fastest iteration, no external dependencies. +4. Check stability tier of the file you're touching ([docs/md/stability.md](docs/md/stability.md)). +5. Update [CHANGELOG.md](CHANGELOG.md) under `## [Unreleased]`. +6. If touching a public name, also update `jvspatial.__all__`. + +### Fix a bug + +1. Reproduce in a failing test. +2. Resolve root cause; do not paper over. +3. If the bug reveals a SPEC inaccuracy, update SPEC. +4. CHANGELOG entry under `### Fixed`. + +### Touch auth, secrets, or security + +1. Read [docs/md/security-review.md](docs/md/security-review.md) and [docs/md/security-operational-notes.md](docs/md/security-operational-notes.md). +2. Preserve constant-time comparisons. +3. Add a corresponding security-review entry for the change. + +### Touch a serverless code path + +1. Check `is_serverless_mode()` precedence (SPEC §11.1) before adding new mode-sensitive defaults. +2. Test both modes; reset the detection cache between tests. +3. If adding a Lambda-specific behavior, document the LWA interaction in [docs/md/serverless-mode.md](docs/md/serverless-mode.md). + +### Add a database backend + +1. Subclass `Database` (`jvspatial/db/database.py:48`). +2. Register via `register_database_type("name", factory_fn)` (`jvspatial/db/factory.py`). +3. Implement bulk overrides (`find_many`, `bulk_save`); the defaults are correct but slow. +4. Set `supports_transactions` capability flag. +5. Add backend-specific tests in `tests/db/`. + +--- + +## Boundaries — what NOT to do + +- Do **not** add a sync version of any I/O API. +- Do **not** broaden CORS defaults to wildcards. +- Do **not** replace `hmac.compare_digest` with `==`. +- Do **not** commit `.env` or any secrets. Use `.env.example` for templates. +- Do **not** import from underscore-prefixed modules (`_atomic`, `_path_locks`, `_cache`, `_observable`) — go through the public factory. +- Do **not** modify SPEC.md without modifying the corresponding code in the same commit (or vice versa). +- Do **not** edit `LLM-CODING-GUIDE.md` — it is preserved as a legacy reference. New documentation belongs in `docs/md/` or in PRD/SPEC/ROADMAP. +- Do **not** add features mentioning specific downstream consumers in the library or its docs. jvspatial is a foundation; consumer-specific logic lives downstream. + +--- + +## Repo geography + +``` +jvspatial/ +├── core/ # Entity hierarchy, GraphContext, events +│ ├── entities/ # Object, Node, Edge, Walker, Root +│ ├── annotations/ # @attribute system +│ └── walker_components/ # Trail, protection, queue, events +├── api/ # FastAPI integration +│ ├── auth/ # JWT, API keys, RBAC, sessions +│ ├── components/ # AppBuilder, AuthConfigurator, middleware +│ ├── decorators/ # @endpoint +│ ├── endpoints/ # Registry, factory, router +│ └── integrations/ # Webhooks, scheduler, storage service +├── db/ # Database backends + abstraction +│ ├── jsondb.py, sqlite.py, mongodb.py, dynamodb.py +│ ├── _atomic.py, _path_locks.py, _cache.py, _observable.py # internal +│ └── query.py, factory.py, manager.py +├── storage/ # File storage +│ ├── interfaces/ # Local, S3 +│ └── security/ # path_sanitizer, validator +├── cache/ # Memory, Redis, layered +├── serverless/ # Deferred task dispatch +├── runtime/ # Serverless detection, LWA helpers +├── observability/ # Metrics, OTEL adapter +├── logging/ # Custom levels, persisted logging +├── exceptions.py # Central exception hierarchy +├── env.py # Env helpers +├── env_adapter.py # JVSPATIAL_* allowlist + merge +└── version.py # Single source of version truth + +tests/ # Mirrors jvspatial/ layout +docs/md/ # How-to and reference docs (see docs/md/README.md) +examples/ # Runnable example scripts +``` + +Each top-level subpackage has its own `README.md` (added in this docs pass) — read that before diving into source. + +--- + +## Slash commands and skills relevant in this repo + +- `pytest` and `pytest --benchmark-only` are the canonical test entry points. +- `pre-commit run --all-files` runs the full linter/formatter suite (black, isort, flake8, mypy, detect-secrets). +- For larger refactors, follow the [docs/md/contributing.md](docs/md/contributing.md) workflow; for security-sensitive changes, the workflow in [docs/md/security-review.md](docs/md/security-review.md). + +--- + +## Pointers to authoritative sources + +When in doubt, the source-of-truth order is: + +1. Code (always) +2. [SPEC.md](SPEC.md) (technical contract; should match code) +3. [PRD.md](PRD.md) (product context) +4. [docs/md/](docs/md/) (how-to) +5. [CHANGELOG.md](CHANGELOG.md) (release notes) +6. [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md) (legacy; informational only, not contractual) + +--- + +**Last updated**: Same commit as the PRD/SPEC/ROADMAP docs pass. Update this file in the same commit as any change that alters an invariant or boundary above. diff --git a/LLM-CODING-GUIDE.md b/LLM-CODING-GUIDE.md new file mode 100644 index 0000000..a80c749 --- /dev/null +++ b/LLM-CODING-GUIDE.md @@ -0,0 +1,5284 @@ +# jvspatial Language Model Coding Guide + +This document provides concise guidance for AI language models to generate code that follows jvspatial library standards and conventions. + +## 🎯 Core Philosophy + +jvspatial emphasizes **entity-centric design** with unified MongoDB-style queries across database backends (JSON, MongoDB). The library distinguishes between: + +- **Objects** - For standalone data entities (users, settings, logs) that don't require graph relationships +- **Nodes** - For graph entities that are interconnected by Edges and traversed by Walkers +- **Edges** - For relationships between Nodes in the graph +- **Walkers** - For traversing and processing graph structures + +**Key Principle**: Use Objects for simple data storage, use Nodes when you need graph traversal and relationships. + +## 🔧 Environment Setup + +### Essential Configuration +```python +from dotenv import load_dotenv +load_dotenv() # Load .env file + +from jvspatial.core import GraphContext +from jvspatial.db import create_database, get_database_manager + +# Option 1: Create database explicitly +db = create_database("json", base_path="./jvdb") +ctx = GraphContext(database=db) + +# Option 2: Use current database from manager (defaults to prime) +manager = get_database_manager() +ctx = GraphContext(database=manager.get_current_database()) +``` + +### Environment Variables +```env +# Choose backend +JVSPATIAL_DB_TYPE=json # or 'mongodb' + +# JSON backend (default) +JVSPATIAL_JSONDB_PATH=./jvdb/dev + +# MongoDB backend +JVSPATIAL_MONGODB_URI=mongodb://localhost:27017 +JVSPATIAL_MONGODB_DB_NAME=jvspatial_dev + +# Caching (optional) +JVSPATIAL_CACHE_BACKEND=memory # 'memory', 'redis', or 'layered' +JVSPATIAL_CACHE_SIZE=1000 # Max cached entities (0 to disable) +``` + +## 📝 Entity-Centric Code Patterns + +### Objects vs Nodes: When to Use Each + +**✅ Objects** - For standalone entities without graph relationships: +```python +from jvspatial.core import Object + +class UserProfile(Object): + name: str = "" + email: str = "" + settings: Dict[str, Any] = {} + +# Use for: user profiles, configuration, logs, simple data +profile = await UserProfile.create(name="Alice", email="alice@company.com") +``` + +**✅ Nodes** - For graph entities with relationships and traversal: +```python +from jvspatial.core import Node + +class User(Node): + name: str = "" + department: str = "" + +class City(Node): + name: str = "" + population: int = 0 + +# Use for: entities that connect to other entities via Edges +user = await User.create(name="Alice", department="engineering") +city = await City.create(name="San Francisco", population=800000) +``` + +### Entity Operations +```python +# Entity creation (no save() needed, automatically cached) +entity = await Entity.create(name="value", field="data") + +# Entity retrieval (uses cache after first access) +entity = await Entity.get(entity_id) # Cached by ID +entities = await Entity.find({"context.active": True}) # Not cached + +# Entity updates (save() only needed after property modification, updates cache) +entity = await Entity.get(entity_id) +entity.name = "Updated Name" # Property modified +await entity.save() # save() required to persist changes + update cache + +# Entity deletion (removes from cache) +await entity.delete() + +# Counting and aggregation (not cached) +# Note: Object.count() doesn't exist - use len() with find() instead +results = await Entity.find({"context.department": "engineering"}) +count = len(results) + +# For distinct values, query and extract manually +all_entities = await Entity.find({}) +departments = set(e.department for e in all_entities if hasattr(e, 'department')) +``` + +**Note**: Caching is automatic and transparent. Individual entity retrievals by ID (`Entity.get(id)`) are cached. Queries (`find()`, `all()`) always hit the database as they can change frequently. For counting, use `Entity.count(query)` for efficient counting without loading all records. + +### save() Operation Rules +**✅ save() is ONLY required when:** +1. You modify entity properties after retrieval: `entity.field = "new_value"` +2. You create entities without using `.create()` method + +**❌ save() is NOT needed when:** +1. Using `.create()` method (automatically persists) +2. Using `.delete()` method (automatically persists deletion) +3. Just reading/querying entities + +**❌ AVOID: Direct database access (use entity methods instead)** +```python +# Don't do this - use entity methods instead +from jvspatial.db import create_database +db = create_database("json") +entities = await db.find("object", {"name": "Entity"}) + +# ✅ Do this instead - use entity-centric methods +entities = await Entity.find({"context.name": "Entity"}) +``` + +## 🗄️ Multi-Database Support + +jvspatial supports managing multiple databases within the same application, with a prime database for core persistence operations (authentication, session management) and additional databases for application-specific data. + +### Basic Multi-Database Usage + +```python +from jvspatial.db import ( + create_database, + get_database_manager, + get_prime_database, + get_current_database, + switch_database, + unregister_database, +) +from jvspatial.core.context import GraphContext + +# Get database manager (singleton) +manager = get_database_manager() + +# Prime database is automatically created for core operations +prime_db = get_prime_database() # Used for auth, sessions, system data + +# Create and register additional database +app_db = create_database( + "json", + base_path="./app_data", + register=True, + name="app" +) + +# Switch to application database +switch_database("app") +current_db = get_current_database() # Now returns app_db + +# Use with GraphContext +app_ctx = GraphContext(database=current_db) + +# Switch back to prime database +switch_database("prime") + +# Unregister non-prime database when no longer needed +unregister_database("app") +``` + +### Prime Database + +The prime database is always used for: +- User authentication +- Session management +- System-level configuration +- Core persistence operations + +It cannot be unregistered and is always available as the default. + +### Database Isolation + +Each database maintains complete isolation: +- Entities in one database are not visible in another +- Switching databases changes the context for all operations +- Prime database ensures core operations always have a stable database + +**📖 For comprehensive multi-database documentation:** [Graph Context Guide](docs/md/graph-context.md) and [Multi-Database Example](examples/database/multi_database_example.py) + +## 🔧 Custom Database Integration + +jvspatial supports seamless extension with custom database backends through a registration system. + +### Registering Custom Database Types + +```python +from jvspatial.db import Database, register_database_type, create_database, list_database_types + +# Define custom database implementation +class CustomDatabase(Database): + async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: + # Implementation + pass + + async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: + # Implementation + pass + + async def delete(self, collection: str, id: str) -> None: + # Implementation + pass + + async def find(self, collection: str, query: Dict[str, Any]) -> List[Dict[str, Any]]: + # Implementation + pass + +# Factory function for creating instances +def create_custom_db(**kwargs: Any) -> CustomDatabase: + return CustomDatabase(**kwargs) + +# Register the custom database type +register_database_type("custom", create_custom_db) + +# Now use it like built-in types +db = create_database("custom", connection_string="custom://...") + +# List all available database types +types = list_database_types() +# Returns: {"json": "JSON file-based database", "mongodb": "MongoDB database", "custom": "Custom database: create_custom_db"} +``` + +### Custom Database Requirements + +Custom databases must: +1. Inherit from `Database` abstract base class +2. Implement all abstract methods: `save()`, `get()`, `delete()`, `find()` +3. Optionally implement `count()` and `find_one()` (default implementations provided) +4. Provide a factory function for creation + +**📖 For comprehensive custom database documentation:** [Custom Database Guide](docs/md/custom-database-guide.md) and [Custom Database Example](examples/database/custom_database_example.py) + +## 🔍 MongoDB-Style Query Patterns + +Always use dot notation for nested fields with `context.` prefix: + +```python +# Comparison operators +users = await User.find({"context.age": {"$gte": 35}}) +users = await User.find({"context.role": {"$ne": "admin"}}) + +# Logical operators +users = await User.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.active": True} + ] +}) + +# Array operations +users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) + +# Regular expressions +users = await User.find({ + "context.name": {"$regex": "Johnson", "$options": "i"} +}) +``` + +## 🔒 Attribute Annotations (Protected & Transient) + +jvspatial provides `@protected` and `@transient` decorators for controlling attribute behavior: + +### Protected Attributes +Protected attributes cannot be modified after initialization (ideal for IDs and immutable config): + +```python +from pydantic import Field +from jvspatial.core.annotations import protected + +class Entity(Node): + # id is already protected in Node + uuid: str = protected("", description="Immutable UUID") + created_at: datetime = protected(Field(default_factory=datetime.now)) + +# ✓ Can set during initialization +entity = await Entity.create(uuid="abc-123") + +# ✗ Cannot modify after creation +entity.uuid = "new-uuid" # Raises AttributeProtectionError +``` + +### Transient Attributes +Transient attributes are excluded from database exports (ideal for runtime caches): + +```python +from jvspatial.core.annotations import transient + +class Entity(Node): + data: str = "" + cache: dict = transient(Field(default_factory=dict)) # Not persisted + temp_count: int = transient(Field(default=0)) # Not persisted + +entity.cache["key"] = "value" # Works at runtime +data = await entity.export() # cache excluded from export +``` + +### Compound Decorators +Combine both for internal state that's neither modifiable nor persisted: + +```python +# Both protected AND transient +_internal: dict = protected(transient(Field(default_factory=dict))) +``` + +**Key Points:** +- All `id` fields in `Object`, `Node`, `Edge`, and `Walker` are automatically protected +- Always use `Field(default_factory=dict)` syntax with `@transient` +### Private Attributes +Private attributes are excluded from serialization and database operations (ideal for internal state): + +```python +from jvspatial.core.annotations import private + +class Entity(Node): + _cache: dict = private(default_factory=dict) # Not serialized + _internal_counter: int = private(default=0) # Not serialized + +entity._cache["key"] = "value" # Works at runtime +data = await entity.export() # _cache excluded from export +``` + +### Compound Decorators +Combine decorators for complex behaviors: +```python +# Private AND transient +_internal: dict = private(transient(Field(default_factory=dict))) +``` +- See [Attribute Annotations](docs/md/attribute-annotations.md) for full documentation + +## 🏢 Type Annotations & Error Handling + +### Required Typing Pattern +```python +from typing import List, Optional, Dict, Any +from jvspatial.core import Node, Object +from jvspatial.exceptions import NodeNotFoundError, ValidationError + +class User(Node): + name: str = "" + email: str = "" + age: int = 0 + roles: List[str] = [] + active: bool = True + metadata: Dict[str, Any] = {} + +async def get_user_by_id(user_id: str) -> Optional[User]: + """Get user by ID, returning None if not found.""" + try: + return await User.get(user_id) + except NodeNotFoundError: + return None +``` + +### Error Handling Patterns + +**🎯 Always catch specific exceptions first:** + +```python +import logging +from jvspatial.exceptions import ( + JVSpatialError, + ValidationError, + EntityNotFoundError, + NodeNotFoundError, + DatabaseError, + ConnectionError +) + +logger = logging.getLogger(__name__) + +# Entity operations with error handling +async def safe_user_operation(user_id: str) -> Optional[User]: + try: + user = await User.get(user_id) + return user + except NodeNotFoundError as e: + logger.warning(f"User not found: {e.entity_id}") + return None + except ValidationError as e: + logger.error(f"Validation failed: {e.message}") + if e.field_errors: + for field, error in e.field_errors.items(): + logger.error(f" {field}: {error}") + return None + except DatabaseError as e: + logger.error(f"Database error: {e.message}") + raise # Re-raise for higher-level handling + except JVSpatialError as e: + logger.error(f"jvspatial error: {e.message}") + return None +``` + +**🔄 Database operations with fallback:** + +```python +from jvspatial.exceptions import ConnectionError, QueryError + +async def robust_user_search(query: Dict[str, Any]) -> List[User]: + try: + # Try complex query + return await User.find(query) + except QueryError as e: + logger.warning(f"Complex query failed: {e.message}") + # Fallback to simple query + try: + all_users = await User.all() + # Apply filtering in Python + return [u for u in all_users if u.active] + except Exception: + logger.error("All query methods failed") + return [] + except ConnectionError as e: + logger.error(f"Database connection failed: {e.database_type}") + return [] # Graceful degradation +``` + +**⚠️ Walker error handling:** + +```python +from jvspatial.exceptions import WalkerExecutionError, WalkerTimeoutError + +class SafeUserProcessor(Walker): + @on_visit(User) + async def process_user(self, here: User): + try: + # Potentially risky operation + result = await external_api_call(here) + self.report(result) + except Exception as e: + # Don't let individual errors stop traversal + logger.warning(f"Failed to process user {here.id}: {e}") + self.report({"error": str(e), "user_id": here.id}) + +async def run_safe_walker(): + try: + walker = SafeUserProcessor() + result = await walker.spawn(start_user) + + # Get results and errors + report = await result.get_report() + errors = [r for r in report if isinstance(r, dict) and "error" in r] + logger.info(f"Processed with {len(errors)} errors") + + except WalkerTimeoutError as e: + logger.error(f"Walker timed out after {e.timeout_seconds}s") + # Access partial results if needed + except WalkerExecutionError as e: + logger.error(f"Walker failed: {e.walker_class} - {e.message}") +``` + +## 📄 ObjectPager for Large Datasets + +### Basic Pagination +```python +from jvspatial.core.pager import paginate_objects, ObjectPager + +# Simple pagination +users = await paginate_objects(User, page=1, page_size=20) + +# With filters +active_users = await paginate_objects( + User, + page=1, + page_size=10, + filters={"context.active": True} +) + +# Advanced pager +pager = ObjectPager( + User, + page_size=25, + filters={"context.department": "engineering"}, + order_by="name" +) +users = await pager.get_page(1) +``` + +## ⏰ Scheduler Integration + +### Task Scheduling +```python +from jvspatial.core.scheduler import Scheduler, ScheduledTask +from datetime import datetime, timedelta + +# Create scheduler +scheduler = Scheduler() + +# Schedule recurring tasks +@scheduler.task(interval=timedelta(hours=1)) +async def cleanup_expired_sessions(): + """Clean up expired user sessions hourly.""" + expired = await UserSession.find({ + "context.expires_at": {"$lt": datetime.now()} + }) + for session in expired: + await session.delete() + print(f"Cleaned up {len(expired)} expired sessions") + +# Schedule one-time tasks +@scheduler.task(run_at=datetime.now() + timedelta(minutes=30)) +async def send_reminder_emails(): + """Send reminder emails.""" + users = await User.find({"context.reminder_due": True}) + for user in users: + await send_email(user.email, "Reminder") + user.reminder_due = False + await user.save() + +# Start scheduler +await scheduler.start() +``` + +### Walker-Based Scheduled Tasks +```python +from jvspatial.core import Walker +from jvspatial.core.entities import on_visit + +@scheduler.walker_task(interval=timedelta(days=1)) +class DailyMaintenanceWalker(Walker): + """Perform daily maintenance tasks via graph traversal.""" + + @on_visit("User") + async def check_user_activity(self, here: Node): + """Check user activity and update status.""" + if here.last_active < datetime.now() - timedelta(days=30): + here.status = "inactive" + await here.save() + self.report({"deactivated_user": here.id}) + + @on_visit("DataNode") + async def cleanup_old_data(self, here: Node): + """Remove old data nodes.""" + if here.created_at < datetime.now() - timedelta(days=90): + await here.delete() + self.report({"deleted_data_node": here.id}) + +# Start scheduled walker +await scheduler.start_walker_task(DailyMaintenanceWalker) +``` + +## 🌐 API Server with Server Class + +### Basic Server Setup +```python +from jvspatial.api import Server + +# Create server instance +server = Server( + title="My Spatial API", + description="Graph-based spatial data management API", + version="1.0.0", + host="0.0.0.0", + port=8000 +) + +# Run server +if __name__ == "__main__": + server.run() +``` + +### Walker Endpoints +```python +from jvspatial.api import endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit + +@endpoint("/api/users/process", methods=["POST"]) +class ProcessUser(Walker): + """Process user data with graph traversal.""" + + user_name: str = EndpointField( + description="Name of user to process", + examples=["John Doe"] + ) + + department: str = EndpointField( + default="general", + description="User department" + ) + + @on_visit("User") + async def process_user(self, here: Node): + """Process user nodes - use 'here' for visited node.""" + if here.name == self.user_name: + self.report({ + "found_user": { + "id": here.id, + "name": here.name, + "department": here.department + } + }) + + # Get connected nodes and continue traversal + colleagues = await here.nodes( + node=['User'], + department=self.department + ) + await self.visit(colleagues) +``` + +### Function Endpoints +```python +from jvspatial.api import endpoint + +@endpoint("/api/users/count", methods=["GET"]) +async def get_user_count() -> Dict[str, int]: + """Get total user count.""" + users = await User.all() + return {"total_users": len(users)} + +@endpoint("/api/users/{user_id}", methods=["GET"]) +async def get_user(user_id: str, endpoint) -> Any: + """Get user with semantic response.""" + user = await User.get(user_id) + if not user: + return endpoint.not_found( + message="User not found", + details={"user_id": user_id} + ) + + return endpoint.success( + data={"id": user.id, "name": user.name, "email": user.email} + ) +``` + +### Server with Scheduler Integration +```python +from jvspatial.api import Server +from jvspatial.core.scheduler import Scheduler + +# Create integrated server with scheduler +server = Server(title="Scheduled API", port=8000) +scheduler = Scheduler() + +# Add scheduled tasks +@scheduler.task(interval=timedelta(minutes=5)) +async def periodic_health_check(): + """Check system health every 5 minutes.""" + # Health check logic here + pass + +@server.on_startup +async def startup_tasks(): + """Start scheduler when server starts.""" + await scheduler.start() + print("✅ Server and scheduler started") + +@server.on_shutdown +async def shutdown_tasks(): + """Stop scheduler when server shuts down.""" + await scheduler.stop() + print("🛑 Server and scheduler stopped") + +# Run integrated server +if __name__ == "__main__": + server.run() +``` + +## 🔗 Webhook Integration + +### Basic Webhook Handler +```python +from jvspatial.api import endpoint +from fastapi import Request + +@endpoint("/webhook/{service}/{auth_token}", methods=["POST"], webhook=True) +async def webhook_handler(request: Request) -> Dict[str, Any]: + """Process webhooks with automatic payload parsing.""" + raw_body = request.state.raw_body + content_type = request.state.content_type + current_user = get_current_user(request) + + # Always return 200 for webhooks + try: + # Process webhook logic here + return {"status": "success", "processed_at": datetime.now().isoformat()} + except Exception as e: + logger.error(f"Webhook error: {e}") + return {"status": "received", "error": "logged"} +``` + +## 🏗️ Core Architecture & Walker Patterns + +### Entity Hierarchy +- **Object** - Base class for all entities with unified query interface +- **Node** - Graph nodes with spatial/contextual data (extends Object) +- **Edge** - Relationships between nodes +- **Walker** - Graph traversal and processing logic +- **GraphContext** - Low-level database interface (use sparingly) + +### Walker Traversal (CRITICAL PATTERNS) + +#### Naming Convention for @on_visit Methods +**ALWAYS** use these parameter names: +- **`here`** - The visited node/edge (current location) +- **`visitor`** - The visiting walker (when accessing from node context) + +```python +@on_visit("User") +async def process_user(self, here: Node): + """Use 'here' for visited node.""" + connected_users = await here.nodes(node=['User']) + await self.visit(connected_users) + +@on_visit("City") +async def process_city(self, here: Node): + """Process cities with filtering.""" + # Skip small cities + if here.population < 10000: + self.skip() # Skip to next node + return + + # Get large connected cities + large_cities = await here.nodes( + node=['City'], + population={"$gte": 500000} + ) + await self.visit(large_cities) +``` + +#### Walker Control Flow +```python +class DataWalker(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.max_items = 100 + + @on_visit("Document") + async def process_document(self, here: Node): + """Process with control flow.""" + # Skip invalid documents + if here.status == "invalid": + self.skip() # Continue to next node + + # Stop at limit + if self.processed_count >= self.max_items: + await self.disengage() # Permanently halt walker + return + + # Pause for rate limiting + if self.processed_count % 50 == 0: + self.pause("Rate limit pause") + + # Normal processing + self.processed_count += 1 + next_docs = await here.nodes(node=['Document']) + await self.visit(next_docs) + + @on_exit + async def cleanup(self): + """Called when walker completes/pauses/disengages.""" + print(f"Processed {self.processed_count} documents") +``` + +## 📋 Quick Reference Checklist + +### Entity Operations +- ✅ Use `await Entity.create(**kwargs)` +- ✅ Use `await Entity.find(query_dict)` +- ✅ Use `await entity.save()` only after property modification +- ✅ Use Objects for standalone data, Nodes for graph entities +- ✅ Use `await node1.disconnect(node2)` to remove connections +- ❌ Avoid direct GraphContext database calls + +### Disconnecting Nodes +To remove connections between nodes, use the `disconnect()` method. This removes edges between nodes and deletes the edge objects. + +```python +# Disconnect two nodes +success = await node1.disconnect(node2) + +# Disconnect with specific edge type +success = await node1.disconnect(node2, edge_type=SpecialEdge) +``` + +### Query Patterns +- ✅ Use `"context.field"` dot notation for nested fields +- ✅ Use MongoDB operators: `$gte`, `$in`, `$regex`, `$and`, `$or` +- ✅ Combine dict filters with kwargs: `node=[{'User': {...}}], active=True` + +### Walker Patterns +- ✅ Use `here` parameter for visited nodes +- ✅ Use `await here.nodes()` to get connected nodes +- ✅ Use `await self.visit(nodes)` to continue traversal +- ✅ Use `self.skip()` to skip current node +- ✅ Use `await self.disengage()` to permanently halt +- ✅ Use `self.pause()` for temporary suspension + +### API Patterns +- ✅ Use `@endpoint` for both graph processing and simple functions +- ✅ Use `EndpointField` for parameter configuration +- ✅ Use `endpoint.success()`, `endpoint.not_found()` for responses +- ✅ Always return 200 for webhooks with try/catch + +### Type Safety +- ✅ Always include proper type annotations +- ✅ Import from `typing` and `jvspatial.exceptions` +- ✅ Handle `NodeNotFoundError`, `ValidationError`, `DatabaseError` +- ✅ Use structured error logging + +This guide provides the essential patterns for generating jvspatial-compliant code. Focus on entity-centric operations, proper typing, and following the established naming conventions for walker traversal. + +The **jvspatial webhook system** provides secure, flexible webhook endpoints with built-in authentication, HMAC verification, idempotency keys, and automatic payload processing. Webhooks integrate seamlessly with the FastAPI server and support both function-based handlers and graph traversal processing. + +### Webhook Architecture Overview + +Webhooks in jvspatial are designed for: +- **Security**: Path-based authentication tokens, optional HMAC signature verification +- **Reliability**: Idempotency key support to handle duplicate deliveries +- **Flexibility**: JSON/XML/binary payload support with automatic parsing +- **Integration**: Full compatibility with existing authentication and permission systems +- **Processing**: Always return HTTP 200 for proper webhook etiquette + +### Basic Webhook Endpoint Setup + +```python path=null start=null +from fastapi import Request +from jvspatial.api import endpoint +from jvspatial.api.auth.middleware import get_current_user +from typing import Dict, Any +import json + +# Basic webhook handler function +@endpoint("/webhook/{route}/{auth_token}", methods=["POST"], webhook=True) +async def generic_webhook_handler(request: Request) -> Dict[str, Any]: + """Generic webhook handler for multiple services. + + Processes webhooks from various sources using route-based dispatch. + Middleware handles authentication, HMAC verification, and payload parsing. + """ + # Access processed data from middleware + raw_body = request.state.raw_body # Original bytes + content_type = request.state.content_type # Content-Type header + route = getattr(request.state, "webhook_route", "unknown") # Route parameter + current_user = get_current_user(request) # Authenticated user + + # Parse payload based on content type + processed_data = None + if content_type == "application/json": + try: + processed_data = json.loads(raw_body) + except json.JSONDecodeError: + return {"status": "error", "message": "Invalid JSON payload"} + else: + # Handle other content types (XML, form data, binary) + processed_data = {"raw_length": len(raw_body), "type": content_type} + + # Route-based processing + if route == "stripe": + return await process_stripe_webhook(processed_data, current_user) + elif route == "github": + return await process_github_webhook(processed_data, current_user) + elif route == "slack": + return await process_slack_webhook(processed_data, current_user) + else: + # Generic processing for unknown routes + return { + "status": "received", + "route": route, + "payload_type": content_type, + "user_id": current_user.id if current_user else None + } + +# Service-specific webhook handlers +@endpoint("/webhook/stripe/{auth_token}", methods=["POST"], webhook=True) +async def stripe_webhook_handler(request: Request) -> Dict[str, Any]: + """Dedicated Stripe webhook handler with event processing.""" + raw_body = request.state.raw_body + current_user = get_current_user(request) + + try: + event = json.loads(raw_body) + event_type = event.get("type", "unknown") + + # Process different Stripe event types + if event_type == "payment_intent.succeeded": + await handle_successful_payment(event["data"]["object"], current_user) + elif event_type == "customer.subscription.updated": + await handle_subscription_update(event["data"]["object"], current_user) + elif event_type == "invoice.payment_failed": + await handle_payment_failure(event["data"]["object"], current_user) + + return { + "status": "success", + "event_type": event_type, + "processed_at": datetime.now().isoformat() + } + + except Exception as e: + # Always return 200 for webhooks, log errors internally + print(f"Stripe webhook processing error: {e}") + return {"status": "received", "error": "Processing error logged"} + +# Helper functions for webhook processing +async def process_stripe_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process Stripe webhook events.""" + event_type = data.get("type", "unknown") + return { + "status": "success", + "message": f"Processed Stripe {event_type}", + "user_id": user.id if user else None + } + +async def process_github_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process GitHub webhook events.""" + action = data.get("action", "unknown") + repo_name = data.get("repository", {}).get("name", "unknown") + return { + "status": "success", + "message": f"GitHub {action} on {repo_name}", + "user_id": user.id if user else None + } + +async def process_slack_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: + """Process Slack webhook events.""" + event_type = data.get("type", "unknown") + return { + "status": "success", + "message": f"Slack {event_type} processed", + "user_id": user.id if user else None + } +``` + +### Webhook Security and Authentication + +Webhook endpoints require authentication tokens in the URL path and support additional security measures: + +```python path=null start=null +# Webhook with permission requirements +@endpoint( + "/webhook/admin/{route}/{auth_token}", + methods=["POST"], + webhook=True, + permissions=["process_webhooks", "admin_access"], + roles=["admin", "webhook_manager"] +) +async def admin_webhook_handler(request: Request) -> Dict[str, Any]: + """Administrative webhook handler with strict permissions.""" + current_user = get_current_user(request) + + # User is guaranteed to have required permissions due to middleware + return { + "status": "success", + "message": "Admin webhook processed", + "admin_user": current_user.username, + "permissions": current_user.permissions + } + +# HMAC signature verification (handled by middleware) +@endpoint("/webhook/secure/{service}/{auth_token}", methods=["POST"], webhook=True) +async def secure_webhook_handler(request: Request) -> Dict[str, Any]: + """Webhook with HMAC signature verification. + + Middleware automatically verifies HMAC signatures when present. + Configure HMAC secrets via environment variables or user settings. + """ + # If this handler executes, HMAC verification passed (if configured) + raw_body = request.state.raw_body + hmac_verified = getattr(request.state, "hmac_verified", False) + + return { + "status": "success", + "message": "Secure webhook processed", + "hmac_verified": hmac_verified, + "payload_size": len(raw_body) + } +``` + +### Graph Traversal Webhook Processing (Future Enhancement) + +The architecture supports webhook processing through graph traversal using Walker classes: + +```python path=null start=null + +# from jvspatial.api.auth.decorators import webhook_endpoint +# from jvspatial.core import Walker, Node +# from jvspatial.core.entities import on_visit + +# @endpoint("/webhook/process/{route}/{auth_token}", methods=["POST"]) +# class WebhookProcessingWalker(Walker): +# """Walker-based webhook processing with graph traversal.""" +# +# def __init__(self): +# super().__init__() +# self.webhook_data = None +# self.processing_results = [] +# +# @on_visit("WebhookEvent") +# async def process_webhook_event(self, here: Node): +# """Process webhook events stored as graph nodes.""" +# # Access webhook data from request.state +# payload = self.webhook_data +# +# # Process event based on node data and webhook payload +# result = await self.analyze_event(here, payload) +# self.processing_results.append(result) +# +# # Continue traversal to related events +# related_events = await here.nodes(node=['WebhookEvent']) +# await self.visit(related_events) +# +# async def analyze_event(self, event_node: Node, payload: dict) -> dict: +# """Analyze webhook event against stored data.""" +# return { +# "event_id": event_node.id, +# "payload_type": payload.get("type"), +# "correlation_score": 0.95 # Example analysis result +# } +``` + +### Idempotency and Duplicate Handling + +Webhooks support idempotency keys to handle duplicate deliveries: + +```python path=null start=null +@endpoint("/webhook/idempotent/{auth_token}", methods=["POST"], webhook=True) +async def idempotent_webhook_handler(request: Request) -> Dict[str, Any]: + """Webhook handler with built-in idempotency support. + + Middleware automatically handles idempotency keys in headers: + - Idempotency-Key header + - X-Idempotency-Key header + - Custom idempotency headers + """ + # Access idempotency information from middleware + idempotency_key = getattr(request.state, "idempotency_key", None) + is_duplicate = getattr(request.state, "is_duplicate_request", False) + + if is_duplicate: + # Return cached response for duplicate requests + cached_response = getattr(request.state, "cached_response", {}) + return { + "status": "success", + "message": "Duplicate request, returning cached response", + "idempotency_key": idempotency_key, + "cached_result": cached_response + } + + # Process new request + raw_body = request.state.raw_body + processed_result = await process_unique_webhook(json.loads(raw_body)) + + return { + "status": "success", + "message": "New webhook processed", + "idempotency_key": idempotency_key, + "result": processed_result + } + +async def process_unique_webhook(payload: dict) -> dict: + """Process a unique webhook payload.""" + # Simulate processing logic + import time + processing_start = time.time() + + # Your actual webhook processing logic here + await asyncio.sleep(0.1) # Simulate work + + return { + "processed_at": processing_start, + "data_processed": True, + "payload_keys": list(payload.keys()) + } +``` + +### Server Integration and Middleware Setup + +Webhook endpoints automatically integrate with the jvspatial server middleware stack: + +```python path=null start=null +from jvspatial.api import Server +from jvspatial.api.auth.middleware import ( + AuthenticationMiddleware, + WebhookMiddleware, + HTTPSRedirectMiddleware +) + +# Server setup with webhook middleware +server = Server( + title="Webhook-Enabled Spatial API", + description="API with secure webhook processing", + version="1.0.0", + host="0.0.0.0", + port=8000 +) + +# Middleware stack (order matters) +server.add_middleware(HTTPSRedirectMiddleware) # Force HTTPS +server.add_middleware(WebhookMiddleware) # Webhook processing +server.add_middleware(AuthenticationMiddleware) # Authentication + +# Webhook endpoints are automatically registered +# Access at: POST https://your-domain.com/webhook/{route}/{auth_token} + +# Environment configuration for webhook security +# Set in .env file: +# WEBHOOK_HMAC_SECRET=your-secret-key +# WEBHOOK_HTTPS_REQUIRED=true +# WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour cache +# WEBHOOK_MAX_PAYLOAD_SIZE=1048576 # 1MB limit + +if __name__ == "__main__": + server.run(port=8000) +``` + +### Webhook Testing and Development + +```python path=null start=null +# Testing webhook handlers +import pytest +from fastapi.testclient import TestClient +from unittest.mock import MagicMock + +@pytest.fixture +def test_webhook_request(): + """Create mock webhook request for testing.""" + request = MagicMock() + request.state.raw_body = b'{"type": "test", "data": {"id": 123}}' + request.state.content_type = "application/json" + request.state.webhook_route = "test" + request.state.current_user = MagicMock(id="user_123") + request.state.hmac_verified = True + request.state.idempotency_key = "test-key-123" + return request + +@pytest.mark.asyncio +async def test_webhook_processing(test_webhook_request): + """Test webhook handler processing.""" + result = await generic_webhook_handler(test_webhook_request) + + assert result["status"] == "received" + assert result["route"] == "test" + assert result["user_id"] == "user_123" + +# Development webhook testing with ngrok or similar +# 1. Start your jvspatial server locally +# 2. Use ngrok to expose: ngrok http 8000 +# 3. Configure webhook URLs: https://abc123.ngrok.io/webhook/test/your-auth-token +# 4. Test with curl: +# curl -X POST https://abc123.ngrok.io/webhook/test/your-token \ +# -H "Content-Type: application/json" \ +# -d '{"test": "data"}' +``` + +### Best Practices for Webhook Implementation + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Always return 200 status for webhooks +@endpoint("/webhook/service/{auth_token}", webhook=True) +async def proper_webhook_handler(request: Request) -> Dict[str, Any]: + try: + # Process webhook + result = await process_webhook_data(request.state.raw_body) + return {"status": "success", "result": result} + except Exception as e: + # Log error but still return 200 + logger.error(f"Webhook processing failed: {e}") + return {"status": "received", "error": "logged"} + +# Good: Use route-based dispatch for multiple services +@endpoint("/webhook/{route}/{auth_token}", webhook=True) +async def multi_service_webhook(request: Request) -> Dict[str, Any]: + route = getattr(request.state, "webhook_route", "unknown") + + handlers = { + "stripe": process_stripe_webhook, + "github": process_github_webhook, + "slack": process_slack_webhook + } + + handler = handlers.get(route, process_generic_webhook) + return await handler(request) + +# Good: Validate authentication token format +@endpoint("/webhook/{service}/{auth_token}", webhook=True) +async def secure_webhook_handler(request: Request) -> Dict[str, Any]: + # Token validation is handled by middleware + current_user = get_current_user(request) + if not current_user: + return {"status": "error", "message": "Invalid authentication"} + + return {"status": "success", "user_verified": True} +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Returning non-200 status codes +@endpoint("/webhook/bad/{auth_token}", webhook=True) +async def bad_webhook_handler(request: Request) -> Dict[str, Any]: + try: + process_webhook(request.state.raw_body) + except Exception: + # Don't do this - breaks webhook retry logic + raise HTTPException(status_code=500, detail="Processing failed") + +# Bad: Not handling authentication properly +@endpoint("/webhook/unsecure/{auth_token}", webhook=True) +async def unsecure_webhook_handler(request: Request) -> Dict[str, Any]: + # Don't bypass authentication checks + # Always use get_current_user() or require auth in decorator + return {"status": "processed"} + +# Bad: Not using middleware-processed data +@endpoint("/webhook/manual/{auth_token}", webhook=True) +async def manual_webhook_handler(request: Request) -> Dict[str, Any]: + # Don't manually read request body - use request.state.raw_body + # raw_body = await request.body() # Wrong - middleware already processed + + # Use middleware-processed data instead + raw_body = request.state.raw_body # Correct + return {"status": "processed"} +``` + +--- + +## 🔗 Webhook System Integration + +JVspatial provides an advanced webhook system for handling external service integrations with enterprise-grade security, reliability, and developer experience. The webhook system supports modern decorators, automatic payload processing, HMAC verification, idempotency handling, and seamless authentication integration. + +### Quick Webhook Setup + +```python path=null start=null +from jvspatial.api import endpoint +from jvspatial.api import Server + +# Simple webhook handler +@endpoint("/webhook/payment", webhook=True) +async def payment_webhook(payload: dict, endpoint): + """Process payment webhooks with automatic JSON parsing.""" + payment_id = payload.get("payment_id") + amount = payload.get("amount") + + # Process payment logic here + print(f"Processing payment {payment_id}: ${amount}") + + return endpoint.response( + content={ + "status": "processed", + "message": f"Payment {payment_id} processed successfully" + } + ) + +# Server automatically detects and configures webhook middleware +server = Server(title="My Webhook API") +server.run() # Webhooks ready at /webhook/* paths +``` + +### Advanced Webhook Features + +```python path=null start=null +# Webhook with full security features +@endpoint( + "/webhook/stripe/{key}", + webhook=True, + path_key_auth=True, # API key in URL path + hmac_secret="stripe-webhook-secret", # HMAC signature verification + idempotency_ttl_hours=48, # Duplicate handling for 48h + permissions=["process_payments"] # RBAC permissions +) +async def secure_stripe_webhook(raw_body: bytes, content_type: str, endpoint): + """Stripe webhook with comprehensive security.""" + import json + + if content_type == "application/json": + payload = json.loads(raw_body.decode('utf-8')) + event_type = payload.get("type", "unknown") + + if event_type == "payment_intent.succeeded": + return endpoint.response( + content={ + "status": "processed", + "event_type": event_type, + "message": "Payment successful" + } + ) + + return endpoint.response(content={"status": "received"}) + +# Multi-service webhook dispatcher +@endpoint("/webhook/{service}", webhook=True) +async def multi_service_webhook(payload: dict, service: str, endpoint): + """Route webhooks based on service parameter.""" + handlers = { + "stripe": process_stripe_event, + "github": process_github_event, + "slack": process_slack_event + } + + handler = handlers.get(service, process_generic_event) + result = await handler(payload) + + return endpoint.response( + content={ + "status": "processed", + "service": service, + "result": result + } + ) + +# Helper functions +async def process_stripe_event(payload: dict) -> dict: + return {"stripe_event": payload.get("type", "unknown")} + +async def process_github_event(payload: dict) -> dict: + return {"github_action": payload.get("action", "unknown")} + +async def process_slack_event(payload: dict) -> dict: + return {"slack_event": payload.get("event", {}).get("type", "unknown")} + +async def process_generic_event(payload: dict) -> dict: + return {"processed": True, "keys": list(payload.keys())} +``` + +### Walker-Based Webhook Processing + +```python path=null start=null +# Future feature - Walker-based webhook processing +# @webhook_walker_endpoint("/webhook/location-update") +# class LocationUpdateWalker(Walker): +# """Process location updates through graph traversal.""" +# +# def __init__(self, payload: dict): +# super().__init__() +# self.payload = payload +# # Use the report() method to collect data during traversal +# +# @on_visit(Node) +# async def update_location_data(self, here: Node): +# locations = self.payload.get("locations", []) +# +# for location_data in locations: +# location_id = location_data.get("id") +# coordinates = location_data.get("coordinates") +# +# if location_id and coordinates: +# here.coordinates = coordinates +# await here.save() +# +# self.report({ +# "updated_location": { +# "id": location_id, +# "coordinates": coordinates +# } +# }) +``` + +### Environment Configuration + +Configure webhook behavior via environment variables: + +```env +# Global webhook settings +JVSPATIAL_WEBHOOK_HMAC_SECRET=your-global-hmac-secret +JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE=5242880 # 5MB +JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour +JVSPATIAL_WEBHOOK_HTTPS_REQUIRED=true + +# Service-specific secrets +JVSPATIAL_WEBHOOK_STRIPE_SECRET=whsec_stripe_secret_key +JVSPATIAL_WEBHOOK_GITHUB_SECRET=github_webhook_secret +``` + +### Testing Webhooks + +```bash +# Basic webhook test +curl -X POST "http://localhost:8000/webhook/payment" \ + -H "Content-Type: application/json" \ + -d '{"payment_id": "pay_123", "amount": 99.99}' + +# Webhook with path-based auth +curl -X POST "http://localhost:8000/webhook/stripe/key123:secret456" \ + -H "Content-Type: application/json" \ + -H "X-Signature: sha256=abc123..." \ + -d '{"type": "payment_intent.succeeded"}' + +# With idempotency key +curl -X POST "http://localhost:8000/webhook/payment" \ + -H "Content-Type: application/json" \ + -H "X-Idempotency-Key: unique-123" \ + -d '{"payment_id": "pay_124"}' +``` + +### Webhook Best Practices + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Always return 200 for webhook endpoints +@endpoint("/webhook/service", webhook=True) +async def proper_webhook(payload: dict, endpoint): + try: + result = await process_webhook_data(payload) + return endpoint.response(content={"status": "success", "result": result}) + except Exception as e: + # Log error but still return 200 + logger.error(f"Webhook processing failed: {e}") + return endpoint.response(content={"status": "received", "error": "logged"}) + +# Good: Use route-based dispatch for multiple services +@endpoint("/webhook/{service}", webhook=True) +async def multi_service_webhook(payload: dict, service: str, endpoint): + handlers = { + "stripe": process_stripe, + "github": process_github + } + + handler = handlers.get(service, process_generic) + return await handler(payload, endpoint) + +# Good: Validate webhook signatures when available +@endpoint("/webhook/secure", webhook=True, hmac_secret="webhook-secret") +async def secure_webhook(raw_body: bytes, endpoint): + # HMAC verification is automatic when secret is provided + return endpoint.response(content={"status": "verified"}) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Returning non-200 status codes +@endpoint("/webhook/bad", webhook=True) +async def bad_webhook(payload: dict, endpoint): + if payload.get("invalid"): + # Don't do this - breaks webhook retry logic + raise HTTPException(status_code=400, detail="Invalid payload") + +# Bad: Not handling errors gracefully +@endpoint("/webhook/risky", webhook=True) +async def risky_webhook(payload: dict, endpoint): + # Unhandled exceptions will return 500 - webhooks will retry + result = dangerous_operation(payload) # Might throw + return endpoint.response(content={"result": result}) + +# Bad: Bypassing security features +@endpoint("/webhook/insecure", webhook=True) +async def insecure_webhook(request: Request, endpoint): + # Don't manually read request body - use automatic payload injection + raw_body = await request.body() # Wrong - middleware already processed + return endpoint.response(content={"status": "received"}) +``` + +> **📖 For complete webhook documentation and advanced patterns:** [Webhook Architecture Guide](docs/md/webhook-architecture.md) | [Webhook Quickstart](docs/md/webhooks-quickstart.md) + +--- + +## 📁 File Storage Quickstart + +jvspatial includes a powerful file storage system with multi-backend support and URL proxy capabilities for secure file sharing. + +### Basic Setup + +```python +from jvspatial.api import Server + +server = Server( + title="File Upload API", + file_storage_enabled=True, + file_storage_provider="local", + file_storage_root=".files", + proxy_enabled=True +) + +if __name__ == "__main__": + server.run() +``` + +### Upload a File + +```bash +curl -X POST -F "file=@document.pdf" \ + http://localhost:8000/storage/upload +``` + +**Response:** +```json +{ + "success": true, + "file_path": "2025/01/05/document-abc123.pdf", + "file_size": 102400, + "content_type": "application/pdf" +} +``` + +### Create a Shareable Link + +```bash +curl -X POST http://localhost:8000/storage/proxy \ + -H "Content-Type: application/json" \ + -d '{ + "file_path": "2025/01/05/document-abc123.pdf", + "expires_in": 3600 + }' +``` + +**Response:** +```json +{ + "success": true, + "proxy_code": "a1b2c3d4", + "proxy_url": "http://localhost:8000/p/a1b2c3d4", + "expires_at": "2025-01-05T23:00:00Z" +} +``` + +### Access via Short URL + +```bash +curl http://localhost:8000/p/a1b2c3d4 +``` + +The file is served directly with appropriate headers. + +### Use in Walkers + +```python +from jvspatial.storage import get_file_interface +from jvspatial.core import Walker, on_visit, Node + +@server.walker("/process-upload") +class ProcessUpload(Walker): + file_path: str + + @on_visit(Node) + async def process(self, here: Node): + # Get file storage interface + storage = get_file_interface( + provider="local", + root_dir=".files" + ) + + # Read file content + content = await storage.get_file(self.file_path) + + # Process file content + self.report({ + "processed_file": { + "path": self.file_path, + "size": len(content), + "status": "success" + } + }) +``` + +### AWS S3 Configuration + +```python +server = Server( + title="S3 File API", + file_storage_enabled=True, + file_storage_provider="s3", + file_storage_s3_bucket="my-bucket", + file_storage_s3_region="us-east-1", + proxy_enabled=True +) +``` + +**Environment Variables:** +```env +JVSPATIAL_FILE_STORAGE_ENABLED=true +JVSPATIAL_FILE_STORAGE_PROVIDER=s3 +JVSPATIAL_FILE_STORAGE_S3_BUCKET=my-bucket +JVSPATIAL_FILE_STORAGE_S3_REGION=us-east-1 +AWS_ACCESS_KEY_ID=your-key-id +AWS_SECRET_ACCESS_KEY=your-secret-key +``` + +### Advanced Usage: Custom Upload Path + +```bash +curl -X POST -F "file=@image.jpg" \ + -F "custom_path=avatars/user123.jpg" \ + http://localhost:8000/storage/upload +``` + +### List Files + +```bash +curl http://localhost:8000/storage/files?prefix=2025/01/ +``` + +**Response:** +```json +{ + "success": true, + "files": [ + { + "path": "2025/01/05/document-abc123.pdf", + "size": 102400, + "modified": "2025-01-05T20:30:00Z" + }, + { + "path": "2025/01/05/image-def456.jpg", + "size": 51200, + "modified": "2025-01-05T21:15:00Z" + } + ] +} +``` + +### Security Features + +```python +server = Server( + title="Secure File API", + file_storage_enabled=True, + file_storage_provider="local", + file_storage_root=".files", + file_storage_max_size=10485760, # 10MB limit + file_storage_allowed_types=["image/jpeg", "image/png", "application/pdf"], + proxy_enabled=True, + proxy_default_ttl=3600 # 1 hour default expiration +) +``` + +### Best Practices + +**✅ Recommended Patterns:** + +```python +# Good: Use environment variables for configuration +from dotenv import load_dotenv +load_dotenv() + +server = Server( + title="Production File API", + file_storage_enabled=True, + # Provider configured via JVSPATIAL_FILE_STORAGE_PROVIDER + # Other settings loaded from environment +) + +# Good: Validate files before processing +@endpoint("/validate-upload") +class ValidateUpload(Walker): + file_path: str + + @on_visit(Node) + async def validate(self, here: Node): + storage = get_file_interface() + + # Check file exists + if not await storage.file_exists(self.file_path): + self.report({"error": "File not found"}) + return + + # Get file metadata + metadata = await storage.get_metadata(self.file_path) + + # Validate size + if metadata.get("size", 0) > 5242880: # 5MB + self.report({"error": "File too large"}) + return + + self.report({"status": "valid", "metadata": metadata}) + +# Good: Use proxy URLs for temporary access +async def create_temp_link(file_path: str, hours: int = 1): + """Create temporary shareable link.""" + response = await storage.create_proxy( + file_path=file_path, + expires_in=hours * 3600 + ) + return response["proxy_url"] +``` + +**❌ Avoided Patterns:** + +```python +# Bad: Hardcoding credentials +server = Server( + file_storage_s3_bucket="my-bucket", + file_storage_s3_access_key="AKIAIOSFODNN7EXAMPLE" # Don't do this! +) + +# Bad: No file validation +@endpoint("/unsafe-upload") +class UnsafeUpload(Walker): + file_path: str + + @on_visit(Node) + async def process(self, here: Node): + # No validation - could process malicious files + content = await storage.get_file(self.file_path) + # Direct processing without checks + +# Bad: Permanent public URLs without expiration +# Always use proxy URLs with expiration for security +``` + +See [File Storage Documentation](docs/md/file-storage-usage.md) for advanced usage and all configuration options. + +--- + +## 🔀 Router Decorators + +jvspatial provides a unified `@endpoint` decorator for all API endpoints: + +1. `@endpoint` - For public endpoints (both functions and Walker classes) +2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) +3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) +4. `@endpoint(..., auth=True, roles=["admin"])` - For admin-only endpoints + +```python +from jvspatial.api import endpoint + +# Function endpoint +@endpoint("/api/users", methods=["GET"]) +async def get_users() -> Dict[str, Any]: + users = await User.all() + return {"users": users} + +# Walker endpoint +@endpoint("/api/graph/traverse", methods=["POST"]) +class GraphTraversal(Walker): + pass + +# Authenticated function endpoint +@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) +async def get_admin_stats() -> Dict[str, Any]: + return {"stats": "admin only"} + +# Authenticated walker endpoint (uses same decorator) +@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) +class SecureProcessor(Walker): + pass + +# Admin-only endpoint +@endpoint("/api/admin/users", auth=True, roles=["admin"], methods=["GET"]) +async def manage_users() -> Dict[str, Any]: + return {"users": "admin access"} +``` + +**❌ DO NOT USE alternative decorators like:** +- `@route` +- `@server.route` +- `@server.walker` +- `@walker_endpoint` (deprecated - use `@endpoint` instead) +- `@auth_endpoint` (deprecated - use `@endpoint(..., auth=True)` instead) +- `@admin_endpoint` (deprecated - use `@endpoint(..., auth=True, roles=["admin"])` instead) + +These are internal or deprecated. + +## 📌 Consolidated Endpoint System + +jvspatial uses a **unified endpoint registration system** where all endpoints (walkers and functions) are registered through a single consolidated mechanism. This ensures clean, maintainable code without backward compatibility cruft. + +### Key Architecture + +All decorators follow the same registration path: + +1. **Decorator** → Attaches metadata to function/walker +2. **Server Detection** → Gets current server from context +3. **Registration** → Registers with `server.endpoint_router` +4. **Tracking** → Tracked by `server._endpoint_registry` + +### Important: Decorator Order + +Always create the server **before** decorating endpoints: + +```python +# ✅ CORRECT +server = Server(title="My API") + +@endpoint("/test") +class TestWalker(Walker): + pass + +# ✗ INCORRECT - endpoint will not be registered +@endpoint("/test") +class TestWalker(Walker): + pass + +server = Server(title="My API") # Created too late +``` + +### Default HTTP Methods + +- **Walkers**: Default to `["POST"]` +- **Functions**: Default to `["GET"]` + +Override with the `methods` parameter: +```python +@endpoint("/data", methods=["GET", "POST"]) +class DataWalker(Walker): + pass +``` + +### Available Response Methods + +Function endpoints can receive an `endpoint` parameter for response formatting: + +```python +@endpoint("/info") +async def get_info(endpoint): + # Use endpoint.success(), endpoint.error(), etc. + return endpoint.success(data={"info": "value"}) +``` + +Walkers automatically have `self.endpoint` available: + +```python +@endpoint("/process") +class ProcessWalker(Walker): + async def process(self): + self.response = self.endpoint.success(data={"result": "done"}) +``` + +**Response Methods:** + +```python +# Success responses +endpoint.success(data=result, message="Success") # 200 OK +endpoint.created(data=new_item, message="Created") # 201 Created +endpoint.no_content() # 204 No Content + +# Error responses +endpoint.bad_request(message="Invalid input") # 400 Bad Request +endpoint.unauthorized(message="Auth required") # 401 Unauthorized +endpoint.forbidden(message="Access denied") # 403 Forbidden +endpoint.not_found(message="Resource not found") # 404 Not Found +endpoint.conflict(message="Resource exists") # 409 Conflict +endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity + +# Flexible custom response +endpoint.response( + content={"custom": "data"}, + status_code=202, + headers={"X-Custom": "value"} +) + +# Generic error with custom status code +endpoint.error( + message="Custom error", + status_code=418, + details={"reason": "custom"} +) +``` + +### Querying Registered Endpoints + +```python +server = Server(title="My API") + +# ... register some endpoints ... + +# List all endpoints +all_endpoints = server.list_all_endpoints() +print(f"Walkers: {len(all_endpoints['walkers'])}") +print(f"Functions: {len(all_endpoints['functions'])}") + +# List just walkers +walkers = server.list_walker_endpoints() + +# List just functions +functions = server.list_function_endpoints() + +# Get registry stats +registry = server._endpoint_registry +counts = registry.count_endpoints() +print(f"Total: {counts['total']}") +``` + +### Unified Registration Benefits + +1. **Single Source of Truth**: All endpoints are registered through `EndpointRouter` +2. **Cleaner Code**: No backward compatibility cruft or deprecated methods +3. **Consistent API**: All decorators follow the same pattern +4. **Better Maintainability**: Future endpoint features only need to be added once +5. **Auto-Detection**: Decorators automatically detect walker vs function + +### Migration from Deprecated Patterns + +If you have code using removed patterns: + +**❌ Old Pattern (Removed)** +```python +# These no longer exist +server._custom_routes.append(...) +server._register_custom_routes(app) +server._setup_webhook_walker_endpoints() +``` + +**✅ New Pattern (Current)** +```python +# Use the decorators - they handle registration automatically +@endpoint("/my-route") +def my_handler(): + pass + +# Or register programmatically +server.register_walker_class(MyWalker, "/my-route", methods=["POST"]) +``` + +## 🌐 API Integration with FastAPI Server + +The **jvspatial API** provides seamless integration with FastAPI to expose your graph operations as REST endpoints. It supports flexible endpoint registration using decorators and automatic parameter model generation from Walker and function properties. + +### Server Setup and Configuration + +```python path=null start=null +from jvspatial.api import Server, ServerConfig, endpoint, walker_endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Node, Walker +from jvspatial.core.entities import on_visit + +# Basic server setup +server = Server( + title="My Spatial API", + description="Graph-based spatial data management API", + version="1.0.0", + host="0.0.0.0", + port=8000, + debug=True # Enable for development +) + +# Advanced server configuration +advanced_config = ServerConfig( + title="Production Spatial API", + description="Enterprise graph data API", + version="2.0.0", + host="0.0.0.0", + port=8080, + # Database configuration + db_type="mongodb", + db_connection_string="mongodb://localhost:27017", + db_database_name="spatial_db", + # CORS settings + cors_enabled=True, + cors_origins=["https://myapp.com", "http://localhost:3000"], + # API documentation + docs_url="/api/docs", + redoc_url="/api/redoc", + log_level="info" +) + +production_server = Server(config=advanced_config) +``` + +### @walker_endpoint Decorator for Walker Classes + +The `@walker_endpoint` decorator automatically exposes Walker classes as API endpoints: + +```python path=null start=null +from jvspatial.api import endpoint +from jvspatial.api.decorators import EndpointField +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit +from typing import List, Optional + +# Define your node types +class User(Node): + name: str = "" + email: str = "" + department: str = "" + active: bool = True + +class City(Node): + name: str = "" + population: int = 0 + state: str = "" + +# Walker with endpoint configuration using EndpointField +@endpoint("/api/users/process", methods=["POST"]) +class ProcessUser(Walker): + """Process user data with graph traversal.""" + + # Endpoint-exposed fields with configuration + user_name: str = EndpointField( + description="Name of the user to process", + examples=["John Doe", "Jane Smith"], + min_length=2, + max_length=100 + ) + + department: str = EndpointField( + default="general", + description="User department", + examples=["engineering", "marketing", "sales"] + ) + + include_inactive: bool = EndpointField( + default=False, + description="Whether to include inactive users in processing" + ) + + # Field excluded from API endpoint + internal_state: str = EndpointField( + default="processing", + exclude_endpoint=True # Won't appear in API schema + ) + + # Optional configuration field + max_connections: int = EndpointField( + default=10, + description="Maximum number of connections to traverse", + ge=1, + le=100 + ) + + @on_visit("User") + async def process_user(self, here: User): + """Process user nodes during traversal. + + Args: + here: The visited User node + """ + # Check if user matches criteria + if here.name == self.user_name: + self.report({ + "found_user": { + "id": here.id, + "name": here.name, + "email": here.email, + "department": here.department + } + }) + + # Find connected users in same department + colleagues = await here.nodes( + node=['User'], + department=self.department, + active=True if not self.include_inactive else None + ) + + self.report({ + "colleagues": [ + {"name": u.name, "email": u.email} + for u in colleagues[:self.max_connections] + ] + }) + + @on_visit("City") + async def process_city_connection(self, here: City): + """Process city connections if user has location data. + + Args: + here: The visited City node + """ + self.report({ + "city_info": { + "name": here.name, + "population": here.population, + "state": here.state + } + }) + +# Advanced walker with field grouping +@endpoint("/api/analytics/user-report", methods=["POST"]) +class UserAnalytics(Walker): + """Generate user analytics reports.""" + + # Grouped fields for better API organization + report_type: str = EndpointField( + description="Type of report to generate", + examples=["summary", "detailed", "connections"], + endpoint_group="report_config" + ) + + date_range: str = EndpointField( + default="30d", + description="Date range for report", + examples=["7d", "30d", "90d", "1y"], + endpoint_group="report_config" + ) + + include_inactive: bool = EndpointField( + default=False, + description="Include inactive users", + endpoint_group="filters" + ) + + departments: List[str] = EndpointField( + default_factory=list, + description="Departments to include (empty = all)", + examples=[["engineering", "product"], ["marketing"]], + endpoint_group="filters" + ) + + @on_visit("User") + async def analyze_user(self, here: User): + """Analyze user data for report. + + Args: + here: The visited User node + """ + # Filter by department if specified + if self.departments and here.department not in self.departments: + return + + # Filter by active status if needed + if not self.include_inactive and not here.active: + return + + # Report individual user analysis + self.report({ + "user_analyzed": { + "id": here.id, + "name": here.name, + "department": here.department, + "active": here.active + } + }) + } +``` + +### Enhanced Response Handling with endpoint.response() + +The `@walker_endpoint` and `@endpoint` decorators now automatically inject semantic response helpers to make crafting HTTP responses clean and flexible: + +**Walker Endpoints with self.endpoint:** + +```python path=null start=null +@endpoint("/api/users/profile", methods=["POST"]) +class UserProfileWalker(Walker): + """Walker demonstrating semantic response patterns.""" + + user_id: str = EndpointField(description="User ID to retrieve") + include_details: bool = EndpointField( + default=False, + description="Include detailed profile information" + ) + + @on_visit("User") + async def get_user_profile(self, here: User): + """Get user profile with semantic responses.""" + if here.id != self.user_id: + return # Continue traversal + + # User not found scenario + if not here.data: + return self.endpoint.not_found( + message="User profile not found", + details={"user_id": self.user_id} + ) + + # Authorization check + if here.private and not self.include_details: + return self.endpoint.forbidden( + message="Insufficient permissions", + details={"required_permission": "view_details"} + ) + + # Successful response + profile_data = { + "id": here.id, + "name": here.name, + "email": here.email + } + + if self.include_details: + profile_data["department"] = here.department + profile_data["created_at"] = here.created_at + + return self.endpoint.success( + data=profile_data, + message="User profile retrieved successfully" + ) + +@endpoint("/api/users/create", methods=["POST"]) +class CreateUserWalker(Walker): + """Walker for creating users with proper HTTP status codes.""" + + name: str = EndpointField(description="User name") + email: str = EndpointField(description="User email") + + @on_visit("Root") + async def create_user(self, here): + """Create a new user with validation.""" + + # Validation example + if "@" not in self.email: + return self.endpoint.unprocessable_entity( + message="Invalid email format", + details={"email": self.email} + ) + + # Check for conflicts + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await User.find({"context.email": self.email}) + existing_user = users[0] if users else None + if existing_user: + return self.endpoint.conflict( + message="User with this email already exists", + details={"email": self.email} + ) + + # Create user + user = await User.create( + name=self.name, + email=self.email + ) + + # Return 201 Created with location header + return self.endpoint.created( + data={ + "id": user.id, + "name": user.name, + "email": user.email + }, + message="User created successfully", + headers={"Location": f"/api/users/{user.id}"} + ) +``` + +**Function Endpoints with endpoint parameter:** + +```python path=null start=null +@endpoint("/api/health", methods=["GET"]) +async def health_check(endpoint) -> Any: + """Health check with semantic response.""" + return endpoint.success( + data={"status": "healthy", "version": "1.0.0"}, + message="Service is running normally" + ) + +@endpoint("/api/users/{user_id}/status", methods=["PUT"]) +async def update_user_status(user_id: str, status: str, endpoint) -> Any: + """Update user status with validation and error handling.""" + + # Validation + valid_statuses = ["active", "inactive", "suspended"] + if status not in valid_statuses: + return endpoint.bad_request( + message="Invalid status value", + details={"provided": status, "valid_options": valid_statuses} + ) + + # Find user + user = await User.get(user_id) + if not user: + return endpoint.not_found( + message="User not found", + details={"user_id": user_id} + ) + + # Update status + user.status = status + await user.save() + + return endpoint.success( + data={"id": user.id, "status": user.status}, + message=f"User status updated to {status}" + ) + +@endpoint("/api/export", methods=["GET"]) +async def export_data(format: str, endpoint) -> Any: + """Export data with custom response formatting.""" + + if format not in ["json", "csv", "xml"]: + return endpoint.error( + message="Unsupported export format", + status_code=406, # Not Acceptable + details={"format": format, "supported": ["json", "csv", "xml"]} + ) + + # Generate export data + export_data = { + "format": format, + "records": 1500, + "export_id": "exp_20250921", + "download_url": f"/downloads/export.{format}" + } + + # Use flexible response() method for custom headers + return endpoint.response( + content={ + "data": export_data, + "message": f"Export ready in {format} format" + }, + status_code=200, + headers={ + "X-Export-Format": format, + "X-Record-Count": "1500" + } + ) +``` + +**Available Response Methods:** + +```python path=null start=null +# Success responses +endpoint.success(data=result, message="Success") # 200 OK +endpoint.created(data=new_item, message="Created") # 201 Created +endpoint.no_content() # 204 No Content + +# Error responses +endpoint.bad_request(message="Invalid input") # 400 Bad Request +endpoint.unauthorized(message="Auth required") # 401 Unauthorized +endpoint.forbidden(message="Access denied") # 403 Forbidden +endpoint.not_found(message="Resource not found") # 404 Not Found +endpoint.conflict(message="Resource exists") # 409 Conflict +endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity + +# Flexible custom response +endpoint.response( + content={"custom": "data"}, + status_code=202, + headers={"X-Custom": "value"} +) + +# Generic error with custom status code +endpoint.error( + message="Custom error", + status_code=418, # I'm a teapot + details={"reason": "custom"} +) +``` + +### @endpoint Decorator for Regular Functions +The `@endpoint` decorator exposes regular functions as API endpoints: + +```python path=null start=null +from jvspatial.api import endpoint +from fastapi import HTTPException +from typing import Dict, Any, List + +# Simple function endpoint +@endpoint("/api/users/count", methods=["GET"]) +async def get_user_count() -> Dict[str, int]: + """Get total count of users in the system.""" + users = await User.all() + return {"total_users": len(users)} + +# Function endpoint with path parameters +@endpoint("/api/cities/{state}", methods=["GET"]) +async def get_cities_by_state(state: str) -> Dict[str, Any]: + """Get all cities in a specific state.""" + cities = await City.find({"context.state": state}) + + if not cities: + raise HTTPException( + status_code=404, + detail=f"No cities found in state: {state}" + ) + + return { + "state": state, + "cities": [ + { + "name": city.name, + "population": city.population + } for city in cities + ], + "total_count": len(cities) + } + +# Function endpoint with request body +@endpoint("/api/cities/search", methods=["POST"]) +async def search_cities(search_request: Dict[str, Any]) -> Dict[str, Any]: + """Search cities based on criteria.""" + # Extract search parameters + name_pattern = search_request.get("name_pattern") + min_population = search_request.get("min_population", 0) + state = search_request.get("state") + + # Build MongoDB-style query + query = {} + + if name_pattern: + query["context.name"] = {"$regex": name_pattern, "$options": "i"} + + if min_population > 0: + query["context.population"] = {"$gte": min_population} + + if state: + query["context.state"] = state + + # Execute search + cities = await City.find(query) + + return { + "search_criteria": search_request, + "results": [ + { + "id": city.id, + "name": city.name, + "population": city.population, + "state": city.state + } for city in cities + ], + "total_results": len(cities) + } + +# Function endpoint with pagination integration +@endpoint("/api/users/paginated", methods=["GET"]) +async def get_users_paginated( + page: int = 1, + page_size: int = 20, + department: Optional[str] = None, + active_only: bool = True +) -> Dict[str, Any]: + """Get paginated list of users with filtering.""" + from jvspatial.core.pager import ObjectPager + + # Build filters + filters = {} + if department: + filters["context.department"] = department + if active_only: + filters["context.active"] = True + + # Create pager + pager = ObjectPager( + User, + page_size=page_size, + filters=filters, + order_by="name", + order_direction="asc" + ) + + # Get page data + users = await pager.get_page(page) + pagination_info = pager.to_dict() + + return { + "users": [ + { + "id": user.id, + "name": user.name, + "email": user.email, + "department": user.department, + "active": user.active + } for user in users + ], + "pagination": pagination_info + } +``` + +### Server Method Registration + +You can also register endpoints directly on server instances: + +```python path=null start=null +# Using server instance decorators +@server.walker("/process-data", methods=["POST"]) +class DataProcessor(Walker): + """Process data using server instance registration.""" + + data_type: str = EndpointField( + description="Type of data to process", + examples=["user", "city", "connection"] + ) + + batch_size: int = EndpointField( + default=10, + description="Batch size for processing", + ge=1, + le=1000 + ) + + @on_visit("Node") + async def process_any_node(self, here: Node): + """Process any type of node. + + Args: + here: The visited Node + """ + # Use report() to collect processed node information + self.report({ + "processed_node": { + "id": here.id, + "type": here.__class__.__name__, + "processed_at": datetime.now().isoformat() + } + }) + +@server.route("/health-detailed", methods=["GET"]) +async def detailed_health_check() -> Dict[str, Any]: + """Detailed health check endpoint.""" + try: + # Test database connectivity + users_count = await User.count() + cities_count = await City.count() + + return { + "status": "healthy", + "database": "connected", + "statistics": { + "total_users": users_count, + "total_cities": cities_count + }, + "timestamp": datetime.now().isoformat() + } + except Exception as e: + raise HTTPException( + status_code=503, + detail=f"Health check failed: {str(e)}" + ) +``` + +### EndpointField Configuration Options + +The `EndpointField` provides extensive configuration for API parameters: + +```python path=null start=null +from jvspatial.api.decorators import EndpointField +from typing import List, Optional + +@endpoint("/api/advanced-example", methods=["POST"]) +class AdvancedEndpointExample(Walker): + """Demonstrate all EndpointField configuration options.""" + + # Basic field with validation + username: str = EndpointField( + description="User identifier", + examples=["john_doe", "jane_smith"], + min_length=3, + max_length=50, + pattern=r"^[a-zA-Z0-9_]+$" # Alphanumeric with underscores + ) + + # Numeric field with constraints + user_score: float = EndpointField( + description="User score rating", + examples=[85.5, 92.0, 78.3], + ge=0.0, # Greater than or equal to 0 + le=100.0 # Less than or equal to 100 + ) + + # Field with custom endpoint name + internal_id: str = EndpointField( + endpoint_name="user_id", # Will appear as 'user_id' in API + description="Internal user identifier" + ) + + # Optional field made required for endpoint + optional_field: Optional[str] = EndpointField( + default=None, + endpoint_required=True, # Required in API despite being Optional in Walker + description="Field that's optional in Walker but required in API" + ) + + # Required field made optional for endpoint + required_field: str = EndpointField( + endpoint_required=False, # Optional in API despite being required in Walker + description="Field that's required in Walker but optional in API" + ) + + # Grouped fields for organized API schema + config_timeout: int = EndpointField( + default=30, + description="Timeout in seconds", + endpoint_group="configuration", + ge=1, + le=300 + ) + + config_retries: int = EndpointField( + default=3, + description="Number of retries", + endpoint_group="configuration", + ge=0, + le=10 + ) + + # Hidden field (not shown in API docs) + debug_mode: bool = EndpointField( + default=False, + endpoint_hidden=True, # Hidden from OpenAPI documentation + description="Debug mode flag" + ) + + # Deprecated field + legacy_option: Optional[str] = EndpointField( + default=None, + endpoint_deprecated=True, # Marked as deprecated in API docs + description="Legacy option - use new_option instead" + ) + + # Field with additional constraints + email_domain: str = EndpointField( + description="Allowed email domain", + endpoint_constraints={ + "format": "hostname", # Additional OpenAPI constraint + "example": "company.com" + } + ) + + # List field with validation + tags: List[str] = EndpointField( + default_factory=list, + description="User tags", + examples=[["admin", "power-user"], ["guest"]] + ) + + @on_visit("User") + async def process_with_config(self, here: User): + """Process user with advanced configuration. + + Args: + here: The visited User node + """ + self.report({ + "processed_user": { + "username": self.username, + "score": self.user_score, + "config": { + "timeout": self.config_timeout, + "retries": self.config_retries + }, + "tags": self.tags, + "debug_enabled": self.debug_mode + } + }) +``` + +### Running the Server + +```python path=null start=null +# Development server +if __name__ == "__main__": + server.run( + host="0.0.0.0", + port=8000, + reload=True # Auto-reload for development + ) + +# Production server with custom configuration +async def run_production_server(): + """Run server asynchronously for production deployment.""" + await server.run_async( + host="0.0.0.0", + port=8080 + ) + +# Custom startup and shutdown hooks +@server.on_startup +async def startup_tasks(): + """Tasks to run on server startup.""" + print("🚀 Server starting up...") + # Initialize data, warm up caches, etc. + +@server.on_shutdown +async def shutdown_tasks(): + """Tasks to run on server shutdown.""" + print("🛑 Server shutting down...") + # Cleanup resources, save state, etc. + +# Access the underlying FastAPI app if needed +fastapi_app = server.get_app() + +# Custom middleware +@server.middleware("http") +async def log_requests(request, call_next): + """Log all API requests.""" + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print(f"{request.method} {request.url} - {response.status_code} ({process_time:.2f}s)") + return response +``` + +### Router Decorators + +jvspatial provides four standard router decorators for API endpoints. These are the ONLY decorators that should be used for routing: + +1. `@endpoint` - For public endpoints (both functions and Walker classes) +2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) +3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) + +```python +# Function endpoint +@endpoint("/api/users", methods=["GET"]) +async def get_users() -> Dict[str, Any]: + users = await User.all() + return {"users": users} + +# Walker endpoint +@endpoint("/api/graph/traverse", methods=["POST"]) +class GraphTraversal(Walker): + pass + +# Authenticated function endpoint +@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) +async def get_admin_stats() -> Dict[str, Any]: + return {"stats": "admin only"} + +# Authenticated walker endpoint (uses same decorator) +@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) +class SecureProcessor(Walker): + pass +``` + +DO NOT use alternative decorators like `@route`, `@server.route`, or `@server.walker`. These are internal or deprecated. + +### API Usage Examples + +Once your server is running, endpoints are automatically available: + +```bash +# Walker endpoint - POST request with parameters +curl -X POST "http://localhost:8000/api/users/process" \ + -H "Content-Type: application/json" \ + -d '{ + "user_name": "John Doe", + "department": "engineering", + "include_inactive": false, + "max_connections": 5, + "start_node": "n:Root:root" + }' + +# Function endpoint - GET request +curl "http://localhost:8000/api/users/count" + +# Function endpoint with path parameters +curl "http://localhost:8000/api/cities/CA" + +# Function endpoint with query parameters +curl "http://localhost:8000/api/users/paginated?page=1&page_size=10&department=engineering" + +# API documentation is automatically available +# http://localhost:8000/docs (Swagger UI) +# http://localhost:8000/redoc (ReDoc) +``` + +### Best Practices + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Use descriptive endpoint paths +@endpoint("/api/users/analyze-connections", methods=["POST"]) +class AnalyzeUserConnections(Walker): + pass + +# Good: Provide comprehensive field documentation +field_name: str = EndpointField( + description="Clear description of what this field does", + examples=["example1", "example2"], + min_length=1, + max_length=100 +) + +# Good: Use appropriate HTTP methods +@endpoint("/api/users", methods=["GET"]) # Retrieve data +@endpoint("/api/users", methods=["POST"]) # Create data +@endpoint("/api/process", methods=["POST"]) # Process/execute + +# Good: Group related fields +config_field: str = EndpointField( + endpoint_group="configuration", + description="Configuration parameter" +) + +# Good: Handle errors appropriately in functions +@endpoint("/api/data") +async def get_data(): + try: + data = await fetch_data() + return {"data": data} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Vague endpoint paths +@endpoint("/process") # Too generic +@endpoint("/api/thing") # Unclear purpose + +# Bad: Missing field documentation +field_name: str = EndpointField() # No description or examples + +# Bad: Exposing internal fields +internal_state: str = EndpointField() # Should use exclude_endpoint=True + +# Bad: Not handling errors in function endpoints +@endpoint("/api/data") +async def get_data(): + data = await risky_operation() # No error handling + return data # Could fail with 500 error + +# Bad: Using wrong HTTP methods +@endpoint("/api/users/delete", methods=["GET"]) # Should be DELETE +@endpoint("/api/data/get", methods=["POST"]) # Should be GET for retrieval +``` + +--- + +## 🏗️ Library Architecture Concepts + +**jvspatial** is an asynchronous, object-spatial Python library designed for building robust persistence and business logic application layers. Inspired by Jaseci's object-spatial paradigm and leveraging Python's async capabilities, jvspatial empowers developers to model complex relationships, traverse object graphs, and implement agent-based architectures that scale with modern cloud-native concurrency requirements. Key capabilities: + +- Typed node/edge modeling via Pydantic +- Precise control over graph traversal +- Multi-backend persistence (JSON, MongoDB, etc.) +- Integrated REST API endpoints +- Async/await architecture + +### Core Entities + +1. **Object** - Base class for all entities with unified query interface; used to store non-graph data +2. **Node** - Extends Object, represents graph nodes with spatial/contextual data; used only as part of a graph +3. **Edge** - Represents relationships between nodes on a graph +4. **Walker** - Implements graph traversal and pathfinding algorithms +5. **GraphContext** - Low-level database interface (use sparingly) + +Once a graph is established (nodes and edges are connected in a meaningful way), a walker may be spawned on the root node or anywhere on the graph. The visit method enacts the walker's traversal using a starting point or a list of nodes on the walker's visit queue. + +As the walker traverses, it may conditionally trigger methods depending on its position on the graph. This is accomplished by the @on_visit annotation on the walker class. Similarly, as the walker traverses over nodes and edges, these entities may conditionally trigger their methods based on the walker's visitation; also accomplished through the @on_visit annotation on the node/edge class. + +**Execution Order**: When a walker visits a node/edge: +1. **Walker hooks** (methods decorated with `@on_visit` on the walker class) execute first +2. **Node/Edge hooks** (methods decorated with `@on_visit` on the node/edge class) are automatically executed after + +Node/Edge hooks are automatically invoked by the walker - no explicit call is needed. The walker binds the hook to the node/edge instance and calls it with the walker as a parameter. + +### Walker Traversal Pattern + +The **recommended approach** for walker traversal is to use the `nodes()` method to get connected nodes for continued traversal. + +#### Naming Convention for @on_visit Methods + +**IMPORTANT**: When writing `@on_visit` decorated methods, use the following naming convention: + +- **`here`**: Parameter name for the visited node/edge (the current location) +- **`visitor`**: Parameter name for the visiting walker (when accessing walker from node context) + +```python path=null start=null +# ✅ RECOMMENDED: Use 'here' for the visited node +@on_visit("User") +async def process_user(self, here: Node): + """Process user node - 'here' represents the current User being visited. + + Args: + here: The visited User node + """ + print(f"Currently visiting user: {here.name}") + connected_users = await here.nodes(node=['User']) + await self.visit(connected_users) + +# ✅ RECOMMENDED: Use 'visitor' when walker is passed to node methods +def node_method_example(self, visitor: Walker): + """Node method that receives the visiting walker. + + Args: + visitor: The walker currently visiting this node + """ + print(f"Being visited by walker: {visitor.__class__.__name__}") + +# ❌ AVOID: Generic parameter names +@on_visit("User") +async def process_user(self, node: Node): # Less clear + pass + +@on_visit("User") +async def process_user(self, n: Node): # Too abbreviated + pass +``` + +**Why this convention?** +- **`here`** clearly indicates the current location in graph traversal +- **`visitor`** clearly indicates the active walker performing the traversal +- Consistent with spatial/navigational metaphors used throughout jvspatial +- Makes code more readable and self-documenting +- Aligns with the library's entity-centric philosophy + +```python +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit, on_exit + +class DataCollector(Walker): + def __init__(self): + super().__init__() + self.collected_data = [] + self.processed_count = 0 + + @on_visit("User") + async def collect_user_data(self, here: Node): + """Called when walker visits a User node. + + Args: + here: The visited User node + """ + self.collected_data.append(here.name) + + # RECOMMENDED: Use nodes() method to get connected nodes + # Default direction="out" follows outgoing edges (forward traversal) + next_nodes = await here.nodes() + await self.visit(next_nodes) + + # Example with semantic filtering - filter connected Users by department + engineering_users = await here.nodes( + node=['User'], # Only User nodes + department="engineering", # Simple kwargs filtering + active=True # Multiple simple filters + ) + await self.visit(engineering_users) + + @on_visit("City") + async def process_city(self, here: Node): + """Process city nodes with filtering and control flow. + + Args: + here: The visited City node + """ + self.processed_count += 1 + + # Skip processing for certain conditions + if here.population < 10000: + print(f"Skipping small city: {here.name}") + self.skip() # Skip to next node in queue + return # This line won't be reached + + # Disengage if we've processed enough + if self.processed_count >= 10: + print("Processed enough cities, disengaging...") + await self.disengage() # Permanently halt and remove from graph + return + + # Continue with normal processing + large_cities = await here.nodes( + node=[{'City': {"context.population": {"$gte": 500_000}}}], + direction="out" # Explicit direction for clarity + ) + await self.visit(large_cities) + + # Example: Mixed filtering approach - dict filters + kwargs + nearby_cities = await here.nodes( + node=[{'City': {"context.region": here.region}}], # Complex filter + state="NY", # Simple kwargs filter + active=True # Additional simple filter + ) + await self.visit(nearby_cities) + + @on_exit + async def cleanup_and_report(self): + """Called when walker completes or disengages.""" + print(f"Walker finished! Collected {len(self.collected_data)} items") + print(f"Processed {self.processed_count} cities") + # Perform cleanup, save results, send notifications, etc. +``` + +### Walker Control Flow Methods + +#### `skip()` - Skip Current Node Processing +The `skip()` method allows you to immediately halt processing of the current node and proceed to the next node in the queue, similar to `continue` in a loop: + +```python +class ConditionalWalker(Walker): + @on_visit("Product") + async def process_product(self, here: Node): + """Process product nodes with conditional skipping. + + Args: + here: The visited Product node + """ + # Skip discontinued products + if here.status == "discontinued": + self.skip() # Jump to next node in queue + # Code below won't execute + + # Skip products outside price range + if not (10 <= here.price <= 1000): + print(f"Skipping {here.name} - price out of range") + self.skip() + + # Normal processing for valid products + print(f"Processing product: {here.name}") + connected_products = await here.nodes(node=['Product']) + await self.visit(connected_products) +``` + +#### `disengage()` - Permanently Halt and Remove Walker from Graph +The `disengage()` method permanently halts the walker and removes it from the graph. Once disengaged, the walker **cannot be resumed** and is considered finished: + +```python +class CompletionWalker(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.max_items = 100 + self.critical_error = False + + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with completion tracking. + + Args: + here: The visited Document node + """ + try: + # Process document + await self.process_item(here) + self.processed_count += 1 + + # Disengage when reaching target or on critical error + if self.processed_count >= self.max_items: + print(f"Target reached: {self.processed_count} items processed") + await self.disengage() # Permanently finish + return + + if self.critical_error: + print("Critical error encountered, disengaging walker") + await self.disengage() # Permanently halt due to error + return + + # Continue to next documents + next_docs = await here.nodes(node=['Document']) + await self.visit(next_docs) + + except CriticalError as e: + print(f"Critical error: {e}") + self.critical_error = True + await self.disengage() # Permanently halt on critical error + + @on_exit + async def final_cleanup(self): + """Called when walker disengages - perform final cleanup.""" + print(f"Walker disengaged. Final count: {self.processed_count}") + # Perform final cleanup, save state, notify completion + await self.save_final_results() + + async def process_item(self, node): + """Process individual item.""" + # Simulate processing that might fail + if node.status == "corrupted": + raise CriticalError("Corrupted data detected") + await asyncio.sleep(0.01) + + async def save_final_results(self): + """Save final processing results.""" + print("💾 Saving final results...") + +# Usage - disengage() creates permanent termination +walker = CompletionWalker() +root = await Root.get(None) + +# Start and run to completion (or error) +walker = await walker.spawn(root) +print(f"Walker finished. Status: {'disengaged' if walker.paused else 'completed'}") + +# NOTE: Once disengaged, walker cannot be resumed +# walker.resume() would not work - walker is permanently finished + +# For pausable/resumable patterns, use different approaches: +# - Save walker state and create new walker instances +# - Use external queue/state management +# - Implement custom pause/resume logic in @on_visit methods +``` + +#### `pause()` and `resume()` - Temporary Walker Suspension +Walkers can be paused during traversal and resumed later, preserving their queue and state. Unlike `disengage()`, paused walkers can be resumed: + +```python +class BatchProcessor(Walker): + def __init__(self): + super().__init__() + self.processed_count = 0 + self.batch_size = 50 + self.total_batches = 0 + + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with batch pausing. + + Args: + here: The visited Document node + """ + # Process document + await self.heavy_processing(here) + self.processed_count += 1 + + # Pause after processing a batch + if self.processed_count % self.batch_size == 0: + self.total_batches += 1 + print(f"Batch {self.total_batches} complete ({self.processed_count} items)") + print("Pausing for rate limiting...") + + # Clean pause using pause() method + self.pause(f"Batch {self.total_batches} processing pause") + + # Continue to next documents + next_docs = await here.nodes(node=['Document'], status="active") + await self.visit(next_docs) + + async def heavy_processing(self, node): + """Simulate expensive processing.""" + await asyncio.sleep(0.1) # Simulate API calls, file I/O, etc. + + @on_exit + async def report_completion(self): + """Called when walker completes or is paused.""" + if self.paused: + print(f"Walker paused after processing {self.processed_count} items") + else: + print(f"Walker completed! Total processed: {self.processed_count}") + +# Usage - pause and resume cycle +walker = BatchProcessor() +root = await Root.get(None) + +# Start processing - will pause after first batch +walker = await walker.spawn(root) +print(f"Walker state: {'paused' if walker.paused else 'completed'}") +print(f"Queue remaining: {len(walker.queue)} items") + +# Simulate processing delay (rate limiting, API cooldown, etc.) +print("\n⏳ Waiting for rate limit cooldown...") +await asyncio.sleep(2.0) + +# Resume processing - will continue from where it left off +print("\n▶️ Resuming processing...") +walker = await walker.resume() +print(f"Walker state after resume: {'paused' if walker.paused else 'completed'}") + +# Can resume multiple times if walker gets paused again +while walker.paused and walker.queue: + print(f"\n🔄 Walker paused again, {len(walker.queue)} items remaining") + await asyncio.sleep(1.0) # Brief pause + walker = await walker.resume() + +print("\n✅ All processing complete!") +``` + +#### Advanced Pause/Resume Patterns + +```python +class SmartProcessor(Walker): + def __init__(self): + super().__init__() + self.api_calls = 0 + self.max_api_calls = 100 + self.error_count = 0 + self.max_errors = 5 + + @on_visit("DataNode") + async def process_data(self, here: Node): + """Process data nodes with smart rate limiting. + + Args: + here: The visited DataNode + """ + try: + # Check rate limits + if self.api_calls >= self.max_api_calls: + print(f"Rate limit reached ({self.api_calls} calls), pausing...") + self.api_calls = 0 # Reset counter + self.pause("Rate limit reached") + + # Check error threshold + if self.error_count >= self.max_errors: + print(f"Too many errors ({self.error_count}), pausing for investigation") + self.pause(f"Error threshold reached: {self.error_count} errors") + + # Process node + result = await self.call_external_api(here) + self.api_calls += 1 + + # Continue traversal based on result + if result.should_continue: + related_nodes = await here.nodes( + node=['DataNode'], + status="pending", + priority={"$gte": result.priority_threshold} + ) + await self.visit(related_nodes) + + except ApiError as e: + self.error_count += 1 + print(f"API error #{self.error_count}: {e}") + # Continue processing - don't pause on single errors + + except CriticalError as e: + print(f"Critical error: {e}") + self.pause(f"Critical error: {e}") + + async def call_external_api(self, node): + """Simulate external API call that might fail.""" + await asyncio.sleep(0.05) # Simulate API latency + # Simulate occasional failures + if random.random() < 0.1: # 10% failure rate + raise ApiError("Temporary API failure") + return APIResult(should_continue=True, priority_threshold=5) + + def pause_for_maintenance(self): + """Manually pause walker for maintenance.""" + print("🔧 Pausing for scheduled maintenance...") + self.paused = True + + @on_exit + async def maintenance_check(self): + """Check if maintenance is needed when paused.""" + if self.paused: + print("Walker paused - performing maintenance checks...") + # Reset error counters, clear caches, etc. + self.error_count = 0 + print("Maintenance complete - ready to resume") + +# Usage with external control +walker = SmartProcessor() +root = await Root.get(None) + +# Start processing +print("🚀 Starting smart processing...") +walker = await walker.spawn(root) + +# External monitoring and control +while walker.paused and walker.queue: + print(f"⏸️ Walker paused - {len(walker.queue)} items remaining") + + # Simulate external decision making + if should_resume_processing(): # Your logic here + print("🔄 Conditions met, resuming...") + walker = await walker.resume() + else: + print("⏳ Waiting for conditions to improve...") + await asyncio.sleep(5.0) + +# Helper classes for example +class ApiError(Exception): pass +class CriticalError(Exception): pass +class APIResult: + def __init__(self, should_continue, priority_threshold): + self.should_continue = should_continue + self.priority_threshold = priority_threshold + +def should_resume_processing(): + """External logic to determine if processing should resume.""" + return True # Simplified for example +``` + +#### `@on_exit` - Cleanup and Finalization +The `@on_exit` decorator marks methods to execute when the walker completes traversal or disengages: + +```python +class AnalyticsWalker(Walker): + def __init__(self): + super().__init__() + self.start_time = time.time() + self.nodes_visited = 0 + self.errors_encountered = 0 + + @on_visit("User") + async def analyze_user(self, here: Node): + """Analyze user behavior and traverse to related users. + + Args: + here: The visited User node + """ + try: + self.nodes_visited += 1 + # Perform analysis + await self.analyze_user_behavior(here) + + # Continue traversal + related_users = await here.nodes(node=['User']) + await self.visit(related_users) + except Exception as e: + self.errors_encountered += 1 + print(f"Error processing user {here.id}: {e}") + + @on_exit + async def generate_report(self): + """Generate analytics report when traversal completes.""" + duration = time.time() - self.start_time + print("\n📊 Analytics Report") + print(f"Duration: {duration:.2f} seconds") + print(f"Nodes visited: {self.nodes_visited}") + print(f"Errors: {self.errors_encountered}") + print(f"Success rate: {(1 - self.errors_encountered/max(self.nodes_visited, 1))*100:.1f}%") + + @on_exit + def save_results(self): + """Save results to database (sync version).""" + # Save analytics data + print("💾 Saving results to database...") + + @on_exit + async def send_notifications(self): + """Send completion notifications (async version).""" + # Send email/slack notifications + print("📧 Sending completion notifications...") + + async def analyze_user_behavior(self, user): + """Simulate user behavior analysis.""" + await asyncio.sleep(0.01) # Simulate work +``` + +### Walker Reporting System + +Walkers feature a simplified reporting system that allows you to collect and aggregate any data during traversal. The reporting system eliminates complex nested structures and provides direct access to collected data. + +#### Basic Reporting + +```python path=null start=null +from jvspatial.core import Walker, on_visit, on_exit + +class DataCollectionWalker(Walker): + """Walker demonstrating the simple reporting system.""" + + def __init__(self): + super().__init__() + self.processed_count = 0 + + @on_visit('User') + async def collect_user_data(self, here: Node): + """Collect user data using the report system.""" + # Report any data - strings, dicts, numbers, lists, etc. + self.report({ + "user_processed": { + "id": here.id, + "name": here.name, + "department": here.department, + "timestamp": time.time() + } + }) + + # Report simple values + self.report(f"Processed user: {here.name}") + + # Report lists + if hasattr(here, 'skills'): + self.report(["skills", here.skills]) + + self.processed_count += 1 + + @on_exit + async def generate_summary(self): + """Generate final summary in the report.""" + report_items = await self.get_report() + + self.report({ + "summary": { + "total_items_collected": len(report_items), + "users_processed": self.processed_count, + "collection_complete": True + } + }) + +# Usage +walker = DataCollectionWalker() +result_walker = await walker.spawn() # spawn() returns the walker instance + +# Access collected data directly as a simple list +report = await result_walker.get_report() +print(f"Total items collected: {len(report)}") + +# Iterate through all collected data +for item in report: + if isinstance(item, dict) and "user_processed" in item: + user_data = item["user_processed"] + print(f"User: {user_data['name']} from {user_data['department']}") + elif isinstance(item, str): + print(f"Log: {item}") +``` + +#### Advanced Reporting Patterns + +```python path=null start=null +class AnalyticsWalker(Walker): + """Walker with advanced reporting for analytics.""" + + def __init__(self): + super().__init__() + self.department_counts = {} + self.error_count = 0 + + @on_visit('User') + async def analyze_user(self, here: Node): + """Analyze user and report findings.""" + try: + # Track department statistics + dept = here.department or "unknown" + self.department_counts[dept] = self.department_counts.get(dept, 0) + 1 + + # Report individual analysis + analysis = await self.perform_user_analysis(here) + self.report({ + "user_analysis": { + "user_id": here.id, + "department": dept, + "performance_score": analysis.get("score", 0), + "risk_level": analysis.get("risk", "low"), + "recommendations": analysis.get("recommendations", []) + } + }) + + except Exception as e: + self.error_count += 1 + self.report({ + "error": { + "user_id": here.id, + "error_message": str(e), + "error_type": type(e).__name__ + } + }) + + @on_exit + async def generate_analytics_report(self): + """Generate comprehensive analytics.""" + all_data = await self.get_report() + + # Analyze collected data + user_analyses = [item for item in all_data + if isinstance(item, dict) and "user_analysis" in item] + errors = [item for item in all_data + if isinstance(item, dict) and "error" in item] + + # Calculate metrics + avg_score = sum(ua["user_analysis"]["performance_score"] + for ua in user_analyses) / len(user_analyses) if user_analyses else 0 + + high_risk_users = [ua for ua in user_analyses + if ua["user_analysis"]["risk_level"] == "high"] + + # Report final analytics + self.report({ + "final_analytics": { + "total_users_analyzed": len(user_analyses), + "average_performance_score": round(avg_score, 2), + "department_breakdown": self.department_counts, + "high_risk_users_count": len(high_risk_users), + "error_rate": self.error_count / max(len(user_analyses), 1), + "processing_summary": { + "success": len(user_analyses), + "errors": self.error_count, + "total_items_in_report": len(all_data) + } + } + }) + + async def perform_user_analysis(self, user): + """Simulate user analysis.""" + import random + return { + "score": random.randint(1, 100), + "risk": random.choice(["low", "medium", "high"]), + "recommendations": ["Update profile", "Complete training"] + } +``` + +### Walker Event System + +Walkers can communicate with each other during traversal using an event system. This enables real-time coordination, data sharing, and complex multi-walker workflows. + +#### Basic Event Communication + +```python path=null start=null +from jvspatial.core.events import on_emit + +class AlertWalker(Walker): + """Walker that emits alerts when finding critical issues.""" + + @on_visit('SystemNode') + async def check_system_health(self, here: Node): + """Check system health and emit alerts.""" + if hasattr(here, 'cpu_usage') and here.cpu_usage > 90: + # Emit event to other walkers + await self.emit("high_cpu_alert", { + "node_id": here.id, + "cpu_usage": here.cpu_usage, + "severity": "critical", + "walker_id": self.id + }) + + self.report({"alert_sent": f"High CPU on {here.id}"}) + +class MonitoringWalker(Walker): + """Walker that receives and processes alerts from other walkers.""" + + def __init__(self): + super().__init__() + self.alerts_received = 0 + + @on_emit("high_cpu_alert") + async def handle_cpu_alert(self, event_data): + """Handle high CPU alerts from AlertWalker.""" + self.alerts_received += 1 + + self.report({ + "alert_processed": { + "from_walker": event_data.get("walker_id"), + "node_id": event_data.get("node_id"), + "cpu_usage": event_data.get("cpu_usage"), + "action_taken": "Notification sent to admin", + "handler_id": self.id + } + }) + + # Take action based on alert + await self.send_notification(event_data) + + @on_visit('SystemNode') + async def log_system_visit(self, here: Node): + """Log system node visits.""" + self.report({"system_visited": here.id}) + + async def send_notification(self, alert_data): + """Send notification to administrators.""" + print(f"🚨 ALERT: High CPU {alert_data['cpu_usage']}% on {alert_data['node_id']}") + +# Run multiple walkers concurrently +import asyncio + +alert_walker = AlertWalker() +monitoring_walker = MonitoringWalker() + +# Start both walkers concurrently +tasks = [ + alert_walker.spawn(), + monitoring_walker.spawn() +] + +walkers = await asyncio.gather(*tasks) + +# Check reports from both walkers +alert_report = await alert_walker.get_report() +monitoring_report = await monitoring_walker.get_report() + +print(f"Alerts sent: {len([r for r in alert_report if 'alert_sent' in str(r)])}") +print(f"Alerts processed: {monitoring_walker.alerts_received}") +``` + +#### Advanced Event Patterns + +```python path=null start=null +class DataProcessorWalker(Walker): + """Walker that processes data and emits completion events.""" + + def __init__(self, batch_id: str): + super().__init__() + self.batch_id = batch_id + self.processed_items = 0 + + @on_visit('DataNode') + async def process_data(self, here: Node): + """Process data nodes.""" + # Simulate processing + await asyncio.sleep(0.01) + self.processed_items += 1 + + self.report({"data_processed": here.id}) + + # Emit progress event every 10 items + if self.processed_items % 10 == 0: + await self.emit("batch_progress", { + "batch_id": self.batch_id, + "processed_count": self.processed_items, + "processor_id": self.id + }) + + @on_exit + async def emit_completion(self): + """Emit batch completion event.""" + await self.emit("batch_complete", { + "batch_id": self.batch_id, + "total_processed": self.processed_items, + "processor_id": self.id + }) + + self.report({"batch_completed": self.batch_id}) + +class BatchCoordinator(Walker): + """Walker that coordinates multiple batch processors.""" + + def __init__(self): + super().__init__() + self.batch_progress = {} + self.completed_batches = [] + + @on_emit("batch_progress") + async def track_progress(self, event_data): + """Track progress from batch processors.""" + batch_id = event_data.get("batch_id") + processed_count = event_data.get("processed_count") + + self.batch_progress[batch_id] = processed_count + + self.report({ + "progress_update": { + "batch_id": batch_id, + "items_processed": processed_count, + "coordinator_id": self.id + } + }) + + @on_emit("batch_complete") + async def handle_completion(self, event_data): + """Handle batch completion events.""" + batch_id = event_data.get("batch_id") + total_processed = event_data.get("total_processed") + + self.completed_batches.append(batch_id) + + self.report({ + "batch_completed": { + "batch_id": batch_id, + "total_items": total_processed, + "completed_batches_count": len(self.completed_batches) + } + }) + + # Check if all batches are complete + if len(self.completed_batches) >= 3: # Expecting 3 batches + await self.emit("all_batches_complete", { + "total_batches": len(self.completed_batches), + "coordinator_id": self.id + }) + + @on_emit("all_batches_complete") + async def finalize_processing(self, event_data): + """Finalize when all processing is complete.""" + self.report({ + "processing_finalized": { + "total_batches_completed": event_data.get("total_batches"), + "finalization_time": time.time() + } + }) + +# Example: Run coordinated batch processing +coordinator = BatchCoordinator() +processors = [ + DataProcessorWalker("batch_1"), + DataProcessorWalker("batch_2"), + DataProcessorWalker("batch_3") +] + +# Start all walkers +all_walkers = [coordinator] + processors +tasks = [walker.spawn() for walker in all_walkers] +results = await asyncio.gather(*tasks) + +# Check final reports +for walker in all_walkers: + report = await walker.get_report() + print(f"Walker {walker.id}: {len(report)} items in report") +``` + +#### Key Reporting & Event Features + +**Reporting System:** +- `walker.report(any_data)` - Add any data to walker's report +- `await walker.get_report()` - Get simple list of all reported items (async) +- No complex nested structures - direct access to your data +- Support for any data type (strings, dicts, lists, numbers, etc.) + +**Event System:** +- `await walker.emit(event_name, payload)` - Send events to other walkers +- `@on_emit(event_name)` - Handle specific events +- Multiple walkers can receive the same event +- Events enable real-time coordination between concurrent walkers +- Both Walkers and Nodes can use `@on_emit` decorators + +**Best Practices:** +- Use `self.report()` to add data, never return values from decorated methods +- Access reports after traversal: `report = await walker.get_report()` +- Use events for walker-to-walker communication during traversal +- Filter reported data by checking item structure/content +- Leverage `@on_exit` hooks for final summaries and cleanup + +### Walker Trail Tracking + +Walkers include built-in **trail tracking** capabilities to monitor and record the complete path taken during graph traversal. This is invaluable for debugging, analytics, audit trails, and optimizing traversal strategies. + +#### Basic Trail Tracking + +```python path=null start=null +from jvspatial.core import Walker, on_visit + +class TrailTrackingWalker(Walker): + def __init__(self): + super().__init__() + # Enable trail tracking with memory management + self.trail_enabled = True + self.max_trail_length = 100 # Keep last 100 steps (0 = unlimited) + + @on_visit('User') + async def process_user_with_trail(self, here: Node): + """Process user nodes while tracking the traversal trail. + + Args: + here: The visited User node + """ + print(f"Processing user: {here.name}") + + # Access current trail information + current_trail = self.get_trail() # List of node IDs + trail_length = self.get_trail_length() # Current trail size + recent_steps = self.get_recent_trail(count=3) # Last 3 steps + + print(f"Trail length: {trail_length}, Recent: {recent_steps}") + + # Avoid revisiting recently visited nodes + if here.id in recent_steps[:-1]: # Exclude current node + print(f"Recently visited {here.name}, skipping deeper traversal") + self.skip() + + # Continue normal traversal + colleagues = await here.nodes( + node=['User'], + department=here.department, + active=True + ) + await self.visit(colleagues) + + @on_exit + async def generate_trail_report(self): + """Generate comprehensive trail analysis report.""" + # Get trail with actual node objects (database lookups) + trail_nodes = await self.get_trail_nodes() + + # Get complete path with connecting edges + trail_path = await self.get_trail_path() + + # Generate detailed report using report() method + trail_report = { + 'summary': { + 'total_steps': self.get_trail_length(), + 'unique_nodes': len(set(self.get_trail())), + 'efficiency_ratio': len(set(self.get_trail())) / max(self.get_trail_length(), 1) + }, + 'visited_nodes': [ + {'step': i+1, 'node_type': node.__class__.__name__, 'node_name': getattr(node, 'name', node.id)} + for i, node in enumerate(trail_nodes) + ], + 'path_analysis': [ + { + 'step': i+1, + 'node': node.name if hasattr(node, 'name') else node.id, + 'via_edge': edge.edge_type if edge else 'start' + } + for i, (node, edge) in enumerate(trail_path) + ] + } + + # Report the trail data + self.report(trail_report) + + print(f"\n📊 Trail Report Generated:") + print(f" - Total steps: {trail_report['summary']['total_steps']}") + print(f" - Unique nodes: {trail_report['summary']['unique_nodes']}") + print(f" - Path efficiency: {trail_report['summary']['efficiency_ratio']:.2%}") + +# Usage example +walker = TrailTrackingWalker() +root = await Root.get(None) +await walker.spawn(root) + +# Access trail data +final_trail = walker.get_trail() +print(f"Final trail: {final_trail}") +# Access the trail report from walker's collected data +report = await walker.get_report() +trail_reports = [item for item in report if isinstance(item, dict) and 'trail_report' in str(item)] +print(f"Trail report: {trail_reports[0] if trail_reports else 'No trail report found'}") +``` + +#### Advanced Trail Use Cases + +```python path=null start=null +class AdvancedTrailWalker(Walker): + def __init__(self): + super().__init__() + self.trail_enabled = True + self.max_trail_length = 0 # Unlimited for comprehensive analysis + self.visited_nodes = set() # For cycle detection + self.performance_metrics = [] + + @on_visit('Document') + async def process_with_cycle_detection(self, here: Node): + """Process documents with cycle detection using trail data. + + Args: + here: The visited Document node + """ + import time + start_time = time.time() + + # Cycle detection using trail + if here.id in self.visited_nodes: + trail = self.get_trail() + first_visit_index = trail.index(here.id) + cycle_length = len(trail) - first_visit_index - 1 + + print(f"🔄 Cycle detected at {here.id}! Length: {cycle_length} steps") + + self.report({ + 'cycle_detected': { + 'node_id': here.id, + 'cycle_length': cycle_length, + 'first_visit_step': first_visit_index + 1, + 'detection_step': len(trail) + } + }) + + # Stop to avoid infinite loop + await self.disengage() + return + + self.visited_nodes.add(here.id) + + # Process document + await self.analyze_document(here) + + # Record performance metrics + processing_time = time.time() - start_time + self.performance_metrics.append({ + 'node_id': here.id, + 'processing_time': processing_time, + 'step_number': self.get_trail_length(), + 'metadata': self.get_trail_metadata() # Get current step metadata + }) + + # Continue traversal with trail-aware filtering + related_docs = await here.nodes( + node=['Document'], + status='active' + ) + + # Filter out recently visited to avoid cycles + recent_trail = self.get_recent_trail(count=10) + unvisited_docs = [doc for doc in related_docs if doc.id not in recent_trail] + + if unvisited_docs: + await self.visit(unvisited_docs) + else: + print("All related documents recently visited, exploring alternatives") + + @on_visit('User') + async def audit_user_access(self, here: Node): + """Create audit trail for user access. + + Args: + here: The visited User node + """ + # Get current trail metadata (automatically includes timestamp, node_type, queue_length) + metadata = self.get_trail_metadata() + + audit_entry = { + 'timestamp': metadata.get('timestamp'), + 'action': 'USER_ACCESS', + 'user_id': here.id, + 'user_name': getattr(here, 'name', 'Unknown'), + 'trail_step': self.get_trail_length(), + 'access_context': { + 'queue_size': metadata.get('queue_length'), + 'node_type': metadata.get('node_type'), + 'previous_steps': self.get_recent_trail(count=3)[:-1] # Exclude current + } + } + + self.report({'audit_entry': audit_entry}) + print(f"📝 Audit: Accessed user {here.id} at step {audit_entry['trail_step']}") + + @on_exit + async def comprehensive_analysis(self): + """Generate comprehensive trail and performance analysis.""" + trail_path = await self.get_trail_path() + + # Performance analysis + avg_processing_time = sum(m['processing_time'] for m in self.performance_metrics) / len(self.performance_metrics) if self.performance_metrics else 0 + + # Path efficiency analysis + total_steps = self.get_trail_length() + unique_nodes = len(set(self.get_trail())) + + # Get report once for all analysis + report = await self.get_report() + + comprehensive_analysis = { + 'trail_summary': { + 'total_steps': total_steps, + 'unique_nodes_visited': unique_nodes, + 'path_efficiency': unique_nodes / total_steps if total_steps > 0 else 0, + 'cycles_detected': len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item]), + 'trail_enabled': self.trail_enabled, + 'trail_limit': self.max_trail_length + }, + 'performance_metrics': { + 'avg_processing_time': avg_processing_time, + 'total_processing_time': sum(m['processing_time'] for m in self.performance_metrics), + 'slowest_step': max(self.performance_metrics, key=lambda x: x['processing_time']) if self.performance_metrics else None + }, + 'audit_summary': { + 'total_audit_entries': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item]), + 'user_accesses': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item and item.get('audit_entry', {}).get('action') == 'USER_ACCESS']) + } + } + + # Report the comprehensive analysis + self.report(comprehensive_analysis) + + print("\n📈 Comprehensive Analysis Complete:") + print(f" - Path efficiency: {comprehensive_analysis['trail_summary']['path_efficiency']:.2%}") + print(f" - Average processing time: {avg_processing_time:.3f}s") + print(f" - Cycles detected: {len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item])}") + + async def analyze_document(self, doc): + """Simulate document analysis.""" + import asyncio + await asyncio.sleep(0.02) # Simulate processing time + +# Usage with trail management +walker = AdvancedTrailWalker() + +# Enable debug mode for detailed trail information +walker.debug_mode = True + +# Spawn and run analysis +root = await Root.get(None) +await walker.spawn(root) + +# Access comprehensive results from walker's report +report = await walker.get_report() +analysis = next((item for item in report if isinstance(item, dict) and 'trail_summary' in item), {}) +cycles = [item for item in report if isinstance(item, dict) and 'cycle_detected' in item] +audit_entries = [item for item in report if isinstance(item, dict) and 'audit_entry' in item] + +print(f"Analysis complete. Efficiency: {analysis.get('trail_summary', {}).get('path_efficiency', 0):.2%}") +print(f"Cycles detected: {len(cycles)}, Audit entries: {len(audit_entries)}") +``` + +#### Trail API Quick Reference + +**Configuration (Read/Write):** +- `self.trail_enabled = True` - Enable trail tracking +- `self.max_trail_length = N` - Limit trail to N steps (0 = unlimited) + +**Trail Data (Read-Only Properties):** +- `self.trail` - List of visited node IDs (returns copy) +- `self.trail_edges` - Edge IDs between nodes (returns copy) +- `self.trail_metadata` - Metadata per step (returns deep copy) + +**Trail Access Methods (O(1) operations):** +- `self.get_trail()` - Get list of visited node IDs +- `self.get_trail_length()` - Get current trail length +- `self.get_recent_trail(count=N)` - Get last N trail steps +- `self.clear_trail()` - Clear entire trail (only way to modify trail) + +**Advanced Access (Database operations):** +- `await self.get_trail_nodes()` - Get actual Node objects from trail +- `await self.get_trail_path()` - Get trail with connecting edges +- `self.get_trail_metadata(step=-1)` - Get metadata for specific step + +**Use Cases:** +- **Debugging**: Track walker path for troubleshooting +- **Cycle Detection**: Avoid infinite loops in graph traversal +- **Performance Analysis**: Measure processing time per step +- **Audit Trails**: Comprehensive access logging for compliance +- **Path Optimization**: Analyze and improve traversal efficiency + +### Walker Queue Manipulation Methods + +Walkers maintain an internal queue (deque) of nodes to visit during traversal. Advanced queue manipulation provides fine-grained control over traversal order, priority handling, and dynamic path planning. These methods allow you to programmatically manage the walker's traversal queue: + +#### Core Queue Methods + +```python path=null start=null +from jvspatial.core import Walker, Node +from jvspatial.core.entities import on_visit +from typing import List, Optional + +class QueueMasterWalker(Walker): + def __init__(self): + super().__init__() + self.priority_nodes = [] + self.deferred_nodes = [] + self.processed_count = 0 + + @on_visit("TaskNode") + async def process_task(self, here: Node): + """Demonstrate queue manipulation methods. + + Args: + here: The visited TaskNode + """ + + # 1. INSPECT QUEUE STATE + current_queue = self.get_queue() # Get queue as list + print(f"Current queue size: {len(current_queue)}") + print(f"Queue contents: {[n.name for n in current_queue]}") + + # Check if specific node is queued + if current_queue: + next_node = current_queue[0] # Peek at next node + print(f"Next node to process: {next_node.name}") + + # 2. ADD NODES TO QUEUE + # Find high-priority nodes to add immediately + urgent_tasks = await here.nodes( + node=['TaskNode'], + priority={"$gte": 9}, + status="pending" + ) + + # Add to front of queue (high priority) + if urgent_tasks: + added = self.prepend(urgent_tasks) # Add to front + print(f"Added {len(added)} urgent tasks to front of queue") + + # Find normal tasks to add to end + normal_tasks = await here.nodes( + node=['TaskNode'], + priority={"$lt": 9, "$gte": 5}, + status="pending" + ) + + # Add to end of queue (normal priority) + if normal_tasks: + added = self.append(normal_tasks) # Add to end + print(f"Added {len(added)} normal tasks to end of queue") + + # Alternative: Use visit() method (equivalent to append) + additional_tasks = await here.nodes(node=['TaskNode'], status="new") + if additional_tasks: + self.visit(additional_tasks) # Same as append() + + # Add nodes right after current processing + immediate_tasks = await here.nodes( + node=['TaskNode'], + priority=10, # Highest priority + status="urgent" + ) + if immediate_tasks: + added = self.add_next(immediate_tasks) # Add next in queue + print(f"Added {len(added)} tasks to process immediately next") + + # 3. CONDITIONAL QUEUE MANIPULATION + # Check if we have too many items in queue + if len(self.get_queue()) > 100: + print("Queue overflow detected, deferring low-priority items") + + # Get current queue and filter it + current_queue = self.get_queue() + low_priority = [] + high_priority = [] + + for item in current_queue: + if hasattr(item, 'priority') and item.priority < 5: + low_priority.append(item) + else: + high_priority.append(item) + + # Clear queue and rebuild with high priority items only + self.clear_queue() + if high_priority: + self.append(high_priority) + + # Store deferred items for later + self.deferred_nodes.extend(low_priority) + print(f"Deferred {len(low_priority)} low-priority items") + + # 4. TARGETED QUEUE MANIPULATION + # Remove specific completed nodes from queue + completed_nodes = await here.nodes( + node=['TaskNode'], + status="completed" + ) + + for completed in completed_nodes: + if self.is_queued(completed): + removed = self.dequeue(completed) + print(f"Removed {len(removed)} completed tasks from queue") + + # 5. PRECISE QUEUE INSERTION + # Find a specific node in queue to insert after + checkpoint_tasks = await here.nodes( + node=['TaskNode'], + task_type="checkpoint" + ) + + followup_tasks = await here.nodes( + node=['TaskNode'], + depends_on=here.id + ) + + for checkpoint in checkpoint_tasks: + if self.is_queued(checkpoint) and followup_tasks: + try: + # Insert followup tasks right after checkpoint + inserted = self.insert_after(checkpoint, followup_tasks) + print(f"Inserted {len(inserted)} followup tasks after checkpoint") + except ValueError as e: + print(f"Could not insert after checkpoint: {e}") + + self.processed_count += 1 + + @on_visit("CompletionNode") + async def handle_completion(self, here: Node): + """Handle task completion and queue cleanup. + + Args: + here: The visited CompletionNode + """ + + # Add back any deferred nodes if queue is manageable + current_queue_size = len(self.get_queue()) + if current_queue_size < 20 and self.deferred_nodes: + reactivated = self.deferred_nodes[:10] # Add up to 10 back + self.deferred_nodes = self.deferred_nodes[10:] # Remove from deferred + + self.append(reactivated) + print(f"Reactivated {len(reactivated)} deferred nodes") + + # Insert critical tasks right at the beginning + critical_tasks = await here.nodes( + node=['TaskNode'], + priority=10, + status="critical" + ) + + if critical_tasks: + # Find first non-critical task and insert before it + queue = self.get_queue() + for i, queued_node in enumerate(queue): + if hasattr(queued_node, 'priority') and queued_node.priority < 10: + try: + inserted = self.insert_before(queued_node, critical_tasks) + print(f"Inserted {len(inserted)} critical tasks before normal task") + break + except ValueError: + # If insertion fails, just prepend + self.prepend(critical_tasks) + break + + @on_exit + async def final_report(self): + """Report final queue statistics.""" + final_queue = self.get_queue() + print(f"\n📊 Queue Processing Complete") + print(f"Total processed: {self.processed_count}") + print(f"Remaining in queue: {len(final_queue)}") + print(f"Deferred nodes: {len(self.deferred_nodes)}") + + if final_queue: + print("Remaining nodes:") + for node in final_queue[:5]: # Show first 5 + print(f" - {node.name}") + if len(final_queue) > 5: + print(f" ... and {len(final_queue) - 5} more") +``` + +#### Queue Method Reference + +Based on the actual Walker implementation in jvspatial: + +**Basic Queue Operations:** +- `self.visit(nodes)` - Add nodes to end of queue (equivalent to append) +- `self.append(nodes)` - Add nodes to end of queue +- `self.prepend(nodes)` - Add nodes to front of queue +- `self.add_next(nodes)` - Add nodes next in queue after current item +- `self.get_queue()` - Return entire queue as a list +- `self.clear_queue()` - Clear all nodes from queue +- `self.is_queued(node)` - Check if specific node is in queue + +**Advanced Queue Operations:** +- `self.dequeue(nodes)` - Remove specific nodes from queue +- `self.insert_after(target_node, nodes)` - Insert nodes after target node +- `self.insert_before(target_node, nodes)` - Insert nodes before target node + +#### Priority-Based Queue Management + +```python path=null start=null +class PriorityQueueWalker(Walker): + def __init__(self): + super().__init__() + self.priority_buckets = { + 'urgent': [], # Priority 9-10 + 'high': [], # Priority 7-8 + 'normal': [], # Priority 4-6 + 'low': [] # Priority 1-3 + } + + @on_visit("WorkItem") + async def process_work_item(self, here: Node): + """Process work items with priority-based queuing. + + Args: + here: The visited WorkItem node + """ + + # Get connected work items + connected_items = await here.nodes( + node=['WorkItem'], + status="pending" + ) + + if connected_items: + # Sort into priority buckets + self._sort_into_priority_buckets(connected_items) + + # Process highest priority items first + self._add_by_priority_order() + + def _sort_into_priority_buckets(self, nodes: List[Node]): + """Sort nodes into priority-based buckets.""" + for node in nodes: + priority = getattr(node, 'priority', 5) + + if priority >= 9: + self.priority_buckets['urgent'].append(node) + elif priority >= 7: + self.priority_buckets['high'].append(node) + elif priority >= 4: + self.priority_buckets['normal'].append(node) + else: + self.priority_buckets['low'].append(node) + + print(f"Sorted into buckets: " + f"urgent={len(self.priority_buckets['urgent'])}, " + f"high={len(self.priority_buckets['high'])}, " + f"normal={len(self.priority_buckets['normal'])}, " + f"low={len(self.priority_buckets['low'])}") + + def _add_by_priority_order(self): + """Add nodes to walker queue in priority order.""" + # Process urgent items first (add to front) + if self.priority_buckets['urgent']: + self.prepend(self.priority_buckets['urgent']) + self.priority_buckets['urgent'].clear() + + # Add high priority items to front (after urgent) + if self.priority_buckets['high']: + # Insert at beginning but after urgent items + current_queue = self.get_queue() + if current_queue: + # Find first non-urgent item and insert before it + high_items = self.priority_buckets['high'] + self.priority_buckets['high'].clear() + + try: + # Try to find insertion point + first_non_urgent = None + for node in current_queue: + if hasattr(node, 'priority') and node.priority < 9: + first_non_urgent = node + break + + if first_non_urgent: + self.insert_before(first_non_urgent, high_items) + else: + self.append(high_items) # All items are urgent + + except ValueError: + # Fallback to prepend if insertion fails + self.prepend(high_items) + else: + self.prepend(self.priority_buckets['high']) + self.priority_buckets['high'].clear() + + # Add normal priority items to end + if self.priority_buckets['normal']: + self.append(self.priority_buckets['normal']) + self.priority_buckets['normal'].clear() + + # Only add low priority if queue is small + current_queue_size = len(self.get_queue()) + if current_queue_size < 50 and self.priority_buckets['low']: + low_items = self.priority_buckets['low'][:10] # Limit low priority + self.priority_buckets['low'] = self.priority_buckets['low'][10:] + self.append(low_items) +``` + +#### Dynamic Queue Filtering and Manipulation + +```python path=null start=null +class AdaptiveQueueWalker(Walker): + def __init__(self): + super().__init__() + self.queue_stats = { + 'added': 0, + 'removed': 0, + 'filtered': 0, + 'reordered': 0 + } + + @on_visit("FilterNode") + async def adaptive_filtering(self, here: Node): + """Demonstrate dynamic queue filtering and manipulation. + + Args: + here: The visited FilterNode + """ + + # Add new nodes based on current context + candidates = await here.nodes( + node=['ProcessNode'], + active=True + ) + + if candidates: + # Filter candidates before adding to queue + filtered_candidates = self._filter_candidates(candidates) + + if filtered_candidates: + # Smart queue insertion based on current load + current_queue_size = len(self.get_queue()) + + if current_queue_size < 20: + # Low load: add all to end + self.append(filtered_candidates) + self.queue_stats['added'] += len(filtered_candidates) + else: + # High load: add only high-priority to front + high_priority = [ + c for c in filtered_candidates + if getattr(c, 'priority', 0) >= 8 + ] + if high_priority: + self.prepend(high_priority) + self.queue_stats['added'] += len(high_priority) + + # Periodic queue maintenance + if hasattr(here, 'name') and here.name.endswith('_maintenance'): + self._perform_queue_maintenance() + + def _filter_candidates(self, candidates: List[Node]) -> List[Node]: + """Filter candidates based on various criteria.""" + filtered = [] + current_queue = self.get_queue() + + for candidate in candidates: + # Check if already in queue (avoid duplicates) + if self.is_queued(candidate): + continue + + # Check resource constraints (mock implementation) + if hasattr(candidate, 'resource_requirement'): + if candidate.resource_requirement > self._get_available_resources(): + continue + + # Check dependencies (mock implementation) + if hasattr(candidate, 'dependencies'): + if not self._dependencies_met(candidate.dependencies): + continue + + filtered.append(candidate) + + self.queue_stats['filtered'] += len(candidates) - len(filtered) + return filtered + + def _perform_queue_maintenance(self): + """Perform queue cleanup and optimization.""" + current_queue = self.get_queue() + if not current_queue: + return + + print("🔧 Performing queue maintenance...") + + # 1. Remove stale items (mock - would check expiration) + non_stale = [] + removed_stale = 0 + + for item in current_queue: + if hasattr(item, 'expires_at'): + # Mock expiration check + if not getattr(item, 'is_expired', False): + non_stale.append(item) + else: + removed_stale += 1 + else: + non_stale.append(item) + + # 2. Deduplicate items by ID + seen_ids = set() + deduplicated = [] + removed_duplicates = 0 + + for item in non_stale: + if item.id not in seen_ids: + seen_ids.add(item.id) + deduplicated.append(item) + else: + removed_duplicates += 1 + + # 3. Reorder by priority + optimized = sorted( + deduplicated, + key=lambda x: getattr(x, 'priority', 0), + reverse=True + ) + + # 4. Rebuild queue with optimized order + self.clear_queue() + if optimized: + self.append(optimized) + + self.queue_stats['removed'] += removed_stale + removed_duplicates + self.queue_stats['reordered'] += 1 + + print(f"Maintenance complete: removed {removed_stale} stale, " + f"{removed_duplicates} duplicates, optimized {len(optimized)} items") + + def _get_available_resources(self) -> int: + """Mock implementation - get available system resources.""" + return 100 + + def _dependencies_met(self, dependencies: List[str]) -> bool: + """Mock implementation - check if dependencies are satisfied.""" + return True + + @on_exit + async def report_queue_stats(self): + """Report queue manipulation statistics.""" + print("\n📈 Queue Statistics:") + for stat, value in self.queue_stats.items(): + print(f" {stat.capitalize()}: {value}") +``` + +#### Best Practices for Queue Manipulation + +**✅ Recommended Patterns:** + +```python path=null start=null +# Good: Check queue state before manipulation +current_queue = self.get_queue() +if current_queue: + next_item = current_queue[0] # Look at next item + # Make decisions based on next item + +# Good: Use appropriate method for insertion priority +if item.priority >= 8: + self.prepend([item]) # High priority to front +else: + self.append([item]) # Normal priority to end + +# Good: Check if node is queued before operations +if self.is_queued(completed_node): + self.dequeue(completed_node) + +# Good: Batch queue operations for efficiency +new_items = await node.nodes(filters) +if new_items: + self.append(new_items) # Add all at once + +# Good: Safe queue iteration and modification +current_queue = self.get_queue() # Get snapshot +filtered_items = [n for n in current_queue if meets_criteria(n)] +self.clear_queue() +if filtered_items: + self.append(filtered_items) + +# Good: Precise insertion with error handling +try: + self.insert_after(target_node, new_nodes) +except ValueError: + # Target not found, use alternative + self.prepend(new_nodes) +``` + +**❌ Avoided Patterns:** + +```python path=null start=null +# Bad: Modifying queue during iteration +for item in self.get_queue(): # Don't iterate over changing queue + if condition: + self.dequeue(item) # Modifying during iteration + +# Bad: Not handling insertion errors +self.insert_after(target_node, nodes) # Could raise ValueError + +# Bad: Inefficient repeated operations +for item in items: + self.append([item]) # Many small operations +# Better: self.append(items) # Single batch operation + +# Bad: Not checking queue state +first_item = self.get_queue()[0] # Could cause IndexError if empty + +# Bad: Assuming nodes are still queued +self.dequeue(node) # Node might not be in queue +# Better: if self.is_queued(node): self.dequeue(node) +``` + +**Key Points:** +- Use `await node.nodes()` to get connected nodes for traversal (NOT `get_edges()`) +- Default `direction="out"` follows outgoing edges (recommended for forward traversal) +- Use `direction="in"` for reverse traversal along incoming edges +- Use `direction="both"` for bidirectional traversal +- **Semantic Filtering Approaches:** + - **Simple filtering**: Use kwargs for connected node properties: `state="NY"` + - **Complex node filtering**: `node=[{'City': {"context.population": {"$gte": 500000}}}]` + - **Complex edge filtering**: `edge=[{'Highway': {"context.condition": {"$ne": "poor"}}}]` + - **Mixed approaches**: Combine dict filters with kwargs for maximum flexibility +- **Database-Level Optimization**: All filtering happens at database level for performance +- **MongoDB-Style Operators**: `$gte`, `$lt`, `$ne`, `$in`, `$nin`, `$regex`, etc. +- **Walker Control Flow:** + - **`skip()`**: Skip current node processing, continue to next (like `continue` in loops) + - **`pause()`/`resume()`**: Temporarily pause walker (use `self.pause()`), can be resumed later + - **`disengage()`**: Permanently halt walker and remove from graph (cannot be resumed) + - **`@on_exit`**: Methods called when walker completes, pauses, or disengages (cleanup) +- The `nodes()` method returns a list that can be directly passed to `walker.visit()` + +### Inheritance Hierarchy + +``` +Object (base class with unified query interface) +├── Node (spatial graph nodes) +├── Edge (relationships) +└── Custom entities (inherit from Node/Object) +``` + +### Database Backends + +- **JSONDatabase** - File-based storage for development/testing +- **MongoDatabase** - MongoDB backend for production +- **Custom databases** - Implement DatabaseInterface + +## 📚 Documentation Maintenance + +### README Updates + +When adding new features: + +1. **Review existing README.md** +2. **Update feature list** if adding major functionality +3. **Update installation/setup** if dependencies change +4. **Update usage examples** to reflect new capabilities +5. **Maintain consistency** with existing documentation style + +### Documentation Directory + +Always review and update `docs/` directory: + +``` +docs/ +├── api/ # API reference documentation +├── guides/ # User guides and tutorials +├── architecture/ # System design documents +└── examples/ # Code examples and tutorials +``` + +**Update procedure:** +1. Check if new feature requires new documentation files +2. Update existing API documentation for modified classes/methods +3. Add user guides for complex features +4. Update architecture docs if design patterns change + +## 💡 Examples Maintenance + +### Example Directory Structure + +``` +examples/ +├── basic/ # Simple usage examples +├── advanced/ # Complex scenarios +├── query_interface_example.py # Comprehensive entity-centric CRUD operations +├── semantic_filtering.py # Advanced semantic filtering with Node.nodes() +└── migration/ # Migration guides (if needed) +``` + +### Example Update Procedure + +1. **Review existing examples** for relevance to new features +2. **Update outdated examples** to use modern entity-centric syntax +3. **Create new examples** for significant new features +4. **Ensure examples are runnable** and well-documented +5. **Remove or archive obsolete examples** +6. **Update key reference examples:** + - `query_interface_example.py` - Showcase latest entity-centric patterns + - `semantic_filtering.py` - Demonstrate advanced Node.nodes() filtering + - Basic examples - Keep simple, focused, and beginner-friendly + - Advanced examples - Show complex real-world scenarios + +### Example Code Standards + +Examples should: +- Use entity-centric syntax exclusively +- Include comprehensive comments +- Demonstrate best practices +- Be self-contained and runnable +- Show error handling patterns + +## 🧪 Testing Requirements + +### Test Organization + +``` +tests/ +├── api/ # FastAPI endpoint tests +├── core/ # Core entity and logic tests +├── db/ # Database backend tests +└── integration/ # End-to-end integration tests +``` + +### Testing Procedure for New Features + +1. **Unit Tests** - Test individual methods and classes +2. **Integration Tests** - Test feature interaction with database +3. **API Tests** - Test HTTP endpoints if applicable +4. **Performance Tests** - For database-heavy features + +### Test Code Standards + +```python +import pytest +from typing import List +from jvspatial.core import Node, Walker, Edge +from jvspatial.core.entities import on_visit +from jvspatial.exceptions import NodeNotFoundError, ValidationError + +class TestUser(Node): + name: str = "" + email: str = "" + department: str = "" + age: int = 0 + active: bool = True + skills: List[str] = [] + +class TestDepartment(Node): + name: str = "" + location: str = "" + budget: int = 0 + +class TestWalker(Walker): + def __init__(self): + super().__init__() + self.visited_users = [] + + @on_visit("TestUser") + async def visit_user(self, node: TestUser): + self.visited_users.append(node.name) + # Test semantic filtering in walker + connected_users = await node.nodes( + node=['TestUser'], + department=node.department, + active=True + ) + await self.visit(connected_users) + +# Entity Creation Tests +@pytest.mark.asyncio +async def test_user_creation(): + """Test entity-centric user creation with full data.""" + user = await TestUser.create( + name="Alice Johnson", + email="alice@company.com", + department="engineering", + age=30, + skills=["python", "javascript"] + ) + assert user.name == "Alice Johnson" + assert user.email == "alice@company.com" + assert user.department == "engineering" + assert user.id is not None + assert "python" in user.skills + +# MongoDB-Style Query Tests +@pytest.mark.asyncio +async def test_mongodb_queries(): + """Test comprehensive MongoDB-style queries.""" + # Setup test data + await TestUser.create(name="Bob Smith", email="bob@test.com", age=25, department="engineering") + await TestUser.create(name="Carol Davis", email="carol@test.com", age=35, department="marketing") + await TestUser.create(name="David Brown", email="david@test.com", age=45, department="engineering") + + # Test regex queries + b_users = await TestUser.find({"context.name": {"$regex": "^B", "$options": "i"}}) + assert len(b_users) >= 1 + assert any(u.name.startswith("B") for u in b_users) + + # Test comparison operators + senior_users = await TestUser.find({"context.age": {"$gte": 35}}) + assert all(u.age >= 35 for u in senior_users) + + # Test logical operators + senior_engineers = await TestUser.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.age": {"$gte": 30}} + ] + }) + assert all(u.department == "engineering" and u.age >= 30 for u in senior_engineers) + + # Test array operations + tech_users = await TestUser.find({"context.skills": {"$in": ["python", "javascript"]}}) + assert all(any(skill in u.skills for skill in ["python", "javascript"]) for u in tech_users) + +# Semantic Filtering Tests +@pytest.mark.asyncio +async def test_semantic_filtering(): + """Test Node.nodes() semantic filtering capabilities.""" + # Create test nodes and edges + dept = await TestDepartment.create(name="Engineering", location="SF") + user1 = await TestUser.create(name="Alice", department="engineering", active=True) + user2 = await TestUser.create(name="Bob", department="engineering", active=False) + user3 = await TestUser.create(name="Carol", department="marketing", active=True) + + # Create connections + await Edge.create(source=dept, target=user1, edge_type="employs") + await Edge.create(source=dept, target=user2, edge_type="employs") + await Edge.create(source=user1, target=user3, edge_type="collaborates") + + # Test simple filtering + active_employees = await dept.nodes( + node=['TestUser'], + active=True + ) + assert len(active_employees) == 1 + assert active_employees[0].name == "Alice" + + # Test complex filtering + engineering_employees = await dept.nodes( + node=[{'TestUser': {"context.department": "engineering"}}], + active=True + ) + assert all(u.department == "engineering" and u.active for u in engineering_employees) + + # Test mixed filtering + collaborators = await user1.nodes( + node=[{'TestUser': {"context.active": True}}], + department="marketing" + ) + assert len(collaborators) >= 1 + +# Walker Tests +@pytest.mark.asyncio +async def test_walker_traversal(): + """Test walker with semantic filtering.""" + # Setup graph + root = await TestUser.create(name="Root", department="engineering") + user1 = await TestUser.create(name="User1", department="engineering", active=True) + user2 = await TestUser.create(name="User2", department="marketing", active=True) + + await Edge.create(source=root, target=user1, edge_type="manages") + await Edge.create(source=root, target=user2, edge_type="manages") + await Edge.create(source=user1, target=user2, edge_type="collaborates") + + # Test walker + walker = TestWalker() + await walker.spawn(root) + + # Verify walker visited users + assert "Root" in walker.visited_users + assert len(walker.visited_users) >= 1 + +# Error Handling Tests +@pytest.mark.asyncio +async def test_error_handling(): + """Test proper error handling patterns.""" + # Test NodeNotFoundError + with pytest.raises(NodeNotFoundError): + await TestUser.get("nonexistent-id") + + # Test safe retrieval + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await TestUser.find({"context.email": "nonexistent@test.com"}) + user = users[0] if users else None + assert user is None + + # Test validation errors (if validation is implemented) + with pytest.raises((ValidationError, ValueError)): + await TestUser.create(name="", email="invalid-email") + +# Performance Tests +@pytest.mark.asyncio +async def test_bulk_operations(): + """Test bulk operations performance.""" + # Create multiple users + users_data = [ + {"name": f"User{i}", "email": f"user{i}@test.com", "department": "engineering"} + for i in range(10) + ] + + # Bulk create + users = [] + for user_data in users_data: + user = await TestUser.create(**user_data) + users.append(user) + + assert len(users) == 10 + + # Bulk query + engineering_users = await TestUser.find({"context.department": "engineering"}) + assert len(engineering_users) >= 10 + + # Bulk update + for user in users[:5]: + user.active = False + await user.save() + + # Verify updates + inactive_users = await TestUser.find({"context.active": False}) + assert len(inactive_users) >= 5 + +# Integration Tests +@pytest.mark.asyncio +async def test_full_workflow(): + """Test complete entity lifecycle with relationships.""" + # Create department + dept = await TestDepartment.create(name="Product", location="NYC", budget=1000000) + + # Create users + manager = await TestUser.create( + name="Jane Manager", + email="jane@company.com", + department="product", + age=40 + ) + + employee = await TestUser.create( + name="John Employee", + email="john@company.com", + department="product", + age=28 + ) + + # Create relationships + await Edge.create(source=dept, target=manager, edge_type="employs") + await Edge.create(source=dept, target=employee, edge_type="employs") + await Edge.create(source=manager, target=employee, edge_type="manages") + + # Test traversal + department_employees = await dept.nodes(node=['TestUser']) + assert len(department_employees) == 2 + + managed_employees = await manager.nodes( + node=['TestUser'], + direction="out", + edge=['manages'] + ) + assert len(managed_employees) == 1 + assert managed_employees[0].name == "John Employee" + + # Test complex query + young_employees = await dept.nodes( + node=[{'TestUser': {"context.age": {"$lt": 30}}}] + ) + assert len(young_employees) == 1 + assert young_employees[0].age < 30 + + # Cleanup + await manager.delete() + await employee.delete() + await dept.delete() + + # Verify deletion + # Note: Object.find_one() doesn't exist - use find() and get first result + users = await TestUser.find({"context.email": "jane@company.com"}) + deleted_manager = users[0] if users else None + assert deleted_manager is None +``` + +## 🗑️ Cleanup and Maintenance + +### Deprecation Procedure + +When evolving the library: + +1. **Identify obsolete code** - Old patterns, unused utilities +2. **Mark as deprecated** - Add deprecation warnings before removal +3. **Update documentation** - Remove references to deprecated features +4. **Update examples** - Remove or update examples using deprecated code +5. **Clean removal** - Remove deprecated code after grace period + +### File Cleanup Checklist + +- [ ] Remove unused import statements +- [ ] Delete empty or obsolete modules +- [ ] Archive outdated examples to `examples/archive/` +- [ ] Update `__all__` exports in `__init__.py` files +- [ ] Remove commented-out code blocks +- [ ] Clean up temporary test files + +### Migration Strategy + +When making breaking changes: + +1. **Create migration guide** in `docs/migration/` +2. **Provide before/after examples** +3. **Update all existing examples** to new patterns +4. **Add runtime warnings** for deprecated usage +5. **Version appropriately** using semantic versioning + +## 🚀 Development Workflow + +### Pre-commit Checklist + +- [ ] Code passes `black --check .` +- [ ] Code passes `flake8 .` +- [ ] Code passes `mypy .` +- [ ] All tests pass: `pytest` +- [ ] Examples are updated and runnable +- [ ] Documentation reflects changes +- [ ] Deprecated code is cleaned up + +### Code Review Focus Areas + +1. **Entity-centric patterns** - Ensure new code uses preferred syntax +2. **Query interface consistency** - MongoDB-style queries throughout +3. **Type safety** - Proper annotations and mypy compliance +4. **Test coverage** - Adequate testing for new features +5. **Documentation completeness** - Examples and guides updated + +## 📋 Quick Reference + +### Preferred Patterns + +```python path=null start=null +# ✅ Entity creation +user = await User.create(name="Alice", email="alice@company.com", department="engineering") + +# ✅ Entity queries with MongoDB-style operators +users = await User.find({"context.active": True}) +# Note: Object.find_one() doesn't exist - use find() and get first result +users_by_email = await User.find({"context.email": email}) +user = users_by_email[0] if users_by_email else None +senior_engineers = await User.find({ + "$and": [ + {"context.department": "engineering"}, + {"context.age": {"$gte": 35}} + ] +}) +tech_users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) + +# ✅ Counting and aggregation +# Note: Object.count() doesn't exist - use len() with find() instead +results = await User.find({"context.department": "engineering"}) +count = len(results) + +# Note: Object.distinct() doesn't exist - query and extract manually +all_users = await User.find({}) +departments = set(u.department for u in all_users if hasattr(u, 'department')) + +# ✅ Entity updates +user.name = "Alice Johnson" +user.active = True +await user.save() + +# ✅ Bulk updates +users = await User.find({"context.department": "old_dept"}) +for user in users: + user.department = "new_dept" + await user.save() + +# ✅ Entity deletion +await user.delete() + +# ✅ Walker traversal with semantic filtering +@on_visit("User") +async def process_user(self, here: Node): + """Process user nodes with semantic filtering. + + Args: + here: The visited User node + """ + # Use nodes() method with semantic filtering + connected_users = await here.nodes( + node=['User'], # Type filtering + department="engineering", # Simple kwargs + active=True # Multiple simple filters + ) + await self.visit(connected_users) + + # Complex filtering with MongoDB operators + senior_connections = await here.nodes( + node=[{'User': {"context.age": {"$gte": 35}}}], + direction="out" + ) + await self.visit(senior_connections) + +# ✅ Walker control flow +class MyWalker(Walker): + @on_visit("Document") + async def process_document(self, here: Node): + """Process document nodes with control flow. + + Args: + here: The visited Document node + """ + if here.status == "archived": + self.skip() # Skip to next node + + if self.processed_count >= 100: + await self.disengage() # Permanently halt + + # Normal processing + next_docs = await here.nodes(node=['Document'], active=True) + await self.visit(next_docs) + + @on_exit + async def cleanup(self): + print(f"Processed {self.processed_count} documents") + +# ✅ Error handling +try: + user = await User.get(user_id) +except NodeNotFoundError: + logger.warning(f"User {user_id} not found") + return None +except DatabaseError as e: + logger.error(f"Database error: {e}") + raise +``` + +### Avoided Patterns + +```python path=null start=null +# ❌ Direct database access (discouraged - use entity methods) +from jvspatial.db import create_database +db = create_database("json") +await db.save("node", data) # Use entity.save() instead +await db.find("node", {"name": "User"}) # Use Entity.find() instead + +# ❌ GraphContext methods for simple operations (use entity methods) +from jvspatial.core import GraphContext +ctx = GraphContext(database=db) +# Prefer entity-centric methods: +# await Node.create(...) instead of ctx.create_node(...) +# await node.get_edges() instead of ctx.get_edges(node_id) + +# ❌ Non-standard query formats +await User.find({"age": 25}) # Missing context. prefix +await User.find({"name": "Alice"}) # Should be context.name + +# ❌ Old traversal patterns (deprecated) +walker.get_edges(node) # Use node.nodes() instead +walker.traverse_edges() # Use semantic filtering + +# ❌ Synchronous operations +user = User.create_sync(**data) # Use async await User.create() +users = User.find_sync(query) # Use async await User.find() + +# ❌ Manual edge management in walkers (show proper naming even in bad examples) +@on_visit("User") +async def visit_user(self, here: Node): + """DEPRECATED: Manual edge management (avoid this pattern). + + Args: + here: The visited User node + """ + # Avoid manual edge retrieval + edges = await here.get_edges() # Deprecated + for edge in edges: + target = await edge.get_target() + await self.visit([target]) + + # Instead, use semantic filtering + connected = await here.nodes() # Preferred + await self.visit(connected) + +# ❌ Missing error handling +user = await User.get(user_id) # Should handle NodeNotFoundError +user.field = value +await user.save() # Should handle ValidationError + +# ❌ Inefficient queries +# Don't fetch all then filter in Python +all_users = await User.find({}) +engineers = [u for u in all_users if u.department == "engineering"] + +# Instead, filter at database level +engineers = await User.find({"context.department": "engineering"}) + +# ❌ Blocking operations in async context +@on_visit("DataNode") +async def process_node(self, here: Node): + """BAD EXAMPLE: Blocking operations in async context. + + Args: + here: The visited DataNode + """ + # Avoid blocking operations + time.sleep(1.0) # Blocks event loop + + # Use async alternatives + await asyncio.sleep(1.0) # Non-blocking +``` + +--- + +## ⏰ Scheduler Integration (Optional) + +jvspatial includes optional scheduler support for background task automation. Install with: + +```bash +pip install jvspatial[scheduler] +``` + +### Basic Scheduled Tasks + +```python path=null start=null +from jvspatial.api import Server +from jvspatial.api.scheduler import on_schedule +from jvspatial.core import Object +from datetime import datetime +from typing import Optional +import logging + +logger = logging.getLogger(__name__) + +# Define entity for job tracking (entity-centric pattern) +class ScheduledJob(Object): + """Entity representing scheduled job execution records.""" + job_name: str = "" + execution_time: datetime = datetime.now() + status: str = "pending" # pending, completed, failed + duration_seconds: Optional[float] = None + error_message: Optional[str] = None + +class SystemMetrics(Object): + """Entity for system metrics collection.""" + timestamp: datetime = datetime.now() + cpu_usage: float = 0.0 + memory_usage: float = 0.0 + active_jobs: int = 0 + +# Scheduled function with proper error handling +@on_schedule("every 30 minutes", description="System cleanup with job tracking") +async def cleanup_system(): + """Automated cleanup with entity-centric job tracking.""" + start_time = datetime.now() + + try: + logger.info("🧹 Starting system cleanup") + + # Perform cleanup work + cleanup_count = await perform_cleanup_work() + + # Create success record + await ScheduledJob.create( + job_name="system_cleanup", + execution_time=start_time, + status="completed", + duration_seconds=(datetime.now() - start_time).total_seconds() + ) + + logger.info(f"✅ Cleanup completed: {cleanup_count} items processed") + + except Exception as e: + # Create error record + await ScheduledJob.create( + job_name="system_cleanup", + execution_time=start_time, + status="failed", + error_message=str(e), + duration_seconds=(datetime.now() - start_time).total_seconds() + ) + logger.error(f"❌ Cleanup failed: {str(e)}") + raise + +# Metrics collection with MongoDB-style queries +@on_schedule("every 5 minutes", retry_count=2, description="Collect system metrics") +async def collect_metrics(): + """Collect system metrics with entity queries.""" + import psutil + + # Get system metrics + cpu_percent = psutil.cpu_percent() + memory = psutil.virtual_memory() + + # Count active jobs using MongoDB-style query + # Note: Object.count() doesn't exist - use len() with find() instead + active_jobs_list = await ScheduledJob.find({ + "context.status": {"$in": ["pending", "running"]} + }) + active_jobs = len(active_jobs_list) + + # Create metrics record + await SystemMetrics.create( + timestamp=datetime.now(), + cpu_usage=cpu_percent, + memory_usage=memory.percent, + active_jobs=active_jobs + ) + + logger.info(f"📊 Metrics: CPU {cpu_percent:.1f}%, Memory {memory.percent:.1f}%") + +async def perform_cleanup_work() -> int: + """Simulate cleanup work.""" + # Query old records using entity-centric approach + cutoff_time = datetime.now().timestamp() - (7 * 24 * 3600) # 7 days ago + + old_jobs = await ScheduledJob.find({ + "context.execution_time": {"$lt": cutoff_time} + }) + + # Delete old records + for job in old_jobs: + await job.delete() + + return len(old_jobs) +``` + +### Server Integration + +```python path=null start=null +from jvspatial.api import Server, endpoint +from jvspatial.api.scheduler import register_scheduled_tasks +from typing import Dict, Any +from dotenv import load_dotenv + +# Load environment configuration (jvspatial pattern) +load_dotenv() + +# Create server with scheduler enabled +server = Server( + title="My Scheduled App", + description="Application with integrated scheduler", + version="1.0.0", + scheduler_enabled=True, # Enable scheduler + scheduler_interval=1, # Check every second + scheduler_timezone="UTC", # Timezone for scheduling +) + +# Register all decorated scheduled tasks +if hasattr(server, 'scheduler_service') and server.scheduler_service: + register_scheduled_tasks(server.scheduler_service) + logger.info("✅ Scheduled tasks registered") + +# Add monitoring endpoint +@endpoint("/api/scheduler/status", methods=["GET"]) +async def get_scheduler_status() -> Dict[str, Any]: + """Get scheduler status with entity-centric job statistics.""" + # Get job statistics using entity queries + # Note: Object.count() doesn't exist - use len() with find() instead + all_jobs = await ScheduledJob.find({}) + total_jobs = len(all_jobs) + + completed_jobs_list = await ScheduledJob.find({"context.status": "completed"}) + completed_jobs = len(completed_jobs_list) + + failed_jobs_list = await ScheduledJob.find({"context.status": "failed"}) + failed_jobs = len(failed_jobs_list) + + return { + "scheduler": "running", + "job_statistics": { + "total_jobs": total_jobs, + "completed_jobs": completed_jobs, + "failed_jobs": failed_jobs, + "success_rate": (completed_jobs / total_jobs * 100) if total_jobs > 0 else 0 + }, + "timestamp": datetime.now().isoformat() + } + +if __name__ == "__main__": + server.run(host="0.0.0.0", port=8000) # Scheduler runs automatically +``` + +**📖 For comprehensive scheduler documentation:** [Scheduler Integration Guide](docs/md/scheduler.md) + +--- + +## 🎯 Key Naming Conventions + +**CRITICAL**: When writing `@on_visit` methods, always use these parameter names: + +- **`here`** - The visited node/edge (current location in traversal) +- **`visitor`** - The walker performing the traversal (when passed to node methods) + +```python path=null start=null +# ✅ CORRECT naming convention +@on_visit("User") +async def process_user(self, here: Node): + """Args: here = visited User node""" + connected = await here.nodes() + await self.visit(connected) + +# ❌ AVOID generic names +@on_visit("User") +async def process_user(self, node: Node): # Less clear + pass +``` + +--- + +**Remember**: This library prioritizes **clean, maintainable code** with **consistent patterns** across all database backends. Always favor the entity-centric approach, MongoDB-style queries, and the **`here`/`visitor`** naming convention for the best developer experience. diff --git a/PRD.md b/PRD.md new file mode 100644 index 0000000..c820f62 --- /dev/null +++ b/PRD.md @@ -0,0 +1,195 @@ +# jvspatial — Product Requirements Document + +> **Purpose**: Capture *why* jvspatial exists, who it serves, and what success looks like. This is the product context for the technical contract in [SPEC.md](SPEC.md). +> +> **Audience**: Maintainers (human and AI), reviewers, future contributors evaluating whether a proposed change is in scope. +> +> **Status**: Living document. Update in the same commit as any change that alters scope, target users, or success criteria. + +--- + +## 1. Positioning + +jvspatial is an **async-first, serverless-compatible, object-spatial Python library** for building persistence and business-logic application layers on top of a graph data model. + +It is *not*: + +- A graph database engine. jvspatial layers a graph model on top of existing backends (MongoDB, SQLite, JSON files, DynamoDB) and unifies query semantics across them. +- A web framework. It builds on FastAPI; it does not replace it. +- A workflow engine. Walkers traverse data; they are not a generic task orchestrator. +- A UI library. Visualization helpers exist (DOT, Mermaid), but rendering is out of scope. + +It *is*: + +- A substrate for graph-native business logic, with an entity-centric API that hides storage details from domain code. +- A FastAPI integration layer that turns decorated entities and walkers into HTTP endpoints with consistent auth, validation, and observability. +- A serverless-aware library that adapts its defaults (deferred saves, index creation, hashing cost) to the deployment environment. + +The library is inspired by Jaseci's object-spatial paradigm. The implementation is pure Python, async-throughout, and uses Pydantic v2 for schemas. + +--- + +## 2. Target Users + +Described by role, not by named product. jvspatial serves callers in three roles: + +### 2.1 Backend engineers building graph-shaped applications + +People modeling domains where relationships are first-class: knowledge graphs, social structures, agent memory, organizational hierarchies, recommendation systems, geographic networks. + +**What they need**: an entity model with first-class edges, traversal primitives that respect graph semantics, async I/O so a request handler can fan out across the graph without blocking. + +### 2.2 Platform teams building serverless APIs over graph data + +People deploying graph-backed services to AWS Lambda, Google Cloud Run, Azure Functions, Vercel — environments that need stateless request handling, fast cold starts, and explicit boundaries between in-request work and deferred work. + +**What they need**: serverless mode detection that flips opinionated defaults (deferred saves off, indexes off, lower bcrypt cost, `/tmp` DB paths), a deferred-task dispatcher that survives invocation boundaries, idempotency-friendly handler registration. + +### 2.3 Library authors needing a persistence + traversal substrate + +People building higher-level frameworks that ship a graph model to their own users. They want a foundation they can extend without forking — custom database backends, custom storage backends, custom auth flows, custom log levels. + +**What they need**: stable extension points (`register_database_type`, `FileStorageInterface`, lifecycle hooks, deferred-task handlers), a documented stability contract so their downstream depends only on stable surface, and a clean separation between public API and internal helpers. + +--- + +## 3. Core Value Propositions + +| # | Value | What it means | Where it lives | +|---|---|---|---| +| 1 | **Unified query DSL across backends** | One Mongo-style query syntax works against JSON, SQLite, MongoDB, and DynamoDB. Switch backends without rewriting queries. | `jvspatial/db/query.py` | +| 2 | **Entity-centric API** | `Node.create()`, `Node.get()`, `node.save()` — domain code calls entity methods, never raw DB clients. | `jvspatial/core/entities/object.py` | +| 3 | **First-class walkers** | Graph traversal is a built-in primitive with infinite-loop protection, trail tracking, and event hooks. Not a hand-rolled BFS in every handler. | `jvspatial/core/entities/walker.py` | +| 4 | **Identity model with explicit discriminators** | Every entity carries `type_code.entity_name.uuid`. `__entity_name__` lets disjoint hierarchies coexist. | SPEC §1 | +| 5 | **Serverless ergonomics by default** | One env var (`SERVERLESS_MODE=true`) flips a coherent set of defaults — no scattered serverless special cases in application code. | `jvspatial/runtime/serverless.py` | +| 6 | **Security defaults that fail closed** | JWT secret required, constant-time secret comparisons, content-based file MIME validation, restrictive CORS, strict CSP on app routes, optional `/docs` gating. | SPEC §15 | +| 7 | **FastAPI integration without lock-in** | `@endpoint` exposes functions and walkers as routes. The underlying FastAPI app is accessible for middleware, custom routers, and lifecycle hooks. | `jvspatial/api/server.py` | +| 8 | **Observable by construction** | Structured DB op logs, `MetricsRecorder` protocol, GraphContext `PerformanceMonitor`, OpenTelemetry adapter. | `jvspatial/observability/`, `jvspatial/db/_observable.py` | + +--- + +## 4. Non-Goals + +Explicit out-of-scope statements. These represent decisions, not gaps. + +| Non-goal | Why | +|---|---| +| Replace a real graph database (Neo4j, ArangoDB, Neptune) | jvspatial layers on document stores; it does not implement graph-native storage, indexes, or query planning. Use a graph DB when the workload demands it. | +| Provide a graph traversal query language | Walkers are imperative async Python. There is no Cypher or Gremlin equivalent and no plan to add one. | +| Synchronous API surface | The library is async-only by design. A sync wrapper would force `asyncio.run` per call and degrade performance. Callers stuck in sync contexts should run jvspatial in a worker. | +| Distributed transactions across backends | Multi-database operations are best-effort. Only single-backend transactions are supported (and only where the backend natively supports them — MongoDB replica sets). | +| Built-in caching strategy beyond the wrappers provided | We provide memory, redis, layered. Beyond that, callers compose their own. | +| Schema migrations | No migration framework. Adding optional fields with defaults is forward-safe; everything else is the caller's responsibility. | +| UI / dashboard | Graph export to DOT/Mermaid is provided. Rendering, visualization tooling, and a Web UI are downstream. | +| First-party multi-tenancy primitives | Multi-tenancy is doable via context scoping but is not codified. Tenant isolation is a caller responsibility. | + +--- + +## 5. Constraints + +### 5.1 Language and runtime + +- Python 3.9+ supported (3.8 declared in `pyproject.toml` classifiers but should be considered legacy). +- Pydantic v2 is required — Pydantic v1 compatibility is not maintained. +- FastAPI / Starlette pinned to versions that retain the `Router(on_startup, on_shutdown)` API (Starlette `<1.0.0`). + +### 5.2 Concurrency + +- Async-only I/O. Every database call, network call, and file-system call is a coroutine. +- No global mutable state that survives a request, except the database manager and module-level registries (decorator-collected endpoints, deferred task handlers). All such state is initialized at app build, not per-request. +- Sessions, rate-limit counters, and JWT blacklists are **per-process / per-worker** by default. Cross-worker sharing requires Redis or a similar shared store. + +### 5.3 Serverless + +- Cold-start cost is a budget, not a goal. Reduced bcrypt rounds and skipped index creation in serverless mode reflect this. +- No assumption that any in-memory state survives across invocations. Walker trails, sessions, and caches reset. +- `/tmp` is the only writable filesystem on Lambda; JSON DB defaults route there. + +### 5.4 Security + +- Secrets are never logged. 500 error responses are sanitized. +- Comparison of any secret, key, token, or hash uses `hmac.compare_digest`. This is non-negotiable. +- New auth flows, key types, or token formats require a security review (see [SECURITY.md](SECURITY.md) and `docs/md/security-review.md`). + +### 5.5 Compatibility + +- The `jvspatial/__init__.py` `__all__` export list is the public contract. Submodule import paths are *not* stable; promote symbols through the top-level package. +- Breaking changes to public API require a major version bump (post-1.0) or a minor bump (pre-1.0) with a `**BREAKING**` callout in [CHANGELOG.md](CHANGELOG.md). +- Deprecations follow the policy in [docs/md/stability.md](docs/md/stability.md): mark with `@deprecated`, warn for at least one minor cycle, then remove. + +--- + +## 6. Success Criteria + +### 6.1 Qualitative + +- **A caller building a new graph-backed service can get to a working CRUD endpoint in under 30 minutes**, following [docs/md/quick-start-guide.md](docs/md/quick-start-guide.md) and one of the [examples/api/](examples/api/) reference implementations, without reading any source code. +- **A caller can switch backends (JSON → SQLite → MongoDB → DynamoDB) without rewriting queries or entities** — only configuration changes. +- **A caller deploying to Lambda does not need to write Lambda-specific code**, beyond the entry adapter for their function. Defaults adapt; deferred work has a single registration point. +- **An AI agent maintaining the library can locate the right code path from a single CLAUDE.md read**, without having to crawl the docs tree. +- **A security reviewer can audit the trust boundary by reading SPEC §15** and confirm every claim against `file:line` citations. + +### 6.2 Quantitative + +Quantitative budgets are deferred to the upcoming foundation audit. Placeholders that will be filled in during that pass: + +- p50 / p99 latency budgets for `Entity.get`, `Entity.find`, `Walker.walk` against each backend. +- Cold-start envelope on Lambda (target: well under the platform's 1s default timeout for HTTP API + Lambda). +- Memory footprint per active walker. +- Test-coverage gate (current floor: 60%; CHANGELOG implies a tighter floor is desired but not yet enforced). +- Benchmark regression detection: the existing pytest-benchmark suite must remain green in CI. + +--- + +## 7. Stability Tiers (Ratification) + +The library distinguishes three tiers; [docs/md/stability.md](docs/md/stability.md) is the canonical reference. + +| Tier | Contract | Examples | +|---|---|---| +| **Stable / Public** | Names in `jvspatial.__all__`. Breaking changes require a major version bump (post-1.0) or a minor bump with a `**BREAKING**` callout (pre-1.0). At least one deprecation cycle before removal. | `Object`, `Node`, `Edge`, `Walker`, `Root`, `GraphContext`, `Server`, `ServerConfig`, `Database`, `@endpoint`, `@attribute`, `DeferredSaveMixin`, `is_serverless_mode`, `dispatch_deferred_task`, `MetricsRecorder`. | +| **Internal** | Module path begins with `_` or is documented internal. Can change in any release without notice. Callers should not import directly. | `jvspatial/db/_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`; `jvspatial/api/server_*.py`; `jvspatial/runtime/lwa.py`. | +| **Experimental** | Public but opt-in. May change or be removed in any minor release. Marked via `@experimental` decorator, emits a once-per-process warning. | `JsonDBTransaction(best_effort=True)`. | + +The PRD ratifies this tiering. Promotion of a feature from experimental to stable, or demotion of an internal helper to public, is a product decision and must be reflected here as well as in `docs/md/stability.md`. + +--- + +## 8. Decision Boundaries + +When evaluating proposed changes, apply these decision rules: + +| Question | Answer | +|---|---| +| Does this change introduce a new sync API on a code path that touches I/O? | **Reject.** Async-only is a core constraint (§5.2). | +| Does this change broaden the CORS default to a wildcard, or remove a constant-time comparison? | **Reject.** Security defaults must fail closed (§5.4). | +| Does this change add a feature that duplicates an existing public API? | **Reject** unless the existing API is being deprecated in the same change. | +| Does this change add a new env var? | **Allowed**, but must be added to the `JVSPATIAL_*` allowlist (`jvspatial/env_adapter.py`) and documented in [docs/md/environment-keys-reference.md](docs/md/environment-keys-reference.md). | +| Does this change add a new public name? | **Allowed**, but must be added to `__all__` and [docs/md/stability.md](docs/md/stability.md). | +| Does this change extend the database ABC contract? | **Allowed** only if all four built-in adapters implement it or fall through a sensible default. | +| Does this change couple jvspatial to a specific upstream consumer? | **Reject.** jvspatial is a foundation; consumer-specific logic lives in the consumer. | + +--- + +## 9. Open Questions + +Items the PRD does not yet decide. These are inputs to the upcoming foundation audit. + +- Quantitative success criteria (§6.2 placeholders). +- Whether session state, rate-limit counters, and JWT blacklists should have first-party shared-storage adapters or remain per-worker by default. +- Whether the multi-database manager should grow first-class tenant scoping primitives or remain a thin registry. +- Whether `count()` semantics should guarantee consistency vs. allow eventual count from DynamoDB. +- Whether the experimental `JsonDBTransaction(best_effort=True)` should be promoted, demoted, or removed. + +--- + +## 10. Living-Document Discipline + +This PRD is authored to remain accurate over time. Update rules: + +1. Adding a public API → update §3 (value props) and §7 (stability tiers). +2. Adding a non-goal → update §4 with one-line justification. +3. Changing a constraint → update §5 and reflect in [SPEC.md](SPEC.md) where contract changes. +4. Closing an open question → move from §9 into the appropriate section. + +Any code change that contradicts a current statement here must update this file in the same commit, or document an exception in [ROADMAP.md](ROADMAP.md). diff --git a/README.md b/README.md index 23348ee..67c4084 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,12 @@ server = Server( - [Security Policy](https://github.com/TrueSelph/jvspatial/blob/main/SECURITY.md) - Vulnerability disclosure - [Code of Conduct](https://github.com/TrueSelph/jvspatial/blob/main/CODE_OF_CONDUCT.md) - Contributor Covenant 2.1 +## Authors & maintainers + +jvspatial — a foundational object-spatial application development framework — was created by **Eldon Marks** ([@eldonm](https://github.com/eldonm)), who serves as its lead maintainer. + +See [AUTHORS](https://github.com/TrueSelph/jvspatial/blob/main/AUTHORS) for the full list of authors and contributors. Copyright and licensing terms are set out in the [LICENSE](https://github.com/TrueSelph/jvspatial/blob/main/LICENSE). + ## Contributors

diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..39e9107 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,154 @@ +# jvspatial — Roadmap + +> **Purpose**: Forward-looking maintenance direction. Captures current focus, known gaps and tech debt, areas under active hardening, and the release process. Not a feature wishlist — every entry has a citation or a justification. +> +> **Audience**: Maintainers planning the next pass. AI agents looking for "what should I work on next" should start here. +> +> **Status**: Living document. Update when scope shifts, when a known gap is closed, or when a new area enters active hardening. + +--- + +## 1. Current Focus + +Derived from recent commit trajectory ([CHANGELOG.md](CHANGELOG.md), `git log`). + +### 1.1 Security hardening (active) + +Recent waves have systematically eliminated constant-time-comparison gaps, fail-open auth defaults, and CSP holes. Continuing work: + +- Maintain the audit cadence from `docs/md/security-review.md`. Each new auth flow or secret-touching path requires a review entry. +- Keep `JVSPATIAL_DOCS_DISABLED` as the standard production posture and ensure new app routes inherit strict CSP. +- Per-path CSP / docs gating: monitor for new routes added under `/docs` or `/openapi.json` namespaces. + +### 1.2 Performance and bulk APIs (active) + +- `find_many`, `bulk_save`, native `count()` pushdown landed in 0.0.7. Continue: +- Native bulk-update equivalents on backends that support them (Mongo `bulk_write`, SQLite `executemany UPDATE`). +- Continue surfacing slow-query telemetry via the observability layer; tune `slow_query_ms` defaults per backend. + +### 1.3 Observability (active) + +- `MetricsRecorder` protocol and OTEL adapter shipped in 0.0.7. Continue: +- Expand the per-op metric set (cache hits, hook latencies, walker step counts) beyond DB ops. +- Validate OTEL adapter against a real OTLP collector and add an integration test. + +### 1.4 Stability contract enforcement (active) + +- The public / internal / experimental tiers in `docs/md/stability.md` are now defined. Continue: +- Audit every public name in `jvspatial/__init__.py` for tier annotation. +- Wire `@experimental` warnings to any opt-in surface still missing them. + +--- + +## 2. Known Gaps and Tech Debt + +Items called out by SPEC audit (this docs pass) and prior work. Each carries a code-path citation. + +### 2.1 Schema migration story is implicit + +No migration framework. Field removals or renames break existing records silently. Adapters do not enforce schemas. + +- **Decision needed**: ship a thin migration helper or document the per-application pattern. +- **Location**: cross-cutting; would touch `jvspatial/core/entities/object.py` field handling and `jvspatial/db/*` adapters. + +### 2.2 Multi-database transactions are best-effort only + +Only MongoDB supports ACID transactions. Cross-backend operations have no built-in coordination. + +- **Decision needed**: document the limit explicitly in handler patterns or build an opt-in saga primitive. +- **Location**: `jvspatial/db/transaction.py`, `jvspatial/db/database.py:48+`. + +### 2.3 Session state, rate-limit counters, JWT blacklist are per-worker + +In multi-worker deployments, configured limits multiply by worker count. Cross-worker invalidation requires a shared store the library does not ship. + +- **Decision needed**: provide first-party Redis-backed adapters for these stores or remain caller-owned. +- **Location**: `jvspatial/api/auth/enhanced.py` (SessionManager), `jvspatial/api/middleware/rate_limit*.py`. + +### 2.4 `count()` consistency varies by backend + +DynamoDB `count()` is eventually consistent; MongoDB and SQLite are strongly consistent; JsonDB is read-at-call-time. + +- **Decision needed**: document the divergence per adapter, or add a `consistency=` argument. +- **Location**: `jvspatial/db/database.py:144`, per-backend overrides. + +### 2.5 Walker trail does not persist across serverless invocations + +By design (in-memory only). Long-running traversals across cold-starts cannot resume. + +- **Decision needed**: add an optional persistent-trail adapter or accept the limit explicitly. +- **Location**: `jvspatial/core/entities/walker_components/walker_trail.py`. + +### 2.6 Quantitative success criteria are unset + +[PRD §6.2](PRD.md#62-quantitative) lists placeholders for latency, cold-start, and memory budgets. The upcoming foundation audit must fill these in. + +### 2.7 Legacy `LLM-CODING-GUIDE.md` overlap with `docs/md/` + +The legacy file contains code patterns that duplicate parts of `docs/md/quick-start-guide.md`, `entity-reference.md`, and others. Consider deprecating sections that are fully covered elsewhere. + +- **Location**: [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md). + +### 2.8 Python 3.8 declared but not really supported + +`pyproject.toml` classifiers include Python 3.8, but several features (notably `typing.Literal` and `ParamSpec` usage) prefer 3.9+. Either confirm 3.8 support with CI coverage or drop the classifier. + +- **Location**: `pyproject.toml`. + +### 2.9 Experimental status of `JsonDBTransaction(best_effort=True)` unresolved + +[PRD §9](PRD.md#9-open-questions) flags this. Promote, demote, or remove. + +- **Location**: `jvspatial/db/transaction.py`. + +--- + +## 3. Areas Under Active Hardening + +Code paths receiving sustained attention. Changes here warrant elevated review. + +| Area | Why | Recent changes | +|---|---|---| +| `jvspatial/api/auth/*` | Auth touches every request; security regressions are high-blast-radius. | 0.0.7 hardening wave; ongoing per-flow reviews. | +| `jvspatial/storage/security/*` | File-upload trust boundary; path traversal is a known threat surface. | Internal-marker handling, validator extension to markdown files. | +| `jvspatial/db/_atomic.py`, `_path_locks.py` | Crash safety and concurrent-write correctness. | 0.0.7 atomic-write introduction; benchmark coverage added. | +| `jvspatial/runtime/serverless.py` | Detection precedence + mode-dependent defaults; subtle changes have wide effects. | 0.0.7 LWA env defaults, bcrypt rounds adjustment. | +| `jvspatial/db/_observable.py`, `jvspatial/observability/*` | Public log-field schema is part of the stability contract. | 0.0.7 introduction; field-set changes require deprecation. | +| `jvspatial/core/entities/walker.py` and `walker_components/*` | Protection limits prevent DOS; subclass extension surface. | Ongoing; trail-tracking refinements. | + +--- + +## 4. Versioning and Release Policy + +Authoritative process lives in [RELEASING.md](RELEASING.md). Summary: + +- Version is read from `jvspatial/version.py`; the release workflow auto-creates the tag. +- Breaking changes pre-1.0 require a `**BREAKING**` callout in [CHANGELOG.md](CHANGELOG.md) and a one-cycle deprecation where reasonable. +- Deprecation pattern: `@deprecated(replacement=..., remove_in=...)` from `jvspatial/utils/deprecation.py`. Emits a once-per-process `DeprecationWarning`. +- Removal happens at the earliest in the second minor release after deprecation. + +Stability tier reference: [docs/md/stability.md](docs/md/stability.md). + +--- + +## 5. Out-of-Scope (Reaffirmed) + +These items appeared in past planning discussions and remain out of scope. Listed so they are not reopened without a deliberate decision. + +| Item | Rejected because | +|---|---| +| Built-in graph query language (Cypher / Gremlin equivalent) | Walkers are imperative async Python by design (PRD §4). | +| Synchronous API surface | Library is async-only (PRD §5.2). | +| Built-in dashboard / web UI | Visualization helpers (DOT, Mermaid) are sufficient; rendering is downstream. | +| First-party multi-tenancy primitives | Tenant isolation is caller-owned through context scoping. | + +--- + +## 6. How to Use This Roadmap + +- **Adding a new gap**: include the code-path citation and the decision needed. Do not list aspirations without a justification. +- **Closing a gap**: remove the entry from §2 in the same commit that lands the fix. Add a corresponding line to [CHANGELOG.md](CHANGELOG.md). +- **Promoting a focus area**: move from §2 (Gaps) into §1 (Current Focus) when sustained work begins. +- **Reaffirming out-of-scope**: append to §5 with the rejecting rationale. + +The roadmap is for orientation. It is not a backlog and not a contract. Commitments live in CHANGELOG and milestones. diff --git a/SPEC.md b/SPEC.md index a80c749..29a2a84 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,5284 +1,785 @@ -# jvspatial Language Model Coding Guide +# jvspatial — Technical Specification + +> **Purpose**: Authoritative technical contract for `jvspatial`. Describes what the library guarantees, what callers must not assume, and where the source of truth for each behavior lives. Cite this document by section number when justifying changes. +> +> **Scope**: Library behavior as of `jvspatial/version.py` → see file. This is a living spec; when behavior changes, this file must change in the same commit. +> +> **Companion documents**: +> - [PRD.md](PRD.md) — *why* the library exists (product context, users, success criteria) +> - [ROADMAP.md](ROADMAP.md) — forward-looking direction and known gaps +> - [CLAUDE.md](CLAUDE.md) — operational guidance for AI agents maintaining this repo +> - [docs/md/README.md](docs/md/README.md) — index of detailed how-to documentation +> - [LLM-CODING-GUIDE.md](LLM-CODING-GUIDE.md) — usage cookbook (code patterns for callers) -This document provides concise guidance for AI language models to generate code that follows jvspatial library standards and conventions. - -## 🎯 Core Philosophy - -jvspatial emphasizes **entity-centric design** with unified MongoDB-style queries across database backends (JSON, MongoDB). The library distinguishes between: - -- **Objects** - For standalone data entities (users, settings, logs) that don't require graph relationships -- **Nodes** - For graph entities that are interconnected by Edges and traversed by Walkers -- **Edges** - For relationships between Nodes in the graph -- **Walkers** - For traversing and processing graph structures - -**Key Principle**: Use Objects for simple data storage, use Nodes when you need graph traversal and relationships. - -## 🔧 Environment Setup - -### Essential Configuration -```python -from dotenv import load_dotenv -load_dotenv() # Load .env file - -from jvspatial.core import GraphContext -from jvspatial.db import create_database, get_database_manager - -# Option 1: Create database explicitly -db = create_database("json", base_path="./jvdb") -ctx = GraphContext(database=db) - -# Option 2: Use current database from manager (defaults to prime) -manager = get_database_manager() -ctx = GraphContext(database=manager.get_current_database()) -``` - -### Environment Variables -```env -# Choose backend -JVSPATIAL_DB_TYPE=json # or 'mongodb' - -# JSON backend (default) -JVSPATIAL_JSONDB_PATH=./jvdb/dev - -# MongoDB backend -JVSPATIAL_MONGODB_URI=mongodb://localhost:27017 -JVSPATIAL_MONGODB_DB_NAME=jvspatial_dev - -# Caching (optional) -JVSPATIAL_CACHE_BACKEND=memory # 'memory', 'redis', or 'layered' -JVSPATIAL_CACHE_SIZE=1000 # Max cached entities (0 to disable) -``` - -## 📝 Entity-Centric Code Patterns - -### Objects vs Nodes: When to Use Each - -**✅ Objects** - For standalone entities without graph relationships: -```python -from jvspatial.core import Object - -class UserProfile(Object): - name: str = "" - email: str = "" - settings: Dict[str, Any] = {} - -# Use for: user profiles, configuration, logs, simple data -profile = await UserProfile.create(name="Alice", email="alice@company.com") -``` - -**✅ Nodes** - For graph entities with relationships and traversal: -```python -from jvspatial.core import Node - -class User(Node): - name: str = "" - department: str = "" - -class City(Node): - name: str = "" - population: int = 0 - -# Use for: entities that connect to other entities via Edges -user = await User.create(name="Alice", department="engineering") -city = await City.create(name="San Francisco", population=800000) -``` - -### Entity Operations -```python -# Entity creation (no save() needed, automatically cached) -entity = await Entity.create(name="value", field="data") - -# Entity retrieval (uses cache after first access) -entity = await Entity.get(entity_id) # Cached by ID -entities = await Entity.find({"context.active": True}) # Not cached - -# Entity updates (save() only needed after property modification, updates cache) -entity = await Entity.get(entity_id) -entity.name = "Updated Name" # Property modified -await entity.save() # save() required to persist changes + update cache - -# Entity deletion (removes from cache) -await entity.delete() - -# Counting and aggregation (not cached) -# Note: Object.count() doesn't exist - use len() with find() instead -results = await Entity.find({"context.department": "engineering"}) -count = len(results) - -# For distinct values, query and extract manually -all_entities = await Entity.find({}) -departments = set(e.department for e in all_entities if hasattr(e, 'department')) -``` - -**Note**: Caching is automatic and transparent. Individual entity retrievals by ID (`Entity.get(id)`) are cached. Queries (`find()`, `all()`) always hit the database as they can change frequently. For counting, use `Entity.count(query)` for efficient counting without loading all records. - -### save() Operation Rules -**✅ save() is ONLY required when:** -1. You modify entity properties after retrieval: `entity.field = "new_value"` -2. You create entities without using `.create()` method - -**❌ save() is NOT needed when:** -1. Using `.create()` method (automatically persists) -2. Using `.delete()` method (automatically persists deletion) -3. Just reading/querying entities - -**❌ AVOID: Direct database access (use entity methods instead)** -```python -# Don't do this - use entity methods instead -from jvspatial.db import create_database -db = create_database("json") -entities = await db.find("object", {"name": "Entity"}) - -# ✅ Do this instead - use entity-centric methods -entities = await Entity.find({"context.name": "Entity"}) -``` - -## 🗄️ Multi-Database Support - -jvspatial supports managing multiple databases within the same application, with a prime database for core persistence operations (authentication, session management) and additional databases for application-specific data. - -### Basic Multi-Database Usage - -```python -from jvspatial.db import ( - create_database, - get_database_manager, - get_prime_database, - get_current_database, - switch_database, - unregister_database, -) -from jvspatial.core.context import GraphContext - -# Get database manager (singleton) -manager = get_database_manager() - -# Prime database is automatically created for core operations -prime_db = get_prime_database() # Used for auth, sessions, system data - -# Create and register additional database -app_db = create_database( - "json", - base_path="./app_data", - register=True, - name="app" -) - -# Switch to application database -switch_database("app") -current_db = get_current_database() # Now returns app_db - -# Use with GraphContext -app_ctx = GraphContext(database=current_db) - -# Switch back to prime database -switch_database("prime") - -# Unregister non-prime database when no longer needed -unregister_database("app") -``` - -### Prime Database - -The prime database is always used for: -- User authentication -- Session management -- System-level configuration -- Core persistence operations - -It cannot be unregistered and is always available as the default. - -### Database Isolation - -Each database maintains complete isolation: -- Entities in one database are not visible in another -- Switching databases changes the context for all operations -- Prime database ensures core operations always have a stable database - -**📖 For comprehensive multi-database documentation:** [Graph Context Guide](docs/md/graph-context.md) and [Multi-Database Example](examples/database/multi_database_example.py) - -## 🔧 Custom Database Integration - -jvspatial supports seamless extension with custom database backends through a registration system. - -### Registering Custom Database Types - -```python -from jvspatial.db import Database, register_database_type, create_database, list_database_types - -# Define custom database implementation -class CustomDatabase(Database): - async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: - # Implementation - pass - - async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: - # Implementation - pass - - async def delete(self, collection: str, id: str) -> None: - # Implementation - pass - - async def find(self, collection: str, query: Dict[str, Any]) -> List[Dict[str, Any]]: - # Implementation - pass - -# Factory function for creating instances -def create_custom_db(**kwargs: Any) -> CustomDatabase: - return CustomDatabase(**kwargs) - -# Register the custom database type -register_database_type("custom", create_custom_db) - -# Now use it like built-in types -db = create_database("custom", connection_string="custom://...") - -# List all available database types -types = list_database_types() -# Returns: {"json": "JSON file-based database", "mongodb": "MongoDB database", "custom": "Custom database: create_custom_db"} -``` - -### Custom Database Requirements - -Custom databases must: -1. Inherit from `Database` abstract base class -2. Implement all abstract methods: `save()`, `get()`, `delete()`, `find()` -3. Optionally implement `count()` and `find_one()` (default implementations provided) -4. Provide a factory function for creation - -**📖 For comprehensive custom database documentation:** [Custom Database Guide](docs/md/custom-database-guide.md) and [Custom Database Example](examples/database/custom_database_example.py) - -## 🔍 MongoDB-Style Query Patterns - -Always use dot notation for nested fields with `context.` prefix: - -```python -# Comparison operators -users = await User.find({"context.age": {"$gte": 35}}) -users = await User.find({"context.role": {"$ne": "admin"}}) - -# Logical operators -users = await User.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.active": True} - ] -}) - -# Array operations -users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) - -# Regular expressions -users = await User.find({ - "context.name": {"$regex": "Johnson", "$options": "i"} -}) -``` - -## 🔒 Attribute Annotations (Protected & Transient) - -jvspatial provides `@protected` and `@transient` decorators for controlling attribute behavior: - -### Protected Attributes -Protected attributes cannot be modified after initialization (ideal for IDs and immutable config): - -```python -from pydantic import Field -from jvspatial.core.annotations import protected - -class Entity(Node): - # id is already protected in Node - uuid: str = protected("", description="Immutable UUID") - created_at: datetime = protected(Field(default_factory=datetime.now)) - -# ✓ Can set during initialization -entity = await Entity.create(uuid="abc-123") - -# ✗ Cannot modify after creation -entity.uuid = "new-uuid" # Raises AttributeProtectionError -``` - -### Transient Attributes -Transient attributes are excluded from database exports (ideal for runtime caches): - -```python -from jvspatial.core.annotations import transient - -class Entity(Node): - data: str = "" - cache: dict = transient(Field(default_factory=dict)) # Not persisted - temp_count: int = transient(Field(default=0)) # Not persisted - -entity.cache["key"] = "value" # Works at runtime -data = await entity.export() # cache excluded from export -``` +--- -### Compound Decorators -Combine both for internal state that's neither modifiable nor persisted: +## Table of Contents + +1. [Identity Model](#1-identity-model) +2. [Entity Hierarchy](#2-entity-hierarchy) +3. [Async Contract](#3-async-contract) +4. [Persistence Layer](#4-persistence-layer) +5. [Query Interface](#5-query-interface) +6. [Walker Traversal Semantics](#6-walker-traversal-semantics) +7. [GraphContext and Dependency Injection](#7-graphcontext-and-dependency-injection) +8. [API Surface](#8-api-surface) +9. [Authentication and Authorization](#9-authentication-and-authorization) +10. [Configuration and Environment](#10-configuration-and-environment) +11. [Serverless Constraints](#11-serverless-constraints) +12. [File Storage](#12-file-storage) +13. [Caching](#13-caching) +14. [Observability](#14-observability) +15. [Security Boundaries](#15-security-boundaries) +16. [Extension Points](#16-extension-points) +17. [Error Taxonomy](#17-error-taxonomy) +18. [Stability Tiers](#18-stability-tiers) -```python -# Both protected AND transient -_internal: dict = protected(transient(Field(default_factory=dict))) -``` +--- -**Key Points:** -- All `id` fields in `Object`, `Node`, `Edge`, and `Walker` are automatically protected -- Always use `Field(default_factory=dict)` syntax with `@transient` -### Private Attributes -Private attributes are excluded from serialization and database operations (ideal for internal state): +## 1. Identity Model -```python -from jvspatial.core.annotations import private +### 1.1 ID format -class Entity(Node): - _cache: dict = private(default_factory=dict) # Not serialized - _internal_counter: int = private(default=0) # Not serialized +All persistable entities carry an `id` field with the format: -entity._cache["key"] = "value" # Works at runtime -data = await entity.export() # _cache excluded from export -``` - -### Compound Decorators -Combine decorators for complex behaviors: -```python -# Private AND transient -_internal: dict = private(transient(Field(default_factory=dict))) ``` -- See [Attribute Annotations](docs/md/attribute-annotations.md) for full documentation - -## 🏢 Type Annotations & Error Handling - -### Required Typing Pattern -```python -from typing import List, Optional, Dict, Any -from jvspatial.core import Node, Object -from jvspatial.exceptions import NodeNotFoundError, ValidationError - -class User(Node): - name: str = "" - email: str = "" - age: int = 0 - roles: List[str] = [] - active: bool = True - metadata: Dict[str, Any] = {} - -async def get_user_by_id(user_id: str) -> Optional[User]: - """Get user by ID, returning None if not found.""" - try: - return await User.get(user_id) - except NodeNotFoundError: - return None +{type_code}.{entity_name}.{hex_id} ``` -### Error Handling Patterns +Where: -**🎯 Always catch specific exceptions first:** +- `type_code` ∈ `{n, e, w, o}` (Node, Edge, Walker, Object) +- `entity_name` is the persisted discriminator (see 1.2) +- `hex_id` is the first 24 hex chars of `uuid.uuid4().hex` -```python -import logging -from jvspatial.exceptions import ( - JVSpatialError, - ValidationError, - EntityNotFoundError, - NodeNotFoundError, - DatabaseError, - ConnectionError -) +**Source of truth**: `jvspatial/core/utils.py:11-22` (`generate_id`). -logger = logging.getLogger(__name__) - -# Entity operations with error handling -async def safe_user_operation(user_id: str) -> Optional[User]: - try: - user = await User.get(user_id) - return user - except NodeNotFoundError as e: - logger.warning(f"User not found: {e.entity_id}") - return None - except ValidationError as e: - logger.error(f"Validation failed: {e.message}") - if e.field_errors: - for field, error in e.field_errors.items(): - logger.error(f" {field}: {error}") - return None - except DatabaseError as e: - logger.error(f"Database error: {e.message}") - raise # Re-raise for higher-level handling - except JVSpatialError as e: - logger.error(f"jvspatial error: {e.message}") - return None -``` +**Invariants**: +- `id` is `protected=True` — set at construction, cannot be reassigned after `__init__` completes (`jvspatial/core/entities/object.py:46-48`). +- `id` collisions are not detected at insertion; callers depending on uniqueness must coordinate generation. UUID4 collision probability is the only practical guarantee. +- `type_code` is per-class default: `o` for `Object`, `n` for `Node` (`jvspatial/core/entities/node.py:43`), `e` for `Edge` (`jvspatial/core/entities/edge.py:41`), `w` for `Walker` (`jvspatial/core/entities/walker.py:118`). -**🔄 Database operations with fallback:** +### 1.2 Entity name discriminator (`__entity_name__`) -```python -from jvspatial.exceptions import ConnectionError, QueryError - -async def robust_user_search(query: Dict[str, Any]) -> List[User]: - try: - # Try complex query - return await User.find(query) - except QueryError as e: - logger.warning(f"Complex query failed: {e.message}") - # Fallback to simple query - try: - all_users = await User.all() - # Apply filtering in Python - return [u for u in all_users if u.active] - except Exception: - logger.error("All query methods failed") - return [] - except ConnectionError as e: - logger.error(f"Database connection failed: {e.database_type}") - return [] # Graceful degradation -``` - -**⚠️ Walker error handling:** +The `entity_name` segment of the ID, and the persisted `entity` field, default to `cls.__name__`. Subclasses may override by setting `__entity_name__` as a `ClassVar`: ```python -from jvspatial.exceptions import WalkerExecutionError, WalkerTimeoutError - -class SafeUserProcessor(Walker): - @on_visit(User) - async def process_user(self, here: User): - try: - # Potentially risky operation - result = await external_api_call(here) - self.report(result) - except Exception as e: - # Don't let individual errors stop traversal - logger.warning(f"Failed to process user {here.id}: {e}") - self.report({"error": str(e), "user_id": here.id}) - -async def run_safe_walker(): - try: - walker = SafeUserProcessor() - result = await walker.spawn(start_user) - - # Get results and errors - report = await result.get_report() - errors = [r for r in report if isinstance(r, dict) and "error" in r] - logger.info(f"Processed with {len(errors)} errors") - - except WalkerTimeoutError as e: - logger.error(f"Walker timed out after {e.timeout_seconds}s") - # Access partial results if needed - except WalkerExecutionError as e: - logger.error(f"Walker failed: {e.walker_class} - {e.message}") +class App(Node): + __entity_name__: ClassVar[Optional[str]] = "HostApp" ``` -## 📄 ObjectPager for Large Datasets - -### Basic Pagination -```python -from jvspatial.core.pager import paginate_objects, ObjectPager - -# Simple pagination -users = await paginate_objects(User, page=1, page_size=20) +**Source of truth**: `jvspatial/core/entities/object.py:35-44` (`_entity_name` classmethod). -# With filters -active_users = await paginate_objects( - User, - page=1, - page_size=10, - filters={"context.active": True} -) +**Resolution rule**: `cls.__dict__.get("__entity_name__") or cls.__name__`. Not inherited from a parent that set it — each subclass decides for itself. -# Advanced pager -pager = ObjectPager( - User, - page_size=25, - filters={"context.department": "engineering"}, - order_by="name" -) -users = await pager.get_page(1) -``` +**Use case**: disambiguating unrelated `Object` subtrees that share a Python class name (e.g. host-app `App` vs library-internal `App`) so they remain distinguishable at the storage layer. -## ⏰ Scheduler Integration +**Implications**: +- `find_subclass_by_name` (`jvspatial/core/utils.py:58-89`) honors the override and caches positive hits. Negative hits are not cached (avoids bootstrap poisoning when classes are imported later). +- The persisted `entity` field tracks `_entity_name()` at creation time, not class name. Renaming a class without `__entity_name__` will orphan existing records. -### Task Scheduling -```python -from jvspatial.core.scheduler import Scheduler, ScheduledTask -from datetime import datetime, timedelta - -# Create scheduler -scheduler = Scheduler() - -# Schedule recurring tasks -@scheduler.task(interval=timedelta(hours=1)) -async def cleanup_expired_sessions(): - """Clean up expired user sessions hourly.""" - expired = await UserSession.find({ - "context.expires_at": {"$lt": datetime.now()} - }) - for session in expired: - await session.delete() - print(f"Cleaned up {len(expired)} expired sessions") - -# Schedule one-time tasks -@scheduler.task(run_at=datetime.now() + timedelta(minutes=30)) -async def send_reminder_emails(): - """Send reminder emails.""" - users = await User.find({"context.reminder_due": True}) - for user in users: - await send_email(user.email, "Reminder") - user.reminder_due = False - await user.save() - -# Start scheduler -await scheduler.start() -``` +### 1.3 Root singleton -### Walker-Based Scheduled Tasks -```python -from jvspatial.core import Walker -from jvspatial.core.entities import on_visit - -@scheduler.walker_task(interval=timedelta(days=1)) -class DailyMaintenanceWalker(Walker): - """Perform daily maintenance tasks via graph traversal.""" - - @on_visit("User") - async def check_user_activity(self, here: Node): - """Check user activity and update status.""" - if here.last_active < datetime.now() - timedelta(days=30): - here.status = "inactive" - await here.save() - self.report({"deactivated_user": here.id}) - - @on_visit("DataNode") - async def cleanup_old_data(self, here: Node): - """Remove old data nodes.""" - if here.created_at < datetime.now() - timedelta(days=90): - await here.delete() - self.report({"deleted_data_node": here.id}) - -# Start scheduled walker -await scheduler.start_walker_task(DailyMaintenanceWalker) -``` +`Root` is the canonical entry node for a graph. Its ID is fixed: `n.Root.root` (`jvspatial/core/entities/root.py`). Created via an async lock to guarantee single instantiation per database. All graph traversals can originate from `Root` by convention; not enforced. -## 🌐 API Server with Server Class +--- -### Basic Server Setup -```python -from jvspatial.api import Server - -# Create server instance -server = Server( - title="My Spatial API", - description="Graph-based spatial data management API", - version="1.0.0", - host="0.0.0.0", - port=8000 -) +## 2. Entity Hierarchy -# Run server -if __name__ == "__main__": - server.run() ``` - -### Walker Endpoints -```python -from jvspatial.api import endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit - -@endpoint("/api/users/process", methods=["POST"]) -class ProcessUser(Walker): - """Process user data with graph traversal.""" - - user_name: str = EndpointField( - description="Name of user to process", - examples=["John Doe"] - ) - - department: str = EndpointField( - default="general", - description="User department" - ) - - @on_visit("User") - async def process_user(self, here: Node): - """Process user nodes - use 'here' for visited node.""" - if here.name == self.user_name: - self.report({ - "found_user": { - "id": here.id, - "name": here.name, - "department": here.department - } - }) - - # Get connected nodes and continue traversal - colleagues = await here.nodes( - node=['User'], - department=self.department - ) - await self.visit(colleagues) +AttributeMixin + pydantic.BaseModel + │ + Object ─────── (type_code="o") + │ │ + │ ├── Node ─────── (type_code="n") ─── Root (singleton) + │ │ + │ └── Edge ─────── (type_code="e") + │ + Walker (AttributeMixin + BaseModel, not Object) ── (type_code="w") ``` -### Function Endpoints -```python -from jvspatial.api import endpoint - -@endpoint("/api/users/count", methods=["GET"]) -async def get_user_count() -> Dict[str, int]: - """Get total user count.""" - users = await User.all() - return {"total_users": len(users)} - -@endpoint("/api/users/{user_id}", methods=["GET"]) -async def get_user(user_id: str, endpoint) -> Any: - """Get user with semantic response.""" - user = await User.get(user_id) - if not user: - return endpoint.not_found( - message="User not found", - details={"user_id": user_id} - ) - - return endpoint.success( - data={"id": user.id, "name": user.name, "email": user.email} - ) -``` +### 2.1 Object — base persistable entity -### Server with Scheduler Integration -```python -from jvspatial.api import Server -from jvspatial.core.scheduler import Scheduler - -# Create integrated server with scheduler -server = Server(title="Scheduled API", port=8000) -scheduler = Scheduler() - -# Add scheduled tasks -@scheduler.task(interval=timedelta(minutes=5)) -async def periodic_health_check(): - """Check system health every 5 minutes.""" - # Health check logic here - pass - -@server.on_startup -async def startup_tasks(): - """Start scheduler when server starts.""" - await scheduler.start() - print("✅ Server and scheduler started") - -@server.on_shutdown -async def shutdown_tasks(): - """Stop scheduler when server shuts down.""" - await scheduler.stop() - print("🛑 Server and scheduler stopped") - -# Run integrated server -if __name__ == "__main__": - server.run() -``` +`jvspatial/core/entities/object.py`. Provides: +- `id`, `entity`, `type_code` fields +- CRUD methods: `create()`, `get()`, `find()`, `save()`, `delete()`, `count()`, `export()` +- Context lookup: `set_context()`, `get_context()` (default via `get_default_context()`) +- Collection mapping via `get_collection_name()` → `{n: node, e: edge, o: object, w: walker}` -## 🔗 Webhook Integration +**Invariant**: `__setattr__` validates field names against the class hierarchy. Setting an undeclared attribute on an `Object` (post-init) is rejected. Prevents schema injection through attribute assignment. -### Basic Webhook Handler -```python -from jvspatial.api import endpoint -from fastapi import Request - -@endpoint("/webhook/{service}/{auth_token}", methods=["POST"], webhook=True) -async def webhook_handler(request: Request) -> Dict[str, Any]: - """Process webhooks with automatic payload parsing.""" - raw_body = request.state.raw_body - content_type = request.state.content_type - current_user = get_current_user(request) - - # Always return 200 for webhooks - try: - # Process webhook logic here - return {"status": "success", "processed_at": datetime.now().isoformat()} - except Exception as e: - logger.error(f"Webhook error: {e}") - return {"status": "received", "error": "logged"} -``` +### 2.2 Node — graph node -## 🏗️ Core Architecture & Walker Patterns +`jvspatial/core/entities/node.py:34+`. Adds: +- `edge_ids: List[str]` — transient in memory, persisted at the top level as `edges` (`Node._get_top_level_fields` line 55-60) +- `_visitor_ref: weakref` — currently visiting walker, transient +- `_visit_hooks: ClassVar` — populated by `__init_subclass__` from `@on_visit`-decorated methods (line 62+) -### Entity Hierarchy -- **Object** - Base class for all entities with unified query interface -- **Node** - Graph nodes with spatial/contextual data (extends Object) -- **Edge** - Relationships between nodes -- **Walker** - Graph traversal and processing logic -- **GraphContext** - Low-level database interface (use sparingly) +**Hooks**: methods decorated with `@on_visit(WalkerType | "string_name")` are registered per-class at class-creation time. String names allow forward references; resolved when the walker visits. -### Walker Traversal (CRITICAL PATTERNS) +### 2.3 Edge — graph relationship -#### Naming Convention for @on_visit Methods -**ALWAYS** use these parameter names: -- **`here`** - The visited node/edge (current location) -- **`visitor`** - The visiting walker (when accessing from node context) +`jvspatial/core/entities/edge.py:29+`. Top-level persisted fields: `source: str`, `target: str`, `bidirectional: bool = True` (line 42-49). -```python -@on_visit("User") -async def process_user(self, here: Node): - """Use 'here' for visited node.""" - connected_users = await here.nodes(node=['User']) - await self.visit(connected_users) - -@on_visit("City") -async def process_city(self, here: Node): - """Process cities with filtering.""" - # Skip small cities - if here.population < 10000: - self.skip() # Skip to next node - return - - # Get large connected cities - large_cities = await here.nodes( - node=['City'], - population={"$gte": 500000} - ) - await self.visit(large_cities) -``` +**Direction**: `await edge.direction` returns `"both"` if `bidirectional`, else `"out"` (line 57-64). Asymmetric edges flow source → target. -#### Walker Control Flow -```python -class DataWalker(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.max_items = 100 - - @on_visit("Document") - async def process_document(self, here: Node): - """Process with control flow.""" - # Skip invalid documents - if here.status == "invalid": - self.skip() # Continue to next node - - # Stop at limit - if self.processed_count >= self.max_items: - await self.disengage() # Permanently halt walker - return - - # Pause for rate limiting - if self.processed_count % 50 == 0: - self.pause("Rate limit pause") - - # Normal processing - self.processed_count += 1 - next_docs = await here.nodes(node=['Document']) - await self.visit(next_docs) - - @on_exit - async def cleanup(self): - """Called when walker completes/pauses/disengages.""" - print(f"Processed {self.processed_count} documents") -``` +### 2.4 Walker — traversal agent -## 📋 Quick Reference Checklist +`jvspatial/core/entities/walker.py:83+`. Walkers are **not** `Object` subclasses; they share `AttributeMixin` + `BaseModel`. Walkers are **not persisted to the database** by default — they exist for the lifetime of a traversal. -### Entity Operations -- ✅ Use `await Entity.create(**kwargs)` -- ✅ Use `await Entity.find(query_dict)` -- ✅ Use `await entity.save()` only after property modification -- ✅ Use Objects for standalone data, Nodes for graph entities -- ✅ Use `await node1.disconnect(node2)` to remove connections -- ❌ Avoid direct GraphContext database calls +**Components** (composition, in `walker_components/`): +- `WalkerTrail` — visited node IDs, edge IDs, timestamps, optional per-step metadata +- `TraversalProtection` — enforces step/visit/time/queue limits +- `WalkerQueue` — FIFO queue with `max_queue_size` cap +- `WalkerEventSystem` — in-memory event bus for traversal lifecycle -### Disconnecting Nodes -To remove connections between nodes, use the `disconnect()` method. This removes edges between nodes and deletes the edge objects. +**Default protection limits** (Walker class attrs): +- `max_steps`: 10 000 +- `max_visits_per_node`: 100 +- `max_execution_time`: 300 seconds +- `max_queue_size`: 1 000 +- `protection_enabled`: True -```python -# Disconnect two nodes -success = await node1.disconnect(node2) +These are *defaults* — subclasses or callers can override. Disabling protection (`protection_enabled=False`) is allowed but strongly discouraged in untrusted code paths. -# Disconnect with specific edge type -success = await node1.disconnect(node2, edge_type=SpecialEdge) -``` +### 2.5 Attribute system -### Query Patterns -- ✅ Use `"context.field"` dot notation for nested fields -- ✅ Use MongoDB operators: `$gte`, `$in`, `$regex`, `$and`, `$or` -- ✅ Combine dict filters with kwargs: `node=[{'User': {...}}], active=True` - -### Walker Patterns -- ✅ Use `here` parameter for visited nodes -- ✅ Use `await here.nodes()` to get connected nodes -- ✅ Use `await self.visit(nodes)` to continue traversal -- ✅ Use `self.skip()` to skip current node -- ✅ Use `await self.disengage()` to permanently halt -- ✅ Use `self.pause()` for temporary suspension - -### API Patterns -- ✅ Use `@endpoint` for both graph processing and simple functions -- ✅ Use `EndpointField` for parameter configuration -- ✅ Use `endpoint.success()`, `endpoint.not_found()` for responses -- ✅ Always return 200 for webhooks with try/catch - -### Type Safety -- ✅ Always include proper type annotations -- ✅ Import from `typing` and `jvspatial.exceptions` -- ✅ Handle `NodeNotFoundError`, `ValidationError`, `DatabaseError` -- ✅ Use structured error logging - -This guide provides the essential patterns for generating jvspatial-compliant code. Focus on entity-centric operations, proper typing, and following the established naming conventions for walker traversal. - -The **jvspatial webhook system** provides secure, flexible webhook endpoints with built-in authentication, HMAC verification, idempotency keys, and automatic payload processing. Webhooks integrate seamlessly with the FastAPI server and support both function-based handlers and graph traversal processing. - -### Webhook Architecture Overview - -Webhooks in jvspatial are designed for: -- **Security**: Path-based authentication tokens, optional HMAC signature verification -- **Reliability**: Idempotency key support to handle duplicate deliveries -- **Flexibility**: JSON/XML/binary payload support with automatic parsing -- **Integration**: Full compatibility with existing authentication and permission systems -- **Processing**: Always return HTTP 200 for proper webhook etiquette - -### Basic Webhook Endpoint Setup - -```python path=null start=null -from fastapi import Request -from jvspatial.api import endpoint -from jvspatial.api.auth.middleware import get_current_user -from typing import Dict, Any -import json - -# Basic webhook handler function -@endpoint("/webhook/{route}/{auth_token}", methods=["POST"], webhook=True) -async def generic_webhook_handler(request: Request) -> Dict[str, Any]: - """Generic webhook handler for multiple services. - - Processes webhooks from various sources using route-based dispatch. - Middleware handles authentication, HMAC verification, and payload parsing. - """ - # Access processed data from middleware - raw_body = request.state.raw_body # Original bytes - content_type = request.state.content_type # Content-Type header - route = getattr(request.state, "webhook_route", "unknown") # Route parameter - current_user = get_current_user(request) # Authenticated user - - # Parse payload based on content type - processed_data = None - if content_type == "application/json": - try: - processed_data = json.loads(raw_body) - except json.JSONDecodeError: - return {"status": "error", "message": "Invalid JSON payload"} - else: - # Handle other content types (XML, form data, binary) - processed_data = {"raw_length": len(raw_body), "type": content_type} - - # Route-based processing - if route == "stripe": - return await process_stripe_webhook(processed_data, current_user) - elif route == "github": - return await process_github_webhook(processed_data, current_user) - elif route == "slack": - return await process_slack_webhook(processed_data, current_user) - else: - # Generic processing for unknown routes - return { - "status": "received", - "route": route, - "payload_type": content_type, - "user_id": current_user.id if current_user else None - } - -# Service-specific webhook handlers -@endpoint("/webhook/stripe/{auth_token}", methods=["POST"], webhook=True) -async def stripe_webhook_handler(request: Request) -> Dict[str, Any]: - """Dedicated Stripe webhook handler with event processing.""" - raw_body = request.state.raw_body - current_user = get_current_user(request) - - try: - event = json.loads(raw_body) - event_type = event.get("type", "unknown") - - # Process different Stripe event types - if event_type == "payment_intent.succeeded": - await handle_successful_payment(event["data"]["object"], current_user) - elif event_type == "customer.subscription.updated": - await handle_subscription_update(event["data"]["object"], current_user) - elif event_type == "invoice.payment_failed": - await handle_payment_failure(event["data"]["object"], current_user) - - return { - "status": "success", - "event_type": event_type, - "processed_at": datetime.now().isoformat() - } - - except Exception as e: - # Always return 200 for webhooks, log errors internally - print(f"Stripe webhook processing error: {e}") - return {"status": "received", "error": "Processing error logged"} - -# Helper functions for webhook processing -async def process_stripe_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process Stripe webhook events.""" - event_type = data.get("type", "unknown") - return { - "status": "success", - "message": f"Processed Stripe {event_type}", - "user_id": user.id if user else None - } - -async def process_github_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process GitHub webhook events.""" - action = data.get("action", "unknown") - repo_name = data.get("repository", {}).get("name", "unknown") - return { - "status": "success", - "message": f"GitHub {action} on {repo_name}", - "user_id": user.id if user else None - } - -async def process_slack_webhook(data: Dict[str, Any], user) -> Dict[str, Any]: - """Process Slack webhook events.""" - event_type = data.get("type", "unknown") - return { - "status": "success", - "message": f"Slack {event_type} processed", - "user_id": user.id if user else None - } -``` +`jvspatial/core/annotations` (re-exported as `jvspatial.core.annotations`): -### Webhook Security and Authentication +| Flag | Effect | +|---|---| +| `protected=True` | Field cannot be reassigned after `__init__` completes | +| `transient=True` | Field is not persisted to the database | +| `private=True` | Field is hidden from `export()` output (and typically transient) | +| `default` / `default_factory` | Pydantic-style defaults | +| `description` | Documentation surfaced in OpenAPI schemas | -Webhook endpoints require authentication tokens in the URL path and support additional security measures: +Indexed fields (`@attribute(indexed=True)`) and compound indexes (`@attribute(compound_index=...)`) inform backend index creation (where supported). -```python path=null start=null -# Webhook with permission requirements -@endpoint( - "/webhook/admin/{route}/{auth_token}", - methods=["POST"], - webhook=True, - permissions=["process_webhooks", "admin_access"], - roles=["admin", "webhook_manager"] -) -async def admin_webhook_handler(request: Request) -> Dict[str, Any]: - """Administrative webhook handler with strict permissions.""" - current_user = get_current_user(request) - - # User is guaranteed to have required permissions due to middleware - return { - "status": "success", - "message": "Admin webhook processed", - "admin_user": current_user.username, - "permissions": current_user.permissions - } - -# HMAC signature verification (handled by middleware) -@endpoint("/webhook/secure/{service}/{auth_token}", methods=["POST"], webhook=True) -async def secure_webhook_handler(request: Request) -> Dict[str, Any]: - """Webhook with HMAC signature verification. - - Middleware automatically verifies HMAC signatures when present. - Configure HMAC secrets via environment variables or user settings. - """ - # If this handler executes, HMAC verification passed (if configured) - raw_body = request.state.raw_body - hmac_verified = getattr(request.state, "hmac_verified", False) - - return { - "status": "success", - "message": "Secure webhook processed", - "hmac_verified": hmac_verified, - "payload_size": len(raw_body) - } -``` +--- -### Graph Traversal Webhook Processing (Future Enhancement) - -The architecture supports webhook processing through graph traversal using Walker classes: - -```python path=null start=null - -# from jvspatial.api.auth.decorators import webhook_endpoint -# from jvspatial.core import Walker, Node -# from jvspatial.core.entities import on_visit - -# @endpoint("/webhook/process/{route}/{auth_token}", methods=["POST"]) -# class WebhookProcessingWalker(Walker): -# """Walker-based webhook processing with graph traversal.""" -# -# def __init__(self): -# super().__init__() -# self.webhook_data = None -# self.processing_results = [] -# -# @on_visit("WebhookEvent") -# async def process_webhook_event(self, here: Node): -# """Process webhook events stored as graph nodes.""" -# # Access webhook data from request.state -# payload = self.webhook_data -# -# # Process event based on node data and webhook payload -# result = await self.analyze_event(here, payload) -# self.processing_results.append(result) -# -# # Continue traversal to related events -# related_events = await here.nodes(node=['WebhookEvent']) -# await self.visit(related_events) -# -# async def analyze_event(self, event_node: Node, payload: dict) -> dict: -# """Analyze webhook event against stored data.""" -# return { -# "event_id": event_node.id, -# "payload_type": payload.get("type"), -# "correlation_score": 0.95 # Example analysis result -# } -``` +## 3. Async Contract -### Idempotency and Duplicate Handling - -Webhooks support idempotency keys to handle duplicate deliveries: - -```python path=null start=null -@endpoint("/webhook/idempotent/{auth_token}", methods=["POST"], webhook=True) -async def idempotent_webhook_handler(request: Request) -> Dict[str, Any]: - """Webhook handler with built-in idempotency support. - - Middleware automatically handles idempotency keys in headers: - - Idempotency-Key header - - X-Idempotency-Key header - - Custom idempotency headers - """ - # Access idempotency information from middleware - idempotency_key = getattr(request.state, "idempotency_key", None) - is_duplicate = getattr(request.state, "is_duplicate_request", False) - - if is_duplicate: - # Return cached response for duplicate requests - cached_response = getattr(request.state, "cached_response", {}) - return { - "status": "success", - "message": "Duplicate request, returning cached response", - "idempotency_key": idempotency_key, - "cached_result": cached_response - } - - # Process new request - raw_body = request.state.raw_body - processed_result = await process_unique_webhook(json.loads(raw_body)) - - return { - "status": "success", - "message": "New webhook processed", - "idempotency_key": idempotency_key, - "result": processed_result - } - -async def process_unique_webhook(payload: dict) -> dict: - """Process a unique webhook payload.""" - # Simulate processing logic - import time - processing_start = time.time() - - # Your actual webhook processing logic here - await asyncio.sleep(0.1) # Simulate work - - return { - "processed_at": processing_start, - "data_processed": True, - "payload_keys": list(payload.keys()) - } -``` +### 3.1 Async-only I/O -### Server Integration and Middleware Setup +Every operation that touches the database, network, or file system is `async`. This includes — without exception: -Webhook endpoints automatically integrate with the jvspatial server middleware stack: +- `Entity.create`, `Entity.get`, `Entity.find`, `Entity.find_one`, `Entity.save`, `Entity.delete`, `Entity.count`, `Entity.export` +- `Database.save`, `Database.get`, `Database.find`, `Database.delete`, `Database.count`, `Database.find_one`, `Database.find_many`, `Database.bulk_save` +- All Walker traversal methods (`spawn`, `step`, `walk`, `enqueue`, `pause`, `resume`) +- Storage backends (`upload`, `download`, `list_versions`, …) +- All API server lifecycle hooks (`on_startup`, `on_shutdown`, …) -```python path=null start=null -from jvspatial.api import Server -from jvspatial.api.auth.middleware import ( - AuthenticationMiddleware, - WebhookMiddleware, - HTTPSRedirectMiddleware -) +### 3.2 Sync operations -# Server setup with webhook middleware -server = Server( - title="Webhook-Enabled Spatial API", - description="API with secure webhook processing", - version="1.0.0", - host="0.0.0.0", - port=8000 -) +Sync functions are reserved for **pure computation only**: -# Middleware stack (order matters) -server.add_middleware(HTTPSRedirectMiddleware) # Force HTTPS -server.add_middleware(WebhookMiddleware) # Webhook processing -server.add_middleware(AuthenticationMiddleware) # Authentication +- `generate_id()`, `find_subclass_by_name()` — `jvspatial/core/utils.py` +- `get_collection_name()` — `jvspatial/core/entities/object.py:78` +- `is_serverless_mode()`, `detect_serverless_provider()` — `jvspatial/runtime/serverless.py` +- Walker trail read-only queries (`get_trail()`, `has_visited()`, `get_trail_summary()`) +- All `@attribute`-related helpers -# Webhook endpoints are automatically registered -# Access at: POST https://your-domain.com/webhook/{route}/{auth_token} +Calling sync code from async is always safe. Calling async code from sync without an event loop is a programming error. -# Environment configuration for webhook security -# Set in .env file: -# WEBHOOK_HMAC_SECRET=your-secret-key -# WEBHOOK_HTTPS_REQUIRED=true -# WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour cache -# WEBHOOK_MAX_PAYLOAD_SIZE=1048576 # 1MB limit +### 3.3 Blocking ops to avoid -if __name__ == "__main__": - server.run(port=8000) -``` +- **Never** do synchronous file or network I/O inside an async handler — it blocks the event loop. +- **Never** spawn a thread to run an async coroutine — use `asyncio.create_task()` or `asyncio.gather()` instead. +- **Never** forget `await` on a database call — at best you receive a coroutine object; at worst you proceed with stale data. -### Webhook Testing and Development - -```python path=null start=null -# Testing webhook handlers -import pytest -from fastapi.testclient import TestClient -from unittest.mock import MagicMock - -@pytest.fixture -def test_webhook_request(): - """Create mock webhook request for testing.""" - request = MagicMock() - request.state.raw_body = b'{"type": "test", "data": {"id": 123}}' - request.state.content_type = "application/json" - request.state.webhook_route = "test" - request.state.current_user = MagicMock(id="user_123") - request.state.hmac_verified = True - request.state.idempotency_key = "test-key-123" - return request - -@pytest.mark.asyncio -async def test_webhook_processing(test_webhook_request): - """Test webhook handler processing.""" - result = await generic_webhook_handler(test_webhook_request) - - assert result["status"] == "received" - assert result["route"] == "test" - assert result["user_id"] == "user_123" - -# Development webhook testing with ngrok or similar -# 1. Start your jvspatial server locally -# 2. Use ngrok to expose: ngrok http 8000 -# 3. Configure webhook URLs: https://abc123.ngrok.io/webhook/test/your-auth-token -# 4. Test with curl: -# curl -X POST https://abc123.ngrok.io/webhook/test/your-token \ -# -H "Content-Type: application/json" \ -# -d '{"test": "data"}' -``` +### 3.4 Deferred saves (optional batching) -### Best Practices for Webhook Implementation - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Always return 200 status for webhooks -@endpoint("/webhook/service/{auth_token}", webhook=True) -async def proper_webhook_handler(request: Request) -> Dict[str, Any]: - try: - # Process webhook - result = await process_webhook_data(request.state.raw_body) - return {"status": "success", "result": result} - except Exception as e: - # Log error but still return 200 - logger.error(f"Webhook processing failed: {e}") - return {"status": "received", "error": "logged"} - -# Good: Use route-based dispatch for multiple services -@endpoint("/webhook/{route}/{auth_token}", webhook=True) -async def multi_service_webhook(request: Request) -> Dict[str, Any]: - route = getattr(request.state, "webhook_route", "unknown") - - handlers = { - "stripe": process_stripe_webhook, - "github": process_github_webhook, - "slack": process_slack_webhook - } - - handler = handlers.get(route, process_generic_webhook) - return await handler(request) - -# Good: Validate authentication token format -@endpoint("/webhook/{service}/{auth_token}", webhook=True) -async def secure_webhook_handler(request: Request) -> Dict[str, Any]: - # Token validation is handled by middleware - current_user = get_current_user(request) - if not current_user: - return {"status": "error", "message": "Invalid authentication"} - - return {"status": "success", "user_verified": True} -``` +When `DeferredSaveMixin` is mixed into an entity *and* `deferred_saves_globally_allowed()` returns `True` (see §11.3), `await entity.save()` marks the entity dirty without writing. Persistence happens on explicit `await entity.flush()` or context exit. -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Returning non-200 status codes -@endpoint("/webhook/bad/{auth_token}", webhook=True) -async def bad_webhook_handler(request: Request) -> Dict[str, Any]: - try: - process_webhook(request.state.raw_body) - except Exception: - # Don't do this - breaks webhook retry logic - raise HTTPException(status_code=500, detail="Processing failed") - -# Bad: Not handling authentication properly -@endpoint("/webhook/unsecure/{auth_token}", webhook=True) -async def unsecure_webhook_handler(request: Request) -> Dict[str, Any]: - # Don't bypass authentication checks - # Always use get_current_user() or require auth in decorator - return {"status": "processed"} - -# Bad: Not using middleware-processed data -@endpoint("/webhook/manual/{auth_token}", webhook=True) -async def manual_webhook_handler(request: Request) -> Dict[str, Any]: - # Don't manually read request body - use request.state.raw_body - # raw_body = await request.body() # Wrong - middleware already processed - - # Use middleware-processed data instead - raw_body = request.state.raw_body # Correct - return {"status": "processed"} -``` +**MRO requirement**: the mixin **must precede** the base class: `class MyEntity(DeferredSaveMixin, Node)` — not the reverse. Wrong MRO silently disables batching. --- -## 🔗 Webhook System Integration +## 4. Persistence Layer -JVspatial provides an advanced webhook system for handling external service integrations with enterprise-grade security, reliability, and developer experience. The webhook system supports modern decorators, automatic payload processing, HMAC verification, idempotency handling, and seamless authentication integration. +### 4.1 Database abstraction -### Quick Webhook Setup +`jvspatial/db/database.py:48+` — `Database` ABC. All adapters must implement: -```python path=null start=null -from jvspatial.api import endpoint -from jvspatial.api import Server +| Method | Required | Description | +|---|---|---| +| `save(collection, data)` | Yes | Insert-or-replace by ID; returns saved record | +| `get(collection, id)` | Yes | Fetch by ID or `None` | +| `delete(collection, id)` | Yes | Idempotent delete by ID | +| `find(collection, query, *, limit, sort)` | Yes | Mongo-style query; returns list | +| `count(collection, query=None)` | Default impl | Default counts the result of `find`; adapters should override for efficiency | +| `find_one(collection, query)` | Default impl | First match or `None` | +| `find_many(collection, ids)` | Default impl | Bulk-fetch by ID; default is N sequential `get`s — adapters override for round-trip efficiency | +| `find_one_and_update` | Default impl | Read-modify-write; **not atomic** except where overridden (MongoDB) | +| `find_one_and_delete` | Default impl | Read-then-delete; **not atomic** except where overridden (MongoDB) | +| `bulk_save` | Default impl | Multi-record save; partial-success semantics vary by adapter | +| `begin_transaction` | Optional | Returns a transaction context manager if `supports_transactions=True` | -# Simple webhook handler -@endpoint("/webhook/payment", webhook=True) -async def payment_webhook(payload: dict, endpoint): - """Process payment webhooks with automatic JSON parsing.""" - payment_id = payload.get("payment_id") - amount = payload.get("amount") +### 4.2 Capability flags - # Process payment logic here - print(f"Processing payment {payment_id}: ${amount}") +Adapters declare capabilities as class attributes: - return endpoint.response( - content={ - "status": "processed", - "message": f"Payment {payment_id} processed successfully" - } - ) +- `supports_transactions: bool` — `True` for MongoDB (replica set); `False` for SQLite (best-effort), JSON, DynamoDB. -# Server automatically detects and configures webhook middleware -server = Server(title="My Webhook API") -server.run() # Webhooks ready at /webhook/* paths -``` +Callers branching on capabilities should test the flag, not the adapter class. -### Advanced Webhook Features - -```python path=null start=null -# Webhook with full security features -@endpoint( - "/webhook/stripe/{key}", - webhook=True, - path_key_auth=True, # API key in URL path - hmac_secret="stripe-webhook-secret", # HMAC signature verification - idempotency_ttl_hours=48, # Duplicate handling for 48h - permissions=["process_payments"] # RBAC permissions -) -async def secure_stripe_webhook(raw_body: bytes, content_type: str, endpoint): - """Stripe webhook with comprehensive security.""" - import json - - if content_type == "application/json": - payload = json.loads(raw_body.decode('utf-8')) - event_type = payload.get("type", "unknown") - - if event_type == "payment_intent.succeeded": - return endpoint.response( - content={ - "status": "processed", - "event_type": event_type, - "message": "Payment successful" - } - ) - - return endpoint.response(content={"status": "received"}) - -# Multi-service webhook dispatcher -@endpoint("/webhook/{service}", webhook=True) -async def multi_service_webhook(payload: dict, service: str, endpoint): - """Route webhooks based on service parameter.""" - handlers = { - "stripe": process_stripe_event, - "github": process_github_event, - "slack": process_slack_event - } - - handler = handlers.get(service, process_generic_event) - result = await handler(payload) - - return endpoint.response( - content={ - "status": "processed", - "service": service, - "result": result - } - ) - -# Helper functions -async def process_stripe_event(payload: dict) -> dict: - return {"stripe_event": payload.get("type", "unknown")} - -async def process_github_event(payload: dict) -> dict: - return {"github_action": payload.get("action", "unknown")} - -async def process_slack_event(payload: dict) -> dict: - return {"slack_event": payload.get("event", {}).get("type", "unknown")} - -async def process_generic_event(payload: dict) -> dict: - return {"processed": True, "keys": list(payload.keys())} -``` - -### Walker-Based Webhook Processing - -```python path=null start=null -# Future feature - Walker-based webhook processing -# @webhook_walker_endpoint("/webhook/location-update") -# class LocationUpdateWalker(Walker): -# """Process location updates through graph traversal.""" -# -# def __init__(self, payload: dict): -# super().__init__() -# self.payload = payload -# # Use the report() method to collect data during traversal -# -# @on_visit(Node) -# async def update_location_data(self, here: Node): -# locations = self.payload.get("locations", []) -# -# for location_data in locations: -# location_id = location_data.get("id") -# coordinates = location_data.get("coordinates") -# -# if location_id and coordinates: -# here.coordinates = coordinates -# await here.save() -# -# self.report({ -# "updated_location": { -# "id": location_id, -# "coordinates": coordinates -# } -# }) -``` +### 4.3 Built-in adapters -### Environment Configuration +| Adapter | File | Transactions | Notes | +|---|---|---|---| +| JSON | `jvspatial/db/jsondb.py` | No | File-per-record, atomic writes (`_atomic.py`), per-file path locks (`_path_locks.py`) | +| SQLite | `jvspatial/db/sqlite.py` | No (single-conn fsync) | `aiosqlite`; Mongo→SQL via `SQLiteTranslator` | +| MongoDB | `jvspatial/db/mongodb.py` | Yes | `motor`; native bulk writes; native compound ops | +| DynamoDB | `jvspatial/db/dynamodb.py` | No | `aioboto3`; throttle-retry; `BatchGetItem` chunks of 100 | -Configure webhook behavior via environment variables: +### 4.4 Atomic IO (JSON) -```env -# Global webhook settings -JVSPATIAL_WEBHOOK_HMAC_SECRET=your-global-hmac-secret -JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE=5242880 # 5MB -JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL=3600 # 1 hour -JVSPATIAL_WEBHOOK_HTTPS_REQUIRED=true - -# Service-specific secrets -JVSPATIAL_WEBHOOK_STRIPE_SECRET=whsec_stripe_secret_key -JVSPATIAL_WEBHOOK_GITHUB_SECRET=github_webhook_secret -``` +`jvspatial/db/_atomic.py` provides crash-safe writes: temp file → `fsync` → `rename` → `fsync(directory)`. Per-file mutex via `PathLockManager` (`_path_locks.py`) serializes concurrent writes to the same record. Bounded LRU prevents lock-table growth. -### Testing Webhooks - -```bash -# Basic webhook test -curl -X POST "http://localhost:8000/webhook/payment" \ - -H "Content-Type: application/json" \ - -d '{"payment_id": "pay_123", "amount": 99.99}' - -# Webhook with path-based auth -curl -X POST "http://localhost:8000/webhook/stripe/key123:secret456" \ - -H "Content-Type: application/json" \ - -H "X-Signature: sha256=abc123..." \ - -d '{"type": "payment_intent.succeeded"}' - -# With idempotency key -curl -X POST "http://localhost:8000/webhook/payment" \ - -H "Content-Type: application/json" \ - -H "X-Idempotency-Key: unique-123" \ - -d '{"payment_id": "pay_124"}' -``` +### 4.5 Multi-database -### Webhook Best Practices - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Always return 200 for webhook endpoints -@endpoint("/webhook/service", webhook=True) -async def proper_webhook(payload: dict, endpoint): - try: - result = await process_webhook_data(payload) - return endpoint.response(content={"status": "success", "result": result}) - except Exception as e: - # Log error but still return 200 - logger.error(f"Webhook processing failed: {e}") - return endpoint.response(content={"status": "received", "error": "logged"}) - -# Good: Use route-based dispatch for multiple services -@endpoint("/webhook/{service}", webhook=True) -async def multi_service_webhook(payload: dict, service: str, endpoint): - handlers = { - "stripe": process_stripe, - "github": process_github - } - - handler = handlers.get(service, process_generic) - return await handler(payload, endpoint) - -# Good: Validate webhook signatures when available -@endpoint("/webhook/secure", webhook=True, hmac_secret="webhook-secret") -async def secure_webhook(raw_body: bytes, endpoint): - # HMAC verification is automatic when secret is provided - return endpoint.response(content={"status": "verified"}) -``` +`jvspatial/db/manager.py` — one "prime" database for core ops (auth, sessions, API keys), plus additional databases for specialized use. Auth state is **always** on the prime database; this cannot be relocated. -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Returning non-200 status codes -@endpoint("/webhook/bad", webhook=True) -async def bad_webhook(payload: dict, endpoint): - if payload.get("invalid"): - # Don't do this - breaks webhook retry logic - raise HTTPException(status_code=400, detail="Invalid payload") - -# Bad: Not handling errors gracefully -@endpoint("/webhook/risky", webhook=True) -async def risky_webhook(payload: dict, endpoint): - # Unhandled exceptions will return 500 - webhooks will retry - result = dangerous_operation(payload) # Might throw - return endpoint.response(content={"result": result}) - -# Bad: Bypassing security features -@endpoint("/webhook/insecure", webhook=True) -async def insecure_webhook(request: Request, endpoint): - # Don't manually read request body - use automatic payload injection - raw_body = await request.body() # Wrong - middleware already processed - return endpoint.response(content={"status": "received"}) -``` +### 4.6 Schema migration -> **📖 For complete webhook documentation and advanced patterns:** [Webhook Architecture Guide](docs/md/webhook-architecture.md) | [Webhook Quickstart](docs/md/webhooks-quickstart.md) +No built-in migration framework. Adapters do not enforce schemas. Adding optional fields with defaults is backward-compatible on read. Removing fields breaks existing records that still contain them. Application owners are responsible for migration scripts. --- -## 📁 File Storage Quickstart +## 5. Query Interface -jvspatial includes a powerful file storage system with multi-backend support and URL proxy capabilities for secure file sharing. +### 5.1 Mongo-style operators -### Basic Setup +`jvspatial/db/query.py` — unified query DSL across adapters. Supported operators: -```python -from jvspatial.api import Server - -server = Server( - title="File Upload API", - file_storage_enabled=True, - file_storage_provider="local", - file_storage_root=".files", - proxy_enabled=True -) +| Operator | Meaning | +|---|---| +| `$eq`, `$ne` | Equality / inequality | +| `$gt`, `$gte`, `$lt`, `$lte` | Comparison | +| `$in`, `$nin` | Membership | +| `$exists` | Field presence | +| `$and`, `$or` | Logical combinators | +| `$regex` | Regex match (string fields) | -if __name__ == "__main__": - server.run() -``` +### 5.2 Pushdown vs in-memory -### Upload a File +- **MongoDB**: native pushdown; queries run server-side. +- **SQLite**: translated to SQL via `SQLiteTranslator` (subset; complex `$or` chains may fall back). +- **DynamoDB**: limited pushdown via `Select=COUNT` and key conditions; remainder filtered client-side. +- **JSON**: full in-memory evaluation after loading matching collection. -```bash -curl -X POST -F "file=@document.pdf" \ - http://localhost:8000/storage/upload -``` +### 5.3 Compiled query cache -**Response:** -```json -{ - "success": true, - "file_path": "2025/01/05/document-abc123.pdf", - "file_size": 102400, - "content_type": "application/pdf" -} -``` +`QueryEngine` caches compiled queries (default LRU size 1024, configurable per database via `query_cache_size`). Cache bounds prevent unbounded growth from dynamic query construction. -### Create a Shareable Link +### 5.4 `Entity.find` field paths -```bash -curl -X POST http://localhost:8000/storage/proxy \ - -H "Content-Type: application/json" \ - -d '{ - "file_path": "2025/01/05/document-abc123.pdf", - "expires_in": 3600 - }' -``` - -**Response:** -```json -{ - "success": true, - "proxy_code": "a1b2c3d4", - "proxy_url": "http://localhost:8000/p/a1b2c3d4", - "expires_at": "2025-01-05T23:00:00Z" -} -``` +Caller-facing queries on `Entity.find({...})` operate on the persisted document shape. Entity attributes (other than top-level fields like `id`, `entity`, `edges`, `source`, `target`, `bidirectional`) live under a `context` sub-object. Use `"context.field_name"` as the query key for attribute fields. -### Access via Short URL - -```bash -curl http://localhost:8000/p/a1b2c3d4 -``` - -The file is served directly with appropriate headers. - -### Use in Walkers - -```python -from jvspatial.storage import get_file_interface -from jvspatial.core import Walker, on_visit, Node - -@server.walker("/process-upload") -class ProcessUpload(Walker): - file_path: str - - @on_visit(Node) - async def process(self, here: Node): - # Get file storage interface - storage = get_file_interface( - provider="local", - root_dir=".files" - ) - - # Read file content - content = await storage.get_file(self.file_path) - - # Process file content - self.report({ - "processed_file": { - "path": self.file_path, - "size": len(content), - "status": "success" - } - }) -``` - -### AWS S3 Configuration - -```python -server = Server( - title="S3 File API", - file_storage_enabled=True, - file_storage_provider="s3", - file_storage_s3_bucket="my-bucket", - file_storage_s3_region="us-east-1", - proxy_enabled=True -) -``` - -**Environment Variables:** -```env -JVSPATIAL_FILE_STORAGE_ENABLED=true -JVSPATIAL_FILE_STORAGE_PROVIDER=s3 -JVSPATIAL_FILE_STORAGE_S3_BUCKET=my-bucket -JVSPATIAL_FILE_STORAGE_S3_REGION=us-east-1 -AWS_ACCESS_KEY_ID=your-key-id -AWS_SECRET_ACCESS_KEY=your-secret-key -``` - -### Advanced Usage: Custom Upload Path - -```bash -curl -X POST -F "file=@image.jpg" \ - -F "custom_path=avatars/user123.jpg" \ - http://localhost:8000/storage/upload -``` +--- -### List Files +## 6. Walker Traversal Semantics -```bash -curl http://localhost:8000/storage/files?prefix=2025/01/ -``` +### 6.1 Lifecycle -**Response:** -```json -{ - "success": true, - "files": [ - { - "path": "2025/01/05/document-abc123.pdf", - "size": 102400, - "modified": "2025-01-05T20:30:00Z" - }, - { - "path": "2025/01/05/image-def456.jpg", - "size": 51200, - "modified": "2025-01-05T21:15:00Z" - } - ] -} -``` +1. Instantiate walker: `walker = MyWalker(...)`. +2. Begin: `await walker.spawn(start_node)` or `await walker.walk(start_node)`. +3. Per step: dequeue node, set visitor, run matching `@on_visit` hooks, optionally enqueue more nodes. +4. End: queue empty, `pause()` called, protection limit hit, or `WalkerTimeoutError` raised. -### Security Features - -```python -server = Server( - title="Secure File API", - file_storage_enabled=True, - file_storage_provider="local", - file_storage_root=".files", - file_storage_max_size=10485760, # 10MB limit - file_storage_allowed_types=["image/jpeg", "image/png", "application/pdf"], - proxy_enabled=True, - proxy_default_ttl=3600 # 1 hour default expiration -) -``` +### 6.2 Visit hooks -### Best Practices +- `@on_visit(NodeType)` on a Walker method — fires when the walker visits `NodeType` (or its subclass). +- `@on_visit(WalkerType)` on a Node method — fires when `WalkerType` visits *this* node. +- `@on_visit` with no target — fires for every visit. -**✅ Recommended Patterns:** +Hooks are registered at class-creation time in `__init_subclass__` (`node.py:62`, `edge.py:66`). -```python -# Good: Use environment variables for configuration -from dotenv import load_dotenv -load_dotenv() - -server = Server( - title="Production File API", - file_storage_enabled=True, - # Provider configured via JVSPATIAL_FILE_STORAGE_PROVIDER - # Other settings loaded from environment -) +### 6.3 Protection invariants -# Good: Validate files before processing -@endpoint("/validate-upload") -class ValidateUpload(Walker): - file_path: str - - @on_visit(Node) - async def validate(self, here: Node): - storage = get_file_interface() - - # Check file exists - if not await storage.file_exists(self.file_path): - self.report({"error": "File not found"}) - return - - # Get file metadata - metadata = await storage.get_metadata(self.file_path) - - # Validate size - if metadata.get("size", 0) > 5242880: # 5MB - self.report({"error": "File too large"}) - return - - self.report({"status": "valid", "metadata": metadata}) - -# Good: Use proxy URLs for temporary access -async def create_temp_link(file_path: str, hours: int = 1): - """Create temporary shareable link.""" - response = await storage.create_proxy( - file_path=file_path, - expires_in=hours * 3600 - ) - return response["proxy_url"] -``` +When `protection_enabled=True` (default): -**❌ Avoided Patterns:** +- Step counter increments on every visit; exceeding `max_steps` raises `WalkerExecutionError`. +- Per-node visit counter increments on every visit; exceeding `max_visits_per_node` raises `InfiniteLoopError`. +- Wall-clock timer started at `spawn`; exceeding `max_execution_time` raises `WalkerTimeoutError`. +- Enqueue refuses when queue length would exceed `max_queue_size` (silent drop, logged). -```python -# Bad: Hardcoding credentials -server = Server( - file_storage_s3_bucket="my-bucket", - file_storage_s3_access_key="AKIAIOSFODNN7EXAMPLE" # Don't do this! -) +Protection is *advisory* for safety, not for security — untrusted user input should not influence walker construction. -# Bad: No file validation -@endpoint("/unsafe-upload") -class UnsafeUpload(Walker): - file_path: str +### 6.4 Trail tracking - @on_visit(Node) - async def process(self, here: Node): - # No validation - could process malicious files - content = await storage.get_file(self.file_path) - # Direct processing without checks +`WalkerTrail` records `(node_id, edge_id, timestamp, node_type, queue_length, metadata)` for every visit (when `trail_enabled=True`). `max_trail_length=0` is unlimited; bounded by memory. In serverless deployments, trails do not persist across invocations. -# Bad: Permanent public URLs without expiration -# Always use proxy URLs with expiration for security -``` +### 6.5 Control flow -See [File Storage Documentation](docs/md/file-storage-usage.md) for advanced usage and all configuration options. +- `walker.pause()` — raises `TraversalPaused`; caller catches to suspend. +- `walker.skip()` — raises `TraversalSkipped`; advances to next queued node. +- `walker.resume()` — re-enters traversal from saved state. --- -## 🔀 Router Decorators - -jvspatial provides a unified `@endpoint` decorator for all API endpoints: - -1. `@endpoint` - For public endpoints (both functions and Walker classes) -2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) -3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) -4. `@endpoint(..., auth=True, roles=["admin"])` - For admin-only endpoints - -```python -from jvspatial.api import endpoint - -# Function endpoint -@endpoint("/api/users", methods=["GET"]) -async def get_users() -> Dict[str, Any]: - users = await User.all() - return {"users": users} - -# Walker endpoint -@endpoint("/api/graph/traverse", methods=["POST"]) -class GraphTraversal(Walker): - pass - -# Authenticated function endpoint -@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) -async def get_admin_stats() -> Dict[str, Any]: - return {"stats": "admin only"} - -# Authenticated walker endpoint (uses same decorator) -@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) -class SecureProcessor(Walker): - pass - -# Admin-only endpoint -@endpoint("/api/admin/users", auth=True, roles=["admin"], methods=["GET"]) -async def manage_users() -> Dict[str, Any]: - return {"users": "admin access"} -``` - -**❌ DO NOT USE alternative decorators like:** -- `@route` -- `@server.route` -- `@server.walker` -- `@walker_endpoint` (deprecated - use `@endpoint` instead) -- `@auth_endpoint` (deprecated - use `@endpoint(..., auth=True)` instead) -- `@admin_endpoint` (deprecated - use `@endpoint(..., auth=True, roles=["admin"])` instead) - -These are internal or deprecated. - -## 📌 Consolidated Endpoint System - -jvspatial uses a **unified endpoint registration system** where all endpoints (walkers and functions) are registered through a single consolidated mechanism. This ensures clean, maintainable code without backward compatibility cruft. +## 7. GraphContext and Dependency Injection -### Key Architecture +`jvspatial/core/context.py` — `GraphContext` binds a `Database` + optional `Cache` + `PerformanceMonitor` to a scope. -All decorators follow the same registration path: +### 7.1 Resolution order -1. **Decorator** → Attaches metadata to function/walker -2. **Server Detection** → Gets current server from context -3. **Registration** → Registers with `server.endpoint_router` -4. **Tracking** → Tracked by `server._endpoint_registry` +When an entity needs a context, it resolves in this order: +1. Explicitly set via `await entity.set_context(ctx)`. +2. Default context: `get_default_context()` (singleton, lazily initialized from environment). -### Important: Decorator Order +### 7.2 Scoping -Always create the server **before** decorating endpoints: +`GraphContext` is request-scoped by convention. The API server installs a per-request context via middleware (`jvspatial/api/components/auth_middleware.py` and lifecycle), so endpoint handlers reach the correct database without manual injection. -```python -# ✅ CORRECT -server = Server(title="My API") - -@endpoint("/test") -class TestWalker(Walker): - pass - -# ✗ INCORRECT - endpoint will not be registered -@endpoint("/test") -class TestWalker(Walker): - pass - -server = Server(title="My API") # Created too late -``` +### 7.3 Performance monitoring -### Default HTTP Methods +`PerformanceMonitor` (within `GraphContext`) records: +- DB operation counts and latencies +- Hook execution counts and latencies +- Cache hits and misses -- **Walkers**: Default to `["POST"]` -- **Functions**: Default to `["GET"]` +Surfaced via `observability/` for export to metrics sinks. -Override with the `methods` parameter: -```python -@endpoint("/data", methods=["GET", "POST"]) -class DataWalker(Walker): - pass -``` +--- -### Available Response Methods +## 8. API Surface -Function endpoints can receive an `endpoint` parameter for response formatting: +### 8.1 Server class -```python -@endpoint("/info") -async def get_info(endpoint): - # Use endpoint.success(), endpoint.error(), etc. - return endpoint.success(data={"info": "value"}) -``` +`jvspatial/api/server.py` — composition of four mixins: -Walkers automatically have `self.endpoint` available: +| Mixin | File | Concern | +|---|---|---| +| `AppFactoryMixin` | `api/server_app_factory.py` | Build FastAPI app, init DB, set up CORS | +| `RegistrationMixin` | `api/server_registration.py` | Register routes from `@endpoint`-decorated targets | +| `LifecycleMixin` | `api/server_lifecycle.py` | Startup/shutdown hooks, context wiring | +| `RunMixin` | `api/server_run.py` | Uvicorn invocation, host/port resolution | -```python -@endpoint("/process") -class ProcessWalker(Walker): - async def process(self): - self.response = self.endpoint.success(data={"result": "done"}) -``` +### 8.2 `@endpoint` decorator -**Response Methods:** +`jvspatial/api/decorators/route.py` — single decorator for both async functions and `Walker` subclasses: ```python -# Success responses -endpoint.success(data=result, message="Success") # 200 OK -endpoint.created(data=new_item, message="Created") # 201 Created -endpoint.no_content() # 204 No Content - -# Error responses -endpoint.bad_request(message="Invalid input") # 400 Bad Request -endpoint.unauthorized(message="Auth required") # 401 Unauthorized -endpoint.forbidden(message="Access denied") # 403 Forbidden -endpoint.not_found(message="Resource not found") # 404 Not Found -endpoint.conflict(message="Resource exists") # 409 Conflict -endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity - -# Flexible custom response -endpoint.response( - content={"custom": "data"}, - status_code=202, - headers={"X-Custom": "value"} -) +@endpoint("/users/{user_id}", methods=["GET"], auth=True, roles=["admin"]) +async def get_user(user_id: str): ... -# Generic error with custom status code -endpoint.error( - message="Custom error", - status_code=418, - details={"reason": "custom"} -) +@endpoint("/walk", methods=["POST"]) +class MyWalker(Walker): + ... ``` -### Querying Registered Endpoints - -```python -server = Server(title="My API") +**Parameters**: +- `path` — FastAPI path with parameters +- `methods` — HTTP methods list +- `auth: bool` — require authentication (default `False`) +- `roles: list[str]` — RBAC roles +- `webhook: bool` / `webhook_model` — webhook handler mode with signature verification +- `response` — response schema using `ResponseField` + `success_response()` -# ... register some endpoints ... +### 8.3 Deferred registry -# List all endpoints -all_endpoints = server.list_all_endpoints() -print(f"Walkers: {len(all_endpoints['walkers'])}") -print(f"Functions: {len(all_endpoints['functions'])}") +Endpoints decorated at import time are collected in a deferred registry (`api/endpoints/registry.py`). The server resolves them at app-build time, after the database context is available. This allows entity-bound endpoints to be declared at module scope. -# List just walkers -walkers = server.list_walker_endpoints() +### 8.4 Built-in endpoints -# List just functions -functions = server.list_function_endpoints() +When `auth_enabled=True`, the server registers `/auth/register`, `/auth/login`, `/auth/logout`, plus token refresh and password reset endpoints. With `auth_enabled=False`, no auth endpoints are registered. -# Get registry stats -registry = server._endpoint_registry -counts = registry.count_endpoints() -print(f"Total: {counts['total']}") -``` +OpenAPI docs at `/docs` and `/redoc` unless `JVSPATIAL_DOCS_DISABLED` is truthy (see §10.5). -### Unified Registration Benefits +--- -1. **Single Source of Truth**: All endpoints are registered through `EndpointRouter` -2. **Cleaner Code**: No backward compatibility cruft or deprecated methods -3. **Consistent API**: All decorators follow the same pattern -4. **Better Maintainability**: Future endpoint features only need to be added once -5. **Auto-Detection**: Decorators automatically detect walker vs function +## 9. Authentication and Authorization -### Migration from Deprecated Patterns +### 9.1 Authentication mechanisms -If you have code using removed patterns: +- **JWT**: `PyJWT`-based. Secret required at startup (`jwt_secret` config or `JVSPATIAL_JWT_SECRET_KEY` env). Server **fails fast** on missing secret. +- **API keys**: SHA-256 hashed at rest, plaintext returned **only once** on creation. Verification uses `hmac.compare_digest` (constant-time). +- **Refresh tokens**: rotate on use; previous tokens invalidated. +- **Password reset**: `token_lookup` field provides O(1) lookup; constant-time comparison. -**❌ Old Pattern (Removed)** -```python -# These no longer exist -server._custom_routes.append(...) -server._register_custom_routes(app) -server._setup_webhook_walker_endpoints() -``` +### 9.2 Authorization (RBAC) -**✅ New Pattern (Current)** -```python -# Use the decorators - they handle registration automatically -@endpoint("/my-route") -def my_handler(): - pass +`jvspatial/api/auth/rbac.py` — roles map to permission unions. Wildcard support (e.g. `users:*`). Admin-only routes enforced on `/status`, `/logs`, and `/graph` subtrees by default. -# Or register programmatically -server.register_walker_class(MyWalker, "/my-route", methods=["POST"]) -``` +### 9.3 Session management -## 🌐 API Integration with FastAPI Server +`SessionManager` tracks active sessions per user. **Per-process, in-memory** — true limits in multi-worker deployments are `limit × workers`. Document this when configuring session caps. -The **jvspatial API** provides seamless integration with FastAPI to expose your graph operations as REST endpoints. It supports flexible endpoint registration using decorators and automatic parameter model generation from Walker and function properties. +### 9.4 Logout blacklist -### Server Setup and Configuration +JWT tokens are blacklisted on logout. Blacklist storage is per-worker in default config. Cross-worker invalidation requires a shared blacklist store (caller-provided). -```python path=null start=null -from jvspatial.api import Server, ServerConfig, endpoint, walker_endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Node, Walker -from jvspatial.core.entities import on_visit +### 9.5 Webhook authentication -# Basic server setup -server = Server( - title="My Spatial API", - description="Graph-based spatial data management API", - version="1.0.0", - host="0.0.0.0", - port=8000, - debug=True # Enable for development -) +`jvspatial/api/integrations/webhooks/` — HMAC signature verification with constant-time comparison. Per-source secret rotation supported. Replay protection via timestamp window (configurable). -# Advanced server configuration -advanced_config = ServerConfig( - title="Production Spatial API", - description="Enterprise graph data API", - version="2.0.0", - host="0.0.0.0", - port=8080, - # Database configuration - db_type="mongodb", - db_connection_string="mongodb://localhost:27017", - db_database_name="spatial_db", - # CORS settings - cors_enabled=True, - cors_origins=["https://myapp.com", "http://localhost:3000"], - # API documentation - docs_url="/api/docs", - redoc_url="/api/redoc", - log_level="info" -) +--- -production_server = Server(config=advanced_config) -``` +## 10. Configuration and Environment -### @walker_endpoint Decorator for Walker Classes - -The `@walker_endpoint` decorator automatically exposes Walker classes as API endpoints: - -```python path=null start=null -from jvspatial.api import endpoint -from jvspatial.api.decorators import EndpointField -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit -from typing import List, Optional - -# Define your node types -class User(Node): - name: str = "" - email: str = "" - department: str = "" - active: bool = True - -class City(Node): - name: str = "" - population: int = 0 - state: str = "" - -# Walker with endpoint configuration using EndpointField -@endpoint("/api/users/process", methods=["POST"]) -class ProcessUser(Walker): - """Process user data with graph traversal.""" - - # Endpoint-exposed fields with configuration - user_name: str = EndpointField( - description="Name of the user to process", - examples=["John Doe", "Jane Smith"], - min_length=2, - max_length=100 - ) - - department: str = EndpointField( - default="general", - description="User department", - examples=["engineering", "marketing", "sales"] - ) - - include_inactive: bool = EndpointField( - default=False, - description="Whether to include inactive users in processing" - ) - - # Field excluded from API endpoint - internal_state: str = EndpointField( - default="processing", - exclude_endpoint=True # Won't appear in API schema - ) - - # Optional configuration field - max_connections: int = EndpointField( - default=10, - description="Maximum number of connections to traverse", - ge=1, - le=100 - ) - - @on_visit("User") - async def process_user(self, here: User): - """Process user nodes during traversal. - - Args: - here: The visited User node - """ - # Check if user matches criteria - if here.name == self.user_name: - self.report({ - "found_user": { - "id": here.id, - "name": here.name, - "email": here.email, - "department": here.department - } - }) - - # Find connected users in same department - colleagues = await here.nodes( - node=['User'], - department=self.department, - active=True if not self.include_inactive else None - ) - - self.report({ - "colleagues": [ - {"name": u.name, "email": u.email} - for u in colleagues[:self.max_connections] - ] - }) - - @on_visit("City") - async def process_city_connection(self, here: City): - """Process city connections if user has location data. - - Args: - here: The visited City node - """ - self.report({ - "city_info": { - "name": here.name, - "population": here.population, - "state": here.state - } - }) - -# Advanced walker with field grouping -@endpoint("/api/analytics/user-report", methods=["POST"]) -class UserAnalytics(Walker): - """Generate user analytics reports.""" - - # Grouped fields for better API organization - report_type: str = EndpointField( - description="Type of report to generate", - examples=["summary", "detailed", "connections"], - endpoint_group="report_config" - ) - - date_range: str = EndpointField( - default="30d", - description="Date range for report", - examples=["7d", "30d", "90d", "1y"], - endpoint_group="report_config" - ) - - include_inactive: bool = EndpointField( - default=False, - description="Include inactive users", - endpoint_group="filters" - ) - - departments: List[str] = EndpointField( - default_factory=list, - description="Departments to include (empty = all)", - examples=[["engineering", "product"], ["marketing"]], - endpoint_group="filters" - ) - - @on_visit("User") - async def analyze_user(self, here: User): - """Analyze user data for report. - - Args: - here: The visited User node - """ - # Filter by department if specified - if self.departments and here.department not in self.departments: - return - - # Filter by active status if needed - if not self.include_inactive and not here.active: - return - - # Report individual user analysis - self.report({ - "user_analyzed": { - "id": here.id, - "name": here.name, - "department": here.department, - "active": here.active - } - }) - } -``` +### 10.1 ServerConfig -### Enhanced Response Handling with endpoint.response() - -The `@walker_endpoint` and `@endpoint` decorators now automatically inject semantic response helpers to make crafting HTTP responses clean and flexible: - -**Walker Endpoints with self.endpoint:** - -```python path=null start=null -@endpoint("/api/users/profile", methods=["POST"]) -class UserProfileWalker(Walker): - """Walker demonstrating semantic response patterns.""" - - user_id: str = EndpointField(description="User ID to retrieve") - include_details: bool = EndpointField( - default=False, - description="Include detailed profile information" - ) - - @on_visit("User") - async def get_user_profile(self, here: User): - """Get user profile with semantic responses.""" - if here.id != self.user_id: - return # Continue traversal - - # User not found scenario - if not here.data: - return self.endpoint.not_found( - message="User profile not found", - details={"user_id": self.user_id} - ) - - # Authorization check - if here.private and not self.include_details: - return self.endpoint.forbidden( - message="Insufficient permissions", - details={"required_permission": "view_details"} - ) - - # Successful response - profile_data = { - "id": here.id, - "name": here.name, - "email": here.email - } - - if self.include_details: - profile_data["department"] = here.department - profile_data["created_at"] = here.created_at - - return self.endpoint.success( - data=profile_data, - message="User profile retrieved successfully" - ) - -@endpoint("/api/users/create", methods=["POST"]) -class CreateUserWalker(Walker): - """Walker for creating users with proper HTTP status codes.""" - - name: str = EndpointField(description="User name") - email: str = EndpointField(description="User email") - - @on_visit("Root") - async def create_user(self, here): - """Create a new user with validation.""" - - # Validation example - if "@" not in self.email: - return self.endpoint.unprocessable_entity( - message="Invalid email format", - details={"email": self.email} - ) - - # Check for conflicts - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await User.find({"context.email": self.email}) - existing_user = users[0] if users else None - if existing_user: - return self.endpoint.conflict( - message="User with this email already exists", - details={"email": self.email} - ) - - # Create user - user = await User.create( - name=self.name, - email=self.email - ) - - # Return 201 Created with location header - return self.endpoint.created( - data={ - "id": user.id, - "name": user.name, - "email": user.email - }, - message="User created successfully", - headers={"Location": f"/api/users/{user.id}"} - ) -``` +`jvspatial/api/config.py` — Pydantic model. Hierarchical groups: -**Function Endpoints with endpoint parameter:** - -```python path=null start=null -@endpoint("/api/health", methods=["GET"]) -async def health_check(endpoint) -> Any: - """Health check with semantic response.""" - return endpoint.success( - data={"status": "healthy", "version": "1.0.0"}, - message="Service is running normally" - ) - -@endpoint("/api/users/{user_id}/status", methods=["PUT"]) -async def update_user_status(user_id: str, status: str, endpoint) -> Any: - """Update user status with validation and error handling.""" - - # Validation - valid_statuses = ["active", "inactive", "suspended"] - if status not in valid_statuses: - return endpoint.bad_request( - message="Invalid status value", - details={"provided": status, "valid_options": valid_statuses} - ) - - # Find user - user = await User.get(user_id) - if not user: - return endpoint.not_found( - message="User not found", - details={"user_id": user_id} - ) - - # Update status - user.status = status - await user.save() - - return endpoint.success( - data={"id": user.id, "status": user.status}, - message=f"User status updated to {status}" - ) - -@endpoint("/api/export", methods=["GET"]) -async def export_data(format: str, endpoint) -> Any: - """Export data with custom response formatting.""" - - if format not in ["json", "csv", "xml"]: - return endpoint.error( - message="Unsupported export format", - status_code=406, # Not Acceptable - details={"format": format, "supported": ["json", "csv", "xml"]} - ) - - # Generate export data - export_data = { - "format": format, - "records": 1500, - "export_id": "exp_20250921", - "download_url": f"/downloads/export.{format}" - } - - # Use flexible response() method for custom headers - return endpoint.response( - content={ - "data": export_data, - "message": f"Export ready in {format} format" - }, - status_code=200, - headers={ - "X-Export-Format": format, - "X-Record-Count": "1500" - } - ) ``` - -**Available Response Methods:** - -```python path=null start=null -# Success responses -endpoint.success(data=result, message="Success") # 200 OK -endpoint.created(data=new_item, message="Created") # 201 Created -endpoint.no_content() # 204 No Content - -# Error responses -endpoint.bad_request(message="Invalid input") # 400 Bad Request -endpoint.unauthorized(message="Auth required") # 401 Unauthorized -endpoint.forbidden(message="Access denied") # 403 Forbidden -endpoint.not_found(message="Resource not found") # 404 Not Found -endpoint.conflict(message="Resource exists") # 409 Conflict -endpoint.unprocessable_entity(message="Validation failed") # 422 Unprocessable Entity - -# Flexible custom response -endpoint.response( - content={"custom": "data"}, - status_code=202, - headers={"X-Custom": "value"} +ServerConfig( + title, description, version, debug, docs_url, redoc_url, + host, port, serverless_mode, deferred_task_provider, + scheduler_enabled, scheduler_interval, + database=DatabaseConfig(...), + security=SecurityConfig(...), + cors=CORSConfig(...), + auth=AuthConfig(...), + rate_limit=RateLimitConfig(...), + file_storage=FileStorageConfig(...), + webhook=WebhookConfig(...), + proxy=ProxyConfig(...), + log_level, startup_hooks, shutdown_hooks, + graph_endpoint_enabled=True, ) - -# Generic error with custom status code -endpoint.error( - message="Custom error", - status_code=418, # I'm a teapot - details={"reason": "custom"} -) -``` - -### @endpoint Decorator for Regular Functions -The `@endpoint` decorator exposes regular functions as API endpoints: - -```python path=null start=null -from jvspatial.api import endpoint -from fastapi import HTTPException -from typing import Dict, Any, List - -# Simple function endpoint -@endpoint("/api/users/count", methods=["GET"]) -async def get_user_count() -> Dict[str, int]: - """Get total count of users in the system.""" - users = await User.all() - return {"total_users": len(users)} - -# Function endpoint with path parameters -@endpoint("/api/cities/{state}", methods=["GET"]) -async def get_cities_by_state(state: str) -> Dict[str, Any]: - """Get all cities in a specific state.""" - cities = await City.find({"context.state": state}) - - if not cities: - raise HTTPException( - status_code=404, - detail=f"No cities found in state: {state}" - ) - - return { - "state": state, - "cities": [ - { - "name": city.name, - "population": city.population - } for city in cities - ], - "total_count": len(cities) - } - -# Function endpoint with request body -@endpoint("/api/cities/search", methods=["POST"]) -async def search_cities(search_request: Dict[str, Any]) -> Dict[str, Any]: - """Search cities based on criteria.""" - # Extract search parameters - name_pattern = search_request.get("name_pattern") - min_population = search_request.get("min_population", 0) - state = search_request.get("state") - - # Build MongoDB-style query - query = {} - - if name_pattern: - query["context.name"] = {"$regex": name_pattern, "$options": "i"} - - if min_population > 0: - query["context.population"] = {"$gte": min_population} - - if state: - query["context.state"] = state - - # Execute search - cities = await City.find(query) - - return { - "search_criteria": search_request, - "results": [ - { - "id": city.id, - "name": city.name, - "population": city.population, - "state": city.state - } for city in cities - ], - "total_results": len(cities) - } - -# Function endpoint with pagination integration -@endpoint("/api/users/paginated", methods=["GET"]) -async def get_users_paginated( - page: int = 1, - page_size: int = 20, - department: Optional[str] = None, - active_only: bool = True -) -> Dict[str, Any]: - """Get paginated list of users with filtering.""" - from jvspatial.core.pager import ObjectPager - - # Build filters - filters = {} - if department: - filters["context.department"] = department - if active_only: - filters["context.active"] = True - - # Create pager - pager = ObjectPager( - User, - page_size=page_size, - filters=filters, - order_by="name", - order_direction="asc" - ) - - # Get page data - users = await pager.get_page(page) - pagination_info = pager.to_dict() - - return { - "users": [ - { - "id": user.id, - "name": user.name, - "email": user.email, - "department": user.department, - "active": user.active - } for user in users - ], - "pagination": pagination_info - } -``` - -### Server Method Registration - -You can also register endpoints directly on server instances: - -```python path=null start=null -# Using server instance decorators -@server.walker("/process-data", methods=["POST"]) -class DataProcessor(Walker): - """Process data using server instance registration.""" - - data_type: str = EndpointField( - description="Type of data to process", - examples=["user", "city", "connection"] - ) - - batch_size: int = EndpointField( - default=10, - description="Batch size for processing", - ge=1, - le=1000 - ) - - @on_visit("Node") - async def process_any_node(self, here: Node): - """Process any type of node. - - Args: - here: The visited Node - """ - # Use report() to collect processed node information - self.report({ - "processed_node": { - "id": here.id, - "type": here.__class__.__name__, - "processed_at": datetime.now().isoformat() - } - }) - -@server.route("/health-detailed", methods=["GET"]) -async def detailed_health_check() -> Dict[str, Any]: - """Detailed health check endpoint.""" - try: - # Test database connectivity - users_count = await User.count() - cities_count = await City.count() - - return { - "status": "healthy", - "database": "connected", - "statistics": { - "total_users": users_count, - "total_cities": cities_count - }, - "timestamp": datetime.now().isoformat() - } - except Exception as e: - raise HTTPException( - status_code=503, - detail=f"Health check failed: {str(e)}" - ) -``` - -### EndpointField Configuration Options - -The `EndpointField` provides extensive configuration for API parameters: - -```python path=null start=null -from jvspatial.api.decorators import EndpointField -from typing import List, Optional - -@endpoint("/api/advanced-example", methods=["POST"]) -class AdvancedEndpointExample(Walker): - """Demonstrate all EndpointField configuration options.""" - - # Basic field with validation - username: str = EndpointField( - description="User identifier", - examples=["john_doe", "jane_smith"], - min_length=3, - max_length=50, - pattern=r"^[a-zA-Z0-9_]+$" # Alphanumeric with underscores - ) - - # Numeric field with constraints - user_score: float = EndpointField( - description="User score rating", - examples=[85.5, 92.0, 78.3], - ge=0.0, # Greater than or equal to 0 - le=100.0 # Less than or equal to 100 - ) - - # Field with custom endpoint name - internal_id: str = EndpointField( - endpoint_name="user_id", # Will appear as 'user_id' in API - description="Internal user identifier" - ) - - # Optional field made required for endpoint - optional_field: Optional[str] = EndpointField( - default=None, - endpoint_required=True, # Required in API despite being Optional in Walker - description="Field that's optional in Walker but required in API" - ) - - # Required field made optional for endpoint - required_field: str = EndpointField( - endpoint_required=False, # Optional in API despite being required in Walker - description="Field that's required in Walker but optional in API" - ) - - # Grouped fields for organized API schema - config_timeout: int = EndpointField( - default=30, - description="Timeout in seconds", - endpoint_group="configuration", - ge=1, - le=300 - ) - - config_retries: int = EndpointField( - default=3, - description="Number of retries", - endpoint_group="configuration", - ge=0, - le=10 - ) - - # Hidden field (not shown in API docs) - debug_mode: bool = EndpointField( - default=False, - endpoint_hidden=True, # Hidden from OpenAPI documentation - description="Debug mode flag" - ) - - # Deprecated field - legacy_option: Optional[str] = EndpointField( - default=None, - endpoint_deprecated=True, # Marked as deprecated in API docs - description="Legacy option - use new_option instead" - ) - - # Field with additional constraints - email_domain: str = EndpointField( - description="Allowed email domain", - endpoint_constraints={ - "format": "hostname", # Additional OpenAPI constraint - "example": "company.com" - } - ) - - # List field with validation - tags: List[str] = EndpointField( - default_factory=list, - description="User tags", - examples=[["admin", "power-user"], ["guest"]] - ) - - @on_visit("User") - async def process_with_config(self, here: User): - """Process user with advanced configuration. - - Args: - here: The visited User node - """ - self.report({ - "processed_user": { - "username": self.username, - "score": self.user_score, - "config": { - "timeout": self.config_timeout, - "retries": self.config_retries - }, - "tags": self.tags, - "debug_enabled": self.debug_mode - } - }) ``` -### Running the Server - -```python path=null start=null -# Development server -if __name__ == "__main__": - server.run( - host="0.0.0.0", - port=8000, - reload=True # Auto-reload for development - ) - -# Production server with custom configuration -async def run_production_server(): - """Run server asynchronously for production deployment.""" - await server.run_async( - host="0.0.0.0", - port=8080 - ) - -# Custom startup and shutdown hooks -@server.on_startup -async def startup_tasks(): - """Tasks to run on server startup.""" - print("🚀 Server starting up...") - # Initialize data, warm up caches, etc. - -@server.on_shutdown -async def shutdown_tasks(): - """Tasks to run on server shutdown.""" - print("🛑 Server shutting down...") - # Cleanup resources, save state, etc. - -# Access the underlying FastAPI app if needed -fastapi_app = server.get_app() - -# Custom middleware -@server.middleware("http") -async def log_requests(request, call_next): - """Log all API requests.""" - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - print(f"{request.method} {request.url} - {response.status_code} ({process_time:.2f}s)") - return response -``` - -### Router Decorators - -jvspatial provides four standard router decorators for API endpoints. These are the ONLY decorators that should be used for routing: +Flat keyword arguments (e.g. `Server(db_type=..., jwt_secret=...)`) are mapped to the appropriate group by a model validator (lines 103-128). The flat form is convenient; the hierarchical form is canonical. -1. `@endpoint` - For public endpoints (both functions and Walker classes) -2. `@endpoint(..., auth=True)` - For authenticated endpoints (both functions and Walker classes) -3. `@endpoint(..., webhook=True)` - For webhook endpoints (both functions and Walker classes) +### 10.2 Merge order -```python -# Function endpoint -@endpoint("/api/users", methods=["GET"]) -async def get_users() -> Dict[str, Any]: - users = await User.all() - return {"users": users} - -# Walker endpoint -@endpoint("/api/graph/traverse", methods=["POST"]) -class GraphTraversal(Walker): - pass - -# Authenticated function endpoint -@endpoint("/api/admin/stats", auth=True, methods=["GET"], roles=["admin"]) -async def get_admin_stats() -> Dict[str, Any]: - return {"stats": "admin only"} - -# Authenticated walker endpoint (uses same decorator) -@endpoint("/api/secure/process", auth=True, methods=["POST"], permissions=["process_data"]) -class SecureProcessor(Walker): - pass -``` +`Server(...)` arguments merge in this order (later wins): -DO NOT use alternative decorators like `@route`, `@server.route`, or `@server.walker`. These are internal or deprecated. +1. `ServerConfig` defaults +2. Allowlisted `JVSPATIAL_*` environment variables (see `jvspatial/env_adapter.py`) +3. `config=` dict or keyword arguments -### API Usage Examples +Unknown `JVSPATIAL_*` keys are rejected at startup to catch typos and removed settings. -Once your server is running, endpoints are automatically available: +### 10.3 Environment variable allowlist -```bash -# Walker endpoint - POST request with parameters -curl -X POST "http://localhost:8000/api/users/process" \ - -H "Content-Type: application/json" \ - -d '{ - "user_name": "John Doe", - "department": "engineering", - "include_inactive": false, - "max_connections": 5, - "start_node": "n:Root:root" - }' +`jvspatial/env_adapter.py` lists every accepted `JVSPATIAL_*` key with type coercion rules. The canonical reference is [docs/md/environment-keys-reference.md](docs/md/environment-keys-reference.md). -# Function endpoint - GET request -curl "http://localhost:8000/api/users/count" +### 10.4 Database environment -# Function endpoint with path parameters -curl "http://localhost:8000/api/cities/CA" +`jvspatial/env.py` provides `resolve_db_paths()`, `parse_bool()`, and friends. Database type defaults: +- `JVSPATIAL_DB_TYPE` — backend selector +- `JVSPATIAL_DB_PATH` — backend-specific path or URI -# Function endpoint with query parameters -curl "http://localhost:8000/api/users/paginated?page=1&page_size=10&department=engineering" +### 10.5 Docs gating -# API documentation is automatically available -# http://localhost:8000/docs (Swagger UI) -# http://localhost:8000/redoc (ReDoc) -``` - -### Best Practices - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Use descriptive endpoint paths -@endpoint("/api/users/analyze-connections", methods=["POST"]) -class AnalyzeUserConnections(Walker): - pass - -# Good: Provide comprehensive field documentation -field_name: str = EndpointField( - description="Clear description of what this field does", - examples=["example1", "example2"], - min_length=1, - max_length=100 -) +`JVSPATIAL_DOCS_DISABLED` (truthy values: `1`, `true`, `yes`, `on`) disables `/docs`, `/redoc`, `/openapi.json`, and `/docs/oauth2-redirect` at app build time. CSP headers are relaxed only on docs paths to allow the Swagger UI CDN; app routes retain strict CSP. -# Good: Use appropriate HTTP methods -@endpoint("/api/users", methods=["GET"]) # Retrieve data -@endpoint("/api/users", methods=["POST"]) # Create data -@endpoint("/api/process", methods=["POST"]) # Process/execute - -# Good: Group related fields -config_field: str = EndpointField( - endpoint_group="configuration", - description="Configuration parameter" -) - -# Good: Handle errors appropriately in functions -@endpoint("/api/data") -async def get_data(): - try: - data = await fetch_data() - return {"data": data} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) -``` - -**❌ Avoided Patterns:** - -```python path=null start=null -# Bad: Vague endpoint paths -@endpoint("/process") # Too generic -@endpoint("/api/thing") # Unclear purpose - -# Bad: Missing field documentation -field_name: str = EndpointField() # No description or examples - -# Bad: Exposing internal fields -internal_state: str = EndpointField() # Should use exclude_endpoint=True - -# Bad: Not handling errors in function endpoints -@endpoint("/api/data") -async def get_data(): - data = await risky_operation() # No error handling - return data # Could fail with 500 error - -# Bad: Using wrong HTTP methods -@endpoint("/api/users/delete", methods=["GET"]) # Should be DELETE -@endpoint("/api/data/get", methods=["POST"]) # Should be GET for retrieval -``` +**Source of truth**: `jvspatial/api/components/app_builder.py`. --- -## 🏗️ Library Architecture Concepts - -**jvspatial** is an asynchronous, object-spatial Python library designed for building robust persistence and business logic application layers. Inspired by Jaseci's object-spatial paradigm and leveraging Python's async capabilities, jvspatial empowers developers to model complex relationships, traverse object graphs, and implement agent-based architectures that scale with modern cloud-native concurrency requirements. Key capabilities: - -- Typed node/edge modeling via Pydantic -- Precise control over graph traversal -- Multi-backend persistence (JSON, MongoDB, etc.) -- Integrated REST API endpoints -- Async/await architecture - -### Core Entities - -1. **Object** - Base class for all entities with unified query interface; used to store non-graph data -2. **Node** - Extends Object, represents graph nodes with spatial/contextual data; used only as part of a graph -3. **Edge** - Represents relationships between nodes on a graph -4. **Walker** - Implements graph traversal and pathfinding algorithms -5. **GraphContext** - Low-level database interface (use sparingly) - -Once a graph is established (nodes and edges are connected in a meaningful way), a walker may be spawned on the root node or anywhere on the graph. The visit method enacts the walker's traversal using a starting point or a list of nodes on the walker's visit queue. - -As the walker traverses, it may conditionally trigger methods depending on its position on the graph. This is accomplished by the @on_visit annotation on the walker class. Similarly, as the walker traverses over nodes and edges, these entities may conditionally trigger their methods based on the walker's visitation; also accomplished through the @on_visit annotation on the node/edge class. - -**Execution Order**: When a walker visits a node/edge: -1. **Walker hooks** (methods decorated with `@on_visit` on the walker class) execute first -2. **Node/Edge hooks** (methods decorated with `@on_visit` on the node/edge class) are automatically executed after - -Node/Edge hooks are automatically invoked by the walker - no explicit call is needed. The walker binds the hook to the node/edge instance and calls it with the walker as a parameter. - -### Walker Traversal Pattern - -The **recommended approach** for walker traversal is to use the `nodes()` method to get connected nodes for continued traversal. - -#### Naming Convention for @on_visit Methods +## 11. Serverless Constraints -**IMPORTANT**: When writing `@on_visit` decorated methods, use the following naming convention: +### 11.1 Detection precedence -- **`here`**: Parameter name for the visited node/edge (the current location) -- **`visitor`**: Parameter name for the visiting walker (when accessing walker from node context) +`is_serverless_mode(config=None)` (`jvspatial/runtime/serverless.py:66-87`) resolves in this order: -```python path=null start=null -# ✅ RECOMMENDED: Use 'here' for the visited node -@on_visit("User") -async def process_user(self, here: Node): - """Process user node - 'here' represents the current User being visited. +1. Explicit `config.serverless_mode` if set (not `None`) +2. Same via `get_current_server()` when `config` is omitted +3. `SERVERLESS_MODE` env var (`true`/`1`/`yes`/`enabled`) +4. Auto-detection from platform env vars (Lambda, Azure Functions, Cloud Run, Vercel) - Args: - here: The visited User node - """ - print(f"Currently visiting user: {here.name}") - connected_users = await here.nodes(node=['User']) - await self.visit(connected_users) +Detection results are memoized via `lru_cache`; tests call `reset_serverless_mode_cache()` between cases. -# ✅ RECOMMENDED: Use 'visitor' when walker is passed to node methods -def node_method_example(self, visitor: Walker): - """Node method that receives the visiting walker. +### 11.2 Mode-dependent behavior - Args: - visitor: The walker currently visiting this node - """ - print(f"Being visited by walker: {visitor.__class__.__name__}") +| Behavior | Standard | Serverless | +|---|---|---| +| Deferred saves enabled by env | Yes (`JVSPATIAL_ENABLE_DEFERRED_SAVES`) | **No (forced off)** | +| Default JSON DB path | Working directory | `/tmp` (Lambda-writable) | +| bcrypt rounds | 12 | 10 (faster cold start) | +| Auto-create DB indexes | Yes | No (CloudWatch log cost) | +| JsonDB orphan cleanup | Yes | Skipped | +| Walker trail persistence | In-memory only | In-memory only (lost on cold start) | -# ❌ AVOID: Generic parameter names -@on_visit("User") -async def process_user(self, node: Node): # Less clear - pass +### 11.3 Deferred task dispatch -@on_visit("User") -async def process_user(self, n: Node): # Too abbreviated - pass -``` - -**Why this convention?** -- **`here`** clearly indicates the current location in graph traversal -- **`visitor`** clearly indicates the active walker performing the traversal -- Consistent with spatial/navigational metaphors used throughout jvspatial -- Makes code more readable and self-documenting -- Aligns with the library's entity-centric philosophy - -```python -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit, on_exit - -class DataCollector(Walker): - def __init__(self): - super().__init__() - self.collected_data = [] - self.processed_count = 0 - - @on_visit("User") - async def collect_user_data(self, here: Node): - """Called when walker visits a User node. - - Args: - here: The visited User node - """ - self.collected_data.append(here.name) - - # RECOMMENDED: Use nodes() method to get connected nodes - # Default direction="out" follows outgoing edges (forward traversal) - next_nodes = await here.nodes() - await self.visit(next_nodes) - - # Example with semantic filtering - filter connected Users by department - engineering_users = await here.nodes( - node=['User'], # Only User nodes - department="engineering", # Simple kwargs filtering - active=True # Multiple simple filters - ) - await self.visit(engineering_users) - - @on_visit("City") - async def process_city(self, here: Node): - """Process city nodes with filtering and control flow. - - Args: - here: The visited City node - """ - self.processed_count += 1 - - # Skip processing for certain conditions - if here.population < 10000: - print(f"Skipping small city: {here.name}") - self.skip() # Skip to next node in queue - return # This line won't be reached - - # Disengage if we've processed enough - if self.processed_count >= 10: - print("Processed enough cities, disengaging...") - await self.disengage() # Permanently halt and remove from graph - return - - # Continue with normal processing - large_cities = await here.nodes( - node=[{'City': {"context.population": {"$gte": 500_000}}}], - direction="out" # Explicit direction for clarity - ) - await self.visit(large_cities) - - # Example: Mixed filtering approach - dict filters + kwargs - nearby_cities = await here.nodes( - node=[{'City': {"context.region": here.region}}], # Complex filter - state="NY", # Simple kwargs filter - active=True # Additional simple filter - ) - await self.visit(nearby_cities) - - @on_exit - async def cleanup_and_report(self): - """Called when walker completes or disengages.""" - print(f"Walker finished! Collected {len(self.collected_data)} items") - print(f"Processed {self.processed_count} cities") - # Perform cleanup, save results, send notifications, etc. -``` - -### Walker Control Flow Methods - -#### `skip()` - Skip Current Node Processing -The `skip()` method allows you to immediately halt processing of the current node and proceed to the next node in the queue, similar to `continue` in a loop: - -```python -class ConditionalWalker(Walker): - @on_visit("Product") - async def process_product(self, here: Node): - """Process product nodes with conditional skipping. - - Args: - here: The visited Product node - """ - # Skip discontinued products - if here.status == "discontinued": - self.skip() # Jump to next node in queue - # Code below won't execute - - # Skip products outside price range - if not (10 <= here.price <= 1000): - print(f"Skipping {here.name} - price out of range") - self.skip() - - # Normal processing for valid products - print(f"Processing product: {here.name}") - connected_products = await here.nodes(node=['Product']) - await self.visit(connected_products) -``` - -#### `disengage()` - Permanently Halt and Remove Walker from Graph -The `disengage()` method permanently halts the walker and removes it from the graph. Once disengaged, the walker **cannot be resumed** and is considered finished: +`jvspatial/serverless/` — `dispatch_deferred_task(task_type, payload)` invokes a registered async handler out-of-band. On AWS, transport is Lambda async-invoke or EventBridge (configurable). -```python -class CompletionWalker(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.max_items = 100 - self.critical_error = False - - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with completion tracking. - - Args: - here: The visited Document node - """ - try: - # Process document - await self.process_item(here) - self.processed_count += 1 - - # Disengage when reaching target or on critical error - if self.processed_count >= self.max_items: - print(f"Target reached: {self.processed_count} items processed") - await self.disengage() # Permanently finish - return - - if self.critical_error: - print("Critical error encountered, disengaging walker") - await self.disengage() # Permanently halt due to error - return - - # Continue to next documents - next_docs = await here.nodes(node=['Document']) - await self.visit(next_docs) - - except CriticalError as e: - print(f"Critical error: {e}") - self.critical_error = True - await self.disengage() # Permanently halt on critical error - - @on_exit - async def final_cleanup(self): - """Called when walker disengages - perform final cleanup.""" - print(f"Walker disengaged. Final count: {self.processed_count}") - # Perform final cleanup, save state, notify completion - await self.save_final_results() - - async def process_item(self, node): - """Process individual item.""" - # Simulate processing that might fail - if node.status == "corrupted": - raise CriticalError("Corrupted data detected") - await asyncio.sleep(0.01) - - async def save_final_results(self): - """Save final processing results.""" - print("💾 Saving final results...") - -# Usage - disengage() creates permanent termination -walker = CompletionWalker() -root = await Root.get(None) - -# Start and run to completion (or error) -walker = await walker.spawn(root) -print(f"Walker finished. Status: {'disengaged' if walker.paused else 'completed'}") - -# NOTE: Once disengaged, walker cannot be resumed -# walker.resume() would not work - walker is permanently finished - -# For pausable/resumable patterns, use different approaches: -# - Save walker state and create new walker instances -# - Use external queue/state management -# - Implement custom pause/resume logic in @on_visit methods -``` - -#### `pause()` and `resume()` - Temporary Walker Suspension -Walkers can be paused during traversal and resumed later, preserving their queue and state. Unlike `disengage()`, paused walkers can be resumed: +Register handlers with `@deferred_invoke_handler("task.name")`. Handlers **must be idempotent**; the framework provides no exactly-once guarantee. -```python -class BatchProcessor(Walker): - def __init__(self): - super().__init__() - self.processed_count = 0 - self.batch_size = 50 - self.total_batches = 0 - - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with batch pausing. - - Args: - here: The visited Document node - """ - # Process document - await self.heavy_processing(here) - self.processed_count += 1 - - # Pause after processing a batch - if self.processed_count % self.batch_size == 0: - self.total_batches += 1 - print(f"Batch {self.total_batches} complete ({self.processed_count} items)") - print("Pausing for rate limiting...") - - # Clean pause using pause() method - self.pause(f"Batch {self.total_batches} processing pause") - - # Continue to next documents - next_docs = await here.nodes(node=['Document'], status="active") - await self.visit(next_docs) - - async def heavy_processing(self, node): - """Simulate expensive processing.""" - await asyncio.sleep(0.1) # Simulate API calls, file I/O, etc. - - @on_exit - async def report_completion(self): - """Called when walker completes or is paused.""" - if self.paused: - print(f"Walker paused after processing {self.processed_count} items") - else: - print(f"Walker completed! Total processed: {self.processed_count}") - -# Usage - pause and resume cycle -walker = BatchProcessor() -root = await Root.get(None) - -# Start processing - will pause after first batch -walker = await walker.spawn(root) -print(f"Walker state: {'paused' if walker.paused else 'completed'}") -print(f"Queue remaining: {len(walker.queue)} items") - -# Simulate processing delay (rate limiting, API cooldown, etc.) -print("\n⏳ Waiting for rate limit cooldown...") -await asyncio.sleep(2.0) - -# Resume processing - will continue from where it left off -print("\n▶️ Resuming processing...") -walker = await walker.resume() -print(f"Walker state after resume: {'paused' if walker.paused else 'completed'}") - -# Can resume multiple times if walker gets paused again -while walker.paused and walker.queue: - print(f"\n🔄 Walker paused again, {len(walker.queue)} items remaining") - await asyncio.sleep(1.0) # Brief pause - walker = await walker.resume() - -print("\n✅ All processing complete!") -``` +### 11.4 Lambda Web Adapter -#### Advanced Pause/Resume Patterns +When LWA is detected, `Server` applies best-effort defaults for `AWS_LWA_PASS_THROUGH_PATH` and `AWS_LWA_INVOKE_MODE`. The LWA extension reads these *before* Python starts, so IaC should still set them explicitly for reliability. -```python -class SmartProcessor(Walker): - def __init__(self): - super().__init__() - self.api_calls = 0 - self.max_api_calls = 100 - self.error_count = 0 - self.max_errors = 5 - - @on_visit("DataNode") - async def process_data(self, here: Node): - """Process data nodes with smart rate limiting. - - Args: - here: The visited DataNode - """ - try: - # Check rate limits - if self.api_calls >= self.max_api_calls: - print(f"Rate limit reached ({self.api_calls} calls), pausing...") - self.api_calls = 0 # Reset counter - self.pause("Rate limit reached") - - # Check error threshold - if self.error_count >= self.max_errors: - print(f"Too many errors ({self.error_count}), pausing for investigation") - self.pause(f"Error threshold reached: {self.error_count} errors") - - # Process node - result = await self.call_external_api(here) - self.api_calls += 1 - - # Continue traversal based on result - if result.should_continue: - related_nodes = await here.nodes( - node=['DataNode'], - status="pending", - priority={"$gte": result.priority_threshold} - ) - await self.visit(related_nodes) - - except ApiError as e: - self.error_count += 1 - print(f"API error #{self.error_count}: {e}") - # Continue processing - don't pause on single errors - - except CriticalError as e: - print(f"Critical error: {e}") - self.pause(f"Critical error: {e}") - - async def call_external_api(self, node): - """Simulate external API call that might fail.""" - await asyncio.sleep(0.05) # Simulate API latency - # Simulate occasional failures - if random.random() < 0.1: # 10% failure rate - raise ApiError("Temporary API failure") - return APIResult(should_continue=True, priority_threshold=5) - - def pause_for_maintenance(self): - """Manually pause walker for maintenance.""" - print("🔧 Pausing for scheduled maintenance...") - self.paused = True - - @on_exit - async def maintenance_check(self): - """Check if maintenance is needed when paused.""" - if self.paused: - print("Walker paused - performing maintenance checks...") - # Reset error counters, clear caches, etc. - self.error_count = 0 - print("Maintenance complete - ready to resume") - -# Usage with external control -walker = SmartProcessor() -root = await Root.get(None) - -# Start processing -print("🚀 Starting smart processing...") -walker = await walker.spawn(root) - -# External monitoring and control -while walker.paused and walker.queue: - print(f"⏸️ Walker paused - {len(walker.queue)} items remaining") - - # Simulate external decision making - if should_resume_processing(): # Your logic here - print("🔄 Conditions met, resuming...") - walker = await walker.resume() - else: - print("⏳ Waiting for conditions to improve...") - await asyncio.sleep(5.0) - -# Helper classes for example -class ApiError(Exception): pass -class CriticalError(Exception): pass -class APIResult: - def __init__(self, should_continue, priority_threshold): - self.should_continue = should_continue - self.priority_threshold = priority_threshold - -def should_resume_processing(): - """External logic to determine if processing should resume.""" - return True # Simplified for example -``` - -#### `@on_exit` - Cleanup and Finalization -The `@on_exit` decorator marks methods to execute when the walker completes traversal or disengages: - -```python -class AnalyticsWalker(Walker): - def __init__(self): - super().__init__() - self.start_time = time.time() - self.nodes_visited = 0 - self.errors_encountered = 0 - - @on_visit("User") - async def analyze_user(self, here: Node): - """Analyze user behavior and traverse to related users. - - Args: - here: The visited User node - """ - try: - self.nodes_visited += 1 - # Perform analysis - await self.analyze_user_behavior(here) - - # Continue traversal - related_users = await here.nodes(node=['User']) - await self.visit(related_users) - except Exception as e: - self.errors_encountered += 1 - print(f"Error processing user {here.id}: {e}") - - @on_exit - async def generate_report(self): - """Generate analytics report when traversal completes.""" - duration = time.time() - self.start_time - print("\n📊 Analytics Report") - print(f"Duration: {duration:.2f} seconds") - print(f"Nodes visited: {self.nodes_visited}") - print(f"Errors: {self.errors_encountered}") - print(f"Success rate: {(1 - self.errors_encountered/max(self.nodes_visited, 1))*100:.1f}%") - - @on_exit - def save_results(self): - """Save results to database (sync version).""" - # Save analytics data - print("💾 Saving results to database...") - - @on_exit - async def send_notifications(self): - """Send completion notifications (async version).""" - # Send email/slack notifications - print("📧 Sending completion notifications...") - - async def analyze_user_behavior(self, user): - """Simulate user behavior analysis.""" - await asyncio.sleep(0.01) # Simulate work -``` - -### Walker Reporting System - -Walkers feature a simplified reporting system that allows you to collect and aggregate any data during traversal. The reporting system eliminates complex nested structures and provides direct access to collected data. - -#### Basic Reporting - -```python path=null start=null -from jvspatial.core import Walker, on_visit, on_exit - -class DataCollectionWalker(Walker): - """Walker demonstrating the simple reporting system.""" - - def __init__(self): - super().__init__() - self.processed_count = 0 - - @on_visit('User') - async def collect_user_data(self, here: Node): - """Collect user data using the report system.""" - # Report any data - strings, dicts, numbers, lists, etc. - self.report({ - "user_processed": { - "id": here.id, - "name": here.name, - "department": here.department, - "timestamp": time.time() - } - }) - - # Report simple values - self.report(f"Processed user: {here.name}") - - # Report lists - if hasattr(here, 'skills'): - self.report(["skills", here.skills]) - - self.processed_count += 1 - - @on_exit - async def generate_summary(self): - """Generate final summary in the report.""" - report_items = await self.get_report() - - self.report({ - "summary": { - "total_items_collected": len(report_items), - "users_processed": self.processed_count, - "collection_complete": True - } - }) - -# Usage -walker = DataCollectionWalker() -result_walker = await walker.spawn() # spawn() returns the walker instance - -# Access collected data directly as a simple list -report = await result_walker.get_report() -print(f"Total items collected: {len(report)}") - -# Iterate through all collected data -for item in report: - if isinstance(item, dict) and "user_processed" in item: - user_data = item["user_processed"] - print(f"User: {user_data['name']} from {user_data['department']}") - elif isinstance(item, str): - print(f"Log: {item}") -``` - -#### Advanced Reporting Patterns - -```python path=null start=null -class AnalyticsWalker(Walker): - """Walker with advanced reporting for analytics.""" - - def __init__(self): - super().__init__() - self.department_counts = {} - self.error_count = 0 - - @on_visit('User') - async def analyze_user(self, here: Node): - """Analyze user and report findings.""" - try: - # Track department statistics - dept = here.department or "unknown" - self.department_counts[dept] = self.department_counts.get(dept, 0) + 1 - - # Report individual analysis - analysis = await self.perform_user_analysis(here) - self.report({ - "user_analysis": { - "user_id": here.id, - "department": dept, - "performance_score": analysis.get("score", 0), - "risk_level": analysis.get("risk", "low"), - "recommendations": analysis.get("recommendations", []) - } - }) - - except Exception as e: - self.error_count += 1 - self.report({ - "error": { - "user_id": here.id, - "error_message": str(e), - "error_type": type(e).__name__ - } - }) - - @on_exit - async def generate_analytics_report(self): - """Generate comprehensive analytics.""" - all_data = await self.get_report() - - # Analyze collected data - user_analyses = [item for item in all_data - if isinstance(item, dict) and "user_analysis" in item] - errors = [item for item in all_data - if isinstance(item, dict) and "error" in item] - - # Calculate metrics - avg_score = sum(ua["user_analysis"]["performance_score"] - for ua in user_analyses) / len(user_analyses) if user_analyses else 0 - - high_risk_users = [ua for ua in user_analyses - if ua["user_analysis"]["risk_level"] == "high"] - - # Report final analytics - self.report({ - "final_analytics": { - "total_users_analyzed": len(user_analyses), - "average_performance_score": round(avg_score, 2), - "department_breakdown": self.department_counts, - "high_risk_users_count": len(high_risk_users), - "error_rate": self.error_count / max(len(user_analyses), 1), - "processing_summary": { - "success": len(user_analyses), - "errors": self.error_count, - "total_items_in_report": len(all_data) - } - } - }) - - async def perform_user_analysis(self, user): - """Simulate user analysis.""" - import random - return { - "score": random.randint(1, 100), - "risk": random.choice(["low", "medium", "high"]), - "recommendations": ["Update profile", "Complete training"] - } -``` - -### Walker Event System - -Walkers can communicate with each other during traversal using an event system. This enables real-time coordination, data sharing, and complex multi-walker workflows. +--- -#### Basic Event Communication +## 12. File Storage -```python path=null start=null -from jvspatial.core.events import on_emit +`jvspatial/storage/` — abstract `FileStorageInterface` (`interfaces/base.py`) with two built-in implementations: -class AlertWalker(Walker): - """Walker that emits alerts when finding critical issues.""" +| Adapter | File | Backing | Versioning | +|---|---|---|---| +| Local | `interfaces/local.py` | Filesystem | Version directory per file | +| S3 | `interfaces/s3.py` | AWS S3 | Native S3 versioning + multipart ≥8 MiB | - @on_visit('SystemNode') - async def check_system_health(self, here: Node): - """Check system health and emit alerts.""" - if hasattr(here, 'cpu_usage') and here.cpu_usage > 90: - # Emit event to other walkers - await self.emit("high_cpu_alert", { - "node_id": here.id, - "cpu_usage": here.cpu_usage, - "severity": "critical", - "walker_id": self.id - }) +### 12.1 Security layer - self.report({"alert_sent": f"High CPU on {here.id}"}) +- `storage/security/path_sanitizer.py` — five-stage validation: regex blocklist (11 patterns), normalization with re-check, hidden-file allowlist, symlink resolution, base-directory confinement. +- `storage/security/validator.py` — content-based MIME via `python-magic`. ~25 allowed types, 14 blocked types, 19 blocked extensions. Internal markers bypassed via metadata validation only (never user-supplied). -class MonitoringWalker(Walker): - """Walker that receives and processes alerts from other walkers.""" +### 12.2 Upload contract - def __init__(self): - super().__init__() - self.alerts_received = 0 +All uploads pass through `path_sanitizer` then `validator`. Failures raise `ValidationError` with `field_errors` populated. Successful uploads return a versioned descriptor. - @on_emit("high_cpu_alert") - async def handle_cpu_alert(self, event_data): - """Handle high CPU alerts from AlertWalker.""" - self.alerts_received += 1 +### 12.3 S3 specifics - self.report({ - "alert_processed": { - "from_walker": event_data.get("walker_id"), - "node_id": event_data.get("node_id"), - "cpu_usage": event_data.get("cpu_usage"), - "action_taken": "Notification sent to admin", - "handler_id": self.id - } - }) +- Multipart upload threshold: 8 MiB (chunks of 8 MiB). +- Throttle retry on `ThrottlingException` with exponential backoff. +- Credentials required if `file_storage_enabled=True` and backend is `s3`. - # Take action based on alert - await self.send_notification(event_data) +--- - @on_visit('SystemNode') - async def log_system_visit(self, here: Node): - """Log system node visits.""" - self.report({"system_visited": here.id}) +## 13. Caching - async def send_notification(self, alert_data): - """Send notification to administrators.""" - print(f"🚨 ALERT: High CPU {alert_data['cpu_usage']}% on {alert_data['node_id']}") +`jvspatial/cache/` — pluggable cache layer. -# Run multiple walkers concurrently -import asyncio +### 13.1 Backends -alert_walker = AlertWalker() -monitoring_walker = MonitoringWalker() +| Backend | Use case | +|---|---| +| `memory` | Per-process LRU; default for single-instance deployments | +| `redis` | Shared cache across workers/instances | +| `layered` | Memory L1 + Redis L2; promotes hot keys to L1 | -# Start both walkers concurrently -tasks = [ - alert_walker.spawn(), - monitoring_walker.spawn() -] +### 13.2 Wrapping pattern -walkers = await asyncio.gather(*tasks) +`jvspatial/db/_cache.py` provides a read-through wrapper for `Database`. Enabled per database via `cache_get_size` parameter in `create_database(...)`. Auto-invalidates on `save` and `delete`. -# Check reports from both walkers -alert_report = await alert_walker.get_report() -monitoring_report = await monitoring_walker.get_report() +### 13.3 TTL and invalidation -print(f"Alerts sent: {len([r for r in alert_report if 'alert_sent' in str(r)])}") -print(f"Alerts processed: {monitoring_walker.alerts_received}") -``` +TTL is per-cache-instance; default `None` (cache until eviction). Invalidation is event-driven: write operations on the wrapped database publish invalidation events. Cross-instance invalidation requires the Redis or layered backend. -#### Advanced Event Patterns - -```python path=null start=null -class DataProcessorWalker(Walker): - """Walker that processes data and emits completion events.""" - - def __init__(self, batch_id: str): - super().__init__() - self.batch_id = batch_id - self.processed_items = 0 - - @on_visit('DataNode') - async def process_data(self, here: Node): - """Process data nodes.""" - # Simulate processing - await asyncio.sleep(0.01) - self.processed_items += 1 - - self.report({"data_processed": here.id}) - - # Emit progress event every 10 items - if self.processed_items % 10 == 0: - await self.emit("batch_progress", { - "batch_id": self.batch_id, - "processed_count": self.processed_items, - "processor_id": self.id - }) - - @on_exit - async def emit_completion(self): - """Emit batch completion event.""" - await self.emit("batch_complete", { - "batch_id": self.batch_id, - "total_processed": self.processed_items, - "processor_id": self.id - }) - - self.report({"batch_completed": self.batch_id}) - -class BatchCoordinator(Walker): - """Walker that coordinates multiple batch processors.""" - - def __init__(self): - super().__init__() - self.batch_progress = {} - self.completed_batches = [] - - @on_emit("batch_progress") - async def track_progress(self, event_data): - """Track progress from batch processors.""" - batch_id = event_data.get("batch_id") - processed_count = event_data.get("processed_count") - - self.batch_progress[batch_id] = processed_count - - self.report({ - "progress_update": { - "batch_id": batch_id, - "items_processed": processed_count, - "coordinator_id": self.id - } - }) - - @on_emit("batch_complete") - async def handle_completion(self, event_data): - """Handle batch completion events.""" - batch_id = event_data.get("batch_id") - total_processed = event_data.get("total_processed") - - self.completed_batches.append(batch_id) - - self.report({ - "batch_completed": { - "batch_id": batch_id, - "total_items": total_processed, - "completed_batches_count": len(self.completed_batches) - } - }) - - # Check if all batches are complete - if len(self.completed_batches) >= 3: # Expecting 3 batches - await self.emit("all_batches_complete", { - "total_batches": len(self.completed_batches), - "coordinator_id": self.id - }) - - @on_emit("all_batches_complete") - async def finalize_processing(self, event_data): - """Finalize when all processing is complete.""" - self.report({ - "processing_finalized": { - "total_batches_completed": event_data.get("total_batches"), - "finalization_time": time.time() - } - }) - -# Example: Run coordinated batch processing -coordinator = BatchCoordinator() -processors = [ - DataProcessorWalker("batch_1"), - DataProcessorWalker("batch_2"), - DataProcessorWalker("batch_3") -] - -# Start all walkers -all_walkers = [coordinator] + processors -tasks = [walker.spawn() for walker in all_walkers] -results = await asyncio.gather(*tasks) - -# Check final reports -for walker in all_walkers: - report = await walker.get_report() - print(f"Walker {walker.id}: {len(report)} items in report") -``` +--- -#### Key Reporting & Event Features - -**Reporting System:** -- `walker.report(any_data)` - Add any data to walker's report -- `await walker.get_report()` - Get simple list of all reported items (async) -- No complex nested structures - direct access to your data -- Support for any data type (strings, dicts, lists, numbers, etc.) - -**Event System:** -- `await walker.emit(event_name, payload)` - Send events to other walkers -- `@on_emit(event_name)` - Handle specific events -- Multiple walkers can receive the same event -- Events enable real-time coordination between concurrent walkers -- Both Walkers and Nodes can use `@on_emit` decorators - -**Best Practices:** -- Use `self.report()` to add data, never return values from decorated methods -- Access reports after traversal: `report = await walker.get_report()` -- Use events for walker-to-walker communication during traversal -- Filter reported data by checking item structure/content -- Leverage `@on_exit` hooks for final summaries and cleanup - -### Walker Trail Tracking - -Walkers include built-in **trail tracking** capabilities to monitor and record the complete path taken during graph traversal. This is invaluable for debugging, analytics, audit trails, and optimizing traversal strategies. - -#### Basic Trail Tracking - -```python path=null start=null -from jvspatial.core import Walker, on_visit - -class TrailTrackingWalker(Walker): - def __init__(self): - super().__init__() - # Enable trail tracking with memory management - self.trail_enabled = True - self.max_trail_length = 100 # Keep last 100 steps (0 = unlimited) - - @on_visit('User') - async def process_user_with_trail(self, here: Node): - """Process user nodes while tracking the traversal trail. - - Args: - here: The visited User node - """ - print(f"Processing user: {here.name}") - - # Access current trail information - current_trail = self.get_trail() # List of node IDs - trail_length = self.get_trail_length() # Current trail size - recent_steps = self.get_recent_trail(count=3) # Last 3 steps - - print(f"Trail length: {trail_length}, Recent: {recent_steps}") - - # Avoid revisiting recently visited nodes - if here.id in recent_steps[:-1]: # Exclude current node - print(f"Recently visited {here.name}, skipping deeper traversal") - self.skip() - - # Continue normal traversal - colleagues = await here.nodes( - node=['User'], - department=here.department, - active=True - ) - await self.visit(colleagues) - - @on_exit - async def generate_trail_report(self): - """Generate comprehensive trail analysis report.""" - # Get trail with actual node objects (database lookups) - trail_nodes = await self.get_trail_nodes() - - # Get complete path with connecting edges - trail_path = await self.get_trail_path() - - # Generate detailed report using report() method - trail_report = { - 'summary': { - 'total_steps': self.get_trail_length(), - 'unique_nodes': len(set(self.get_trail())), - 'efficiency_ratio': len(set(self.get_trail())) / max(self.get_trail_length(), 1) - }, - 'visited_nodes': [ - {'step': i+1, 'node_type': node.__class__.__name__, 'node_name': getattr(node, 'name', node.id)} - for i, node in enumerate(trail_nodes) - ], - 'path_analysis': [ - { - 'step': i+1, - 'node': node.name if hasattr(node, 'name') else node.id, - 'via_edge': edge.edge_type if edge else 'start' - } - for i, (node, edge) in enumerate(trail_path) - ] - } - - # Report the trail data - self.report(trail_report) - - print(f"\n📊 Trail Report Generated:") - print(f" - Total steps: {trail_report['summary']['total_steps']}") - print(f" - Unique nodes: {trail_report['summary']['unique_nodes']}") - print(f" - Path efficiency: {trail_report['summary']['efficiency_ratio']:.2%}") - -# Usage example -walker = TrailTrackingWalker() -root = await Root.get(None) -await walker.spawn(root) - -# Access trail data -final_trail = walker.get_trail() -print(f"Final trail: {final_trail}") -# Access the trail report from walker's collected data -report = await walker.get_report() -trail_reports = [item for item in report if isinstance(item, dict) and 'trail_report' in str(item)] -print(f"Trail report: {trail_reports[0] if trail_reports else 'No trail report found'}") -``` +## 14. Observability -#### Advanced Trail Use Cases - -```python path=null start=null -class AdvancedTrailWalker(Walker): - def __init__(self): - super().__init__() - self.trail_enabled = True - self.max_trail_length = 0 # Unlimited for comprehensive analysis - self.visited_nodes = set() # For cycle detection - self.performance_metrics = [] - - @on_visit('Document') - async def process_with_cycle_detection(self, here: Node): - """Process documents with cycle detection using trail data. - - Args: - here: The visited Document node - """ - import time - start_time = time.time() - - # Cycle detection using trail - if here.id in self.visited_nodes: - trail = self.get_trail() - first_visit_index = trail.index(here.id) - cycle_length = len(trail) - first_visit_index - 1 - - print(f"🔄 Cycle detected at {here.id}! Length: {cycle_length} steps") - - self.report({ - 'cycle_detected': { - 'node_id': here.id, - 'cycle_length': cycle_length, - 'first_visit_step': first_visit_index + 1, - 'detection_step': len(trail) - } - }) - - # Stop to avoid infinite loop - await self.disengage() - return - - self.visited_nodes.add(here.id) - - # Process document - await self.analyze_document(here) - - # Record performance metrics - processing_time = time.time() - start_time - self.performance_metrics.append({ - 'node_id': here.id, - 'processing_time': processing_time, - 'step_number': self.get_trail_length(), - 'metadata': self.get_trail_metadata() # Get current step metadata - }) - - # Continue traversal with trail-aware filtering - related_docs = await here.nodes( - node=['Document'], - status='active' - ) - - # Filter out recently visited to avoid cycles - recent_trail = self.get_recent_trail(count=10) - unvisited_docs = [doc for doc in related_docs if doc.id not in recent_trail] - - if unvisited_docs: - await self.visit(unvisited_docs) - else: - print("All related documents recently visited, exploring alternatives") - - @on_visit('User') - async def audit_user_access(self, here: Node): - """Create audit trail for user access. - - Args: - here: The visited User node - """ - # Get current trail metadata (automatically includes timestamp, node_type, queue_length) - metadata = self.get_trail_metadata() - - audit_entry = { - 'timestamp': metadata.get('timestamp'), - 'action': 'USER_ACCESS', - 'user_id': here.id, - 'user_name': getattr(here, 'name', 'Unknown'), - 'trail_step': self.get_trail_length(), - 'access_context': { - 'queue_size': metadata.get('queue_length'), - 'node_type': metadata.get('node_type'), - 'previous_steps': self.get_recent_trail(count=3)[:-1] # Exclude current - } - } - - self.report({'audit_entry': audit_entry}) - print(f"📝 Audit: Accessed user {here.id} at step {audit_entry['trail_step']}") - - @on_exit - async def comprehensive_analysis(self): - """Generate comprehensive trail and performance analysis.""" - trail_path = await self.get_trail_path() - - # Performance analysis - avg_processing_time = sum(m['processing_time'] for m in self.performance_metrics) / len(self.performance_metrics) if self.performance_metrics else 0 - - # Path efficiency analysis - total_steps = self.get_trail_length() - unique_nodes = len(set(self.get_trail())) - - # Get report once for all analysis - report = await self.get_report() - - comprehensive_analysis = { - 'trail_summary': { - 'total_steps': total_steps, - 'unique_nodes_visited': unique_nodes, - 'path_efficiency': unique_nodes / total_steps if total_steps > 0 else 0, - 'cycles_detected': len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item]), - 'trail_enabled': self.trail_enabled, - 'trail_limit': self.max_trail_length - }, - 'performance_metrics': { - 'avg_processing_time': avg_processing_time, - 'total_processing_time': sum(m['processing_time'] for m in self.performance_metrics), - 'slowest_step': max(self.performance_metrics, key=lambda x: x['processing_time']) if self.performance_metrics else None - }, - 'audit_summary': { - 'total_audit_entries': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item]), - 'user_accesses': len([item for item in report if isinstance(item, dict) and 'audit_entry' in item and item.get('audit_entry', {}).get('action') == 'USER_ACCESS']) - } - } - - # Report the comprehensive analysis - self.report(comprehensive_analysis) - - print("\n📈 Comprehensive Analysis Complete:") - print(f" - Path efficiency: {comprehensive_analysis['trail_summary']['path_efficiency']:.2%}") - print(f" - Average processing time: {avg_processing_time:.3f}s") - print(f" - Cycles detected: {len([item for item in report if isinstance(item, dict) and 'cycle_detected' in item])}") - - async def analyze_document(self, doc): - """Simulate document analysis.""" - import asyncio - await asyncio.sleep(0.02) # Simulate processing time - -# Usage with trail management -walker = AdvancedTrailWalker() - -# Enable debug mode for detailed trail information -walker.debug_mode = True - -# Spawn and run analysis -root = await Root.get(None) -await walker.spawn(root) - -# Access comprehensive results from walker's report -report = await walker.get_report() -analysis = next((item for item in report if isinstance(item, dict) and 'trail_summary' in item), {}) -cycles = [item for item in report if isinstance(item, dict) and 'cycle_detected' in item] -audit_entries = [item for item in report if isinstance(item, dict) and 'audit_entry' in item] - -print(f"Analysis complete. Efficiency: {analysis.get('trail_summary', {}).get('path_efficiency', 0):.2%}") -print(f"Cycles detected: {len(cycles)}, Audit entries: {len(audit_entries)}") -``` +`jvspatial/observability/` — metrics and tracing surface. -#### Trail API Quick Reference - -**Configuration (Read/Write):** -- `self.trail_enabled = True` - Enable trail tracking -- `self.max_trail_length = N` - Limit trail to N steps (0 = unlimited) - -**Trail Data (Read-Only Properties):** -- `self.trail` - List of visited node IDs (returns copy) -- `self.trail_edges` - Edge IDs between nodes (returns copy) -- `self.trail_metadata` - Metadata per step (returns deep copy) - -**Trail Access Methods (O(1) operations):** -- `self.get_trail()` - Get list of visited node IDs -- `self.get_trail_length()` - Get current trail length -- `self.get_recent_trail(count=N)` - Get last N trail steps -- `self.clear_trail()` - Clear entire trail (only way to modify trail) - -**Advanced Access (Database operations):** -- `await self.get_trail_nodes()` - Get actual Node objects from trail -- `await self.get_trail_path()` - Get trail with connecting edges -- `self.get_trail_metadata(step=-1)` - Get metadata for specific step - -**Use Cases:** -- **Debugging**: Track walker path for troubleshooting -- **Cycle Detection**: Avoid infinite loops in graph traversal -- **Performance Analysis**: Measure processing time per step -- **Audit Trails**: Comprehensive access logging for compliance -- **Path Optimization**: Analyze and improve traversal efficiency - -### Walker Queue Manipulation Methods - -Walkers maintain an internal queue (deque) of nodes to visit during traversal. Advanced queue manipulation provides fine-grained control over traversal order, priority handling, and dynamic path planning. These methods allow you to programmatically manage the walker's traversal queue: - -#### Core Queue Methods - -```python path=null start=null -from jvspatial.core import Walker, Node -from jvspatial.core.entities import on_visit -from typing import List, Optional - -class QueueMasterWalker(Walker): - def __init__(self): - super().__init__() - self.priority_nodes = [] - self.deferred_nodes = [] - self.processed_count = 0 - - @on_visit("TaskNode") - async def process_task(self, here: Node): - """Demonstrate queue manipulation methods. - - Args: - here: The visited TaskNode - """ - - # 1. INSPECT QUEUE STATE - current_queue = self.get_queue() # Get queue as list - print(f"Current queue size: {len(current_queue)}") - print(f"Queue contents: {[n.name for n in current_queue]}") - - # Check if specific node is queued - if current_queue: - next_node = current_queue[0] # Peek at next node - print(f"Next node to process: {next_node.name}") - - # 2. ADD NODES TO QUEUE - # Find high-priority nodes to add immediately - urgent_tasks = await here.nodes( - node=['TaskNode'], - priority={"$gte": 9}, - status="pending" - ) - - # Add to front of queue (high priority) - if urgent_tasks: - added = self.prepend(urgent_tasks) # Add to front - print(f"Added {len(added)} urgent tasks to front of queue") - - # Find normal tasks to add to end - normal_tasks = await here.nodes( - node=['TaskNode'], - priority={"$lt": 9, "$gte": 5}, - status="pending" - ) - - # Add to end of queue (normal priority) - if normal_tasks: - added = self.append(normal_tasks) # Add to end - print(f"Added {len(added)} normal tasks to end of queue") - - # Alternative: Use visit() method (equivalent to append) - additional_tasks = await here.nodes(node=['TaskNode'], status="new") - if additional_tasks: - self.visit(additional_tasks) # Same as append() - - # Add nodes right after current processing - immediate_tasks = await here.nodes( - node=['TaskNode'], - priority=10, # Highest priority - status="urgent" - ) - if immediate_tasks: - added = self.add_next(immediate_tasks) # Add next in queue - print(f"Added {len(added)} tasks to process immediately next") - - # 3. CONDITIONAL QUEUE MANIPULATION - # Check if we have too many items in queue - if len(self.get_queue()) > 100: - print("Queue overflow detected, deferring low-priority items") - - # Get current queue and filter it - current_queue = self.get_queue() - low_priority = [] - high_priority = [] - - for item in current_queue: - if hasattr(item, 'priority') and item.priority < 5: - low_priority.append(item) - else: - high_priority.append(item) - - # Clear queue and rebuild with high priority items only - self.clear_queue() - if high_priority: - self.append(high_priority) - - # Store deferred items for later - self.deferred_nodes.extend(low_priority) - print(f"Deferred {len(low_priority)} low-priority items") - - # 4. TARGETED QUEUE MANIPULATION - # Remove specific completed nodes from queue - completed_nodes = await here.nodes( - node=['TaskNode'], - status="completed" - ) - - for completed in completed_nodes: - if self.is_queued(completed): - removed = self.dequeue(completed) - print(f"Removed {len(removed)} completed tasks from queue") - - # 5. PRECISE QUEUE INSERTION - # Find a specific node in queue to insert after - checkpoint_tasks = await here.nodes( - node=['TaskNode'], - task_type="checkpoint" - ) - - followup_tasks = await here.nodes( - node=['TaskNode'], - depends_on=here.id - ) - - for checkpoint in checkpoint_tasks: - if self.is_queued(checkpoint) and followup_tasks: - try: - # Insert followup tasks right after checkpoint - inserted = self.insert_after(checkpoint, followup_tasks) - print(f"Inserted {len(inserted)} followup tasks after checkpoint") - except ValueError as e: - print(f"Could not insert after checkpoint: {e}") - - self.processed_count += 1 - - @on_visit("CompletionNode") - async def handle_completion(self, here: Node): - """Handle task completion and queue cleanup. - - Args: - here: The visited CompletionNode - """ - - # Add back any deferred nodes if queue is manageable - current_queue_size = len(self.get_queue()) - if current_queue_size < 20 and self.deferred_nodes: - reactivated = self.deferred_nodes[:10] # Add up to 10 back - self.deferred_nodes = self.deferred_nodes[10:] # Remove from deferred - - self.append(reactivated) - print(f"Reactivated {len(reactivated)} deferred nodes") - - # Insert critical tasks right at the beginning - critical_tasks = await here.nodes( - node=['TaskNode'], - priority=10, - status="critical" - ) - - if critical_tasks: - # Find first non-critical task and insert before it - queue = self.get_queue() - for i, queued_node in enumerate(queue): - if hasattr(queued_node, 'priority') and queued_node.priority < 10: - try: - inserted = self.insert_before(queued_node, critical_tasks) - print(f"Inserted {len(inserted)} critical tasks before normal task") - break - except ValueError: - # If insertion fails, just prepend - self.prepend(critical_tasks) - break - - @on_exit - async def final_report(self): - """Report final queue statistics.""" - final_queue = self.get_queue() - print(f"\n📊 Queue Processing Complete") - print(f"Total processed: {self.processed_count}") - print(f"Remaining in queue: {len(final_queue)}") - print(f"Deferred nodes: {len(self.deferred_nodes)}") - - if final_queue: - print("Remaining nodes:") - for node in final_queue[:5]: # Show first 5 - print(f" - {node.name}") - if len(final_queue) > 5: - print(f" ... and {len(final_queue) - 5} more") -``` +### 14.1 MetricsRecorder protocol -#### Queue Method Reference - -Based on the actual Walker implementation in jvspatial: - -**Basic Queue Operations:** -- `self.visit(nodes)` - Add nodes to end of queue (equivalent to append) -- `self.append(nodes)` - Add nodes to end of queue -- `self.prepend(nodes)` - Add nodes to front of queue -- `self.add_next(nodes)` - Add nodes next in queue after current item -- `self.get_queue()` - Return entire queue as a list -- `self.clear_queue()` - Clear all nodes from queue -- `self.is_queued(node)` - Check if specific node is in queue - -**Advanced Queue Operations:** -- `self.dequeue(nodes)` - Remove specific nodes from queue -- `self.insert_after(target_node, nodes)` - Insert nodes after target node -- `self.insert_before(target_node, nodes)` - Insert nodes before target node - -#### Priority-Based Queue Management - -```python path=null start=null -class PriorityQueueWalker(Walker): - def __init__(self): - super().__init__() - self.priority_buckets = { - 'urgent': [], # Priority 9-10 - 'high': [], # Priority 7-8 - 'normal': [], # Priority 4-6 - 'low': [] # Priority 1-3 - } - - @on_visit("WorkItem") - async def process_work_item(self, here: Node): - """Process work items with priority-based queuing. - - Args: - here: The visited WorkItem node - """ - - # Get connected work items - connected_items = await here.nodes( - node=['WorkItem'], - status="pending" - ) - - if connected_items: - # Sort into priority buckets - self._sort_into_priority_buckets(connected_items) - - # Process highest priority items first - self._add_by_priority_order() - - def _sort_into_priority_buckets(self, nodes: List[Node]): - """Sort nodes into priority-based buckets.""" - for node in nodes: - priority = getattr(node, 'priority', 5) - - if priority >= 9: - self.priority_buckets['urgent'].append(node) - elif priority >= 7: - self.priority_buckets['high'].append(node) - elif priority >= 4: - self.priority_buckets['normal'].append(node) - else: - self.priority_buckets['low'].append(node) - - print(f"Sorted into buckets: " - f"urgent={len(self.priority_buckets['urgent'])}, " - f"high={len(self.priority_buckets['high'])}, " - f"normal={len(self.priority_buckets['normal'])}, " - f"low={len(self.priority_buckets['low'])}") - - def _add_by_priority_order(self): - """Add nodes to walker queue in priority order.""" - # Process urgent items first (add to front) - if self.priority_buckets['urgent']: - self.prepend(self.priority_buckets['urgent']) - self.priority_buckets['urgent'].clear() - - # Add high priority items to front (after urgent) - if self.priority_buckets['high']: - # Insert at beginning but after urgent items - current_queue = self.get_queue() - if current_queue: - # Find first non-urgent item and insert before it - high_items = self.priority_buckets['high'] - self.priority_buckets['high'].clear() - - try: - # Try to find insertion point - first_non_urgent = None - for node in current_queue: - if hasattr(node, 'priority') and node.priority < 9: - first_non_urgent = node - break - - if first_non_urgent: - self.insert_before(first_non_urgent, high_items) - else: - self.append(high_items) # All items are urgent - - except ValueError: - # Fallback to prepend if insertion fails - self.prepend(high_items) - else: - self.prepend(self.priority_buckets['high']) - self.priority_buckets['high'].clear() - - # Add normal priority items to end - if self.priority_buckets['normal']: - self.append(self.priority_buckets['normal']) - self.priority_buckets['normal'].clear() - - # Only add low priority if queue is small - current_queue_size = len(self.get_queue()) - if current_queue_size < 50 and self.priority_buckets['low']: - low_items = self.priority_buckets['low'][:10] # Limit low priority - self.priority_buckets['low'] = self.priority_buckets['low'][10:] - self.append(low_items) -``` +`MetricsRecorder` defines `record_db_op`, `record_hook`, `record_cache_event`. Implementations may push to Prometheus, StatsD, or any sink. The library provides: -#### Dynamic Queue Filtering and Manipulation - -```python path=null start=null -class AdaptiveQueueWalker(Walker): - def __init__(self): - super().__init__() - self.queue_stats = { - 'added': 0, - 'removed': 0, - 'filtered': 0, - 'reordered': 0 - } - - @on_visit("FilterNode") - async def adaptive_filtering(self, here: Node): - """Demonstrate dynamic queue filtering and manipulation. - - Args: - here: The visited FilterNode - """ - - # Add new nodes based on current context - candidates = await here.nodes( - node=['ProcessNode'], - active=True - ) - - if candidates: - # Filter candidates before adding to queue - filtered_candidates = self._filter_candidates(candidates) - - if filtered_candidates: - # Smart queue insertion based on current load - current_queue_size = len(self.get_queue()) - - if current_queue_size < 20: - # Low load: add all to end - self.append(filtered_candidates) - self.queue_stats['added'] += len(filtered_candidates) - else: - # High load: add only high-priority to front - high_priority = [ - c for c in filtered_candidates - if getattr(c, 'priority', 0) >= 8 - ] - if high_priority: - self.prepend(high_priority) - self.queue_stats['added'] += len(high_priority) - - # Periodic queue maintenance - if hasattr(here, 'name') and here.name.endswith('_maintenance'): - self._perform_queue_maintenance() - - def _filter_candidates(self, candidates: List[Node]) -> List[Node]: - """Filter candidates based on various criteria.""" - filtered = [] - current_queue = self.get_queue() - - for candidate in candidates: - # Check if already in queue (avoid duplicates) - if self.is_queued(candidate): - continue - - # Check resource constraints (mock implementation) - if hasattr(candidate, 'resource_requirement'): - if candidate.resource_requirement > self._get_available_resources(): - continue - - # Check dependencies (mock implementation) - if hasattr(candidate, 'dependencies'): - if not self._dependencies_met(candidate.dependencies): - continue - - filtered.append(candidate) - - self.queue_stats['filtered'] += len(candidates) - len(filtered) - return filtered - - def _perform_queue_maintenance(self): - """Perform queue cleanup and optimization.""" - current_queue = self.get_queue() - if not current_queue: - return - - print("🔧 Performing queue maintenance...") - - # 1. Remove stale items (mock - would check expiration) - non_stale = [] - removed_stale = 0 - - for item in current_queue: - if hasattr(item, 'expires_at'): - # Mock expiration check - if not getattr(item, 'is_expired', False): - non_stale.append(item) - else: - removed_stale += 1 - else: - non_stale.append(item) - - # 2. Deduplicate items by ID - seen_ids = set() - deduplicated = [] - removed_duplicates = 0 - - for item in non_stale: - if item.id not in seen_ids: - seen_ids.add(item.id) - deduplicated.append(item) - else: - removed_duplicates += 1 - - # 3. Reorder by priority - optimized = sorted( - deduplicated, - key=lambda x: getattr(x, 'priority', 0), - reverse=True - ) - - # 4. Rebuild queue with optimized order - self.clear_queue() - if optimized: - self.append(optimized) - - self.queue_stats['removed'] += removed_stale + removed_duplicates - self.queue_stats['reordered'] += 1 - - print(f"Maintenance complete: removed {removed_stale} stale, " - f"{removed_duplicates} duplicates, optimized {len(optimized)} items") - - def _get_available_resources(self) -> int: - """Mock implementation - get available system resources.""" - return 100 - - def _dependencies_met(self, dependencies: List[str]) -> bool: - """Mock implementation - check if dependencies are satisfied.""" - return True - - @on_exit - async def report_queue_stats(self): - """Report queue manipulation statistics.""" - print("\n📈 Queue Statistics:") - for stat, value in self.queue_stats.items(): - print(f" {stat.capitalize()}: {value}") -``` +- `NullRecorder` — default; no-op. +- OpenTelemetry adapter (`observability/otel.py`) — optional, requires `[otel]` extra. -#### Best Practices for Queue Manipulation - -**✅ Recommended Patterns:** - -```python path=null start=null -# Good: Check queue state before manipulation -current_queue = self.get_queue() -if current_queue: - next_item = current_queue[0] # Look at next item - # Make decisions based on next item - -# Good: Use appropriate method for insertion priority -if item.priority >= 8: - self.prepend([item]) # High priority to front -else: - self.append([item]) # Normal priority to end - -# Good: Check if node is queued before operations -if self.is_queued(completed_node): - self.dequeue(completed_node) - -# Good: Batch queue operations for efficiency -new_items = await node.nodes(filters) -if new_items: - self.append(new_items) # Add all at once - -# Good: Safe queue iteration and modification -current_queue = self.get_queue() # Get snapshot -filtered_items = [n for n in current_queue if meets_criteria(n)] -self.clear_queue() -if filtered_items: - self.append(filtered_items) - -# Good: Precise insertion with error handling -try: - self.insert_after(target_node, new_nodes) -except ValueError: - # Target not found, use alternative - self.prepend(new_nodes) -``` +### 14.2 Structured DB logging -**❌ Avoided Patterns:** +`jvspatial/db/_observable.py` — wraps a `Database` to emit structured log records per op: `{op, collection, duration_ms, query_size, result_size}`. Configurable slow-query threshold (`slow_query_ms`) emits warnings. -```python path=null start=null -# Bad: Modifying queue during iteration -for item in self.get_queue(): # Don't iterate over changing queue - if condition: - self.dequeue(item) # Modifying during iteration +### 14.3 GraphContext PerformanceMonitor -# Bad: Not handling insertion errors -self.insert_after(target_node, nodes) # Could raise ValueError +In-context counters for DB ops, hook execs, cache stats. Read at end-of-request to surface aggregate metrics. -# Bad: Inefficient repeated operations -for item in items: - self.append([item]) # Many small operations -# Better: self.append(items) # Single batch operation +--- -# Bad: Not checking queue state -first_item = self.get_queue()[0] # Could cause IndexError if empty +## 15. Security Boundaries -# Bad: Assuming nodes are still queued -self.dequeue(node) # Node might not be in queue -# Better: if self.is_queued(node): self.dequeue(node) -``` +### 15.1 User-input boundary -**Key Points:** -- Use `await node.nodes()` to get connected nodes for traversal (NOT `get_edges()`) -- Default `direction="out"` follows outgoing edges (recommended for forward traversal) -- Use `direction="in"` for reverse traversal along incoming edges -- Use `direction="both"` for bidirectional traversal -- **Semantic Filtering Approaches:** - - **Simple filtering**: Use kwargs for connected node properties: `state="NY"` - - **Complex node filtering**: `node=[{'City': {"context.population": {"$gte": 500000}}}]` - - **Complex edge filtering**: `edge=[{'Highway': {"context.condition": {"$ne": "poor"}}}]` - - **Mixed approaches**: Combine dict filters with kwargs for maximum flexibility -- **Database-Level Optimization**: All filtering happens at database level for performance -- **MongoDB-Style Operators**: `$gte`, `$lt`, `$ne`, `$in`, `$nin`, `$regex`, etc. -- **Walker Control Flow:** - - **`skip()`**: Skip current node processing, continue to next (like `continue` in loops) - - **`pause()`/`resume()`**: Temporarily pause walker (use `self.pause()`), can be resumed later - - **`disengage()`**: Permanently halt walker and remove from graph (cannot be resumed) - - **`@on_exit`**: Methods called when walker completes, pauses, or disengages (cleanup) -- The `nodes()` method returns a list that can be directly passed to `walker.visit()` - -### Inheritance Hierarchy +User input crosses the trust boundary at: +1. **HTTP request handlers** — validated by Pydantic at the FastAPI layer. +2. **`Entity.update()`** — validates all field names against the class hierarchy; rejects undeclared attributes (`object.py` setter). +3. **File uploads** — content-based MIME validation, not extension-based. +4. **Path inputs** — `path_sanitizer.py` before any filesystem access. -``` -Object (base class with unified query interface) -├── Node (spatial graph nodes) -├── Edge (relationships) -└── Custom entities (inherit from Node/Object) -``` +### 15.2 Constant-time comparisons -### Database Backends +All secret/key/token/hash comparisons use `hmac.compare_digest`. Affected paths: +- API key verification +- Refresh token comparison +- Password reset token comparison +- Webhook HMAC signature verification +- Deferred-invoke secret comparison +- Legacy bcrypt fallback hash verification -- **JSONDatabase** - File-based storage for development/testing -- **MongoDatabase** - MongoDB backend for production -- **Custom databases** - Implement DatabaseInterface +Removing constant-time comparisons in favor of `==` is a security regression. -## 📚 Documentation Maintenance +### 15.3 Security headers -### README Updates +`jvspatial/api/components/security_headers_middleware.py` injects: +- `X-Content-Type-Options: nosniff` +- `X-Frame-Options: DENY` +- `X-XSS-Protection: 1; mode=block` +- `Content-Security-Policy` — strict on app routes; relaxed only on `/docs`, `/redoc`, `/openapi.json` (CDN allowed). +- `Strict-Transport-Security` (HSTS) when HTTPS termination is configured. -When adding new features: +### 15.4 CORS defaults -1. **Review existing README.md** -2. **Update feature list** if adding major functionality -3. **Update installation/setup** if dependencies change -4. **Update usage examples** to reflect new capabilities -5. **Maintain consistency** with existing documentation style +Default CORS is restrictive (no wildcards). Wildcard origins must be set explicitly and trigger a startup warning. -### Documentation Directory +### 15.5 Secret handling -Always review and update `docs/` directory: +- JWT secret required at startup; empty/weak values rejected. +- S3 credentials required if file storage is enabled. +- API key secrets shown plaintext **only** on creation response; stored hashed. +- 500 error responses sanitize secret-bearing fields. -``` -docs/ -├── api/ # API reference documentation -├── guides/ # User guides and tutorials -├── architecture/ # System design documents -└── examples/ # Code examples and tutorials -``` +--- -**Update procedure:** -1. Check if new feature requires new documentation files -2. Update existing API documentation for modified classes/methods -3. Add user guides for complex features -4. Update architecture docs if design patterns change +## 16. Extension Points -## 💡 Examples Maintenance +### 16.1 Custom databases -### Example Directory Structure +`register_database_type("name", factory_fn)` (`jvspatial/db/factory.py`) registers a custom adapter. The factory must return a `Database` subclass instance. After registration, callers use `create_database("name", ...)` like a built-in. -``` -examples/ -├── basic/ # Simple usage examples -├── advanced/ # Complex scenarios -├── query_interface_example.py # Comprehensive entity-centric CRUD operations -├── semantic_filtering.py # Advanced semantic filtering with Node.nodes() -└── migration/ # Migration guides (if needed) -``` +### 16.2 Endpoint registration -### Example Update Procedure +`@endpoint(...)` collects targets at import time. Either decorate functions or `Walker` subclasses. The framework registers routes at server build. -1. **Review existing examples** for relevance to new features -2. **Update outdated examples** to use modern entity-centric syntax -3. **Create new examples** for significant new features -4. **Ensure examples are runnable** and well-documented -5. **Remove or archive obsolete examples** -6. **Update key reference examples:** - - `query_interface_example.py` - Showcase latest entity-centric patterns - - `semantic_filtering.py` - Demonstrate advanced Node.nodes() filtering - - Basic examples - Keep simple, focused, and beginner-friendly - - Advanced examples - Show complex real-world scenarios +### 16.3 Hooks -### Example Code Standards +`Server(...)` accepts lifecycle and event hooks: -Examples should: -- Use entity-centric syntax exclusively -- Include comprehensive comments -- Demonstrate best practices -- Be self-contained and runnable -- Show error handling patterns +| Hook | Fires on | +|---|---| +| `on_startup` | App boot, after DB ready | +| `on_shutdown` | App teardown | +| `on_admin_bootstrapped` | First admin user created | +| `on_user_registered` | New user creation | +| `on_enrich_current_user` | Per-request user enrichment | +| `on_password_reset_requested` | Password reset flow | -## 🧪 Testing Requirements +### 16.4 Deferred task handlers -### Test Organization +`@deferred_invoke_handler("task.type")` registers an async handler. Handlers must be idempotent. -``` -tests/ -├── api/ # FastAPI endpoint tests -├── core/ # Core entity and logic tests -├── db/ # Database backend tests -└── integration/ # End-to-end integration tests -``` +### 16.5 Custom storage backends -### Testing Procedure for New Features +Implement `FileStorageInterface` (`storage/interfaces/base.py`) and register via the storage manager (`storage/managers/proxy.py`). -1. **Unit Tests** - Test individual methods and classes -2. **Integration Tests** - Test feature interaction with database -3. **API Tests** - Test HTTP endpoints if applicable -4. **Performance Tests** - For database-heavy features +### 16.6 Middleware -### Test Code Standards +`server.middleware_manager.add(...)` accepts any Starlette-compatible middleware. -```python -import pytest -from typing import List -from jvspatial.core import Node, Walker, Edge -from jvspatial.core.entities import on_visit -from jvspatial.exceptions import NodeNotFoundError, ValidationError - -class TestUser(Node): - name: str = "" - email: str = "" - department: str = "" - age: int = 0 - active: bool = True - skills: List[str] = [] - -class TestDepartment(Node): - name: str = "" - location: str = "" - budget: int = 0 - -class TestWalker(Walker): - def __init__(self): - super().__init__() - self.visited_users = [] - - @on_visit("TestUser") - async def visit_user(self, node: TestUser): - self.visited_users.append(node.name) - # Test semantic filtering in walker - connected_users = await node.nodes( - node=['TestUser'], - department=node.department, - active=True - ) - await self.visit(connected_users) - -# Entity Creation Tests -@pytest.mark.asyncio -async def test_user_creation(): - """Test entity-centric user creation with full data.""" - user = await TestUser.create( - name="Alice Johnson", - email="alice@company.com", - department="engineering", - age=30, - skills=["python", "javascript"] - ) - assert user.name == "Alice Johnson" - assert user.email == "alice@company.com" - assert user.department == "engineering" - assert user.id is not None - assert "python" in user.skills - -# MongoDB-Style Query Tests -@pytest.mark.asyncio -async def test_mongodb_queries(): - """Test comprehensive MongoDB-style queries.""" - # Setup test data - await TestUser.create(name="Bob Smith", email="bob@test.com", age=25, department="engineering") - await TestUser.create(name="Carol Davis", email="carol@test.com", age=35, department="marketing") - await TestUser.create(name="David Brown", email="david@test.com", age=45, department="engineering") - - # Test regex queries - b_users = await TestUser.find({"context.name": {"$regex": "^B", "$options": "i"}}) - assert len(b_users) >= 1 - assert any(u.name.startswith("B") for u in b_users) - - # Test comparison operators - senior_users = await TestUser.find({"context.age": {"$gte": 35}}) - assert all(u.age >= 35 for u in senior_users) - - # Test logical operators - senior_engineers = await TestUser.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.age": {"$gte": 30}} - ] - }) - assert all(u.department == "engineering" and u.age >= 30 for u in senior_engineers) - - # Test array operations - tech_users = await TestUser.find({"context.skills": {"$in": ["python", "javascript"]}}) - assert all(any(skill in u.skills for skill in ["python", "javascript"]) for u in tech_users) - -# Semantic Filtering Tests -@pytest.mark.asyncio -async def test_semantic_filtering(): - """Test Node.nodes() semantic filtering capabilities.""" - # Create test nodes and edges - dept = await TestDepartment.create(name="Engineering", location="SF") - user1 = await TestUser.create(name="Alice", department="engineering", active=True) - user2 = await TestUser.create(name="Bob", department="engineering", active=False) - user3 = await TestUser.create(name="Carol", department="marketing", active=True) - - # Create connections - await Edge.create(source=dept, target=user1, edge_type="employs") - await Edge.create(source=dept, target=user2, edge_type="employs") - await Edge.create(source=user1, target=user3, edge_type="collaborates") - - # Test simple filtering - active_employees = await dept.nodes( - node=['TestUser'], - active=True - ) - assert len(active_employees) == 1 - assert active_employees[0].name == "Alice" - - # Test complex filtering - engineering_employees = await dept.nodes( - node=[{'TestUser': {"context.department": "engineering"}}], - active=True - ) - assert all(u.department == "engineering" and u.active for u in engineering_employees) - - # Test mixed filtering - collaborators = await user1.nodes( - node=[{'TestUser': {"context.active": True}}], - department="marketing" - ) - assert len(collaborators) >= 1 - -# Walker Tests -@pytest.mark.asyncio -async def test_walker_traversal(): - """Test walker with semantic filtering.""" - # Setup graph - root = await TestUser.create(name="Root", department="engineering") - user1 = await TestUser.create(name="User1", department="engineering", active=True) - user2 = await TestUser.create(name="User2", department="marketing", active=True) - - await Edge.create(source=root, target=user1, edge_type="manages") - await Edge.create(source=root, target=user2, edge_type="manages") - await Edge.create(source=user1, target=user2, edge_type="collaborates") - - # Test walker - walker = TestWalker() - await walker.spawn(root) - - # Verify walker visited users - assert "Root" in walker.visited_users - assert len(walker.visited_users) >= 1 - -# Error Handling Tests -@pytest.mark.asyncio -async def test_error_handling(): - """Test proper error handling patterns.""" - # Test NodeNotFoundError - with pytest.raises(NodeNotFoundError): - await TestUser.get("nonexistent-id") - - # Test safe retrieval - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await TestUser.find({"context.email": "nonexistent@test.com"}) - user = users[0] if users else None - assert user is None - - # Test validation errors (if validation is implemented) - with pytest.raises((ValidationError, ValueError)): - await TestUser.create(name="", email="invalid-email") - -# Performance Tests -@pytest.mark.asyncio -async def test_bulk_operations(): - """Test bulk operations performance.""" - # Create multiple users - users_data = [ - {"name": f"User{i}", "email": f"user{i}@test.com", "department": "engineering"} - for i in range(10) - ] - - # Bulk create - users = [] - for user_data in users_data: - user = await TestUser.create(**user_data) - users.append(user) - - assert len(users) == 10 - - # Bulk query - engineering_users = await TestUser.find({"context.department": "engineering"}) - assert len(engineering_users) >= 10 - - # Bulk update - for user in users[:5]: - user.active = False - await user.save() - - # Verify updates - inactive_users = await TestUser.find({"context.active": False}) - assert len(inactive_users) >= 5 - -# Integration Tests -@pytest.mark.asyncio -async def test_full_workflow(): - """Test complete entity lifecycle with relationships.""" - # Create department - dept = await TestDepartment.create(name="Product", location="NYC", budget=1000000) - - # Create users - manager = await TestUser.create( - name="Jane Manager", - email="jane@company.com", - department="product", - age=40 - ) - - employee = await TestUser.create( - name="John Employee", - email="john@company.com", - department="product", - age=28 - ) - - # Create relationships - await Edge.create(source=dept, target=manager, edge_type="employs") - await Edge.create(source=dept, target=employee, edge_type="employs") - await Edge.create(source=manager, target=employee, edge_type="manages") - - # Test traversal - department_employees = await dept.nodes(node=['TestUser']) - assert len(department_employees) == 2 - - managed_employees = await manager.nodes( - node=['TestUser'], - direction="out", - edge=['manages'] - ) - assert len(managed_employees) == 1 - assert managed_employees[0].name == "John Employee" - - # Test complex query - young_employees = await dept.nodes( - node=[{'TestUser': {"context.age": {"$lt": 30}}}] - ) - assert len(young_employees) == 1 - assert young_employees[0].age < 30 - - # Cleanup - await manager.delete() - await employee.delete() - await dept.delete() - - # Verify deletion - # Note: Object.find_one() doesn't exist - use find() and get first result - users = await TestUser.find({"context.email": "jane@company.com"}) - deleted_manager = users[0] if users else None - assert deleted_manager is None -``` +### 16.7 Custom log levels -## 🗑️ Cleanup and Maintenance +`jvspatial/logging/` permits adding domain levels (e.g. `AUDIT`, `SECURITY`, `TRACE`) above standard levels. See `docs/md/custom-log-levels.md`. -### Deprecation Procedure - -When evolving the library: - -1. **Identify obsolete code** - Old patterns, unused utilities -2. **Mark as deprecated** - Add deprecation warnings before removal -3. **Update documentation** - Remove references to deprecated features -4. **Update examples** - Remove or update examples using deprecated code -5. **Clean removal** - Remove deprecated code after grace period - -### File Cleanup Checklist - -- [ ] Remove unused import statements -- [ ] Delete empty or obsolete modules -- [ ] Archive outdated examples to `examples/archive/` -- [ ] Update `__all__` exports in `__init__.py` files -- [ ] Remove commented-out code blocks -- [ ] Clean up temporary test files - -### Migration Strategy - -When making breaking changes: - -1. **Create migration guide** in `docs/migration/` -2. **Provide before/after examples** -3. **Update all existing examples** to new patterns -4. **Add runtime warnings** for deprecated usage -5. **Version appropriately** using semantic versioning - -## 🚀 Development Workflow - -### Pre-commit Checklist - -- [ ] Code passes `black --check .` -- [ ] Code passes `flake8 .` -- [ ] Code passes `mypy .` -- [ ] All tests pass: `pytest` -- [ ] Examples are updated and runnable -- [ ] Documentation reflects changes -- [ ] Deprecated code is cleaned up - -### Code Review Focus Areas - -1. **Entity-centric patterns** - Ensure new code uses preferred syntax -2. **Query interface consistency** - MongoDB-style queries throughout -3. **Type safety** - Proper annotations and mypy compliance -4. **Test coverage** - Adequate testing for new features -5. **Documentation completeness** - Examples and guides updated - -## 📋 Quick Reference - -### Preferred Patterns - -```python path=null start=null -# ✅ Entity creation -user = await User.create(name="Alice", email="alice@company.com", department="engineering") - -# ✅ Entity queries with MongoDB-style operators -users = await User.find({"context.active": True}) -# Note: Object.find_one() doesn't exist - use find() and get first result -users_by_email = await User.find({"context.email": email}) -user = users_by_email[0] if users_by_email else None -senior_engineers = await User.find({ - "$and": [ - {"context.department": "engineering"}, - {"context.age": {"$gte": 35}} - ] -}) -tech_users = await User.find({"context.skills": {"$in": ["python", "javascript"]}}) - -# ✅ Counting and aggregation -# Note: Object.count() doesn't exist - use len() with find() instead -results = await User.find({"context.department": "engineering"}) -count = len(results) - -# Note: Object.distinct() doesn't exist - query and extract manually -all_users = await User.find({}) -departments = set(u.department for u in all_users if hasattr(u, 'department')) - -# ✅ Entity updates -user.name = "Alice Johnson" -user.active = True -await user.save() - -# ✅ Bulk updates -users = await User.find({"context.department": "old_dept"}) -for user in users: - user.department = "new_dept" - await user.save() - -# ✅ Entity deletion -await user.delete() - -# ✅ Walker traversal with semantic filtering -@on_visit("User") -async def process_user(self, here: Node): - """Process user nodes with semantic filtering. - - Args: - here: The visited User node - """ - # Use nodes() method with semantic filtering - connected_users = await here.nodes( - node=['User'], # Type filtering - department="engineering", # Simple kwargs - active=True # Multiple simple filters - ) - await self.visit(connected_users) - - # Complex filtering with MongoDB operators - senior_connections = await here.nodes( - node=[{'User': {"context.age": {"$gte": 35}}}], - direction="out" - ) - await self.visit(senior_connections) - -# ✅ Walker control flow -class MyWalker(Walker): - @on_visit("Document") - async def process_document(self, here: Node): - """Process document nodes with control flow. - - Args: - here: The visited Document node - """ - if here.status == "archived": - self.skip() # Skip to next node - - if self.processed_count >= 100: - await self.disengage() # Permanently halt - - # Normal processing - next_docs = await here.nodes(node=['Document'], active=True) - await self.visit(next_docs) - - @on_exit - async def cleanup(self): - print(f"Processed {self.processed_count} documents") - -# ✅ Error handling -try: - user = await User.get(user_id) -except NodeNotFoundError: - logger.warning(f"User {user_id} not found") - return None -except DatabaseError as e: - logger.error(f"Database error: {e}") - raise -``` +--- -### Avoided Patterns - -```python path=null start=null -# ❌ Direct database access (discouraged - use entity methods) -from jvspatial.db import create_database -db = create_database("json") -await db.save("node", data) # Use entity.save() instead -await db.find("node", {"name": "User"}) # Use Entity.find() instead - -# ❌ GraphContext methods for simple operations (use entity methods) -from jvspatial.core import GraphContext -ctx = GraphContext(database=db) -# Prefer entity-centric methods: -# await Node.create(...) instead of ctx.create_node(...) -# await node.get_edges() instead of ctx.get_edges(node_id) - -# ❌ Non-standard query formats -await User.find({"age": 25}) # Missing context. prefix -await User.find({"name": "Alice"}) # Should be context.name - -# ❌ Old traversal patterns (deprecated) -walker.get_edges(node) # Use node.nodes() instead -walker.traverse_edges() # Use semantic filtering - -# ❌ Synchronous operations -user = User.create_sync(**data) # Use async await User.create() -users = User.find_sync(query) # Use async await User.find() - -# ❌ Manual edge management in walkers (show proper naming even in bad examples) -@on_visit("User") -async def visit_user(self, here: Node): - """DEPRECATED: Manual edge management (avoid this pattern). - - Args: - here: The visited User node - """ - # Avoid manual edge retrieval - edges = await here.get_edges() # Deprecated - for edge in edges: - target = await edge.get_target() - await self.visit([target]) - - # Instead, use semantic filtering - connected = await here.nodes() # Preferred - await self.visit(connected) - -# ❌ Missing error handling -user = await User.get(user_id) # Should handle NodeNotFoundError -user.field = value -await user.save() # Should handle ValidationError - -# ❌ Inefficient queries -# Don't fetch all then filter in Python -all_users = await User.find({}) -engineers = [u for u in all_users if u.department == "engineering"] - -# Instead, filter at database level -engineers = await User.find({"context.department": "engineering"}) - -# ❌ Blocking operations in async context -@on_visit("DataNode") -async def process_node(self, here: Node): - """BAD EXAMPLE: Blocking operations in async context. - - Args: - here: The visited DataNode - """ - # Avoid blocking operations - time.sleep(1.0) # Blocks event loop - - # Use async alternatives - await asyncio.sleep(1.0) # Non-blocking -``` +## 17. Error Taxonomy + +`jvspatial/exceptions.py` centralizes all exceptions. Hierarchy: + +``` +JVSpatialError +├── EntityError +│ ├── EntityNotFoundError +│ │ ├── NodeNotFoundError +│ │ ├── EdgeNotFoundError +│ │ └── ObjectNotFoundError +│ └── DuplicateEntityError +├── ValidationError +│ └── FieldValidationError +├── ConfigurationError +│ ├── InvalidConfigurationError +│ └── MissingConfigurationError +├── DatabaseError +│ ├── ConnectionError +│ ├── QueryError +│ ├── TransactionError +│ └── VersionConflictError +├── GraphError +│ ├── InvalidGraphStructureError +│ ├── CircularReferenceError +│ └── EdgeConnectionError +├── WalkerError +│ ├── WalkerExecutionError +│ ├── WalkerTimeoutError +│ └── InfiniteLoopError +├── APIError (via jvspatial.api.exceptions) +│ ├── JVSpatialAPIException +│ ├── EndpointError +│ ├── ParameterError +│ ├── AuthenticationError +│ ├── AuthorizationError +│ ├── RateLimitError +│ ├── InvalidCredentialsError +│ └── RegistrationDisabledError +└── SecurityError + └── PermissionDeniedError +``` + +Control-flow exceptions for walkers live in `jvspatial.core.entities`: +- `TraversalPaused` — caller catches to suspend traversal +- `TraversalSkipped` — caller catches to skip current node + +### 17.1 Propagation rules + +- Database errors propagate to the caller; entity methods may wrap with context. +- Validation errors raise `ValidationError` with `field_errors`. The API layer maps to HTTP 400. +- Authentication errors map to HTTP 401; authorization errors to HTTP 403. +- Walker errors are logged and surface in `walker.response["errors"]`; traversal halts. --- -## ⏰ Scheduler Integration (Optional) +## 18. Stability Tiers -jvspatial includes optional scheduler support for background task automation. Install with: +`docs/md/stability.md` is the canonical tier reference. Summary: -```bash -pip install jvspatial[scheduler] -``` +| Tier | Guarantee | +|---|---| +| **Stable** | Public API. Breaking changes require a major version bump and two-version deprecation grace. | +| **Internal** | May change between minor versions. Documented for callers who need the detail, but not contracted. | +| **Experimental** | May change in any release. Opt-in only. Examples: `JsonDBTransaction(best_effort=True)`, some deferred-save edge cases. | -### Basic Scheduled Tasks - -```python path=null start=null -from jvspatial.api import Server -from jvspatial.api.scheduler import on_schedule -from jvspatial.core import Object -from datetime import datetime -from typing import Optional -import logging - -logger = logging.getLogger(__name__) - -# Define entity for job tracking (entity-centric pattern) -class ScheduledJob(Object): - """Entity representing scheduled job execution records.""" - job_name: str = "" - execution_time: datetime = datetime.now() - status: str = "pending" # pending, completed, failed - duration_seconds: Optional[float] = None - error_message: Optional[str] = None - -class SystemMetrics(Object): - """Entity for system metrics collection.""" - timestamp: datetime = datetime.now() - cpu_usage: float = 0.0 - memory_usage: float = 0.0 - active_jobs: int = 0 - -# Scheduled function with proper error handling -@on_schedule("every 30 minutes", description="System cleanup with job tracking") -async def cleanup_system(): - """Automated cleanup with entity-centric job tracking.""" - start_time = datetime.now() - - try: - logger.info("🧹 Starting system cleanup") - - # Perform cleanup work - cleanup_count = await perform_cleanup_work() - - # Create success record - await ScheduledJob.create( - job_name="system_cleanup", - execution_time=start_time, - status="completed", - duration_seconds=(datetime.now() - start_time).total_seconds() - ) - - logger.info(f"✅ Cleanup completed: {cleanup_count} items processed") - - except Exception as e: - # Create error record - await ScheduledJob.create( - job_name="system_cleanup", - execution_time=start_time, - status="failed", - error_message=str(e), - duration_seconds=(datetime.now() - start_time).total_seconds() - ) - logger.error(f"❌ Cleanup failed: {str(e)}") - raise - -# Metrics collection with MongoDB-style queries -@on_schedule("every 5 minutes", retry_count=2, description="Collect system metrics") -async def collect_metrics(): - """Collect system metrics with entity queries.""" - import psutil - - # Get system metrics - cpu_percent = psutil.cpu_percent() - memory = psutil.virtual_memory() - - # Count active jobs using MongoDB-style query - # Note: Object.count() doesn't exist - use len() with find() instead - active_jobs_list = await ScheduledJob.find({ - "context.status": {"$in": ["pending", "running"]} - }) - active_jobs = len(active_jobs_list) - - # Create metrics record - await SystemMetrics.create( - timestamp=datetime.now(), - cpu_usage=cpu_percent, - memory_usage=memory.percent, - active_jobs=active_jobs - ) - - logger.info(f"📊 Metrics: CPU {cpu_percent:.1f}%, Memory {memory.percent:.1f}%") - -async def perform_cleanup_work() -> int: - """Simulate cleanup work.""" - # Query old records using entity-centric approach - cutoff_time = datetime.now().timestamp() - (7 * 24 * 3600) # 7 days ago - - old_jobs = await ScheduledJob.find({ - "context.execution_time": {"$lt": cutoff_time} - }) - - # Delete old records - for job in old_jobs: - await job.delete() - - return len(old_jobs) -``` +**Stable** includes (non-exhaustive): `Object`/`Node`/`Edge`/`Walker` public methods, `@endpoint` and `@attribute` decorators, `Database` ABC contract, `Server` constructor. -### Server Integration - -```python path=null start=null -from jvspatial.api import Server, endpoint -from jvspatial.api.scheduler import register_scheduled_tasks -from typing import Dict, Any -from dotenv import load_dotenv - -# Load environment configuration (jvspatial pattern) -load_dotenv() - -# Create server with scheduler enabled -server = Server( - title="My Scheduled App", - description="Application with integrated scheduler", - version="1.0.0", - scheduler_enabled=True, # Enable scheduler - scheduler_interval=1, # Check every second - scheduler_timezone="UTC", # Timezone for scheduling -) +**Internal** includes: contents of `jvspatial/db/_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`; `jvspatial/api/components/*` implementation details; `jvspatial/storage/managers/*` internals. -# Register all decorated scheduled tasks -if hasattr(server, 'scheduler_service') and server.scheduler_service: - register_scheduled_tasks(server.scheduler_service) - logger.info("✅ Scheduled tasks registered") - -# Add monitoring endpoint -@endpoint("/api/scheduler/status", methods=["GET"]) -async def get_scheduler_status() -> Dict[str, Any]: - """Get scheduler status with entity-centric job statistics.""" - # Get job statistics using entity queries - # Note: Object.count() doesn't exist - use len() with find() instead - all_jobs = await ScheduledJob.find({}) - total_jobs = len(all_jobs) - - completed_jobs_list = await ScheduledJob.find({"context.status": "completed"}) - completed_jobs = len(completed_jobs_list) - - failed_jobs_list = await ScheduledJob.find({"context.status": "failed"}) - failed_jobs = len(failed_jobs_list) - - return { - "scheduler": "running", - "job_statistics": { - "total_jobs": total_jobs, - "completed_jobs": completed_jobs, - "failed_jobs": failed_jobs, - "success_rate": (completed_jobs / total_jobs * 100) if total_jobs > 0 else 0 - }, - "timestamp": datetime.now().isoformat() - } - -if __name__ == "__main__": - server.run(host="0.0.0.0", port=8000) # Scheduler runs automatically -``` +**Experimental** includes: anything explicitly flagged in `docs/md/stability.md`. -**📖 For comprehensive scheduler documentation:** [Scheduler Integration Guide](docs/md/scheduler.md) +Deprecations emit warnings via `jvspatial/utils/deprecation.py` (see file). Removal happens no sooner than two minor versions after the deprecation warning lands. --- -## 🎯 Key Naming Conventions +## Appendix A — Critical File Map + +| Concern | Primary file | +|---|---| +| Identity / `__entity_name__` | `jvspatial/core/entities/object.py:35-44`, `jvspatial/core/utils.py:11-89` | +| Node model | `jvspatial/core/entities/node.py` | +| Edge model | `jvspatial/core/entities/edge.py` | +| Walker model | `jvspatial/core/entities/walker.py`, `walker_components/*` | +| Root singleton | `jvspatial/core/entities/root.py` | +| GraphContext | `jvspatial/core/context.py` | +| Database ABC | `jvspatial/db/database.py:48+` | +| Built-in DB adapters | `jvspatial/db/{jsondb,sqlite,mongodb,dynamodb}.py` | +| Query engine | `jvspatial/db/query.py` | +| Atomic IO | `jvspatial/db/_atomic.py` | +| Path locks | `jvspatial/db/_path_locks.py` | +| DB cache wrapper | `jvspatial/db/_cache.py` | +| DB observability wrapper | `jvspatial/db/_observable.py` | +| Server class | `jvspatial/api/server.py` | +| App builder + CSP/docs gating | `jvspatial/api/components/app_builder.py` | +| `@endpoint` decorator | `jvspatial/api/decorators/route.py` | +| Endpoint registry | `jvspatial/api/endpoints/registry.py` | +| Auth service | `jvspatial/api/auth/service.py` | +| RBAC | `jvspatial/api/auth/rbac.py` | +| API key service | `jvspatial/api/auth/api_key_service.py` | +| Config + env merge | `jvspatial/api/config.py`, `jvspatial/env.py`, `jvspatial/env_adapter.py` | +| Serverless detection | `jvspatial/runtime/serverless.py` | +| Deferred task dispatch | `jvspatial/serverless/deferred_invoke.py`, `serverless/tasks/*` | +| Storage interfaces | `jvspatial/storage/interfaces/{base,local,s3}.py` | +| Storage security | `jvspatial/storage/security/{path_sanitizer,validator}.py` | +| Cache backends | `jvspatial/cache/{base,memory,redis,layered}.py` | +| Observability | `jvspatial/observability/{metrics,otel}.py` | +| Logging service | `jvspatial/logging/{config,service,models}.py` | +| Exceptions | `jvspatial/exceptions.py`, `jvspatial/api/exceptions.py` | +| Version | `jvspatial/version.py` | -**CRITICAL**: When writing `@on_visit` methods, always use these parameter names: - -- **`here`** - The visited node/edge (current location in traversal) -- **`visitor`** - The walker performing the traversal (when passed to node methods) +--- -```python path=null start=null -# ✅ CORRECT naming convention -@on_visit("User") -async def process_user(self, here: Node): - """Args: here = visited User node""" - connected = await here.nodes() - await self.visit(connected) +## Appendix B — Maintenance Checklist for Spec Changes -# ❌ AVOID generic names -@on_visit("User") -async def process_user(self, node: Node): # Less clear - pass -``` - ---- +When code changes alter behavior described in this document: -**Remember**: This library prioritizes **clean, maintainable code** with **consistent patterns** across all database backends. Always favor the entity-centric approach, MongoDB-style queries, and the **`here`/`visitor`** naming convention for the best developer experience. +1. Update the affected section, citing the new `file:line` if structure changed. +2. If the change is breaking, also update [ROADMAP.md](ROADMAP.md) (release notes) and [CHANGELOG.md](CHANGELOG.md). +3. If the change affects stability tier, also update [docs/md/stability.md](docs/md/stability.md). +4. If the change affects a security invariant (§15), also update [docs/md/security-review.md](docs/md/security-review.md). +5. Add a test that exercises the new behavior. diff --git a/docs/md/README.md b/docs/md/README.md index 44088c5..f826efc 100644 --- a/docs/md/README.md +++ b/docs/md/README.md @@ -1,353 +1,145 @@ -# JVspatial Documentation +# jvspatial Documentation Index -**Version**: 0.0.6 -**Last Updated**: 2025-03-16 +This directory holds detailed how-to and reference documentation. For higher-level orientation start at the repo root: -Welcome to the jvspatial documentation! This guide will help you understand and use the jvspatial library effectively. +- [README.md](../../README.md) — project overview and installation +- [PRD.md](../../PRD.md) — *why* the library exists, target users, non-goals +- [SPEC.md](../../SPEC.md) — *what* the library guarantees (technical contract) +- [ROADMAP.md](../../ROADMAP.md) — forward direction and known gaps +- [CLAUDE.md](../../CLAUDE.md) — agent maintenance guide +- [CHANGELOG.md](../../CHANGELOG.md) — release history -> **Developer Quickstart**: For minimal setup (Server with database + auth, import API modules, run), see [Auth Quickstart](auth-quickstart.md) and [Endpoint Registration Guide](endpoint-registration-guide.md). +The documents below answer *how* to use each subsystem. Every link resolves; entries marked **NEW** were added since the previous index revision. --- -## 📚 **Documentation Index** - -### **Getting Started** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Quick Start Guide](quick-start-guide.md) | Get started in 5 minutes | Beginners | -| [Examples](examples.md) | Code examples and tutorials | All levels | -| [Installation Guide](../README.md) | Installation and setup | Beginners | - -### **Core Concepts** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Graph Traversal](graph-traversal.md) | Walker pattern and graph operations | Intermediate | -| [Graph Visualization](graph-visualization.md) | Export graphs in DOT/Mermaid formats | All levels | -| [Entity Reference](entity-reference.md) | Node, Edge, Walker classes | All levels | -| [Context Management](context-management-guide.md) | GraphContext, ServerContext usage | Intermediate | -| [Node Operations](node-operations.md) | Working with nodes | All levels | - -### **API & Server** - -| Document | Description | Audience | -|----------|-------------|----------| -| [REST API](rest-api.md) | API design and endpoints | All levels | -| [API Architecture](api-architecture.md) | Server architecture | Advanced | -| [Server API](server-api.md) | Server configuration | Intermediate | -| [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended entrypoint and auto-registration | Intermediate | -| [Examples](examples.md) | **Standard implementation examples** | ⭐ **Start Here** | -| [Decorator Reference](decorator-reference.md) | All decorators explained | All levels | - -### **Authentication & Security** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Authentication](authentication.md) | Auth system overview | Intermediate | -| [Auth Quickstart](auth-quickstart.md) | Get auth working fast | Beginners | -| [Testing Guide](testing-guide.md) | Test auth mode and isolated databases | Intermediate | -| [Password Migration Guide](password-migration-guide.md) | bcrypt upgrade and transparent migration | Intermediate | - -### **Integrations** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Webhooks Architecture](webhook-architecture.md) | Webhook system design | Advanced | -| [Webhooks Quickstart](webhooks-quickstart.md) | Using webhooks | Intermediate | -| [Scheduler](scheduler.md) | Background job scheduling | Intermediate | -| [File Storage](file-storage-architecture.md) | File storage system | Intermediate | -| [File Storage Usage](file-storage-usage.md) | Using file storage | All levels | - -### **Database & Caching** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Graph Context](graph-context.md) | Database management and multi-database support (JSON, SQLite, MongoDB, DynamoDB) | Intermediate | -| [DynamoDB Guide](dynamodb-guide.md) | DynamoDB setup and configuration | Intermediate | -| [MongoDB Query Interface](mongodb-query-interface.md) | Database queries | Intermediate | -| [Custom Database Guide](custom-database-guide.md) | Implementing custom database backends | Advanced | -| [Caching](caching.md) | Cache strategies | Intermediate | -| [Text Normalization](text-normalization.md) | Unicode to ASCII text normalization | All levels | - -### **Logging** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Logging Service](logging-service.md) | Database logging with automatic persistence | All levels | -| [Custom Log Levels](custom-log-levels.md) | Domain-specific log levels (AUDIT, SECURITY, etc.) | Intermediate | - -### **Advanced Topics** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Architectural Decisions](architectural-decisions.md) | ADRs and design rationale | Advanced | -| [Module Responsibility Matrix](module-responsibility-matrix.md) | Module organization | Advanced | -| [Import Patterns](import-patterns.md) | Best practices for imports | Intermediate | -| [Design Decisions](design-decisions.md) | Design philosophy | Advanced | -| [Optimization](optimization.md) | Performance tuning | Advanced | -| [Error Handling](error-handling.md) | Error patterns | Intermediate | - -### **Development** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Contributing](contributing.md) | Contribution guide | Developers | -| [Custom Database Guide](custom-database-guide.md) | Extending with custom databases | Advanced | -| [Troubleshooting](troubleshooting.md) | Common issues | All levels | -| [Migration Guide](migration.md) | Adopting jvspatial | Users | - -### **Reference** - -| Document | Description | Audience | -|----------|-------------|----------| -| [Attribute Annotations](attribute-annotations.md) | @attribute | All levels | -| [Walker Events](walker-reporting-events.md) | Walker event system | Intermediate | -| [Walker Queue](walker-queue-operations.md) | Queue management | Advanced | -| [Walker Trail](walker-trail-tracking.md) | Trail tracking | Advanced | -| [Pagination](pagination.md) | Paginating results | Intermediate | -| [Environment Config](environment-configuration.md) | Configuration options | All levels | -| [Environment Keys Reference](environment-keys-reference.md) | Canonical valid env key inventory | All levels | -| [Infinite Walk Protection](infinite-walk-protection.md) | Preventing infinite loops | Advanced | - ---- - -## 🎯 **Learning Paths** - -### **Path 1: Beginner → Intermediate** - -1. ✅ [Quick Start Guide](quick-start-guide.md) -2. ✅ [Examples](examples.md) -3. ✅ [Entity Reference](entity-reference.md) -4. ✅ [Graph Traversal](graph-traversal.md) -5. ✅ [REST API](rest-api.md) -6. ✅ [Decorator Reference](decorator-reference.md) - -### **Path 2: API Development** - -1. ✅ [Quick Start Guide](quick-start-guide.md) -2. ✅ [REST API](rest-api.md) -3. ✅ [Decorator Reference](decorator-reference.md) -4. ✅ [Endpoint Registration Guide](endpoint-registration-guide.md) -5. ✅ [Authentication](authentication.md) -6. ✅ [Server API](server-api.md) -7. ✅ [Webhooks Quickstart](webhooks-quickstart.md) - -### **Path 3: Advanced Architecture** - -1. ✅ [Module Responsibility Matrix](module-responsibility-matrix.md) -2. ✅ [Import Patterns](import-patterns.md) -3. ✅ [API Architecture](api-architecture.md) -4. ✅ [Architectural Decisions](architectural-decisions.md) -5. ✅ [Design Decisions](design-decisions.md) -6. ✅ [Optimization](optimization.md) - ---- - -## 🔍 **Quick Reference** - -### **Common Tasks** - -| Task | Document | Section | -|------|----------|---------| -| Create a node | [Quick Start](quick-start-guide.md) | Pattern 1 | -| Define an endpoint | [Quick Start](quick-start-guide.md) | Step 3 | -| Setup endpoint registration | [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended Entrypoint | -| Build a walker | [Graph Traversal](graph-traversal.md) | Basic Walker | -| Visualize graph | [Graph Visualization](graph-visualization.md) | Quick Start | -| Query database | [MongoDB Query](mongodb-query-interface.md) | Query Builder | -| Add authentication | [Auth Quickstart](auth-quickstart.md) | Setup | -| Setup caching | [Caching](caching.md) | Configuration | -| Handle files | [File Storage Usage](file-storage-usage.md) | Basic Usage | -| Schedule jobs | [Scheduler](scheduler.md) | Basic Tasks | -| Setup logging | [Logging Service](logging-service.md) | Quick Start | -| Use custom log levels | [Custom Log Levels](custom-log-levels.md) | Quick Start | - -### **Common Issues** - -| Issue | Document | Solution | -|-------|----------|----------| -| Import errors | [Troubleshooting](troubleshooting.md) | Import Issues | -| Context errors | [Context Management](context-management-guide.md) | Usage Patterns | -| Import patterns | [Import Patterns](import-patterns.md) | Best Practices | -| Authentication fails | [Troubleshooting](troubleshooting.md) | Auth Issues | -| 401 with valid token | [Troubleshooting](troubleshooting.md) | Authentication: 401 with valid token | -| Database path wrong | [Troubleshooting](troubleshooting.md) | Database path wrong | - ---- - -## 📖 **Document Categories** - -### **By Complexity** - -**Beginner** (⭐): -- Quick Start Guide -- Examples -- Entity Reference -- Node Operations -- REST API -- Auth Quickstart -- Webhooks Quickstart -- File Storage Usage - -**Intermediate** (⭐⭐): -- Graph Traversal -- Context Management -- Authentication -- Server API -- Scheduler -- MongoDB Query Interface -- Caching -- Import Patterns -- Error Handling - -**Advanced** (⭐⭐⭐): -- API Architecture -- Module Responsibility Matrix -- Architectural Decisions -- Design Decisions -- Optimization -- Webhook Architecture -- File Storage Architecture -- Walker Queue Operations -- Walker Trail Tracking -- Infinite Walk Protection - -### **By Topic** - -**Core Graph**: -- Graph Traversal -- Graph Visualization -- Entity Reference -- Node Operations -- Walker Events -- Walker Queue Operations -- Walker Trail Tracking - -**API Development**: -- REST API -- API Architecture -- Server API -- Endpoint Registration Guide -- Decorator Reference -- Error Handling - -**Authentication & Security**: -- Authentication -- Auth Quickstart -- Attribute Annotations - -**Integrations**: -- Webhooks Architecture -- Webhooks Quickstart -- Scheduler -- File Storage Architecture -- File Storage Usage - -**Data Management**: -- MongoDB Query Interface -- Caching -- Pagination -- Text Normalization - -**Logging**: -- Logging Service -- Custom Log Levels - -**Architecture**: -- Module Responsibility Matrix -- Import Patterns -- Architectural Decisions -- Design Decisions - ---- - -## 🎓 **Glossary** - -| Term | Definition | -|------|------------| -| **Node** | A data point in the graph | -| **Edge** | A relationship between nodes | -| **Walker** | A pattern for traversing the graph | -| **Context** | Manages database and configuration | -| **Root** | Entry point to the graph | -| **Endpoint** | An API route | -| **Query Builder** | Fluent interface for database queries | -| **Cache Backend** | Storage for cached data | -| **Storage Interface** | File storage abstraction | -| **Decorator** | Function/class modifier | +## Getting Started + +| Document | What's in it | +|---|---| +| [Quick Start Guide](quick-start-guide.md) | Minimal end-to-end setup: install, create a Server, define a Node, expose an endpoint. | +| [Examples](examples.md) | Index of runnable scripts in [examples/](../../examples/). | +| [Auth Quickstart](auth-quickstart.md) | Fast path to authenticated endpoints. | +| [Endpoint Registration Guide](endpoint-registration-guide.md) | Recommended entrypoint pattern and auto-registration semantics. | +| [Migration Guide](migration.md) | Adopting jvspatial in an existing project. | + +## Core Concepts + +| Document | What's in it | +|---|---| +| [Entity Reference](entity-reference.md) | `Object`, `Node`, `Edge`, `Walker`, `Root` — fields, lifecycle, persistence shape. | +| [Attribute Annotations](attribute-annotations.md) | `@attribute(protected, transient, private, indexed, …)` semantics. | +| [Graph Context](graph-context.md) | Database + cache + monitor binding; multi-database setup. | +| [Context Management Guide](context-management-guide.md) | When and how to scope `GraphContext` / `ServerContext`. | +| [Graph Traversal](graph-traversal.md) | Walker pattern, queue semantics, visit hooks. | +| [Graph Visualization](graph-visualization.md) | DOT / Mermaid export. | +| [Node Operations](node-operations.md) | Connect / disconnect / neighbor queries. | +| [Walker Events](walker-reporting-events.md) | Walker event bus and reporting. | +| [Walker Queue Operations](walker-queue-operations.md) | Queue manipulation patterns. | +| [Walker Trail Tracking](walker-trail-tracking.md) | Trail capture, metadata, summary. | +| [Infinite Walk Protection](infinite-walk-protection.md) | `max_steps` / `max_visits_per_node` / `max_execution_time`. | + +## API and Server + +| Document | What's in it | +|---|---| +| [REST API](rest-api.md) | Endpoint patterns and conventions. | +| [Server API](server-api.md) | `Server` configuration surface. | +| [API Architecture](api-architecture.md) | Mixin composition, request lifecycle, middleware stack. | +| [Decorator Reference](decorator-reference.md) | Every decorator the library ships: `@endpoint`, `@attribute`, `@on_visit`, etc. | +| [Pagination](pagination.md) | `ObjectPager` usage. | +| [Rate Limiting](rate-limiting.md) | **NEW** Token-bucket rate limit configuration. | +| [Error Handling](error-handling.md) | Exception taxonomy and propagation. | + +## Authentication and Security + +| Document | What's in it | +|---|---| +| [Authentication](authentication.md) | JWT, API key, refresh-token flows. | +| [API Keys](api-keys.md) | **NEW** Key management endpoints and hashing model. | +| [Password Migration Guide](password-migration-guide.md) | bcrypt upgrade path. | +| [Security Review](security-review.md) | **NEW** Audit findings and resolved fixes. | +| [Security Operational Notes](security-operational-notes.md) | **NEW** Runtime security guidance. | +| [Production Deployment](production-deployment.md) | **NEW** Hardening checklist for production. | + +## Database and Storage + +| Document | What's in it | +|---|---| +| [MongoDB Query Interface](mongodb-query-interface.md) | Mongo-style operators across all backends. | +| [Custom Database Guide](custom-database-guide.md) | Implementing and registering a new `Database` adapter. | +| [DynamoDB Guide](dynamodb-guide.md) | DynamoDB-specific setup and limits. | +| [File Storage Architecture](file-storage-architecture.md) | Storage interface, security layer, version model. | +| [File Storage Usage](file-storage-usage.md) | Upload, list, download, versioning. | +| [Caching](caching.md) | Memory / Redis / layered caches; read-through DB wrapper. | +| [Text Normalization](text-normalization.md) | Unicode → ASCII normalization for query stability. | + +## Observability and Operations + +| Document | What's in it | +|---|---| +| [Observability](observability.md) | **NEW** Structured DB op logs, `MetricsRecorder`, slow-query threshold, OTEL adapter. | +| [Logging Service](logging-service.md) | Persistent application logging. | +| [Custom Log Levels](custom-log-levels.md) | Adding domain levels (`AUDIT`, `SECURITY`, …). | +| [Benchmarks](benchmarks.md) | **NEW** Regression-detection bench suite. | +| [Serverless Mode](serverless-mode.md) | **NEW** Detection precedence, mode-dependent defaults, LWA env. | +| [Scheduler](scheduler.md) | In-process scheduler and decorators. | +| [Webhook Architecture](webhook-architecture.md) | Webhook routing, signature verification. | +| [Webhooks Quickstart](webhooks-quickstart.md) | Common webhook patterns. | +| [Troubleshooting](troubleshooting.md) | Symptoms → root causes for common issues. | + +## Architecture and Conventions + +| Document | What's in it | +|---|---| +| [Architectural Decisions](architectural-decisions.md) | ADR-style design rationale. | +| [Design Decisions](design-decisions.md) | Philosophy and tradeoffs. | +| [Module Responsibility Matrix](module-responsibility-matrix.md) | Which package owns which concern. | +| [Import Patterns](import-patterns.md) | Stable vs internal imports. | +| [Stability](stability.md) | **NEW** Public / internal / experimental tiers + deprecation policy. | +| [Optimization](optimization.md) | Performance tuning playbook. | + +## Configuration + +| Document | What's in it | +|---|---| +| [Environment Configuration](environment-configuration.md) | `JVSPATIAL_*` allowlist behavior and merge order. | +| [Environment Keys Reference](environment-keys-reference.md) | Canonical inventory of every valid env key. | + +## Contributing + +| Document | What's in it | +|---|---| +| [Contributing](contributing.md) | Dev loop, conventions, label glossary. | +| [Testing Guide](testing-guide.md) | Async test patterns, fixtures, auth-test isolation. | +| [License](license.md) | MIT license reference. | --- -## 🔧 **API Reference** - -### **Core Modules** - -```python -# Core entities -from jvspatial import Object, Node, Edge, Walker, Root - -# Graph operations -from jvspatial.core import GraphContext, on_visit, on_exit - -# Graph visualization -from jvspatial.core.graph import generate_graph_dot, generate_graph_mermaid, export_graph +## Quick Reference -# API -from jvspatial.api import Server, ServerConfig, endpoint, get_auth_service +### Common Tasks -# Database -from jvspatial.db import create_database, Database +| Task | Start here | +|---|---| +| Create your first node + endpoint | [Quick Start Guide](quick-start-guide.md) | +| Switch from JSON to MongoDB | [Graph Context](graph-context.md) | +| Add authentication | [Auth Quickstart](auth-quickstart.md) | +| Implement a custom database backend | [Custom Database Guide](custom-database-guide.md) | +| Deploy to AWS Lambda | [Serverless Mode](serverless-mode.md), [Production Deployment](production-deployment.md) | +| Build a walker | [Graph Traversal](graph-traversal.md), [Walker Events](walker-reporting-events.md) | +| Surface metrics | [Observability](observability.md) | +| Verify SLA / catch regressions | [Benchmarks](benchmarks.md) | +| Audit security posture | [Security Review](security-review.md), [Security Operational Notes](security-operational-notes.md) | -# Cache -from jvspatial.cache import create_cache +### Authoritative Sources -# Storage -from jvspatial.storage.interfaces import LocalFileInterface +When documents disagree, this is the order of trust: -# Utils -from jvspatial.utils import memoize, retry, NodeId -``` - ---- - -## 📊 **Version History** - -| Version | Date | Changes | -|---------|------|---------| -| **0.0.6** | 2025-03-16 | Auth streamlining: get_auth_service(), bootstrap_admin, on_user_registered/on_admin_bootstrapped/on_enrich_current_user callbacks, built-in /auth/me, user_id guarantee, exempt path auto-discovery, argon2 fix | -| **0.0.5** | 2025-03-15 | JWT auth uses prime DB (fixes 401 with valid token), db_path_resolve, docs updates | -| **0.0.4** | 2025-03-14 | Auto-registration for endpoints, Endpoint Registration Guide | -| **0.0.3** | 2025-02-22 | Documentation updates, DynamoDB support | -| **0.0.2** | 2025-xx-xx | Bug fixes and improvements | -| **0.0.1** | 2025-xx-xx | Initial release | - ---- - -## 🤝 **Contributing** - -Want to improve the documentation? - -1. Read the [Contributing Guide](contributing.md) -2. Check for open documentation issues -3. Submit a pull request - ---- - -## 📧 **Support** - -- **Documentation Issues**: Open a GitHub issue -- **Questions**: GitHub Discussions -- **Email**: support@jvspatial.com - ---- - -## 📝 **License** - -See [License](license.md) for details. - ---- +1. Source code (cited in [SPEC.md](../../SPEC.md)) +2. [SPEC.md](../../SPEC.md) — contract +3. [PRD.md](../../PRD.md) — product context +4. [docs/md/](.) — how-to documentation +5. [LLM-CODING-GUIDE.md](../../LLM-CODING-GUIDE.md) — legacy code-pattern cookbook (no longer authoritative on contracts) -**Last Updated**: 2025-03-16 -**Version**: 0.0.6 -**Maintainer**: JVspatial Team +If you find a doc in this directory that contradicts SPEC, the doc is wrong — open an issue. diff --git a/docs/md/entity-reference.md b/docs/md/entity-reference.md index f3bb1bf..33fecff 100644 --- a/docs/md/entity-reference.md +++ b/docs/md/entity-reference.md @@ -93,6 +93,15 @@ class Edge(Object): async def count(cls, query: Optional[dict] = None, **kwargs) -> int # Inherited from Object ``` +#### Convention: `Node` vs `Object` + +The library lets you persist either as a `Node` (graph member) or an `Object` (flat record). Pick correctly — orphaned nodes and graph-traversed objects are both anti-patterns. Conventions for applications built on jvspatial: + +- **A `Node` MUST be connected to the graph.** Every node must be reachable from `Root`, directly or through intermediate nodes. If a record is never traversed to or from, it should not be a `Node`. +- **Declare an application-root node.** Create a single top-level class — `App`, `MyApp`, or similar — extending `Node`. Instantiate once and connect to `Root`. Treat this app-root as the entry point for all application state. +- **All application nodes hang off the app-root.** Connect them directly to the app-root, or beneath a branch/category node under it (e.g. `App -> Users -> User`, `App -> Catalog -> Product`). Do not connect application nodes directly to `Root`. +- **Use `Object` for record-style data with no graph relationships.** Audit logs, change events, denormalized snapshots, ledger entries, and raw inbound payloads belong in `Object`, not `Node`. **`ChangeEvent` is an `Object`.** Rule of thumb: if nothing will ever traverse to or from the record, it is not a node. + #### `Walker` Graph traversal agent with hook-based logic. diff --git a/docs/md/environment-keys-reference.md b/docs/md/environment-keys-reference.md index e96763e..932db0f 100644 --- a/docs/md/environment-keys-reference.md +++ b/docs/md/environment-keys-reference.md @@ -31,6 +31,7 @@ For full examples and default values, see: - `JVSPATIAL_API_HEALTH` - Health route path. - `JVSPATIAL_API_ROOT` - Root route path. - `JVSPATIAL_GRAPH_ENDPOINT_ENABLED` - Enables graph REST endpoint. +- `JVSPATIAL_DOCS_DISABLED` - When truthy (`1`/`true`/`yes`/`on`), unpublishes the documentation surface entirely: `/docs`, `/redoc`, `/openapi.json`, and `/docs/oauth2-redirect` are not registered (return 404). Recommended for production. ### CORS - `JVSPATIAL_CORS_ENABLED` - Enables CORS middleware. diff --git a/docs/md/production-deployment.md b/docs/md/production-deployment.md index 16706c9..8e4613d 100644 --- a/docs/md/production-deployment.md +++ b/docs/md/production-deployment.md @@ -51,6 +51,25 @@ Security headers (X-Content-Type-Options, X-Frame-Options, X-XSS-Protection) are server = Server(security=dict(security_headers_enabled=False)) ``` +The `Content-Security-Policy` header is strict by default on application +routes. The bundled Swagger UI / ReDoc pages (`/docs`, `/redoc`, +`/openapi.json`) get a relaxed CSP that permits `cdn.jsdelivr.net` so the +docs render — no extra config needed in dev. + +#### Unpublishing the docs surface in production + +Set a single env var to remove `/docs`, `/redoc`, `/openapi.json`, and +`/docs/oauth2-redirect` from the registered routes entirely (FastAPI +returns 404 — no Swagger HTML, no OpenAPI JSON): + +``` +JVSPATIAL_DOCS_DISABLED=1 +``` + +Truthy values: `1`, `true`, `yes`, `on` (case-insensitive). Unset or any +other value keeps the docs published. Use this in production when the API +surface should not be self-documenting. + ### 4. CORS Configuration The default CORS configuration allows localhost origins. For production, restrict to your actual frontend domain(s): @@ -104,6 +123,7 @@ JVSPATIAL_WEBHOOK_HMAC_SECRET=your-webhook-secret-minimum-32-chars | `JVSPATIAL_CORS_ORIGINS` | Your frontend domain(s), not `*` | | `JVSPATIAL_WEBHOOK_HMAC_SECRET` | Unique per environment, 32+ chars | | `JVSPATIAL_WEBHOOK_HTTPS_REQUIRED` | `true` | +| `JVSPATIAL_DOCS_DISABLED` | `1` (unpublishes `/docs`, `/redoc`, `/openapi.json`) | ## Kubernetes / Container Deployment diff --git a/docs/md/quick-start-guide.md b/docs/md/quick-start-guide.md index 98fa9db..cf52c18 100644 --- a/docs/md/quick-start-guide.md +++ b/docs/md/quick-start-guide.md @@ -27,9 +27,11 @@ pip install jvspatial[all] # All optional deps jvspatial is built on three key concepts: ### **1. Graph Entities** -- **Nodes**: Data points in your graph +- **Nodes**: Data points in your graph — every `Node` must be reachable from `Root` (directly or indirectly) - **Edges**: Relationships between nodes -- **Root**: Entry point to your graph +- **Root**: Library-owned entry point to the graph +- **App-root node**: Convention — define your own top-level node (e.g. `class App(Node)`), connect it to `Root` once, and hang all application nodes off it. See [Entity Reference → Convention: Node vs Object](entity-reference.md#convention-node-vs-object). +- **Objects**: Record-style data with no graph relationships (audit logs, change events, snapshots). Use `Object`, not `Node`. ### **2. Walkers** - Traverse the graph diff --git a/jvgraph-ui/package-lock.json b/jvgraph-ui/package-lock.json index 86832be..b7d9fcd 100644 --- a/jvgraph-ui/package-lock.json +++ b/jvgraph-ui/package-lock.json @@ -1280,12 +1280,12 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.14.0.tgz", - "integrity": "sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", + "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.11", + "follow-redirects": "^1.16.0", "form-data": "^4.0.5", "proxy-from-env": "^2.1.0" } @@ -1637,9 +1637,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", @@ -1843,9 +1843,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/loose-envify": { @@ -1965,9 +1965,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", + "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", "dev": true, "funding": [ { @@ -2211,9 +2211,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", + "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", "dev": true, "license": "MIT", "peer": true, diff --git a/jvspatial/api/README.md b/jvspatial/api/README.md new file mode 100644 index 0000000..86bd83f --- /dev/null +++ b/jvspatial/api/README.md @@ -0,0 +1,83 @@ +# jvspatial/api + +FastAPI integration: `Server` class, endpoint decorator, auth, middleware, and lifecycle. + +> **Read first**: [SPEC §8-10](../../SPEC.md), [docs/md/api-architecture.md](../../docs/md/api-architecture.md) + +--- + +## Purpose + +`api/` adapts the entity layer to FastAPI. The `Server` class composes app construction, route registration, lifecycle, and uvicorn invocation. `@endpoint` turns functions or Walker classes into HTTP routes with consistent auth, validation, and response shaping. + +## Layout + +``` +api/ +├── server.py # Server (composition of 4 mixins) +├── server_app_factory.py # AppFactoryMixin (internal) +├── server_registration.py # RegistrationMixin (internal) +├── server_lifecycle.py # LifecycleMixin (internal) +├── server_run.py # RunMixin (internal) +├── config.py # ServerConfig (Pydantic) +├── config_groups.py # DatabaseConfig, AuthConfig, etc. +├── context.py # ServerContext + per-request helpers +├── exceptions.py # APIError hierarchy +├── auth/ # JWT, API keys, RBAC, sessions +├── components/ # AppBuilder, AuthConfigurator, middleware +├── decorators/ # @endpoint, deferred registry, endpoint fields +├── endpoints/ # Registry, factory, router, response +├── integrations/ # Webhooks, scheduler, storage service +├── middleware/ # Rate limit, manager +├── services/ # Endpoint discovery +└── utils/ # Misc helpers +``` + +## Public API (from `jvspatial.api`) + +| Name | What it does | +|---|---| +| `Server` | Main server class (SPEC §8.1) | +| `create_server` | Functional constructor | +| `ServerConfig` | Pydantic config (SPEC §10.1) | +| `ServerContext`, `get_current_server`, `set_current_server` | Per-request server binding | +| `get_auth_service` | Singleton accessor for `AuthenticationService` | +| `@endpoint` | Unified function / Walker route decorator (SPEC §8.2) | +| `BaseRouter`, `EndpointRouter` | Mountable routers | +| `endpoint_field`, `EndpointField`, `EndpointFieldInfo` | Endpoint parameter helpers | +| `format_response`, `ResponseHelper` | Response shaping | +| `register_deferred_endpoint`, `flush_deferred_endpoints`, `get_deferred_endpoint_count`, `clear_deferred_endpoints`, `sync_endpoint_modules` | Deferred-registry utilities (SPEC §8.3) | + +## Invariants + +- **Auth state lives on the prime database only.** Users, sessions, API keys, refresh tokens, password-reset tokens — all on prime DB. Not relocatable. (`auth/service.py`) +- **JWT secret is required when auth is enabled.** Server fails fast with a clear error if `JVSPATIAL_JWT_SECRET_KEY` is empty or a placeholder. (`auth/service.py`, [CHANGELOG 0.0.7](../../CHANGELOG.md)) +- **All secret comparisons use `hmac.compare_digest`.** No `==` for tokens, keys, or hashes. +- **CORS defaults are restrictive.** Wildcards must be explicit and trigger a startup warning. (`components/cors_configurator.py`) +- **CSP is strict on app routes, relaxed only on `/docs`, `/redoc`, `/openapi.json`.** (`components/app_builder.py`) +- **`JVSPATIAL_DOCS_DISABLED` removes the entire docs surface.** No spec leak when truthy. (`components/app_builder.py`) +- **Sessions and rate-limit counters are per-process.** Multi-worker deployments multiply configured limits by worker count. +- **Endpoint registration is deferred.** `@endpoint` collects targets at import; `Server` resolves them at app build time. + +## Modification patterns + +- Adding a new endpoint kind: extend `decorators/route.py` and update `endpoints/factory.py`. New decorator forms must be auth/role/webhook-aware. +- Adding new middleware: register via `Server.middleware_manager.add(...)`. Built-ins live in `middleware/` and `components/`. +- Adding a new auth flow: extend `auth/service.py` and `auth/rbac.py`. Add a security-review entry per [docs/md/security-review.md](../../docs/md/security-review.md). +- Adding a new lifecycle hook: declare in `Server` constructor signature, wire through `LifecycleMixin`, document under [docs/md/server-api.md](../../docs/md/server-api.md). + +## Related docs + +- [docs/md/api-architecture.md](../../docs/md/api-architecture.md) +- [docs/md/server-api.md](../../docs/md/server-api.md) +- [docs/md/endpoint-registration-guide.md](../../docs/md/endpoint-registration-guide.md) +- [docs/md/decorator-reference.md](../../docs/md/decorator-reference.md) +- [docs/md/authentication.md](../../docs/md/authentication.md) +- [docs/md/api-keys.md](../../docs/md/api-keys.md) +- [docs/md/auth-quickstart.md](../../docs/md/auth-quickstart.md) +- [docs/md/rate-limiting.md](../../docs/md/rate-limiting.md) +- [docs/md/webhook-architecture.md](../../docs/md/webhook-architecture.md) + +## Stability + +Public names listed above are stable. The `server_*` mixin modules are internal — assemble through `Server` only. `components/`, `middleware/`, `services/`, and `integrations/` internals can change between minor versions; cross them only through the public surface. diff --git a/jvspatial/api/auth/api_key_service.py b/jvspatial/api/auth/api_key_service.py index b24d1bf..8d10653 100644 --- a/jvspatial/api/auth/api_key_service.py +++ b/jvspatial/api/auth/api_key_service.py @@ -23,11 +23,15 @@ def __init__(self, context: Optional[GraphContext] = None): Args: context: GraphContext instance for database operations. - If None, uses the default context. + If None, defaults to the prime database — never the + application default context. Auth state must live on + the prime DB (SPEC §9 / CLAUDE.md §1; audit §4.4). """ - from jvspatial.core.context import get_default_context + if context is None: + from jvspatial.db import get_prime_database - self.context = context or get_default_context() + context = GraphContext(database=get_prime_database()) + self.context = context self._logger = logging.getLogger(__name__) self.key_prefix = "sk_" # Default prefix, can be configured self.key_length = 32 # Length of random part after prefix @@ -242,6 +246,17 @@ async def revoke_key(self, key_id: str, user_id: str) -> bool: api_key._graph_context = self.context await self.context.save(api_key) + # Drop the webhook-layer cache entry so revocation is effective + # immediately rather than after the 5-minute TTL (audit §4.5). + try: + from jvspatial.api.integrations.webhooks.webhook_auth import ( + invalidate_api_key_cache_hash, + ) + + invalidate_api_key_cache_hash(api_key.key_hash) + except Exception: # pragma: no cover — webhook module is optional + pass + self._logger.info(f"Revoked API key {key_id} for user {user_id}") return True diff --git a/jvspatial/api/auth/enhanced.py b/jvspatial/api/auth/enhanced.py index ddc3c87..dae2001 100644 --- a/jvspatial/api/auth/enhanced.py +++ b/jvspatial/api/auth/enhanced.py @@ -4,6 +4,7 @@ rate limiting, brute force protection, and session management. """ +import asyncio import hashlib import logging import time @@ -244,6 +245,11 @@ def __init__(self, session_timeout: int = 3600, max_sessions_per_user: int = 5): self._sessions: Dict[str, Dict[str, Any]] = {} self._user_sessions: Dict[str, Set[str]] = {} self._logger = logging.getLogger(__name__) + # Single lock guards both ``_sessions`` and ``_user_sessions`` so + # concurrent create/invalidate/cleanup cannot raise + # ``RuntimeError: dictionary changed size during iteration`` and + # ``max_sessions_per_user`` enforcement is not racy (audit §4.8). + self._lock = asyncio.Lock() async def create_session(self, user_id: str, user_data: Dict[str, Any]) -> str: """Create a new session for a user. @@ -260,22 +266,33 @@ async def create_session(self, user_id: str, user_data: Dict[str, Any]) -> str: session_id = str(uuid.uuid4()) now = time.time() - # Clean up old sessions for this user - await self._cleanup_user_sessions(user_id) - - # Create new session - self._sessions[session_id] = { - "user_id": user_id, - "created_at": now, - "last_accessed": now, - "user_data": user_data, - "is_active": True, - } - - # Track user sessions - if user_id not in self._user_sessions: - self._user_sessions[user_id] = set() - self._user_sessions[user_id].add(session_id) + async with self._lock: + # Inline expired-session sweep under the lock so the cap + # check below sees a coherent set. + self._sweep_user_sessions_locked(user_id, now) + + # Enforce per-user cap. + existing = self._user_sessions.get(user_id, set()) + while len(existing) >= self.max_sessions_per_user: + # Drop the oldest by last_accessed. + oldest = min( + (s for s in existing if s in self._sessions), + key=lambda s: self._sessions[s].get("last_accessed", 0), + default=None, + ) + if oldest is None: + break + self._invalidate_session_locked(oldest) + existing = self._user_sessions.get(user_id, set()) + + self._sessions[session_id] = { + "user_id": user_id, + "created_at": now, + "last_accessed": now, + "user_data": user_data, + "is_active": True, + } + self._user_sessions.setdefault(user_id, set()).add(session_id) self._logger.info(f"Created session {session_id} for user {user_id}") return session_id @@ -289,21 +306,22 @@ async def validate_session(self, session_id: str) -> Optional[Dict[str, Any]]: Returns: User data if session is valid, None otherwise """ - if session_id not in self._sessions: - return None + async with self._lock: + if session_id not in self._sessions: + return None - session = self._sessions[session_id] - now = time.time() + session = self._sessions[session_id] + now = time.time() - # Check if session is expired - if now - session["last_accessed"] > self.session_timeout: - await self.invalidate_session(session_id) - return None + # Check if session is expired + if now - session["last_accessed"] > self.session_timeout: + self._invalidate_session_locked(session_id) + return None - # Update last accessed time - session["last_accessed"] = now + # Update last accessed time + session["last_accessed"] = now - return session["user_data"] + return session["user_data"] async def invalidate_session(self, session_id: str) -> bool: """Invalidate a session. @@ -314,6 +332,11 @@ async def invalidate_session(self, session_id: str) -> bool: Returns: True if session was invalidated, False otherwise """ + async with self._lock: + return self._invalidate_session_locked(session_id) + + def _invalidate_session_locked(self, session_id: str) -> bool: + """Remove ``session_id`` from internal maps. Caller holds ``_lock``.""" if session_id not in self._sessions: return False @@ -341,38 +364,31 @@ async def invalidate_user_sessions(self, user_id: str) -> int: Returns: Number of sessions invalidated """ - if user_id not in self._user_sessions: - return 0 - - session_ids = list(self._user_sessions[user_id]) - invalidated_count = 0 - - for session_id in session_ids: - if await self.invalidate_session(session_id): - invalidated_count += 1 - - return invalidated_count - - async def _cleanup_user_sessions(self, user_id: str) -> None: - """Clean up old sessions for a user. - - Args: - user_id: User ID to clean up sessions for - """ + async with self._lock: + session_ids = list(self._user_sessions.get(user_id, set())) + count = 0 + for session_id in session_ids: + if self._invalidate_session_locked(session_id): + count += 1 + return count + + def _sweep_user_sessions_locked(self, user_id: str, now: float) -> None: + """Drop expired sessions for ``user_id``. Caller holds ``_lock``.""" if user_id not in self._user_sessions: return - now = time.time() - sessions_to_remove = [] - - for session_id in self._user_sessions[user_id]: - if session_id in self._sessions: - session = self._sessions[session_id] - if now - session["last_accessed"] > self.session_timeout: - sessions_to_remove.append(session_id) + # Iterate a snapshot — _invalidate_session_locked mutates the set. + for session_id in list(self._user_sessions[user_id]): + session = self._sessions.get(session_id) + if session is None: + continue + if now - session["last_accessed"] > self.session_timeout: + self._invalidate_session_locked(session_id) - for session_id in sessions_to_remove: - await self.invalidate_session(session_id) + async def _cleanup_user_sessions(self, user_id: str) -> None: + """Clean up old sessions for a user (acquires the lock).""" + async with self._lock: + self._sweep_user_sessions_locked(user_id, time.time()) async def cleanup_expired_sessions(self) -> int: """Clean up all expired sessions. @@ -380,17 +396,17 @@ async def cleanup_expired_sessions(self) -> int: Returns: Number of sessions cleaned up """ - now = time.time() - expired_sessions = [] - - for session_id, session in self._sessions.items(): - if now - session["last_accessed"] > self.session_timeout: - expired_sessions.append(session_id) - - for session_id in expired_sessions: - await self.invalidate_session(session_id) - - return len(expired_sessions) + async with self._lock: + now = time.time() + # Snapshot keys so the locked invalidate helper can mutate. + expired = [ + sid + for sid, s in self._sessions.items() + if now - s["last_accessed"] > self.session_timeout + ] + for sid in expired: + self._invalidate_session_locked(sid) + return len(expired) class AuthenticationEnhancer: diff --git a/jvspatial/api/auth/service.py b/jvspatial/api/auth/service.py index e25793c..3e49278 100644 --- a/jvspatial/api/auth/service.py +++ b/jvspatial/api/auth/service.py @@ -364,7 +364,12 @@ def _verify_refresh_token(self, token: str, hashed: str) -> bool: except Exception: return False else: - return hashlib.sha256(token.encode()).hexdigest() == hashed + # Constant-time comparison is non-negotiable for any secret + # comparison (SPEC §15.2, CLAUDE.md invariant §2). The earlier + # ``==`` form was timing-leakable on partial hex prefix matches. + return hmac.compare_digest( + hashlib.sha256(token.encode()).hexdigest(), hashed + ) def _generate_jwt_token( self, @@ -423,8 +428,10 @@ def _decode_jwt_token(self, token: str) -> Optional[Dict[str, Any]]: import logging logger = logging.getLogger(__name__) + # Do not log secret length — that narrows the search space + # for a brute force (audit §4.11 / SPEC §15.5). logger.debug( - f"JWT token decode failed: {e}, secret length: {len(self.jwt_secret) if self.jwt_secret else 0}" + f"JWT token decode failed: {e}, secret_configured={bool(self.jwt_secret)}" ) return None @@ -1049,13 +1056,13 @@ async def validate_token(self, token: str) -> Optional[UserResponse]: ) if not user: - db_path = getattr( - getattr(self.context, "database", None), "base_path", None - ) + # Avoid logging the filesystem path of the database — leaks + # on-disk layout to log sinks (audit §4.13 / SPEC §15.5). + db_type = type(getattr(self.context, "database", None)).__name__ self._logger.warning( - "[validate_token] failed: user %s not found in database (db_path=%s)", + "[validate_token] failed: user %s not found in database (db_type=%s)", user_id, - db_path, + db_type, ) return None diff --git a/jvspatial/api/components/app_builder.py b/jvspatial/api/components/app_builder.py index 0c29fc4..b9fa6a4 100644 --- a/jvspatial/api/components/app_builder.py +++ b/jvspatial/api/components/app_builder.py @@ -45,13 +45,30 @@ def create_app(self, lifespan: Optional[Any] = None) -> FastAPI: Returns: Configured FastAPI application instance + + Production unpublish: setting ``JVSPATIAL_DOCS_DISABLED=1`` (or + ``true``/``yes``/``on``) forces ``docs_url``, ``redoc_url``, + ``openapi_url``, and ``swagger_ui_oauth2_redirect_url`` to + ``None``. FastAPI then never registers any of those routes — they + return 404 with no body, the OpenAPI schema is not served, and no + Swagger UI HTML is emitted. Use this in production when the API + surface should not be self-documenting. """ - app_kwargs = { + # Use the canonical boolean parser instead of an ad-hoc inline + # set so behavior matches the rest of the codebase + # (audit §7.2 / §7.12). + from jvspatial.env import env, parse_bool + + docs_disabled = bool( + env("JVSPATIAL_DOCS_DISABLED", default=False, parse=parse_bool) + ) + + app_kwargs: Dict[str, Any] = { "title": self.config.title, "description": self.config.description, "version": self.config.version, - "docs_url": self.config.docs_url, - "redoc_url": self.config.redoc_url, + "docs_url": None if docs_disabled else self.config.docs_url, + "redoc_url": None if docs_disabled else self.config.redoc_url, "debug": self.config.debug, # Configure OpenAPI tags order - ensure "App" appears after "default" # Tags are ordered by their appearance in this list @@ -60,6 +77,11 @@ def create_app(self, lifespan: Optional[Any] = None) -> FastAPI: {"name": "App", "description": "Application-specific endpoints"}, ], } + if docs_disabled: + # Explicitly nuke the OpenAPI JSON spec and the Swagger OAuth2 + # redirect endpoint too — FastAPI defaults them on otherwise. + app_kwargs["openapi_url"] = None + app_kwargs["swagger_ui_oauth2_redirect_url"] = None # Add lifespan if provided if lifespan is not None: diff --git a/jvspatial/api/components/database_configurator.py b/jvspatial/api/components/database_configurator.py index b99bb6a..a486c64 100644 --- a/jvspatial/api/components/database_configurator.py +++ b/jvspatial/api/components/database_configurator.py @@ -8,7 +8,7 @@ import logging import os from pathlib import Path -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from jvspatial.core.context import GraphContext, set_default_context from jvspatial.db.factory import create_database @@ -79,6 +79,17 @@ def _resolve_mongodb_connection(self) -> Tuple[str, str]: db_name = (db.db_database_name or "").strip() or "jvdb" return uri, db_name + def _resolve_observability_kwargs(self) -> Dict[str, Any]: + """Resolve optional DB observability wrapper kwargs from env.""" + raw_enabled = str(os.environ.get("JVSPATIAL_OBSERVABILITY_ENABLED", "")).lower() + observe = raw_enabled in ("1", "true", "yes", "on") + slow_raw = os.environ.get("JVSPATIAL_SLOW_QUERY_MS", "") + try: + slow_query_ms = float(slow_raw) if slow_raw != "" else 100.0 + except ValueError: + slow_query_ms = 100.0 + return {"observe": observe, "slow_query_ms": slow_query_ms} + def initialize_graph_context(self) -> Optional[GraphContext]: """Initialize GraphContext with current database configuration. @@ -102,6 +113,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: try: # Create prime database based on configuration FIRST. prime_db = None + observability_kwargs = self._resolve_observability_kwargs() if db_type == "json": # Check if db_path is an S3 path (not supported for file-based databases) @@ -121,6 +133,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: prime_db = create_database( db_type="json", base_path=db_path, + **observability_kwargs, ) elif db_type == "mongodb": mongo_uri, mongo_db_name = self._resolve_mongodb_connection() @@ -128,6 +141,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: db_type="mongodb", uri=mongo_uri, db_name=mongo_db_name, + **observability_kwargs, ) elif db_type == "sqlite": # Check if db_path is an S3 path (not supported for file-based databases) @@ -147,6 +161,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: prime_db = create_database( db_type="sqlite", db_path=db_path, + **observability_kwargs, ) elif db_type == "dynamodb": prime_db = create_database( @@ -156,6 +171,7 @@ def initialize_graph_context(self) -> Optional[GraphContext]: endpoint_url=self.config.database.dynamodb_endpoint_url, aws_access_key_id=self.config.database.dynamodb_access_key_id, aws_secret_access_key=self.config.database.dynamodb_secret_access_key, + **observability_kwargs, ) else: raise ValueError(f"Unsupported database type: {db_type}") diff --git a/jvspatial/api/components/error_handler.py b/jvspatial/api/components/error_handler.py index c68b745..a3f5925 100644 --- a/jvspatial/api/components/error_handler.py +++ b/jvspatial/api/components/error_handler.py @@ -22,9 +22,46 @@ ) +def _is_production_environment() -> bool: + """Return True when the runtime is signalled as production. + + Looks at ``JVSPATIAL_ENVIRONMENT`` (canonical) and ``ENVIRONMENT`` + (fallback) for values ``prod`` / ``production``. Used to fail-closed + on the ``JVSPATIAL_EXPOSE_ERROR_DETAILS`` toggle (audit §4.10 / + SPEC §15.5). + """ + import os + + for var in ("JVSPATIAL_ENVIRONMENT", "ENVIRONMENT"): + raw = (os.environ.get(var) or "").strip().lower() + if raw in ("prod", "production"): + return True + return False + + def _expose_error_details_to_clients() -> bool: - """When true, unhandled 500 responses may include exception text (dev only).""" - return bool(env("JVSPATIAL_EXPOSE_ERROR_DETAILS", default=False, parse=parse_bool)) + """When true, unhandled 500 responses may include exception text. + + Refuses to honor the flag in production environments — leaking raw + exception text (which can include DB query fragments, file paths, + and secret-bearing config values) is a footgun (audit §4.10). + Emits a one-shot warning when the flag is requested but suppressed. + """ + requested = bool( + env("JVSPATIAL_EXPOSE_ERROR_DETAILS", default=False, parse=parse_bool) + ) + if requested and _is_production_environment(): + if not getattr(_expose_error_details_to_clients, "_warned", False): + logging.getLogger(__name__).warning( + "JVSPATIAL_EXPOSE_ERROR_DETAILS is set but the runtime " + "appears to be production (JVSPATIAL_ENVIRONMENT). " + "Ignoring the flag — 500 responses will use the generic " + "message. Unset JVSPATIAL_ENVIRONMENT or change the value " + "to enable detailed errors." + ) + _expose_error_details_to_clients._warned = True # type: ignore[attr-defined] + return False + return requested # Context variable to track exceptions that have been logged by our handler diff --git a/jvspatial/api/config_groups.py b/jvspatial/api/config_groups.py index a6c5cfe..b519d44 100644 --- a/jvspatial/api/config_groups.py +++ b/jvspatial/api/config_groups.py @@ -79,6 +79,36 @@ class CORSConfig(BaseModel): cors_headers: List[str] = Field( default_factory=lambda: ["Content-Type", "Authorization", "X-API-Key"] ) + # Set to ``True`` to silence the startup warning that fires when + # ``cors_origins`` contains a wildcard. Default opt-in is loud so + # accidental wildcards in production surface as logs (audit §4.12 / + # SPEC §15.4). + cors_allow_wildcard: bool = False + + @model_validator(mode="after") + def _warn_on_wildcard_origin(self) -> "CORSConfig": + """Emit a startup warning when CORS origins include a wildcard. + + SPEC §15.4: "Wildcard origins must be set explicitly and trigger + a startup warning." Browsers reject ``allow_origins=["*"]`` when + ``allow_credentials=True`` at runtime, but the surprise factor + of a permissive wildcard left in by mistake warrants a loud + warning here too (audit §4.12). + """ + import logging + + if not self.cors_enabled: + return self + wildcards = [o for o in self.cors_origins if o.strip() == "*" or "*" in o] + if wildcards and not self.cors_allow_wildcard: + logging.getLogger(__name__).warning( + "CORSConfig: wildcard origins detected: %s. This permits " + "any origin and is rarely correct in production. Set " + "cors_allow_wildcard=True on CORSConfig to silence this " + "warning.", + wildcards, + ) + return self class AuthConfig(BaseModel): diff --git a/jvspatial/api/decorators/function_wrappers.py b/jvspatial/api/decorators/function_wrappers.py index a9d28e4..4d38d31 100644 --- a/jvspatial/api/decorators/function_wrappers.py +++ b/jvspatial/api/decorators/function_wrappers.py @@ -45,10 +45,42 @@ def _is_request_type(param_annotation: Any) -> bool: return False -def _find_request_parameter(func_sig: inspect.Signature) -> Tuple[bool, Optional[str]]: - """Find Request parameter in function signature.""" +def _resolve_type_hints(func: Callable) -> Dict[str, Any]: + """Best-effort ``typing.get_type_hints`` for ``func``. + + Resolves PEP 563 / ``from __future__ import annotations`` string + forward refs to real types so ``_is_request_type`` sees a class + object instead of the literal string ``"Request"``. Falls back to + an empty dict when resolution fails (unresolvable forward ref, + missing import) — callers then degrade to the raw + ``param.annotation`` value. + """ + from typing import get_type_hints + + try: + return get_type_hints(func) + except Exception: + return {} + + +def _find_request_parameter( + func_sig: inspect.Signature, func: Optional[Callable] = None +) -> Tuple[bool, Optional[str]]: + """Find Request parameter in function signature. + + When ``func`` is supplied, annotations are resolved via + ``typing.get_type_hints`` so callers using + ``from __future__ import annotations`` (PEP 563) — which yields + string-typed ``param.annotation`` values — are still recognized + correctly. Mirrors the resolution that + ``ParameterModelFactory._create_function_model`` already performs. + """ + type_hints: Dict[str, Any] = _resolve_type_hints(func) if func is not None else {} for param_name, param in func_sig.parameters.items(): - if _is_request_type(param.annotation): + # Prefer the resolved type-hint when available; fall back to + # the raw signature annotation otherwise. + annotation = type_hints.get(param_name, param.annotation) + if _is_request_type(annotation): return True, param_name return False, None @@ -121,7 +153,7 @@ def wrap_function_auth_only( from fastapi import Request as FastAPIRequest func_sig = inspect.signature(func) - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -203,7 +235,7 @@ def wrap_function_with_params( for p in (*AUTH_INJECTED_PARAMS, *AUTH_INJECTED_USER_PARAMS) ) if needs_auth_injection: - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -372,7 +404,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: body_data.pop(excluded_param, None) combined = {**kwargs, **body_data} - orig_has_request_param, _ = _find_request_parameter(func_sig) + orig_has_request_param, _ = _find_request_parameter(func_sig, func) if ( orig_has_request_param and request_param_name @@ -426,7 +458,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: p in func_sig.parameters for p in (*AUTH_INJECTED_PARAMS, *AUTH_INJECTED_USER_PARAMS) ): - has_request, _ = _find_request_parameter(func_sig) + has_request, _ = _find_request_parameter(func_sig, func) auth_required = getattr(func, "_jvspatial_endpoint_config", {}).get( "auth_required", False ) @@ -557,7 +589,7 @@ async def wrapped_func(*args: Any, **kwargs: Any) -> Any: # type: ignore[assign for excluded_param in EXCLUDED_BODY_PARAMS: data.pop(excluded_param, None) - orig_has_request_param, _ = _find_request_parameter(func_sig) + orig_has_request_param, _ = _find_request_parameter(func_sig, func) if ( orig_has_request_param and request_param_name diff --git a/jvspatial/api/deferred_invoke_route.py b/jvspatial/api/deferred_invoke_route.py index be48b18..396525f 100644 --- a/jvspatial/api/deferred_invoke_route.py +++ b/jvspatial/api/deferred_invoke_route.py @@ -26,9 +26,23 @@ def _deferred_invoke_disabled() -> bool: def _deferred_invoke_secret_ok(request: Request) -> bool: + """Authorize the internal deferred-invoke endpoint. + + Fail-closed when ``JVSPATIAL_DEFERRED_INVOKE_SECRET`` is unset or + empty: the previous "no secret = allow everything" semantics were + a footgun — a misconfigured deployment exposed the internal + endpoint to any caller (audit §4.16 / SPEC §15.2). Disable the + route entirely via ``JVSPATIAL_DEFERRED_INVOKE_DISABLED=true`` if + you do not need it. + """ secret = env("JVSPATIAL_DEFERRED_INVOKE_SECRET") or "" if not secret: - return True + logger.warning( + "Deferred-invoke route rejected: " + "JVSPATIAL_DEFERRED_INVOKE_SECRET is unset. Either set a " + "secret or set JVSPATIAL_DEFERRED_INVOKE_DISABLED=true." + ) + return False hdr = (request.headers.get("X-JVSPATIAL-Deferred-Authorize") or "").strip() auth = request.headers.get("Authorization") or "" bearer = "" diff --git a/jvspatial/api/integrations/storage/service.py b/jvspatial/api/integrations/storage/service.py index a085cf9..bea61b1 100644 --- a/jvspatial/api/integrations/storage/service.py +++ b/jvspatial/api/integrations/storage/service.py @@ -156,7 +156,10 @@ async def handle_delete(self, file_path: str) -> Dict[str, Any]: HTTPException: On storage error """ try: - success = self.file_interface.delete_file(file_path) + # ``delete_file`` is async on every backend; missing ``await`` + # silently skipped the delete and assigned the coroutine to + # ``success`` (audit §3.3). + success = await self.file_interface.delete_file(file_path) return {"success": success, "file_path": file_path} except StorageError as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/jvspatial/api/integrations/webhooks/helpers.py b/jvspatial/api/integrations/webhooks/helpers.py index d14c9f0..43b442d 100644 --- a/jvspatial/api/integrations/webhooks/helpers.py +++ b/jvspatial/api/integrations/webhooks/helpers.py @@ -132,7 +132,11 @@ def inject_walker_webhook_payload(walker_class: type) -> type: """ original_init = walker_class.__init__ # type: ignore - async def enhanced_init(self, *args: Any, **kwargs: Any) -> None: + # ``__init__`` is a sync protocol — Python ignores ``async def __init__`` + # and returns the coroutine to the constructor, which then never gets + # awaited. The earlier ``async def`` here leaked a coroutine on every + # webhook walker construction (audit §3.1 / SPEC §3.1). + def enhanced_init(self, *args: Any, **kwargs: Any) -> None: # Extract webhook data from kwargs if present webhook_data = kwargs.pop("webhook_data", {}) @@ -212,9 +216,12 @@ async def webhook_wrapper(request: Request) -> Any: # For any other required parameters, we'll let the function handle it # and raise an error if needed - # Call original function with only the parameters it expects + # Call original function with only the parameters it expects. + # The async branch must ``await``; previously both arms were + # identical, so coroutines leaked unawaited (audit §3.2 / + # SPEC §3.1). if asyncio.iscoroutinefunction(endpoint_func): - result = endpoint_func(**kwargs) + result = await endpoint_func(**kwargs) else: result = endpoint_func(**kwargs) diff --git a/jvspatial/api/integrations/webhooks/utils.py b/jvspatial/api/integrations/webhooks/utils.py index a889a28..336e014 100644 --- a/jvspatial/api/integrations/webhooks/utils.py +++ b/jvspatial/api/integrations/webhooks/utils.py @@ -172,8 +172,13 @@ def verify_hmac_signature( payload, secret, algorithm, "" ) # No prefix for comparison - # Use constant-time comparison to prevent timing attacks - return hmac.compare_digest(signature, expected_signature[len(prefix) :]) + # ``signature`` is the incoming digest after prefix stripping above, + # and ``expected_signature`` is generated with ``prefix=""`` so it is + # already the bare hex digest. The earlier ``[len(prefix):]`` slice + # truncated 7 chars off a 64-char SHA-256 digest, making + # ``compare_digest`` always return False — webhook HMAC verification + # was effectively broken (audit §4.3 / SPEC §15.2). + return hmac.compare_digest(signature, expected_signature) except Exception: return False diff --git a/jvspatial/api/integrations/webhooks/webhook_auth.py b/jvspatial/api/integrations/webhooks/webhook_auth.py index dacb59d..2f53da1 100644 --- a/jvspatial/api/integrations/webhooks/webhook_auth.py +++ b/jvspatial/api/integrations/webhooks/webhook_auth.py @@ -20,16 +20,48 @@ _API_KEY_CACHE_TTL = 300.0 _API_KEY_CACHE_MAX_SIZE = 500 _API_KEY_VALIDATE_TIMEOUT = 10.0 +# Single lock guards every read/write/eviction of ``_API_KEY_CACHE`` so +# concurrent webhook requests cannot trip a ``KeyError`` when the size-cap +# eviction races a cache read (audit §4.7). +_API_KEY_CACHE_LOCK = asyncio.Lock() -def _api_key_cache_cleanup() -> None: - """Remove expired entries from API key cache.""" +def _api_key_cache_cleanup_locked() -> None: + """Remove expired entries from API key cache. Caller holds the lock.""" now = time.time() expired = [k for k, (exp, _) in _API_KEY_CACHE.items() if exp <= now] for k in expired: del _API_KEY_CACHE[k] +async def invalidate_api_key_cache(api_key: Optional[str] = None) -> None: + """Drop a single key (or all keys) from the webhook API-key cache. + + ``APIKeyService.revoke_key`` calls this so revoking a key is reflected + immediately, not after the up-to-5-minute TTL (audit §4.5). + """ + async with _API_KEY_CACHE_LOCK: + if api_key is None: + _API_KEY_CACHE.clear() + return + cache_key = hashlib.sha256(api_key.encode()).hexdigest() + _API_KEY_CACHE.pop(cache_key, None) + + +def invalidate_api_key_cache_hash(cache_key: str) -> None: + """Drop a single hashed cache key (sync variant). + + Callers that already hold the cache hash (e.g. ``APIKeyService`` + working with the persisted hash, not the plaintext key) can invoke + this from sync code paths such as ``APIKeyService.revoke_key``. + + The unlocked ``pop`` accepts a one-shot race where a concurrent + reader sees the stale entry once — preferable to failing in callers + that have no bound event loop. + """ + _API_KEY_CACHE.pop(cache_key, None) + + async def authenticate_webhook_api_key( request: Request, auth_mode: str, @@ -144,17 +176,24 @@ async def authenticate_webhook_api_key( cache_key = hashlib.sha256(api_key.encode()).hexdigest() now = time.time() cache_hit = False - if cache_key in _API_KEY_CACHE: - cached_expiry, cached_entity = _API_KEY_CACHE[cache_key] - if cached_expiry > now and cached_entity is not None: - api_key_entity = cached_entity - cache_hit = True - logger.debug(f"Webhook API key cache hit: key_id={cached_entity.id}") + # Single lock around read + eviction + miss-population so a + # concurrent size-cap cleanup cannot ``KeyError`` a reader and + # the size cap is not racy (audit §4.7). + async with _API_KEY_CACHE_LOCK: + cached = _API_KEY_CACHE.get(cache_key) + if cached is not None: + cached_expiry, cached_entity = cached + if cached_expiry > now and cached_entity is not None: + api_key_entity = cached_entity + cache_hit = True + logger.debug( + f"Webhook API key cache hit: key_id={cached_entity.id}" + ) + else: + api_key_entity = None + _API_KEY_CACHE.pop(cache_key, None) else: api_key_entity = None - del _API_KEY_CACHE[cache_key] - else: - api_key_entity = None prime_ctx = None service = None @@ -175,12 +214,13 @@ async def authenticate_webhook_api_key( detail="Service temporarily unavailable", ) if api_key_entity: - if len(_API_KEY_CACHE) >= _API_KEY_CACHE_MAX_SIZE: - _api_key_cache_cleanup() - _API_KEY_CACHE[cache_key] = ( - now + _API_KEY_CACHE_TTL, - api_key_entity, - ) + async with _API_KEY_CACHE_LOCK: + if len(_API_KEY_CACHE) >= _API_KEY_CACHE_MAX_SIZE: + _api_key_cache_cleanup_locked() + _API_KEY_CACHE[cache_key] = ( + now + _API_KEY_CACHE_TTL, + api_key_entity, + ) if not api_key_entity: raise HTTPException(status_code=401, detail="Invalid or expired API key") diff --git a/jvspatial/api/middleware/manager.py b/jvspatial/api/middleware/manager.py index 8b0e10c..3c5826b 100644 --- a/jvspatial/api/middleware/manager.py +++ b/jvspatial/api/middleware/manager.py @@ -8,7 +8,7 @@ """ import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, List +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -20,22 +20,89 @@ from jvspatial.api.server import Server +# Strict default — applied to application routes. +_DEFAULT_CSP = "default-src 'self'; frame-ancestors 'none'" + +# Relaxed CSP for FastAPI's bundled Swagger UI / ReDoc pages. Those pages +# pull swagger-ui-dist + redoc bundles from cdn.jsdelivr.net and a favicon +# from fastapi.tiangolo.com, plus run a small inline bootstrap script. The +# strict default blocks all of that and the docs render blank. Scoped to +# documentation paths only — application routes keep the strict policy. +# +# The single production knob is `JVSPATIAL_DOCS_DISABLED=1` (read at app +# build time in `AppBuilder.create_app`); that flag unpublishes /docs, +# /redoc, and /openapi.json entirely so this CSP never matters in prod. +# When docs ARE published (dev / staging) this relaxation makes Swagger +# UI render — no further env knobs needed. +_DOCS_CSP = ( + "default-src 'self' https://cdn.jsdelivr.net; " + "base-uri 'self'; " + "frame-ancestors 'none'; " + "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " + "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " + "img-src 'self' data: blob: https://cdn.jsdelivr.net https://fastapi.tiangolo.com; " + "font-src 'self' data: https://cdn.jsdelivr.net; " + "connect-src 'self' https://cdn.jsdelivr.net; " + "worker-src 'self' blob:; " + "object-src 'none'" +) + +# Default documentation-page path prefixes. Both ``/docs`` and ``/redoc`` +# may carry trailing path segments (Swagger's oauth2-redirect, ReDoc +# assets); match exact path or ``/...``. ``/openapi.json`` is +# the JSON spec consumed by both UIs. +# +# These are FastAPI's *defaults*. When ``ServerConfig`` overrides +# ``docs_url`` / ``redoc_url`` / ``openapi_url``, the middleware reads +# the custom prefixes from the server config at construction time +# (audit §7.13). +_DEFAULT_DOCS_PATH_PREFIXES = ("/docs", "/redoc", "/openapi.json") + + +def _is_docs_path( + path: str, prefixes: Tuple[str, ...] = _DEFAULT_DOCS_PATH_PREFIXES +) -> bool: + """True for FastAPI Swagger / ReDoc / OpenAPI surfaces.""" + return any(path == p or path.startswith(p + "/") for p in prefixes) + + class SecurityHeadersMiddleware(BaseHTTPMiddleware): """Middleware that adds security headers to all responses. Headers applied: - X-Content-Type-Options: nosniff (MIME sniffing prevention) - X-Frame-Options: DENY (clickjacking prevention) - - Content-Security-Policy: default-src 'self'; frame-ancestors 'none' + - Content-Security-Policy: strict on app routes; a relaxed variant + that permits ``cdn.jsdelivr.net`` is emitted for ``/docs``, + ``/redoc``, and ``/openapi.json`` so FastAPI's bundled Swagger UI + and ReDoc pages render. Without this scoped relaxation the docs + load blank because the strict default blocks the CDN-hosted JS/CSS. + For production lockdown set ``JVSPATIAL_DOCS_DISABLED=1`` — + ``AppBuilder.create_app`` then unpublishes the docs surface entirely + and this CSP never applies (no routes registered). - Strict-Transport-Security: max-age=31536000; includeSubDomains (if enabled) HSTS is only applied when the server configures hsts_enabled=True (off by default in development, on in production). """ - def __init__(self, app, hsts_enabled: bool = False): + def __init__( + self, + app, + hsts_enabled: bool = False, + docs_path_prefixes: Optional[Tuple[str, ...]] = None, + ): super().__init__(app) self._hsts_enabled = hsts_enabled + # Derived from ``ServerConfig`` (``docs_url`` / ``redoc_url`` / + # ``openapi_url``) at install time so customizing the docs URL + # does not silently break Swagger UI with strict CSP + # (audit §7.13). + self._docs_path_prefixes: Tuple[str, ...] = ( + docs_path_prefixes + if docs_path_prefixes is not None + else _DEFAULT_DOCS_PATH_PREFIXES + ) async def dispatch(self, request, call_next): """Process request and add security headers to response.""" @@ -43,7 +110,9 @@ async def dispatch(self, request, call_next): response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["Content-Security-Policy"] = ( - "default-src 'self'; frame-ancestors 'none'" + _DOCS_CSP + if _is_docs_path(request.url.path, self._docs_path_prefixes) + else _DEFAULT_CSP ) if self._hsts_enabled: response.headers["Strict-Transport-Security"] = ( @@ -127,7 +196,25 @@ def _configure_security_headers(self, app: FastAPI) -> None: return hsts_enabled = getattr(self.server.config.security, "hsts_enabled", False) - app.add_middleware(SecurityHeadersMiddleware, hsts_enabled=hsts_enabled) + # Derive docs-path prefixes from ServerConfig so customized + # ``docs_url`` / ``redoc_url`` / ``openapi_url`` keep Swagger UI + # rendering under the relaxed CSP (audit §7.13). + cfg = self.server.config + configured_prefixes = tuple( + p + for p in ( + getattr(cfg, "docs_url", None), + getattr(cfg, "redoc_url", None), + getattr(cfg, "openapi_url", None), + ) + if p + ) + docs_path_prefixes = configured_prefixes or _DEFAULT_DOCS_PATH_PREFIXES + app.add_middleware( + SecurityHeadersMiddleware, + hsts_enabled=hsts_enabled, + docs_path_prefixes=docs_path_prefixes, + ) self._logger.debug(f"{LogIcons.SUCCESS} Security headers middleware configured") def _configure_cors(self, app: FastAPI) -> None: diff --git a/jvspatial/cache/README.md b/jvspatial/cache/README.md new file mode 100644 index 0000000..98461c4 --- /dev/null +++ b/jvspatial/cache/README.md @@ -0,0 +1,56 @@ +# jvspatial/cache + +Cache backends: memory, Redis, layered. Plus a factory and the abstract base. + +> **Read first**: [SPEC §13](../../SPEC.md), [docs/md/caching.md](../../docs/md/caching.md) + +--- + +## Purpose + +`cache/` provides pluggable cache backends. The most common use is the read-through DB wrapper enabled via `create_database(cache_get_size=N)` — this package exposes the underlying backends and a factory so callers can build their own cache strategies. + +## Layout + +``` +cache/ +├── base.py # CacheBackend ABC +├── factory.py # create_cache, create_default_cache +├── memory.py # MemoryCache (LRU) +├── redis.py # RedisCache (optional) +└── layered.py # LayeredCache (memory L1 + redis L2) +``` + +## Public API (from `jvspatial.cache`) + +| Name | What it does | +|---|---| +| `CacheBackend` | ABC for cache backends | +| `create_cache(backend, **kwargs)` | Factory entry point | +| `create_default_cache()` | Env-driven default | +| `MemoryCache` | Per-process LRU | +| `RedisCache` | Shared cross-process cache (requires `redis-py`) | +| `LayeredCache` | Memory L1 + Redis L2, promotes hot keys to L1 | + +## Invariants + +- **Memory cache is per-process.** Cross-worker invalidation requires Redis or layered backend. +- **TTL is per-cache-instance, default `None`.** Cache until eviction. +- **DB-wrapped caches invalidate on `save` / `delete`.** No stale read after a write through the wrapped database. (`jvspatial/db/_cache.py`, internal) +- **`bulk_save` and `find_one_and_update` refresh the cache** for affected keys; other writes invalidate. +- **Cache is skipped in serverless mode** when wrapped via `create_database(cache_get_size=N)`. Cold-start memory does not survive invocations anyway. + +## Modification patterns + +- **Adding a backend**: implement `CacheBackend`. Register in `create_cache` factory. Decide eviction semantics up front. +- **Changing invalidation rules**: edit `jvspatial/db/_cache.py` (internal). Add tests that cover save / delete / bulk_save / find_one_and_update. +- **Tuning the read-through wrapper**: pass `cache_get_size=`, `cache_get_ttl=` to `create_database`. Do not import `CachingDatabase` directly. + +## Related docs + +- [docs/md/caching.md](../../docs/md/caching.md) +- [docs/md/optimization.md](../../docs/md/optimization.md) + +## Stability + +`CacheBackend`, `create_cache`, `MemoryCache` are stable. `RedisCache` and `LayeredCache` are stable when their optional dependency is installed. Internal wrappers (`jvspatial/db/_cache.py`) are not part of the public surface. diff --git a/jvspatial/core/README.md b/jvspatial/core/README.md new file mode 100644 index 0000000..b7818bf --- /dev/null +++ b/jvspatial/core/README.md @@ -0,0 +1,71 @@ +# jvspatial/core + +Entity hierarchy, graph context, traversal primitives, and decorators. + +> **Read first**: [SPEC §1-7](../../SPEC.md), [docs/md/entity-reference.md](../../docs/md/entity-reference.md) + +--- + +## Purpose + +`core/` defines the graph data model. Everything jvspatial persists is one of the entity types here: `Object`, `Node`, `Edge`, `Walker`, or `Root`. Traversal is performed by walkers; database I/O is brokered through `GraphContext`. + +## Layout + +``` +core/ +├── entities/ # Object, Node, Edge, Walker, Root + walker_components +├── annotations/ # @attribute system + index helpers +├── decorators/ # @on_visit, @on_exit +├── mixins/ # DeferredSaveMixin and globals +├── walker_components/ # Trail, protection, queue, event system (under entities/) +├── context.py # GraphContext + scoping helpers +├── events.py # Global event bus + @on_emit +├── graph.py # DOT / Mermaid export +├── graph_expansion.py # BFS / subgraph utilities +├── pager.py # ObjectPager +└── utils.py # generate_id, find_subclass_by_name, datetime helpers +``` + +## Public API (from `jvspatial.core`) + +| Name | What it does | +|---|---| +| `Object`, `Node`, `Edge`, `Walker`, `Root` | Entity classes (SPEC §2) | +| `NodeQuery` | Typed node-query helper | +| `GraphContext` | Database + cache + monitor binding (SPEC §7) | +| `get_default_context` / `set_default_context` / `scoped_default_context` | Context lifecycle | +| `graph_context` / `async_graph_context` | Sync / async context managers | +| `@on_visit`, `@on_exit`, `@on_emit` | Hook decorators | +| `DeferredSaveMixin`, `deferred_saves_globally_allowed`, `flush_deferred_entities` | Save-batching opt-in | +| `ObjectPager`, `paginate_objects`, `paginate_by_field` | Pagination | +| `generate_id`, `find_subclass_by_name`, `serialize_datetime` | Utilities | +| `export_graph`, `generate_graph_dot`, `generate_graph_mermaid`, `expand_node`, `subgraph_bfs` | Visualization / expansion | + +## Invariants + +- **`__entity_name__` is per-subclass.** Resolution: `cls.__dict__.get("__entity_name__") or cls.__name__`. Not inherited. (`entities/object.py:35-44`) +- **`id` is protected.** Set in `__init__`, cannot be reassigned. (`entities/object.py:46-48`) +- **Walker protection is on by default.** `max_steps=10000`, `max_visits_per_node=100`, `max_execution_time=300s`, `max_queue_size=1000`. Disabling globally is forbidden. (`entities/walker.py:106-115`) +- **Subclass lookup honors entity-name override and caches positive hits only.** Negative caching would break later imports. (`utils.py:58-89`) +- **Root is a singleton with fixed ID `n.Root.root`.** Created under async lock. (`entities/root.py`) +- **MRO matters for mixins.** `DeferredSaveMixin` must precede the base class. (`mixins/`) + +## Modification patterns + +- Adding a new entity field: declare with `@attribute(...)`. Top-level persisted fields are added via `_get_top_level_fields()`. Update tests under `tests/core/`. +- Adding a new walker hook: decorate with `@on_visit(NodeType | "string_name")`. Hooks are registered at `__init_subclass__` time. +- Adding a new graph utility: prefer `core/graph.py` (export) or `core/graph_expansion.py` (traversal). Both have sibling tests. + +## Related docs + +- [docs/md/entity-reference.md](../../docs/md/entity-reference.md) +- [docs/md/graph-traversal.md](../../docs/md/graph-traversal.md) +- [docs/md/graph-context.md](../../docs/md/graph-context.md) +- [docs/md/walker-trail-tracking.md](../../docs/md/walker-trail-tracking.md) +- [docs/md/infinite-walk-protection.md](../../docs/md/infinite-walk-protection.md) +- [docs/md/attribute-annotations.md](../../docs/md/attribute-annotations.md) + +## Stability + +All names listed in "Public API" are part of `jvspatial.__all__` and follow the stable-tier contract (see [docs/md/stability.md](../../docs/md/stability.md)). The contents of `walker_components/`, `mixins/_internal/`, and any underscore-prefixed module are internal — do not import directly. diff --git a/jvspatial/core/context.py b/jvspatial/core/context.py index c4ead7d..56e8dc3 100644 --- a/jvspatial/core/context.py +++ b/jvspatial/core/context.py @@ -1,8 +1,10 @@ """GraphContext for managing database dependencies.""" import asyncio +import base64 import contextvars import inspect +import json import logging import time from contextlib import asynccontextmanager, contextmanager, suppress @@ -14,8 +16,10 @@ List, Optional, Set, + Tuple, Type, TypeVar, + Union, cast, ) @@ -741,23 +745,30 @@ async def save( else: # For Objects, export returns nested format (id, entity, context) record = await entity.export() - # Ensure ID follows proper format: type_code.ClassName.hex_id - # Check entity's ID directly (it's transient so not in export) + # Ensure ID follows proper format: type_code.entity_name.hex_id + # entity_name honors ``__entity_name__`` override (SPEC §1.2) + # so classes that disambiguate from a same-``__name__`` peer + # round-trip through save/load without ID rewriting. entity_id = getattr(entity, "id", None) if entity_id: id_parts = entity_id.split(".") - # Check if ID matches expected format: type_code.ClassName.hex_id + entity_name_resolver = getattr( + entity.__class__, "_entity_name", None + ) + expected_entity_name = ( + entity_name_resolver() + if callable(entity_name_resolver) + else entity.__class__.__name__ + ) if ( len(id_parts) != 3 or id_parts[0] != entity.type_code - or id_parts[1] != entity.__class__.__name__ + or id_parts[1] != expected_entity_name ): # ID doesn't match expected format, regenerate it from jvspatial.core.utils import generate_id - new_id = generate_id( - entity.type_code, entity.__class__.__name__ - ) + new_id = generate_id(entity.type_code, expected_entity_name) object.__setattr__(entity, "id", new_id) record["id"] = new_id # ID is in record from export() @@ -899,7 +910,7 @@ async def find( return await self.find_nodes(entity_class, query, limit=limit) collection = self._get_collection_name(entity_type_code) - db_query = {"entity": entity_class.__name__, **query} + db_query = {"entity": entity_class._entity_name(), **query} results = await self.database.find(collection, db_query, limit=limit) entities = [] @@ -1263,7 +1274,7 @@ async def find_nodes( collection = self._get_collection_name(self._get_entity_type_code(node_class)) # Add class name filter to query for type safety - db_query = {"entity": node_class.__name__, **query} + db_query = {"entity": node_class._entity_name(), **query} db = self.database results = await db.find(collection, db_query) @@ -1282,6 +1293,196 @@ async def find_nodes( return nodes + @staticmethod + def _resolve_entity_name(value: Any) -> Optional[str]: + """Normalize filter tokens (class/string) to persisted entity names.""" + if isinstance(value, str): + return value + if isinstance(value, type): + resolver = getattr(value, "_entity_name", None) + return resolver() if callable(resolver) else value.__name__ + return None + + async def find_page( + self, + collection: str, + query: Dict[str, Any], + sort: List[Tuple[str, int]], + after: Optional[Union[str, Dict[str, Any]]] = None, + limit: int = 50, + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """Find one keyset-paginated page and return ``(items, next_cursor)``. + + ``sort`` is expected to include at least one field; ``id`` is appended as + a deterministic tiebreaker when missing. + """ + page_limit = max(1, int(limit)) + if not sort: + sort = [("id", 1)] + sort_fields: List[Tuple[str, int]] = list(sort) + if not any(field == "id" for field, _ in sort_fields): + sort_fields.append(("id", sort_fields[0][1])) + + final_query: Dict[str, Any] = dict(query or {}) + cursor_payload: Optional[Dict[str, Any]] = None + if isinstance(after, str) and after: + try: + cursor_payload = json.loads( + base64.urlsafe_b64decode(after.encode()).decode() + ) + except Exception: + cursor_payload = None + elif isinstance(after, dict): + cursor_payload = after + + primary_field, primary_dir = sort_fields[0] + id_dir = sort_fields[-1][1] + if cursor_payload and "id" in cursor_payload and "sort" in cursor_payload: + sort_op = "$lt" if primary_dir < 0 else "$gt" + id_op = "$lt" if id_dir < 0 else "$gt" + keyset_filter = { + "$or": [ + {primary_field: {sort_op: cursor_payload["sort"]}}, + { + "$and": [ + {primary_field: cursor_payload["sort"]}, + {"id": {id_op: cursor_payload["id"]}}, + ] + }, + ] + } + final_query = ( + {"$and": [final_query, keyset_filter]} if final_query else keyset_filter + ) + + rows = await self.database.find( + collection, final_query, limit=page_limit + 1, sort=sort_fields + ) + has_more = len(rows) > page_limit + page_rows = rows[:page_limit] + + next_cursor: Optional[str] = None + if has_more and page_rows: + last = page_rows[-1] + payload = {"id": last.get("id"), "sort": last.get(primary_field)} + next_cursor = base64.urlsafe_b64encode( + json.dumps(payload, separators=(",", ":")).encode() + ).decode() + + return page_rows, next_cursor + + async def nodes_bulk( + self, + node_ids: List[str], + *, + direction: str = "out", + edge: Optional[List[Union[str, Type[Any]]]] = None, + node: Optional[List[Union[str, Type[Any]]]] = None, + edge_filter: Optional[Dict[str, Any]] = None, + node_filter: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + ) -> Dict[str, List[Any]]: + """Batch traversal for many source IDs in one edge-query pass.""" + from .entities.edge import Edge + from .entities.node import Node + + unique_ids = [nid for nid in dict.fromkeys(node_ids) if nid] + out: Dict[str, List[Any]] = {nid: [] for nid in unique_ids} + if not unique_ids: + return out + + edge_entities = [ + en + for en in (self._resolve_entity_name(e) for e in (edge or [])) + if en is not None + ] + node_entities = [ + en + for en in (self._resolve_entity_name(n) for n in (node or [])) + if en is not None + ] + + if direction not in ("out", "in", "both"): + direction = "out" + + edge_query: Dict[str, Any] = {} + if direction == "out": + edge_query["source"] = {"$in": unique_ids} + elif direction == "in": + edge_query["target"] = {"$in": unique_ids} + else: + edge_query["$or"] = [ + {"source": {"$in": unique_ids}}, + {"target": {"$in": unique_ids}}, + ] + + if edge_entities: + edge_query["entity"] = {"$in": edge_entities} + if edge_filter: + for k, v in edge_filter.items(): + edge_query[f"context.{k}"] = v + + edge_docs = await self.database.find( + self._get_collection_name(self._get_entity_type_code(Edge)), + edge_query, + limit=limit, + ) + + neighbor_ids: set = set() + links: Dict[str, List[str]] = {nid: [] for nid in unique_ids} + seen_pairs: set = set() + + for edge_doc in edge_docs: + src = edge_doc.get("source") + dst = edge_doc.get("target") + if direction in ("out", "both") and src in links and dst: + pair = (src, dst) + if pair not in seen_pairs: + links[src].append(dst) + seen_pairs.add(pair) + neighbor_ids.add(dst) + if direction in ("in", "both") and dst in links and src: + pair = (dst, src) + if pair not in seen_pairs: + links[dst].append(src) + seen_pairs.add(pair) + neighbor_ids.add(src) + + if not neighbor_ids: + return out + + node_docs = await self.database.find( + self._get_collection_name(self._get_entity_type_code(Node)), + {"id": {"$in": list(neighbor_ids)}}, + ) + node_by_id: Dict[str, Any] = {} + for doc in node_docs: + if node_entities and doc.get("entity") not in node_entities: + continue + if node_filter: + fail = False + for k, v in node_filter.items(): + if doc.get("context", {}).get(k) != v: + fail = True + break + if fail: + continue + obj = await self._deserialize_entity(Node, doc) + if obj is not None: + node_by_id[doc["id"]] = obj + + for src_id, target_ids in links.items(): + matched: List[Any] = [] + for tid in target_ids: + obj = node_by_id.get(tid) + if obj is None: + continue + matched.append(obj) + if limit is not None and len(matched) >= max(1, limit): + break + out[src_id] = matched + return out + async def ensure_indexes(self, entity_class: Type[T]) -> None: """Ensure indexes are created for the given entity class. @@ -1319,7 +1520,7 @@ async def ensure_indexes(self, entity_class: Type[T]) -> None: collection = self._get_collection_name(type_code) # Check if we've already ensured indexes for this collection - collection_key = f"{collection}:{entity_class.__name__}" + collection_key = f"{collection}:{entity_class._entity_name()}" if collection_key in _ensured_indexes: return # Already ensured @@ -1398,9 +1599,16 @@ async def find_edges_between( if target_id is not None: query["target"] = target_id - # Filter by edge class if specified + # Filter by edge class if specified — honor ``__entity_name__`` + # override so subclasses persisted under a custom discriminator are + # found (SPEC §1.2). if edge_class: - query["entity"] = edge_class.__name__ + entity_name_resolver = getattr(edge_class, "_entity_name", None) + query["entity"] = ( + entity_name_resolver() + if callable(entity_name_resolver) + else edge_class.__name__ + ) # Add additional property filters for key, value in kwargs.items(): @@ -1438,7 +1646,7 @@ async def _deserialize_entity( from .utils import find_subclass_by_name # Use entity field for class identification - stored_entity = data.get("entity", entity_class.__name__) + stored_entity = data.get("entity", entity_class._entity_name()) entity_type_code = self._get_entity_type_code(entity_class) # Prefer requested class subtree, then (for Nodes) scan the entire Node hierarchy. diff --git a/jvspatial/core/entities/__init__.py b/jvspatial/core/entities/__init__.py index 1488424..6bb8f3d 100644 --- a/jvspatial/core/entities/__init__.py +++ b/jvspatial/core/entities/__init__.py @@ -7,6 +7,8 @@ simplified decorator support and other improvements. """ +from jvspatial.exceptions import JVSpatialError + from .edge import Edge from .node import Node @@ -22,6 +24,27 @@ from .walker_components.walker_queue import WalkerQueue from .walker_components.walker_trail import WalkerTrail + +class TraversalSkipped(JVSpatialError): + """Walker-skip signal raised by ``Walker.skip()``. + + Abandons processing of the current node and continues with the + next queued node. Replaces the historical + ``raise JVSpatialError("Node skipped")`` pattern that relied on + substring-matching the error message — a fragile contract that any + unrelated exception containing the phrase would silently trigger + (audit §2.9 / SPEC §6.5). + """ + + +class TraversalPaused(JVSpatialError): + """Walker-pause signal raised by ``Walker.pause()``. + + Callers catch this exception to resume later via + ``Walker.resume()``. + """ + + __all__ = [ # Enhanced entity classes (maintaining original hierarchy) "Object", @@ -35,4 +58,7 @@ "WalkerQueue", "WalkerTrail", "WalkerEventSystem", + # Walker control-flow exceptions + "TraversalSkipped", + "TraversalPaused", ] diff --git a/jvspatial/core/entities/edge.py b/jvspatial/core/entities/edge.py index 2f8d391..71299d0 100644 --- a/jvspatial/core/entities/edge.py +++ b/jvspatial/core/entities/edge.py @@ -63,8 +63,13 @@ async def direction(self: "Edge") -> str: """ return "both" if self.bidirectional else "out" - def __init_subclass__(cls: Type["Edge"]) -> None: - """Initialize subclass by registering visit hooks.""" + def __init_subclass__(cls: Type["Edge"], **kwargs: Any) -> None: + """Initialize subclass by registering visit hooks. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs (audit §6.2). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} cls._is_visit_hook = {} @@ -165,7 +170,7 @@ def __init__( # Don't override ID if already provided if "id" not in kwargs: - kwargs["id"] = generate_id("e", self.__class__.__name__) + kwargs["id"] = generate_id("e", self.__class__._entity_name()) kwargs.update( {"source": source, "target": target, "bidirectional": bidirectional} @@ -235,7 +240,19 @@ async def export( @classmethod def get_indexes(cls: Type["Edge"]) -> List[Dict[str, Any]]: - """Get index definitions including unique compound index on (source, target, entity).""" + """Default indexes for every edge collection. + + ``idx_source_target_entity_unique`` enforces edge uniqueness and serves + outgoing traversal via leftmost-prefix on ``source``. The remaining + indexes cover traversal shapes the unique index cannot: + + - ``idx_target_entity`` — incoming traversal (``direction="in"``). + - ``idx_entity_source`` / ``idx_entity_target`` — typed-edge sweeps + that filter by ``entity`` without pinning a node. + + All four are domain-agnostic — they only reference fields every Edge + document carries (``entity``, ``source``, ``target``). + """ indexes = super().get_indexes() indexes.append( { @@ -244,6 +261,27 @@ def get_indexes(cls: Type["Edge"]) -> List[Dict[str, Any]]: "name": "idx_source_target_entity_unique", } ) + indexes.append( + { + "fields": [("target", 1), ("entity", 1)], + "unique": False, + "name": "idx_target_entity", + } + ) + indexes.append( + { + "fields": [("entity", 1), ("source", 1)], + "unique": False, + "name": "idx_entity_source", + } + ) + indexes.append( + { + "fields": [("entity", 1), ("target", 1)], + "unique": False, + "name": "idx_entity_target", + } + ) return indexes @classmethod @@ -321,7 +359,7 @@ async def all(cls: Type["Edge"]) -> List["Object"]: bidirectional = data["context"].get("bidirectional", True) # Handle subclass instantiation based on stored entity - stored_entity = data.get("entity", cls.__name__) + stored_entity = data.get("entity", cls._entity_name()) target_class = find_subclass_by_name(cls, stored_entity) or cls context_data = { diff --git a/jvspatial/core/entities/node.py b/jvspatial/core/entities/node.py index cad6351..6e59b0d 100644 --- a/jvspatial/core/entities/node.py +++ b/jvspatial/core/entities/node.py @@ -59,8 +59,37 @@ def _get_top_level_fields(cls: Type["Node"]) -> set: "id", } # edge_ids is stored as "edges" at top level; id is top-level on node docs - def __init_subclass__(cls: Type["Node"]) -> None: - """Initialize subclass by registering visit hooks.""" + @classmethod + def get_indexes(cls: Type["Node"]) -> List[Dict[str, Any]]: + """Default node indexes. + + Adds an index on the top-level ``entity`` discriminator. Every typed + find (``ClassName.find(...)`` / ``GraphContext.find_by_class(...)``) + is rewritten to ``{"entity": "", ...}``, so this index is + the universal first cut for any multi-type node collection. Generic + — it only touches fields jvspatial itself attaches to every node + document. + """ + indexes = super().get_indexes() + indexes.append( + { + "field": "entity", + "unique": False, + "direction": 1, + "name": "idx_node_entity", + } + ) + return indexes + + def __init_subclass__(cls: Type["Node"], **kwargs: Any) -> None: + """Initialize subclass by registering visit hooks. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs and registers + ``protected`` / ``transient`` / ``private`` attribute metadata + for Node subclasses (audit §6.1). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} for _name, method in inspect.getmembers(cls, inspect.isfunction): @@ -383,6 +412,36 @@ async def nodes( **kwargs, ) + @classmethod + async def nodes_bulk( + cls, + node_ids: List[str], + *, + direction: str = "out", + edge: Optional[List[Union[str, Type["Edge"]]]] = None, + node: Optional[List[Union[str, Type["Node"]]]] = None, + edge_filter: Optional[Dict[str, Any]] = None, + node_filter: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + ) -> Dict[str, List["Node"]]: + """Batch version of ``nodes()`` for many source IDs. + + Returns a mapping of ``source_id -> connected nodes`` using a single + edge-query pass in the active GraphContext. + """ + from ..context import get_default_context + + context = get_default_context() + return await context.nodes_bulk( + node_ids, + direction=direction, + edge=edge, + node=node, + edge_filter=edge_filter, + node_filter=node_filter, + limit=limit, + ) + async def count_neighbors( self, direction: str = "out", @@ -423,7 +482,10 @@ async def count_neighbors( if isinstance(node, str): entity_name = node elif isinstance(node, type): - entity_name = node.__name__ + # Honor ``__entity_name__`` so subclasses with custom + # discriminators match their persisted ID prefix. + resolver = getattr(node, "_entity_name", None) + entity_name = resolver() if callable(resolver) else node.__name__ if entity_name is not None: try: from ..context import get_default_context @@ -558,7 +620,12 @@ async def _node_query( "$or": [{"source": self.id}, {"target": self.id}] } if isinstance(edge_filter, type): - edge_query["name"] = edge_filter.__name__ + # Persisted discriminator field is ``entity``; honor + # ``__entity_name__`` override (SPEC §1.2). + resolver = getattr(edge_filter, "_entity_name", None) + edge_query["entity"] = ( + resolver() if callable(resolver) else edge_filter.__name__ + ) # Cap edge fan-out so hub nodes don't accidentally load unbounded sets. edge_results = await context.database.find( @@ -590,7 +657,10 @@ async def _node_query( # Find incoming edges (where this node is the target) query: Dict[str, Any] = {"target": self.id} if isinstance(edge_filter, type): - query["name"] = edge_filter.__name__ + resolver = getattr(edge_filter, "_entity_name", None) + query["entity"] = ( + resolver() if callable(resolver) else edge_filter.__name__ + ) edge_results = await context.database.find("edge", query) for edge_data in edge_results: @@ -663,9 +733,17 @@ def _matches_node_filter( Returns: True if node matches the filter """ + # ``_entity_name()`` honors the ``__entity_name__`` override + # (SPEC §1.2). Falls back to ``__name__`` if absent so this helper + # works on non-Object types passed by mistake. + resolver = getattr(node_obj.__class__, "_entity_name", None) + obj_entity_name = ( + resolver() if callable(resolver) else node_obj.__class__.__name__ + ) + if isinstance(node_filter, str): - # Simple string filter - match by class name - return node_obj.__class__.__name__ == node_filter + # Simple string filter - match by entity_name + return obj_entity_name == node_filter elif isinstance(node_filter, type): # Class type filter - match by class or inheritance @@ -674,18 +752,18 @@ def _matches_node_filter( elif isinstance(node_filter, list): for filter_item in node_filter: if isinstance(filter_item, str): - # String in list - match by class name - if node_obj.__class__.__name__ == filter_item: + # String in list - match by entity_name + if obj_entity_name == filter_item: return True elif isinstance(filter_item, type): # Class type in list - match by class or inheritance if isinstance(node_obj, filter_item): return True elif isinstance(filter_item, dict): - # Dict filter - match by class name and criteria + # Dict filter - match by entity_name and criteria for class_name, criteria in filter_item.items(): if ( - node_obj.__class__.__name__ == class_name + obj_entity_name == class_name and self._matches_property_filter(node_obj, criteria) ): return True diff --git a/jvspatial/core/entities/object.py b/jvspatial/core/entities/object.py index 72af6d5..0a89b74 100644 --- a/jvspatial/core/entities/object.py +++ b/jvspatial/core/entities/object.py @@ -1,7 +1,7 @@ """Base Object class for jvspatial entities.""" import inspect -from typing import Any, Dict, List, Optional, Set, Type, Union +from typing import Any, ClassVar, Dict, List, Optional, Set, Type, Union from pydantic import BaseModel, ConfigDict @@ -28,6 +28,21 @@ class Object(AttributeMixin, BaseModel): model_config = ConfigDict(extra="ignore") + # Per-class override of the persisted ``entity`` discriminator. Defaults to + # ``cls.__name__``. Set this on a subclass when two unrelated ``Object`` + # subtrees share a Python class name (e.g. host-app ``App`` vs library + # ``App``) and must remain distinguishable at the storage layer. + __entity_name__: ClassVar[Optional[str]] = None + + @classmethod + def _entity_name(cls) -> str: + """Return the persisted entity discriminator for this class. + + Honors ``__entity_name__`` when set on the class (not inherited from a + parent that happens to set it), otherwise falls back to ``cls.__name__``. + """ + return cls.__dict__.get("__entity_name__") or cls.__name__ + id: str = attribute( protected=True, transient=True, description="Unique identifier for the object" ) @@ -82,10 +97,10 @@ def __init__(self: "Object", **kwargs: Any) -> None: type_code = type_code_field.default else: type_code = "o" # Default type code - kwargs["id"] = generate_id(type_code, self.__class__.__name__) + kwargs["id"] = generate_id(type_code, self.__class__._entity_name()) # Set entity to class name if not provided (protected attribute) if "entity" not in kwargs: - kwargs["entity"] = self.__class__.__name__ + kwargs["entity"] = self.__class__._entity_name() # Call super().__init__() first to initialize Pydantic model (including __pydantic_private__) # The _initializing attribute defaults to True, so it's already set during Pydantic initialization @@ -592,7 +607,7 @@ def _collect_class_names(cls: Type["Object"]) -> Set[str]: Returns: Set of class names including the class itself and all imported subclasses """ - names: Set[str] = {cls.__name__} + names: Set[str] = {cls._entity_name()} # Add all imported subclasses recursively for subclass in cls.__subclasses__(): diff --git a/jvspatial/core/entities/walker.py b/jvspatial/core/entities/walker.py index ffe3584..219beab 100644 --- a/jvspatial/core/entities/walker.py +++ b/jvspatial/core/entities/walker.py @@ -21,8 +21,6 @@ from .node import Node from .edge import Edge -from jvspatial.exceptions import JVSpatialError - from ..annotations import AttributeMixin, attribute from ..events import event_bus from ..utils import generate_id @@ -115,6 +113,21 @@ class Walker(AttributeMixin, BaseModel): """ model_config = ConfigDict(extra="allow") + + # Per-class override of the persisted ``entity`` discriminator. Mirrors + # ``Object.__entity_name__`` so Walker subclasses can disambiguate from + # same-``__name__`` peers without subclassing Object. + __entity_name__: ClassVar[Optional[str]] = None + + @classmethod + def _entity_name(cls) -> str: + """Return the persisted entity discriminator for this walker class. + + Honors ``__entity_name__`` when set directly on the class (not + inherited), otherwise falls back to ``cls.__name__``. + """ + return cls.__dict__.get("__entity_name__") or cls.__name__ + type_code: str = attribute(transient=True, default="w") id: str = attribute( protected=True, transient=True, description="Unique identifier for the walker" @@ -137,7 +150,9 @@ class Walker(AttributeMixin, BaseModel): ] = {} _paused: bool = attribute(private=True, default=False) - # Trail tracking + # Trail tracking. Default factory creates an unbounded trail; Walker + # ``__init__`` rebuilds with ``max_trail_length`` from kwargs/env so + # SPEC §6.4 ``max_trail_length`` is actually wired (audit §2.3). _trail_tracker: WalkerTrail = attribute(private=True, default_factory=WalkerTrail) # Trail-related methods (sync for pure computations, async for I/O) @@ -302,13 +317,22 @@ def _get_edge_class(cls) -> Type["Edge"]: def __init__(self: "Walker", **kwargs: Any) -> None: """Initialize a walker with auto-generated ID if not provided.""" + entity_name = self.__class__._entity_name() + # Force ``type_code='w'`` so SPEC §1.1 ID-format invariant + # (``w.EntityName.``) cannot be corrupted by a caller + # passing a different value (audit §2.10). + if kwargs.get("type_code", "w") != "w": + raise ValueError( + f"Walker.type_code must be 'w'; got {kwargs.get('type_code')!r}" + ) + kwargs["type_code"] = "w" if "id" not in kwargs: - # Use class-level type_code or default from Field - type_code = kwargs.get("type_code", "w") - kwargs["id"] = generate_id(type_code, self.__class__.__name__) - # Set entity to class name if not provided (protected attribute) + kwargs["id"] = generate_id("w", entity_name) + # Set entity to class entity_name if not provided (protected attribute). + # Honors ``__entity_name__`` override so subclasses with the same + # Python ``__name__`` as a sibling persist a distinct discriminator. if "entity" not in kwargs: - kwargs["entity"] = self.__class__.__name__ + kwargs["entity"] = entity_name # Extract component configuration from kwargs (before super().__init__) from jvspatial.env import env, parse_bool_basic @@ -328,6 +352,12 @@ def __init__(self: "Walker", **kwargs: Any) -> None: "max_queue_size", int(env("JVSPATIAL_WALKER_MAX_QUEUE_SIZE", default="1000")), ) + # 0 = unlimited (SPEC §6.4). Bounds the in-memory trail so long + # traversals cannot blow memory (audit §2.3). + max_trail_length = kwargs.pop( + "max_trail_length", + int(env("JVSPATIAL_WALKER_MAX_TRAIL_LENGTH", default="0")), + ) paused = kwargs.pop("paused", False) super().__init__(**kwargs) @@ -347,7 +377,9 @@ def __init__(self: "Walker", **kwargs: Any) -> None: # Initialize composition components self._queue: deque[Any] = deque() # Create new deque for queue manager self.queue = WalkerQueue(backing_deque=self._queue, max_size=max_queue_size) - # Trail is initialized as _trail_tracker class attribute + # Replace default unbounded trail with a (possibly bounded) one so + # ``max_trail_length`` from kwargs/env is honored (SPEC §6.4). + self._trail_tracker = WalkerTrail(max_length=max_trail_length) self._protection = TraversalProtection( max_steps=max_steps, max_visits_per_node=max_visits_per_node, @@ -363,8 +395,15 @@ def __init__(self: "Walker", **kwargs: Any) -> None: # Register with global event bus # Note: We need to register asynchronously, so we'll do it in spawn() - def __init_subclass__(cls: Type["Walker"]) -> None: - """Handle subclass initialization.""" + def __init_subclass__(cls: Type["Walker"], **kwargs: Any) -> None: + """Handle subclass initialization. + + Forwards through ``super().__init_subclass__`` so + ``AttributeMixin.__init_subclass__`` runs and registers + ``protected`` / ``transient`` / ``private`` attribute metadata + for Walker subclasses (audit §6.3). + """ + super().__init_subclass__(**kwargs) cls._visit_hooks = {} for _name, method in inspect.getmembers(cls, inspect.isfunction): @@ -647,8 +686,19 @@ async def run(self: "Walker") -> List[Any]: Returns: List of all reported items """ - # Reset protection state - await self._protection.reset() + # Initialize protection state once per traversal session. + # ``start_if_needed`` is idempotent so ``resume()`` does not reset + # the step / visit / timer counters (audit §2.2 / SPEC §6.3). + await self._protection.start_if_needed() + + # Late import to avoid circular dependency at module load. + from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, + ) + + from .walker_components.protection import ProtectionViolation # Process queue until empty or paused while self.queue and not self._paused: @@ -681,8 +731,33 @@ async def run(self: "Walker") -> List[Any]: # Execute visit hooks await self._execute_visit_hooks(current) + except ProtectionViolation as pv: + # SPEC §6.3 / §17: surface protection violations as the + # documented exception types instead of swallowing them + # into the walker report (audit §2.1). + walker_class = type(self).__name__ + ptype = pv.protection_type + if ptype == "max_visits_per_node": + raise InfiniteLoopError( + walker_class=walker_class, + node_id=pv.details.get("node_id", ""), + visit_count=pv.details.get("visit_count", 0), + ) from pv + if ptype == "timeout": + raise WalkerTimeoutError( + walker_class=walker_class, + timeout_seconds=pv.details.get( + "max_execution_time", self._max_execution_time + ), + ) from pv + # ``max_steps`` and any future protection types. + raise WalkerExecutionError( + walker_class=walker_class, + reason=f"protection_triggered: {ptype}", + details=pv.details, + ) from pv except Exception as e: - # Handle errors gracefully + # Handle other (non-protection) errors gracefully. await self.report(f"Error during traversal: {e}") break @@ -732,9 +807,11 @@ async def _execute_visit_hooks(self, target: Union["Node", "Edge"]) -> None: else: hook(self, target) except Exception as e: - # Check if this is a skip exception - if "Node skipped" in str(e): - # Skip this node and continue + # Use the dedicated exception class instead of a fragile + # substring match (audit §2.9 / SPEC §6.5). + from . import TraversalSkipped + + if isinstance(e, TraversalSkipped): walker_hook_skipped = True return else: @@ -795,9 +872,9 @@ async def _execute_visit_hooks(self, target: Union["Node", "Edge"]) -> None: else: bound_hook(self) except Exception as e: - # Check if this is a skip exception - if "Node skipped" in str(e): - # Skip this node and continue + from . import TraversalSkipped + + if isinstance(e, TraversalSkipped): return else: # Report error as structured data @@ -853,11 +930,14 @@ async def spawn( async def skip(self) -> None: """Skip the current node and continue traversal. - This method allows the walker to skip processing the current node - and continue with the next node in the queue. + Raises ``TraversalSkipped`` so the caller can recognize the + skip via ``except TraversalSkipped`` rather than the historical + substring match on ``"Node skipped"`` (audit §2.9 / SPEC §6.5). """ - # Raise an exception to stop current node processing - raise JVSpatialError("Node skipped") + # Late import to avoid circular dep at module load. + from . import TraversalSkipped + + raise TraversalSkipped("Node skipped") def pause(self, message: str = "Traversal paused") -> None: """Pause the walker's traversal. diff --git a/jvspatial/core/entities/walker_components/protection.py b/jvspatial/core/entities/walker_components/protection.py index 6b230ac..147d549 100644 --- a/jvspatial/core/entities/walker_components/protection.py +++ b/jvspatial/core/entities/walker_components/protection.py @@ -53,6 +53,10 @@ def __init__( # Track visit violations for O(1) check_limits() - set when any node # reaches max_visits_per_node (before raising exception) self._visit_violation_detected: bool = False + # Whether the traversal session has begun. ``start_if_needed`` flips + # this once per session so ``resume()`` cannot reset step / visit / + # timer counters (audit §2.2 / SPEC §6.3). + self._started: bool = False async def increment_step(self) -> None: """Increment the step counter and check protection limits.""" @@ -116,11 +120,30 @@ def _check_timeout(self) -> None: ) async def reset(self) -> None: - """Reset protection state and start timing.""" + """Reset protection state and start timing. + + Explicit reset for callers that want to restart a traversal from + scratch. ``Walker.run()`` should call :meth:`start_if_needed` + instead — calling ``reset`` on every ``run`` (including from + ``resume()``) lets pause/resume cycle around the configured limits + (audit §2.2 / SPEC §6.3). + """ self._steps = 0 self._visit_counts.clear() self._visit_violation_detected = False self._start_time = time.time() + self._started = True + + async def start_if_needed(self) -> None: + """Initialize the timer / counters on first call, then no-op. + + Idempotent across ``Walker.run()`` invocations so ``resume()`` does + not wipe accumulated step / visit / timer state. Callers wanting a + fresh session must call :meth:`reset` explicitly. + """ + if self._started: + return + await self.reset() @property def step_count(self) -> int: diff --git a/jvspatial/core/entities/walker_components/walker_queue.py b/jvspatial/core/entities/walker_components/walker_queue.py index e3e8cad..917609a 100644 --- a/jvspatial/core/entities/walker_components/walker_queue.py +++ b/jvspatial/core/entities/walker_components/walker_queue.py @@ -6,9 +6,12 @@ from __future__ import annotations +import logging from collections import deque from typing import Deque, Iterable, List, Optional +logger = logging.getLogger(__name__) + class WalkerQueue: """Manages walker traversal queue operations.""" @@ -27,13 +30,35 @@ def __init__( backing_deque if backing_deque is not None else deque() ) self._max_size = max_size + # One-shot warning when ``max_size`` is hit — repeated drops are + # rate-limited so a runaway producer cannot flood logs. + self._drop_warned: bool = False + + def _has_capacity(self) -> bool: + return self._max_size <= 0 or len(self._backing) < self._max_size + + def _warn_drop(self, op: str, count: int) -> None: + """Warn once per queue when ``max_size`` causes drops (SPEC §6.3).""" + if not self._drop_warned: + logger.warning( + "WalkerQueue.%s: queue at max_size=%d, dropped %d node(s); " + "subsequent drops will be silent.", + op, + self._max_size, + count, + ) + self._drop_warned = True async def visit(self, nodes: Iterable[object]) -> None: """Add nodes to queue, respecting max_size limit.""" + dropped = 0 for n in nodes: - # Only add if we haven't hit the limit - if self._max_size <= 0 or len(self._backing) < self._max_size: + if self._has_capacity(): self._backing.append(n) + else: + dropped += 1 + if dropped: + self._warn_drop("visit", dropped) async def dequeue(self, nodes: object) -> List[object]: """Remove specified node(s) from the queue. @@ -67,11 +92,21 @@ async def dequeue(self, nodes: object) -> List[object]: async def prepend(self, nodes: Iterable[object]) -> None: """Add nodes to the front of the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3) — earlier versions + of this method bypassed the cap, providing a silent protection + bypass via front-of-queue inserts. + Args: nodes: Nodes to prepend to the queue """ + dropped = 0 for n in reversed(list(nodes)): - self._backing.appendleft(n) + if self._has_capacity(): + self._backing.appendleft(n) + else: + dropped += 1 + if dropped: + self._warn_drop("prepend", dropped) async def append(self, nodes: Iterable[object]) -> None: """Add nodes to the end of the queue. @@ -79,19 +114,31 @@ async def append(self, nodes: Iterable[object]) -> None: Args: nodes: Nodes to append to the queue """ + dropped = 0 for n in nodes: - # Only add if we haven't hit the limit - if self._max_size <= 0 or len(self._backing) < self._max_size: + if self._has_capacity(): self._backing.append(n) + else: + dropped += 1 + if dropped: + self._warn_drop("append", dropped) async def add_next(self, nodes: Iterable[object]) -> None: """Add nodes to the front of the queue in the order provided. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). + Args: nodes: Nodes to add to the front of the queue """ + dropped = 0 for n in reversed(list(nodes)): - self._backing.appendleft(n) + if self._has_capacity(): + self._backing.appendleft(n) + else: + dropped += 1 + if dropped: + self._warn_drop("add_next", dropped) async def clear(self) -> None: """Clear all nodes from the queue.""" @@ -137,12 +184,15 @@ async def insert_after( ) -> List[object]: """Insert nodes after a target node in the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). Excess nodes are + dropped from the tail of the input rather than inserted. + Args: target_node: The node to insert after nodes: Nodes to insert Returns: - List of inserted nodes + List of nodes that were actually inserted Raises: ValueError: If target node is not found in the queue @@ -157,23 +207,32 @@ async def insert_after( except ValueError: raise ValueError(f"Target node {target_node} not found in queue") - # Insert after the target - for i, node in enumerate(nodes_list): - self._backing.insert(target_index + 1 + i, node) + inserted: List[object] = [] + dropped = 0 + for node in nodes_list: + if self._has_capacity(): + self._backing.insert(target_index + 1 + len(inserted), node) + inserted.append(node) + else: + dropped += 1 + if dropped: + self._warn_drop("insert_after", dropped) - return nodes_list + return inserted async def insert_before( self, target_node: object, nodes: Iterable[object] ) -> List[object]: """Insert nodes before a target node in the queue. + Respects ``max_size`` (audit §2.4 / SPEC §6.3). + Args: target_node: The node to insert before nodes: Nodes to insert Returns: - List of inserted nodes + List of nodes that were actually inserted Raises: ValueError: If target node is not found in the queue @@ -188,8 +247,15 @@ async def insert_before( except ValueError: raise ValueError(f"Target node {target_node} not found in queue") - # Insert before the target - for i, node in enumerate(nodes_list): - self._backing.insert(target_index + i, node) - - return nodes_list + inserted: List[object] = [] + dropped = 0 + for node in nodes_list: + if self._has_capacity(): + self._backing.insert(target_index + len(inserted), node) + inserted.append(node) + else: + dropped += 1 + if dropped: + self._warn_drop("insert_before", dropped) + + return inserted diff --git a/jvspatial/core/entities/walker_components/walker_trail.py b/jvspatial/core/entities/walker_components/walker_trail.py index fb11d1c..3fdda51 100644 --- a/jvspatial/core/entities/walker_components/walker_trail.py +++ b/jvspatial/core/entities/walker_components/walker_trail.py @@ -6,15 +6,31 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional +from collections import deque +from typing import Any, Deque, Dict, List, Optional class WalkerTrail: - """Tracks traversal steps and metadata.""" + """Tracks traversal steps and metadata. - def __init__(self) -> None: - """Initialize the trail tracker.""" - self._trail: List[Dict[str, Any]] = [] + ``max_length`` bounds the trail so long traversals cannot blow memory + (SPEC §6.4 — ``0`` means unlimited, the documented Walker contract). + Older steps are dropped from the front when the bound is hit. + """ + + def __init__(self, max_length: int = 0) -> None: + """Initialize the trail tracker. + + Args: + max_length: Maximum number of steps retained. ``0`` (default) + means unlimited. Use a positive integer to cap memory on + long-running traversals. + """ + self._max_length = max(0, int(max_length)) + # ``maxlen=None`` makes the deque unbounded. Annotating in one place + # so mypy knows the element type regardless of which branch ran. + bound: Optional[int] = self._max_length if self._max_length > 0 else None + self._trail: Deque[Dict[str, Any]] = deque(maxlen=bound) def record_step( self, node_id: Any, edge_id: Optional[Any] = None, **metadata: Any @@ -41,7 +57,9 @@ async def get_recent(self, count: int = 5) -> List[str]: """Get most recent node IDs from trail.""" if count <= 0: return [] - return [step["node"] for step in self._trail[-count:]] + # ``deque`` does not support slice access; materialize once. + trail_list = list(self._trail) + return [step["node"] for step in trail_list[-count:]] def get_length(self) -> int: """Get trail length.""" diff --git a/jvspatial/core/graph.py b/jvspatial/core/graph.py index f035e59..5f75f30 100644 --- a/jvspatial/core/graph.py +++ b/jvspatial/core/graph.py @@ -169,9 +169,12 @@ async def generate_graph_dot( # Optionally save to file if output_file: + import asyncio from pathlib import Path - Path(output_file).write_text(result, encoding="utf-8") + # ``write_text`` is sync I/O and blocks the event loop in an async + # function (audit §3.4 / SPEC §3.3). + await asyncio.to_thread(Path(output_file).write_text, result, encoding="utf-8") return result @@ -327,9 +330,12 @@ async def generate_graph_mermaid( # Optionally save to file if output_file: + import asyncio from pathlib import Path - Path(output_file).write_text(result, encoding="utf-8") + # Same blocking-write issue as generate_graph_dot + # (audit §3.5 / SPEC §3.3). + await asyncio.to_thread(Path(output_file).write_text, result, encoding="utf-8") return result diff --git a/jvspatial/core/pager.py b/jvspatial/core/pager.py index fb5881e..a411a53 100644 --- a/jvspatial/core/pager.py +++ b/jvspatial/core/pager.py @@ -67,11 +67,12 @@ def __init__( self.total_pages = 1 self.has_previous = False self.has_next = False + # Result caching removed (audit §8.2). The previous ``_cache`` + # was never invalidated on writes so callers got stale rows after + # any save/delete on the underlying collection. Backend-level + # caches (``create_database(cache_get_size=...)``) remain. self.is_cached = False - # Internal cache for results (includes filter hash in key) - self._cache: Dict[str, List[T]] = {} - async def get_page( self, page: int = 1, @@ -117,16 +118,20 @@ async def get_page( # --- Keyset (cursor) pagination --- if after_id is not None: + # Cursor semantics only hold when the sort key matches the + # cursor field. ``after_id`` filters by ``id > after_id`` so + # sorting by anything else would skip rows / return duplicates + # on writes between pages. Reject the combo loudly rather + # than silently return broken pages (audit §8.1). + if self.order_by: + raise ValueError( + "ObjectPager: ``after_id`` (keyset pagination) cannot be " + "combined with ``order_by``; the cursor only tracks id. " + "Use offset pagination if you need a custom sort key." + ) keyset_filter = dict(db_filter) keyset_filter["id"] = {"$gt": after_id} sort: Optional[List[Any]] = [("id", 1)] - if self.order_by: - sort = [ - ( - f"context.{self.order_by}", - 1 if self.order_direction.lower() == "asc" else -1, - ) - ] raw_items = await db.find( collection, keyset_filter, limit=self.page_size + 1, sort=sort ) @@ -138,17 +143,11 @@ async def get_page( obj = await context._deserialize_entity(self.object_class, item_data) if obj: objects.append(obj) - cache_key = f"keyset_{after_id}_{hash(str(additional_filters))}" - self._cache[cache_key] = objects # type: ignore[assignment] self.is_cached = False return objects # --- Page-number (offset) pagination --- self.current_page = max(1, page) - cache_key = f"{self.current_page}_{hash(str(additional_filters))}" - if cache_key in self._cache: - self.is_cached = True - return self._cache[cache_key] # type: ignore[return-value] self.is_cached = False self.total_items = await db.count(collection, db_filter) @@ -211,7 +210,6 @@ async def get_page( if obj: page_objects.append(obj) - self._cache[cache_key] = page_objects # type: ignore[assignment] return page_objects async def next_page( diff --git a/jvspatial/core/utils.py b/jvspatial/core/utils.py index 24e7365..277953b 100644 --- a/jvspatial/core/utils.py +++ b/jvspatial/core/utils.py @@ -1,5 +1,6 @@ """Utility functions for jvspatial core module.""" +import contextlib import uuid from typing import Dict, Optional, Tuple, Type @@ -22,7 +23,12 @@ def generate_id(type_: str, class_name: str) -> str: async def generate_id_async(type_: str, class_name: str) -> str: - """Generate an ID string for graph objects (async version). + """Deprecated async alias for :func:`generate_id`. + + ID generation is pure computation (no I/O); the async signature was + a vestige of an earlier design. SPEC §3.2 documents ``generate_id`` + as the canonical sync API (audit §3.11). Will be removed in a + future minor release. Args: type_: Object type ('n' for node, 'e' for edge, 'w' for walker, 'o' for object) @@ -31,22 +37,50 @@ async def generate_id_async(type_: str, class_name: str) -> str: Returns: Unique ID string in the format "type.class_name.hex_id" """ - hex_id = uuid.uuid4().hex[:24] - return f"{type_}.{class_name}.{hex_id}" + # Lazy import — deprecation helper lives outside the core hot path. + from jvspatial.utils.deprecation import deprecated + + @deprecated( + replacement="jvspatial.core.utils.generate_id", + remove_in="0.1.0", + name="jvspatial.core.utils.generate_id_async", + ) + def _emit() -> None: + return None + + _emit() + return generate_id(type_, class_name) # Cache for subclass lookups to avoid repeated tree traversals _subclass_cache: Dict[Tuple[Type, str], Optional[Type]] = {} +def _class_entity_name(cls: Type) -> str: + """Return the persisted entity discriminator for ``cls``. + + Mirrors ``Object._entity_name()`` but is safe to call on arbitrary types — + falls back to ``cls.__name__`` when ``_entity_name`` is absent. Lets + ``find_subclass_by_name`` honor ``__entity_name__`` overrides without + forcing every caller to pass an ``Object`` descendant. + """ + fn = getattr(cls, "_entity_name", None) + if callable(fn): + with contextlib.suppress(Exception): + return fn() + return cls.__name__ + + def find_subclass_by_name(base_class: Type, name: str) -> Optional[Type]: """Find a subclass by name recursively with caching. - Returns the base class if it matches the name, otherwise returns - the first matching subclass found. Uses caching for performance. + Matches against each class's ``_entity_name()`` (which honors the + ``__entity_name__`` override) and falls back to ``cls.__name__``. + Returns the base class if it matches, otherwise the first matching + subclass found. Uses caching for performance. """ # Check base class first - if base_class.__name__ == name: + if _class_entity_name(base_class) == name: return base_class # Check cache @@ -56,7 +90,7 @@ def find_subclass_by_name(base_class: Type, name: str) -> Optional[Type]: def find_subclass(cls: Type) -> Optional[Type]: for subclass in cls.__subclasses__(): - if subclass.__name__ == name: + if _class_entity_name(subclass) == name: return subclass found = find_subclass(subclass) if found: diff --git a/jvspatial/db/README.md b/jvspatial/db/README.md new file mode 100644 index 0000000..5fa4ee8 --- /dev/null +++ b/jvspatial/db/README.md @@ -0,0 +1,87 @@ +# jvspatial/db + +Database abstraction and backends — JSON, SQLite, MongoDB, DynamoDB. Plus query engine, transactions, atomic IO, path locks, cache wrapper, and observability wrapper. + +> **Read first**: [SPEC §4-5](../../SPEC.md), [docs/md/mongodb-query-interface.md](../../docs/md/mongodb-query-interface.md) + +--- + +## Purpose + +`db/` defines the `Database` ABC and ships four production backends, all sharing a Mongo-style query DSL. Internal wrappers (`_cache`, `_observable`) add cross-cutting concerns to any backend through factory flags. + +## Layout + +``` +db/ +├── database.py # Database ABC + finalize_find_results +├── factory.py # create_database, register_database_type, switch_database +├── manager.py # DatabaseManager + prime DB convention +├── query.py # QueryEngine + Mongo-style operators +├── transaction.py # Transaction context + JsonDBTransaction +├── jsondb.py # JSON file-per-record backend +├── sqlite.py # aiosqlite backend + Mongo→SQL translator +├── mongodb.py # motor backend +├── dynamodb.py # aioboto3 backend +├── _atomic.py # internal: crash-safe write helper +├── _path_locks.py # internal: bounded-LRU per-path locks +├── _cache.py # internal: read-through cache wrapper +├── _observable.py # internal: structured log + metrics wrapper +└── _sqlite_translate.py # internal: Mongo → SQL translator +``` + +## Backend matrix + +| Backend | Transactions | Bulk APIs | Native count | Notes | +|---|---|---|---|---| +| JSON | No (best-effort opt-in only) | Parallel reads/writes | Dirent fast path | Atomic writes, per-file locks | +| SQLite | No (single-conn fsync) | `executemany` + `IN` | Mongo→SQL pushdown | Translator covers `$eq/$ne/$gt/$gte/$lt/$lte/$in/$nin/$exists`, AND, `$and/$or` | +| MongoDB | **Yes** (replica set required) | `bulk_write`, `$in` | `count_documents` / `estimated_document_count` | Native compound ops; shared retry helper | +| DynamoDB | No | `BatchGetItem`/`BatchWriteItem` (100/batch) | `Select="COUNT"` | Throttle retry with backoff | + +## Public API (from `jvspatial.db`) + +| Name | What it does | +|---|---| +| `Database` | ABC every adapter implements (SPEC §4.1) | +| `create_database(type, ...)` | Factory entry point | +| `create_default_database()` | Environment-driven default | +| `register_database_type(name, factory)` | Register a custom adapter | +| `unregister_database`, `unregister_database_type` | Cleanup | +| `list_database_types()` | Inventory | +| `get_prime_database`, `get_current_database`, `switch_database` | Multi-DB lookup | +| `DatabaseManager`, `get_database_manager`, `set_database_manager` | Manager surface | +| `DatabaseError`, `VersionConflictError` | DB-specific exceptions | +| `JsonDB`, `SQLiteDB`, `MongoDB`, `DynamoDB` | Concrete adapters (optional extras for non-JSON) | + +## Invariants + +- **`Database.supports_transactions` is a capability flag.** Branch on it; do not sniff adapter class. (`database.py:84`) +- **`find_many` and `bulk_save` are public and benefit from native overrides.** Defaults exist but are slow. (`database.py:176+`) +- **`find_one_and_update` / `find_one_and_delete` are NOT atomic by default.** Only MongoDB overrides with native atomic versions. +- **Atomic JSON writes use `temp + fsync + rename + fsync(dir)`.** No partial records survive a crash. (`_atomic.py`) +- **Per-file locks serialize concurrent writes to the same record only.** Different files run in parallel. (`_path_locks.py`) +- **`QueryEngine` LRU is bounded.** Default 1024; configurable. Unbounded query construction will not leak memory. (`query.py`) +- **Prime database is unique.** Auth state, sessions, API keys live there. Cannot be switched. + +## Modification patterns + +- **Adding a custom backend**: subclass `Database`, implement abstract methods, override `find_many` / `bulk_save` if you can, set `supports_transactions`, register via `register_database_type`. Add tests under `tests/db/`. +- **Adding a query operator**: extend `QueryEngine.evaluate_*`. If pushdown is feasible, update `SQLiteTranslator` and the DynamoDB query path. +- **Adding a backend capability flag**: declare as class attribute on `Database`, default to `False` (or the safe value). Document in SPEC §4.2. +- **Touching internal wrappers** (`_atomic.py`, `_path_locks.py`, `_cache.py`, `_observable.py`): they are not part of the public API but **the observability log-field schema IS public** (see [docs/md/stability.md](../../docs/md/stability.md)). + +## Related docs + +- [docs/md/mongodb-query-interface.md](../../docs/md/mongodb-query-interface.md) +- [docs/md/custom-database-guide.md](../../docs/md/custom-database-guide.md) +- [docs/md/dynamodb-guide.md](../../docs/md/dynamodb-guide.md) +- [docs/md/graph-context.md](../../docs/md/graph-context.md) +- [docs/md/caching.md](../../docs/md/caching.md) +- [docs/md/observability.md](../../docs/md/observability.md) +- [docs/md/optimization.md](../../docs/md/optimization.md) +- [docs/md/benchmarks.md](../../docs/md/benchmarks.md) + +## Stability + +Public surface (above) is stable. Underscore-prefixed modules are internal. The structured log field set from `ObservableDatabase` is part of the public contract — schema changes require a deprecation cycle. diff --git a/jvspatial/db/_atomic.py b/jvspatial/db/_atomic.py index 1d87998..bc83d95 100644 --- a/jvspatial/db/_atomic.py +++ b/jvspatial/db/_atomic.py @@ -169,12 +169,23 @@ def cleanup_orphan_tmp_files(roots: Iterable[Union[str, Path]]) -> int: if is_serverless_mode(): return 0 + current_pid = os.getpid() removed = 0 for root in roots: root_path = Path(root) if not root_path.exists() or not root_path.is_dir(): continue for orphan in root_path.rglob(f"*{TMP_SUFFIX}"): + # Tmp filenames embed the writer's pid (see ``_make_temp_path``) + # so the sweep can distinguish in-flight writes (current process) + # from orphans left by a previously-crashed process. Never reap + # a file owned by the current pid — it's either mid-write right + # now (deleting it races ``os.replace`` and corrupts the write) + # or will get reaped on the next process startup. Filename shape: + # ``...jvtmp``. + parts = orphan.name.rsplit(".", 3) + if len(parts) == 4 and parts[1].isdigit() and int(parts[1]) == current_pid: + continue try: orphan.unlink() removed += 1 diff --git a/jvspatial/db/_observable.py b/jvspatial/db/_observable.py index b2cbc9a..ceee1b2 100644 --- a/jvspatial/db/_observable.py +++ b/jvspatial/db/_observable.py @@ -38,6 +38,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union from jvspatial.db.database import Database +from jvspatial.observability import db_op_counter from jvspatial.observability.metrics import ( MetricsRecorder, NullMetricsRecorder, @@ -118,6 +119,8 @@ async def _instrument( success = False raise finally: + # Increment per-flow DB operation count even on failures. + db_op_counter.set(db_op_counter.get() + 1) duration_s = time.monotonic() - start duration_ms = duration_s * 1000.0 result_count: Optional[int] = None diff --git a/jvspatial/db/database.py b/jvspatial/db/database.py index eb444a5..5ab5716 100644 --- a/jvspatial/db/database.py +++ b/jvspatial/db/database.py @@ -6,11 +6,34 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass, field from functools import partial from typing import Any, Dict, List, Optional, Tuple, Union from jvspatial.db.query import QueryEngine + +@dataclass(frozen=True) +class BulkSaveResult: + """Structured outcome of a :meth:`Database.bulk_save_detailed` call. + + Adapters with partial-success semantics (JsonDB, DynamoDB, MongoDB + ``ordered=False``) report the per-record breakdown so callers can + distinguish "all saved" from "some lost" — previously every adapter + returned the same ``int`` and silent drops were indistinguishable + from success (audit §5.7 / §5.13). + """ + + attempted: int + saved: int + failed_ids: List[str] = field(default_factory=list) + + @property + def all_saved(self) -> bool: + """Return True when every attempted record persisted.""" + return self.saved == self.attempted and not self.failed_ids + + logger = logging.getLogger(__name__) @@ -20,6 +43,24 @@ def _find_sort_key(record: Dict[str, Any], field: str) -> Tuple[bool, Any]: return (value is None, value) +def _normalize_id_query(query: Dict[str, Any]) -> Dict[str, Any]: + """Map ``_id`` → ``id`` when only ``_id`` is present. + + The default ``find_one_and_update`` / ``find_one_and_delete`` impls + feed the query into ``QueryEngine.match`` against records stored by + non-Mongo backends (JsonDB / SQLite / DynamoDB) which only persist + ``id``. Callers that follow the Mongo-style convention of querying + by ``_id`` would otherwise silently miss every row on those backends + (audit §5.3). When both keys are present the caller's intent is + preserved verbatim. + """ + if "_id" in query and "id" not in query: + normalized = {k: v for k, v in query.items() if k != "_id"} + normalized["id"] = query["_id"] + return normalized + return query + + def finalize_find_results( records: List[Dict[str, Any]], *, @@ -35,9 +76,9 @@ def finalize_find_results( out = records if sort: out = list(records) - for field, direction in reversed(sort): + for sort_field, direction in reversed(sort): out.sort( - key=partial(_find_sort_key, field=field), + key=partial(_find_sort_key, field=sort_field), reverse=(direction == -1), ) if limit is not None: @@ -220,34 +261,68 @@ async def bulk_save(self, collection: str, records: List[Dict[str, Any]]) -> int Returns: Number of records successfully saved. + For partial-success visibility (which IDs failed) use + :meth:`bulk_save_detailed`. Adapters with all-or-nothing + semantics (SQLite) raise rather than return a partial count. + """ + result = await self.bulk_save_detailed(collection, records) + return result.saved + + async def bulk_save_detailed( + self, collection: str, records: List[Dict[str, Any]] + ) -> BulkSaveResult: + """Save many records, returning a structured per-record outcome. + + Args: + collection: Collection name. + records: Iterable of record dicts. Each must have an ``id`` + field. Records without ``id`` raise ``ValueError`` (the + check is per-batch, before any save attempt). + + Returns: + :class:`BulkSaveResult` with ``attempted`` / ``saved`` / + ``failed_ids``. + Atomicity (per backend): - * MongoDB: ``bulk_write`` with ``ordered=False`` -- partial + * MongoDB: ``bulk_write`` with ``ordered=False`` — partial successes are reported; failures don't block other writes. * SQLite: single transaction with ``executemany``; **all records or none** land. A constraint violation rolls back - the whole batch. + the whole batch and raises (``failed_ids`` is unset). * DynamoDB: ``BatchWriteItem`` with unprocessed-item retry; - partial successes possible. - * JsonDB: parallel atomic per-file writes; partial successes - possible. + unprocessed items after retries land in ``failed_ids``. + * JsonDB: parallel atomic per-file writes; per-record + exceptions land in ``failed_ids``. The default implementation in this base class is a serial - loop of ``save()`` calls (partial success on failure), which - is correct but slow -- adapters should override. + loop of ``save()`` calls catching per-record exceptions. + Adapters should override for round-trip efficiency. """ if not records: - return 0 - for r in records: + return BulkSaveResult(attempted=0, saved=0, failed_ids=[]) + for idx, r in enumerate(records): if "id" not in r: - raise ValueError( - "bulk_save requires every record to have an 'id' field" - ) - # Sequential save() calls; on any failure the exception - # propagates and the caller sees no return value, so reaching - # the ``return`` always means every record persisted. + # Caller error — surface the offending index for fast + # debugging (audit §5.15). + raise ValueError(f"bulk_save: record at index {idx} has no 'id' field") + + saved = 0 + failed_ids: List[str] = [] for r in records: - await self.save(collection, dict(r)) - return len(records) + try: + await self.save(collection, dict(r)) + saved += 1 + except Exception as e: # pragma: no cover — defensive + logger.warning( + "%s.bulk_save: record id=%r failed: %s", + type(self).__name__, + r.get("id"), + e, + ) + failed_ids.append(str(r.get("id", ""))) + return BulkSaveResult( + attempted=len(records), saved=saved, failed_ids=failed_ids + ) async def find_one_and_delete( self, collection: str, query: Dict[str, Any] @@ -268,7 +343,10 @@ async def find_one_and_delete( Returns: Deleted document if found, ``None`` otherwise """ - doc = await self.find_one(collection, query) + # Non-Mongo backends store records keyed by ``id`` only. Normalize + # the Mongo-style ``_id`` filter so default matching works + # uniformly (audit §5.3 / SPEC §4.1). + doc = await self.find_one(collection, _normalize_id_query(query)) if doc is None: return None record_id = doc.get("_id", doc.get("id")) @@ -301,7 +379,11 @@ async def find_one_and_update( Returns: Updated document, or ``None`` if no match and ``upsert`` is ``False`` """ - doc = await self.find_one(collection, query) + # Non-Mongo backends store records keyed by ``id`` only. Normalize + # the Mongo-style ``_id`` filter so default matching works + # uniformly (audit §5.3 / SPEC §4.1). + normalized = _normalize_id_query(query) + doc = await self.find_one(collection, normalized) is_new = doc is None if is_new: if not upsert: diff --git a/jvspatial/db/dynamodb.py b/jvspatial/db/dynamodb.py index fbff8d7..2e1b0d0 100644 --- a/jvspatial/db/dynamodb.py +++ b/jvspatial/db/dynamodb.py @@ -16,6 +16,7 @@ import asyncio import json import logging +from functools import partial from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union try: @@ -637,8 +638,13 @@ async def process_batch(batch_ids: List[str]) -> List[Dict[str, Any]]: } try: - # Execute batch get - response = await client.batch_get_item(RequestItems=request_items) + # Throttle-retry wraps the wire call so transient + # ProvisionedThroughputExceeded / ThrottlingException are + # backed off (audit §5.1 / SPEC §4.3). + response = await self._run_with_throttle_retry( + "batch_get_item", + lambda: client.batch_get_item(RequestItems=request_items), + ) # Process responses items = response.get("Responses", {}).get(table_name, []) @@ -653,10 +659,11 @@ async def process_batch(batch_ids: List[str]) -> List[Dict[str, Any]]: logger.debug( f"Unprocessed keys in batch_get for collection '{collection}': {len(unprocessed.get(table_name, {}).get('Keys', []))}" ) - # Retry unprocessed keys once + # Retry unprocessed keys once (also throttle-aware). retry_items = {table_name: unprocessed[table_name]} - retry_response = await client.batch_get_item( - RequestItems=retry_items + retry_response = await self._run_with_throttle_retry( + "batch_get_item_retry", + lambda: client.batch_get_item(RequestItems=retry_items), ) retry_items_list = retry_response.get("Responses", {}).get( table_name, [] @@ -739,8 +746,11 @@ async def process_batch(batch_items: List[Dict[str, Any]]) -> None: request_items = {table_name: write_requests} try: - # Execute batch write - response = await client.batch_write_item(RequestItems=request_items) + # Throttle-retry wraps the wire call (audit §5.1 / SPEC §4.3). + response = await self._run_with_throttle_retry( + "batch_write_item", + lambda: client.batch_write_item(RequestItems=request_items), + ) # Handle unprocessed items with retry logic unprocessed = response.get("UnprocessedItems", {}) @@ -756,8 +766,13 @@ async def process_batch(batch_items: List[Dict[str, Any]]) -> None: # Wait before retry (exponential backoff) await asyncio.sleep(0.1 * (2 ** (retry_count - 1))) - retry_response = await client.batch_write_item( - RequestItems=unprocessed + # ``partial`` snapshots kwargs at construction time, so + # the closure does not refer to the loop variable + # ``unprocessed`` (flake8-bugbear B023). The retry + # helper awaits the resulting coroutine inline. + retry_response = await self._run_with_throttle_retry( + "batch_write_item_retry", + partial(client.batch_write_item, RequestItems=unprocessed), ) unprocessed = retry_response.get("UnprocessedItems", {}) @@ -993,7 +1008,10 @@ async def count( total = 0 while True: - resp = await client.query(**params) + resp = await self._run_with_throttle_retry( + "count_query", + lambda: client.query(**params), + ) total += int(resp.get("Count", 0)) if "LastEvaluatedKey" not in resp: break @@ -1042,7 +1060,10 @@ async def count( } total = 0 while True: - resp = await client.scan(**scan_params) + resp = await self._run_with_throttle_retry( + "count_scan", + lambda: client.scan(**scan_params), + ) total += int(resp.get("Count", 0)) if "LastEvaluatedKey" not in resp: break @@ -1146,7 +1167,10 @@ async def find( if filter_expr: query_params["FilterExpression"] = filter_expr - response = await client.query(**query_params) + response = await self._run_with_throttle_retry( + "find_query", + lambda: client.query(**query_params), + ) results = [] for item in response.get("Items", []): @@ -1170,7 +1194,10 @@ async def find( query_params["ExclusiveStartKey"] = response["LastEvaluatedKey"] if fetch_limit: query_params["Limit"] = fetch_limit - len(results) - response = await client.query(**query_params) + response = await self._run_with_throttle_retry( + "find_query", + lambda: client.query(**query_params), + ) for item in response.get("Items", []): data = json.loads(item["data"]["S"]) @@ -1226,7 +1253,10 @@ async def find( if fetch_limit: scan_params["Limit"] = fetch_limit - response = await client.scan(**scan_params) + response = await self._run_with_throttle_retry( + "find_scan", + lambda: client.scan(**scan_params), + ) results = [] for item in response.get("Items", []): @@ -1247,7 +1277,10 @@ async def find( scan_params["ExclusiveStartKey"] = response["LastEvaluatedKey"] if fetch_limit: scan_params["Limit"] = fetch_limit - len(results) - response = await client.scan(**scan_params) + response = await self._run_with_throttle_retry( + "find_scan", + lambda: client.scan(**scan_params), + ) for item in response.get("Items", []): data = json.loads(item["data"]["S"]) diff --git a/jvspatial/db/jsondb.py b/jvspatial/db/jsondb.py index ddb5de3..9a671c3 100644 --- a/jvspatial/db/jsondb.py +++ b/jvspatial/db/jsondb.py @@ -149,6 +149,20 @@ def _get_record_path(self, collection: str, record_id: str) -> Path: collection_dir = self._get_collection_dir(collection) return collection_dir / f"{record_id}.json" + @staticmethod + def _list_collection_json_files(collection_dir: Path) -> List[Path]: + """Enumerate persisted ``*.json`` records. + + Called from inside ``asyncio.to_thread`` by ``count`` and ``find`` + so the directory scan does not block the event loop. + + Note: in-flight ``*.jvtmp`` files are named + ``.json...jvtmp`` (see ``_atomic._make_temp_path``) + so the ``*.json`` glob already excludes them — the historical + ``not endswith('.jvtmp')`` filter was dead (audit §5.16). + """ + return list(collection_dir.glob("*.json")) + async def _async_write_json(self, path: Path, data: Dict[str, Any]) -> None: """Write JSON data to file asynchronously. @@ -165,7 +179,10 @@ async def _async_read_json(self, path: Path) -> Optional[Dict[str, Any]]: Uses aiofiles if available, otherwise falls back to asyncio.to_thread. Returns None if file doesn't exist or is invalid. """ - if not path.exists(): + # ``Path.exists`` performs a stat() syscall — blocking I/O inside + # an async function (audit §3.6 / SPEC §3.3). Offload to the + # default executor instead. + if not await asyncio.to_thread(path.exists): return None try: @@ -247,12 +264,14 @@ async def count( """ q = query or {} collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync ``exists``/``glob`` block the event loop — offload both + # (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return 0 - json_files = [ - p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") - ] + json_files = await asyncio.to_thread( + self._list_collection_json_files, collection_dir + ) if not q: return len(json_files) @@ -283,7 +302,8 @@ async def find_many( # Build (id, path) pairs first so we can short-circuit when # the collection dir doesn't exist. collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync stat blocks the event loop (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return {} paths = [ (rec_id, self._get_record_path(collection, rec_id)) for rec_id in unique_ids @@ -352,15 +372,17 @@ async def find( """ collection_dir = self._get_collection_dir(collection) - if not collection_dir.exists(): + # Sync ``exists``/``glob`` block the event loop — offload both + # (audit §3.6 / SPEC §3.3). + if not await asyncio.to_thread(collection_dir.exists): return [] # Get all JSON files in the collection directory. # Skip ``*.jvtmp`` files left behind by an in-flight write -- they # are not yet part of the published dataset. - json_files = [ - p for p in collection_dir.glob("*.json") if not p.name.endswith(".jvtmp") - ] + json_files = await asyncio.to_thread( + self._list_collection_json_files, collection_dir + ) if not json_files: return [] diff --git a/jvspatial/db/mongodb.py b/jvspatial/db/mongodb.py index 2fda38a..8350f6f 100644 --- a/jvspatial/db/mongodb.py +++ b/jvspatial/db/mongodb.py @@ -62,8 +62,12 @@ class MongoDB(Database): """Simplified MongoDB-based database implementation.""" # Advertised capability. The adapter implements the transactional API; - # the deployment must be a replica set (even single-node) for the - # actual ``begin_transaction()`` call to succeed at runtime. + # actual transactional semantics also require a replica-set (or + # ``sharded`` 4.2+) deployment. The class attribute stays ``True`` + # so callers branching on the static flag (the documented form) still + # see "MongoDB supports transactions". Use :meth:`is_transactional` + # for a runtime probe that honors the deployment topology + # (audit §5.9 / SPEC §4.2). supports_transactions: bool = True def __init__( @@ -100,6 +104,10 @@ def __init__( self._created_indexes: Dict[str, Set[str]] = ( {} ) # collection -> set of index names + # Memoized result of the transactional-topology probe — ``None`` + # until first checked, ``True`` if the deployment supports + # transactions, ``False`` otherwise. See :meth:`is_transactional`. + self._transactional_probe: Optional[bool] = None async def _ensure_connected(self) -> None: """Ensure database connection is established. @@ -696,6 +704,41 @@ async def create_index( except PyMongoError as e: raise DatabaseError(f"MongoDB index creation error: {e}") from e + async def is_transactional(self) -> bool: + """Return True only when the deployment supports MongoDB transactions. + + Probes the cluster topology on first call (``hello`` admin command) + and caches the result on the instance. Standalone deployments and + DocumentDB instances without transaction support return ``False``; + replica sets (single- or multi-node) and sharded clusters at + version 4.2+ return ``True``. + + Use this instead of branching on the static + :attr:`supports_transactions` class attribute when the caller + intends to actually open a transaction (audit §5.9 / SPEC §4.2). + """ + if self._transactional_probe is not None: + return self._transactional_probe + await self._ensure_connected() + if self._client is None: + self._transactional_probe = False + return False + try: + info = await self._client.admin.command("hello") + # Replica sets expose ``setName``; sharded clusters expose + # ``msg: "isdbgrid"``. Standalone deployments have neither. + is_replica_set = bool(info.get("setName")) + is_mongos = info.get("msg") == "isdbgrid" + self._transactional_probe = is_replica_set or is_mongos + except Exception: + logger.debug( + "MongoDB transaction-support probe failed; treating as " + "non-transactional", + exc_info=True, + ) + self._transactional_probe = False + return self._transactional_probe + async def begin_transaction(self): """Start a MongoDB transaction (requires replica set). @@ -710,6 +753,11 @@ async def begin_transaction(self): await self._ensure_connected() if self._client is None or self._db is None: return None + # Refuse to attempt a transaction on a topology that does not + # support it — saves an exception round-trip on every call after + # the first (audit §5.9). + if not await self.is_transactional(): + return None try: import uuid @@ -718,6 +766,8 @@ async def begin_transaction(self): return MongoDBTransaction(str(uuid.uuid4()), session, self._db) except Exception: logger.debug("Transactions not available on this deployment", exc_info=True) + # Cache the negative probe so subsequent calls short-circuit. + self._transactional_probe = False return None async def commit_transaction(self, txn) -> None: diff --git a/jvspatial/db/query.py b/jvspatial/db/query.py index 23b5cc0..93f7d73 100644 --- a/jvspatial/db/query.py +++ b/jvspatial/db/query.py @@ -10,6 +10,8 @@ from collections import OrderedDict from typing import Any, Callable, Dict, List, Optional, Union +from jvspatial.exceptions import QueryError + # Default upper bound for the per-instance ``optimize_query`` cache. The # previous implementation grew unboundedly which is fine for short-lived # processes but a slow leak in long-lived servers. 1024 entries holds @@ -17,6 +19,28 @@ DEFAULT_QUERY_CACHE_SIZE = 1024 +# Mapping from MongoDB ``$type`` operand strings to Python types used by +# the in-memory matcher. BSON type codes are not carried through this +# layer; Mongo callers using the native driver's ``$type`` get native +# behavior. (Audit §5.2.) +_TYPE_NAME_MAP = { + "string": str, + "str": str, + "int": int, + "long": int, + "double": float, + "float": float, + "bool": bool, + "boolean": bool, + "list": list, + "array": list, + "dict": dict, + "object": dict, + "null": type(None), + "none": type(None), +} + + # Unified evaluation and builder in a single module @@ -315,6 +339,13 @@ def unset_field_value(document: Dict[str, Any], field: str) -> None: elif isinstance(current, dict): current.pop(last, None) + # Operators recognized at the top-level of a query document. Anything + # else is rejected to surface typos rather than silently match nothing + # (audit §5.2). Field-name keys are checked separately below. + _TOP_LEVEL_LOGICAL_OPERATORS = frozenset({"$and", "$or", "$nor", "$not"}) + # Markers the query optimizer may inject; ignored by the matcher. + _IGNORED_TOP_LEVEL_MARKERS = frozenset({"$hint", "$select"}) + @staticmethod def match(document: Dict[str, Any], query: Optional[Dict[str, Any]]) -> bool: """Check if a document matches a query. @@ -335,9 +366,31 @@ def match(document: Dict[str, Any], query: Optional[Dict[str, Any]]) -> bool: elif key == "$or": if not any(QueryEngine.match(document, sub) for sub in condition or []): return False + elif key == "$nor": + # Audit §5.2 / SPEC §5.1: QueryBuilder.nor_() existed + # but match() ignored ``$nor`` entirely. Now: ``$nor`` + # matches when none of the sub-conditions match. + if any(QueryEngine.match(document, sub) for sub in condition or []): + return False elif key == "$not": if QueryEngine.match(document, condition): return False + elif key in QueryEngine._IGNORED_TOP_LEVEL_MARKERS: + # Optimizer hints — irrelevant to in-memory matching. + continue + elif key.startswith("$"): + # Unknown top-level operator. Refuse silently-matching + # nothing — raise so callers see the bug immediately + # (audit §5.2). + raise QueryError( + query=str(query), + reason=( + f"unsupported top-level query operator: {key!r}. " + "Supported: $and, $or, $nor, $not. Field-level " + "operators (e.g. $regex, $mod, $type, $size) live " + "inside a field condition dict." + ), + ) else: value = QueryEngine.get_field_value(document, key) if not QueryEngine._match_value(value, condition): @@ -422,8 +475,56 @@ def _match_value(value: Any, condition: Any) -> bool: for elem in value ): return False + elif op == "$mod": + # MongoDB ``$mod`` — operand is ``[divisor, remainder]``. + # Previously the QueryBuilder.mod() helper produced this + # but the engine returned False for it (audit §5.2). + try: + divisor, remainder = operand + if not isinstance(value, (int, float)): + return False + if value % divisor != remainder: + return False + except (TypeError, ValueError, ZeroDivisionError): + return False + elif op == "$all": + # ``$all`` — value must be a list containing every operand + # element. Previously silently no-matched (audit §5.2). + if not isinstance(value, list): + return False + try: + operand_iter = list(operand) + except TypeError: + return False + if not all(item in value for item in operand_iter): + return False + elif op == "$type": + # MongoDB ``$type`` — accept a Python type name string + # (``"int"``, ``"string"``, ``"list"`` …) since we do not + # carry BSON type codes through. + expected = _TYPE_NAME_MAP.get( + operand.lower() if isinstance(operand, str) else None + ) + if expected is None: + return False + if not isinstance(value, expected): + return False + elif op == "$not": + # Field-level negation — operand is another condition dict. + if QueryEngine._match_value(value, operand): + return False else: - return False + # Unknown field-level operator. Refuse silent no-match — + # raise so callers see the bug (audit §5.2). + raise QueryError( + query=str(condition), + reason=( + f"unsupported field-level query operator: {op!r}. " + "Supported: $eq, $ne, $gt, $gte, $lt, $lte, $in, " + "$nin, $exists, $regex, $size, $elemMatch, $mod, " + "$all, $type, $not." + ), + ) return True @staticmethod diff --git a/jvspatial/db/sqlite.py b/jvspatial/db/sqlite.py index 0cdc784..69f8c45 100644 --- a/jvspatial/db/sqlite.py +++ b/jvspatial/db/sqlite.py @@ -113,9 +113,47 @@ def __init__( self._created_indexes: Dict[str, Set[str]] = ( {} ) # collection -> set of index names + # The event loop that owns ``self._connection``. ``aiosqlite`` + # binds its connection to a single loop; using the connection + # from a different loop produces opaque "Future attached to a + # different loop" errors. We track the binding here and raise + # a clear ``DatabaseError`` on cross-loop reuse (audit §5.10). + self._owning_loop: Optional[asyncio.AbstractEventLoop] = None async def _get_connection(self) -> "Connection": - """Get or create the SQLite connection.""" + """Get or create the SQLite connection. + + Cross-loop reuse handling (audit §5.10 / SPEC §4.3): + + * For **file-backed databases**, the connection is silently + rebound to the current loop — the old loop is presumed gone, + and on-disk state persists across the rebind so callers do + not observe data loss. + * For ``:memory:`` databases, data lives in the connection + itself; rebinding would silently truncate the dataset. We + keep the existing connection and trust aiosqlite's internal + thread to dispatch queries from any loop (this matches + historical behavior — the audit's concern was an opaque + failure mode, not data loss). + """ + current_loop = asyncio.get_running_loop() + if ( + self._connection is not None + and self._owning_loop is not None + and self._owning_loop is not current_loop + and self.db_path_str != ":memory:" + ): + logger.debug( + "SQLiteDB rebinding to a new event loop; abandoning " + "connection owned by %r and reconnecting on %r", + self._owning_loop, + current_loop, + ) + self._connection = None + self._owning_loop = None + self._initialized = False + # Reset per-loop state; index re-creation is idempotent. + self._created_indexes.clear() if self._connection is None: # Ensure parent directory exists before connecting (for file paths) if self.db_path_str != ":memory:": @@ -134,6 +172,7 @@ async def _get_connection(self) -> "Connection": self._connection = await aiosqlite.connect( self.db_path_str, timeout=self.timeout ) + self._owning_loop = current_loop self._connection.row_factory = aiosqlite.Row await self._connection.execute(f"PRAGMA journal_mode={self.journal_mode};") await self._connection.execute(f"PRAGMA synchronous={self.synchronous};") @@ -250,12 +289,17 @@ async def create_index( ) async def close(self) -> None: - """Close the underlying SQLite connection.""" + """Close the underlying SQLite connection. + + Clears the owning-loop binding so the instance can be reused on + a fresh event loop (audit §5.10). + """ if self._connection is not None: await self._connection.close() self._connection = None self._initialized = False self._created_indexes.clear() + self._owning_loop = None async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: """Save a record to the database. @@ -271,7 +315,13 @@ async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: connection = await self._get_connection() record = data.copy() - record_id = record.setdefault("id", str(uuid.uuid4())) + # Coerce id to ``str`` so non-string ids (int, uuid.UUID) + # round-trip cleanly through SQLite's TEXT column. The + # legacy code only stringified the default uuid and bound + # the raw value; ``get(collection, id)`` then missed when + # callers passed an int-typed id (audit §5.20). + record_id = str(record.setdefault("id", str(uuid.uuid4()))) + record["id"] = record_id payload = json.dumps(record) await connection.execute( diff --git a/jvspatial/db/transaction.py b/jvspatial/db/transaction.py index ec923d6..d36f42a 100644 --- a/jvspatial/db/transaction.py +++ b/jvspatial/db/transaction.py @@ -17,8 +17,7 @@ commit. Intended for testing, scripting, and local-dev workflows where the trade-off is acceptable. * **None.** Calling ``JsonDBTransaction()`` (without ``best_effort=True``) - or any operation on ``JSONTransaction`` raises - :class:`NotImplementedError`. Callers can detect this via the + raises :class:`NotImplementedError`. Callers can detect this via the ``Database.supports_transactions`` capability flag and fall back to non-transactional writes. """ @@ -240,9 +239,11 @@ def __init__(self, database, *, best_effort: bool = False): if best_effort: # Emit a once-per-process ExperimentalWarning so adopters know # this surface may change. See docs/md/stability.md. - from jvspatial.utils.stability import _emit_once + # Uses the public ``emit_experimental_once`` hook rather than + # the underscore-prefixed implementation (audit §7.7). + from jvspatial.utils.stability import emit_experimental_once - _emit_once( + emit_experimental_once( "JsonDBTransaction(best_effort=True)", "Buffered-commit semantics are weaker than ACID and the " "interface may evolve; track docs/md/stability.md.", @@ -360,59 +361,6 @@ async def rollback(self) -> None: self.is_rolled_back = True -class JSONTransaction(Transaction): - """JSON database transaction implementation (no-op for file-based storage).""" - - def __init__(self, transaction_id: str): - """Initialize JSON transaction. - - Args: - transaction_id: Unique identifier for this transaction - """ - super().__init__(transaction_id) - self.is_active = True - # JSON database doesn't support true transactions, so we simulate them - - async def save(self, collection: str, data: Dict[str, Any]) -> Dict[str, Any]: - """Save a record within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction save not implemented") - - async def get(self, collection: str, id: str) -> Optional[Dict[str, Any]]: - """Retrieve a record by ID within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction get not implemented") - - async def delete(self, collection: str, id: str) -> bool: - """Delete a record within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction delete not implemented") - - async def find( - self, - collection: str, - query: Dict[str, Any], - *, - limit: Optional[int] = None, - sort: Optional[List[Tuple[str, int]]] = None, - ) -> List[Dict[str, Any]]: - """Find records matching query within this JSON transaction (simulated).""" - # For JSON database, we just track operations but don't implement true transactions - raise NotImplementedError("JSON transaction find not implemented") - - async def commit(self) -> None: - """Commit this JSON transaction (simulated).""" - if self.is_active and not self.is_committed and not self.is_rolled_back: - self.is_active = False - self.is_committed = True - - async def rollback(self) -> None: - """Rollback this JSON transaction (simulated).""" - if self.is_active and not self.is_committed and not self.is_rolled_back: - self.is_active = False - self.is_rolled_back = True - - @asynccontextmanager async def transaction_context(database, transaction_id: Optional[str] = None): """Context manager for database transactions. @@ -449,6 +397,5 @@ async def transaction_context(database, transaction_id: Optional[str] = None): "Transaction", "MongoDBTransaction", "JsonDBTransaction", - "JSONTransaction", "transaction_context", ] diff --git a/jvspatial/env_adapter.py b/jvspatial/env_adapter.py index 74b9794..a89f492 100644 --- a/jvspatial/env_adapter.py +++ b/jvspatial/env_adapter.py @@ -2,14 +2,33 @@ from __future__ import annotations +import logging import os from typing import Any, Dict, List, Optional +from jvspatial.env import parse_bool from jvspatial.runtime.eventbridge_readiness import resolve_eventbridge_lambda_arn +logger = logging.getLogger(__name__) + def _parse_bool(val: str) -> bool: - return str(val).strip().lower() in ("true", "1", "yes") + """Permissive boolean parser; consolidated with :func:`jvspatial.env.parse_bool`. + + Accepts ``true/false``, ``1/0``, ``yes/no``, ``on/off`` (case + insensitive). Falls back to ``False`` for unrecognized values rather + than raising — preserves prior behavior for misconfigured env values + (audit §7.2-§7.3). + """ + try: + return parse_bool(val) + except ValueError: + logger.warning( + "env_adapter: unrecognized boolean env value %r; treating as False. " + "Use one of: true/false, 1/0, yes/no, on/off.", + val, + ) + return False def _split_csv_list(raw: Optional[str]) -> Optional[List[str]]: @@ -193,8 +212,198 @@ def server_config_overrides_from_env() -> Dict[str, Any]: return o +# Canonical allowlist of every ``JVSPATIAL_*`` environment variable the +# library reads. Anything outside this set is rejected at startup so +# typos (``JVSPATIAL_JWT_SECRET`` for ``JVSPATIAL_JWT_SECRET_KEY`` and +# similar) surface immediately rather than silently no-op'ing. +# SPEC §10.2: "Unknown JVSPATIAL_* keys are rejected at startup to catch +# typos and removed settings." Audit §7.1 closed. +ALLOWED_ENV_KEYS: frozenset[str] = frozenset( + { + # API metadata / server runtime + "JVSPATIAL_TITLE", + "JVSPATIAL_API_TITLE", + "JVSPATIAL_DESCRIPTION", + "JVSPATIAL_API_DESCRIPTION", + "JVSPATIAL_VERSION", + "JVSPATIAL_API_VERSION", + "JVSPATIAL_API_PREFIX", + "JVSPATIAL_API_HEALTH", + "JVSPATIAL_API_ROOT", + "JVSPATIAL_HOST", + "JVSPATIAL_PORT", + "JVSPATIAL_DEBUG", + "JVSPATIAL_LOG_LEVEL", + "JVSPATIAL_GRAPH_ENDPOINT_ENABLED", + "JVSPATIAL_ENVIRONMENT", + "JVSPATIAL_DOCS_DISABLED", + # Database + "JVSPATIAL_DB_TYPE", + "JVSPATIAL_DB_PATH", + "JVSPATIAL_MONGODB_URI", + "JVSPATIAL_MONGODB_DB_NAME", + "JVSPATIAL_MONGODB_MAX_POOL_SIZE", + "JVSPATIAL_MONGODB_MIN_POOL_SIZE", + "JVSPATIAL_DYNAMODB_TABLE_NAME", + "JVSPATIAL_DYNAMODB_REGION", + "JVSPATIAL_DYNAMODB_ENDPOINT_URL", + "JVSPATIAL_DYNAMODB_WAIT_FOR_INDEX", + "JVSPATIAL_AUTO_CREATE_INDEXES", + # Auth + "JVSPATIAL_AUTH_ENABLED", + "JVSPATIAL_AUTH_STRICT_HASHING", + "JVSPATIAL_AUTH_BLACKLIST_FAIL_CLOSED", + "JVSPATIAL_JWT_SECRET_KEY", + "JVSPATIAL_JWT_ALGORITHM", + "JVSPATIAL_JWT_EXPIRE_MINUTES", + "JVSPATIAL_JWT_REFRESH_EXPIRE_DAYS", + "JVSPATIAL_BCRYPT_ROUNDS", + "JVSPATIAL_BCRYPT_ROUNDS_SERVERLESS", + # CORS + "JVSPATIAL_CORS_ENABLED", + "JVSPATIAL_CORS_ORIGINS", + "JVSPATIAL_CORS_METHODS", + "JVSPATIAL_CORS_HEADERS", + # Rate limiting + "JVSPATIAL_RATE_LIMIT_ENABLED", + "JVSPATIAL_RATE_LIMIT_DEFAULT_REQUESTS", + "JVSPATIAL_RATE_LIMIT_DEFAULT_WINDOW", + # File storage + "JVSPATIAL_FILE_STORAGE_ENABLED", + "JVSPATIAL_FILE_STORAGE_PROVIDER", + "JVSPATIAL_FILE_STORAGE_BASE_URL", + "JVSPATIAL_FILE_STORAGE_MAX_SIZE", + "JVSPATIAL_FILE_STORAGE_SERVERLESS_SHARED", + "JVSPATIAL_FILES_ROOT_PATH", + "JVSPATIAL_FILES_PUBLIC_READ", + "JVSPATIAL_FILE_INTERFACE", + "JVSPATIAL_S3_BUCKET_NAME", + "JVSPATIAL_S3_REGION", + "JVSPATIAL_S3_ACCESS_KEY", + "JVSPATIAL_S3_SECRET_KEY", + "JVSPATIAL_S3_ENDPOINT_URL", + "JVSPATIAL_S3_MULTIPART_THRESHOLD", + # URL proxy + "JVSPATIAL_PROXY_ENABLED", + "JVSPATIAL_PROXY_DEFAULT_EXPIRATION", + "JVSPATIAL_PROXY_MAX_EXPIRATION", + "JVSPATIAL_PROXY_PREFIX", + # Cache + "JVSPATIAL_CACHE_BACKEND", + "JVSPATIAL_CACHE_SIZE", + "JVSPATIAL_REDIS_URL", + "JVSPATIAL_REDIS_TTL", + "JVSPATIAL_REDIS_SERIALIZATION", + # Scheduler / deferred / serverless + "JVSPATIAL_SCHEDULER_ENABLED", + "JVSPATIAL_SCHEDULER_INTERVAL", + "JVSPATIAL_DEFERRED_TASK_PROVIDER", + "JVSPATIAL_DEFERRED_INVOKE_DISABLED", + "JVSPATIAL_DEFERRED_INVOKE_SECRET", + "JVSPATIAL_ENABLE_DEFERRED_SAVES", + "JVSPATIAL_AWS_DEFERRED_TRANSPORT", + "JVSPATIAL_AWS_SQS_QUEUE_URL", + "JVSPATIAL_EVENTBRIDGE_LAMBDA_ARN", + "JVSPATIAL_EVENTBRIDGE_ROLE_ARN", + "JVSPATIAL_EVENTBRIDGE_SCHEDULER_ENABLED", + "JVSPATIAL_EVENTBRIDGE_SCHEDULER_GROUP", + "JVSPATIAL_LWA_ENV_DEFAULTS", + # Webhooks + "JVSPATIAL_WEBHOOK_HMAC_ALGORITHM", + "JVSPATIAL_WEBHOOK_HMAC_SECRET", + "JVSPATIAL_WEBHOOK_HTTPS_REQUIRED", + "JVSPATIAL_WEBHOOK_IDEMPOTENCY_TTL", + "JVSPATIAL_WEBHOOK_MAX_PAYLOAD_SIZE", + # Walkers + "JVSPATIAL_WALKER_MAX_STEPS", + "JVSPATIAL_WALKER_MAX_VISITS_PER_NODE", + "JVSPATIAL_WALKER_MAX_EXECUTION_TIME", + "JVSPATIAL_WALKER_MAX_QUEUE_SIZE", + "JVSPATIAL_WALKER_MAX_TRAIL_LENGTH", + "JVSPATIAL_WALKER_PROTECTION_ENABLED", + # Logging + "JVSPATIAL_DB_LOGGING_ENABLED", + "JVSPATIAL_DB_LOGGING_API_ENABLED", + "JVSPATIAL_DB_LOGGING_DB_NAME", + "JVSPATIAL_DB_LOGGING_LEVELS", + "JVSPATIAL_DB_LOG_SERVERLESS_ASYNC", + "JVSPATIAL_DB_LOG_SERVERLESS_JOIN_TIMEOUT", + "JVSPATIAL_LOG_DB_TYPE", + "JVSPATIAL_LOG_DB_NAME", + "JVSPATIAL_LOG_DB_PATH", + "JVSPATIAL_LOG_DB_URI", + "JVSPATIAL_LOG_DB_REGION", + "JVSPATIAL_LOG_DB_TABLE_NAME", + "JVSPATIAL_LOG_DB_ENDPOINT_URL", + # Collections + "JVSPATIAL_COLLECTION_API_KEYS", + "JVSPATIAL_COLLECTION_SCHEDULED_TASKS", + "JVSPATIAL_COLLECTION_SESSIONS", + "JVSPATIAL_COLLECTION_USERS", + "JVSPATIAL_COLLECTION_WEBHOOKS", + "JVSPATIAL_COLLECTION_WEBHOOK_REQUESTS", + # Work-claim helpers + "JVSPATIAL_WORK_CLAIM_STALE_SECONDS", + # Misc + "JVSPATIAL_TEXT_NORMALIZATION_ENABLED", + "JVSPATIAL_EXPOSE_ERROR_DETAILS", + "JVSPATIAL_STRICT_ENV_ALLOWLIST", + } +) + + +def discover_unknown_jvspatial_env_keys() -> List[str]: + """Return any ``JVSPATIAL_*`` env keys not present in :data:`ALLOWED_ENV_KEYS`. + + Pure helper — callers decide the strictness of the response. + """ + return sorted( + k + for k in os.environ + if k.startswith("JVSPATIAL_") and k not in ALLOWED_ENV_KEYS + ) + + +def enforce_env_allowlist() -> None: + """Reject (strict) or warn (default) on unknown ``JVSPATIAL_*`` env keys. + + Toggled by ``JVSPATIAL_STRICT_ENV_ALLOWLIST``: when truthy, unknown + keys raise ``ValueError`` at server startup, surfacing typos + immediately. Default emits a single warning per unknown key per + process so existing deployments don't break on upgrade. + + Closes audit §7.1 / SPEC §10.2. + """ + unknown = discover_unknown_jvspatial_env_keys() + if not unknown: + return + + strict = False + raw_strict = os.environ.get("JVSPATIAL_STRICT_ENV_ALLOWLIST", "").strip() + if raw_strict: + try: + strict = parse_bool(raw_strict) + except ValueError: + strict = False + + if strict: + raise ValueError( + "Unknown JVSPATIAL_* environment variables detected: " + + ", ".join(unknown) + + ". Either remove the variable or add it to ALLOWED_ENV_KEYS in " + "jvspatial/env_adapter.py." + ) + for key in unknown: + logger.warning( + "Unknown JVSPATIAL_* env var %r ignored. Set " + "JVSPATIAL_STRICT_ENV_ALLOWLIST=true to fail-fast on typos.", + key, + ) + + def validate_server_config_requirements(config: Any) -> None: """Raise ``ValueError`` when required settings for enabled features are missing.""" + enforce_env_allowlist() auth = config.auth if auth.auth_enabled: secret = (auth.jwt_secret or "").strip() diff --git a/jvspatial/logging/README.md b/jvspatial/logging/README.md new file mode 100644 index 0000000..1bbf7af --- /dev/null +++ b/jvspatial/logging/README.md @@ -0,0 +1,87 @@ +# jvspatial/logging + +Structured console logging, persistent database logging, custom log levels. + +> **Read first**: [docs/md/logging-service.md](../../docs/md/logging-service.md), [docs/md/custom-log-levels.md](../../docs/md/custom-log-levels.md) + +--- + +## Purpose + +`logging/` covers two surfaces: + +1. **Structured console logging** — `JVSpatialLogger`, optionally backed by `structlog`. Falls back to stdlib `logging` if `structlog` is not installed. +2. **Database logging service** — `BaseLoggingService` persists log records to a backing database via `DBLogHandler`. Useful for audit trails and post-hoc query. + +Custom log levels (`AUDIT`, `SECURITY`, etc.) are supported and integrate with both surfaces. + +## Layout + +``` +logging/ +├── __init__.py # Public surface (this file documents) +├── config.py # Initialization helpers +├── service.py # BaseLoggingService +├── handler.py # DBLogHandler, install_db_log_handler, exception hook +├── models.py # DBLog entity +├── custom_levels.py # add_custom_log_level, CUSTOM_LEVEL_NUMBER +└── endpoints.py # /logs API endpoints (admin) +``` + +## Public API (from `jvspatial.logging`) + +### Database logging + +| Name | What it does | +|---|---| +| `BaseLoggingService` | Service that persists log records | +| `get_logging_service()` | Singleton accessor | +| `DBLog` | Pydantic model for persisted records | +| `get_logging_config`, `initialize_logging_database` | Initialization helpers | +| `DBLogHandler`, `install_db_log_handler`, `install_exception_hook` | Wiring | + +### Custom log levels + +| Name | What it does | +|---|---| +| `add_custom_log_level(name, level_number)` | Register a custom level | +| `get_custom_levels()` | Inventory | +| `is_custom_level(name)` | Predicate | +| `CUSTOM_LEVEL_NUMBER` | Default integer for ad-hoc custom level | + +### Structured console logging + +| Name | What it does | +|---|---| +| `JVSpatialLogger` | Logger with optional structlog backing | +| `StructuredLoggingConfig` | JSON / coloring configuration | +| `PerformanceLogger` | Convenience for op duration logging | +| `SecurityLogger` | Convenience for auth / rate-limit / brute-force events | +| `get_logger(name)` | Factory | +| `configure_logging(...)` | Configure structlog (JSON / colors) | +| `configure_standard_logging(...)` | Configure stdlib handler (level / colors / preserved handlers) | +| `performance_logger`, `security_logger` | Module-global instances | + +## Invariants + +- **Custom levels are global state.** Registering twice is idempotent; registering with a conflicting number is rejected. +- **`SecurityLogger` events feed downstream auth analysis.** Auth attempts, rate-limit hits, and brute-force detections must call into `SecurityLogger` or the equivalent, not silently log elsewhere. +- **`DBLogHandler` persists asynchronously.** Synchronous failures (DB down) must not block the request handler — they downgrade to local logging. +- **`structlog` is optional.** Code must not assume it; use the fallback path when `STRUCTLOG_AVAILABLE` is `False`. +- **Secrets must never reach the log pipeline.** Sanitize before logging — see the 500-response sanitizer pattern. + +## Modification patterns + +- **Adding a custom log level**: call `add_custom_log_level("FOO", 25)` once at startup. Reuse by `logger.log(25, ...)` or extend `JVSpatialLogger`. +- **Persisting a new event type**: extend `DBLog` model or add a sibling model. Update the handler wiring in `handler.py`. +- **Adding a security-relevant log call**: prefer `SecurityLogger` over ad-hoc info logging. Helps downstream audit tooling find events consistently. + +## Related docs + +- [docs/md/logging-service.md](../../docs/md/logging-service.md) +- [docs/md/custom-log-levels.md](../../docs/md/custom-log-levels.md) +- [docs/md/security-operational-notes.md](../../docs/md/security-operational-notes.md) + +## Stability + +All names in the public API tables above are stable. `endpoints.py` (admin `/logs` routes) is wired via the API layer and follows the same stability tier as the rest of `jvspatial/api`. diff --git a/jvspatial/observability/README.md b/jvspatial/observability/README.md new file mode 100644 index 0000000..63b0664 --- /dev/null +++ b/jvspatial/observability/README.md @@ -0,0 +1,55 @@ +# jvspatial/observability + +Metrics protocol, default no-op recorder, OpenTelemetry adapter. + +> **Read first**: [SPEC §14](../../SPEC.md), [docs/md/observability.md](../../docs/md/observability.md) + +--- + +## Purpose + +`observability/` defines a tiny protocol (`MetricsRecorder`) that any metrics backend can implement. The library wires `ObservableDatabase` (internal, in `jvspatial/db/_observable.py`) to emit a structured log line and one metric per DB operation, with WARNING-level elevation past a configurable slow-query threshold. + +## Layout + +``` +observability/ +├── metrics.py # MetricsRecorder Protocol + NullMetricsRecorder +└── otel.py # OpenTelemetryMetricsRecorder (optional, [otel] extra) +``` + +## Public API (from `jvspatial.observability`) + +| Name | What it does | +|---|---| +| `MetricsRecorder` | Protocol implemented by metrics backends | +| `NullMetricsRecorder` | Default no-op | + +From `jvspatial.observability.otel` (requires `pip install jvspatial[otel]`): + +| Name | What it does | +|---|---| +| `OpenTelemetryMetricsRecorder` | OTEL implementation of `MetricsRecorder` | + +## Invariants + +- **The structured log field set from `ObservableDatabase` is public.** Schema: `backend`, `op`, `collection`, `duration_ms`, `success`, `result_count`. Breaking changes require a deprecation cycle. (See [docs/md/stability.md](../../docs/md/stability.md).) +- **`NullMetricsRecorder` is zero-overhead.** Calls dispatch to no-op methods; do not add allocations or branching here. +- **Slow-query threshold elevates the log to WARNING.** Configurable per database via `create_database(..., slow_query_ms=N)`. +- **Four metric names are emitted per DB op**: `jvspatial.db.op.duration_seconds`, `jvspatial.db.op.count`, `jvspatial.db.op.slow_count`, `jvspatial.db.op.result_count`. + +## Modification patterns + +- **Adding a metrics backend**: implement the `MetricsRecorder` protocol. No subclassing required; structural typing applies. Pass via `create_database(..., metrics=YourRecorder())`. +- **Adding a new metric**: extend the protocol with a new method, default-implement on `NullMetricsRecorder` so existing recorders do not break. Document in `docs/md/observability.md` and add to the stability tier table. +- **Changing emitted log fields**: requires a deprecation cycle. Add the new field alongside the old, deprecate the old, remove after one minor cycle. + +## Related docs + +- [docs/md/observability.md](../../docs/md/observability.md) +- [docs/md/benchmarks.md](../../docs/md/benchmarks.md) +- [docs/md/stability.md](../../docs/md/stability.md) + +## Stability + +`MetricsRecorder`, `NullMetricsRecorder`, and the `ObservableDatabase` log field set are all stable. `OpenTelemetryMetricsRecorder` is stable when the `[otel]` extra is installed. diff --git a/jvspatial/observability/__init__.py b/jvspatial/observability/__init__.py index 4561b4a..203e1eb 100644 --- a/jvspatial/observability/__init__.py +++ b/jvspatial/observability/__init__.py @@ -15,12 +15,20 @@ contract and ``docs/md/stability.md`` for the stability tier. """ +from contextvars import ContextVar + from jvspatial.observability.metrics import ( MetricsRecorder, NullMetricsRecorder, ) +# Per-request (or per-logical execution flow) database operation counter. +# ObservableDatabase increments this on every wrapped operation so callers +# can cheaply surface round-trip count in headers/logs. +db_op_counter: ContextVar[int] = ContextVar("jvspatial_db_op_counter", default=0) + __all__ = [ "MetricsRecorder", "NullMetricsRecorder", + "db_op_counter", ] diff --git a/jvspatial/runtime/README.md b/jvspatial/runtime/README.md new file mode 100644 index 0000000..0846bff --- /dev/null +++ b/jvspatial/runtime/README.md @@ -0,0 +1,54 @@ +# jvspatial/runtime + +Runtime capability detection — primarily serverless mode and Lambda Web Adapter glue. + +> **Read first**: [SPEC §11](../../SPEC.md), [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) + +--- + +## Purpose + +`runtime/` answers questions about the deployment environment: "are we in a serverless runtime?", "which provider?", "are we behind Lambda Web Adapter?". Detection results inform mode-dependent defaults (deferred saves, index creation, bcrypt cost) without leaking environment-checking code throughout the library. + +## Layout + +``` +runtime/ +├── serverless.py # is_serverless_mode, detect_serverless_provider, cache reset +├── lwa.py # internal: Lambda Web Adapter glue +└── eventbridge.py # internal: EventBridge wiring helpers +``` + +## Public API (from `jvspatial.runtime`) + +| Name | What it does | +|---|---| +| `is_serverless_mode(config=None)` | Effective mode using precedence (SPEC §11.1) | +| `reset_serverless_mode_cache()` | Clear memoization (tests only) | + +From `jvspatial.runtime.serverless` (re-exported through `jvspatial`): + +| Name | What it does | +|---|---| +| `detect_serverless_provider()` | Returns one of `"aws"`, `"azure"`, `"gcp"`, `"vercel"`, `"unknown"` | + +## Invariants + +- **Detection precedence is fixed**: explicit `config.serverless_mode` → `get_current_server().config.serverless_mode` → `SERVERLESS_MODE` env → auto-detect from platform env vars. Do not bypass with custom env reads. +- **Auto-detection is memoized via `lru_cache`.** Tests must call `reset_serverless_mode_cache()` between cases that toggle env. +- **Provider detection is best-effort.** Falls back to `"unknown"`; do not rely on provider-specific behavior outside of well-known providers. +- **`SERVERLESS_MODE` env values**: `true`, `1`, `yes`, `enabled` (case-insensitive) → True. Everything else → False. + +## Modification patterns + +- **Adding a new serverless provider**: extend `_detect_serverless_mode` and `_detect_serverless_provider` with the relevant env var checks. Add a `ServerlessProvider` literal entry. Update tests in `tests/runtime/`. +- **Adding a new mode-dependent default**: read `is_serverless_mode()` at the point of decision, not at import time. Make the default explicit in the relevant SPEC table (§11.2). + +## Related docs + +- [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) +- [docs/md/production-deployment.md](../../docs/md/production-deployment.md) + +## Stability + +`is_serverless_mode`, `reset_serverless_mode_cache`, and `detect_serverless_provider` are stable. `lwa.py` and `eventbridge.py` are internal — go through the public serverless helpers in `jvspatial.serverless`. diff --git a/jvspatial/runtime/serverless.py b/jvspatial/runtime/serverless.py index 2bd24ed..74cbb23 100644 --- a/jvspatial/runtime/serverless.py +++ b/jvspatial/runtime/serverless.py @@ -8,7 +8,38 @@ def _parse_bool(val: str) -> bool: - return str(val).strip().lower() in ("true", "1", "yes", "enabled") + """Truthy ``SERVERLESS_MODE`` env override. + + Delegates to :func:`jvspatial.env.parse_bool` for the canonical set + (``true/false``, ``1/0``, ``yes/no``, ``on/off``) plus the historical + ``enabled``/``disabled`` aliases kept for backward compatibility + (audit §7.2). + + Unrecognized non-empty values are now logged (audit §7.3): silently + mapping garbage to ``False`` hid typos. Returns ``False`` either way + so existing deployments with typo'd ``SERVERLESS_MODE`` keep their + previous effective behavior — but the warning surfaces the typo. + """ + import logging + + # Late import to avoid circular dependency at module load. + from jvspatial.env import parse_bool + + s = str(val).strip().lower() + if s == "enabled": + return True + if s == "disabled": + return False + try: + return parse_bool(s) + except ValueError: + logging.getLogger(__name__).warning( + "Unrecognized boolean env value %r for SERVERLESS_MODE; " + "treating as False. Use one of: true/false, 1/0, yes/no, " + "on/off, enabled/disabled.", + val, + ) + return False @lru_cache(maxsize=1) diff --git a/jvspatial/serverless/README.md b/jvspatial/serverless/README.md new file mode 100644 index 0000000..c1a847e --- /dev/null +++ b/jvspatial/serverless/README.md @@ -0,0 +1,59 @@ +# jvspatial/serverless + +Deferred task dispatch for serverless environments. Sister to `jvspatial/runtime` (detection). + +> **Read first**: [SPEC §11](../../SPEC.md), [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) + +--- + +## Purpose + +`serverless/` provides a dispatcher for tasks that must run outside the current HTTP invocation: long-running work, retryable steps, fan-out workflows. On AWS this typically routes through Lambda async-invoke or EventBridge; the API is provider-agnostic. + +## Layout + +``` +serverless/ +├── factory.py # dispatch_deferred_task, get_task_scheduler +├── deferred_invoke.py # register_deferred_invoke_handler, dispatch_deferred_invoke, normalize_deferred_envelope +└── tasks/ # Provider-specific scheduler implementations + └── __init__.py +``` + +## Public API (from `jvspatial.serverless`) + +| Name | What it does | +|---|---| +| `dispatch_deferred_task(task_type, payload)` | Enqueue a task for out-of-band execution | +| `get_task_scheduler()` | Return the configured scheduler instance | + +Plus from `jvspatial.serverless.deferred_invoke`: + +| Name | What it does | +|---|---| +| `register_deferred_invoke_handler(task_type, handler)` | Register an async handler for a task type | +| `dispatch_deferred_invoke(task_type, payload)` | Dispatch and immediately invoke registered handler (in-process testing path) | +| `normalize_deferred_envelope(event)` | Flatten provider-specific event envelopes (SQS, EventBridge) | + +## Invariants + +- **Handlers must be idempotent.** The framework provides no exactly-once guarantee. Retries may deliver the same payload multiple times. +- **Handler registration is process-global.** Late registration after the first dispatch is allowed but discouraged. +- **Envelope normalization is provider-aware.** New providers should extend `normalize_deferred_envelope`, not bypass it. +- **Secrets in deferred-invoke headers use constant-time comparison.** (See `deferred_invoke.py`.) +- **Lambda Web Adapter env defaults are set best-effort by `Server`** when LWA is detected. IaC should set them explicitly. (SPEC §11.4) + +## Modification patterns + +- **Adding a provider**: implement a scheduler in `tasks/`, register it via configuration. Extend `normalize_deferred_envelope` if the new provider has a distinct event envelope. +- **Adding a handler**: use `@deferred_invoke_handler("task.name")` or `register_deferred_invoke_handler(...)`. Document idempotency requirements at the call site. +- **Adding a new envelope flattener**: keep flattening in `normalize_deferred_envelope` rather than per-handler. + +## Related docs + +- [docs/md/serverless-mode.md](../../docs/md/serverless-mode.md) +- [docs/md/production-deployment.md](../../docs/md/production-deployment.md) + +## Stability + +`dispatch_deferred_task`, `get_task_scheduler`, `register_deferred_invoke_handler`, `dispatch_deferred_invoke`, `normalize_deferred_envelope` are stable. `tasks/` internals can change. LWA-specific env wiring in `jvspatial/runtime/lwa.py` is internal. diff --git a/jvspatial/serverless/tasks/stub.py b/jvspatial/serverless/tasks/stub.py index 5cd7c81..7087c96 100644 --- a/jvspatial/serverless/tasks/stub.py +++ b/jvspatial/serverless/tasks/stub.py @@ -23,6 +23,13 @@ def schedule( retry_config: Optional[RetryConfig] = None, run_at: Optional[float] = None, ) -> str: - """Log and return a synthetic reference; see base class.""" - logger.warning("%s (task_type=%s)", self._message, task_type) + """Log and return a synthetic reference; see base class. + + Downgraded to DEBUG so a misconfigured serverless deployment does + not flood CloudWatch with one WARNING per dispatch. The + once-per-process startup error from + ``serverless.factory._note_noop_in_serverless`` is sufficient + (audit §7.14 / SPEC §11.2). + """ + logger.debug("%s (task_type=%s)", self._message, task_type) return f"noop-{uuid.uuid4()}" diff --git a/jvspatial/storage/README.md b/jvspatial/storage/README.md new file mode 100644 index 0000000..0cbed4a --- /dev/null +++ b/jvspatial/storage/README.md @@ -0,0 +1,71 @@ +# jvspatial/storage + +File storage system: interfaces, security layer, version model, URL proxy manager. + +> **Read first**: [SPEC §12](../../SPEC.md), [docs/md/file-storage-architecture.md](../../docs/md/file-storage-architecture.md) + +--- + +## Purpose + +`storage/` provides a secure abstraction over file backends. Every upload passes a five-stage path sanitizer and a content-based MIME validator before reaching the backend. Local-filesystem and S3 backends ship today; the interface is open for additional providers. + +## Layout + +``` +storage/ +├── interfaces/ +│ ├── base.py # FileStorageInterface ABC +│ ├── local.py # LocalFileInterface +│ └── s3.py # S3FileInterface (optional extra) +├── security/ +│ ├── path_sanitizer.py # Five-stage validation +│ └── validator.py # Content-based MIME validation +├── managers/ +│ ├── proxy.py # URL proxy lifecycle +│ └── *.py # Other management helpers +├── models.py # Pydantic file/version/proxy models +└── exceptions.py # Storage-specific exceptions +``` + +## Public API (from `jvspatial.storage`) + +| Name | What it does | +|---|---| +| `create_storage(provider, **kwargs)` | Factory entry point (provider ∈ `local`, `s3`) | +| `create_default_storage()` | Env-driven default (`JVSPATIAL_FILE_INTERFACE`) | +| `FileStorageInterface` | ABC for storage backends | +| `LocalFileInterface` | Filesystem backend | +| `S3FileInterface` | S3 backend (requires boto3) | +| `PathSanitizer` | Path validation | +| `FileValidator` | MIME / size validation | +| `URLProxy`, `URLProxyManager`, `get_proxy_manager` | Time-limited / one-time download URLs | +| `StorageError`, `PathTraversalError`, `InvalidPathError`, `ValidationError`, `FileNotFoundError`, `FileSizeLimitError`, `InvalidMimeTypeError`, `StorageProviderError`, `AccessDeniedError` | Exception types | + +## Invariants + +- **Every upload passes `PathSanitizer` then `FileValidator`.** No bypass for "trusted" callers. (`security/path_sanitizer.py`, `security/validator.py`) +- **MIME validation is content-based, not extension-based.** Uses `python-magic`. Renaming `.exe` → `.txt` does not bypass blocking. +- **Path sanitization is five stages**: regex blocklist → normalization with re-check → hidden-file allowlist → symlink resolution → base-dir confinement. +- **Internal directory markers bypass user-input checks via metadata only.** Not via filenames. (`storage/security/path_sanitizer.py`) +- **Atomic writes for local backend.** Same `temp + fsync + rename + fsync(dir)` helper as JsonDB. +- **S3 multipart at ≥8 MiB.** Configurable via constructor or `JVSPATIAL_S3_MULTIPART_THRESHOLD`. +- **S3 throttle retry** uses the shared retry helper with exponential backoff + jitter. + +## Modification patterns + +- **Adding a backend**: implement `FileStorageInterface`. Wire into `create_storage(provider="...")` and the storage manager. Add tests under `tests/storage/`. +- **Changing path sanitization**: review all five stages; add tests to `tests/storage/test_path_sanitizer.py`. Each stage has its own test surface. +- **Extending allowed MIME types**: update `FileValidator.ALLOWED_MIME_TYPES` and add an integration test that round-trips the new type. +- **Adding a security rule**: prefer adding to the sanitizer (paths) or validator (content) rather than ad-hoc in the backend. + +## Related docs + +- [docs/md/file-storage-architecture.md](../../docs/md/file-storage-architecture.md) +- [docs/md/file-storage-usage.md](../../docs/md/file-storage-usage.md) +- [docs/md/security-review.md](../../docs/md/security-review.md) +- [docs/md/security-operational-notes.md](../../docs/md/security-operational-notes.md) + +## Stability + +`create_storage`, `FileStorageInterface`, `LocalFileInterface`, `S3FileInterface`, the security classes, and the exception hierarchy are public. `managers/` internals can change between minor versions; cross them through `get_proxy_manager` and the documented API. diff --git a/jvspatial/storage/interfaces/local.py b/jvspatial/storage/interfaces/local.py index 6bbc9b6..b0e4d28 100644 --- a/jvspatial/storage/interfaces/local.py +++ b/jvspatial/storage/interfaces/local.py @@ -159,6 +159,30 @@ def _get_full_path(self, file_path: str) -> Path: return full_path + def _sanitized_version_base(self, file_path: str) -> Path: + """Return a sandboxed base path for versioning helpers. + + ``create_version`` / ``get_version`` / ``list_versions`` / + ``delete_version`` / ``get_latest_version`` all derive ``.versions`` + and ``.latest`` paths from ``file_path``. Without sanitization here, + a caller-supplied ``file_path`` like ``../../etc/passwd`` escapes + ``self.root_dir`` (audit §4.2 / SPEC §15.1). Centralizing the check + in one helper guarantees uniform coverage across every versioning + entry point. + """ + sanitized = PathSanitizer.sanitize_path(file_path, base_dir=str(self.root_dir)) + base = self.root_dir / sanitized + try: + # ``resolve(strict=False)`` traverses any existing symlinks but + # tolerates non-existent leaves (versioned bases often don't + # exist yet). The ``relative_to`` check then enforces the root. + resolved = base.resolve() + resolved.relative_to(self.root_dir) + except (ValueError, RuntimeError): + logger.error(f"Version path escape attempt blocked: {file_path}") + raise PathTraversalError("Path escapes root directory", path=file_path) + return base + def _http_file_url(self, file_path: str) -> str: """Full URL for HTTP GET ``{FILES_ROOT}/{file_path}`` (FileStorageService).""" return f"{self.base_url.rstrip('/')}{APIRoutes.FILES_ROOT}/{file_path}" @@ -203,8 +227,9 @@ async def create_version( if version is None: version = f"v{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" - # Create version directory (use .versions suffix to avoid conflicts) - version_dir = self.root_dir / f"{file_path}.versions" + # Sanitize ``file_path`` and derive sandboxed version paths. + version_base = self._sanitized_version_base(file_path) + version_dir = version_base.with_name(version_base.name + ".versions") await to_thread(version_dir.mkdir, parents=True, exist_ok=True) version_metadata = { @@ -217,7 +242,7 @@ async def create_version( version_file = version_dir / f"{version}.bin" metadata_file = version_dir / f"{version}.meta.json" - latest_file = self.root_dir / f"{file_path}.latest" + latest_file = version_base.with_name(version_base.name + ".latest") # 1) Write content atomically (fully durable). await to_thread(atomic_write_bytes, version_file, content) @@ -249,7 +274,9 @@ async def get_version(self, file_path: str, version: str) -> Dict[str, Any]: Returns: Dictionary with version information and content """ - version_file = self.root_dir / f"{file_path}.versions" / f"{version}.bin" + version_base = self._sanitized_version_base(file_path) + version_dir = version_base.with_name(version_base.name + ".versions") + version_file = version_dir / f"{version}.bin" if not await to_thread(version_file.exists): return None @@ -257,7 +284,7 @@ async def get_version(self, file_path: str, version: str) -> Dict[str, Any]: content = await to_thread(version_file.read_bytes) # Try to get metadata - metadata_file = self.root_dir / f"{file_path}.versions" / f"{version}.meta.json" + metadata_file = version_dir / f"{version}.meta.json" metadata = {} if await to_thread(metadata_file.exists): try: @@ -285,7 +312,8 @@ async def list_versions(self, file_path: str) -> List[Dict[str, Any]]: Returns: List of version information dictionaries """ - versions_dir = self.root_dir / f"{file_path}.versions" + version_base = self._sanitized_version_base(file_path) + versions_dir = version_base.with_name(version_base.name + ".versions") if not await to_thread(versions_dir.exists): return [] @@ -324,7 +352,8 @@ async def delete_version(self, file_path: str, version: str) -> bool: Returns: True if version was deleted, False otherwise """ - versions_dir = self.root_dir / f"{file_path}.versions" + version_base = self._sanitized_version_base(file_path) + versions_dir = version_base.with_name(version_base.name + ".versions") if not await to_thread(versions_dir.exists): return False @@ -353,7 +382,8 @@ async def get_latest_version(self, file_path: str) -> Optional[Dict[str, Any]]: Returns: Latest version information dictionary, or None if no versions exist """ - latest_file = self.root_dir / f"{file_path}.latest" + version_base = self._sanitized_version_base(file_path) + latest_file = version_base.with_name(version_base.name + ".latest") if not await to_thread(latest_file.exists): return None diff --git a/jvspatial/storage/security/path_sanitizer.py b/jvspatial/storage/security/path_sanitizer.py index 6c51830..0c662fb 100644 --- a/jvspatial/storage/security/path_sanitizer.py +++ b/jvspatial/storage/security/path_sanitizer.py @@ -69,6 +69,49 @@ class PathSanitizer: # Known internal storage markers (directory placeholders, sandbox roots) _ALLOWED_HIDDEN_SEGMENTS = frozenset({".jvdirectory", ".jvagent_sandbox"}) + # Windows-reserved filename stems. CMD / Win32 reject these regardless + # of extension, so allowing them in a cross-platform storage layer + # creates write-failure footguns and lets an attacker DoS a Windows + # host by uploading e.g. ``CON.txt`` (audit §4.18 / SPEC §15.1). + _WINDOWS_RESERVED_NAMES = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", + } + ) + + @classmethod + def _is_windows_reserved(cls, part: str) -> bool: + """Return True when ``part`` matches a Windows-reserved name. + + Checked regardless of host OS so cross-platform storage is + uniformly safe. + """ + # Match the stem before any extension. ``CON.txt`` is reserved + # because Windows resolves the device first. + stem = part.split(".", 1)[0].upper() + return stem in cls._WINDOWS_RESERVED_NAMES + @classmethod def sanitize_path( cls, file_path: str, base_dir: Optional[str] = None, allow_hidden: bool = False @@ -170,6 +213,13 @@ def sanitize_path( path=file_path, ) + # Reject Windows-reserved filenames cross-platform (audit §4.18). + if cls._is_windows_reserved(part): + raise InvalidPathError( + f"Reserved Windows filename not allowed: {part}", + path=file_path, + ) + # Check filename length if len(part) > cls.MAX_FILENAME_LENGTH: raise InvalidPathError( @@ -238,6 +288,10 @@ def sanitize_filename(cls, filename: str, allow_hidden: bool = False) -> str: filename = re.sub(r"[^a-zA-Z0-9_\-\.]", "_", filename) logger.info("Filename sanitized by removing invalid characters") + # Reject Windows-reserved filenames cross-platform (audit §4.18). + if cls._is_windows_reserved(filename): + raise InvalidPathError(f"Reserved Windows filename not allowed: {filename}") + # Enforce length limit if len(filename) > cls.MAX_FILENAME_LENGTH: # Preserve extension diff --git a/jvspatial/utils/stability.py b/jvspatial/utils/stability.py index fb263da..0831f8c 100644 --- a/jvspatial/utils/stability.py +++ b/jvspatial/utils/stability.py @@ -89,6 +89,23 @@ def _emit_once(name: str, message: str) -> None: ) +def emit_experimental_once(name: str, message: str = "") -> None: + """Public re-export of the once-per-process experimental warning hook. + + Some opt-in surfaces — e.g. ``JsonDBTransaction(best_effort=True)`` — + need to emit the experimental warning without going through the + ``@experimental`` decorator (because the decision happens on a + constructor flag, not the symbol itself). Use this hook instead of + reaching into the underscore-prefixed implementation (audit §7.7 / + SPEC §18 stability-tier discipline). + + Args: + name: Stable identifier for the API (deduplicated per-process). + message: Optional context appended to the warning body. + """ + _emit_once(name, message) + + def experimental( name: Optional[str] = None, note: str = "", diff --git a/jvspatial/version.py b/jvspatial/version.py index 2b835ec..d3e0d82 100644 --- a/jvspatial/version.py +++ b/jvspatial/version.py @@ -9,4 +9,4 @@ # - MAJOR: Breaking changes # - MINOR: New features, backward compatible # - PATCH: Bug fixes, backward compatible -__version__ = "0.0.7" +__version__ = "0.0.8" diff --git a/tests/api/components/test_app_builder_docs_disable.py b/tests/api/components/test_app_builder_docs_disable.py new file mode 100644 index 0000000..4069138 --- /dev/null +++ b/tests/api/components/test_app_builder_docs_disable.py @@ -0,0 +1,86 @@ +"""AppBuilder — JVSPATIAL_DOCS_DISABLED unpublish behavior. + +Verifies that the single env knob fully removes the documentation surface +(`/docs`, `/redoc`, `/openapi.json`, `/docs/oauth2-redirect`) without +otherwise affecting the application. +""" + +from __future__ import annotations + +from typing import Optional + +import pytest +from fastapi.testclient import TestClient + +from jvspatial.api.components.app_builder import AppBuilder +from jvspatial.api.config import ServerConfig + + +def _build_client(config: Optional[ServerConfig] = None) -> TestClient: + """Construct an AppBuilder-built FastAPI app + add a probe route.""" + cfg = config or ServerConfig() + app = AppBuilder(cfg).create_app() + + @app.get("/__probe") + def _probe() -> dict: + return {"ok": True} + + return TestClient(app) + + +def test_docs_published_by_default() -> None: + """Without the env flag, /docs and /openapi.json render.""" + client = _build_client() + assert client.get("/docs").status_code == 200 + assert client.get("/redoc").status_code == 200 + assert client.get("/openapi.json").status_code == 200 + + +@pytest.mark.parametrize("flag", ["1", "true", "True", "yes", "on"]) +def test_docs_unpublished_via_env_flag(monkeypatch, flag: str) -> None: + """`JVSPATIAL_DOCS_DISABLED=` returns 404 on every doc surface.""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", flag) + client = _build_client() + assert client.get("/docs").status_code == 404 + assert client.get("/redoc").status_code == 404 + assert client.get("/openapi.json").status_code == 404 + # Swagger's OAuth2 redirect helper is also stripped. + assert client.get("/docs/oauth2-redirect").status_code == 404 + + +def test_app_routes_unaffected_when_docs_disabled(monkeypatch) -> None: + """Disabling docs leaves application routes alone.""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", "1") + client = _build_client() + r = client.get("/__probe") + assert r.status_code == 200 + assert r.json() == {"ok": True} + + +@pytest.mark.parametrize("flag", ["", "0", "false", "no", "off", "garbage"]) +def test_falsey_or_unrelated_values_keep_docs_published(monkeypatch, flag: str) -> None: + """Only the documented truthy values disable docs.""" + if flag: + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", flag) + else: + monkeypatch.delenv("JVSPATIAL_DOCS_DISABLED", raising=False) + client = _build_client() + assert client.get("/docs").status_code == 200 + assert client.get("/openapi.json").status_code == 200 + + +def test_disabled_state_strips_openapi_schema_endpoint(monkeypatch) -> None: + """`/openapi.json` is the metadata leak that matters most in prod — + confirm it is gone (not just inaccessible).""" + monkeypatch.setenv("JVSPATIAL_DOCS_DISABLED", "1") + cfg = ServerConfig() + app = AppBuilder(cfg).create_app() + + # FastAPI exposes the openapi mount via app.openapi_url; should be None. + assert app.openapi_url is None + # And the spec generator returns nothing usable when openapi_url is off. + schema = app.openapi() + # FastAPI still synthesizes a schema in-memory, but the route is not + # registered — the public surface is what matters and we already + # asserted 404 above. Sanity-check the schema is a dict (no crash). + assert isinstance(schema, dict) diff --git a/tests/api/components/test_database_configurator.py b/tests/api/components/test_database_configurator.py index 3fd89c6..05b72c7 100644 --- a/tests/api/components/test_database_configurator.py +++ b/tests/api/components/test_database_configurator.py @@ -11,6 +11,12 @@ class TestDatabaseConfigurator: """Test DatabaseConfigurator functionality.""" + @pytest.fixture(autouse=True) + def _isolate_observability_env(self, monkeypatch): + """create_database observability kwargs must not leak from host .env.""" + monkeypatch.delenv("JVSPATIAL_OBSERVABILITY_ENABLED", raising=False) + monkeypatch.delenv("JVSPATIAL_SLOW_QUERY_MS", raising=False) + @pytest.fixture def config(self): """Create test server config.""" @@ -62,6 +68,8 @@ async def test_initialize_graph_context_json(self, configurator): mock_create.assert_called_once_with( db_type="json", base_path="./test_db", + observe=False, + slow_query_ms=100.0, ) @pytest.mark.asyncio @@ -107,6 +115,8 @@ async def test_initialize_graph_context_mongodb(self, monkeypatch): db_type="mongodb", uri="mongodb://localhost:27017", db_name="testdb", + observe=False, + slow_query_ms=100.0, ) def test_resolve_mongodb_uses_config_uri_only(self, monkeypatch): @@ -134,6 +144,54 @@ def test_resolve_mongodb_config_only_when_env_absent(self, monkeypatch): assert uri == "mongodb://cfg:27017" assert db_name == "cfgdb" + def test_resolve_observability_kwargs_defaults(self, configurator): + assert configurator._resolve_observability_kwargs() == { + "observe": False, + "slow_query_ms": 100.0, + } + + def test_resolve_observability_kwargs_from_env(self, configurator, monkeypatch): + monkeypatch.setenv("JVSPATIAL_OBSERVABILITY_ENABLED", "true") + monkeypatch.setenv("JVSPATIAL_SLOW_QUERY_MS", "250") + assert configurator._resolve_observability_kwargs() == { + "observe": True, + "slow_query_ms": 250.0, + } + + @pytest.mark.asyncio + async def test_initialize_graph_context_passes_observability_from_env( + self, configurator, monkeypatch + ): + monkeypatch.setenv("JVSPATIAL_OBSERVABILITY_ENABLED", "1") + monkeypatch.setenv("JVSPATIAL_SLOW_QUERY_MS", "42") + + with patch( + "jvspatial.api.components.database_configurator.create_database" + ) as mock_create: + mock_create.return_value = MagicMock() + with patch( + "jvspatial.api.components.database_configurator.get_database_manager", + side_effect=RuntimeError(), + ): + with patch( + "jvspatial.api.components.database_configurator.set_database_manager" + ): + with patch( + "jvspatial.api.components.database_configurator.GraphContext", + return_value=MagicMock(), + ): + with patch( + "jvspatial.api.components.database_configurator.set_default_context" + ): + configurator.initialize_graph_context() + + mock_create.assert_called_once_with( + db_type="json", + base_path="./test_db", + observe=True, + slow_query_ms=42.0, + ) + def test_resolve_mongodb_empty_config_uses_localhost_jvdb(self, monkeypatch): monkeypatch.delenv("JVSPATIAL_MONGODB_URI", raising=False) monkeypatch.delenv("JVSPATIAL_MONGODB_DB_NAME", raising=False) @@ -187,6 +245,8 @@ async def test_initialize_graph_context_dynamodb(self): endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, + observe=False, + slow_query_ms=100.0, ) @pytest.mark.asyncio @@ -250,6 +310,8 @@ async def test_db_path_resolve_app_resolves_relative_path(self, configurator): mock_create.assert_called_once_with( db_type="json", base_path="/opt/backend/track75_db", + observe=False, + slow_query_ms=100.0, ) def test_resolve_db_path_absolute_unchanged(self, configurator): diff --git a/tests/api/middleware/test_security_headers.py b/tests/api/middleware/test_security_headers.py new file mode 100644 index 0000000..b719bc1 --- /dev/null +++ b/tests/api/middleware/test_security_headers.py @@ -0,0 +1,88 @@ +"""SecurityHeadersMiddleware — per-path CSP behavior. + +Production unpublish lives elsewhere: `JVSPATIAL_DOCS_DISABLED=1` in +`AppBuilder.create_app` removes the routes outright, so this middleware +only ever sees app routes. When docs ARE published (dev/staging) the +relaxed `_DOCS_CSP` permits the Swagger UI / ReDoc CDN bundles. +""" + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from jvspatial.api.middleware.manager import ( + _DEFAULT_CSP, + _DOCS_CSP, + SecurityHeadersMiddleware, +) + + +def _build_client() -> TestClient: + """Spin a minimal FastAPI app + a docs sub-path probe.""" + app = FastAPI() + app.add_middleware(SecurityHeadersMiddleware) + + @app.get("/health") + def _health() -> dict: + return {"ok": True} + + @app.get("/docs/extra") + def _docs_subpath() -> dict: + return {"docs": True} + + return TestClient(app) + + +def test_app_route_gets_strict_csp() -> None: + """Application routes receive the locked-down default CSP.""" + client = _build_client() + r = client.get("/health") + assert r.headers["content-security-policy"] == _DEFAULT_CSP + + +def test_docs_route_gets_relaxed_csp() -> None: + """`/docs` (FastAPI default) gets the CDN-permitting CSP so Swagger UI loads.""" + app = FastAPI() # FastAPI auto-mounts /docs and /openapi.json + app.add_middleware(SecurityHeadersMiddleware) + client = TestClient(app) + + r = client.get("/docs") + assert r.status_code == 200 + assert r.headers["content-security-policy"] == _DOCS_CSP + + r2 = client.get("/openapi.json") + assert r2.status_code == 200 + assert r2.headers["content-security-policy"] == _DOCS_CSP + + +def test_docs_subpath_also_relaxed() -> None: + """Docs path matching includes sub-paths (e.g. /docs/oauth2-redirect).""" + client = _build_client() + r = client.get("/docs/extra") + assert r.headers["content-security-policy"] == _DOCS_CSP + + +def test_constant_security_headers_always_present() -> None: + """X-Content-Type-Options + X-Frame-Options ride along on every response.""" + client = _build_client() + r = client.get("/health") + assert r.headers["x-content-type-options"] == "nosniff" + assert r.headers["x-frame-options"] == "DENY" + + +def test_hsts_off_by_default() -> None: + client = _build_client() + r = client.get("/health") + assert "strict-transport-security" not in {k.lower() for k in r.headers} + + +def test_hsts_on_when_constructor_flag_enabled() -> None: + app = FastAPI() + app.add_middleware(SecurityHeadersMiddleware, hsts_enabled=True) + + @app.get("/x") + def _x() -> dict: + return {"ok": True} + + client = TestClient(app) + r = client.get("/x") + assert "max-age=31536000" in r.headers["strict-transport-security"] diff --git a/tests/api/test_cors_wildcard_and_error_detail_audit.py b/tests/api/test_cors_wildcard_and_error_detail_audit.py new file mode 100644 index 0000000..6d9a5c7 --- /dev/null +++ b/tests/api/test_cors_wildcard_and_error_detail_audit.py @@ -0,0 +1,63 @@ +"""CORS wildcard warning + EXPOSE_ERROR_DETAILS prod guard. + +Audit §4.10 / §4.12 / SPEC §15.4-§15.5. +""" + +import logging +import os +from unittest.mock import patch + +from jvspatial.api.components.error_handler import _expose_error_details_to_clients +from jvspatial.api.config_groups import CORSConfig + + +def test_wildcard_origin_emits_warning(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_origins=["*"]) + assert any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_wildcard_with_opt_in_silences_warning(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_origins=["*"], cors_allow_wildcard=True) + assert not any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_disabled_cors_does_not_warn(caplog): + caplog.set_level(logging.WARNING, logger="jvspatial.api.config_groups") + CORSConfig(cors_enabled=False, cors_origins=["*"]) + assert not any( + "wildcard origins detected" in record.getMessage() for record in caplog.records + ) + + +def test_expose_error_details_suppressed_in_production(): + """Production-marked runtime ignores JVSPATIAL_EXPOSE_ERROR_DETAILS.""" + with patch.dict( + os.environ, + { + "JVSPATIAL_EXPOSE_ERROR_DETAILS": "true", + "JVSPATIAL_ENVIRONMENT": "production", + }, + clear=False, + ): + # Reset the once-per-process flag so the warning re-arms. + if hasattr(_expose_error_details_to_clients, "_warned"): + delattr(_expose_error_details_to_clients, "_warned") + assert _expose_error_details_to_clients() is False + + +def test_expose_error_details_honored_outside_production(): + with patch.dict( + os.environ, + { + "JVSPATIAL_EXPOSE_ERROR_DETAILS": "true", + "JVSPATIAL_ENVIRONMENT": "development", + }, + clear=False, + ): + assert _expose_error_details_to_clients() is True diff --git a/tests/api/test_deferred_invoke_fail_closed_audit.py b/tests/api/test_deferred_invoke_fail_closed_audit.py new file mode 100644 index 0000000..24c3cb2 --- /dev/null +++ b/tests/api/test_deferred_invoke_fail_closed_audit.py @@ -0,0 +1,50 @@ +"""Deferred-invoke fail-closed when secret unset (audit §4.16 / SPEC §15.2).""" + +import os +from unittest.mock import MagicMock, patch + +from jvspatial.api.deferred_invoke_route import _deferred_invoke_secret_ok + + +def _fake_request(headers: dict) -> MagicMock: + req = MagicMock() + req.headers.get = lambda k, default=None: headers.get(k, default) + req.headers.__getitem__ = lambda _self, k: headers[k] + return req + + +def test_no_secret_set_denies_access(): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("JVSPATIAL_DEFERRED_INVOKE_SECRET", None) + req = _fake_request({}) + assert _deferred_invoke_secret_ok(req) is False + + +def test_matching_header_allows(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"X-JVSPATIAL-Deferred-Authorize": "shh"}) + assert _deferred_invoke_secret_ok(req) is True + + +def test_matching_bearer_allows(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"Authorization": "Bearer shh"}) + assert _deferred_invoke_secret_ok(req) is True + + +def test_mismatched_secret_denies(): + with patch.dict( + os.environ, + {"JVSPATIAL_DEFERRED_INVOKE_SECRET": "shh"}, # pragma: allowlist secret + clear=False, + ): + req = _fake_request({"X-JVSPATIAL-Deferred-Authorize": "wrong"}) + assert _deferred_invoke_secret_ok(req) is False diff --git a/tests/api/test_env_allowlist_audit.py b/tests/api/test_env_allowlist_audit.py new file mode 100644 index 0000000..519d334 --- /dev/null +++ b/tests/api/test_env_allowlist_audit.py @@ -0,0 +1,73 @@ +"""JVSPATIAL_* env allowlist enforcement (audit §7.1 / SPEC §10.2). + +SPEC §10.2 promises "Unknown JVSPATIAL_* keys are rejected at startup +to catch typos." The earlier implementation only read enumerated keys +in ``env_adapter``; stray ``JVSPATIAL_*`` env vars went silently +ignored. ``enforce_env_allowlist`` now scans the environment and +warns (default) or raises (strict mode) on unknown keys. +""" + +import os +from unittest.mock import patch + +import pytest + +from jvspatial.env_adapter import ( + ALLOWED_ENV_KEYS, + discover_unknown_jvspatial_env_keys, + enforce_env_allowlist, +) + + +def test_discover_unknown_returns_empty_with_known_keys(): + with patch.dict(os.environ, {"JVSPATIAL_DB_TYPE": "json"}, clear=False): + assert "JVSPATIAL_DB_TYPE" not in discover_unknown_jvspatial_env_keys() + + +def test_discover_unknown_returns_typo_key(): + with patch.dict( + os.environ, + {"JVSPATIAL_JWT_SECRET": "oops"}, # pragma: allowlist secret - typo + clear=False, + ): + unknown = discover_unknown_jvspatial_env_keys() + assert "JVSPATIAL_JWT_SECRET" in unknown + + +def test_enforce_warns_by_default(caplog): + with patch.dict( + os.environ, + {"JVSPATIAL_JWT_SECRET": "oops"}, # pragma: allowlist secret - typo + clear=False, + ): + # No exception expected — default is warn. + enforce_env_allowlist() + # Restoring caplog level for the helper logger is enough — we don't + # assert on the captured record because the helper uses + # ``logger.warning`` directly via ``logging.getLogger`` and caplog + # propagation depends on pytest configuration. + + +def test_enforce_raises_in_strict_mode(): + with patch.dict( + os.environ, + { + "JVSPATIAL_JWT_SECRET": "oops", # pragma: allowlist secret - typo + "JVSPATIAL_STRICT_ENV_ALLOWLIST": "true", + }, + clear=False, + ): + with pytest.raises(ValueError, match="Unknown JVSPATIAL_"): + enforce_env_allowlist() + + +def test_allowlist_contains_canonical_keys(): + # Spot-check: a handful of well-known keys must be present. + expected = { + "JVSPATIAL_DB_TYPE", + "JVSPATIAL_JWT_SECRET_KEY", + "JVSPATIAL_DOCS_DISABLED", + "JVSPATIAL_WALKER_MAX_STEPS", + "JVSPATIAL_CORS_ORIGINS", + } + assert expected.issubset(ALLOWED_ENV_KEYS) diff --git a/tests/api/test_request_detection_pep563_audit.py b/tests/api/test_request_detection_pep563_audit.py new file mode 100644 index 0000000..7f6b732 --- /dev/null +++ b/tests/api/test_request_detection_pep563_audit.py @@ -0,0 +1,57 @@ +"""Request-parameter detection with ``from __future__ import annotations``. + +Upstream report: ``wrap_function_with_params`` relied on raw +``param.annotation`` values. With PEP 563 (or quoted forward refs) +the annotation is a *string* like ``"Request"`` — never matches +``FastAPIRequest`` / ``StarletteRequest`` and never matches the +``__name__`` / ``__module__`` heuristic. Result: the wrapper failed +to detect the caller's Request parameter and corrupted the route +signature. + +``ParameterModelFactory._create_function_model`` already calls +``typing.get_type_hints(func)``; this fix mirrors that resolution in +``_find_request_parameter`` so both paths agree. +""" + +from __future__ import annotations + +import inspect + +from fastapi import Request + +from jvspatial.api.decorators.function_wrappers import _find_request_parameter + + +def _handler_with_pep563(request: Request, x: int) -> dict: + """Function whose annotations are strings under PEP 563.""" + return {"x": x} + + +def _handler_no_request(x: int, y: int) -> dict: + return {"x": x, "y": y} + + +def test_request_detected_under_pep563_when_func_passed(): + sig = inspect.signature(_handler_with_pep563) + # Raw annotation IS a string under ``from __future__ import annotations``. + assert isinstance(sig.parameters["request"].annotation, str) + + # ``_find_request_parameter`` resolves type hints when ``func`` is supplied. + has_request, name = _find_request_parameter(sig, _handler_with_pep563) + assert has_request is True + assert name == "request" + + +def test_no_request_param_returns_false(): + sig = inspect.signature(_handler_no_request) + has_request, name = _find_request_parameter(sig, _handler_no_request) + assert has_request is False + assert name is None + + +def test_back_compat_without_func_argument(): + """Calling without ``func`` still works (legacy signature).""" + sig = inspect.signature(_handler_no_request) + has_request, _ = _find_request_parameter(sig) + # No request param: still False either way. + assert has_request is False diff --git a/tests/api/test_webhook_hmac_audit_fix.py b/tests/api/test_webhook_hmac_audit_fix.py new file mode 100644 index 0000000..ebbe117 --- /dev/null +++ b/tests/api/test_webhook_hmac_audit_fix.py @@ -0,0 +1,47 @@ +"""HMAC verification regression coverage (audit §4.3 / SPEC §15.2). + +``verify_hmac_signature`` previously sliced 7 chars off +``expected_signature`` before comparison, while the incoming signature +had already had its prefix stripped. The mismatched lengths fed into +``hmac.compare_digest`` always returned False — webhook signature auth +rejected every request. +""" + +import pytest + +from jvspatial.api.integrations.webhooks.utils import ( + generate_hmac_signature, + verify_hmac_signature, +) + + +def test_valid_hmac_with_default_prefix_verifies(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + signature_with_prefix = generate_hmac_signature(payload, secret) + assert signature_with_prefix.startswith("sha256=") + + assert verify_hmac_signature(payload, signature_with_prefix, secret) is True + + +def test_valid_hmac_without_prefix_verifies(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + bare = generate_hmac_signature(payload, secret, prefix="") + + assert verify_hmac_signature(payload, bare, secret, prefix="") is True + + +def test_invalid_hmac_rejects(): + payload = b'{"event":"ping"}' + secret = "shared-secret" # pragma: allowlist secret + # Tampered signature + bad = "sha256=" + ("0" * 64) + assert verify_hmac_signature(payload, bad, secret) is False + + +def test_empty_signature_or_secret_rejects(): + assert ( + verify_hmac_signature(b"x", "", "secret") is False + ) # pragma: allowlist secret + assert verify_hmac_signature(b"x", "sha256=abc", "") is False diff --git a/tests/core/test_entity_name_override.py b/tests/core/test_entity_name_override.py new file mode 100644 index 0000000..264c56d --- /dev/null +++ b/tests/core/test_entity_name_override.py @@ -0,0 +1,101 @@ +"""Tests for ``__entity_name__`` per-class override of the persisted entity discriminator. + +Covers the case where two unrelated ``Node`` (or ``Object``) subclasses share a +Python class name and must remain distinguishable at the storage layer (e.g. +host-app ``App`` vs library ``App``). Setting ``__entity_name__`` on one of +them decouples ``cls.__name__`` from the ``entity`` field jvspatial uses to +discriminate rows. +""" + +import tempfile +import uuid +from unittest.mock import patch + +import pytest + +from jvspatial.core.context import GraphContext +from jvspatial.core.entities import Node +from jvspatial.core.utils import _class_entity_name, find_subclass_by_name +from jvspatial.db import create_database + + +# Two classes with the same Python ``__name__`` but distinct entity discriminators. +class App(Node): + """Host-app App. Persists with the override entity name.""" + + __entity_name__ = "HostApp" + title: str = "" + + +# Define a second class also literally named ``App`` in this module's local scope +# by reusing the class statement. Pytest's collector preserves the binding so we +# rebind via type() to keep both alive. +_LibApp = type( + "App", + (Node,), + { + "__module__": __name__, + "__doc__": "Library App. Persists with default entity name 'App'.", + "__annotations__": {"label": str}, + "label": "", + }, +) + + +@pytest.fixture +def json_context(): + with tempfile.TemporaryDirectory() as tmpdir: + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + yield GraphContext(database=database) + + +def test_entity_name_helper_honors_override(): + assert _class_entity_name(App) == "HostApp" + assert _class_entity_name(_LibApp) == "App" + + # Subclasses without their own override inherit ``__name__`` semantics, + # NOT the parent's override. + class ChildOfHost(App): + pass + + assert _class_entity_name(ChildOfHost) == "ChildOfHost" + + +def test_find_subclass_by_name_routes_by_entity_name(): + # ``App`` (host) matches lookup for "HostApp", not "App". + assert find_subclass_by_name(Node, "HostApp") is App + # ``_LibApp`` (the other class also named App in Python) matches "App". + assert find_subclass_by_name(Node, "App") is _LibApp + + +@pytest.mark.asyncio +async def test_persisted_entity_field_uses_override(json_context): + host = App(title="host-side") + lib = _LibApp(label="lib-side") + + # ``entity`` attribute on the live instance reflects the override at create-time. + assert host.entity == "HostApp" + assert lib.entity == "App" + + # ID prefix follows entity name (``n..``). + assert host.id.startswith("n.HostApp.") + assert lib.id.startswith("n.App.") + + await json_context.save(host) + await json_context.save(lib) + + with patch("jvspatial.core.context.get_default_context", return_value=json_context): + # Cross-class query isolation: each class only sees its own rows. + hosts = await App.find({}) + libs = await _LibApp.find({}) + + assert len(hosts) == 1 + assert len(libs) == 1 + assert hosts[0].id == host.id + assert libs[0].id == lib.id + # Deserialization picks the right class for each entity discriminator. + assert isinstance(hosts[0], App) + assert isinstance(libs[0], _LibApp) + assert hosts[0].title == "host-side" + assert libs[0].label == "lib-side" diff --git a/tests/core/test_entity_name_walker_and_save.py b/tests/core/test_entity_name_walker_and_save.py new file mode 100644 index 0000000..e0d9f63 --- /dev/null +++ b/tests/core/test_entity_name_walker_and_save.py @@ -0,0 +1,114 @@ +"""Walker and save-cycle coverage for ``__entity_name__`` override (audit §1). + +Walker now ships its own ``__entity_name__`` classmethod parallel to +``Object._entity_name`` so walker IDs and the persisted ``entity`` field +honor the per-subclass discriminator. The previous behavior used +``cls.__name__`` unconditionally, which broke the override for walkers +entirely. + +``GraphContext.save_object`` previously regenerated entity IDs when +``id_parts[1] != cls.__name__`` — that check used ``__name__`` instead of +``_entity_name()``, so every save of an override-using class rewrote the +ID with the wrong discriminator. The fix uses ``_entity_name()`` so saves +are stable. + +``GraphContext.find_edges_between`` previously queried +``entity == edge_class.__name__``; an edge subclass with +``__entity_name__`` would never match its own rows. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.core.context import GraphContext +from jvspatial.core.entities import Edge, Node, Walker +from jvspatial.db import create_database + + +class CustomNamedWalker(Walker): + __entity_name__ = "FleetTraversalWalker" + + +class CustomNamedEdge(Edge): + __entity_name__ = "FleetLink" + weight: float = 1.0 + + +class FleetNode(Node): + label: str = "" + + +@pytest.fixture +def json_context(): + with tempfile.TemporaryDirectory() as tmpdir: + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + yield GraphContext(database=database) + + +def test_walker_id_honors_entity_name_override(): + walker = CustomNamedWalker() + # ID prefix and persisted entity must reflect the override, not the + # Python class name. + assert walker.id.startswith("w.FleetTraversalWalker.") + assert walker.entity == "FleetTraversalWalker" + + +def test_walker_entity_name_classmethod(): + assert CustomNamedWalker._entity_name() == "FleetTraversalWalker" + + # Per-subclass — not inherited from a parent that happens to override. + class GrandchildWalker(CustomNamedWalker): + pass + + assert GrandchildWalker._entity_name() == "GrandchildWalker" + + +@pytest.mark.asyncio +async def test_save_does_not_rewrite_override_ids(json_context): + """``GraphContext.save_object`` must not regenerate IDs that use the + override. + + Before the §1 fix, the ID-validation check at context.py:753 compared + against ``__name__``, so any class with ``__entity_name__`` had its + ID rewritten on every save — silent data corruption. + """ + + class WidgetNode(Node): + __entity_name__ = "MarketplaceWidget" + name: str = "" + + n = WidgetNode(name="alpha") + original_id = n.id + assert original_id.startswith("n.MarketplaceWidget.") + + await json_context.save(n) + # ID is unchanged after save. + assert n.id == original_id + + # And on a second save (covers the regeneration branch). + await json_context.save(n) + assert n.id == original_id + + +@pytest.mark.asyncio +async def test_find_edges_between_honors_override(json_context): + """``find_edges_between`` builds an ``entity == edge_class._entity_name()`` + query rather than ``__name__`` so override edges are findable.""" + a = FleetNode(label="A") + b = FleetNode(label="B") + await json_context.save(a) + await json_context.save(b) + + edge = CustomNamedEdge(source=a.id, target=b.id, weight=2.0) + await json_context.save(edge) + assert edge.entity == "FleetLink" + assert edge.id.startswith("e.FleetLink.") + + found = await json_context.find_edges_between( + source_id=a.id, target_id=b.id, edge_class=CustomNamedEdge + ) + assert len(found) == 1 + assert found[0].id == edge.id diff --git a/tests/core/test_graph_context.py b/tests/core/test_graph_context.py index 77158e9..e123ed5 100644 --- a/tests/core/test_graph_context.py +++ b/tests/core/test_graph_context.py @@ -855,3 +855,52 @@ async def test_performance_stats_structure(self, temp_context): # Check structure assert "total_operations" in stats assert isinstance(stats["total_operations"], int) + + +class TestGraphContextFindPage: + @pytest.fixture + def temp_context(self): + with tempfile.TemporaryDirectory() as tmpdir: + import uuid + + unique_path = f"{tmpdir}/test_{uuid.uuid4().hex}" + database = create_database("json", base_path=unique_path) + context = GraphContext(database=database) + set_default_context(context) + yield context + + @pytest.mark.asyncio + async def test_find_page_returns_next_cursor(self, temp_context): + await temp_context.database.save("node", {"id": "a", "ts": 3}) + await temp_context.database.save("node", {"id": "b", "ts": 2}) + await temp_context.database.save("node", {"id": "c", "ts": 1}) + + page1, cursor = await temp_context.find_page( + "node", {}, sort=[("ts", -1)], limit=2 + ) + assert [row["id"] for row in page1] == ["a", "b"] + assert cursor + + page2, next2 = await temp_context.find_page( + "node", {}, sort=[("ts", -1)], after=cursor, limit=2 + ) + assert [row["id"] for row in page2] == ["c"] + assert next2 is None + + @pytest.mark.asyncio + async def test_find_page_accepts_dict_cursor_payload(self, temp_context): + await temp_context.database.save("node", {"id": "a", "ts": 2}) + await temp_context.database.save("node", {"id": "b", "ts": 2}) + await temp_context.database.save("node", {"id": "c", "ts": 1}) + + page1, _ = await temp_context.find_page("node", {}, sort=[("ts", -1)], limit=1) + assert page1[0]["id"] in ("a", "b") + + page2, _ = await temp_context.find_page( + "node", + {}, + sort=[("ts", -1)], + after={"sort": page1[0]["ts"], "id": page1[0]["id"]}, + limit=2, + ) + assert len(page2) >= 1 diff --git a/tests/core/test_pager_audit_fixes.py b/tests/core/test_pager_audit_fixes.py new file mode 100644 index 0000000..9dcff91 --- /dev/null +++ b/tests/core/test_pager_audit_fixes.py @@ -0,0 +1,69 @@ +"""ObjectPager fixes (audit §8.1, §8.2). + +Wave 2 changes: + +* Drop the in-memory ``_cache`` entirely — entries were never invalidated + on writes, so callers got stale rows after any save/delete on the + underlying collection. +* Reject ``after_id`` combined with ``order_by`` — the cursor only tracks + ``id`` so a non-id sort key would skip or duplicate rows on writes + between pages. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.core.context import GraphContext, scoped_default_context_async +from jvspatial.core.entities import Node +from jvspatial.core.pager import ObjectPager +from jvspatial.db import create_database + + +class PageNode(Node): + name: str = "" + value: int = 0 + + +@pytest.fixture +async def context(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + ctx = GraphContext(database=db) + async with scoped_default_context_async(ctx): + yield ctx + + +def test_object_pager_has_no_cache_attribute(): + """The previously-stale ``_cache`` attribute is gone.""" + pager = ObjectPager(PageNode, page_size=3) + assert not hasattr(pager, "_cache") + + +@pytest.mark.asyncio +async def test_after_id_with_order_by_rejected(): + pager = ObjectPager(PageNode, page_size=3, order_by="value") + with pytest.raises(ValueError, match="after_id"): + await pager.get_page(after_id="n.PageNode.abc") + + +@pytest.mark.asyncio +async def test_get_page_no_stale_cache_after_write(context): + # Seed: 4 nodes. + nodes = [PageNode(name=f"n{i}", value=i) for i in range(4)] + for n in nodes: + await context.save(n) + + pager = ObjectPager(PageNode, page_size=10) + first = await pager.get_page(page=1) + assert len(first) == 4 + + # Add a node and re-page — must see the new node, not a cached + # 4-item snapshot. + extra = PageNode(name="n_extra", value=99) + await context.save(extra) + + refreshed = await pager.get_page(page=1) + assert len(refreshed) == 5 + assert any(n.name == "n_extra" for n in refreshed) diff --git a/tests/core/test_pagination.py b/tests/core/test_pagination.py index 677ffe7..319c3b0 100644 --- a/tests/core/test_pagination.py +++ b/tests/core/test_pagination.py @@ -281,8 +281,9 @@ async def mock_deserialize(cls, data): assert pager.has_next_page() assert len(page1) == 2 - # Clear cache to ensure next call hits the database - pager._cache.clear() + # Audit §8.2: pager no longer has an in-memory ``_cache`` — + # every ``get_page`` hits the database. No explicit clear + # needed. # Get next page page2 = await pager.next_page() @@ -340,11 +341,17 @@ async def test_has_previous_page(self): class TestObjectPagerCaching: - """Test ObjectPager caching behavior.""" + """Pager no longer caches results (audit §8.2).""" @pytest.mark.asyncio - async def test_page_caching(self, mock_context, sample_data): - """Test that pages are cached properly.""" + async def test_get_page_does_not_cache(self, mock_context, sample_data): + """Every ``get_page`` call must hit the database. + + The previous in-memory ``_cache`` was never invalidated on writes, + so callers got stale rows after any save/delete on the underlying + collection. The cache has been removed; ``is_cached`` is always + False. + """ with patch( "jvspatial.core.context.get_default_context", return_value=mock_context ): @@ -357,16 +364,17 @@ async def mock_deserialize(cls, data): mock_context._deserialize_entity.side_effect = mock_deserialize pager = ObjectPager(PaginationTestObject, page_size=2) + assert not hasattr(pager, "_cache") - # First call should hit the database results1 = await pager.get_page(1) assert not pager.is_cached - assert mock_context.database.find.call_count == 1 + first_call_count = mock_context.database.find.call_count + assert first_call_count >= 1 - # Second call to same page should use cache + # Second call to same page MUST re-fetch. results2 = await pager.get_page(1) - assert pager.is_cached - assert mock_context.database.find.call_count == 1 # No additional calls + assert not pager.is_cached + assert mock_context.database.find.call_count > first_call_count assert len(results1) == len(results2) diff --git a/tests/core/test_walker_protection.py b/tests/core/test_walker_protection.py index 8ed3dd4..2a941c0 100644 --- a/tests/core/test_walker_protection.py +++ b/tests/core/test_walker_protection.py @@ -27,6 +27,16 @@ Root, Walker, ) +from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, +) + +# Per SPEC §6.3 / audit §2.1: protection limits now raise the documented +# exception types instead of being swallowed into the walker report. Tests +# that intentionally trip a limit must wrap ``spawn`` in this tuple. +PROTECTION_EXCS = (WalkerExecutionError, InfiniteLoopError, WalkerTimeoutError) class ProtectionTestNode(Node): @@ -230,8 +240,9 @@ async def test_max_steps_protection_triggers(self): walker = StepCountTestWalker(max_steps=5) start_node = ProtectionTestNode(name="start", value=0) - # Run walker and expect protection to trigger - await walker.spawn(start_node) + # SPEC §6.3: exceeding max_steps raises WalkerExecutionError. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check protection was triggered by checking step count and limits # The protection system stops traversal when limits are exceeded @@ -250,8 +261,10 @@ async def test_max_steps_protection_disabled(self): walker = StepCountTestWalker(max_steps=50) start_node = ProtectionTestNode(name="start", value=0) - # Run walker - it will stop at 50 steps due to the walker's internal limit - await walker.spawn(start_node) + # Walker still hits the cap, but at 50 it is "effectively disabled" + # from the test's perspective; SPEC §6.3 says it still raises. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check that walker ran more than the original 5 steps # but still stopped due to protection at 50 @@ -270,7 +283,9 @@ async def test_step_counting_accuracy(self): walker = StepCountTestWalker(max_steps=10) start_node = ProtectionTestNode(name="start", value=0) - await walker.spawn(start_node) + # StepCountTestWalker is designed to trip max_steps. + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check step counting matches expected # The Walker should have taken some steps and stopped @@ -291,7 +306,9 @@ async def test_max_visits_per_node_protection(self): walker = NodeRevisitWalker(max_visits_per_node=3) start_node = ProtectionTestNode(name="revisit_test", value=0) - await walker.spawn(start_node) + # SPEC §6.3: max_visits_per_node raises InfiniteLoopError. + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check protection was triggered by checking visit counts # The protection system stops traversal when limits are exceeded @@ -309,7 +326,8 @@ async def test_node_visit_counting(self): walker = NodeRevisitWalker(max_visits_per_node=5) start_node = ProtectionTestNode(name="count_test", value=0) - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check visit counting through protection component visit_counts = walker._protection.visit_counts @@ -334,7 +352,8 @@ async def visit_multiple_nodes(self, here): walker = MultiNodeWalker(max_steps=20) start_node = ProtectionTestNode(name="multi_start") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check that multiple nodes were tracked visit_counts = walker._protection.visit_counts @@ -397,7 +416,8 @@ async def test_queue_size_protection(self): ) start_node = ProtectionTestNode(name="queue_test") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # Check that protection triggered (should be max_steps since queue limiting doesn't stop traversal) # The protection system stops traversal when limits are exceeded @@ -426,7 +446,8 @@ async def test_queue_size_protection_disabled(self): start_node = ProtectionTestNode(name="queue_unlimited_test") - await walker.spawn(start_node) + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # With high queue limit, queue can grow larger # But step protection should still stop the walker @@ -493,8 +514,9 @@ async def test_protection_status_during_execution(self): initial_step_count = walker.step_count initial_queue_size = len(walker.queue) - # Run walker - await walker.spawn(start_node) + # Run walker (expected to trip max_steps). + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check final status final_step_count = walker.step_count @@ -534,7 +556,8 @@ async def test_protection_with_trail_tracking(self): ) start_node = ProtectionTestNode(name="trail_protection_test") - await walker.spawn(start_node) + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check both protection and trail worked # The protection system stops traversal when limits are exceeded @@ -570,7 +593,8 @@ async def set_response_data(self, here): walker = ResponseTestWalker(max_steps=5) start_node = ProtectionTestNode(name="response_test") - await walker.spawn(start_node) + with pytest.raises(WalkerExecutionError): + await walker.spawn(start_node) # Check protection data was added without overwriting custom data # The protection system stops traversal when limits are exceeded @@ -597,7 +621,10 @@ async def test_multiple_protection_triggers(self): ) start_node = ProtectionTestNode(name="multi_protection_test") - await walker.spawn(start_node) + # Any of the three limits can fire first — accept all three exception + # types (SPEC §6.3). + with pytest.raises(PROTECTION_EXCS): + await walker.spawn(start_node) # One of the protections should have triggered # The protection system stops traversal when limits are exceeded diff --git a/tests/core/test_walker_protection_audit_fixes.py b/tests/core/test_walker_protection_audit_fixes.py new file mode 100644 index 0000000..e89064c --- /dev/null +++ b/tests/core/test_walker_protection_audit_fixes.py @@ -0,0 +1,173 @@ +"""Regression coverage for Wave 1 walker-protection fixes (audit §2). + +The earlier implementation: + +* Swallowed every ``ProtectionViolation`` into ``walker.report``, so callers + never saw ``InfiniteLoopError`` / ``WalkerTimeoutError`` / ``WalkerExecutionError`` + even though SPEC §6.3 promised them. +* Called ``self._protection.reset()`` at the top of ``Walker.run()``. ``resume()`` + re-enters ``run()`` → resetting step / visit counters and the wall-clock timer. + Repeated pause/resume cycles cleared protection state, trivially defeating the + cap. +* Ignored ``max_trail_length`` — the docstring promised it, no code wired it. +* Allowed ``WalkerQueue.prepend`` / ``add_next`` / ``insert_after`` / ``insert_before`` + to grow past ``max_size``, providing a silent protection bypass. + +These tests pin the corrected behavior so future regressions are loud. +""" + +import asyncio + +import pytest + +from jvspatial.core.entities import Node, Walker +from jvspatial.core.entities.walker_components.protection import ( + ProtectionViolation, + TraversalProtection, +) +from jvspatial.core.entities.walker_components.walker_queue import WalkerQueue +from jvspatial.core.entities.walker_components.walker_trail import WalkerTrail +from jvspatial.exceptions import ( + InfiniteLoopError, + WalkerExecutionError, + WalkerTimeoutError, +) + +# ---------- TraversalProtection ---------- + + +@pytest.mark.asyncio +async def test_start_if_needed_is_idempotent(): + prot = TraversalProtection(max_steps=10) + await prot.start_if_needed() + first_start = prot._start_time + await prot.increment_step() + await prot.increment_step() + assert prot.step_count == 2 + + # Calling start_if_needed again must NOT reset counters or restart the + # wall-clock timer. Previously ``run()`` called ``reset()`` which did. + await prot.start_if_needed() + assert prot.step_count == 2 + assert prot._start_time == first_start + + +@pytest.mark.asyncio +async def test_reset_is_still_available_for_explicit_restart(): + prot = TraversalProtection(max_steps=10) + await prot.start_if_needed() + await prot.increment_step() + await prot.reset() + # Explicit reset starts a fresh session — start_if_needed becomes a no-op + # after reset because reset sets _started = True. + assert prot.step_count == 0 + + +# ---------- Walker.run() raises documented exceptions ---------- + + +class AlwaysReenqueueNode(Node): + name: str = "" + + +class TightLoopWalker(Walker): + """Visits the same node repeatedly so max_visits_per_node fires.""" + + async def visit(self, target): + await self.queue.append([target]) + + +@pytest.mark.asyncio +async def test_walker_run_raises_infinite_loop_error_on_visit_cap(): + node = AlwaysReenqueueNode(name="cycle") + walker = TightLoopWalker(max_visits_per_node=3) + # Seed the queue with the same node many times so record_visit hits the cap. + await walker.queue.append([node] * 20) + + with pytest.raises(InfiniteLoopError) as exc_info: + await walker.run() + err = exc_info.value + assert err.node_id == node.id + assert err.walker_class == "TightLoopWalker" + + +@pytest.mark.asyncio +async def test_walker_run_raises_walker_execution_error_on_step_cap(): + walker = TightLoopWalker(max_steps=3, max_visits_per_node=10_000) + # Distinct nodes so visit-cap does not fire first. + nodes = [AlwaysReenqueueNode(name=f"n{i}") for i in range(20)] + await walker.queue.append(nodes) + + with pytest.raises(WalkerExecutionError) as exc_info: + await walker.run() + assert "max_steps" in exc_info.value.reason + + +# ---------- max_trail_length wired ---------- + + +def test_walker_trail_unbounded_by_default(): + trail = WalkerTrail() + for i in range(50): + trail.record_step(f"n.X.{i}") + assert trail.get_length() == 50 + + +def test_walker_trail_respects_max_length(): + trail = WalkerTrail(max_length=10) + for i in range(50): + trail.record_step(f"n.X.{i}") + # Only the most recent 10 retained. + assert trail.get_length() == 10 + most_recent_ids = [step["node"] for step in trail.get_trail()] + assert most_recent_ids[0] == "n.X.40" + assert most_recent_ids[-1] == "n.X.49" + + +def test_walker_propagates_max_trail_length(): + """Walker.__init__ pops ``max_trail_length`` and rebuilds the trail tracker.""" + walker = TightLoopWalker(max_trail_length=3) + for i in range(10): + walker._trail_tracker.record_step(f"n.X.{i}") + assert walker._trail_tracker.get_length() == 3 + + +# ---------- WalkerQueue inserts respect max_size ---------- + + +@pytest.mark.asyncio +async def test_prepend_respects_max_size(): + q = WalkerQueue(max_size=3) + await q.visit(["a", "b", "c"]) + # Past cap — prepend must drop, not silently grow. + await q.prepend(["x", "y"]) + assert len(q) == 3 + # Existing items preserved at the head's downstream side. + assert list(q.to_list()) == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_add_next_respects_max_size(): + q = WalkerQueue(max_size=2) + await q.visit(["a", "b"]) + await q.add_next(["x"]) + assert len(q) == 2 + + +@pytest.mark.asyncio +async def test_insert_after_respects_max_size_and_returns_inserted(): + q = WalkerQueue(max_size=3) + await q.visit(["a", "b", "c"]) + inserted = await q.insert_after("a", ["x", "y"]) + assert inserted == [] # No room. + assert len(q) == 3 + + +@pytest.mark.asyncio +async def test_insert_before_respects_max_size_and_returns_inserted(): + q = WalkerQueue(max_size=4) + await q.visit(["a", "b"]) + inserted = await q.insert_before("b", ["x", "y", "z"]) + # First two fit (filling the 4-slot cap), third is dropped. + assert inserted == ["x", "y"] + assert len(q) == 4 diff --git a/tests/core/test_walker_trail_new.py b/tests/core/test_walker_trail_new.py index ad4e23f..34402ab 100644 --- a/tests/core/test_walker_trail_new.py +++ b/tests/core/test_walker_trail_new.py @@ -368,15 +368,24 @@ async def test_trail_during_resume(self): @pytest.mark.asyncio async def test_trail_with_protection(self): - """Test trail recording with protection limits.""" + """Test trail recording with protection limits. + + SPEC §6.3 / audit §2.1: tripping ``max_steps`` now raises + ``WalkerExecutionError`` instead of being silently absorbed into the + walker report. The trail captured up to the violation is still + readable after the exception unwinds. + """ + from jvspatial.exceptions import WalkerExecutionError + walker = TrailTrackingWalker(max_steps=2) nodes = [TrailTestNode(name=f"node{i}") for i in range(5)] # Add nodes to queue await walker.visit(nodes) - # Spawn with protection should record trail steps - await walker.spawn(nodes[0]) + # Spawn with protection should record trail steps and then raise. + with pytest.raises(WalkerExecutionError): + await walker.spawn(nodes[0]) # Check that trail was recorded trail = walker.get_trail() diff --git a/tests/core/test_wave5_walker_audit.py b/tests/core/test_wave5_walker_audit.py new file mode 100644 index 0000000..4e411ad --- /dev/null +++ b/tests/core/test_wave5_walker_audit.py @@ -0,0 +1,29 @@ +"""Walker polish (audit §2.9, §2.10).""" + +import pytest + +from jvspatial.core.entities import TraversalSkipped, Walker + + +class _DemoWalker(Walker): + pass + + +@pytest.mark.asyncio +async def test_walker_skip_raises_traversal_skipped(): + w = _DemoWalker() + with pytest.raises(TraversalSkipped): + await w.skip() + + +def test_walker_type_code_locked_to_w(): + w = _DemoWalker() + assert w.type_code == "w" + assert w.id.startswith("w.") + + +def test_walker_rejects_alternate_type_code(): + """Caller cannot smuggle a different ``type_code`` past the SPEC + §1.1 invariant (audit §2.10).""" + with pytest.raises(ValueError, match="type_code must be 'w'"): + _DemoWalker(type_code="n") diff --git a/tests/db/test_bulk_save_detailed_audit.py b/tests/db/test_bulk_save_detailed_audit.py new file mode 100644 index 0000000..e57cec4 --- /dev/null +++ b/tests/db/test_bulk_save_detailed_audit.py @@ -0,0 +1,62 @@ +"""BulkSaveResult / bulk_save_detailed semantics (audit §5.6 / §5.7). + +The legacy ``bulk_save`` returned a single ``int`` so partial-failure +backends (JsonDB, DynamoDB) could silently drop records without callers +noticing. ``bulk_save_detailed`` returns a structured +:class:`BulkSaveResult` with ``attempted`` / ``saved`` / ``failed_ids``. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.db import create_database +from jvspatial.db.database import BulkSaveResult + + +@pytest.fixture +async def jsondb(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + yield db + + +def test_bulk_save_result_dataclass_fields(): + r = BulkSaveResult(attempted=3, saved=3) + assert r.attempted == 3 + assert r.saved == 3 + assert r.failed_ids == [] + assert r.all_saved is True + + partial = BulkSaveResult(attempted=3, saved=2, failed_ids=["x"]) + assert partial.all_saved is False + + +@pytest.mark.asyncio +async def test_bulk_save_detailed_returns_structured_result(jsondb): + records = [{"id": f"r{i}", "n": i} for i in range(5)] + result = await jsondb.bulk_save_detailed("widgets", records) + assert isinstance(result, BulkSaveResult) + assert result.attempted == 5 + assert result.saved == 5 + assert result.failed_ids == [] + assert result.all_saved + + +@pytest.mark.asyncio +async def test_bulk_save_int_still_returned_for_back_compat(jsondb): + records = [{"id": f"r{i}", "n": i} for i in range(4)] + saved_count = await jsondb.bulk_save("widgets", records) + assert saved_count == 4 + + +@pytest.mark.asyncio +async def test_bulk_save_missing_id_includes_index(jsondb): + records = [ + {"id": "r0", "n": 0}, + {"n": 1}, # missing id at index 1 + {"id": "r2", "n": 2}, + ] + with pytest.raises(ValueError, match="index 1"): + await jsondb.bulk_save_detailed("widgets", records) diff --git a/tests/db/test_default_compound_ops_id_normalization.py b/tests/db/test_default_compound_ops_id_normalization.py new file mode 100644 index 0000000..91d14c3 --- /dev/null +++ b/tests/db/test_default_compound_ops_id_normalization.py @@ -0,0 +1,72 @@ +"""Default ``find_one_and_update`` / ``find_one_and_delete`` must honor +both ``_id`` and ``id`` query keys. + +Audit §5.3 / SPEC §4.1: non-Mongo backends (JsonDB, SQLite, DynamoDB) +persist records keyed by ``id`` only. Callers that follow the +Mongo-style convention of querying by ``_id`` were silently missed by +the default compound-op implementation, which fed the query into +``QueryEngine.match`` against records that have no ``_id`` field. + +The default impls now normalize ``_id`` → ``id`` before matching. +""" + +import tempfile +import uuid + +import pytest + +from jvspatial.db import create_database + + +@pytest.fixture +async def jsondb(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("json", base_path=f"{tmpdir}/{uuid.uuid4().hex}") + yield db + + +@pytest.mark.asyncio +async def test_find_one_and_update_matches_by_underscored_id(jsondb): + await jsondb.save("widgets", {"id": "w1", "qty": 1}) + out = await jsondb.find_one_and_update( + "widgets", {"_id": "w1"}, {"$set": {"qty": 2}} + ) + assert out is not None + assert out["qty"] == 2 + # Side-effect on disk reflects the update. + fresh = await jsondb.get("widgets", "w1") + assert fresh["qty"] == 2 + + +@pytest.mark.asyncio +async def test_find_one_and_update_matches_by_id_unchanged(jsondb): + await jsondb.save("widgets", {"id": "w2", "qty": 5}) + out = await jsondb.find_one_and_update( + "widgets", {"id": "w2"}, {"$set": {"qty": 7}} + ) + assert out is not None + assert out["qty"] == 7 + + +@pytest.mark.asyncio +async def test_find_one_and_delete_matches_by_underscored_id(jsondb): + await jsondb.save("widgets", {"id": "w3", "qty": 9}) + deleted = await jsondb.find_one_and_delete("widgets", {"_id": "w3"}) + assert deleted is not None + assert deleted["id"] == "w3" + assert await jsondb.get("widgets", "w3") is None + + +@pytest.mark.asyncio +async def test_find_one_and_update_upsert_with_underscored_id(jsondb): + out = await jsondb.find_one_and_update( + "widgets", + {"_id": "w_new"}, + {"$set": {"qty": 42}}, + upsert=True, + ) + assert out is not None + assert out["id"] == "w_new" + assert out["qty"] == 42 + persisted = await jsondb.get("widgets", "w_new") + assert persisted is not None diff --git a/tests/db/test_observable_database.py b/tests/db/test_observable_database.py index 09ee943..b23103b 100644 --- a/tests/db/test_observable_database.py +++ b/tests/db/test_observable_database.py @@ -19,6 +19,7 @@ from jvspatial.db._observable import ObservableDatabase from jvspatial.db.factory import create_database from jvspatial.db.jsondb import JsonDB +from jvspatial.observability import db_op_counter class _CapturingRecorder: @@ -161,6 +162,17 @@ async def test_slow_op_increments_slow_count_metric(self, jsondb): ] assert slow_counters, "slow op should emit slow_count counter" + async def test_db_op_counter_increments_per_operation(self, jsondb): + wrapped = ObservableDatabase(jsondb) + tok = db_op_counter.set(0) + try: + await wrapped.save("node", {"id": "x", "v": 1}) + await wrapped.get("node", "x") + await wrapped.find("node", {}) + assert db_op_counter.get() == 3 + finally: + db_op_counter.reset(tok) + # ---------------------- error path ------------------------------------ diff --git a/tests/db/test_query_operator_parity_audit.py b/tests/db/test_query_operator_parity_audit.py new file mode 100644 index 0000000..3222b27 --- /dev/null +++ b/tests/db/test_query_operator_parity_audit.py @@ -0,0 +1,103 @@ +"""QueryEngine operator parity (audit §5.2 / SPEC §5.1). + +The earlier matcher: + +* silently returned False for any field-level operator it did not know; +* silently returned False for the top-level ``$nor`` operator the + QueryBuilder advertised; +* silently passed through optimizer markers (``$hint`` / ``$select``) + which caused match failures when ``optimize_query`` injected them. + +The matcher now raises ``QueryError`` for unsupported operators and +implements ``$nor`` / ``$mod`` / ``$all`` / ``$type`` / ``$not`` so the +QueryBuilder surface and engine behavior line up. +""" + +import pytest + +from jvspatial.db.query import QueryEngine +from jvspatial.exceptions import QueryError + +# ---------- $nor ---------- + + +def test_nor_excludes_documents_matching_any_subcondition(): + doc = {"status": "active", "tier": "free"} + # Document does NOT match {"status": "inactive"} nor {"tier": "pro"}; + # $nor should accept it. + assert ( + QueryEngine.match( + doc, + { + "$nor": [ + {"status": "inactive"}, + {"tier": "pro"}, + ] + }, + ) + is True + ) + # Document DOES match {"status": "active"}; $nor rejects. + assert QueryEngine.match(doc, {"$nor": [{"status": "active"}]}) is False + + +# ---------- $mod ---------- + + +def test_mod_matches_remainder(): + assert QueryEngine.match({"n": 10}, {"n": {"$mod": [3, 1]}}) is True + assert QueryEngine.match({"n": 9}, {"n": {"$mod": [3, 1]}}) is False + assert QueryEngine.match({"n": "x"}, {"n": {"$mod": [3, 0]}}) is False + + +# ---------- $all ---------- + + +def test_all_requires_every_operand_in_value(): + assert QueryEngine.match({"tags": ["a", "b", "c"]}, {"tags": {"$all": ["a", "b"]}}) + assert not QueryEngine.match( + {"tags": ["a", "b", "c"]}, {"tags": {"$all": ["a", "z"]}} + ) + assert not QueryEngine.match({"tags": "ab"}, {"tags": {"$all": ["a"]}}) + + +# ---------- $type ---------- + + +def test_type_accepts_python_type_names(): + assert QueryEngine.match({"x": 5}, {"x": {"$type": "int"}}) + assert QueryEngine.match({"x": "y"}, {"x": {"$type": "string"}}) + assert not QueryEngine.match({"x": "y"}, {"x": {"$type": "int"}}) + assert not QueryEngine.match({"x": 5}, {"x": {"$type": "bogus"}}) + + +# ---------- $not (field-level) ---------- + + +def test_field_level_not_negates(): + assert QueryEngine.match({"n": 5}, {"n": {"$not": {"$gt": 10}}}) + assert not QueryEngine.match({"n": 5}, {"n": {"$not": {"$gt": 1}}}) + + +# ---------- Unsupported operators raise ---------- + + +def test_unsupported_top_level_operator_raises(): + with pytest.raises(QueryError): + QueryEngine.match({}, {"$bogus": []}) + + +def test_unsupported_field_level_operator_raises(): + with pytest.raises(QueryError): + QueryEngine.match({"x": 1}, {"x": {"$bogus": 1}}) + + +# ---------- Optimizer markers are ignored ---------- + + +def test_optimizer_markers_do_not_break_match(): + """``$hint`` / ``$select`` are optimizer-only — match() must skip them.""" + doc = {"name": "alice"} + # With $hint mixed in, the match still passes on the actual condition. + assert QueryEngine.match(doc, {"name": "alice", "$hint": "name_idx"}) + assert QueryEngine.match(doc, {"$select": ["name"]}) diff --git a/tests/db/test_sqlite_cross_loop_audit.py b/tests/db/test_sqlite_cross_loop_audit.py new file mode 100644 index 0000000..272b48c --- /dev/null +++ b/tests/db/test_sqlite_cross_loop_audit.py @@ -0,0 +1,62 @@ +"""SQLite cross-loop detection (audit §5.10 / SPEC §4.3). + +``aiosqlite`` binds its connection to the event loop that opened it. +Reusing a ``SQLiteDB`` instance across loops previously produced an +opaque "Future attached to a different loop" error from inside +``aiosqlite``; the wrapper now detects the binding change and +transparently rebinds to the current loop instead. +""" + +import asyncio +import tempfile + +import pytest + +from jvspatial.db import create_database + + +@pytest.mark.asyncio +async def test_same_loop_reuse_works(): + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + await db.save("widgets", {"id": "w1", "qty": 1}) + got = await db.get("widgets", "w1") + assert got is not None + await db.save("widgets", {"id": "w2", "qty": 2}) + await db.close() + + +def test_cross_loop_reuse_auto_rebinds(): + """Across loops the connection is silently rebuilt on the active loop.""" + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + + async def first() -> None: + await db.save("widgets", {"id": "w1", "qty": 1}) + + asyncio.run(first()) + + async def second() -> None: + # Auto-rebind on a new loop — no error. + await db.save("widgets", {"id": "w2", "qty": 2}) + got = await db.get("widgets", "w2") + assert got is not None + await db.close() + + asyncio.run(second()) + + +def test_owning_loop_tracked(): + """After save the SQLiteDB tracks the loop that owns the connection.""" + with tempfile.TemporaryDirectory() as tmpdir: + db = create_database("sqlite", db_path=f"{tmpdir}/x.db") + + async def go() -> None: + await db.save("widgets", {"id": "w1", "qty": 1}) + # ``_owning_loop`` is private but the contract is part of + # the cross-loop fix; assert it was populated. + assert db._owning_loop is asyncio.get_running_loop() + await db.close() + assert db._owning_loop is None + + asyncio.run(go()) diff --git a/tests/db/test_sqlite_id_coercion_audit.py b/tests/db/test_sqlite_id_coercion_audit.py new file mode 100644 index 0000000..db56e04 --- /dev/null +++ b/tests/db/test_sqlite_id_coercion_audit.py @@ -0,0 +1,19 @@ +"""SQLite ``save()`` id coercion to ``str`` (audit §5.20).""" + +import tempfile + +import pytest + +from jvspatial.db.sqlite import SQLiteDB + + +@pytest.mark.asyncio +async def test_int_id_roundtrips_through_save_and_get(): + with tempfile.TemporaryDirectory() as tmpdir: + db = SQLiteDB(db_path=f"{tmpdir}/x.db") + await db.save("widgets", {"id": 42, "name": "alpha"}) + # The persisted id is stringified — get() by the str form works. + got = await db.get("widgets", "42") + assert got is not None + assert got["name"] == "alpha" + await db.close() diff --git a/tests/serverless/test_deferred_invoke.py b/tests/serverless/test_deferred_invoke.py index 5b98aa7..14a3cdd 100644 --- a/tests/serverless/test_deferred_invoke.py +++ b/tests/serverless/test_deferred_invoke.py @@ -78,7 +78,10 @@ async def test_dispatch_malformed_task_type(body: dict): await dispatch_deferred_invoke(body) -def test_deferred_invoke_http_route(): +def test_deferred_invoke_http_route(monkeypatch): + # Audit §4.16: route is now fail-closed when the secret is unset. + # Set a secret + send matching header for the happy-path test. + monkeypatch.setenv("JVSPATIAL_DEFERRED_INVOKE_SECRET", "test-secret-value") app = FastAPI() async def handler(event: dict) -> dict: @@ -97,17 +100,23 @@ async def handler(event: dict) -> dict: "sender": "u1", "media_batch_window": 1.5, }, + headers={"X-JVSPATIAL-Deferred-Authorize": "test-secret-value"}, ) assert r.status_code == 200 assert r.json() == {"ok": True, "sender": "u1"} -def test_deferred_invoke_http_unknown_returns_404(): +def test_deferred_invoke_http_unknown_returns_404(monkeypatch): + monkeypatch.setenv("JVSPATIAL_DEFERRED_INVOKE_SECRET", "test-secret-value") app = FastAPI() register_deferred_invoke_route(app) client = TestClient(app) path = APIRoutes.deferred_invoke_full_path() - r = client.post(path, json={"task_type": "no.such.task"}) + r = client.post( + path, + json={"task_type": "no.such.task"}, + headers={"X-JVSPATIAL-Deferred-Authorize": "test-secret-value"}, + ) assert r.status_code == 404 assert "Unknown task_type" in r.json()["detail"] diff --git a/tests/storage/test_versioning_path_sanitizer_audit.py b/tests/storage/test_versioning_path_sanitizer_audit.py new file mode 100644 index 0000000..a686721 --- /dev/null +++ b/tests/storage/test_versioning_path_sanitizer_audit.py @@ -0,0 +1,62 @@ +"""Path-traversal coverage for ``LocalFileInterface`` versioning methods. + +Audit §4.2 / SPEC §15.1: ``create_version`` / ``get_version`` / +``list_versions`` / ``delete_version`` / ``get_latest_version`` previously +computed ``self.root_dir / f"{file_path}.versions"`` without sanitizing +``file_path`` — a caller-supplied ``../../etc/passwd`` escaped the storage +root entirely. +""" + +import tempfile + +import pytest + +from jvspatial.storage import create_storage +from jvspatial.storage.exceptions import PathTraversalError + + +@pytest.fixture +def local_storage(): + with tempfile.TemporaryDirectory() as tmpdir: + storage = create_storage("local", root_dir=tmpdir) + yield storage + + +@pytest.mark.asyncio +async def test_create_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.create_version("../../etc/passwd", b"payload") + + +@pytest.mark.asyncio +async def test_get_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.get_version("../../etc/passwd", "v1") + + +@pytest.mark.asyncio +async def test_list_versions_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.list_versions("../../etc/passwd") + + +@pytest.mark.asyncio +async def test_delete_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.delete_version("../../etc/passwd", "v1") + + +@pytest.mark.asyncio +async def test_get_latest_version_rejects_path_traversal(local_storage): + with pytest.raises(PathTraversalError): + await local_storage.get_latest_version("../../etc/passwd") + + +@pytest.mark.asyncio +async def test_create_version_then_get_round_trips_for_safe_path(local_storage): + """Sanitizer must not break the happy path — legitimate paths still work.""" + res = await local_storage.create_version("safe/file.txt", b"hello") + assert res["path"] == "safe/file.txt" + fetched = await local_storage.get_version("safe/file.txt", res["version_id"]) + assert fetched is not None + assert fetched["content"] == b"hello" diff --git a/tests/storage/test_windows_reserved_audit.py b/tests/storage/test_windows_reserved_audit.py new file mode 100644 index 0000000..9a458d7 --- /dev/null +++ b/tests/storage/test_windows_reserved_audit.py @@ -0,0 +1,45 @@ +"""Windows-reserved-name rejection (audit §4.18 / SPEC §15.1).""" + +import pytest + +from jvspatial.storage.exceptions import InvalidPathError +from jvspatial.storage.security.path_sanitizer import PathSanitizer + + +@pytest.mark.parametrize( + "filename", + [ + "CON", + "con.txt", + "PRN.json", + "AUX", + "NUL.bin", + "COM1", + "com9.dat", + "LPT1", + "lpt9.log", + ], +) +def test_sanitize_path_rejects_windows_reserved(filename): + with pytest.raises(InvalidPathError, match="Reserved Windows filename"): + PathSanitizer.sanitize_path(filename) + + +@pytest.mark.parametrize( + "filename", + ["CONFIG.json", "context.txt", "PRNT.log", "COMRADE.bin"], +) +def test_sanitize_path_allows_non_reserved(filename): + """Names that *start* with a reserved stem but are longer must pass.""" + out = PathSanitizer.sanitize_path(filename) + assert out == filename + + +def test_sanitize_filename_rejects_reserved(): + with pytest.raises(InvalidPathError, match="Reserved Windows filename"): + PathSanitizer.sanitize_filename("CON.txt") + + +def test_sanitize_path_blocks_reserved_in_subdir(): + with pytest.raises(InvalidPathError): + PathSanitizer.sanitize_path("uploads/CON.txt") diff --git a/tests/test_node_operations.py b/tests/test_node_operations.py index 3d49dbd..b6da782 100644 --- a/tests/test_node_operations.py +++ b/tests/test_node_operations.py @@ -262,6 +262,38 @@ async def test_node_method_optimizes_with_limit(context): assert isinstance(result, Memory) +@pytest.mark.asyncio +async def test_nodes_bulk_returns_per_source_neighbors(context): + city1 = await City.create(name="NY") + city2 = await City.create(name="SF") + city3 = await City.create(name="LA") + + await city1.connect(city2, edge=LocatedIn) + await city1.connect(city3, edge=LocatedIn) + await city2.connect(city3, edge=LocatedIn) + + result = await Node.nodes_bulk( + [city1.id, city2.id], direction="out", edge=["LocatedIn"], node=["City"] + ) + + assert set(result.keys()) == {city1.id, city2.id} + assert {n.id for n in result[city1.id]} == {city2.id, city3.id} + assert {n.id for n in result[city2.id]} == {city3.id} + + +@pytest.mark.asyncio +async def test_nodes_bulk_supports_incoming_direction(context): + city1 = await City.create(name="One") + city2 = await City.create(name="Two") + city3 = await City.create(name="Three") + await city1.connect(city3, edge=LocatedIn) + await city2.connect(city3, edge=LocatedIn) + + result = await Node.nodes_bulk([city3.id], direction="in", edge=["LocatedIn"]) + assert city3.id in result + assert {n.id for n in result[city3.id]} == {city1.id, city2.id} + + @pytest.mark.asyncio async def test_node_method_use_case_example(context): """Test the real-world use case that motivated this method.""" diff --git a/tests/utils/test_wave4_polish_audit.py b/tests/utils/test_wave4_polish_audit.py new file mode 100644 index 0000000..ddc6b04 --- /dev/null +++ b/tests/utils/test_wave4_polish_audit.py @@ -0,0 +1,46 @@ +"""Wave 4 polish (audit §3.11, §7.7, §5.14).""" + +import warnings + +import pytest + +from jvspatial.utils.stability import ( + ExperimentalWarning, + emit_experimental_once, + reset_experimental_warnings, +) + + +def test_emit_experimental_once_is_public(): + """Public hook so callers can flag opt-in surface without reaching + into ``_emit_once`` (audit §7.7).""" + reset_experimental_warnings() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ExperimentalWarning) + emit_experimental_once("test.api.public_hook", "note") + emit_experimental_once("test.api.public_hook", "note") + # Single emission per (name) regardless of repeated calls. + assert sum(1 for w in caught if issubclass(w.category, ExperimentalWarning)) == 1 + + +@pytest.mark.asyncio +async def test_generate_id_async_emits_deprecation(): + """``generate_id_async`` is a deprecated alias for ``generate_id`` + (audit §3.11). It must still work — the call site only sees a + warning.""" + from jvspatial.core.utils import generate_id_async + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + result = await generate_id_async("n", "Demo") + assert result.startswith("n.Demo.") + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + + +def test_jsontransaction_dead_class_removed(): + """``JSONTransaction`` was unused dead code (audit §5.14). Must no + longer be exported from ``jvspatial.db.transaction``.""" + import jvspatial.db.transaction as txn_mod + + assert "JSONTransaction" not in txn_mod.__all__ + assert not hasattr(txn_mod, "JSONTransaction")