|
| 1 | +{ |
| 2 | + "$comment": "Standing platform test checklist — area ledger. Hand-edited, append-only; validated by scripts/check-platform-checklist.mjs (pnpm check:platform-checklist). Authoring rules: docs/qa/platform-checklist/README.md · execution protocol: RUNNER.md. NOTE: AI is deliberately absent from the showcase app (ADR-0063 — platform-owned), so most items here carry explicit fixture requirements instead of assuming showcase seeds.", |
| 3 | + "area": "ai", |
| 4 | + "title": "AI — agents, tools, skills, MCP", |
| 5 | + "items": [ |
| 6 | + { |
| 7 | + "id": "ai.agent-tool-skill-metadata-roundtrip", |
| 8 | + "title": "agent / tool / skill metadata kinds author, persist and list over the meta + REST surfaces", |
| 9 | + "since": "v15", |
| 10 | + "status": "active", |
| 11 | + "revision": 1, |
| 12 | + "priority": "P1", |
| 13 | + "surface": "api", |
| 14 | + "personas": ["admin"], |
| 15 | + "fixtures": { |
| 16 | + "app": "any", |
| 17 | + "requires": ["a writable package to author into (AI artifacts are not seeded by the showcase — ADR-0063)"], |
| 18 | + "knownGaps": ["no stock example app ships AI seeds; a minimal agent+tool+skill fixture config is needed for the full round-trip"] |
| 19 | + }, |
| 20 | + "variants": ["agent", "tool", "skill"], |
| 21 | + "steps": [ |
| 22 | + "author one metadata item per variant (defineAgent / defineTool / defineSkill shapes from @objectstack/spec src/ai/*.zod.ts) in a writable package", |
| 23 | + "build/boot the app; capture the boot log for AI registration lines", |
| 24 | + "GET /api/v1/meta/types/agent (and tool, skill) to confirm the kinds are served by the registry", |
| 25 | + "list the authored items over the REST surface (routes under /api/v1/ai/** — e.g. GET /api/v1/ai/agents, the REST-plural example of AGENTS.md PD#3)", |
| 26 | + "re-read one item and field-diff it against the authored source" |
| 27 | + ], |
| 28 | + "acceptance": [ |
| 29 | + { |
| 30 | + "clause": "each of the three kinds parses against its Zod schema at build — an off-spec shape fails at authoring, not silently at runtime", |
| 31 | + "oracle": "build", |
| 32 | + "verify": "a deliberately broken variant (unknown key) is rejected with a located error; the valid ones build clean", |
| 33 | + "evidence": "build output for both cases" |
| 34 | + }, |
| 35 | + { |
| 36 | + "clause": "the metadata registry serves all three kinds (they are registered metadata types, singular names per PD#3)", |
| 37 | + "oracle": "api", |
| 38 | + "verify": "GET /api/v1/meta/types/<kind> returns the kind's schema/registration for agent, tool, skill", |
| 39 | + "evidence": "the three responses" |
| 40 | + }, |
| 41 | + { |
| 42 | + "clause": "authored items are listed and readable over /api/v1/ai/** and survive a field-diff against source", |
| 43 | + "oracle": "api", |
| 44 | + "verify": "list + get round-trip; every authored field present with the authored value", |
| 45 | + "evidence": "list/get responses + diff" |
| 46 | + } |
| 47 | + ], |
| 48 | + "negative": [ |
| 49 | + "an unauthenticated request to the /api/v1/ai/** list route → 401 (anonymous-deny holds on the AI family too)" |
| 50 | + ], |
| 51 | + "traps": ["stale-dist"], |
| 52 | + "source": ["packages/spec/src/ai/agent.zod.ts (+tool/skill)", "packages/runtime/src/route-ledger.ts ('* /ai/**')", "packages/spec/liveness/agent.json"], |
| 53 | + "history": [{ "revision": 1, "date": "2026-08-07", "change": "initial — closes the AI hole in capability coverage (no area covered agent/tool/skill kinds)", "ref": "claude/platform-test-checklist-ocwugl" }] |
| 54 | + }, |
| 55 | + { |
| 56 | + "id": "ai.mcp-http-surface", |
| 57 | + "title": "MCP HTTP transport and the public /mcp/skill endpoint behave per config", |
| 58 | + "since": "v15", |
| 59 | + "status": "active", |
| 60 | + "revision": 1, |
| 61 | + "priority": "P1", |
| 62 | + "surface": "api", |
| 63 | + "fixtures": { |
| 64 | + "app": "any", |
| 65 | + "requires": ["one boot with MCP HTTP enabled and one with it disabled (the #3358 sweep hit the disabled config and read 501)"] |
| 66 | + }, |
| 67 | + "steps": [ |
| 68 | + "boot with MCP HTTP disabled; GET /mcp and /mcp/skill; record statuses", |
| 69 | + "boot with MCP HTTP enabled; GET /mcp/skill unauthenticated", |
| 70 | + "open an MCP Streamable HTTP session against /mcp and list tools" |
| 71 | + ], |
| 72 | + "acceptance": [ |
| 73 | + { |
| 74 | + "clause": "disabled config answers a deliberate 501/refusal on /mcp — a clean 'off', not a hang or 500", |
| 75 | + "oracle": "api", |
| 76 | + "verify": "status + body of GET /mcp on the disabled boot", |
| 77 | + "evidence": "the response" |
| 78 | + }, |
| 79 | + { |
| 80 | + "clause": "GET /mcp/skill serves the public SKILL.md for agents without auth (documented server-only surface)", |
| 81 | + "oracle": "api", |
| 82 | + "verify": "200 + markdown body on the enabled boot, unauthenticated", |
| 83 | + "evidence": "the response head" |
| 84 | + }, |
| 85 | + { |
| 86 | + "clause": "an MCP client can complete the handshake and list tools on the enabled boot", |
| 87 | + "oracle": "network", |
| 88 | + "verify": "initialize + tools/list round-trip succeeds over Streamable HTTP", |
| 89 | + "evidence": "the session trace" |
| 90 | + } |
| 91 | + ], |
| 92 | + "negative": ["the enabled boot must still reject tool CALLS that need auth when the session carries none — transport up ≠ authz open"], |
| 93 | + "traps": ["dispatcher-vs-hono-route"], |
| 94 | + "source": ["packages/runtime/src/route-ledger.ts ('* /mcp/**', 'GET /mcp/skill')", "#3358 §9 (MCP HTTP 501 on the swept config)"], |
| 95 | + "history": [{ "revision": 1, "date": "2026-08-07", "change": "initial — the #3358 sweep could not drive MCP because the config was off; both sides are now explicit", "ref": "claude/platform-test-checklist-ocwugl" }] |
| 96 | + }, |
| 97 | + { |
| 98 | + "id": "ai.mcp-stdio-fail-closed", |
| 99 | + "title": "MCP stdio auto-start fails closed without an API key; with a key, reads honor RLS/FLS", |
| 100 | + "since": "v16", |
| 101 | + "status": "active", |
| 102 | + "revision": 1, |
| 103 | + "priority": "P0", |
| 104 | + "surface": "cli", |
| 105 | + "personas": ["restricted member API key (for the RLS half)"], |
| 106 | + "steps": [ |
| 107 | + "attempt MCP stdio auto-start WITHOUT OS_MCP_STDIO_API_KEY set; capture the refusal", |
| 108 | + "start it with a valid key bound to a restricted member; over MCP, read an OWD-private object", |
| 109 | + "read the same object as an admin-keyed session; compare row sets" |
| 110 | + ], |
| 111 | + "acceptance": [ |
| 112 | + { |
| 113 | + "clause": "stdio auto-start without the key refuses to serve (fail-closed), with a clear message naming the missing key", |
| 114 | + "oracle": "log", |
| 115 | + "verify": "the refusal line in the process output", |
| 116 | + "evidence": "the log line" |
| 117 | + }, |
| 118 | + { |
| 119 | + "clause": "with a member-bound key, MCP reads return only RLS-visible rows and FLS-masked fields — identical to the same member over REST", |
| 120 | + "oracle": "api", |
| 121 | + "verify": "row-set + field-set comparison MCP-vs-REST for the same member; admin session shows the full set (both sides)", |
| 122 | + "evidence": "the compared reads" |
| 123 | + } |
| 124 | + ], |
| 125 | + "negative": ["an invalid/revoked key must be rejected, not fall back to an anonymous-but-serving session"], |
| 126 | + "traps": ["wrong-persona"], |
| 127 | + "source": ["#3358 §9 (verified PASS: fail-closed guard present, reads honor RLS/FLS)"], |
| 128 | + "history": [{ "revision": 1, "date": "2026-08-07", "change": "initial import from #3358 §9", "ref": "#3358" }] |
| 129 | + }, |
| 130 | + { |
| 131 | + "id": "ai.mcp-run-action-exposure-gate", |
| 132 | + "title": "MCP run_action requires ai.exposed and audits the call", |
| 133 | + "since": "v15.1", |
| 134 | + "status": "active", |
| 135 | + "revision": 1, |
| 136 | + "priority": "P1", |
| 137 | + "surface": "api", |
| 138 | + "fixtures": { "app": "any", "requires": ["one action declared ai.exposed and one without, on the same object"] }, |
| 139 | + "steps": [ |
| 140 | + "over MCP, call run_action on the undeclared action; capture the refusal", |
| 141 | + "call the ai.exposed action; capture result", |
| 142 | + "read the audit log for the successful call" |
| 143 | + ], |
| 144 | + "acceptance": [ |
| 145 | + { |
| 146 | + "clause": "the undeclared action is rejected fail-closed (declared ≠ exposed is the gate, PD#10 discipline)", |
| 147 | + "oracle": "api", |
| 148 | + "verify": "the MCP call errors with the exposure refusal; the action did NOT execute (verify by its side effect being absent)", |
| 149 | + "evidence": "the refusal + absence check" |
| 150 | + }, |
| 151 | + { |
| 152 | + "clause": "the exposed action executes and lands in the audit log with the caller identity", |
| 153 | + "oracle": "api", |
| 154 | + "verify": "action side effect present + audit row for the call", |
| 155 | + "evidence": "result + audit row" |
| 156 | + } |
| 157 | + ], |
| 158 | + "negative": ["the refusal path must leave zero side effects — a rejected call that half-executed is a FAIL"], |
| 159 | + "source": ["docs/plans/release-15.1-test-plan.md §A9 (#2964)"], |
| 160 | + "history": [{ "revision": 1, "date": "2026-08-07", "change": "initial import from the 15.1 plan §A9", "ref": "claude/platform-test-checklist-ocwugl" }] |
| 161 | + }, |
| 162 | + { |
| 163 | + "id": "ai.mcp-validate-expression", |
| 164 | + "title": "MCP validate_expression returns errors/warnings/inferred type against an object schema", |
| 165 | + "since": "v16", |
| 166 | + "status": "active", |
| 167 | + "revision": 1, |
| 168 | + "priority": "P2", |
| 169 | + "surface": "api", |
| 170 | + "fixtures": { "app": "any", "requires": ["MCP HTTP enabled (see ai.mcp-http-surface)"] }, |
| 171 | + "steps": [ |
| 172 | + "over MCP, validate a correct formula against a seeded object schema", |
| 173 | + "validate a formula with a type error (e.g. date arithmetic) and one referencing a nonexistent field" |
| 174 | + ], |
| 175 | + "acceptance": [ |
| 176 | + { |
| 177 | + "clause": "the valid expression returns ok with an inferred type; the invalid ones return located errors naming the problem (bad arithmetic, unknown field)", |
| 178 | + "oracle": "api", |
| 179 | + "verify": "three calls, three verdicts matching the formula gates the build enforces (api-backend.formula-gates)", |
| 180 | + "evidence": "the three responses" |
| 181 | + } |
| 182 | + ], |
| 183 | + "source": ["#3358 §9 (underlying validateExpression works; MCP path was blocked on the disabled transport)"], |
| 184 | + "history": [{ "revision": 1, "date": "2026-08-07", "change": "initial import from #3358 §9", "ref": "#3358" }] |
| 185 | + } |
| 186 | + ] |
| 187 | +} |
0 commit comments