|
1 | 1 | { |
2 | 2 | "$comment": "Standing platform test checklist — area ledger. Hand-edited, append-only; validated by scripts/check-platform-checklist.mjs (pnpm check:platform-checklist). Authoring rules: docs/qa/platform-checklist/README.md · execution protocol: RUNNER.md.", |
3 | 3 | "area": "cli", |
4 | | - "title": "CLI — os dev/build/migrate/verify boot & exit-code contracts, scaffold first-run, flag/command error UX", |
| 4 | + "title": "CLI — os dev/build/migrate/verify/test boot, load, execution & exit-code contracts, scaffold first-run, flag/command error UX", |
5 | 5 | "items": [ |
6 | 6 | { |
7 | 7 | "id": "cli.dev-boot-contract", |
|
427 | 427 | { "revision": 1, "date": "2026-08-07", "change": "new item: the published first-run experience mirrored step-for-step from scaffold-e2e.yml, template matrix as variants, RC-window protocol refusal recorded as gate-working instead of failure", "ref": "claude/platform-test-checklist-ocwugl" } |
428 | 428 | ] |
429 | 429 | }, |
| 430 | + { |
| 431 | + "id": "cli.qa-suite-execution", |
| 432 | + "title": "os test: a Quality Protocol suite is validated at LOAD, executed against a booted app, and its verdict is the exit code — capture/interpolation thread state, an unevaluable assertion FAILS", |
| 433 | + "since": "v17", |
| 434 | + "status": "active", |
| 435 | + "revision": 1, |
| 436 | + "priority": "P1", |
| 437 | + "surface": "cli", |
| 438 | + "personas": ["operator (local shell)", "suite author (writes qa/*.test.json)"], |
| 439 | + "fixtures": { |
| 440 | + "app": "showcase", |
| 441 | + "requires": [ |
| 442 | + "a free non-default port and an ephemeral boot: `pnpm dev -- --fresh -p <port>` from the repo root (showcase + --seed-admin; dogfood skill §0)", |
| 443 | + "a session token for the data steps — POST /api/v1/auth/sign-in/email with the seeded admin (admin@objectos.ai / admin123) and read `.token`; pass it as `os test --token <token>` (the CLI sends it as `Authorization: Bearer`)", |
| 444 | + "the shipped fixture suite examples/app-showcase/qa/platform-smoke.test.json — the repo's only Quality Protocol suite, authored so `os test`'s DEFAULT glob (`qa/*.test.json`) finds it from that directory with no argument", |
| 445 | + "a scratch directory with its own qa/ for the negative and variant probes — deliberately broken suites are STAGED per run, never committed, so the repo's own suite stays green" |
| 446 | + ], |
| 447 | + "knownGaps": [ |
| 448 | + "five of the eight action types cannot reach a stock server at all (see `negative`): `HttpTestAdapter` builds `${baseUrl}/api/data/:object` while the server serves `{basePath}/data/:object` with basePath `/api/v1`. The variant sweep therefore records five refusals, and that is the honest verdict — not a fixture gap to work around by rewriting the probes as `api_call`", |
| 449 | + "no scenario SELECTION exists: `os test` has exactly two flags (--url, --token), `scenario.tags` filters nothing and `scenario.requires` is never checked (packages/spec/liveness/qa.json rows `tags`/`requires`), so the whole glob always runs and a suite cannot declare a precondition it will be skipped for" |
| 450 | + ] |
| 451 | + }, |
| 452 | + "steps": [ |
| 453 | + "boot the app: from the repo root `pnpm dev -- --fresh -p <port>`; poll `curl -s -m3 -o /dev/null -w '%{http_code}' http://localhost:<port>/api/v1/health` until 200", |
| 454 | + "mint a token: `curl -s -X POST http://localhost:<port>/api/v1/auth/sign-in/email -H 'Content-Type: application/json' -H 'Origin: http://localhost:<port>' -d '{\"email\":\"admin@objectos.ai\",\"password\":\"admin123\"}'` and read `.token`", |
| 455 | + "the green path on the DEFAULT glob: `cd examples/app-showcase && os test --url http://localhost:<port> --token <token>; echo $?` — capture the whole transcript including the `Found N test suites.` line and the summary", |
| 456 | + "prove the teardown ran: list `GET /api/v1/data/showcase_category` before and after and confirm the row the run created is gone (the run leaves the fixture exactly as it found it)", |
| 457 | + "prove the run repeats: execute the same command a second time and confirm it is green again with no leftover row — a suite that only passes once is a suite whose teardown is decorative", |
| 458 | + "stage the negative probes in a scratch dir's qa/: (a) a suite with `steps` misspelled as `stpes`, (b) a suite asserting `data.status equals degraded` against /api/v1/health, (c) a suite asserting `contains` over `data.nope`, a path the health response does not carry. Run `os test 'qa/*.test.json' --url http://localhost:<port>; echo $?` and capture all three refusals in one transcript", |
| 459 | + "prove the malformed suite never ran: the server log carries no request attributable to it, and the transcript shows the two OTHER staged suites still executing after the refusal", |
| 460 | + "teardown-after-failure: stage a scenario that creates a record, captures its id, then fails a deliberate assertion, with a teardown that deletes `{{id}}`; run it and confirm via the API that the record is gone despite the red verdict", |
| 461 | + "variant sweep: stage one scenario per TestActionTypeSchema member (8) and run them in one suite; record each member's own verdict and the exact error text of every refusal — no member inferred from a sibling", |
| 462 | + "zero-match glob: `os test 'qa/nothing-matches-*.test.json' --url http://localhost:<port>; echo $?` and record BOTH the message and the exit code", |
| 463 | + "glob discipline (#7363): `os test 'qa/**/*.test.json'` from the app dir — it must complete in seconds and find the same suite, never the multi-minute walk that preceded the lazy resolver", |
| 464 | + "persona check: re-run the green suite with NO --token and record which scenario survives (health) and which fails (the data roundtrip, HTTP 401)" |
| 465 | + ], |
| 466 | + "acceptance": [ |
| 467 | + { |
| 468 | + "clause": "the green path closes: every scenario reports ✅ with its scenarioId and a duration, the summary reads `SUCCESS: All N scenarios passed.`, and the process exits 0", |
| 469 | + "oracle": "log", |
| 470 | + "verify": "the transcript shows one ✅ line per scenario and `echo $?` prints 0 — AND the evidence quotes the `Found N test suites.` line with N > 0, because a run that discovered nothing also exits 0 (see `negative`)", |
| 471 | + "evidence": "the full os test transcript + exit code" |
| 472 | + }, |
| 473 | + { |
| 474 | + "clause": "state threads through the run: a step's `capture` writes into the scenario context and `{{var}}` interpolation reads it back into a LATER step's action, so the read-back step addresses the record the create step actually made", |
| 475 | + "oracle": "api", |
| 476 | + "verify": "the roundtrip scenario passes AND the id it captured is confirmed independently — list/GET the object over the API and match the record the run created; a green scenario whose URL still contains the literal `{{categoryId}}` would be interpolation dead and the step failing for the wrong reason", |
| 477 | + "evidence": "the scenario transcript + the independent API read of the same id" |
| 478 | + }, |
| 479 | + { |
| 480 | + "clause": "a false assertion FAILS the scenario and the run: the scenario is ❌, the failing step is named, the message names the field, the expected and the actual value, and the process exits 1", |
| 481 | + "oracle": "log", |
| 482 | + "verify": "the staged `equals degraded` probe prints `Assertion failed: data.status expected degraded, got ok` under `Step Failed: <step name>`; `echo $?` prints 1", |
| 483 | + "evidence": "the probe transcript + exit code" |
| 484 | + }, |
| 485 | + { |
| 486 | + "clause": "#7256 — an assertion the engine CANNOT evaluate fails, it does not pass: `contains` against a path the response does not carry names the runtime type it found and says which of the fixture or the assertion is the suspect", |
| 487 | + "oracle": "log", |
| 488 | + "verify": "the `contains` over `data.nope` probe fails with `cannot be evaluated by 'contains' … got undefined` plus the `is_null` hint — a ✅ here is the exact silent-pass regression the clause exists for", |
| 489 | + "evidence": "the probe transcript" |
| 490 | + }, |
| 491 | + { |
| 492 | + "clause": "#6247 — load-site enforcement holds: a document TestSuiteSchema rejects is refused BEFORE execution, naming the file, every offending path, the expected shape and the doc reference; it counts as one failed suite; the rest of the glob still runs; the run exits 1", |
| 493 | + "oracle": "log", |
| 494 | + "verify": "the malformed probe prints `<file> is not a valid Quality Protocol suite (TestSuiteSchema)` with `✗ scenarios.0.steps: …`, the two sibling suites still execute after it, and the summary counts it as failed — never `SUCCESS` and never a TypeError attributed to the runner", |
| 495 | + "evidence": "the three-suite transcript + exit code" |
| 496 | + }, |
| 497 | + { |
| 498 | + "clause": "teardown runs even after a failed step, so a red run does not leave the fixture dirty — and a passing teardown never rescues an already-failed scenario", |
| 499 | + "oracle": "api", |
| 500 | + "verify": "after the staged fail-then-teardown scenario the record created in step 1 is absent from the object listing, while the scenario's verdict stays ❌", |
| 501 | + "evidence": "the scenario verdict + the before/after listings" |
| 502 | + }, |
| 503 | + { |
| 504 | + "clause": "every one of the 8 TestActionTypeSchema members carries its OWN recorded verdict — executed and asserted, or refused with a named error — and none is inferred from a sibling", |
| 505 | + "oracle": "log", |
| 506 | + "verify": "the sweep transcript shows all 8 scenario ids with their individual outcomes; refusals are quoted verbatim (the adapter's `Unsupported action type in HttpAdapter: <type>` and any HTTP error) rather than summarised as 'the record actions don't work'", |
| 507 | + "evidence": "the 8-scenario sweep transcript" |
| 508 | + }, |
| 509 | + { |
| 510 | + "clause": "#7363 glob discipline: the default `qa/*.test.json` resolves relative to the invocation directory, a `**` pattern completes in seconds, and matches run in sorted order", |
| 511 | + "oracle": "log", |
| 512 | + "verify": "the bare `os test` from examples/app-showcase finds the shipped suite with no argument; the quoted `qa/**/*.test.json` run finds the same suite and returns in seconds — the pre-#7363 resolver died at exit 134 after ~7 minutes on this monorepo", |
| 513 | + "evidence": "both transcripts with wall-clock timings" |
| 514 | + } |
| 515 | + ], |
| 516 | + "negative": [ |
| 517 | + "a zero-match glob prints `No test files found matching: <pattern>` and exits **0** (measured 2026-08-11) — a green exit from a run that loaded no suite. A run record that ticks this item without quoting a `Found N test suites.` line with N > 0 is the false positive this item exists to prevent, and a CI job wired to `os test` with a typo'd path would report success forever", |
| 518 | + "a malformed suite that REACHES the runner — reported passed having executed nothing, or dying as a TypeError inside runSuite with no file named — is the #6247 regression", |
| 519 | + "a `contains` against a path the result does not carry reporting ✅ is the #7256 regression", |
| 520 | + "the five record-shaped action types (`create_record`, `read_record`, `update_record`, `delete_record`, `query_records`) answer **HTTP 404** against a stock server: `HttpTestAdapter` builds `${baseUrl}/api/data/:object` while the server serves `{basePath}/data/:object` with basePath `/api/v1` (and `update_record` issues PUT where the route is PATCH). Measured 2026-08-11 on showcase. A run must record those four-oh-fours as the measured verdict — recording them as 'not applicable' hides the finding, and a future run that finds them PASSING means the adapter was repaired, which is a revision of this item rather than a silent green", |
| 521 | + "satisfying this item by executing the unit pins in `source` instead of driving a booted app is not a run: those pins cover the load boundary, the glob resolver and one assertion operator, and none of them proves a suite reaches a real server — which is why this item carries no `automated` entry" |
| 522 | + ], |
| 523 | + "variants": [ |
| 524 | + "api_call — the only action type that reaches the real API surface (target is a path or absolute URL; `payload.method`/`payload.body` shape the request)", |
| 525 | + "wait — local, no HTTP; resolves `{ waited: <ms> }` from `payload.duration`", |
| 526 | + "create_record — refused: POST ${baseUrl}/api/data/:object → 404 (see `negative`)", |
| 527 | + "read_record — refused: GET ${baseUrl}/api/data/:object/:id → 404", |
| 528 | + "update_record — refused: PUT ${baseUrl}/api/data/:object/:id → 404 (and PUT, where the route is PATCH)", |
| 529 | + "delete_record — refused: DELETE ${baseUrl}/api/data/:object/:id → 404", |
| 530 | + "query_records — refused: POST ${baseUrl}/api/data/:object/query → 404", |
| 531 | + "run_script — declared in the enum with NO adapter branch: `Unsupported action type in HttpAdapter: run_script` (loud, recorded in packages/spec/liveness/qa.json)" |
| 532 | + ], |
| 533 | + "enumSource": { |
| 534 | + "file": "packages/spec/src/qa/testing.zod.ts", |
| 535 | + "export": "TestActionTypeSchema", |
| 536 | + "expect": 8 |
| 537 | + }, |
| 538 | + "traps": ["dispatcher-vs-hono-route", "stale-dist", "wrong-persona"], |
| 539 | + "source": [ |
| 540 | + "packages/cli/src/commands/test.ts (the shipped `os test`: the #7363 lazy segment-directed glob with its prune list, `loadTestSuite`'s #6247 boundary parse, the per-scenario report, and the exit 0/1 summary)", |
| 541 | + "packages/core/src/qa/runner.ts (scenario sequencing, `capture` + `{{var}}` interpolation, the assertion operators, setup/teardown semantics, the #7256 unevaluable-`contains` fix)", |
| 542 | + "packages/core/src/qa/http-adapter.ts (the action-type switch — its case labels ARE the enum values; the `/api/data/...` record routes this item's variants measure against the server's `/api/v1/data/...`)", |
| 543 | + "packages/spec/src/qa/testing.zod.ts (TestSuiteSchema — the shape enforced at load; TestActionTypeSchema pinned above)", |
| 544 | + "packages/spec/liveness/qa.json (the ADR-0049 ledger whose existence this item is coverage.json's mapping for — its dead `tags`/`requires` rows are why no scenario selection exists)", |
| 545 | + "packages/cli/test/qa-suite-schema-load.test.ts, packages/cli/test/resolve-glob-lazy-walk.test.ts, packages/core/src/qa/runner.test.ts (the three unit pins — cited so a run knows what is already covered, NOT a substitute for driving a booted app)", |
| 546 | + "content/docs/deployment/cli.mdx §os test (the documented command contract)", |
| 547 | + "examples/app-showcase/qa/platform-smoke.test.json (the fixture suite this item drives)" |
| 548 | + ], |
| 549 | + "history": [ |
| 550 | + { "revision": 1, "date": "2026-08-11", "change": "new item: the `qa` capability's coverage.json mapping, authored rather than waived (#7347 triage ruling). `os test` is a shipped, documented CLI command, so the honest mapping is a surface:cli item that authors a real qa/*.test.json suite and drives it against a booted app — the fixture suite examples/app-showcase/qa/platform-smoke.test.json lands with this item and is the repo's first Quality Protocol suite. Every clause was measured on showcase before it was written: the green path, capture/interpolation, the #6247 load refusal, the #7256 unevaluable-contains failure, teardown-after-failure, the 8-member action-type sweep and the #7363 glob. `since: v17` records the release in which the surface became GOVERNED (liveness ledger seeded + TestSuiteSchema enforced at the load site, #6247 / PR #7255); the command itself predates it. No `automated` entry: the three unit pins cover pieces, none of them proves a suite reaches a real server", "ref": "#7347" } |
| 551 | + ] |
| 552 | + }, |
430 | 553 | { |
431 | 554 | "id": "cli.flag-command-error-ux", |
432 | 555 | "title": "Wrong flags and unknown commands error with usage and a nonzero exit — never silently ignored, never executed anyway", |
|
0 commit comments