diff --git a/.github/workflows/product-validation.yml b/.github/workflows/product-validation.yml new file mode 100644 index 0000000..6736be0 --- /dev/null +++ b/.github/workflows/product-validation.yml @@ -0,0 +1,42 @@ +name: product-validation + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: {node-version: 22} + - name: Validate exact candidate + run: | + set -o pipefail + install -m 755 scripts/tabellio-validator.mjs "$RUNNER_TEMP/tabellio-validator" + PATH="$RUNNER_TEMP:$PATH" node scripts/tabellio-validate.mjs gate \ + --repo . \ + --repo-id IntelIP/Tabellio \ + --base origin/main \ + --commit HEAD \ + --manifest tabellio.validation.json | tee tabellio-validation-result.json + - name: Upload validation result + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: tabellio-validation-${{ github.event.pull_request.head.sha || github.sha }} + path: | + tabellio-validation-result.json + refs/tabellio/validations/ + if-no-files-found: warn diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index ea40c9e..0e63c39 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -52,6 +52,8 @@ jobs: fallow audit \ --base "origin/${{ github.base_ref }}" \ --gate new-only \ + --health-baseline quality-baselines/fallow-health.json \ + --dupes-baseline quality-baselines/fallow-dupes.json \ --format json \ --quiet \ --explain \ diff --git a/.gitignore b/.gitignore index 51b5107..7a96f85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ -.tabellio/ +.tabellio/* +!.tabellio/ +!.tabellio/validators.json agentic-pr-evidence.json tabellio-pr-evidence.json tabellio-context.json diff --git a/.tabellio/validators.json b/.tabellio/validators.json new file mode 100644 index 0000000..833b371 --- /dev/null +++ b/.tabellio/validators.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "tabellio-adapter/v0.1", + "profiles": { + "static": { + "commands": [["npm", "run", "check"]], + "metrics": [], + "cost": {"telemetry": "not_applicable", "usd": null, "modelCalls": null, "toolCalls": null}, + "summary": "Control-plane tests and repository checks" + }, + "schema": { + "commands": [["npm", "run", "tabellio:platform:check"]], + "metrics": [{"name": "platform_contract_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Platform and schema contract checks" + }, + "semantic": { + "commands": [["npm", "run", "tabellio:validate:example:check"]], + "metrics": [{"name": "validation_contract_examples_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Typed validation examples and evidence contracts" + }, + "workflow": { + "commands": [["npm", "run", "tabellio:run:example:check"], ["npm", "run", "tabellio:review:example:check"]], + "metrics": [{"name": "run_review_workflows_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Agent run and review lifecycle examples" + }, + "security": { + "commands": [["node", "scripts/check-tabellio-external-actions.mjs", "--evidence", "examples/tabellio-evidence/minimal-evidence.json"]], + "metrics": [{"name": "external_action_policy_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "External-action and side-effect policy checks" + } + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe8308..603ace1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to Tabellio are recorded here. ## Unreleased +## 0.4.0 - 2026-07-16 + +### Added + +- Product-validity manifests that bind acceptance outcomes, invariants, forbidden outcomes, risk, and required validator types to an exact candidate commit. +- Typed schema, semantic, workflow, visual, operational, and security evidence with bounded metrics, artifact digests, and cost telemetry. +- Fail-closed `passed`, `failed`, and `blocked` decisions with threshold and cost-budget enforcement. +- `tabellio-validate gate` for CI enforcement without external JSON parsing. +- `tabellio-validator` for repository-declared shell-free adapter commands, metric extraction, and common evidence emission. + +### Changed + +- Product-changing work now requires evidence beyond command exit status when the manifest declares typed validators. +- Missing required evidence, missing metrics, command errors, timeouts, and unknown required cost telemetry block readiness. +- Existing v0.1 manifests and v0.1/v0.2 result readers remain supported during migration. + +### Release Gates + +- `tabellio-preflight --profile release` +- `npm run check` +- Fallow whole-repository and changed-code scans +- `npm pack --dry-run --json` +- Exact merged-head Tabellio validation + ## 0.3.0 - 2026-07-15 ### Added diff --git a/README.md b/README.md index 2ff8f8c..dfc6c74 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Tabellio attaches a structured evidence packet to a pull request. The packet is | Git | Repository, base branch, head branch, commit SHA, and PR metadata | | Runtime | Human, CI, or agent runtime that produced the change | | Diff | Changed files | -| Validation | Commands run and check results | +| Validation | Exact-head commands plus structural, semantic, workflow, visual, operational, and security results | | Approvals | Required, granted, denied, or skipped approvals | | Side effects | Deployment, migration, infra, billing, secret, provider, and destructive-action policy | | Artifacts | Evidence files generated by the run | @@ -58,7 +58,7 @@ AI-assisted pull requests should not depend on reviewer trust alone. Tabellio gi | Layer | Tooling | Role | | --- | --- | --- | | Runtime | [Node.js 20+](https://nodejs.org/) | Runs the local writer and validators | -| Validation | `tabellio-validate` on any trusted worker | Runs an exact committed command manifest and stores results on a Git ref | +| Validation | `tabellio-validate` on any trusted worker | Runs exact committed command or typed product validators and stores results on a Git ref | | Evidence contract | [JSON Schema](https://json-schema.org/) | Validates the evidence envelope and external-action policy | | Code storage | [GitHub](https://github.com/) | Stores code refs and tags; provides a thin pull-request shell | | Stacked review | [git-spice](https://abhinav.github.io/git-spice/) | Stack engine for small dependent GitHub pull requests | @@ -103,6 +103,12 @@ npm run tabellio:platform:check node scripts/tabellio-validate.mjs run --repo . --commit HEAD --manifest tabellio.validation.json ``` +Use `gate` in CI. It persists the same exact-head result but exits non-zero unless the final decision is `passed`: + +```bash +tabellio-validate gate --repo . --repo-id owner/repository --base main --commit HEAD --manifest tabellio.validation.json +``` + Keep `origin` limited to ordinary code branches and tags. Configure a separate private GitHub repository under another remote name before publishing control refs. The transport refuses to target `origin`. Before agent or release work, run: @@ -162,6 +168,7 @@ node scripts/tabellio-run.mjs promote --run-id run-42 --repo . ``` See [Agent run lifecycle](docs/agent-run-lifecycle.md) for state and failure behavior. +See [Product validation](docs/product-validation.md) for acceptance contracts, typed evidence, cost gates, and repository adapters. Package scripts: diff --git a/docs/codex-review.md b/docs/codex-review.md index 7edc9af..96272fc 100644 --- a/docs/codex-review.md +++ b/docs/codex-review.md @@ -36,3 +36,5 @@ Codex review is not: - deployment approval - substitute for human ownership - proof that agent output is secure + +When the repository commits a product-validation manifest, review begins only after the exact pull-request head has a `passed` v0.3 decision. A `blocked` decision means the evidence is incomplete or untrustworthy; it is not a product pass and should not be converted into reviewer guesswork. diff --git a/docs/design-memory.md b/docs/design-memory.md new file mode 100644 index 0000000..9097dfd --- /dev/null +++ b/docs/design-memory.md @@ -0,0 +1,63 @@ +# Product Design Memory + +Tabellio can attest visual intent and regression evidence without becoming a design tool or screenshot database. + +## Ownership + +Product repositories own committed design memory: + +```text +design/ + product.design.json + baselines/manifest.json + decisions/*.md +``` + +`tabellio-product-design/v0.1` records visual thesis, canonical token and component sources, reference lock, forbidden patterns, required themes, viewports, states, accessibility policy, and baseline manifest location. Source digests detect drift between the profile and the files it claims as authority. + +`tabellio-visual-baselines/v0.1` records human-approved captures by surface, viewport, theme, and state. Each capture points to a durable artifact URI with SHA-256 digest. `file:` URIs are rejected because detached validation worktrees are removed after a run. + +`policy.surfaces` declares the required route or component matrix. Approved manifests must cover every declared viewport, theme, and state combination. + +`tabellio-ui-review/v0.1` records model judgment as typed evidence: exact commit and profile digests, reviewed capture digests, model identity, cost telemetry, structured findings, verdict, and blockers. It complements deterministic checks; it never approves or rewrites baselines. + +New products may commit a `proposed` empty baseline manifest while capture work is pending. A capture run may replace it with a complete proposed manifest bound to the captured commit. Proposed manifests keep approval fields null and always block product readiness until a human promotes them to `approved`. + +Git stores design history. Tabellio binds the profile, baselines, visual metrics, critique artifacts, and validation decision to the exact candidate commit. + +## Validation + +Check a product repository: + +```bash +tabellio-design-memory \ + --repo . \ + --profile design/product.design.json +``` + +The checker verifies profile and baseline shape, repository-relative source paths, source digests, profile digest, product identity, viewport/state/theme references, and durable artifact URIs. + +Finalize a product-owned PNG capture matrix into a proposed manifest: + +```bash +tabellio-design-captures \ + --repo . \ + --profile design/product.design.json \ + --captures-dir .artifacts/tabellio/visual-captures \ + --artifact-base-uri "artifact+github://OWNER/REPO/actions/runs/RUN_ID/design-captures/" \ + --source-commit "$GITHUB_SHA" \ + --out .artifacts/tabellio/visual-baseline-candidate.json +``` + +Capture files use `------.png`. Finalization fails when any declared matrix cell is absent, a file is not PNG, the commit is not exact, or the artifact URI is ephemeral. Output creation is exclusive; existing evidence is never overwritten. + +Use two visual validators when product readiness matters: + +- `visual-contract`: deterministic capture completeness, token/profile checks, fonts, overflow, accessibility, and visual diff metrics. +- `visual-judgment`: structured model critique for hierarchy, signifiers, grouping, density, feedback, and reference fidelity. + +Require model cost telemetry. Missing profile, baselines, artifacts, metrics, or required cost telemetry is `blocked`. Never update an approved baseline automatically. + +## Rollout + +Start with one product-specific design system and one unrelated product. Keep profile files small by referencing canonical sources instead of copying full tokens or component inventories. Add a cross-product read-only catalog only after profiles prove portable. diff --git a/docs/operations-hardening.md b/docs/operations-hardening.md index 1a63af2..d46d1aa 100644 --- a/docs/operations-hardening.md +++ b/docs/operations-hardening.md @@ -11,9 +11,10 @@ Tabellio keeps code and control state in standard Git objects, but production sa ## Bounded Work -- Validation manifests allow at most 50 commands, 100 arguments per command, and one-hour command timeouts. +- Validation manifests allow at most 50 commands or validators, 100 arguments per entry, and one-hour command timeouts. - Timed-out commands receive `SIGTERM`, then `SIGKILL` after one second. Fail-fast suites mark remaining commands skipped. - Full stdout and stderr are hashed; only the newest 16 KiB of each stream is retained. +- Typed evidence files are limited to 1 MiB, 100 metrics, and 50 immutable artifact references. Required missing cost telemetry blocks validation instead of being treated as zero spend. - Agent reviews allow at most 1,000 findings. Review cycles bound feedback, fixes, check statuses, titles, bodies, summaries, event details, and retained event history. - Remote control-ref reads and atomic pushes use a 15-minute timeout. Local atomic ref updates use a 30-second timeout. diff --git a/docs/product-validation.md b/docs/product-validation.md new file mode 100644 index 0000000..605b23c --- /dev/null +++ b/docs/product-validation.md @@ -0,0 +1,84 @@ +# Product Validation + +Tabellio product validation answers a stronger question than “did tests exit zero?”: did the exact committed candidate satisfy its declared user outcomes while staying inside semantic, visual, operational, cost, and safety boundaries? + +## Contracts + +`tabellio-validation/v0.2` binds three layers in one committed manifest: + +1. `acceptance` records the source request, risk, required outcomes, invariants, forbidden outcomes, and required validator types. +2. `validators` run shell-free argument arrays in the exact-head detached worktree. +3. `policy` applies metric thresholds and cost requirements to a bounded evidence report. + +Validator types are `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security`. Static validators can use command exit status alone. Every other type must emit `tabellio-validator-evidence/v0.1` at its declared relative path. + +## Evidence + +An evidence report contains: + +- `status`: the adapter’s passed, failed, or blocked observation; +- numeric metrics with explicit units; +- cost telemetry state, USD cost, model calls, and tool calls; +- immutable artifact metadata for screenshots, traces, reports, or source snapshots. + +The runner validates the evidence shape and validator identity, embeds the bounded report in the durable result, and evaluates policy itself. Adapter-reported `passed` cannot override a failed metric threshold or exceeded cost cap. + +## Decision Semantics + +| Decision | Meaning | +| --- | --- | +| `passed` | Every required validator completed with trustworthy evidence inside all thresholds. | +| `failed` | Trustworthy evidence proves a required product or budget threshold was missed. | +| `blocked` | Required proof is missing or untrustworthy because of invalid evidence, missing metrics or cost telemetry, command errors, or timeouts. | + +Fail closed. Never translate `blocked` into `passed`, and never treat missing cost telemetry as zero. + +## Repository Adapters + +Keep product knowledge in each repository. The adapter runs the existing eval, browser, design, cost, or security harness and writes the common evidence contract. + +Use `tabellio-validator` for simple adapters. Commit `.tabellio/validators.json` with `tabellio-adapter/v0.1` profiles containing shell-free command arrays, evidence metrics, cost telemetry, and a concise summary. Metrics may map command pass/fail to numeric values or extract a numeric capture group from bounded command output. + +- AI systems: stable golden cases in pull requests; larger live suites on schedules or release gates. +- Web applications: critical journeys, desktop/mobile screenshot comparisons, loaded-font assertions, overflow, and accessibility metrics. +- Operational systems: projected and observed provider usage, worker counts, polling frequency, rate limits, latency, and spend. +- Structured outputs: strict schema validation plus semantic oracles that catch plausible but wrong values. + +Use fixtures and isolated namespaces in pull requests. Live external mutation remains separately approval-gated. + +## Review Integration + +Review sync reads only the newest validation result stored under the exact pull-request head. `passed` becomes a successful local check; `failed` or `blocked` prevents merge readiness. A new head requires new validation. + +Run contract checks with: + +```bash +node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json +node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json +``` + +Run the exact-head suite with: + +```bash +node scripts/tabellio-validate.mjs run \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +Use the fail-closed command in CI: + +```bash +tabellio-validate gate \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +`gate` still records the result. It exits non-zero when the decision is `failed` or `blocked`. + +Artifact upload and path-based automatic validator selection remain scheduler or repository-policy responsibilities. The first contract keeps evaluation portable without creating a second CI or deployment authority. diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md new file mode 100644 index 0000000..b510b73 --- /dev/null +++ b/docs/releases/v0.4.0.md @@ -0,0 +1,12 @@ +Tabellio v0.4.0 adds product-validity and cost-aware validation. + +## Highlights + +- Acceptance contracts declare user outcomes, invariants, forbidden outcomes, risk, and required evidence types. +- Typed validators cover schema, semantics, workflows, visuals, operations, cost, and security. +- Evidence carries numeric metrics, explicit units, cost telemetry, tool and model call counts, and immutable artifact metadata. +- Policy evaluation belongs to the runner. Adapter-reported success cannot override a missed threshold or cost cap. +- Missing or untrustworthy required proof is `blocked`, never an implied pass. +- `tabellio-validate gate` records exact-head results and exits non-zero unless the decision is `passed`. + +Existing v0.1 validation manifests remain readable. Live provider calls, deployments, infrastructure changes, and publication remain separately approval-gated. diff --git a/docs/validation-runner.md b/docs/validation-runner.md index a814a40..b042f22 100644 --- a/docs/validation-runner.md +++ b/docs/validation-runner.md @@ -24,6 +24,12 @@ Tabellio validation replaces the GitHub Actions runtime dependency with a small Commands are executable-plus-argument arrays. Tabellio never invokes a shell, interpolates command strings, or reads a newer working-tree manifest while validating an older commit. +Manifest v0.1 remains supported. Product-facing repositories can use `tabellio-validation/v0.2`, which adds an immutable acceptance contract and typed `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security` validators. Every required validator type must have a required validator. Non-static validators must write a bounded `tabellio-validator-evidence/v0.1` report. + +Validator policy can require numeric metric thresholds, cost telemetry, and a maximum validation cost. Missing or invalid evidence, missing required metrics, unavailable required cost telemetry, command errors, and timeouts produce `blocked`. Valid evidence that misses a product threshold or cost budget produces `failed`. Only complete required evidence inside every boundary produces `passed`. + +See [Product validation](product-validation.md) and [the example manifest](../examples/tabellio-validation/product-manifest.json). + ## Run ```bash @@ -47,7 +53,7 @@ The runner: 7. Removes the worktree even after failure. 8. Writes an integrity-protected result to `refs/tabellio/validations` with compare-and-swap retries. -New results use `tabellio-validation-result/v0.2`. They require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 results. +Command-manifest results use `tabellio-validation-result/v0.2`. Product-validation results use v0.3 and embed the acceptance digest, typed validator results, bounded evidence reports, total observed validation cost, and final policy decision. Both require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 and v0.2 results. Read the newest result for a commit: @@ -64,3 +70,5 @@ Publishing `refs/tabellio/validations` to a remote is a separate approved Git wr ## Security Boundary Detached worktrees and isolated home directories protect host source and common credential paths; they are not a hostile-code sandbox. Validation commands can execute arbitrary repository code and use the network. Run untrusted contributions in a disposable VM, container, or sandbox with scoped credentials and network policy. Never place secrets in manifest arguments or print them: argv and bounded output tails are retained as durable evidence. + +Typed validation does not authorize live writes. Repository adapters should use fixtures, isolated namespaces, read-only credentials, and explicit cost caps. Artifact metadata must point to durable content-addressed storage when a reviewer needs the screenshot, trace, or report after the temporary worktree is removed. diff --git a/docs/workflow-model.md b/docs/workflow-model.md index 19bef23..dfd6dc5 100644 --- a/docs/workflow-model.md +++ b/docs/workflow-model.md @@ -12,6 +12,8 @@ Tabellio turns an agentic coding run into a GitHub-bound context packet and, whe | Workspace | Isolated Git worktree assigned to one agent run | | Context packet | Integrity-protected binding between task, actor, exact commits, changed files, checkpoints, and merge preview | | Evidence envelope | JSON record of Git state, changed files, commands, checks, approvals, side-effect policy, and artifacts | +| Acceptance contract | Committed outcomes, invariants, forbidden outcomes, risk, and required validator types | +| Validator evidence | Bounded metrics, cost telemetry, and immutable artifact references emitted by one product validator | | External action policy | Default-deny policy for deploys, migrations, infra, DNS, billing, secrets, provider reads, and destructive actions | | Pull request | Human review surface for the diff plus evidence summary | | Stack | Ordered PR chain for larger changes | @@ -34,7 +36,7 @@ task -> resolve immutable base commit -> isolated worktree and agent branch -> small commits and mandatory Entire checkpoint metadata - -> deterministic checks + -> deterministic and product-validity checks -> context packet -> read-only merge preview -> evidence envelope @@ -60,7 +62,7 @@ Each PR should expose: The context packet is usable without a pull request. GitHub is the review and code-distribution surface; Tabellio's evidence remains independently verifiable. -Evidence is not a claim that the work is correct. Evidence is the record reviewers inspect before trusting the work. +Raw evidence is not automatically a correctness claim. A typed validation decision proves that the exact committed evidence met the committed thresholds. Human review still owns product judgment outside the declared contract. ## Stack Discipline @@ -112,6 +114,6 @@ Future versions can add: - SLSA provenance export - in-toto link metadata - OpenTelemetry spans -- model/tool eval suites +- path-based validator policy selection - automated git-spice submission and merge orchestration for GitHub - GitHub merge queue metadata diff --git a/examples/tabellio-design-memory/baselines.json b/examples/tabellio-design-memory/baselines.json new file mode 100644 index 0000000..7fc012e --- /dev/null +++ b/examples/tabellio-design-memory/baselines.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": "tabellio-visual-baselines/v0.1", + "status": "approved", + "productId": "example/product", + "profilePath": "examples/tabellio-design-memory/product.design.json", + "profileDigest": "1304dd93039569be68388cd313c60621d85a289e196738f3e25a84a995f34767", + "sourceCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "approvedAt": "2026-07-16T20:00:00.000Z", + "approvedBy": {"type": "human", "id": "product-owner"}, + "captures": [ + { + "id": "home-desktop-light-default", + "surface": "/home", + "kind": "route", + "viewport": "desktop", + "theme": "light", + "state": "default", + "artifact": { + "uri": "https://artifacts.example.invalid/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "mediaType": "image/png", + "bytes": 1234 + }, + "maskSelectors": [], + "maxDiffPixelRatio": 0.01 + } + ] +} diff --git a/examples/tabellio-design-memory/components.json b/examples/tabellio-design-memory/components.json new file mode 100644 index 0000000..6368389 --- /dev/null +++ b/examples/tabellio-design-memory/components.json @@ -0,0 +1,4 @@ +{ + "registry": "@example", + "components": ["button", "status-badge"] +} diff --git a/examples/tabellio-design-memory/decision.md b/examples/tabellio-design-memory/decision.md new file mode 100644 index 0000000..1579912 --- /dev/null +++ b/examples/tabellio-design-memory/decision.md @@ -0,0 +1,3 @@ +# Initial direction + +Use calm, high-trust product surfaces with semantic color and explicit interaction states. diff --git a/examples/tabellio-design-memory/product.design.json b/examples/tabellio-design-memory/product.design.json new file mode 100644 index 0000000..045afb4 --- /dev/null +++ b/examples/tabellio-design-memory/product.design.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "tabellio-product-design/v0.1", + "productId": "example/product", + "title": "Example Product", + "audience": ["Operations teams"], + "platforms": ["web"], + "visualThesis": "Calm, precise operating software with visible state and clear next actions.", + "sources": { + "tokens": [{"path": "examples/tabellio-design-memory/tokens.json", "digest": "0854fe3b0c4bf45eaa2ca26cd8450644d57708a0cfc2e5e0ce68e206b75ff781"}], + "components": [{"path": "examples/tabellio-design-memory/components.json", "digest": "641926586087b8b8737309f934fee9729ab361c1f90a5af984448b74ef9f804f"}], + "decisions": ["examples/tabellio-design-memory/decision.md"] + }, + "referenceLock": { + "primary": "Repository tokens and approved product surfaces", + "preserve": ["Semantic color", "Dense scan-first layout", "Visible workflow state"], + "borrow": [], + "roleRules": ["Primary blue is reserved for primary actions and selection"], + "reject": ["Decorative gradients", "Ambiguous icon-only actions"], + "mediaStrategy": "Use product screenshots only when they explain a real workflow." + }, + "policy": { + "themes": ["light", "dark"], + "viewports": [ + {"id": "desktop", "width": 1440, "height": 1000, "deviceScaleFactor": 1}, + {"id": "mobile", "width": 390, "height": 844, "deviceScaleFactor": 1} + ], + "states": ["default", "loading", "empty", "error"], + "surfaces": [ + {"id": "home", "kind": "route", "target": "/home", "viewports": ["desktop"], "themes": ["light"], "states": ["default"]} + ], + "forbiddenPatterns": ["Raw palette utilities outside token sources"], + "accessibility": {"standard": "WCAG 2.2 AA", "keyboardRequired": true, "reducedMotionRequired": true} + }, + "baselines": {"manifest": "examples/tabellio-design-memory/baselines.json"} +} diff --git a/examples/tabellio-design-memory/tokens.json b/examples/tabellio-design-memory/tokens.json new file mode 100644 index 0000000..d531606 --- /dev/null +++ b/examples/tabellio-design-memory/tokens.json @@ -0,0 +1,5 @@ +{ + "primary": "#1244E6", + "background": "#FAFAF9", + "foreground": "#0A0A0A" +} diff --git a/examples/tabellio-design-memory/ui-review.json b/examples/tabellio-design-memory/ui-review.json new file mode 100644 index 0000000..37b3e89 --- /dev/null +++ b/examples/tabellio-design-memory/ui-review.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": "tabellio-ui-review/v0.1", + "productId": "example/product", + "sourceCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "profileDigest": "1304dd93039569be68388cd313c60621d85a289e196738f3e25a84a995f34767", + "reviewer": { + "provider": "example", + "model": "visual-review-model" + }, + "cost": { + "telemetry": "available", + "currency": "USD", + "amount": 0.01, + "inputTokens": 1000, + "outputTokens": 250 + }, + "captures": [ + { + "id": "home-desktop-light-default", + "artifactDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "summary": "The rendered surface matches the approved visual hierarchy and interaction vocabulary.", + "findings": [], + "verdict": "passed", + "blockers": [] +} diff --git a/examples/tabellio-validation/minimal-validator-evidence.json b/examples/tabellio-validation/minimal-validator-evidence.json new file mode 100644 index 0000000..882a907 --- /dev/null +++ b/examples/tabellio-validation/minimal-validator-evidence.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": "tabellio-validator-evidence/v0.1", + "validatorId": "semantic-golden-set", + "status": "passed", + "summary": "Eight of ten required scenarios passed the strict semantic rubric.", + "metrics": [ + { + "name": "strict_pass_rate", + "value": 0.8, + "unit": "ratio" + } + ], + "cost": { + "telemetry": "available", + "usd": 0.18, + "modelCalls": 10, + "toolCalls": 24 + }, + "artifacts": [] +} diff --git a/examples/tabellio-validation/product-manifest.json b/examples/tabellio-validation/product-manifest.json new file mode 100644 index 0000000..4a7761a --- /dev/null +++ b/examples/tabellio-validation/product-manifest.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": "tabellio-validation/v0.2", + "id": "product-validity", + "failFast": true, + "requireEntireCheckpoint": true, + "acceptance": { + "id": "PLANE-101", + "source": "plane", + "risk": "high", + "outcomes": [ + "The user receives a correct result for the requested workflow." + ], + "invariants": [ + "Validation cannot mutate production resources." + ], + "forbiddenOutcomes": [ + "Unknown cost telemetry is treated as zero cost." + ], + "requiredValidatorTypes": [ + "semantic", + "operational" + ] + }, + "validators": [ + { + "id": "semantic-golden-set", + "type": "semantic", + "argv": [ + "npm", + "run", + "eval:semantic:ci" + ], + "cwd": ".", + "timeoutMs": 600000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/semantic-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "strict_pass_rate", + "operator": "gte", + "value": 0.8 + } + ], + "maxCostUsd": 1, + "requireCostTelemetry": true + } + }, + { + "id": "runtime-cost-guard", + "type": "operational", + "argv": [ + "npm", + "run", + "cost:check" + ], + "cwd": ".", + "timeoutMs": 120000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/cost-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "projected_monthly_cost_usd", + "operator": "lte", + "value": 20 + } + ], + "maxCostUsd": 0.25, + "requireCostTelemetry": true + } + } + ] +} diff --git a/package.json b/package.json index e4698e2..f4ceb25 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@intelip/tabellio", - "version": "0.3.0", + "version": "0.4.0", "description": "Tabellio: GitHub-native context and evidence for agentic development.", "type": "module", "license": "Apache-2.0", @@ -16,6 +16,9 @@ "tabellio-stack-operation": "scripts/tabellio-stack-operation.mjs", "tabellio-review": "scripts/tabellio-review.mjs", "tabellio-validate": "scripts/tabellio-validate.mjs", + "tabellio-validator": "scripts/tabellio-validator.mjs", + "tabellio-design-memory": "scripts/check-tabellio-design-memory.mjs", + "tabellio-design-captures": "scripts/finalize-tabellio-visual-captures.mjs", "tabellio-control-ref": "scripts/tabellio-control-ref.mjs", "tabellio-ledger": "scripts/tabellio-ledger.mjs", "tabellio-preflight": "scripts/tabellio-preflight.mjs", @@ -60,7 +63,8 @@ "tabellio:release": "node scripts/tabellio-release.mjs", "tabellio:release:example:check": "node scripts/check-tabellio-release.mjs --intent examples/tabellio-release/minimal-intent.json", "tabellio:platform:check": "node scripts/check-tabellio-platform.mjs", - "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", + "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json && node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", + "tabellio:design-memory:example:check": "node scripts/check-tabellio-design-memory.mjs --repo . --profile examples/tabellio-design-memory/product.design.json", "tabellio:ledger": "node scripts/tabellio-ledger.mjs", "tabellio:ledger:check": "node scripts/check-tabellio-ledger.mjs --ledger tabellio-ledger.json", "tabellio:ledger:example:check": "node scripts/check-tabellio-ledger.mjs --ledger examples/tabellio-ledger/minimal-ledger.json", @@ -71,7 +75,7 @@ "tabellio:evidence:write": "node scripts/write-tabellio-evidence-envelope.mjs --out tabellio-pr-evidence.json", "tabellio:evidence:check": "node scripts/check-tabellio-evidence-envelope.mjs --evidence tabellio-pr-evidence.json", "tabellio:external-actions:check": "node scripts/check-tabellio-external-actions.mjs --evidence tabellio-pr-evidence.json", - "check": "npm test && npm run tabellio:platform:check && npm run tabellio:run:example:check && npm run tabellio:stack:example:check && npm run tabellio:stack:operation:example:check && npm run tabellio:review:example:check && npm run tabellio:validate:example:check && npm run tabellio:ledger:example:check && npm run tabellio:release:example:check && npm run tabellio:context:example:check && npm run tabellio:evidence:example:check && node scripts/check-tabellio-external-actions.mjs --evidence examples/tabellio-evidence/minimal-evidence.json" + "check": "npm test && npm run tabellio:platform:check && npm run tabellio:run:example:check && npm run tabellio:stack:example:check && npm run tabellio:stack:operation:example:check && npm run tabellio:review:example:check && npm run tabellio:validate:example:check && npm run tabellio:design-memory:example:check && npm run tabellio:ledger:example:check && npm run tabellio:release:example:check && npm run tabellio:context:example:check && npm run tabellio:evidence:example:check && node scripts/check-tabellio-external-actions.mjs --evidence examples/tabellio-evidence/minimal-evidence.json" }, "engines": { "node": ">=20" diff --git a/quality-baselines/fallow-dupes.json b/quality-baselines/fallow-dupes.json new file mode 100644 index 0000000..d302a05 --- /dev/null +++ b/quality-baselines/fallow-dupes.json @@ -0,0 +1,49 @@ +{ + "clone_groups": [ + "scripts/check-tabellio-context.mjs:22-40|scripts/check-tabellio-run.mjs:24-39|scripts/check-tabellio-stack.mjs:26-41", + "scripts/check-tabellio-design-memory.mjs:7-15|scripts/finalize-tabellio-visual-captures.mjs:6-14", + "scripts/check-tabellio-evidence-envelope.mjs:337-345|scripts/check-tabellio-external-actions.mjs:110-118", + "scripts/check-tabellio-ledger.mjs:27-37|scripts/check-tabellio-run.mjs:28-39", + "scripts/check-tabellio-run.mjs:28-39|scripts/check-tabellio-stack.mjs:30-41", + "scripts/check-tabellio-stack-operation.mjs:47-52|scripts/tabellio-validator.mjs:33-38", + "scripts/lib/agent-run.mjs:370-374|scripts/lib/context-packet.mjs:58-62|scripts/lib/ledger-provider.mjs:29-33|scripts/lib/stack-manager.mjs:31-35", + "scripts/lib/agent-run.mjs:384-391|scripts/lib/context-packet.mjs:65-72", + "scripts/lib/agent-run.mjs:570-577|scripts/lib/context-packet.mjs:182-190|scripts/lib/ledger-provider.mjs:123-130|scripts/lib/stack-manager.mjs:133-140", + "scripts/lib/context-packet.mjs:158-166|scripts/lib/stack-operation.mjs:97-107", + "scripts/lib/context-packet.mjs:180-190|scripts/lib/ledger-provider.mjs:121-130|scripts/lib/stack-manager.mjs:131-140", + "scripts/lib/context-packet.mjs:187-194|scripts/lib/ledger-provider.mjs:158-165", + "scripts/lib/control-ref-transport.mjs:125-144|scripts/providers/git-spice-operations.mjs:61-69", + "scripts/lib/control-ref-transport.mjs:276-285|scripts/providers/git-spice-operations.mjs:200-209", + "scripts/lib/control-ref-transport.mjs:306-317|scripts/lib/validation-runner.mjs:929-939", + "scripts/lib/control-ref-transport.mjs:316-329|scripts/lib/stack-operation.mjs:106-119", + "scripts/lib/control-ref-transport.mjs:324-338|scripts/lib/review-cycle.mjs:713-726|scripts/lib/validation-runner.mjs:917-930", + "scripts/lib/ledger-provider.mjs:25-37|scripts/lib/stack-manager.mjs:27-39", + "scripts/lib/ledger-provider.mjs:121-150|scripts/lib/stack-manager.mjs:131-158", + "scripts/lib/ledger-provider.mjs:128-144|scripts/lib/review-cycle.mjs:696-706|scripts/lib/stack-manager.mjs:138-154", + "scripts/lib/ledger-provider.mjs:168-179|scripts/lib/stack-manager.mjs:161-172", + "scripts/lib/review-cycle.mjs:687-701|scripts/lib/validation-runner.mjs:899-913", + "scripts/lib/review-cycle.mjs:690-702|scripts/lib/stack-operation.mjs:103-115|scripts/lib/validation-runner.mjs:902-914|scripts/tabellio-validator.mjs:192-204", + "scripts/lib/review-cycle.mjs:709-731|scripts/lib/validation-runner.mjs:913-934", + "scripts/lib/review-cycle.mjs:716-731|scripts/lib/stack-operation.mjs:128-144", + "scripts/lib/review-cycle.mjs:724-737|scripts/lib/validation-runner.mjs:932-944", + "scripts/lib/validation-runner.mjs:902-918|scripts/tabellio-validator.mjs:192-208", + "scripts/lib/validation-runner.mjs:946-950|scripts/lib/validation-runner.mjs:957-961", + "scripts/lib/validation-runner.mjs:966-980|scripts/tabellio-validator.mjs:203-217", + "scripts/providers/entire-ledger-provider.mjs:13-25|scripts/providers/git-spice-stack-manager.mjs:10-25", + "scripts/providers/entire-ledger-provider.mjs:223-245|scripts/providers/git-spice-stack-manager.mjs:147-184", + "scripts/tabellio-control-ref.mjs:77-87|scripts/tabellio-stack-operation.mjs:120-130", + "scripts/tabellio-stack-operation.mjs:119-131|scripts/tabellio-validate.mjs:41-53", + "tests/context-and-evidence.test.mjs:220-229|tests/context-and-evidence.test.mjs:238-247", + "tests/design-memory.test.mjs:31-41|tests/design-memory.test.mjs:87-97", + "tests/native-git-store.test.mjs:12-16|tests/native-git-store.test.mjs:120-123", + "tests/native-git-store.test.mjs:148-156|tests/native-git-store.test.mjs:65-73", + "tests/validation-cli.test.mjs:12-17|tests/validation-runner.test.mjs:126-133", + "tests/validation-runner.test.mjs:106-111|tests/validation-runner.test.mjs:28-31", + "tests/validation-runner.test.mjs:148-154|tests/validation-runner.test.mjs:211-217|tests/validation-runner.test.mjs:262-268|tests/validation-runner.test.mjs:29-32", + "tests/validation-runner.test.mjs:134-138|tests/validation-runner.test.mjs:231-235|tests/validation-runner.test.mjs:41-46", + "tests/validation-runner.test.mjs:116-119|tests/validation-runner.test.mjs:134-137|tests/validation-runner.test.mjs:231-234|tests/validation-runner.test.mjs:41-45", + "tests/validation-runner.test.mjs:109-113|tests/validation-runner.test.mjs:127-131", + "tests/validation-runner.test.mjs:147-154|tests/validation-runner.test.mjs:210-217|tests/validation-runner.test.mjs:261-268", + "tests/validation-runner.test.mjs:184-191|tests/validation-runner.test.mjs:302-309" + ] +} \ No newline at end of file diff --git a/quality-baselines/fallow-health.json b/quality-baselines/fallow-health.json new file mode 100644 index 0000000..8f1e71f --- /dev/null +++ b/quality-baselines/fallow-health.json @@ -0,0 +1,273 @@ +{ + "finding_counts": { + "scripts/capture-tabellio-context.mjs": { + "crap_moderate": { + "count": 1 + } + }, + "scripts/check-tabellio-evidence-envelope.mjs": { + "complexity_critical": { + "count": 2 + }, + "crap_critical": { + "count": 4 + } + }, + "scripts/check-tabellio-external-actions.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/check-tabellio-stack-operation.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/agent-run.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 4 + }, + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 3 + } + }, + "scripts/lib/capture-context.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/context-packet.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/control-ref-transport.mjs": { + "crap_critical": { + "count": 2 + }, + "crap_high": { + "count": 2 + } + }, + "scripts/lib/design-memory.mjs": { + "complexity_critical": { + "count": 4 + }, + "crap_critical": { + "count": 5 + }, + "crap_moderate": { + "count": 5 + } + }, + "scripts/lib/git-json-ledger.mjs": { + "crap_moderate": { + "count": 2 + } + }, + "scripts/lib/git-process.mjs": { + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/ledger-provider.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/operation-lock.mjs": { + "crap_critical": { + "count": 2 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/repository-identity.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/review-cycle.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 6 + }, + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 4 + } + }, + "scripts/lib/stack-manager.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/stack-operation.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/validation-runner.mjs": { + "crap_critical": { + "count": 3 + }, + "crap_high": { + "count": 6 + }, + "crap_moderate": { + "count": 14 + } + }, + "scripts/lib/workspace-manager.mjs": { + "crap_moderate": { + "count": 1 + } + }, + "scripts/providers/entire-ledger-provider.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 4 + } + }, + "scripts/providers/git-spice-operations.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 2 + } + }, + "scripts/providers/git-spice-stack-manager.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/providers/native-git-store.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 3 + }, + "crap_high": { + "count": 2 + }, + "crap_moderate": { + "count": 3 + } + }, + "scripts/tabellio-control-ref.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + } + }, + "scripts/tabellio-run.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/tabellio-stack-operation.mjs": { + "complexity_critical": { + "count": 2 + }, + "crap_critical": { + "count": 2 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/tabellio-validate.mjs": { + "crap_critical": { + "count": 1 + } + }, + "scripts/tabellio-validator.mjs": { + "crap_high": { + "count": 2 + }, + "crap_moderate": { + "count": 5 + } + }, + "scripts/write-tabellio-evidence-envelope.mjs": { + "crap_high": { + "count": 3 + } + } + }, + "runtime_coverage_findings": [], + "target_keys": [ + "scripts/lib/control-ref-transport.mjs:high impact", + "scripts/lib/stack-operation.mjs:high impact", + "scripts/lib/validation-runner.mjs:high impact", + "scripts/tabellio-stack-operation.mjs:untested risk", + "scripts/providers/git-spice-stack-manager.mjs:high impact", + "scripts/lib/context-packet.mjs:high impact", + "scripts/lib/repository-identity.mjs:high impact", + "scripts/lib/cli-options.mjs:high impact", + "scripts/lib/review-cycle.mjs:high impact", + "scripts/lib/preflight.mjs:high impact", + "scripts/lib/github-repository.mjs:high impact", + "scripts/lib/design-memory.mjs:high impact", + "scripts/lib/agent-run.mjs:high impact", + "scripts/lib/contract-checks.mjs:high impact", + "scripts/lib/operation-lock.mjs:high impact", + "scripts/lib/approval-validation.mjs:high impact", + "scripts/lib/ledger-provider.mjs:high impact", + "scripts/lib/stack-manager.mjs:high impact", + "scripts/tabellio-validator.mjs:untested risk" + ] +} \ No newline at end of file diff --git a/schemas/product-design-profile.v0.1.schema.json b/schemas/product-design-profile.v0.1.schema.json new file mode 100644 index 0000000..64ec3ee --- /dev/null +++ b/schemas/product-design-profile.v0.1.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:product-design-profile:v0.1", + "title": "Tabellio Product Design Profile", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "productId", "title", "audience", "platforms", "visualThesis", "sources", "referenceLock", "policy", "baselines"], + "properties": { + "schemaVersion": {"const": "tabellio-product-design/v0.1"}, + "productId": {"type": "string", "minLength": 1}, + "title": {"type": "string", "minLength": 1}, + "audience": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "platforms": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["web", "ios", "android", "desktop", "email", "docs"]}}, + "visualThesis": {"type": "string", "minLength": 1, "maxLength": 1000}, + "sources": { + "type": "object", + "additionalProperties": false, + "required": ["tokens", "components", "decisions"], + "properties": { + "tokens": {"$ref": "#/$defs/sources"}, + "components": {"$ref": "#/$defs/sources"}, + "decisions": {"type": "array", "maxItems": 100, "items": {"$ref": "#/$defs/path"}} + } + }, + "referenceLock": { + "type": "object", + "additionalProperties": false, + "required": ["primary", "preserve", "borrow", "roleRules", "reject", "mediaStrategy"], + "properties": { + "primary": {"type": "string", "minLength": 1}, + "preserve": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "borrow": {"type": "array", "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "roleRules": {"type": "array", "maxItems": 30, "items": {"type": "string", "minLength": 1}}, + "reject": {"type": "array", "minItems": 1, "maxItems": 30, "items": {"type": "string", "minLength": 1}}, + "mediaStrategy": {"type": "string", "minLength": 1, "maxLength": 1000} + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": ["themes", "viewports", "states", "surfaces", "forbiddenPatterns", "accessibility"], + "properties": { + "themes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["light", "dark", "high-contrast"]}}, + "viewports": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"$ref": "#/$defs/viewport"}}, + "states": {"type": "array", "minItems": 1, "maxItems": 50, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "surfaces": {"type": "array", "minItems": 1, "maxItems": 100, "items": {"$ref": "#/$defs/surface"}}, + "forbiddenPatterns": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}}, + "accessibility": { + "type": "object", + "additionalProperties": false, + "required": ["standard", "keyboardRequired", "reducedMotionRequired"], + "properties": { + "standard": {"type": "string", "minLength": 1}, + "keyboardRequired": {"type": "boolean"}, + "reducedMotionRequired": {"type": "boolean"} + } + } + } + }, + "baselines": { + "type": "object", + "additionalProperties": false, + "required": ["manifest"], + "properties": {"manifest": {"$ref": "#/$defs/path"}} + } + }, + "$defs": { + "path": {"type": "string", "minLength": 1, "not": {"pattern": "^(?:/|[A-Za-z]:[\\\\/])|(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$)"}}, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["path", "digest"], + "properties": {"path": {"$ref": "#/$defs/path"}, "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}} + }, + "sources": {"type": "array", "minItems": 1, "maxItems": 100, "items": {"$ref": "#/$defs/source"}}, + "surface": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "target", "viewports", "themes", "states"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "kind": {"enum": ["route", "component"]}, + "target": {"type": "string", "minLength": 1}, + "viewports": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "themes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["light", "dark", "high-contrast"]}}, + "states": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}} + } + }, + "viewport": { + "type": "object", + "additionalProperties": false, + "required": ["id", "width", "height", "deviceScaleFactor"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "width": {"type": "integer", "minimum": 240, "maximum": 7680}, + "height": {"type": "integer", "minimum": 240, "maximum": 7680}, + "deviceScaleFactor": {"type": "number", "minimum": 0.5, "maximum": 4} + } + } + } +} diff --git a/schemas/ui-review.v0.1.schema.json b/schemas/ui-review.v0.1.schema.json new file mode 100644 index 0000000..be1edef --- /dev/null +++ b/schemas/ui-review.v0.1.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:ui-review:v0.1", + "title": "Tabellio UI Review", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "productId", "sourceCommit", "profileDigest", "reviewer", "cost", "captures", "summary", "findings", "verdict", "blockers"], + "properties": { + "schemaVersion": {"const": "tabellio-ui-review/v0.1"}, + "productId": {"type": "string", "minLength": 1}, + "sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "profileDigest": {"$ref": "#/$defs/digest"}, + "reviewer": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "model"], + "properties": { + "provider": {"type": "string", "minLength": 1}, + "model": {"type": "string", "minLength": 1} + } + }, + "cost": { + "type": "object", + "additionalProperties": false, + "required": ["telemetry", "currency", "amount", "inputTokens", "outputTokens"], + "properties": { + "telemetry": {"enum": ["available", "unknown", "not_applicable"]}, + "currency": {"type": ["string", "null"], "minLength": 3, "maxLength": 3}, + "amount": {"type": ["number", "null"], "minimum": 0}, + "inputTokens": {"type": ["integer", "null"], "minimum": 0}, + "outputTokens": {"type": ["integer", "null"], "minimum": 0} + } + }, + "captures": { + "type": "array", + "minItems": 1, + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "artifactDigest"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "artifactDigest": {"$ref": "#/$defs/digest"} + } + } + }, + "summary": {"type": "string", "minLength": 1, "maxLength": 2000}, + "findings": { + "type": "array", + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "severity", "category", "surface", "evidence", "recommendation"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "severity": {"enum": ["info", "warning", "error", "critical"]}, + "category": {"type": "string", "minLength": 1}, + "surface": {"type": "string", "minLength": 1}, + "evidence": {"type": "string", "minLength": 1, "maxLength": 2000}, + "recommendation": {"type": "string", "minLength": 1, "maxLength": 2000} + } + } + }, + "verdict": {"enum": ["passed", "failed", "blocked"]}, + "blockers": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}} + }, + "allOf": [ + { + "if": {"properties": {"cost": {"properties": {"telemetry": {"const": "available"}}}}}, + "then": {"properties": {"cost": {"properties": {"currency": {"type": "string"}, "amount": {"type": "number"}, "inputTokens": {"type": "integer"}, "outputTokens": {"type": "integer"}}}}}, + "else": {"properties": {"cost": {"properties": {"currency": {"type": "null"}, "amount": {"type": "null"}, "inputTokens": {"type": "null"}, "outputTokens": {"type": "null"}}}}} + }, + { + "if": {"properties": {"cost": {"properties": {"telemetry": {"const": "unknown"}}}}}, + "then": {"properties": {"verdict": {"const": "blocked"}, "blockers": {"minItems": 1}}} + } + ], + "$defs": { + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } +} diff --git a/schemas/validation-manifest.v0.1.schema.json b/schemas/validation-manifest.v0.1.schema.json new file mode 100644 index 0000000..b250820 --- /dev/null +++ b/schemas/validation-manifest.v0.1.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.1", + "title": "Tabellio Validation Manifest v0.1", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.1" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "commands": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "type": "object", + "required": ["id", "argv", "cwd", "timeoutMs", "required"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" } + } + } + } + } +} diff --git a/schemas/validation-manifest.v0.2.schema.json b/schemas/validation-manifest.v0.2.schema.json new file mode 100644 index 0000000..2ce8d2d --- /dev/null +++ b/schemas/validation-manifest.v0.2.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.2", + "title": "Tabellio Product Validation Manifest", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.2" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "acceptance": { "$ref": "#/$defs/acceptance" }, + "validators": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { "$ref": "#/$defs/validator" } + } + }, + "$defs": { + "validatorType": { + "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] + }, + "boundedText": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "acceptance": { + "type": "object", + "required": ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "outcomes": { "type": "array", "minItems": 1, "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "invariants": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "forbiddenOutcomes": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validator": { + "type": "object", + "required": ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { "path": { "type": "string", "minLength": 1 } } + } + ] + }, + "policy": { "$ref": "#/$defs/policy" } + }, + "allOf": [ + { + "if": { "properties": { "type": { "not": { "const": "static" } } }, "required": ["type"] }, + "then": { "properties": { "evidence": { "type": "object" } } } + } + ] + }, + "policy": { + "type": "object", + "required": ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], + "additionalProperties": false, + "properties": { + "metricThresholds": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["metric", "operator", "value"], + "additionalProperties": false, + "properties": { + "metric": { "type": "string", "minLength": 1 }, + "operator": { "enum": ["gte", "lte", "eq"] }, + "value": { "type": "number" } + } + } + }, + "maxCostUsd": { "type": ["number", "null"], "minimum": 0 }, + "requireCostTelemetry": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "maxCostUsd": { "type": "number" } }, "required": ["maxCostUsd"] }, + "then": { "properties": { "requireCostTelemetry": { "const": true } } } + } + ] + } + } +} diff --git a/schemas/validation-result.v0.3.schema.json b/schemas/validation-result.v0.3.schema.json new file mode 100644 index 0000000..83756f1 --- /dev/null +++ b/schemas/validation-result.v0.3.schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-result:v0.3", + "title": "Tabellio Product Validation Result", + "type": "object", + "required": ["schemaVersion", "runId", "repository", "revision", "checkpointRevision", "suite", "runner", "status", "checkpoints", "commands", "acceptance", "validators", "decision", "startedAt", "completedAt", "integrity"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation-result/v0.3" }, + "runId": { "type": "string", "minLength": 1 }, + "repository": { + "type": "object", + "required": ["id"], + "additionalProperties": false, + "properties": { "id": { "type": "string", "minLength": 1 } } + }, + "revision": { "$ref": "#/$defs/revision" }, + "checkpointRevision": { "$ref": "#/$defs/revision" }, + "suite": { + "type": "object", + "required": ["id", "manifestPath", "manifestDigest"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "manifestPath": { "type": "string", "minLength": 1 }, + "manifestDigest": { "$ref": "#/$defs/sha256" } + } + }, + "runner": { + "type": "object", + "required": ["id", "runtime"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "runtime": { "type": "string", "minLength": 1 } + } + }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "checkpoints": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, + "commands": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/commandResult" } }, + "acceptance": { "$ref": "#/$defs/acceptanceResult" }, + "validators": { "type": "array", "minItems": 1, "maxItems": 50, "items": { "$ref": "#/$defs/validatorResult" } }, + "decision": { "$ref": "#/$defs/decision" }, + "startedAt": { "type": "string", "format": "date-time" }, + "completedAt": { "type": "string", "format": "date-time" }, + "integrity": { + "type": "object", + "required": ["algorithm", "digest"], + "additionalProperties": false, + "properties": { + "algorithm": { "const": "sha256" }, + "digest": { "$ref": "#/$defs/sha256" } + } + } + }, + "$defs": { + "oid": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "validatorType": { "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] }, + "revision": { + "type": "object", + "required": ["baseCommit", "mergeBase", "headCommit"], + "additionalProperties": false, + "properties": { + "baseCommit": { "$ref": "#/$defs/oid" }, + "mergeBase": { "$ref": "#/$defs/oid" }, + "headCommit": { "$ref": "#/$defs/oid" } + } + }, + "output": { + "type": "object", + "required": ["bytes", "digest", "tail", "truncated"], + "additionalProperties": false, + "properties": { + "bytes": { "type": "integer", "minimum": 0 }, + "digest": { "$ref": "#/$defs/sha256" }, + "tail": { "type": "string" }, + "truncated": { "type": "boolean" } + } + }, + "commandResult": { + "type": "object", + "required": ["id", "argv", "cwd", "required", "status", "exitCode", "signal", "durationMs", "stdout", "stderr", "startedAt", "completedAt", "error"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "error", "timed_out", "skipped"] }, + "exitCode": { "type": ["integer", "null"] }, + "signal": { "type": ["string", "null"] }, + "durationMs": { "type": "integer", "minimum": 0 }, + "stdout": { "$ref": "#/$defs/output" }, + "stderr": { "$ref": "#/$defs/output" }, + "startedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "completedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "error": { "type": ["string", "null"] } + } + }, + "acceptanceResult": { + "type": "object", + "required": ["id", "source", "risk", "digest", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "digest": { "$ref": "#/$defs/sha256" }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validatorResult": { + "type": "object", + "required": ["id", "type", "required", "status", "evidence", "reasons"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "blocked", "skipped"] }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path", "digest", "report"], + "additionalProperties": false, + "properties": { + "path": { "type": "string", "minLength": 1 }, + "digest": { "$ref": "#/$defs/sha256" }, + "report": { "$ref": "#/$defs/evidenceReport" } + } + } + ] + }, + "reasons": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } } + } + }, + "evidenceReport": { + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { "type": "array", "maxItems": 100, "items": { "$ref": "#/$defs/metric" } }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/artifact" } } + } + }, + "metric": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + }, + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "$ref": "#/$defs/sha256" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + }, + "decision": { + "type": "object", + "required": ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], + "additionalProperties": false, + "properties": { + "status": { "enum": ["passed", "failed", "blocked"] }, + "reasons": { "type": "array", "maxItems": 6400, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } }, + "totalCostUsd": { "type": "number", "minimum": 0 }, + "costTelemetryComplete": { "type": "boolean" } + } + } + } +} diff --git a/schemas/validator-evidence.v0.1.schema.json b/schemas/validator-evidence.v0.1.schema.json new file mode 100644 index 0000000..14fec11 --- /dev/null +++ b/schemas/validator-evidence.v0.1.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validator-evidence:v0.1", + "title": "Tabellio Validator Evidence", + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + } + }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { + "type": "array", + "maxItems": 50, + "items": { "$ref": "#/$defs/artifact" } + } + }, + "$defs": { + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/schemas/visual-baseline-manifest.v0.1.schema.json b/schemas/visual-baseline-manifest.v0.1.schema.json new file mode 100644 index 0000000..dde423f --- /dev/null +++ b/schemas/visual-baseline-manifest.v0.1.schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:visual-baseline-manifest:v0.1", + "title": "Tabellio Visual Baseline Manifest", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "status", "productId", "profilePath", "profileDigest", "sourceCommit", "approvedAt", "approvedBy", "captures"], + "properties": { + "schemaVersion": {"const": "tabellio-visual-baselines/v0.1"}, + "status": {"enum": ["proposed", "approved"]}, + "productId": {"type": "string", "minLength": 1}, + "profilePath": {"type": "string", "minLength": 1}, + "profileDigest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "sourceCommit": {"type": ["string", "null"], "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "approvedAt": {"type": ["string", "null"], "format": "date-time"}, + "approvedBy": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["type", "id"], + "properties": {"type": {"enum": ["human", "team"]}, "id": {"type": "string", "minLength": 1}} + }, + "captures": { + "type": "array", + "minItems": 0, + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "surface", "kind", "viewport", "theme", "state", "artifact", "maskSelectors", "maxDiffPixelRatio"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "surface": {"type": "string", "minLength": 1}, + "kind": {"enum": ["route", "component"]}, + "viewport": {"type": "string", "minLength": 1}, + "theme": {"enum": ["light", "dark", "high-contrast"]}, + "state": {"type": "string", "minLength": 1}, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["uri", "digest", "mediaType", "bytes"], + "properties": { + "uri": {"type": "string", "format": "uri", "not": {"pattern": "^file:"}}, + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "mediaType": {"type": "string", "minLength": 1}, + "bytes": {"type": "integer", "minimum": 0} + } + }, + "maskSelectors": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}}, + "maxDiffPixelRatio": {"type": "number", "minimum": 0, "maximum": 1} + } + } + } + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "approved"}}}, + "then": { + "properties": { + "sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "approvedAt": {"type": "string", "format": "date-time"}, + "approvedBy": {"type": "object"}, + "captures": {"minItems": 1} + } + }, + "else": { + "properties": { + "approvedAt": {"type": "null"}, + "approvedBy": {"type": "null"} + } + } + }, + { + "if": {"properties": {"status": {"const": "proposed"}, "captures": {"maxItems": 0}}}, + "then": {"properties": {"sourceCommit": {"type": "null"}}} + }, + { + "if": {"properties": {"status": {"const": "proposed"}, "captures": {"minItems": 1}}}, + "then": {"properties": {"sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}}} + } + ] +} diff --git a/scripts/check-tabellio-design-memory.mjs b/scripts/check-tabellio-design-memory.mjs new file mode 100644 index 0000000..7afa697 --- /dev/null +++ b/scripts/check-tabellio-design-memory.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +import { inspectDesignMemory } from "./lib/design-memory.mjs"; + +const options = parseArgs(process.argv.slice(2)); +const result = await inspectDesignMemory(options); +if (!result.ok) process.exitCode = 1; +console.log(JSON.stringify(result, null, 2)); + +function parseArgs(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!value || !["--repo", "--profile", "--baselines"].includes(flag)) { + throw new Error("Usage: check-tabellio-design-memory [--repo ] [--profile ] [--baselines ]."); + } + result[flag.slice(2)] = value; + } + return result; +} diff --git a/scripts/check-tabellio-validation.mjs b/scripts/check-tabellio-validation.mjs index 228bc90..be3f1ae 100755 --- a/scripts/check-tabellio-validation.mjs +++ b/scripts/check-tabellio-validation.mjs @@ -3,19 +3,24 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; -import { validateValidationManifest, validateValidationResult } from "./lib/validation-runner.mjs"; +import { + validateValidationManifest, + validateValidationResult, + validateValidatorEvidence, +} from "./lib/validation-runner.mjs"; const type = process.argv[2]; const path = process.argv[3]; const blockers = []; let value = null; try { - if (!path || process.argv.length !== 4 || !["--manifest", "--result"].includes(type)) { - throw new Error("Usage: check-tabellio-validation (--manifest|--result) ."); + if (!path || process.argv.length !== 4 || !["--manifest", "--result", "--evidence"].includes(type)) { + throw new Error("Usage: check-tabellio-validation (--manifest|--result|--evidence) ."); } value = JSON.parse(await readFile(resolve(path), "utf8")); if (type === "--manifest") validateValidationManifest(value); - else validateValidationResult(value); + else if (type === "--result") validateValidationResult(value); + else validateValidatorEvidence(value); } catch (error) { blockers.push(error instanceof Error ? error.message : String(error)); } @@ -23,12 +28,14 @@ const result = { ok: blockers.length === 0, status: blockers.length === 0 ? "validation_contract_ready" : "blocked", checkedAt: new Date().toISOString(), - type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : null, + type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : type === "--evidence" ? "evidence" : null, path, summary: value ? { schemaVersion: value.schemaVersion, - id: value.id ?? value.runId ?? null, + id: value.id ?? value.runId ?? value.validatorId ?? null, commandCount: Array.isArray(value.commands) ? value.commands.length : null, + validatorCount: Array.isArray(value.validators) ? value.validators.length : null, + metricCount: Array.isArray(value.metrics) ? value.metrics.length : null, } : null, blockers, }; diff --git a/scripts/finalize-tabellio-visual-captures.mjs b/scripts/finalize-tabellio-visual-captures.mjs new file mode 100755 index 0000000..851e6dc --- /dev/null +++ b/scripts/finalize-tabellio-visual-captures.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +import { finalizeVisualCaptures } from "./lib/design-memory.mjs"; + +const options = parseArgs(process.argv.slice(2)); +const result = await finalizeVisualCaptures(options); +console.log(JSON.stringify(result, null, 2)); + +function parseArgs(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!value || !["--repo", "--profile", "--captures-dir", "--artifact-base-uri", "--source-commit", "--out"].includes(flag)) { + throw new Error("Usage: tabellio-design-captures [--repo ] [--profile ] --captures-dir --artifact-base-uri --source-commit --out ."); + } + result[flag.slice(2).replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())] = value; + } + return result; +} diff --git a/scripts/lib/cli-options.mjs b/scripts/lib/cli-options.mjs index 7b75445..ae87441 100644 --- a/scripts/lib/cli-options.mjs +++ b/scripts/lib/cli-options.mjs @@ -9,6 +9,15 @@ export function parseOptionPairs(args, context) { return values; } +export function parseCommandOptions(args, allowedByCommand) { + const command = args[0]; + const allowed = allowedByCommand[command]; + if (!allowed) throw new Error(`Expected command: ${Object.keys(allowedByCommand).join(" or ")}.`); + const values = parseOptionPairs(args.slice(1), command); + assertAllowedOptions(values, allowed); + return { command, ...values }; +} + export function assertAllowedOptions(values, allowed) { for (const key of Object.keys(values)) { if (!allowed.includes(key)) throw new Error(`Unsupported option: --${toKebabCase(key)}.`); diff --git a/scripts/lib/design-memory.mjs b/scripts/lib/design-memory.mjs new file mode 100644 index 0000000..571aaa8 --- /dev/null +++ b/scripts/lib/design-memory.mjs @@ -0,0 +1,367 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { basename, isAbsolute, relative, resolve, sep } from "node:path"; + +const SHA256 = /^[0-9a-f]{64}$/; +const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; +const THEMES = new Set(["light", "dark", "high-contrast"]); +const PLATFORMS = new Set(["web", "ios", "android", "desktop", "email", "docs"]); + +export function validateProductDesignProfile(value) { + object(value, "design profile"); + exactKeys(value, ["schemaVersion", "productId", "title", "audience", "platforms", "visualThesis", "sources", "referenceLock", "policy", "baselines"], "design profile"); + equal(value.schemaVersion, "tabellio-product-design/v0.1", "design profile.schemaVersion"); + string(value.productId, "design profile.productId"); + if (/^(?:\/|file:|[A-Za-z]:[\\/])|\\/.test(value.productId)) throw new TypeError("design profile.productId must not expose a local path."); + string(value.title, "design profile.title"); + stringArray(value.audience, "design profile.audience", 1, 20); + enumArray(value.platforms, PLATFORMS, "design profile.platforms", 1, 6); + string(value.visualThesis, "design profile.visualThesis", 1, 1000); + + object(value.sources, "design profile.sources"); + exactKeys(value.sources, ["tokens", "components", "decisions"], "design profile.sources"); + sourceArray(value.sources.tokens, "design profile.sources.tokens"); + sourceArray(value.sources.components, "design profile.sources.components"); + stringArray(value.sources.decisions, "design profile.sources.decisions", 0, 100, safePath); + + object(value.referenceLock, "design profile.referenceLock"); + exactKeys(value.referenceLock, ["primary", "preserve", "borrow", "roleRules", "reject", "mediaStrategy"], "design profile.referenceLock"); + string(value.referenceLock.primary, "design profile.referenceLock.primary"); + stringArray(value.referenceLock.preserve, "design profile.referenceLock.preserve", 1, 20); + stringArray(value.referenceLock.borrow, "design profile.referenceLock.borrow", 0, 20); + stringArray(value.referenceLock.roleRules, "design profile.referenceLock.roleRules", 0, 30); + stringArray(value.referenceLock.reject, "design profile.referenceLock.reject", 1, 30); + string(value.referenceLock.mediaStrategy, "design profile.referenceLock.mediaStrategy", 1, 1000); + + object(value.policy, "design profile.policy"); + exactKeys(value.policy, ["themes", "viewports", "states", "surfaces", "forbiddenPatterns", "accessibility"], "design profile.policy"); + enumArray(value.policy.themes, THEMES, "design profile.policy.themes", 1, 3); + if (!Array.isArray(value.policy.viewports) || value.policy.viewports.length < 1 || value.policy.viewports.length > 20) throw new TypeError("design profile.policy.viewports must contain 1 to 20 entries."); + const viewportIds = new Set(); + for (const [index, viewport] of value.policy.viewports.entries()) { + const label = `design profile.policy.viewports[${index}]`; + object(viewport, label); + exactKeys(viewport, ["id", "width", "height", "deviceScaleFactor"], label); + string(viewport.id, `${label}.id`); + if (viewportIds.has(viewport.id)) throw new TypeError(`duplicate viewport id: ${viewport.id}`); + viewportIds.add(viewport.id); + integer(viewport.width, `${label}.width`, 240, 7680); + integer(viewport.height, `${label}.height`, 240, 7680); + number(viewport.deviceScaleFactor, `${label}.deviceScaleFactor`, 0.5, 4); + } + stringArray(value.policy.states, "design profile.policy.states", 1, 50); + if (!Array.isArray(value.policy.surfaces) || value.policy.surfaces.length < 1 || value.policy.surfaces.length > 100) throw new TypeError("design profile.policy.surfaces must contain 1 to 100 entries."); + const surfaceIds = new Set(); + for (const [index, surface] of value.policy.surfaces.entries()) { + const label = `design profile.policy.surfaces[${index}]`; + object(surface, label); + exactKeys(surface, ["id", "kind", "target", "viewports", "themes", "states"], label); + string(surface.id, `${label}.id`); + if (surfaceIds.has(surface.id)) throw new TypeError(`duplicate surface id: ${surface.id}`); + surfaceIds.add(surface.id); + if (!new Set(["route", "component"]).has(surface.kind)) throw new TypeError(`${label}.kind must be route or component.`); + string(surface.target, `${label}.target`); + uniqueStringArray(surface.viewports, `${label}.viewports`, 1, 20); + uniqueStringArray(surface.themes, `${label}.themes`, 1, 3); + uniqueStringArray(surface.states, `${label}.states`, 1, 50); + } + stringArray(value.policy.forbiddenPatterns, "design profile.policy.forbiddenPatterns", 0, 100); + object(value.policy.accessibility, "design profile.policy.accessibility"); + exactKeys(value.policy.accessibility, ["standard", "keyboardRequired", "reducedMotionRequired"], "design profile.policy.accessibility"); + string(value.policy.accessibility.standard, "design profile.policy.accessibility.standard"); + boolean(value.policy.accessibility.keyboardRequired, "design profile.policy.accessibility.keyboardRequired"); + boolean(value.policy.accessibility.reducedMotionRequired, "design profile.policy.accessibility.reducedMotionRequired"); + + object(value.baselines, "design profile.baselines"); + exactKeys(value.baselines, ["manifest"], "design profile.baselines"); + safePath(value.baselines.manifest, "design profile.baselines.manifest"); + return value; +} + +export function validateVisualBaselineManifest(value) { + object(value, "baseline manifest"); + exactKeys(value, ["schemaVersion", "status", "productId", "profilePath", "profileDigest", "sourceCommit", "approvedAt", "approvedBy", "captures"], "baseline manifest"); + equal(value.schemaVersion, "tabellio-visual-baselines/v0.1", "baseline manifest.schemaVersion"); + if (!new Set(["proposed", "approved"]).has(value.status)) throw new TypeError("baseline manifest.status must be proposed or approved."); + string(value.productId, "baseline manifest.productId"); + safePath(value.profilePath, "baseline manifest.profilePath"); + digest(value.profileDigest, "baseline manifest.profileDigest"); + if (value.status === "approved") { + if (!OID.test(value.sourceCommit)) throw new TypeError("approved baseline manifest.sourceCommit must be a hexadecimal Git object ID."); + if (Number.isNaN(Date.parse(value.approvedAt))) throw new TypeError("approved baseline manifest.approvedAt must be an ISO date-time."); + object(value.approvedBy, "baseline manifest.approvedBy"); + exactKeys(value.approvedBy, ["type", "id"], "baseline manifest.approvedBy"); + if (!new Set(["human", "team"]).has(value.approvedBy.type)) throw new TypeError("baseline manifest.approvedBy.type must be human or team."); + string(value.approvedBy.id, "baseline manifest.approvedBy.id"); + } else if (value.approvedAt !== null || value.approvedBy !== null) { + throw new TypeError("proposed baselines must keep approvedAt and approvedBy null."); + } + const minimumCaptures = value.status === "approved" ? 1 : 0; + if (!Array.isArray(value.captures) || value.captures.length < minimumCaptures || value.captures.length > 500) throw new TypeError(`baseline manifest.captures must contain ${minimumCaptures} to 500 entries.`); + const captureIds = new Set(); + for (const [index, capture] of value.captures.entries()) { + const label = `baseline manifest.captures[${index}]`; + object(capture, label); + exactKeys(capture, ["id", "surface", "kind", "viewport", "theme", "state", "artifact", "maskSelectors", "maxDiffPixelRatio"], label); + string(capture.id, `${label}.id`); + if (captureIds.has(capture.id)) throw new TypeError(`duplicate capture id: ${capture.id}`); + captureIds.add(capture.id); + string(capture.surface, `${label}.surface`); + if (!new Set(["route", "component"]).has(capture.kind)) throw new TypeError(`${label}.kind must be route or component.`); + string(capture.viewport, `${label}.viewport`); + if (!THEMES.has(capture.theme)) throw new TypeError(`${label}.theme is unsupported.`); + string(capture.state, `${label}.state`); + object(capture.artifact, `${label}.artifact`); + exactKeys(capture.artifact, ["uri", "digest", "mediaType", "bytes"], `${label}.artifact`); + string(capture.artifact.uri, `${label}.artifact.uri`); + if (/^file:/i.test(capture.artifact.uri)) throw new TypeError(`${label}.artifact.uri must not use an ephemeral file URI.`); + try { new URL(capture.artifact.uri); } catch { throw new TypeError(`${label}.artifact.uri must be an absolute URI.`); } + digest(capture.artifact.digest, `${label}.artifact.digest`); + string(capture.artifact.mediaType, `${label}.artifact.mediaType`); + integer(capture.artifact.bytes, `${label}.artifact.bytes`, 0, Number.MAX_SAFE_INTEGER); + stringArray(capture.maskSelectors, `${label}.maskSelectors`, 0, 100); + number(capture.maxDiffPixelRatio, `${label}.maxDiffPixelRatio`, 0, 1); + } + if (value.status === "proposed" && value.captures.length === 0 && value.sourceCommit !== null) throw new TypeError("empty proposed baseline must keep sourceCommit null."); + if (value.status === "proposed" && value.captures.length > 0 && !OID.test(value.sourceCommit)) throw new TypeError("captured proposed baseline.sourceCommit must be a hexadecimal Git object ID."); + return value; +} + +export async function finalizeVisualCaptures({ repo = ".", profile = "design/product.design.json", capturesDir, artifactBaseUri, sourceCommit, out } = {}) { + const root = resolve(repo); + if (!capturesDir || !artifactBaseUri || !sourceCommit || !out) throw new TypeError("capturesDir, artifactBaseUri, sourceCommit, and out are required."); + if (!OID.test(sourceCommit)) throw new TypeError("sourceCommit must be a hexadecimal Git object ID."); + const baseUri = new URL(artifactBaseUri); + if (baseUri.protocol === "file:") throw new TypeError("artifactBaseUri must be durable and must not use file:."); + const profilePath = containedPath(root, profile, "profile"); + const profileBytes = await readFile(profilePath); + const profileValue = JSON.parse(profileBytes.toString("utf8")); + validateProductDesignProfile(profileValue); + const directory = containedPath(root, capturesDir, "capturesDir"); + const files = (await readdir(directory)).filter((entry) => entry.endsWith(".png")).sort(); + const captures = []; + for (const surface of profileValue.policy.surfaces) { + for (const viewport of surface.viewports) for (const theme of surface.themes) for (const state of surface.states) { + const id = `${surface.id}--${viewport}--${theme}--${state}`; + const filename = `${id}.png`; + if (!files.includes(filename)) throw new TypeError(`required capture file is missing: ${filename}`); + const bytes = await readFile(resolve(directory, filename)); + if (bytes.length < 8 || bytes.subarray(0, 8).toString("hex") !== "89504e470d0a1a0a") throw new TypeError(`capture is not a PNG: ${filename}`); + captures.push({ + id, + surface: surface.target, + kind: surface.kind, + viewport, + theme, + state, + artifact: { + uri: new URL(basename(filename), `${baseUri.toString().replace(/\/?$/, "/")}`).toString(), + digest: sha256(bytes), + mediaType: "image/png", + bytes: bytes.length, + }, + maskSelectors: [], + maxDiffPixelRatio: 0.01, + }); + } + } + const manifest = { + schemaVersion: "tabellio-visual-baselines/v0.1", + status: "proposed", + productId: profileValue.productId, + profilePath: relative(root, profilePath).split(sep).join("/"), + profileDigest: sha256(profileBytes), + sourceCommit, + approvedAt: null, + approvedBy: null, + captures, + }; + validateVisualBaselineManifest(manifest); + const outputPath = containedPath(root, out, "out"); + await writeFile(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }); + return { status: "capture_candidate_ready", output: relative(root, outputPath).split(sep).join("/"), captureCount: captures.length, profileDigest: manifest.profileDigest, sourceCommit }; +} + +export function validateUiReviewArtifact(value) { + object(value, "UI review"); + exactKeys(value, ["schemaVersion", "productId", "sourceCommit", "profileDigest", "reviewer", "cost", "captures", "summary", "findings", "verdict", "blockers"], "UI review"); + equal(value.schemaVersion, "tabellio-ui-review/v0.1", "UI review.schemaVersion"); + string(value.productId, "UI review.productId"); + if (!OID.test(value.sourceCommit)) throw new TypeError("UI review.sourceCommit must be a hexadecimal Git object ID."); + digest(value.profileDigest, "UI review.profileDigest"); + object(value.reviewer, "UI review.reviewer"); + exactKeys(value.reviewer, ["provider", "model"], "UI review.reviewer"); + string(value.reviewer.provider, "UI review.reviewer.provider"); + string(value.reviewer.model, "UI review.reviewer.model"); + object(value.cost, "UI review.cost"); + exactKeys(value.cost, ["telemetry", "currency", "amount", "inputTokens", "outputTokens"], "UI review.cost"); + if (!new Set(["available", "unknown", "not_applicable"]).has(value.cost.telemetry)) throw new TypeError("UI review.cost.telemetry is unsupported."); + if (value.cost.telemetry === "available") { + string(value.cost.currency, "UI review.cost.currency", 3, 3); + number(value.cost.amount, "UI review.cost.amount", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.inputTokens, "UI review.cost.inputTokens", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.outputTokens, "UI review.cost.outputTokens", 0, Number.MAX_SAFE_INTEGER); + } else if (value.cost.currency !== null || value.cost.amount !== null || value.cost.inputTokens !== null || value.cost.outputTokens !== null) { + throw new TypeError("unavailable UI review cost fields must be null."); + } + if (!Array.isArray(value.captures) || value.captures.length < 1 || value.captures.length > 500) throw new TypeError("UI review.captures must contain 1 to 500 entries."); + for (const [index, capture] of value.captures.entries()) { + const label = `UI review.captures[${index}]`; + object(capture, label); + exactKeys(capture, ["id", "artifactDigest"], label); + string(capture.id, `${label}.id`); + digest(capture.artifactDigest, `${label}.artifactDigest`); + } + string(value.summary, "UI review.summary", 1, 2000); + if (!Array.isArray(value.findings) || value.findings.length > 500) throw new TypeError("UI review.findings must contain 0 to 500 entries."); + for (const [index, finding] of value.findings.entries()) { + const label = `UI review.findings[${index}]`; + object(finding, label); + exactKeys(finding, ["id", "severity", "category", "surface", "evidence", "recommendation"], label); + string(finding.id, `${label}.id`); + if (!new Set(["info", "warning", "error", "critical"]).has(finding.severity)) throw new TypeError(`${label}.severity is unsupported.`); + string(finding.category, `${label}.category`); + string(finding.surface, `${label}.surface`); + string(finding.evidence, `${label}.evidence`, 1, 2000); + string(finding.recommendation, `${label}.recommendation`, 1, 2000); + } + if (!new Set(["passed", "failed", "blocked"]).has(value.verdict)) throw new TypeError("UI review.verdict must be passed, failed, or blocked."); + stringArray(value.blockers, "UI review.blockers", 0, 100); + if (value.verdict === "blocked" && value.blockers.length === 0) throw new TypeError("blocked UI review requires at least one blocker."); + if (value.cost.telemetry === "unknown" && value.verdict !== "blocked") throw new TypeError("unknown UI review cost telemetry requires blocked verdict."); + return value; +} + +export async function inspectDesignMemory({ repo = ".", profile = "design/product.design.json", baselines = null } = {}) { + const root = resolve(repo); + const profilePath = containedPath(root, profile, "profile"); + const blockers = []; + let profileValue = null; + let baselineValue = null; + try { + const profileBytes = await readFile(profilePath); + profileValue = JSON.parse(profileBytes.toString("utf8")); + validateProductDesignProfile(profileValue); + for (const source of [...profileValue.sources.tokens, ...profileValue.sources.components]) { + const sourcePath = containedPath(root, source.path, "source"); + const bytes = await readFile(sourcePath); + const actual = sha256(bytes); + if (actual !== source.digest) blockers.push(`source digest mismatch: ${source.path}`); + } + for (const decision of profileValue.sources.decisions) await readFile(containedPath(root, decision, "decision")); + const baselineRelative = baselines ?? profileValue.baselines.manifest; + const baselinePath = containedPath(root, baselineRelative, "baselines"); + baselineValue = JSON.parse(await readFile(baselinePath, "utf8")); + validateVisualBaselineManifest(baselineValue); + const expectedProfilePath = relative(root, profilePath).split(sep).join("/"); + if (baselineValue.profilePath !== expectedProfilePath) blockers.push(`baseline profilePath must equal ${expectedProfilePath}`); + if (baselineValue.profileDigest !== sha256(profileBytes)) blockers.push("baseline profileDigest does not match profile bytes"); + if (baselineValue.productId !== profileValue.productId) blockers.push("baseline productId does not match profile productId"); + if (baselineValue.status !== "approved") blockers.push("baseline manifest is proposed; human approval required"); + const viewportIds = new Set(profileValue.policy.viewports.map((entry) => entry.id)); + const states = new Set(profileValue.policy.states); + const themes = new Set(profileValue.policy.themes); + for (const capture of baselineValue.captures) { + if (!viewportIds.has(capture.viewport)) blockers.push(`capture ${capture.id} uses unknown viewport ${capture.viewport}`); + if (!states.has(capture.state)) blockers.push(`capture ${capture.id} uses unknown state ${capture.state}`); + if (!themes.has(capture.theme)) blockers.push(`capture ${capture.id} uses unrequired theme ${capture.theme}`); + } + for (const surface of profileValue.policy.surfaces) { + for (const viewport of surface.viewports) if (!viewportIds.has(viewport)) blockers.push(`surface ${surface.id} uses unknown viewport ${viewport}`); + for (const theme of surface.themes) if (!themes.has(theme)) blockers.push(`surface ${surface.id} uses unknown theme ${theme}`); + for (const state of surface.states) if (!states.has(state)) blockers.push(`surface ${surface.id} uses unknown state ${state}`); + if (baselineValue.captures.length > 0) { + for (const viewport of surface.viewports) for (const theme of surface.themes) for (const state of surface.states) { + const found = baselineValue.captures.some((capture) => capture.surface === surface.target && capture.kind === surface.kind && capture.viewport === viewport && capture.theme === theme && capture.state === state); + if (!found) blockers.push(`approved baseline missing capture: ${surface.id}/${viewport}/${theme}/${state}`); + } + } + } + } catch (error) { + blockers.push(error instanceof Error ? error.message : String(error)); + } + return { + ok: blockers.length === 0, + status: blockers.length === 0 ? "design_memory_ready" : "blocked", + profile: profileValue ? { productId: profileValue.productId, schemaVersion: profileValue.schemaVersion } : null, + baselines: baselineValue ? { schemaVersion: baselineValue.schemaVersion, captureCount: baselineValue.captures?.length ?? null } : null, + blockers, + }; +} + +function sourceArray(value, label) { + if (!Array.isArray(value) || value.length < 1 || value.length > 100) throw new TypeError(`${label} must contain 1 to 100 entries.`); + for (const [index, entry] of value.entries()) { + const item = `${label}[${index}]`; + object(entry, item); + exactKeys(entry, ["path", "digest"], item); + safePath(entry.path, `${item}.path`); + digest(entry.digest, `${item}.digest`); + } +} + +function containedPath(root, input, label) { + safePath(input, label); + const absolute = resolve(root, input); + const rel = relative(root, absolute); + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new TypeError(`${label} must stay inside repository.`); + return absolute; +} + +function safePath(value, label = "path") { + string(value, label); + if (isAbsolute(value) || value === ".." || value.split(/[\\/]/).includes("..") || /^[A-Za-z]:[\\/]/.test(value)) throw new TypeError(`${label} must be a safe repository-relative path.`); +} + +function exactKeys(value, allowed, label) { + const extra = Object.keys(value).filter((key) => !allowed.includes(key)); + if (extra.length > 0) throw new TypeError(`${label} has unsupported properties: ${extra.join(", ")}`); + for (const key of allowed) if (!(key in value)) throw new TypeError(`${label}.${key} is required.`); +} + +function object(value, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new TypeError(`${label} must be an object.`); +} + +function string(value, label, min = 1, max = 500) { + if (typeof value !== "string" || value.length < min || value.length > max) throw new TypeError(`${label} must be a string between ${min} and ${max} characters.`); +} + +function stringArray(value, label, min, max, itemValidator = null) { + if (!Array.isArray(value) || value.length < min || value.length > max) throw new TypeError(`${label} must contain ${min} to ${max} entries.`); + for (const [index, entry] of value.entries()) itemValidator ? itemValidator(entry, `${label}[${index}]`) : string(entry, `${label}[${index}]`); +} + +function uniqueStringArray(value, label, min, max) { + stringArray(value, label, min, max); + if (new Set(value).size !== value.length) throw new TypeError(`${label} must contain unique entries.`); +} + +function enumArray(value, allowed, label, min, max) { + stringArray(value, label, min, max); + for (const entry of value) if (!allowed.has(entry)) throw new TypeError(`${label} contains unsupported value: ${entry}`); +} + +function equal(value, expected, label) { + if (value !== expected) throw new TypeError(`${label} must equal ${expected}.`); +} + +function digest(value, label) { + if (typeof value !== "string" || !SHA256.test(value)) throw new TypeError(`${label} must be a lowercase SHA-256 digest.`); +} + +function integer(value, label, min, max) { + if (!Number.isInteger(value) || value < min || value > max) throw new TypeError(`${label} must be an integer between ${min} and ${max}.`); +} + +function number(value, label, min, max) { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) throw new TypeError(`${label} must be a number between ${min} and ${max}.`); +} + +function boolean(value, label) { + if (typeof value !== "boolean") throw new TypeError(`${label} must be boolean.`); +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index 1986498..2875e66 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -1,16 +1,21 @@ import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { mkdir, rm, stat } from "node:fs/promises"; +import { chmod, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; import { LedgerConflictError } from "./git-json-ledger.mjs"; import { runGit } from "./git-process.mjs"; import { digestObject } from "./stack-operation.mjs"; -const VALIDATION_MANIFEST_SCHEMA_VERSION = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V1 = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V2 = "tabellio-validation/v0.2"; const VALIDATION_RESULT_SCHEMA_VERSION_V1 = "tabellio-validation-result/v0.1"; const VALIDATION_RESULT_SCHEMA_VERSION_V2 = "tabellio-validation-result/v0.2"; +const VALIDATION_RESULT_SCHEMA_VERSION_V3 = "tabellio-validation-result/v0.3"; +const VALIDATOR_EVIDENCE_SCHEMA_VERSION = "tabellio-validator-evidence/v0.1"; +const VALIDATOR_TYPES = ["static", "schema", "semantic", "workflow", "visual", "operational", "security"]; const MAX_OUTPUT_TAIL_BYTES = 16 * 1024; +const MAX_EVIDENCE_BYTES = 1024 * 1024; export class ValidationRunner { constructor({ store, ledger }) { @@ -55,50 +60,38 @@ export class ValidationRunner { await mkdir(workspaceRoot, { recursive: true }); if (await stat(workspace).catch(() => null)) throw new Error(`Validation workspace already exists: ${workspace}.`); const startedAt = now.toISOString(); - const commands = []; + const definitions = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V1 + ? manifest.commands + : manifest.validators; + let execution = null; try { await runGit({ args: ["worktree", "add", "--detach", workspace, revision.headCommit], cwd: this.store.repoPath }); await mkdir(resolve(home, "tmp"), { recursive: true }); - let stopped = false; - for (const command of manifest.commands) { - if (stopped) { - commands.push(skippedCommand(command)); - continue; - } - const result = await runValidationCommand(command, workspace, home); - commands.push(result); - if (manifest.failFast && command.required && result.status !== "passed") stopped = true; - } + execution = await runValidationDefinitions({ manifest, definitions, workspace, home }); } finally { await runGit({ args: ["worktree", "remove", "--force", workspace], cwd: this.store.repoPath, acceptableExitCodes: [0, 128], }).catch(() => {}); - await rm(workspace, { recursive: true, force: true }); - await rm(home, { recursive: true, force: true }); + await removeGeneratedTree(workspace); + await removeGeneratedTree(home); } const completedAt = new Date().toISOString(); - const requiredFailed = commands.some((command, index) => manifest.commands[index].required && command.status !== "passed"); - const result = { - schemaVersion: VALIDATION_RESULT_SCHEMA_VERSION_V2, + const result = buildValidationResult({ + manifest, + definitions, + execution, runId, - repository: { id: repositoryId }, + repositoryId, revision, checkpointRevision, - suite: { - id: manifest.id, - manifestPath, - manifestDigest: digestObject(manifest), - }, - runner: { id: runnerId, runtime: `node-${process.version}` }, - status: requiredFailed ? "failed" : "passed", + manifestPath, + runnerId, checkpoints, - commands, startedAt, completedAt, - integrity: { algorithm: "sha256", digest: "0".repeat(64) }, - }; + }); result.integrity.digest = validationResultDigest(result); validateValidationResult(result); const path = validationPath(revision.headCommit, runId); @@ -107,6 +100,121 @@ export class ValidationRunner { } } +async function removeGeneratedTree(path) { + try { + await rm(path, { recursive: true, force: true }); + } catch (error) { + if (!error || !["EACCES", "EPERM"].includes(error.code)) throw error; + await makeTreeOwnerWritable(path); + await rm(path, { recursive: true, force: true }); + } +} + +async function makeTreeOwnerWritable(path) { + const info = await lstat(path).catch((error) => { + if (error?.code === "ENOENT") return null; + throw error; + }); + if (!info) return; + if (info.isSymbolicLink()) return; + if (!info.isDirectory()) { + await chmod(path, info.mode | 0o600).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + return; + } + await chmod(path, info.mode | 0o700).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + const entries = await readdir(path).catch((error) => { + if (error?.code === "ENOENT") return []; + throw error; + }); + for (const entry of entries) { + await makeTreeOwnerWritable(resolve(path, entry)); + } +} + +async function runValidationDefinitions({ manifest, definitions, workspace, home }) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const commands = []; + const validators = []; + let stopped = false; + for (const definition of definitions) { + if (stopped) { + commands.push(skippedCommand(definition)); + if (typed) validators.push(skippedValidator(definition)); + continue; + } + const commandResult = await runValidationCommand(definition, workspace, home); + commands.push(commandResult); + const gateStatus = typed + ? await appendValidatorResult(validators, definition, commandResult, workspace) + : commandResult.status; + stopped = manifest.failFast && definition.required && gateStatus !== "passed"; + } + return { commands, validators }; +} + +async function appendValidatorResult(validators, definition, commandResult, workspace) { + const result = await evaluateValidator(definition, commandResult, workspace); + validators.push(result); + return result.status; +} + +function buildValidationResult({ + manifest, + definitions, + execution, + runId, + repositoryId, + revision, + checkpointRevision, + manifestPath, + runnerId, + checkpoints, + startedAt, + completedAt, +}) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const decision = typed ? validationDecision(execution.validators) : null; + const requiredFailed = execution.commands.some((command, index) => definitions[index].required && command.status !== "passed"); + const result = { + schemaVersion: typed ? VALIDATION_RESULT_SCHEMA_VERSION_V3 : VALIDATION_RESULT_SCHEMA_VERSION_V2, + runId, + repository: { id: repositoryId }, + revision, + checkpointRevision, + suite: { + id: manifest.id, + manifestPath, + manifestDigest: digestObject(manifest), + }, + runner: { id: runnerId, runtime: `node-${process.version}` }, + status: decision?.status ?? (requiredFailed ? "failed" : "passed"), + checkpoints, + commands: execution.commands, + startedAt, + completedAt, + integrity: { algorithm: "sha256", digest: "0".repeat(64) }, + }; + if (!typed) return result; + result.acceptance = acceptanceResult(manifest.acceptance); + result.validators = execution.validators; + result.decision = decision; + return result; +} + +function acceptanceResult(acceptance) { + return { + id: acceptance.id, + source: acceptance.source, + risk: acceptance.risk, + digest: digestObject(acceptance), + requiredValidatorTypes: [...acceptance.requiredValidatorTypes], + }; +} + export async function latestValidationResult(ledger, commit, repositoryId = null) { oid(commit, "commit"); if (repositoryId !== null) requiredString(repositoryId, "repositoryId"); @@ -126,11 +234,25 @@ export async function latestValidationResult(ledger, commit, repositoryId = null export function validateValidationManifest(value) { object(value, "validation manifest"); - exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); - equals(value.schemaVersion, VALIDATION_MANIFEST_SCHEMA_VERSION, "validation manifest.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_MANIFEST_SCHEMA_VERSION_V1, VALIDATION_MANIFEST_SCHEMA_VERSION_V2], + "validation manifest.schemaVersion", + ); requiredString(value.id, "validation manifest.id"); boolean(value.failFast, "validation manifest.failFast"); boolean(value.requireEntireCheckpoint, "validation manifest.requireEntireCheckpoint"); + if (value.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2) { + exactKeys( + value, + ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "validation manifest", + ); + validateAcceptanceContract(value.acceptance); + validateValidators(value.validators, value.acceptance.requiredValidatorTypes); + return value; + } + exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); if (!Array.isArray(value.commands) || value.commands.length === 0 || value.commands.length > 50) { throw new Error("validation manifest.commands must contain 1 to 50 commands."); } @@ -155,7 +277,11 @@ export function validateValidationManifest(value) { export function validateValidationResult(value) { object(value, "validation result"); - member(value.schemaVersion, [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2], "validation result.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3], + "validation result.schemaVersion", + ); exactKeys(value, validationResultKeys(value.schemaVersion), "validation result"); requiredString(value.runId, "validation result.runId"); object(value.repository, "validation result.repository"); @@ -172,7 +298,11 @@ export function validateValidationResult(value) { exactKeys(value.runner, ["id", "runtime"], "validation result.runner"); requiredString(value.runner.id, "validation result.runner.id"); requiredString(value.runner.runtime, "validation result.runner.runtime"); - member(value.status, ["passed", "failed"], "validation result.status"); + member( + value.status, + value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3 ? ["passed", "failed", "blocked"] : ["passed", "failed"], + "validation result.status", + ); stringArray(value.checkpoints, "validation result.checkpoints"); if (!Array.isArray(value.commands) || value.commands.length === 0) throw new Error("validation result.commands must be a non-empty array."); value.commands.forEach((command, index) => validateCommandResult(command, `validation result.commands[${index}]`)); @@ -183,6 +313,21 @@ export function validateValidationResult(value) { equals(value.integrity.algorithm, "sha256", "validation result.integrity.algorithm"); sha256(value.integrity.digest, "validation result.integrity.digest"); if (validationResultDigest(value) !== value.integrity.digest) throw new Error("validation result integrity digest does not match."); + if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + validateAcceptanceResult(value.acceptance); + if (!Array.isArray(value.validators) || value.validators.length === 0) { + throw new Error("validation result.validators must be a non-empty array."); + } + value.validators.forEach((validator, index) => validateValidatorResult(validator, `validation result.validators[${index}]`)); + const commandIds = value.commands.map((command) => command.id); + const validatorIds = value.validators.map((validator) => validator.id); + if (JSON.stringify(commandIds) !== JSON.stringify(validatorIds)) { + throw new Error("validation result validators must align with command ids and order."); + } + validateValidationDecision(value.decision, value.validators); + if (value.status !== value.decision.status) throw new Error("validation result status does not match decision status."); + return value; + } const expectedStatus = value.commands.some((command) => command.required && command.status !== "passed") ? "failed" : "passed"; if (value.status !== expectedStatus) throw new Error("validation result status does not match required command results."); return value; @@ -190,11 +335,14 @@ export function validateValidationResult(value) { function validationResultKeys(schemaVersion) { const keys = ["schemaVersion", "runId", "repository", "revision", "suite", "runner", "status", "checkpoints", "commands", "startedAt", "completedAt", "integrity"]; + if (schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + return [...keys, "checkpointRevision", "acceptance", "validators", "decision"]; + } return schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2 ? [...keys, "checkpointRevision"] : keys; } function validateCheckpointRevision(value) { - if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2) { + if ([VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3].includes(value.schemaVersion)) { validateRevision(value.checkpointRevision, "validation result.checkpointRevision"); } } @@ -207,9 +355,220 @@ function validateRevision(value, path) { oid(value.headCommit, `${path}.headCommit`); } +function validateAcceptanceContract(value) { + object(value, "validation manifest.acceptance"); + exactKeys( + value, + ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "validation manifest.acceptance", + ); + requiredString(value.id, "validation manifest.acceptance.id"); + requiredString(value.source, "validation manifest.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation manifest.acceptance.risk"); + boundedUniqueStrings(value.outcomes, "validation manifest.acceptance.outcomes", { minimum: 1, maximum: 50 }); + boundedUniqueStrings(value.invariants, "validation manifest.acceptance.invariants", { minimum: 0, maximum: 50 }); + boundedUniqueStrings(value.forbiddenOutcomes, "validation manifest.acceptance.forbiddenOutcomes", { minimum: 0, maximum: 50 }); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation manifest.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidators(value, requiredTypes) { + if (!Array.isArray(value) || value.length === 0 || value.length > 50) { + throw new Error("validation manifest.validators must contain 1 to 50 validators."); + } + const ids = new Set(); + for (const [index, validator] of value.entries()) { + validateValidatorDefinition(validator, index); + if (ids.has(validator.id)) throw new Error(`validation manifest.validators contains duplicate id ${validator.id}.`); + ids.add(validator.id); + } + for (const type of requiredTypes) { + if (!value.some((validator) => validator.type === type && validator.required)) { + throw new Error(`validation manifest requires a required ${type} validator.`); + } + } +} + +function validateValidatorDefinition(validator, index) { + const path = `validation manifest.validators[${index}]`; + object(validator, path); + exactKeys(validator, ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], path); + requiredString(validator.id, `${path}.id`); + member(validator.type, VALIDATOR_TYPES, `${path}.type`); + validateValidatorCommand(validator, path); + validateValidatorEvidenceDeclaration(validator, path); + validateValidatorPolicy(validator.policy, `${path}.policy`, validator.evidence !== null); +} + +function validateValidatorCommand(validator, path) { + if (!Array.isArray(validator.argv) || validator.argv.length === 0 || validator.argv.length > 100) { + throw new Error(`${path}.argv must contain 1 to 100 arguments.`); + } + validator.argv.forEach((argument, argumentIndex) => requiredString(argument, `${path}.argv[${argumentIndex}]`)); + validateRelativePath(validator.cwd, `${path}.cwd`, { allowDot: true }); + if (!Number.isInteger(validator.timeoutMs) || validator.timeoutMs < 100 || validator.timeoutMs > 60 * 60 * 1000) { + throw new Error(`${path}.timeoutMs must be between 100 and 3600000.`); + } + boolean(validator.required, `${path}.required`); +} + +function validateValidatorEvidenceDeclaration(validator, path) { + if (validator.evidence === null) { + if (validator.type !== "static") throw new Error(`${path}.evidence is required for ${validator.type} validators.`); + return; + } + object(validator.evidence, `${path}.evidence`); + exactKeys(validator.evidence, ["path"], `${path}.evidence`); + validateRelativePath(validator.evidence.path, `${path}.evidence.path`); +} + +function validateValidatorPolicy(value, path, hasEvidence) { + object(value, path); + exactKeys(value, ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], path); + validateMetricThresholds(value.metricThresholds, `${path}.metricThresholds`); + validateCostPolicy(value, path); + if ((value.metricThresholds.length > 0 || value.requireCostTelemetry || value.maxCostUsd !== null) && !hasEvidence) { + throw new Error(`${path} requires an evidence report.`); + } +} + +function validateMetricThresholds(value, path) { + if (!Array.isArray(value) || value.length > 100) throw new Error(`${path} must contain at most 100 thresholds.`); + const metrics = new Set(); + for (const [index, threshold] of value.entries()) { + const thresholdPath = `${path}[${index}]`; + object(threshold, thresholdPath); + exactKeys(threshold, ["metric", "operator", "value"], thresholdPath); + requiredString(threshold.metric, `${thresholdPath}.metric`); + if (metrics.has(threshold.metric)) throw new Error(`${path} contains duplicate metric ${threshold.metric}.`); + metrics.add(threshold.metric); + member(threshold.operator, ["gte", "lte", "eq"], `${thresholdPath}.operator`); + finiteNumber(threshold.value, `${thresholdPath}.value`); + } +} + +function validateCostPolicy(value, path) { + if (value.maxCostUsd !== null) nonNegativeNumber(value.maxCostUsd, `${path}.maxCostUsd`); + boolean(value.requireCostTelemetry, `${path}.requireCostTelemetry`); + if (value.maxCostUsd !== null && value.requireCostTelemetry !== true) { + throw new Error(`${path}.requireCostTelemetry must be true when maxCostUsd is set.`); + } +} + +export function validateValidatorEvidence(value) { + object(value, "validator evidence"); + exactKeys(value, ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], "validator evidence"); + equals(value.schemaVersion, VALIDATOR_EVIDENCE_SCHEMA_VERSION, "validator evidence.schemaVersion"); + requiredString(value.validatorId, "validator evidence.validatorId"); + member(value.status, ["passed", "failed", "blocked"], "validator evidence.status"); + requiredString(value.summary, "validator evidence.summary"); + maxLength(value.summary, 2_000, "validator evidence.summary"); + validateEvidenceMetrics(value.metrics); + validateEvidenceCost(value.cost); + validateEvidenceArtifacts(value.artifacts); + return value; +} + +function validateEvidenceMetrics(value) { + if (!Array.isArray(value) || value.length > 100) throw new Error("validator evidence.metrics must contain at most 100 metrics."); + const metricNames = new Set(); + for (const [index, metric] of value.entries()) { + const path = `validator evidence.metrics[${index}]`; + object(metric, path); + exactKeys(metric, ["name", "value", "unit"], path); + requiredString(metric.name, `${path}.name`); + if (metricNames.has(metric.name)) throw new Error(`validator evidence.metrics contains duplicate name ${metric.name}.`); + metricNames.add(metric.name); + finiteNumber(metric.value, `${path}.value`); + requiredString(metric.unit, `${path}.unit`); + } +} + +function validateEvidenceArtifacts(value) { + if (!Array.isArray(value) || value.length > 50) throw new Error("validator evidence.artifacts must contain at most 50 artifacts."); + for (const [index, artifact] of value.entries()) validateEvidenceArtifact(artifact, `validator evidence.artifacts[${index}]`); +} + +function validateEvidenceCost(value) { + object(value, "validator evidence.cost"); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], "validator evidence.cost"); + member(value.telemetry, ["available", "unavailable", "not_applicable"], "validator evidence.cost.telemetry"); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, "validator evidence.cost.usd"); + nonNegativeInteger(value.modelCalls, "validator evidence.cost.modelCalls"); + nonNegativeInteger(value.toolCalls, "validator evidence.cost.toolCalls"); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error("validator evidence cost values must be null when telemetry is unavailable or not applicable."); + } +} + +function validateEvidenceArtifact(value, path) { + object(value, path); + exactKeys(value, ["name", "uri", "digest", "mediaType", "bytes"], path); + requiredString(value.name, `${path}.name`); + durableUri(value.uri, `${path}.uri`); + sha256(value.digest, `${path}.digest`); + requiredString(value.mediaType, `${path}.mediaType`); + nonNegativeInteger(value.bytes, `${path}.bytes`); +} + +function validateAcceptanceResult(value) { + object(value, "validation result.acceptance"); + exactKeys(value, ["id", "source", "risk", "digest", "requiredValidatorTypes"], "validation result.acceptance"); + requiredString(value.id, "validation result.acceptance.id"); + requiredString(value.source, "validation result.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation result.acceptance.risk"); + sha256(value.digest, "validation result.acceptance.digest"); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation result.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidatorResult(value, path) { + object(value, path); + exactKeys(value, ["id", "type", "required", "status", "evidence", "reasons"], path); + requiredString(value.id, `${path}.id`); + member(value.type, VALIDATOR_TYPES, `${path}.type`); + boolean(value.required, `${path}.required`); + member(value.status, ["passed", "failed", "blocked", "skipped"], `${path}.status`); + boundedUniqueStrings(value.reasons, `${path}.reasons`, { minimum: 0, maximum: 128 }); + if (value.evidence === null) return; + object(value.evidence, `${path}.evidence`); + exactKeys(value.evidence, ["path", "digest", "report"], `${path}.evidence`); + validateRelativePath(value.evidence.path, `${path}.evidence.path`); + sha256(value.evidence.digest, `${path}.evidence.digest`); + validateValidatorEvidence(value.evidence.report); + if (digestObject(value.evidence.report) !== value.evidence.digest) throw new Error(`${path}.evidence digest does not match report.`); + if (value.evidence.report.validatorId !== value.id) throw new Error(`${path}.evidence validator id does not match.`); +} + +function validateValidationDecision(value, validators) { + object(value, "validation result.decision"); + exactKeys(value, ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], "validation result.decision"); + member(value.status, ["passed", "failed", "blocked"], "validation result.decision.status"); + boundedUniqueStrings(value.reasons, "validation result.decision.reasons", { minimum: 0, maximum: 6_400 }); + nonNegativeNumber(value.totalCostUsd, "validation result.decision.totalCostUsd"); + boolean(value.costTelemetryComplete, "validation result.decision.costTelemetryComplete"); + const expected = validationDecision(validators); + if ( + value.status !== expected.status + || JSON.stringify(value.reasons) !== JSON.stringify(expected.reasons) + || value.totalCostUsd !== expected.totalCostUsd + || value.costTelemetryComplete !== expected.costTelemetryComplete + ) { + throw new Error("validation result decision does not match validator results."); + } +} + async function runValidationCommand(command, workspace, home) { - const cwd = resolve(workspace, command.cwd); - if (relative(workspace, cwd).startsWith("..") || isAbsolute(relative(workspace, cwd))) throw new Error(`Command ${command.id} cwd escapes validation workspace.`); + const cwd = resolveWorkspacePath(workspace, command.cwd, `Command ${command.id} cwd`); const startedAt = new Date(); const execution = await spawnCaptured(command.argv, { cwd, @@ -234,6 +593,156 @@ async function runValidationCommand(command, workspace, home) { }; } +async function evaluateValidator(validator, commandResult, workspace) { + const base = validatorResultBase(validator); + const commandGate = validatorCommandGate(validator, commandResult.status); + if (commandGate) return { ...base, ...commandGate }; + if (validator.evidence === null) return base; + + let evidence; + try { + evidence = await loadValidatorEvidence(workspace, validator.evidence.path, validator.id); + } catch (error) { + return { + ...base, + status: "blocked", + reasons: blockedReasons(validator, `evidence_invalid:${boundedReason(error instanceof Error ? error.message : String(error))}`), + }; + } + const evaluation = evaluateEvidencePolicy(validator, evidence.report); + return { ...base, ...evaluation, evidence }; +} + +function validatorResultBase(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "passed", + evidence: null, + reasons: [], + }; +} + +function validatorCommandGate(validator, status) { + if (status === "timed_out") return { status: "blocked", reasons: blockedReasons(validator, "command_timed_out") }; + if (status === "error") return { status: "blocked", reasons: blockedReasons(validator, "command_error") }; + if (status === "skipped") return { status: "skipped", reasons: ["fail_fast"] }; + if (status !== "failed") return null; + return validator.evidence === null + ? { status: "failed", reasons: ["command_failed"] } + : { status: "blocked", reasons: blockedReasons(validator, "validator_command_failed") }; +} + +function evaluateEvidencePolicy(validator, report) { + const evaluations = [ + reportedEvidenceStatus(report.status), + evaluateMetricPolicy(report.metrics, validator.policy.metricThresholds), + evaluateCostPolicy(report.cost, validator.policy), + ]; + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + const reasons = [...new Set(evaluations.flatMap((evaluation) => evaluation.reasons))].sort(); + return { status, reasons }; +} + +function reportedEvidenceStatus(status) { + if (status === "failed") return { status, reasons: ["evidence_reported_failed"] }; + if (status === "blocked") return { status, reasons: ["evidence_reported_blocked"] }; + return { status: "passed", reasons: [] }; +} + +function evaluateMetricPolicy(metrics, thresholds) { + const evaluations = thresholds.map((threshold) => evaluateMetricThreshold(metrics, threshold)); + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + return { status, reasons: evaluations.flatMap((evaluation) => evaluation.reasons) }; +} + +function evaluateMetricThreshold(metrics, threshold) { + const metric = metrics.find((candidate) => candidate.name === threshold.metric); + if (!metric) return { status: "blocked", reasons: [`metric_missing:${threshold.metric}`] }; + if (thresholdPassed(metric.value, threshold.operator, threshold.value)) return { status: "passed", reasons: [] }; + return { + status: "failed", + reasons: [`threshold_not_met:${threshold.metric}:${threshold.operator}:${threshold.value}`], + }; +} + +function evaluateCostPolicy(cost, policy) { + if (policy.requireCostTelemetry && cost.telemetry !== "available") { + return { status: "blocked", reasons: ["cost_telemetry_unavailable"] }; + } + if (policy.maxCostUsd !== null && cost.telemetry === "available" && cost.usd > policy.maxCostUsd) { + return { status: "failed", reasons: [`cost_budget_exceeded:${policy.maxCostUsd}`] }; + } + return { status: "passed", reasons: [] }; +} + +async function loadValidatorEvidence(workspace, evidencePath, validatorId) { + const path = resolveWorkspacePath(workspace, evidencePath, `Validator ${validatorId} evidence`); + const metadata = await stat(path).catch(() => { + throw new Error("file is missing or unreadable"); + }); + if (!metadata.isFile()) throw new Error("must be a regular file"); + if (metadata.size > MAX_EVIDENCE_BYTES) throw new Error(`exceeds ${MAX_EVIDENCE_BYTES} bytes`); + const source = await readFile(path).catch(() => { + throw new Error("file is unreadable"); + }); + let report; + try { + report = JSON.parse(source.toString("utf8")); + } catch { + throw new Error("is not valid JSON"); + } + validateValidatorEvidence(report); + if (report.validatorId !== validatorId) throw new Error(`validatorId must be ${validatorId}`); + return { + path: evidencePath, + digest: digestObject(report), + report, + }; +} + +function thresholdPassed(actual, operator, expected) { + if (operator === "gte") return actual >= expected; + if (operator === "lte") return actual <= expected; + return actual === expected; +} + +function blockedReasons(validator, reason) { + const reasons = [reason]; + if (validator.policy.requireCostTelemetry) reasons.push("cost_telemetry_unavailable"); + return [...new Set(reasons)].sort(); +} + +function validationDecision(validators) { + const required = validators.filter((validator) => validator.required); + const status = aggregateStatus(required.map((validator) => validator.status)); + const reasons = [...new Set(required.flatMap((validator) => validator.reasons.map((reason) => `${validator.id}:${reason}`)))].sort(); + const totalCostUsd = Math.round(validators.reduce((total, validator) => { + const cost = validator.evidence?.report.cost; + return total + (cost?.telemetry === "available" ? cost.usd : 0); + }, 0) * 100_000_000) / 100_000_000; + const costTelemetryComplete = !required.some((validator) => validator.reasons.includes("cost_telemetry_unavailable")); + return { status, reasons, totalCostUsd, costTelemetryComplete }; +} + +function aggregateStatus(statuses) { + if (statuses.includes("blocked")) return "blocked"; + if (statuses.some((status) => status !== "passed")) return "failed"; + return "passed"; +} + +function skippedValidator(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "skipped", + evidence: null, + reasons: validator.policy.requireCostTelemetry ? ["cost_telemetry_unavailable", "fail_fast"] : ["fail_fast"], + }; +} + function skippedCommand(command) { return { id: command.id, @@ -356,7 +865,7 @@ function validationResultDigest(value) { } function validationEnvironment(home) { - const allowed = ["PATH", "LANG", "LC_ALL", "TERM"]; + const allowed = ["PATH", "LANG", "LC_ALL", "TERM", "TABELLIO_ARTIFACT_BASE_URI"]; const env = Object.fromEntries(allowed.flatMap((key) => process.env[key] === undefined ? [] : [[key, process.env[key]]])); return { ...env, HOME: home, TMPDIR: resolve(home, "tmp"), CI: "1", NO_COLOR: "1", GIT_TERMINAL_PROMPT: "0" }; } @@ -434,6 +943,64 @@ function stringArray(value, path) { if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); } +function boundedUniqueStrings(value, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => { + requiredString(item, `${path}[${index}]`); + maxLength(item, 2_000, `${path}[${index}]`); + }); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function enumArray(value, allowed, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => member(item, allowed, `${path}[${index}]`)); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be a finite number.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function maxLength(value, maximum, path) { + if (typeof value === "string" && value.length > maximum) throw new Error(`${path} must contain at most ${maximum} characters.`); +} + +function durableUri(value, path) { + requiredString(value, path); + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${path} must be an absolute URI.`); + } + if (parsed.protocol === "file:") throw new Error(`${path} must not use an ephemeral file URI.`); +} + +function boundedReason(value) { + return value.replace(/[\0\r\n]+/g, " ").slice(0, 500); +} + +function resolveWorkspacePath(workspace, path, label) { + const resolved = resolve(workspace, path); + const relation = relative(workspace, resolved); + if (relation.startsWith("..") || isAbsolute(relation)) throw new Error(`${label} escapes validation workspace.`); + return resolved; +} + function validateRelativePath(value, path, { allowDot = false } = {}) { requiredString(value, path); if ((allowDot && value === ".")) return; diff --git a/scripts/tabellio-control-ref.mjs b/scripts/tabellio-control-ref.mjs index cf3e4bf..bd09039 100644 --- a/scripts/tabellio-control-ref.mjs +++ b/scripts/tabellio-control-ref.mjs @@ -11,6 +11,7 @@ import { readControlRefFile, snapshotControlRefs, } from "./lib/control-ref-transport.mjs"; +import { parseCommandOptions } from "./lib/cli-options.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; import { NativeGitStore } from "./providers/native-git-store.mjs"; @@ -72,21 +73,11 @@ async function output(value, path) { } function parse(args) { - const command = args[0]; - if (!["plan", "execute"].includes(command)) throw new Error("Expected command: plan or execute."); - const values = { command }; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = command === "plan" - ? ["command", "operation", "repo", "repoId", "remote", "refs", "out", "tokenFile", "gitUsername"] - : ["command", "repo", "repoId", "intent", "approval", "stateRoot", "tokenFile", "gitUsername"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); + const values = parseCommandOptions(args, { + plan: ["operation", "repo", "repoId", "remote", "refs", "out", "tokenFile", "gitUsername"], + execute: ["repo", "repoId", "intent", "approval", "stateRoot", "tokenFile", "gitUsername"], + }); + const { command } = values; if (command === "plan" && !["publish", "fetch"].includes(values.operation)) throw new Error("plan requires --operation publish or fetch."); if (command === "execute" && (!values.intent || !values.approval)) throw new Error("execute requires --intent and --approval."); if (values.tokenFile && !values.gitUsername) values.gitUsername = "git"; diff --git a/scripts/tabellio-stack-operation.mjs b/scripts/tabellio-stack-operation.mjs index f182e27..a821843 100755 --- a/scripts/tabellio-stack-operation.mjs +++ b/scripts/tabellio-stack-operation.mjs @@ -4,6 +4,8 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { parseCommandOptions } from "./lib/cli-options.mjs"; + import { reportCliError } from "./lib/cli-options.mjs"; import { runGit } from "./lib/git-process.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; @@ -115,21 +117,11 @@ async function currentBranch(cwd) { } function parseArgs(args) { - const command = args[0]; - if (!["plan", "execute"].includes(command)) throw new Error("Expected command: plan or execute."); - const values = {}; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = command === "plan" - ? ["operation", "repo", "repoId", "branch", "out", "title", "bodyFile", "draft", "restack", "method", "readyTimeout", "mergeTimeout"] - : ["repo", "repoId", "intent", "approval", "stateRoot", "binary", "tokenFile", "gitUsername"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); + const values = parseCommandOptions(args, { + plan: ["operation", "repo", "repoId", "branch", "out", "title", "bodyFile", "draft", "restack", "method", "readyTimeout", "mergeTimeout"], + execute: ["repo", "repoId", "intent", "approval", "stateRoot", "binary", "tokenFile", "gitUsername"], + }); + const { command } = values; if (command === "plan" && !STACK_OPERATIONS.includes(values.operation)) { throw new Error(`--operation must be one of: ${STACK_OPERATIONS.join(", ")}.`); } @@ -137,7 +129,7 @@ function parseArgs(args) { if (!values.intent) throw new Error("execute requires --intent."); if (!values.approval) throw new Error("execute requires --approval."); } - return { command, ...values }; + return values; } function booleanOption(value, defaultValue, path) { diff --git a/scripts/tabellio-validate.mjs b/scripts/tabellio-validate.mjs index 595d0ee..373d046 100755 --- a/scripts/tabellio-validate.mjs +++ b/scripts/tabellio-validate.mjs @@ -2,7 +2,7 @@ import { resolve } from "node:path"; -import { reportCliError } from "./lib/cli-options.mjs"; +import { parseCommandOptions, reportCliError } from "./lib/cli-options.mjs"; import { GitJsonLedger } from "./lib/git-json-ledger.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; import { latestValidationResult, ValidationRunner } from "./lib/validation-runner.mjs"; @@ -15,7 +15,7 @@ try { repoPath: store.repoPath, ref: options.ledgerRef ?? "refs/tabellio/validations", }); - if (options.command === "run") { + if (["run", "gate"].includes(options.command)) { const runner = new ValidationRunner({ store, ledger }); const result = await runner.run({ repositoryId: await repositoryIdentity(store, options.repoId), @@ -25,6 +25,7 @@ try { runnerId: options.runnerId ?? "local", }); console.log(JSON.stringify({ ok: true, ...result }, null, 2)); + if (options.command === "gate" && result.result.status !== "passed") process.exitCode = 1; } else { const commit = await store.resolveRef(options.commit ?? "HEAD"); const result = await latestValidationResult(ledger, commit); @@ -36,20 +37,9 @@ try { } function parseArgs(args) { - const command = args[0]; - if (!["run", "latest"].includes(command)) throw new Error("Expected command: run or latest."); - const values = {}; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = command === "run" - ? ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"] - : ["repo", "commit", "ledgerRef"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); - return { command, ...values }; + return parseCommandOptions(args, { + run: ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"], + gate: ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"], + latest: ["repo", "commit", "ledgerRef"], + }); } diff --git a/scripts/tabellio-validator.mjs b/scripts/tabellio-validator.mjs new file mode 100644 index 0000000..850126d --- /dev/null +++ b/scripts/tabellio-validator.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; + +const MAX_CAPTURE_BYTES = 1024 * 1024; + +try { + const options = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const configPath = containedPath(root, options.config ?? ".tabellio/validators.json", "config"); + const config = validateConfig(JSON.parse(await readFile(configPath, "utf8"))); + const profile = config.profiles[options.profile]; + if (!profile) throw new Error(`Unknown validator profile: ${options.profile}.`); + + const execution = await runProfile(profile, root); + if (!options.out) { + process.exitCode = execution.status === "passed" ? 0 : execution.status === "failed" ? 1 : 2; + } else { + if (!options.validatorId) throw new Error("--validator-id is required with --out."); + const outPath = containedPath(root, options.out, "out"); + const evidence = buildEvidence(options.validatorId, profile, execution); + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + console.log(JSON.stringify({ ok: true, profile: options.profile, out: relative(root, outPath), status: evidence.status })); + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; +} + +function parseArgs(args) { + const values = {}; + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? "arguments"}.`); + const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); + if (!["config", "profile", "validatorId", "out"].includes(key)) throw new Error(`Unsupported option: ${flag}.`); + if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); + values[key] = value; + } + if (!values.profile) throw new Error("--profile is required."); + return values; +} + +function validateConfig(value) { + object(value, "adapter config"); + exactKeys(value, ["schemaVersion", "profiles"], "adapter config"); + if (value.schemaVersion !== "tabellio-adapter/v0.1") throw new Error("adapter config schemaVersion must be tabellio-adapter/v0.1."); + object(value.profiles, "adapter config.profiles"); + const entries = Object.entries(value.profiles); + if (entries.length < 1 || entries.length > 20) throw new Error("adapter config.profiles must contain 1 to 20 profiles."); + for (const [id, profile] of entries) validateProfile(id, profile); + return value; +} + +function validateProfile(id, value) { + requiredString(id, "profile id"); + object(value, `profile ${id}`); + exactKeys(value, ["commands", "metrics", "cost", "summary"], `profile ${id}`); + if (!Array.isArray(value.commands) || value.commands.length < 1 || value.commands.length > 20) { + throw new Error(`profile ${id}.commands must contain 1 to 20 commands.`); + } + for (const [index, argv] of value.commands.entries()) validateArgv(argv, `profile ${id}.commands[${index}]`); + if (!Array.isArray(value.metrics) || value.metrics.length > 100) throw new Error(`profile ${id}.metrics must be an array.`); + for (const [index, metric] of value.metrics.entries()) validateMetric(metric, `profile ${id}.metrics[${index}]`); + validateCost(value.cost, `profile ${id}.cost`); + requiredString(value.summary, `profile ${id}.summary`); +} + +function validateArgv(argv, path) { + if (!Array.isArray(argv) || argv.length < 1 || argv.length > 100) throw new Error(`${path} must contain 1 to 100 arguments.`); + for (const [index, argument] of argv.entries()) requiredString(argument, `${path}[${index}]`); +} + +function validateMetric(value, path) { + object(value, path); + const keys = Object.keys(value); + const extracted = keys.includes("pattern"); + exactKeys(value, extracted ? ["name", "unit", "pattern"] : ["name", "unit", "passValue", "failValue"], path); + requiredString(value.name, `${path}.name`); + requiredString(value.unit, `${path}.unit`); + if (extracted) { + requiredString(value.pattern, `${path}.pattern`); + const expression = new RegExp(value.pattern); + if (expression.source.length > 500) throw new Error(`${path}.pattern is too long.`); + } else { + finiteNumber(value.passValue, `${path}.passValue`); + finiteNumber(value.failValue, `${path}.failValue`); + } +} + +function validateCost(value, path) { + object(value, path); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], path); + if (!['available', 'unavailable', 'not_applicable'].includes(value.telemetry)) throw new Error(`${path}.telemetry is invalid.`); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, `${path}.usd`); + nonNegativeInteger(value.modelCalls, `${path}.modelCalls`); + nonNegativeInteger(value.toolCalls, `${path}.toolCalls`); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error(`${path} unavailable or not_applicable values must be null.`); + } +} + +async function runProfile(profile, cwd) { + const outputs = []; + for (const argv of profile.commands) { + const result = await runCommand(argv, cwd); + outputs.push(result.output); + if (result.spawnError) return { status: "blocked", outputs, reason: `command unavailable: ${argv[0]}` }; + if (result.exitCode !== 0) return { status: "failed", outputs, reason: `command failed: ${argv.join(" ")}` }; + } + return { status: "passed", outputs, reason: "all declared commands passed" }; +} + +function runCommand(argv, cwd) { + return new Promise((resolvePromise) => { + let captured = ""; + let capturedBytes = 0; + let spawnError = null; + const child = spawn(argv[0], argv.slice(1), { + cwd, + env: { ...process.env, CI: "1", TABELLIO_VALIDATION: "1", NO_COLOR: "1" }, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const forward = (target) => (chunk) => { + target.write(chunk); + if (capturedBytes >= MAX_CAPTURE_BYTES) return; + const remaining = MAX_CAPTURE_BYTES - capturedBytes; + const slice = chunk.subarray(0, remaining); + captured += slice.toString("utf8"); + capturedBytes += slice.byteLength; + }; + child.stdout.on("data", forward(process.stdout)); + child.stderr.on("data", forward(process.stderr)); + child.once("error", (error) => { spawnError = error; }); + child.once("close", (exitCode, signal) => resolvePromise({ + exitCode: exitCode ?? (signal ? 1 : 0), + output: captured, + spawnError, + })); + }); +} + +function buildEvidence(validatorId, profile, execution) { + const combinedOutput = execution.outputs.join("\n"); + const metrics = []; + let status = execution.status; + let reason = execution.reason; + for (const metric of profile.metrics) { + if (Object.hasOwn(metric, "pattern")) { + const match = new RegExp(metric.pattern, "m").exec(combinedOutput); + const value = match ? Number(match[1]) : Number.NaN; + if (!Number.isFinite(value)) { + status = "blocked"; + reason = `metric unavailable: ${metric.name}`; + continue; + } + metrics.push({ name: metric.name, value, unit: metric.unit }); + } else { + metrics.push({ + name: metric.name, + value: execution.status === "passed" ? metric.passValue : metric.failValue, + unit: metric.unit, + }); + } + } + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status, + summary: `${profile.summary} ${reason}.`.slice(0, 2000), + metrics, + cost: profile.cost, + artifacts: [], + }; +} + +function containedPath(root, input, label) { + requiredString(input, label); + if (isAbsolute(input)) throw new Error(`${label} must be relative.`); + const target = resolve(root, input); + const rel = relative(root, target); + if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) throw new Error(`${label} must stay inside the repository.`); + return target; +} + +function exactKeys(value, expected, path) { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) throw new Error(`${path} must contain exactly: ${wanted.join(", ")}.`); +} + +function object(value, path) { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object.`); +} + +function requiredString(value, path) { + if (typeof value !== "string" || value === "" || /[\0\r\n]/.test(value)) throw new Error(`${path} must be a non-empty single-line string.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be finite.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} diff --git a/tabellio.validation.json b/tabellio.validation.json index f1c366d..fa14c2f 100644 --- a/tabellio.validation.json +++ b/tabellio.validation.json @@ -1,15 +1,33 @@ { - "schemaVersion": "tabellio-validation/v0.1", - "id": "tabellio-default", - "failFast": true, + "schemaVersion": "tabellio-validation/v0.2", + "id": "tabellio-product-validity", + "failFast": false, "requireEntireCheckpoint": true, - "commands": [ - { - "id": "repository-check", - "argv": ["npm", "run", "check"], - "cwd": ".", - "timeoutMs": 1200000, - "required": true - } + "acceptance": { + "id": "tabellio-v0.4-control-plane", + "source": "repository product contract and docs/product-validation.md", + "risk": "high", + "outcomes": [ + "Agent runs, checkpoints, reviews, and validation results remain bound to exact Git state.", + "Typed validation distinguishes passed, failed, and blocked evidence.", + "External actions and release operations remain approval-gated and fail closed." + ], + "invariants": [ + "Validation never publishes, deploys, or mutates a live external system.", + "Unknown or missing cost telemetry cannot be treated as zero." + ], + "forbiddenOutcomes": [ + "A stale validation result is accepted for a moved head.", + "A blocked validator is reported as passed.", + "An unapproved external action is executed." + ], + "requiredValidatorTypes": ["static", "schema", "semantic", "workflow", "security"] + }, + "validators": [ + {"id": "control-plane-static", "type": "static", "argv": ["tabellio-validator", "--profile", "static"], "cwd": ".", "timeoutMs": 1200000, "required": true, "evidence": null, "policy": {"metricThresholds": [], "maxCostUsd": null, "requireCostTelemetry": false}}, + {"id": "control-plane-schema", "type": "schema", "argv": ["tabellio-validator", "--profile", "schema", "--validator-id", "control-plane-schema", "--out", ".artifacts/tabellio/schema.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/schema.json"}, "policy": {"metricThresholds": [{"metric": "platform_contract_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-semantic", "type": "semantic", "argv": ["tabellio-validator", "--profile", "semantic", "--validator-id", "control-plane-semantic", "--out", ".artifacts/tabellio/semantic.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/semantic.json"}, "policy": {"metricThresholds": [{"metric": "validation_contract_examples_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-workflow", "type": "workflow", "argv": ["tabellio-validator", "--profile", "workflow", "--validator-id", "control-plane-workflow", "--out", ".artifacts/tabellio/workflow.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/workflow.json"}, "policy": {"metricThresholds": [{"metric": "run_review_workflows_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-security", "type": "security", "argv": ["tabellio-validator", "--profile", "security", "--validator-id", "control-plane-security", "--out", ".artifacts/tabellio/security.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/security.json"}, "policy": {"metricThresholds": [{"metric": "external_action_policy_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}} ] } diff --git a/tests/design-memory.test.mjs b/tests/design-memory.test.mjs new file mode 100644 index 0000000..295f80e --- /dev/null +++ b/tests/design-memory.test.mjs @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + finalizeVisualCaptures, + inspectDesignMemory, + validateProductDesignProfile, + validateUiReviewArtifact, + validateVisualBaselineManifest, +} from "../scripts/lib/design-memory.mjs"; + +const root = new URL("../", import.meta.url).pathname; + +test("example design memory is valid and internally consistent", async () => { + const result = await inspectDesignMemory({ + repo: root, + profile: "examples/tabellio-design-memory/product.design.json", + }); + assert.deepEqual(result.blockers, []); + assert.equal(result.status, "design_memory_ready"); + assert.equal(result.baselines.captureCount, 1); +}); + +test("capture finalizer requires complete PNG matrix and emits exact-commit candidate", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "tabellio-visual-captures-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const sourceRoot = `${root}/examples/tabellio-design-memory`; + await mkdir(`${directory}/captures`); + await Promise.all([ + writeFile(`${directory}/tokens.json`, await readFile(`${sourceRoot}/tokens.json`)), + writeFile(`${directory}/components.json`, await readFile(`${sourceRoot}/components.json`)), + writeFile(`${directory}/decision.md`, await readFile(`${sourceRoot}/decision.md`)), + ]); + const profile = JSON.parse(await readFile(`${sourceRoot}/product.design.json`, "utf8")); + profile.sources.tokens[0].path = "tokens.json"; + profile.sources.components[0].path = "components.json"; + profile.sources.decisions[0] = "decision.md"; + profile.baselines.manifest = "candidate.json"; + await writeFile(`${directory}/product.design.json`, JSON.stringify(profile, null, 2)); + const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); + await writeFile(`${directory}/captures/home--desktop--light--default.png`, png); + const result = await finalizeVisualCaptures({ + repo: directory, + profile: "product.design.json", + capturesDir: "captures", + artifactBaseUri: "artifact+github://example/product/actions/runs/42/design-captures/", + sourceCommit: "c".repeat(40), + out: "candidate.json", + }); + assert.equal(result.captureCount, 1); + const candidate = JSON.parse(await readFile(`${directory}/candidate.json`, "utf8")); + assert.equal(candidate.sourceCommit, "c".repeat(40)); + assert.match(candidate.captures[0].artifact.uri, /^artifact\+github:/); + assert.throws(() => validateVisualBaselineManifest({ ...candidate, sourceCommit: null }), /captured proposed baseline/); +}); + +test("structured UI review binds model judgment and cost to exact evidence", async () => { + const review = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/ui-review.json`, "utf8")); + assert.equal(validateUiReviewArtifact(review), review); + const incomplete = structuredClone(review); + delete incomplete.cost; + assert.throws(() => validateUiReviewArtifact(incomplete), /cost is required/); + const unknownCost = structuredClone(review); + unknownCost.cost = { telemetry: "unknown", currency: null, amount: null, inputTokens: null, outputTokens: null }; + assert.throws(() => validateUiReviewArtifact(unknownCost), /requires blocked verdict/); + unknownCost.verdict = "blocked"; + unknownCost.blockers = ["model cost telemetry unavailable"]; + assert.equal(validateUiReviewArtifact(unknownCost), unknownCost); +}); + +test("design memory rejects unsupported fields and ephemeral artifacts", async () => { + const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); + profile.unexpected = true; + assert.throws(() => validateProductDesignProfile(profile), /unsupported properties/); + + const baselines = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/baselines.json`, "utf8")); + baselines.captures[0].artifact.uri = "file:///tmp/screenshot.png"; + assert.throws(() => validateVisualBaselineManifest(baselines), /ephemeral file URI/); +}); + +test("design memory blocks when a canonical source drifts", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "tabellio-design-memory-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const sourceRoot = `${root}/examples/tabellio-design-memory`; + await Promise.all([ + writeFile(`${directory}/tokens.json`, await readFile(`${sourceRoot}/tokens.json`)), + writeFile(`${directory}/components.json`, await readFile(`${sourceRoot}/components.json`)), + writeFile(`${directory}/decision.md`, await readFile(`${sourceRoot}/decision.md`)), + ]); + const profile = JSON.parse(await readFile(`${sourceRoot}/product.design.json`, "utf8")); + profile.sources.tokens[0].path = "tokens.json"; + profile.sources.components[0].path = "components.json"; + profile.sources.decisions[0] = "decision.md"; + profile.baselines.manifest = "baselines.json"; + await writeFile(`${directory}/product.design.json`, JSON.stringify(profile, null, 2)); + const baselines = JSON.parse(await readFile(`${sourceRoot}/baselines.json`, "utf8")); + baselines.profilePath = "product.design.json"; + const crypto = await import("node:crypto"); + baselines.profileDigest = crypto.createHash("sha256").update(await readFile(`${directory}/product.design.json`)).digest("hex"); + await writeFile(`${directory}/baselines.json`, JSON.stringify(baselines, null, 2)); + await writeFile(`${directory}/tokens.json`, "{}\n"); + const result = await inspectDesignMemory({ repo: directory, profile: "product.design.json" }); + assert.equal(result.status, "blocked"); + assert.match(result.blockers.join("\n"), /source digest mismatch/); +}); + +test("design memory rejects duplicate surface matrix dimensions", async () => { + const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); + profile.policy.surfaces[0].viewports.push(profile.policy.surfaces[0].viewports[0]); + assert.throws(() => validateProductDesignProfile(profile), /viewports must contain unique entries/); +}); diff --git a/tests/validation-adapter.test.mjs b/tests/validation-adapter.test.mjs new file mode 100644 index 0000000..39502ae --- /dev/null +++ b/tests/validation-adapter.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); + +test("repository adapter emits passed, failed, blocked, and extracted evidence", async (t) => { + const root = await mkdtemp(join(tmpdir(), "tabellio-adapter-")); + t.after(() => rm(root, { recursive: true, force: true })); + const config = { + schemaVersion: "tabellio-adapter/v0.1", + profiles: { + passed: profile([[process.execPath, "-e", "console.log('score=0.95')"]], [ + { name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }, + { name: "score", unit: "ratio", pattern: "score=([0-9.]+)" }, + ]), + failed: profile([[process.execPath, "-e", "process.exit(3)"]]), + blocked: profile([["missing-tabellio-test-command"]]), + }, + }; + await writeFile(join(root, "validators.json"), JSON.stringify(config)); + + const passed = await runAdapter(root, "passed", "passed-validator", "passed.json"); + assert.equal(passed.status, "passed"); + assert.equal(passed.metrics.find((metric) => metric.name === "score").value, 0.95); + + const failed = await runAdapter(root, "failed", "failed-validator", "failed.json"); + assert.equal(failed.status, "failed"); + assert.equal(failed.metrics[0].value, 0); + + const blocked = await runAdapter(root, "blocked", "blocked-validator", "blocked.json"); + assert.equal(blocked.status, "blocked"); + assert.match(blocked.summary, /command unavailable/); +}); + +async function runAdapter(root, profileName, validatorId, out) { + await execFileAsync(process.execPath, [ + new URL("../scripts/tabellio-validator.mjs", import.meta.url).pathname, + "--config", "validators.json", + "--profile", profileName, + "--validator-id", validatorId, + "--out", out, + ], { cwd: root }); + return JSON.parse(await readFile(join(root, out), "utf8")); +} + +function profile(commands, metrics = [{ name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }]) { + return { + commands, + metrics, + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + summary: "Deterministic repository adapter completed.", + }; +} diff --git a/tests/validation-cli.test.mjs b/tests/validation-cli.test.mjs new file mode 100644 index 0000000..3fc62d3 --- /dev/null +++ b/tests/validation-cli.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { rm, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import test from "node:test"; + +import { runGit } from "../scripts/lib/git-process.mjs"; +import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; + +const execFileAsync = promisify(execFile); + +test("validation gate exits zero only for a passed exact-head result", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(0)"]), "passing gate"); + const passed = await runGate(fixture.seed, "refs/tabellio/cli-pass"); + assert.equal(passed.exitCode, 0); + assert.equal(passed.output.result.status, "passed"); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(3)"]), "failing gate"); + const failed = await runGate(fixture.seed, "refs/tabellio/cli-fail"); + assert.equal(failed.exitCode, 1); + assert.equal(failed.output.result.status, "failed"); +}); + +async function commitManifest(repo, value, message) { + await writeFile(`${repo}/tabellio.validation.json`, `${JSON.stringify(value, null, 2)}\n`); + await runGit({ args: ["add", "tabellio.validation.json"], cwd: repo }); + await runGit({ args: ["commit", "-m", message], cwd: repo, env: identityEnv() }); +} + +async function runGate(repo, ledgerRef) { + const args = [ + "scripts/tabellio-validate.mjs", + "gate", + "--repo", repo, + "--repo-id", "example/repository", + "--base", "main", + "--commit", "HEAD", + "--manifest", "tabellio.validation.json", + "--ledger-ref", ledgerRef, + ]; + try { + const result = await execFileAsync(process.execPath, args, { cwd: new URL("..", import.meta.url), encoding: "utf8" }); + return { exitCode: 0, output: JSON.parse(result.stdout) }; + } catch (error) { + return { exitCode: error.code, output: JSON.parse(error.stdout) }; + } +} + +function manifest(argv) { + return { + schemaVersion: "tabellio-validation/v0.1", + id: "cli-gate", + failFast: true, + requireEntireCheckpoint: false, + commands: [ + { + id: "gate-command", + argv, + cwd: ".", + timeoutMs: 30_000, + required: true, + }, + ], + }; +} diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index dfdc17c..5bc4802 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -10,11 +10,22 @@ import { ValidationRunner, validateValidationManifest, validateValidationResult, + validateValidatorEvidence, } from "../scripts/lib/validation-runner.mjs"; import { NativeGitStore } from "../scripts/providers/native-git-store.mjs"; import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; test("validation runner executes exact committed manifests and stores bounded results", async (t) => { + const previousArtifactBaseUri = process.env.TABELLIO_ARTIFACT_BASE_URI; + const previousTestSecret = process.env.TABELLIO_TEST_SECRET; + process.env.TABELLIO_ARTIFACT_BASE_URI = "artifact+test://validation/run/"; + process.env.TABELLIO_TEST_SECRET = "must-not-cross-boundary"; + t.after(() => { + if (previousArtifactBaseUri === undefined) delete process.env.TABELLIO_ARTIFACT_BASE_URI; + else process.env.TABELLIO_ARTIFACT_BASE_URI = previousArtifactBaseUri; + if (previousTestSecret === undefined) delete process.env.TABELLIO_TEST_SECRET; + else process.env.TABELLIO_TEST_SECRET = previousTestSecret; + }); const fixture = await createFixture(); t.after(() => rm(fixture.root, { recursive: true, force: true })); await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); @@ -22,6 +33,9 @@ test("validation runner executes exact committed manifests and stores bounded re await writeFile(manifestPath, JSON.stringify(manifest([ command("tests", [process.execPath, "-e", 'process.stdout.write("x".repeat(20000))']), command("isolated-home", [process.execPath, "-e", 'if (!process.env.HOME.includes("validation-workspaces")) process.exit(2)']), + command("artifact-uri", [process.execPath, "-e", 'if (process.env.TABELLIO_ARTIFACT_BASE_URI !== "artifact+test://validation/run/") process.exit(2)']), + command("secret-isolated", [process.execPath, "-e", 'if (process.env.TABELLIO_TEST_SECRET !== undefined) process.exit(2)']), + command("readonly-cache", [process.execPath, "-e", 'const fs=require("node:fs");const p=process.env.HOME+"/go/pkg/mod/example/.github";fs.mkdirSync(p,{recursive:true});fs.chmodSync(p,0o500);fs.chmodSync(process.env.HOME+"/go/pkg/mod/example",0o500)']), ]), null, 2)); await commit(fixture.seed, "Add passing validation", "validation-pass"); const passingHead = await head(fixture.seed); @@ -43,6 +57,9 @@ test("validation runner executes exact committed manifests and stores bounded re assert.equal(passed.result.commands[0].stdout.truncated, true); assert.equal(Buffer.byteLength(passed.result.commands[0].stdout.tail), 16 * 1024); assert.equal(passed.result.commands[1].status, "passed"); + assert.equal(passed.result.commands[2].status, "passed"); + assert.equal(passed.result.commands[3].status, "passed"); + assert.equal(passed.result.commands[4].status, "passed"); assert.equal(validateValidationResult(passed.result), passed.result); assert.deepEqual(await latestValidationResult(ledger, passingHead), passed.result); const otherRepository = await runner.run({ @@ -127,6 +144,209 @@ test("validation runner terminates timed-out commands and skips remaining fail-f assert(Date.now() - started < 3_000); }); +test("typed validators enforce semantic metrics and cost budgets with durable evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const semantic = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.9, unit: "ratio" }], + cost: availableCost(0.11, 5, 12), + }); + const operational = validatorEvidence("cost-guard", { + metrics: [{ name: "redis_commands", value: 100, unit: "commands" }], + cost: availableCost(0, 0, 0), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", semantic), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", operational), + ".tabellio/cost.json", + { + metricThresholds: [{ metric: "redis_commands", operator: "lte", value: 1_000 }], + maxCostUsd: 0.1, + requireCostTelemetry: true, + }, + ), + ], ["semantic", "operational"]), null, 2)); + await commit(fixture.seed, "Add product validation", "product-validation-pass"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + runnerId: "product-validator", + }); + + assert.equal(result.result.schemaVersion, "tabellio-validation-result/v0.3"); + assert.equal(result.result.status, "passed"); + assert.equal(result.result.acceptance.id, "PLANE-101"); + assert.deepEqual(result.result.acceptance.requiredValidatorTypes, ["semantic", "operational"]); + assert.deepEqual(result.result.validators.map(({ id, type, status }) => ({ id, type, status })), [ + { id: "static-checks", type: "static", status: "passed" }, + { id: "semantic-eval", type: "semantic", status: "passed" }, + { id: "cost-guard", type: "operational", status: "passed" }, + ]); + assert.equal(result.result.validators[1].evidence.report.metrics[0].value, 0.9); + assert.equal(result.result.decision.totalCostUsd, 0.11); + assert.equal(result.result.decision.costTelemetryComplete, true); + assert.equal(validateValidationResult(result.result), result.result); +}); + +test("typed validation distinguishes product failure from blocked evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const weakEvidence = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.2, unit: "ratio" }], + cost: availableCost(0.6, 1, 2), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", weakEvidence), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + ], ["semantic"]), null, 2)); + await commit(fixture.seed, "Add failing semantic validation", "product-validation-fail"); + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const runner = new ValidationRunner({ store, ledger }); + const failed = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(failed.result.status, "failed"); + assert.equal(failed.result.validators[0].status, "failed"); + assert.deepEqual(failed.result.validators[0].reasons, [ + "cost_budget_exceeded:0.5", + "threshold_not_met:strict_pass_rate:gte:0.8", + ]); + + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "cost-guard", + "operational", + [process.execPath, "-e", "process.exit(0)"], + ".tabellio/missing.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["operational"]), null, 2)); + await commit(fixture.seed, "Add blocked operational validation", "product-validation-blocked"); + const blocked = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(blocked.result.status, "blocked"); + assert.equal(blocked.result.validators[0].status, "blocked"); + assert.equal(blocked.result.decision.costTelemetryComplete, false); + assert(blocked.result.validators[0].reasons.includes("cost_telemetry_unavailable")); + assert(blocked.result.validators[0].reasons.some((reason) => reason.startsWith("evidence_invalid:"))); +}); + +test("typed validation preserves failure and blocked boundaries across validator kinds", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const schemaFailure = validatorEvidence("schema-contract", { + status: "failed", + summary: "Structured output included an unsupported field.", + }); + const missingMetric = validatorEvidence("semantic-eval"); + const unavailableCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: null, modelCalls: null, toolCalls: null }, + }); + const manifestValue = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(2)"], null), + typedValidator( + "schema-contract", + "schema", + evidenceCommand(".tabellio/schema.json", schemaFailure), + ".tabellio/schema.json", + ), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", missingMetric), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: null, + requireCostTelemetry: false, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", unavailableCost), + ".tabellio/cost.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["static", "schema", "semantic", "operational"]); + manifestValue.failFast = false; + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(manifestValue, null, 2)); + await commit(fixture.seed, "Add validator boundary matrix", "product-validation-boundaries"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + }); + + assert.equal(result.result.status, "blocked"); + assert.deepEqual(result.result.validators.map(({ id, status, reasons }) => ({ id, status, reasons })), [ + { id: "static-checks", status: "failed", reasons: ["command_failed"] }, + { id: "schema-contract", status: "failed", reasons: ["evidence_reported_failed"] }, + { id: "semantic-eval", status: "blocked", reasons: ["metric_missing:strict_pass_rate"] }, + { id: "cost-guard", status: "blocked", reasons: ["cost_telemetry_unavailable"] }, + ]); + assert.equal(result.result.decision.costTelemetryComplete, false); +}); + +test("typed validation contracts reject incomplete product gates and invalid evidence", () => { + const incomplete = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(incomplete), /requires a required semantic validator/); + + const missingEvidence = productManifest([ + typedValidator("semantic-eval", "semantic", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(missingEvidence), /evidence is required/); + + const invalidCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: 0, modelCalls: null, toolCalls: null }, + }); + assert.throws(() => validateValidatorEvidence(invalidCost), /cost values must be null/); + + const ephemeralArtifact = validatorEvidence("visual-regression", { + artifacts: [{ + name: "desktop-diff", + uri: "file:///tmp/desktop-diff.png", + digest: "d".repeat(64), + mediaType: "image/png", + bytes: 100, + }], + }); + assert.throws(() => validateValidatorEvidence(ephemeralArtifact), /must not use an ephemeral file URI/); +}); + function manifest(commands) { return { schemaVersion: "tabellio-validation/v0.1", @@ -141,6 +361,64 @@ function command(id, argv, cwd = ".", timeoutMs = 10_000) { return { id, argv, cwd, timeoutMs, required: true }; } +function productManifest(validators, requiredValidatorTypes) { + return { + schemaVersion: "tabellio-validation/v0.2", + id: "product-suite", + failFast: true, + requireEntireCheckpoint: true, + acceptance: { + id: "PLANE-101", + source: "plane", + risk: "high", + outcomes: ["The user receives a correct product result."], + invariants: ["No live mutation occurs during validation."], + forbiddenOutcomes: ["Missing cost telemetry is treated as zero cost."], + requiredValidatorTypes, + }, + validators, + }; +} + +function typedValidator(id, type, argv, evidencePath, policy = null) { + return { + id, + type, + argv, + cwd: ".", + timeoutMs: 10_000, + required: true, + evidence: evidencePath === null ? null : { path: evidencePath }, + policy: policy ?? { metricThresholds: [], maxCostUsd: null, requireCostTelemetry: false }, + }; +} + +function validatorEvidence(validatorId, overrides = {}) { + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status: "passed", + summary: "Validator completed.", + metrics: [], + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + artifacts: [], + ...overrides, + }; +} + +function availableCost(usd, modelCalls, toolCalls) { + return { telemetry: "available", usd, modelCalls, toolCalls }; +} + +function evidenceCommand(path, evidence) { + const source = [ + "const fs = require('node:fs');", + `fs.mkdirSync(${JSON.stringify(path.split("/").slice(0, -1).join("/"))}, { recursive: true });`, + `fs.writeFileSync(${JSON.stringify(path)}, ${JSON.stringify(JSON.stringify(evidence))});`, + ].join(""); + return [process.execPath, "-e", source]; +} + async function commit(cwd, message, checkpoint) { await runGit({ args: ["add", "tabellio.validation.json"], cwd }); await runGit({