From 8d9f20e05d2774937faf56391bcda596b15ea26a Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 13:13:10 -0700 Subject: [PATCH 01/13] Add product validation control plane --- CHANGELOG.md | 23 + README.md | 11 +- docs/codex-review.md | 2 + docs/operations-hardening.md | 3 +- docs/product-validation.md | 82 +++ docs/releases/v0.4.0.md | 12 + docs/validation-runner.md | 10 +- docs/workflow-model.md | 8 +- .../minimal-validator-evidence.json | 20 + .../tabellio-validation/product-manifest.json | 78 +++ package.json | 4 +- schemas/validation-manifest.v0.1.schema.json | 31 + schemas/validation-manifest.v0.2.schema.json | 101 +++ schemas/validation-result.v0.3.schema.json | 218 +++++++ schemas/validator-evidence.v0.1.schema.json | 78 +++ scripts/check-tabellio-validation.mjs | 19 +- scripts/lib/validation-runner.mjs | 600 +++++++++++++++++- scripts/tabellio-validate.mjs | 7 +- tests/validation-cli.test.mjs | 69 ++ tests/validation-runner.test.mjs | 262 ++++++++ 20 files changed, 1586 insertions(+), 52 deletions(-) create mode 100644 docs/product-validation.md create mode 100644 docs/releases/v0.4.0.md create mode 100644 examples/tabellio-validation/minimal-validator-evidence.json create mode 100644 examples/tabellio-validation/product-manifest.json create mode 100644 schemas/validation-manifest.v0.1.schema.json create mode 100644 schemas/validation-manifest.v0.2.schema.json create mode 100644 schemas/validation-result.v0.3.schema.json create mode 100644 schemas/validator-evidence.v0.1.schema.json create mode 100644 tests/validation-cli.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe8308..9dd4035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to Tabellio are recorded here. ## Unreleased +## 0.4.0 - 2026-07-16 + +### Added + +- Product-validity manifests that bind acceptance outcomes, invariants, forbidden outcomes, risk, and required validator types to an exact candidate commit. +- Typed schema, semantic, workflow, visual, operational, and security evidence with bounded metrics, artifact digests, and cost telemetry. +- Fail-closed `passed`, `failed`, and `blocked` decisions with threshold and cost-budget enforcement. +- `tabellio-validate gate` for CI enforcement without external JSON parsing. + +### Changed + +- Product-changing work now requires evidence beyond command exit status when the manifest declares typed validators. +- Missing required evidence, missing metrics, command errors, timeouts, and unknown required cost telemetry block readiness. +- Existing v0.1 manifests and v0.1/v0.2 result readers remain supported during migration. + +### Release Gates + +- `tabellio-preflight --profile release` +- `npm run check` +- Fallow whole-repository and changed-code scans +- `npm pack --dry-run --json` +- Exact merged-head Tabellio validation + ## 0.3.0 - 2026-07-15 ### Added diff --git a/README.md b/README.md index 2ff8f8c..dfc6c74 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Tabellio attaches a structured evidence packet to a pull request. The packet is | Git | Repository, base branch, head branch, commit SHA, and PR metadata | | Runtime | Human, CI, or agent runtime that produced the change | | Diff | Changed files | -| Validation | Commands run and check results | +| Validation | Exact-head commands plus structural, semantic, workflow, visual, operational, and security results | | Approvals | Required, granted, denied, or skipped approvals | | Side effects | Deployment, migration, infra, billing, secret, provider, and destructive-action policy | | Artifacts | Evidence files generated by the run | @@ -58,7 +58,7 @@ AI-assisted pull requests should not depend on reviewer trust alone. Tabellio gi | Layer | Tooling | Role | | --- | --- | --- | | Runtime | [Node.js 20+](https://nodejs.org/) | Runs the local writer and validators | -| Validation | `tabellio-validate` on any trusted worker | Runs an exact committed command manifest and stores results on a Git ref | +| Validation | `tabellio-validate` on any trusted worker | Runs exact committed command or typed product validators and stores results on a Git ref | | Evidence contract | [JSON Schema](https://json-schema.org/) | Validates the evidence envelope and external-action policy | | Code storage | [GitHub](https://github.com/) | Stores code refs and tags; provides a thin pull-request shell | | Stacked review | [git-spice](https://abhinav.github.io/git-spice/) | Stack engine for small dependent GitHub pull requests | @@ -103,6 +103,12 @@ npm run tabellio:platform:check node scripts/tabellio-validate.mjs run --repo . --commit HEAD --manifest tabellio.validation.json ``` +Use `gate` in CI. It persists the same exact-head result but exits non-zero unless the final decision is `passed`: + +```bash +tabellio-validate gate --repo . --repo-id owner/repository --base main --commit HEAD --manifest tabellio.validation.json +``` + Keep `origin` limited to ordinary code branches and tags. Configure a separate private GitHub repository under another remote name before publishing control refs. The transport refuses to target `origin`. Before agent or release work, run: @@ -162,6 +168,7 @@ node scripts/tabellio-run.mjs promote --run-id run-42 --repo . ``` See [Agent run lifecycle](docs/agent-run-lifecycle.md) for state and failure behavior. +See [Product validation](docs/product-validation.md) for acceptance contracts, typed evidence, cost gates, and repository adapters. Package scripts: diff --git a/docs/codex-review.md b/docs/codex-review.md index 7edc9af..96272fc 100644 --- a/docs/codex-review.md +++ b/docs/codex-review.md @@ -36,3 +36,5 @@ Codex review is not: - deployment approval - substitute for human ownership - proof that agent output is secure + +When the repository commits a product-validation manifest, review begins only after the exact pull-request head has a `passed` v0.3 decision. A `blocked` decision means the evidence is incomplete or untrustworthy; it is not a product pass and should not be converted into reviewer guesswork. diff --git a/docs/operations-hardening.md b/docs/operations-hardening.md index 1a63af2..d46d1aa 100644 --- a/docs/operations-hardening.md +++ b/docs/operations-hardening.md @@ -11,9 +11,10 @@ Tabellio keeps code and control state in standard Git objects, but production sa ## Bounded Work -- Validation manifests allow at most 50 commands, 100 arguments per command, and one-hour command timeouts. +- Validation manifests allow at most 50 commands or validators, 100 arguments per entry, and one-hour command timeouts. - Timed-out commands receive `SIGTERM`, then `SIGKILL` after one second. Fail-fast suites mark remaining commands skipped. - Full stdout and stderr are hashed; only the newest 16 KiB of each stream is retained. +- Typed evidence files are limited to 1 MiB, 100 metrics, and 50 immutable artifact references. Required missing cost telemetry blocks validation instead of being treated as zero spend. - Agent reviews allow at most 1,000 findings. Review cycles bound feedback, fixes, check statuses, titles, bodies, summaries, event details, and retained event history. - Remote control-ref reads and atomic pushes use a 15-minute timeout. Local atomic ref updates use a 30-second timeout. diff --git a/docs/product-validation.md b/docs/product-validation.md new file mode 100644 index 0000000..bff89db --- /dev/null +++ b/docs/product-validation.md @@ -0,0 +1,82 @@ +# Product Validation + +Tabellio product validation answers a stronger question than “did tests exit zero?”: did the exact committed candidate satisfy its declared user outcomes while staying inside semantic, visual, operational, cost, and safety boundaries? + +## Contracts + +`tabellio-validation/v0.2` binds three layers in one committed manifest: + +1. `acceptance` records the source request, risk, required outcomes, invariants, forbidden outcomes, and required validator types. +2. `validators` run shell-free argument arrays in the exact-head detached worktree. +3. `policy` applies metric thresholds and cost requirements to a bounded evidence report. + +Validator types are `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security`. Static validators can use command exit status alone. Every other type must emit `tabellio-validator-evidence/v0.1` at its declared relative path. + +## Evidence + +An evidence report contains: + +- `status`: the adapter’s passed, failed, or blocked observation; +- numeric metrics with explicit units; +- cost telemetry state, USD cost, model calls, and tool calls; +- immutable artifact metadata for screenshots, traces, reports, or source snapshots. + +The runner validates the evidence shape and validator identity, embeds the bounded report in the durable result, and evaluates policy itself. Adapter-reported `passed` cannot override a failed metric threshold or exceeded cost cap. + +## Decision Semantics + +| Decision | Meaning | +| --- | --- | +| `passed` | Every required validator completed with trustworthy evidence inside all thresholds. | +| `failed` | Trustworthy evidence proves a required product or budget threshold was missed. | +| `blocked` | Required proof is missing or untrustworthy because of invalid evidence, missing metrics or cost telemetry, command errors, or timeouts. | + +Fail closed. Never translate `blocked` into `passed`, and never treat missing cost telemetry as zero. + +## Repository Adapters + +Keep product knowledge in each repository. The adapter runs the existing eval, browser, design, cost, or security harness and writes the common evidence contract. + +- AI systems: stable golden cases in pull requests; larger live suites on schedules or release gates. +- Web applications: critical journeys, desktop/mobile screenshot comparisons, loaded-font assertions, overflow, and accessibility metrics. +- Operational systems: projected and observed provider usage, worker counts, polling frequency, rate limits, latency, and spend. +- Structured outputs: strict schema validation plus semantic oracles that catch plausible but wrong values. + +Use fixtures and isolated namespaces in pull requests. Live external mutation remains separately approval-gated. + +## Review Integration + +Review sync reads only the newest validation result stored under the exact pull-request head. `passed` becomes a successful local check; `failed` or `blocked` prevents merge readiness. A new head requires new validation. + +Run contract checks with: + +```bash +node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json +node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json +``` + +Run the exact-head suite with: + +```bash +node scripts/tabellio-validate.mjs run \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +Use the fail-closed command in CI: + +```bash +tabellio-validate gate \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +`gate` still records the result. It exits non-zero when the decision is `failed` or `blocked`. + +Artifact upload and path-based automatic validator selection remain scheduler or repository-policy responsibilities. The first contract keeps evaluation portable without creating a second CI or deployment authority. diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md new file mode 100644 index 0000000..b510b73 --- /dev/null +++ b/docs/releases/v0.4.0.md @@ -0,0 +1,12 @@ +Tabellio v0.4.0 adds product-validity and cost-aware validation. + +## Highlights + +- Acceptance contracts declare user outcomes, invariants, forbidden outcomes, risk, and required evidence types. +- Typed validators cover schema, semantics, workflows, visuals, operations, cost, and security. +- Evidence carries numeric metrics, explicit units, cost telemetry, tool and model call counts, and immutable artifact metadata. +- Policy evaluation belongs to the runner. Adapter-reported success cannot override a missed threshold or cost cap. +- Missing or untrustworthy required proof is `blocked`, never an implied pass. +- `tabellio-validate gate` records exact-head results and exits non-zero unless the decision is `passed`. + +Existing v0.1 validation manifests remain readable. Live provider calls, deployments, infrastructure changes, and publication remain separately approval-gated. diff --git a/docs/validation-runner.md b/docs/validation-runner.md index a814a40..b042f22 100644 --- a/docs/validation-runner.md +++ b/docs/validation-runner.md @@ -24,6 +24,12 @@ Tabellio validation replaces the GitHub Actions runtime dependency with a small Commands are executable-plus-argument arrays. Tabellio never invokes a shell, interpolates command strings, or reads a newer working-tree manifest while validating an older commit. +Manifest v0.1 remains supported. Product-facing repositories can use `tabellio-validation/v0.2`, which adds an immutable acceptance contract and typed `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security` validators. Every required validator type must have a required validator. Non-static validators must write a bounded `tabellio-validator-evidence/v0.1` report. + +Validator policy can require numeric metric thresholds, cost telemetry, and a maximum validation cost. Missing or invalid evidence, missing required metrics, unavailable required cost telemetry, command errors, and timeouts produce `blocked`. Valid evidence that misses a product threshold or cost budget produces `failed`. Only complete required evidence inside every boundary produces `passed`. + +See [Product validation](product-validation.md) and [the example manifest](../examples/tabellio-validation/product-manifest.json). + ## Run ```bash @@ -47,7 +53,7 @@ The runner: 7. Removes the worktree even after failure. 8. Writes an integrity-protected result to `refs/tabellio/validations` with compare-and-swap retries. -New results use `tabellio-validation-result/v0.2`. They require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 results. +Command-manifest results use `tabellio-validation-result/v0.2`. Product-validation results use v0.3 and embed the acceptance digest, typed validator results, bounded evidence reports, total observed validation cost, and final policy decision. Both require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 and v0.2 results. Read the newest result for a commit: @@ -64,3 +70,5 @@ Publishing `refs/tabellio/validations` to a remote is a separate approved Git wr ## Security Boundary Detached worktrees and isolated home directories protect host source and common credential paths; they are not a hostile-code sandbox. Validation commands can execute arbitrary repository code and use the network. Run untrusted contributions in a disposable VM, container, or sandbox with scoped credentials and network policy. Never place secrets in manifest arguments or print them: argv and bounded output tails are retained as durable evidence. + +Typed validation does not authorize live writes. Repository adapters should use fixtures, isolated namespaces, read-only credentials, and explicit cost caps. Artifact metadata must point to durable content-addressed storage when a reviewer needs the screenshot, trace, or report after the temporary worktree is removed. diff --git a/docs/workflow-model.md b/docs/workflow-model.md index 19bef23..dfd6dc5 100644 --- a/docs/workflow-model.md +++ b/docs/workflow-model.md @@ -12,6 +12,8 @@ Tabellio turns an agentic coding run into a GitHub-bound context packet and, whe | Workspace | Isolated Git worktree assigned to one agent run | | Context packet | Integrity-protected binding between task, actor, exact commits, changed files, checkpoints, and merge preview | | Evidence envelope | JSON record of Git state, changed files, commands, checks, approvals, side-effect policy, and artifacts | +| Acceptance contract | Committed outcomes, invariants, forbidden outcomes, risk, and required validator types | +| Validator evidence | Bounded metrics, cost telemetry, and immutable artifact references emitted by one product validator | | External action policy | Default-deny policy for deploys, migrations, infra, DNS, billing, secrets, provider reads, and destructive actions | | Pull request | Human review surface for the diff plus evidence summary | | Stack | Ordered PR chain for larger changes | @@ -34,7 +36,7 @@ task -> resolve immutable base commit -> isolated worktree and agent branch -> small commits and mandatory Entire checkpoint metadata - -> deterministic checks + -> deterministic and product-validity checks -> context packet -> read-only merge preview -> evidence envelope @@ -60,7 +62,7 @@ Each PR should expose: The context packet is usable without a pull request. GitHub is the review and code-distribution surface; Tabellio's evidence remains independently verifiable. -Evidence is not a claim that the work is correct. Evidence is the record reviewers inspect before trusting the work. +Raw evidence is not automatically a correctness claim. A typed validation decision proves that the exact committed evidence met the committed thresholds. Human review still owns product judgment outside the declared contract. ## Stack Discipline @@ -112,6 +114,6 @@ Future versions can add: - SLSA provenance export - in-toto link metadata - OpenTelemetry spans -- model/tool eval suites +- path-based validator policy selection - automated git-spice submission and merge orchestration for GitHub - GitHub merge queue metadata diff --git a/examples/tabellio-validation/minimal-validator-evidence.json b/examples/tabellio-validation/minimal-validator-evidence.json new file mode 100644 index 0000000..882a907 --- /dev/null +++ b/examples/tabellio-validation/minimal-validator-evidence.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": "tabellio-validator-evidence/v0.1", + "validatorId": "semantic-golden-set", + "status": "passed", + "summary": "Eight of ten required scenarios passed the strict semantic rubric.", + "metrics": [ + { + "name": "strict_pass_rate", + "value": 0.8, + "unit": "ratio" + } + ], + "cost": { + "telemetry": "available", + "usd": 0.18, + "modelCalls": 10, + "toolCalls": 24 + }, + "artifacts": [] +} diff --git a/examples/tabellio-validation/product-manifest.json b/examples/tabellio-validation/product-manifest.json new file mode 100644 index 0000000..4a7761a --- /dev/null +++ b/examples/tabellio-validation/product-manifest.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": "tabellio-validation/v0.2", + "id": "product-validity", + "failFast": true, + "requireEntireCheckpoint": true, + "acceptance": { + "id": "PLANE-101", + "source": "plane", + "risk": "high", + "outcomes": [ + "The user receives a correct result for the requested workflow." + ], + "invariants": [ + "Validation cannot mutate production resources." + ], + "forbiddenOutcomes": [ + "Unknown cost telemetry is treated as zero cost." + ], + "requiredValidatorTypes": [ + "semantic", + "operational" + ] + }, + "validators": [ + { + "id": "semantic-golden-set", + "type": "semantic", + "argv": [ + "npm", + "run", + "eval:semantic:ci" + ], + "cwd": ".", + "timeoutMs": 600000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/semantic-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "strict_pass_rate", + "operator": "gte", + "value": 0.8 + } + ], + "maxCostUsd": 1, + "requireCostTelemetry": true + } + }, + { + "id": "runtime-cost-guard", + "type": "operational", + "argv": [ + "npm", + "run", + "cost:check" + ], + "cwd": ".", + "timeoutMs": 120000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/cost-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "projected_monthly_cost_usd", + "operator": "lte", + "value": 20 + } + ], + "maxCostUsd": 0.25, + "requireCostTelemetry": true + } + } + ] +} diff --git a/package.json b/package.json index e4698e2..32ff583 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@intelip/tabellio", - "version": "0.3.0", + "version": "0.4.0", "description": "Tabellio: GitHub-native context and evidence for agentic development.", "type": "module", "license": "Apache-2.0", @@ -60,7 +60,7 @@ "tabellio:release": "node scripts/tabellio-release.mjs", "tabellio:release:example:check": "node scripts/check-tabellio-release.mjs --intent examples/tabellio-release/minimal-intent.json", "tabellio:platform:check": "node scripts/check-tabellio-platform.mjs", - "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", + "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json && node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", "tabellio:ledger": "node scripts/tabellio-ledger.mjs", "tabellio:ledger:check": "node scripts/check-tabellio-ledger.mjs --ledger tabellio-ledger.json", "tabellio:ledger:example:check": "node scripts/check-tabellio-ledger.mjs --ledger examples/tabellio-ledger/minimal-ledger.json", diff --git a/schemas/validation-manifest.v0.1.schema.json b/schemas/validation-manifest.v0.1.schema.json new file mode 100644 index 0000000..b250820 --- /dev/null +++ b/schemas/validation-manifest.v0.1.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.1", + "title": "Tabellio Validation Manifest v0.1", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.1" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "commands": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "type": "object", + "required": ["id", "argv", "cwd", "timeoutMs", "required"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" } + } + } + } + } +} diff --git a/schemas/validation-manifest.v0.2.schema.json b/schemas/validation-manifest.v0.2.schema.json new file mode 100644 index 0000000..2ce8d2d --- /dev/null +++ b/schemas/validation-manifest.v0.2.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.2", + "title": "Tabellio Product Validation Manifest", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.2" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "acceptance": { "$ref": "#/$defs/acceptance" }, + "validators": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { "$ref": "#/$defs/validator" } + } + }, + "$defs": { + "validatorType": { + "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] + }, + "boundedText": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "acceptance": { + "type": "object", + "required": ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "outcomes": { "type": "array", "minItems": 1, "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "invariants": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "forbiddenOutcomes": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validator": { + "type": "object", + "required": ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { "path": { "type": "string", "minLength": 1 } } + } + ] + }, + "policy": { "$ref": "#/$defs/policy" } + }, + "allOf": [ + { + "if": { "properties": { "type": { "not": { "const": "static" } } }, "required": ["type"] }, + "then": { "properties": { "evidence": { "type": "object" } } } + } + ] + }, + "policy": { + "type": "object", + "required": ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], + "additionalProperties": false, + "properties": { + "metricThresholds": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["metric", "operator", "value"], + "additionalProperties": false, + "properties": { + "metric": { "type": "string", "minLength": 1 }, + "operator": { "enum": ["gte", "lte", "eq"] }, + "value": { "type": "number" } + } + } + }, + "maxCostUsd": { "type": ["number", "null"], "minimum": 0 }, + "requireCostTelemetry": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "maxCostUsd": { "type": "number" } }, "required": ["maxCostUsd"] }, + "then": { "properties": { "requireCostTelemetry": { "const": true } } } + } + ] + } + } +} diff --git a/schemas/validation-result.v0.3.schema.json b/schemas/validation-result.v0.3.schema.json new file mode 100644 index 0000000..83756f1 --- /dev/null +++ b/schemas/validation-result.v0.3.schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-result:v0.3", + "title": "Tabellio Product Validation Result", + "type": "object", + "required": ["schemaVersion", "runId", "repository", "revision", "checkpointRevision", "suite", "runner", "status", "checkpoints", "commands", "acceptance", "validators", "decision", "startedAt", "completedAt", "integrity"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation-result/v0.3" }, + "runId": { "type": "string", "minLength": 1 }, + "repository": { + "type": "object", + "required": ["id"], + "additionalProperties": false, + "properties": { "id": { "type": "string", "minLength": 1 } } + }, + "revision": { "$ref": "#/$defs/revision" }, + "checkpointRevision": { "$ref": "#/$defs/revision" }, + "suite": { + "type": "object", + "required": ["id", "manifestPath", "manifestDigest"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "manifestPath": { "type": "string", "minLength": 1 }, + "manifestDigest": { "$ref": "#/$defs/sha256" } + } + }, + "runner": { + "type": "object", + "required": ["id", "runtime"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "runtime": { "type": "string", "minLength": 1 } + } + }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "checkpoints": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, + "commands": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/commandResult" } }, + "acceptance": { "$ref": "#/$defs/acceptanceResult" }, + "validators": { "type": "array", "minItems": 1, "maxItems": 50, "items": { "$ref": "#/$defs/validatorResult" } }, + "decision": { "$ref": "#/$defs/decision" }, + "startedAt": { "type": "string", "format": "date-time" }, + "completedAt": { "type": "string", "format": "date-time" }, + "integrity": { + "type": "object", + "required": ["algorithm", "digest"], + "additionalProperties": false, + "properties": { + "algorithm": { "const": "sha256" }, + "digest": { "$ref": "#/$defs/sha256" } + } + } + }, + "$defs": { + "oid": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "validatorType": { "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] }, + "revision": { + "type": "object", + "required": ["baseCommit", "mergeBase", "headCommit"], + "additionalProperties": false, + "properties": { + "baseCommit": { "$ref": "#/$defs/oid" }, + "mergeBase": { "$ref": "#/$defs/oid" }, + "headCommit": { "$ref": "#/$defs/oid" } + } + }, + "output": { + "type": "object", + "required": ["bytes", "digest", "tail", "truncated"], + "additionalProperties": false, + "properties": { + "bytes": { "type": "integer", "minimum": 0 }, + "digest": { "$ref": "#/$defs/sha256" }, + "tail": { "type": "string" }, + "truncated": { "type": "boolean" } + } + }, + "commandResult": { + "type": "object", + "required": ["id", "argv", "cwd", "required", "status", "exitCode", "signal", "durationMs", "stdout", "stderr", "startedAt", "completedAt", "error"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "error", "timed_out", "skipped"] }, + "exitCode": { "type": ["integer", "null"] }, + "signal": { "type": ["string", "null"] }, + "durationMs": { "type": "integer", "minimum": 0 }, + "stdout": { "$ref": "#/$defs/output" }, + "stderr": { "$ref": "#/$defs/output" }, + "startedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "completedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "error": { "type": ["string", "null"] } + } + }, + "acceptanceResult": { + "type": "object", + "required": ["id", "source", "risk", "digest", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "digest": { "$ref": "#/$defs/sha256" }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validatorResult": { + "type": "object", + "required": ["id", "type", "required", "status", "evidence", "reasons"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "blocked", "skipped"] }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path", "digest", "report"], + "additionalProperties": false, + "properties": { + "path": { "type": "string", "minLength": 1 }, + "digest": { "$ref": "#/$defs/sha256" }, + "report": { "$ref": "#/$defs/evidenceReport" } + } + } + ] + }, + "reasons": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } } + } + }, + "evidenceReport": { + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { "type": "array", "maxItems": 100, "items": { "$ref": "#/$defs/metric" } }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/artifact" } } + } + }, + "metric": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + }, + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "$ref": "#/$defs/sha256" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + }, + "decision": { + "type": "object", + "required": ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], + "additionalProperties": false, + "properties": { + "status": { "enum": ["passed", "failed", "blocked"] }, + "reasons": { "type": "array", "maxItems": 6400, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } }, + "totalCostUsd": { "type": "number", "minimum": 0 }, + "costTelemetryComplete": { "type": "boolean" } + } + } + } +} diff --git a/schemas/validator-evidence.v0.1.schema.json b/schemas/validator-evidence.v0.1.schema.json new file mode 100644 index 0000000..14fec11 --- /dev/null +++ b/schemas/validator-evidence.v0.1.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validator-evidence:v0.1", + "title": "Tabellio Validator Evidence", + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + } + }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { + "type": "array", + "maxItems": 50, + "items": { "$ref": "#/$defs/artifact" } + } + }, + "$defs": { + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/scripts/check-tabellio-validation.mjs b/scripts/check-tabellio-validation.mjs index 228bc90..be3f1ae 100755 --- a/scripts/check-tabellio-validation.mjs +++ b/scripts/check-tabellio-validation.mjs @@ -3,19 +3,24 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; -import { validateValidationManifest, validateValidationResult } from "./lib/validation-runner.mjs"; +import { + validateValidationManifest, + validateValidationResult, + validateValidatorEvidence, +} from "./lib/validation-runner.mjs"; const type = process.argv[2]; const path = process.argv[3]; const blockers = []; let value = null; try { - if (!path || process.argv.length !== 4 || !["--manifest", "--result"].includes(type)) { - throw new Error("Usage: check-tabellio-validation (--manifest|--result) ."); + if (!path || process.argv.length !== 4 || !["--manifest", "--result", "--evidence"].includes(type)) { + throw new Error("Usage: check-tabellio-validation (--manifest|--result|--evidence) ."); } value = JSON.parse(await readFile(resolve(path), "utf8")); if (type === "--manifest") validateValidationManifest(value); - else validateValidationResult(value); + else if (type === "--result") validateValidationResult(value); + else validateValidatorEvidence(value); } catch (error) { blockers.push(error instanceof Error ? error.message : String(error)); } @@ -23,12 +28,14 @@ const result = { ok: blockers.length === 0, status: blockers.length === 0 ? "validation_contract_ready" : "blocked", checkedAt: new Date().toISOString(), - type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : null, + type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : type === "--evidence" ? "evidence" : null, path, summary: value ? { schemaVersion: value.schemaVersion, - id: value.id ?? value.runId ?? null, + id: value.id ?? value.runId ?? value.validatorId ?? null, commandCount: Array.isArray(value.commands) ? value.commands.length : null, + validatorCount: Array.isArray(value.validators) ? value.validators.length : null, + metricCount: Array.isArray(value.metrics) ? value.metrics.length : null, } : null, blockers, }; diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index 1986498..bd34b76 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -1,16 +1,21 @@ import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { mkdir, rm, stat } from "node:fs/promises"; +import { mkdir, readFile, rm, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; import { LedgerConflictError } from "./git-json-ledger.mjs"; import { runGit } from "./git-process.mjs"; import { digestObject } from "./stack-operation.mjs"; -const VALIDATION_MANIFEST_SCHEMA_VERSION = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V1 = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V2 = "tabellio-validation/v0.2"; const VALIDATION_RESULT_SCHEMA_VERSION_V1 = "tabellio-validation-result/v0.1"; const VALIDATION_RESULT_SCHEMA_VERSION_V2 = "tabellio-validation-result/v0.2"; +const VALIDATION_RESULT_SCHEMA_VERSION_V3 = "tabellio-validation-result/v0.3"; +const VALIDATOR_EVIDENCE_SCHEMA_VERSION = "tabellio-validator-evidence/v0.1"; +const VALIDATOR_TYPES = ["static", "schema", "semantic", "workflow", "visual", "operational", "security"]; const MAX_OUTPUT_TAIL_BYTES = 16 * 1024; +const MAX_EVIDENCE_BYTES = 1024 * 1024; export class ValidationRunner { constructor({ store, ledger }) { @@ -55,20 +60,14 @@ export class ValidationRunner { await mkdir(workspaceRoot, { recursive: true }); if (await stat(workspace).catch(() => null)) throw new Error(`Validation workspace already exists: ${workspace}.`); const startedAt = now.toISOString(); - const commands = []; + const definitions = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V1 + ? manifest.commands + : manifest.validators; + let execution = null; try { await runGit({ args: ["worktree", "add", "--detach", workspace, revision.headCommit], cwd: this.store.repoPath }); await mkdir(resolve(home, "tmp"), { recursive: true }); - let stopped = false; - for (const command of manifest.commands) { - if (stopped) { - commands.push(skippedCommand(command)); - continue; - } - const result = await runValidationCommand(command, workspace, home); - commands.push(result); - if (manifest.failFast && command.required && result.status !== "passed") stopped = true; - } + execution = await runValidationDefinitions({ manifest, definitions, workspace, home }); } finally { await runGit({ args: ["worktree", "remove", "--force", workspace], @@ -79,26 +78,20 @@ export class ValidationRunner { await rm(home, { recursive: true, force: true }); } const completedAt = new Date().toISOString(); - const requiredFailed = commands.some((command, index) => manifest.commands[index].required && command.status !== "passed"); - const result = { - schemaVersion: VALIDATION_RESULT_SCHEMA_VERSION_V2, + const result = buildValidationResult({ + manifest, + definitions, + execution, runId, - repository: { id: repositoryId }, + repositoryId, revision, checkpointRevision, - suite: { - id: manifest.id, - manifestPath, - manifestDigest: digestObject(manifest), - }, - runner: { id: runnerId, runtime: `node-${process.version}` }, - status: requiredFailed ? "failed" : "passed", + manifestPath, + runnerId, checkpoints, - commands, startedAt, completedAt, - integrity: { algorithm: "sha256", digest: "0".repeat(64) }, - }; + }); result.integrity.digest = validationResultDigest(result); validateValidationResult(result); const path = validationPath(revision.headCommit, runId); @@ -107,6 +100,86 @@ export class ValidationRunner { } } +async function runValidationDefinitions({ manifest, definitions, workspace, home }) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const commands = []; + const validators = []; + let stopped = false; + for (const definition of definitions) { + if (stopped) { + commands.push(skippedCommand(definition)); + if (typed) validators.push(skippedValidator(definition)); + continue; + } + const commandResult = await runValidationCommand(definition, workspace, home); + commands.push(commandResult); + const gateStatus = typed + ? await appendValidatorResult(validators, definition, commandResult, workspace) + : commandResult.status; + stopped = manifest.failFast && definition.required && gateStatus !== "passed"; + } + return { commands, validators }; +} + +async function appendValidatorResult(validators, definition, commandResult, workspace) { + const result = await evaluateValidator(definition, commandResult, workspace); + validators.push(result); + return result.status; +} + +function buildValidationResult({ + manifest, + definitions, + execution, + runId, + repositoryId, + revision, + checkpointRevision, + manifestPath, + runnerId, + checkpoints, + startedAt, + completedAt, +}) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const decision = typed ? validationDecision(execution.validators) : null; + const requiredFailed = execution.commands.some((command, index) => definitions[index].required && command.status !== "passed"); + const result = { + schemaVersion: typed ? VALIDATION_RESULT_SCHEMA_VERSION_V3 : VALIDATION_RESULT_SCHEMA_VERSION_V2, + runId, + repository: { id: repositoryId }, + revision, + checkpointRevision, + suite: { + id: manifest.id, + manifestPath, + manifestDigest: digestObject(manifest), + }, + runner: { id: runnerId, runtime: `node-${process.version}` }, + status: decision?.status ?? (requiredFailed ? "failed" : "passed"), + checkpoints, + commands: execution.commands, + startedAt, + completedAt, + integrity: { algorithm: "sha256", digest: "0".repeat(64) }, + }; + if (!typed) return result; + result.acceptance = acceptanceResult(manifest.acceptance); + result.validators = execution.validators; + result.decision = decision; + return result; +} + +function acceptanceResult(acceptance) { + return { + id: acceptance.id, + source: acceptance.source, + risk: acceptance.risk, + digest: digestObject(acceptance), + requiredValidatorTypes: [...acceptance.requiredValidatorTypes], + }; +} + export async function latestValidationResult(ledger, commit, repositoryId = null) { oid(commit, "commit"); if (repositoryId !== null) requiredString(repositoryId, "repositoryId"); @@ -126,11 +199,25 @@ export async function latestValidationResult(ledger, commit, repositoryId = null export function validateValidationManifest(value) { object(value, "validation manifest"); - exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); - equals(value.schemaVersion, VALIDATION_MANIFEST_SCHEMA_VERSION, "validation manifest.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_MANIFEST_SCHEMA_VERSION_V1, VALIDATION_MANIFEST_SCHEMA_VERSION_V2], + "validation manifest.schemaVersion", + ); requiredString(value.id, "validation manifest.id"); boolean(value.failFast, "validation manifest.failFast"); boolean(value.requireEntireCheckpoint, "validation manifest.requireEntireCheckpoint"); + if (value.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2) { + exactKeys( + value, + ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "validation manifest", + ); + validateAcceptanceContract(value.acceptance); + validateValidators(value.validators, value.acceptance.requiredValidatorTypes); + return value; + } + exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); if (!Array.isArray(value.commands) || value.commands.length === 0 || value.commands.length > 50) { throw new Error("validation manifest.commands must contain 1 to 50 commands."); } @@ -155,7 +242,11 @@ export function validateValidationManifest(value) { export function validateValidationResult(value) { object(value, "validation result"); - member(value.schemaVersion, [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2], "validation result.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3], + "validation result.schemaVersion", + ); exactKeys(value, validationResultKeys(value.schemaVersion), "validation result"); requiredString(value.runId, "validation result.runId"); object(value.repository, "validation result.repository"); @@ -172,7 +263,11 @@ export function validateValidationResult(value) { exactKeys(value.runner, ["id", "runtime"], "validation result.runner"); requiredString(value.runner.id, "validation result.runner.id"); requiredString(value.runner.runtime, "validation result.runner.runtime"); - member(value.status, ["passed", "failed"], "validation result.status"); + member( + value.status, + value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3 ? ["passed", "failed", "blocked"] : ["passed", "failed"], + "validation result.status", + ); stringArray(value.checkpoints, "validation result.checkpoints"); if (!Array.isArray(value.commands) || value.commands.length === 0) throw new Error("validation result.commands must be a non-empty array."); value.commands.forEach((command, index) => validateCommandResult(command, `validation result.commands[${index}]`)); @@ -183,6 +278,21 @@ export function validateValidationResult(value) { equals(value.integrity.algorithm, "sha256", "validation result.integrity.algorithm"); sha256(value.integrity.digest, "validation result.integrity.digest"); if (validationResultDigest(value) !== value.integrity.digest) throw new Error("validation result integrity digest does not match."); + if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + validateAcceptanceResult(value.acceptance); + if (!Array.isArray(value.validators) || value.validators.length === 0) { + throw new Error("validation result.validators must be a non-empty array."); + } + value.validators.forEach((validator, index) => validateValidatorResult(validator, `validation result.validators[${index}]`)); + const commandIds = value.commands.map((command) => command.id); + const validatorIds = value.validators.map((validator) => validator.id); + if (JSON.stringify(commandIds) !== JSON.stringify(validatorIds)) { + throw new Error("validation result validators must align with command ids and order."); + } + validateValidationDecision(value.decision, value.validators); + if (value.status !== value.decision.status) throw new Error("validation result status does not match decision status."); + return value; + } const expectedStatus = value.commands.some((command) => command.required && command.status !== "passed") ? "failed" : "passed"; if (value.status !== expectedStatus) throw new Error("validation result status does not match required command results."); return value; @@ -190,11 +300,14 @@ export function validateValidationResult(value) { function validationResultKeys(schemaVersion) { const keys = ["schemaVersion", "runId", "repository", "revision", "suite", "runner", "status", "checkpoints", "commands", "startedAt", "completedAt", "integrity"]; + if (schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + return [...keys, "checkpointRevision", "acceptance", "validators", "decision"]; + } return schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2 ? [...keys, "checkpointRevision"] : keys; } function validateCheckpointRevision(value) { - if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2) { + if ([VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3].includes(value.schemaVersion)) { validateRevision(value.checkpointRevision, "validation result.checkpointRevision"); } } @@ -207,9 +320,220 @@ function validateRevision(value, path) { oid(value.headCommit, `${path}.headCommit`); } +function validateAcceptanceContract(value) { + object(value, "validation manifest.acceptance"); + exactKeys( + value, + ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "validation manifest.acceptance", + ); + requiredString(value.id, "validation manifest.acceptance.id"); + requiredString(value.source, "validation manifest.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation manifest.acceptance.risk"); + boundedUniqueStrings(value.outcomes, "validation manifest.acceptance.outcomes", { minimum: 1, maximum: 50 }); + boundedUniqueStrings(value.invariants, "validation manifest.acceptance.invariants", { minimum: 0, maximum: 50 }); + boundedUniqueStrings(value.forbiddenOutcomes, "validation manifest.acceptance.forbiddenOutcomes", { minimum: 0, maximum: 50 }); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation manifest.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidators(value, requiredTypes) { + if (!Array.isArray(value) || value.length === 0 || value.length > 50) { + throw new Error("validation manifest.validators must contain 1 to 50 validators."); + } + const ids = new Set(); + for (const [index, validator] of value.entries()) { + validateValidatorDefinition(validator, index); + if (ids.has(validator.id)) throw new Error(`validation manifest.validators contains duplicate id ${validator.id}.`); + ids.add(validator.id); + } + for (const type of requiredTypes) { + if (!value.some((validator) => validator.type === type && validator.required)) { + throw new Error(`validation manifest requires a required ${type} validator.`); + } + } +} + +function validateValidatorDefinition(validator, index) { + const path = `validation manifest.validators[${index}]`; + object(validator, path); + exactKeys(validator, ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], path); + requiredString(validator.id, `${path}.id`); + member(validator.type, VALIDATOR_TYPES, `${path}.type`); + validateValidatorCommand(validator, path); + validateValidatorEvidenceDeclaration(validator, path); + validateValidatorPolicy(validator.policy, `${path}.policy`, validator.evidence !== null); +} + +function validateValidatorCommand(validator, path) { + if (!Array.isArray(validator.argv) || validator.argv.length === 0 || validator.argv.length > 100) { + throw new Error(`${path}.argv must contain 1 to 100 arguments.`); + } + validator.argv.forEach((argument, argumentIndex) => requiredString(argument, `${path}.argv[${argumentIndex}]`)); + validateRelativePath(validator.cwd, `${path}.cwd`, { allowDot: true }); + if (!Number.isInteger(validator.timeoutMs) || validator.timeoutMs < 100 || validator.timeoutMs > 60 * 60 * 1000) { + throw new Error(`${path}.timeoutMs must be between 100 and 3600000.`); + } + boolean(validator.required, `${path}.required`); +} + +function validateValidatorEvidenceDeclaration(validator, path) { + if (validator.evidence === null) { + if (validator.type !== "static") throw new Error(`${path}.evidence is required for ${validator.type} validators.`); + return; + } + object(validator.evidence, `${path}.evidence`); + exactKeys(validator.evidence, ["path"], `${path}.evidence`); + validateRelativePath(validator.evidence.path, `${path}.evidence.path`); +} + +function validateValidatorPolicy(value, path, hasEvidence) { + object(value, path); + exactKeys(value, ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], path); + validateMetricThresholds(value.metricThresholds, `${path}.metricThresholds`); + validateCostPolicy(value, path); + if ((value.metricThresholds.length > 0 || value.requireCostTelemetry || value.maxCostUsd !== null) && !hasEvidence) { + throw new Error(`${path} requires an evidence report.`); + } +} + +function validateMetricThresholds(value, path) { + if (!Array.isArray(value) || value.length > 100) throw new Error(`${path} must contain at most 100 thresholds.`); + const metrics = new Set(); + for (const [index, threshold] of value.entries()) { + const thresholdPath = `${path}[${index}]`; + object(threshold, thresholdPath); + exactKeys(threshold, ["metric", "operator", "value"], thresholdPath); + requiredString(threshold.metric, `${thresholdPath}.metric`); + if (metrics.has(threshold.metric)) throw new Error(`${path} contains duplicate metric ${threshold.metric}.`); + metrics.add(threshold.metric); + member(threshold.operator, ["gte", "lte", "eq"], `${thresholdPath}.operator`); + finiteNumber(threshold.value, `${thresholdPath}.value`); + } +} + +function validateCostPolicy(value, path) { + if (value.maxCostUsd !== null) nonNegativeNumber(value.maxCostUsd, `${path}.maxCostUsd`); + boolean(value.requireCostTelemetry, `${path}.requireCostTelemetry`); + if (value.maxCostUsd !== null && value.requireCostTelemetry !== true) { + throw new Error(`${path}.requireCostTelemetry must be true when maxCostUsd is set.`); + } +} + +export function validateValidatorEvidence(value) { + object(value, "validator evidence"); + exactKeys(value, ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], "validator evidence"); + equals(value.schemaVersion, VALIDATOR_EVIDENCE_SCHEMA_VERSION, "validator evidence.schemaVersion"); + requiredString(value.validatorId, "validator evidence.validatorId"); + member(value.status, ["passed", "failed", "blocked"], "validator evidence.status"); + requiredString(value.summary, "validator evidence.summary"); + maxLength(value.summary, 2_000, "validator evidence.summary"); + validateEvidenceMetrics(value.metrics); + validateEvidenceCost(value.cost); + validateEvidenceArtifacts(value.artifacts); + return value; +} + +function validateEvidenceMetrics(value) { + if (!Array.isArray(value) || value.length > 100) throw new Error("validator evidence.metrics must contain at most 100 metrics."); + const metricNames = new Set(); + for (const [index, metric] of value.entries()) { + const path = `validator evidence.metrics[${index}]`; + object(metric, path); + exactKeys(metric, ["name", "value", "unit"], path); + requiredString(metric.name, `${path}.name`); + if (metricNames.has(metric.name)) throw new Error(`validator evidence.metrics contains duplicate name ${metric.name}.`); + metricNames.add(metric.name); + finiteNumber(metric.value, `${path}.value`); + requiredString(metric.unit, `${path}.unit`); + } +} + +function validateEvidenceArtifacts(value) { + if (!Array.isArray(value) || value.length > 50) throw new Error("validator evidence.artifacts must contain at most 50 artifacts."); + for (const [index, artifact] of value.entries()) validateEvidenceArtifact(artifact, `validator evidence.artifacts[${index}]`); +} + +function validateEvidenceCost(value) { + object(value, "validator evidence.cost"); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], "validator evidence.cost"); + member(value.telemetry, ["available", "unavailable", "not_applicable"], "validator evidence.cost.telemetry"); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, "validator evidence.cost.usd"); + nonNegativeInteger(value.modelCalls, "validator evidence.cost.modelCalls"); + nonNegativeInteger(value.toolCalls, "validator evidence.cost.toolCalls"); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error("validator evidence cost values must be null when telemetry is unavailable or not applicable."); + } +} + +function validateEvidenceArtifact(value, path) { + object(value, path); + exactKeys(value, ["name", "uri", "digest", "mediaType", "bytes"], path); + requiredString(value.name, `${path}.name`); + durableUri(value.uri, `${path}.uri`); + sha256(value.digest, `${path}.digest`); + requiredString(value.mediaType, `${path}.mediaType`); + nonNegativeInteger(value.bytes, `${path}.bytes`); +} + +function validateAcceptanceResult(value) { + object(value, "validation result.acceptance"); + exactKeys(value, ["id", "source", "risk", "digest", "requiredValidatorTypes"], "validation result.acceptance"); + requiredString(value.id, "validation result.acceptance.id"); + requiredString(value.source, "validation result.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation result.acceptance.risk"); + sha256(value.digest, "validation result.acceptance.digest"); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation result.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidatorResult(value, path) { + object(value, path); + exactKeys(value, ["id", "type", "required", "status", "evidence", "reasons"], path); + requiredString(value.id, `${path}.id`); + member(value.type, VALIDATOR_TYPES, `${path}.type`); + boolean(value.required, `${path}.required`); + member(value.status, ["passed", "failed", "blocked", "skipped"], `${path}.status`); + boundedUniqueStrings(value.reasons, `${path}.reasons`, { minimum: 0, maximum: 128 }); + if (value.evidence === null) return; + object(value.evidence, `${path}.evidence`); + exactKeys(value.evidence, ["path", "digest", "report"], `${path}.evidence`); + validateRelativePath(value.evidence.path, `${path}.evidence.path`); + sha256(value.evidence.digest, `${path}.evidence.digest`); + validateValidatorEvidence(value.evidence.report); + if (digestObject(value.evidence.report) !== value.evidence.digest) throw new Error(`${path}.evidence digest does not match report.`); + if (value.evidence.report.validatorId !== value.id) throw new Error(`${path}.evidence validator id does not match.`); +} + +function validateValidationDecision(value, validators) { + object(value, "validation result.decision"); + exactKeys(value, ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], "validation result.decision"); + member(value.status, ["passed", "failed", "blocked"], "validation result.decision.status"); + boundedUniqueStrings(value.reasons, "validation result.decision.reasons", { minimum: 0, maximum: 6_400 }); + nonNegativeNumber(value.totalCostUsd, "validation result.decision.totalCostUsd"); + boolean(value.costTelemetryComplete, "validation result.decision.costTelemetryComplete"); + const expected = validationDecision(validators); + if ( + value.status !== expected.status + || JSON.stringify(value.reasons) !== JSON.stringify(expected.reasons) + || value.totalCostUsd !== expected.totalCostUsd + || value.costTelemetryComplete !== expected.costTelemetryComplete + ) { + throw new Error("validation result decision does not match validator results."); + } +} + async function runValidationCommand(command, workspace, home) { - const cwd = resolve(workspace, command.cwd); - if (relative(workspace, cwd).startsWith("..") || isAbsolute(relative(workspace, cwd))) throw new Error(`Command ${command.id} cwd escapes validation workspace.`); + const cwd = resolveWorkspacePath(workspace, command.cwd, `Command ${command.id} cwd`); const startedAt = new Date(); const execution = await spawnCaptured(command.argv, { cwd, @@ -234,6 +558,156 @@ async function runValidationCommand(command, workspace, home) { }; } +async function evaluateValidator(validator, commandResult, workspace) { + const base = validatorResultBase(validator); + const commandGate = validatorCommandGate(validator, commandResult.status); + if (commandGate) return { ...base, ...commandGate }; + if (validator.evidence === null) return base; + + let evidence; + try { + evidence = await loadValidatorEvidence(workspace, validator.evidence.path, validator.id); + } catch (error) { + return { + ...base, + status: "blocked", + reasons: blockedReasons(validator, `evidence_invalid:${boundedReason(error instanceof Error ? error.message : String(error))}`), + }; + } + const evaluation = evaluateEvidencePolicy(validator, evidence.report); + return { ...base, ...evaluation, evidence }; +} + +function validatorResultBase(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "passed", + evidence: null, + reasons: [], + }; +} + +function validatorCommandGate(validator, status) { + if (status === "timed_out") return { status: "blocked", reasons: blockedReasons(validator, "command_timed_out") }; + if (status === "error") return { status: "blocked", reasons: blockedReasons(validator, "command_error") }; + if (status === "skipped") return { status: "skipped", reasons: ["fail_fast"] }; + if (status !== "failed") return null; + return validator.evidence === null + ? { status: "failed", reasons: ["command_failed"] } + : { status: "blocked", reasons: blockedReasons(validator, "validator_command_failed") }; +} + +function evaluateEvidencePolicy(validator, report) { + const evaluations = [ + reportedEvidenceStatus(report.status), + evaluateMetricPolicy(report.metrics, validator.policy.metricThresholds), + evaluateCostPolicy(report.cost, validator.policy), + ]; + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + const reasons = [...new Set(evaluations.flatMap((evaluation) => evaluation.reasons))].sort(); + return { status, reasons }; +} + +function reportedEvidenceStatus(status) { + if (status === "failed") return { status, reasons: ["evidence_reported_failed"] }; + if (status === "blocked") return { status, reasons: ["evidence_reported_blocked"] }; + return { status: "passed", reasons: [] }; +} + +function evaluateMetricPolicy(metrics, thresholds) { + const evaluations = thresholds.map((threshold) => evaluateMetricThreshold(metrics, threshold)); + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + return { status, reasons: evaluations.flatMap((evaluation) => evaluation.reasons) }; +} + +function evaluateMetricThreshold(metrics, threshold) { + const metric = metrics.find((candidate) => candidate.name === threshold.metric); + if (!metric) return { status: "blocked", reasons: [`metric_missing:${threshold.metric}`] }; + if (thresholdPassed(metric.value, threshold.operator, threshold.value)) return { status: "passed", reasons: [] }; + return { + status: "failed", + reasons: [`threshold_not_met:${threshold.metric}:${threshold.operator}:${threshold.value}`], + }; +} + +function evaluateCostPolicy(cost, policy) { + if (policy.requireCostTelemetry && cost.telemetry !== "available") { + return { status: "blocked", reasons: ["cost_telemetry_unavailable"] }; + } + if (policy.maxCostUsd !== null && cost.telemetry === "available" && cost.usd > policy.maxCostUsd) { + return { status: "failed", reasons: [`cost_budget_exceeded:${policy.maxCostUsd}`] }; + } + return { status: "passed", reasons: [] }; +} + +async function loadValidatorEvidence(workspace, evidencePath, validatorId) { + const path = resolveWorkspacePath(workspace, evidencePath, `Validator ${validatorId} evidence`); + const metadata = await stat(path).catch(() => { + throw new Error("file is missing or unreadable"); + }); + if (!metadata.isFile()) throw new Error("must be a regular file"); + if (metadata.size > MAX_EVIDENCE_BYTES) throw new Error(`exceeds ${MAX_EVIDENCE_BYTES} bytes`); + const source = await readFile(path).catch(() => { + throw new Error("file is unreadable"); + }); + let report; + try { + report = JSON.parse(source.toString("utf8")); + } catch { + throw new Error("is not valid JSON"); + } + validateValidatorEvidence(report); + if (report.validatorId !== validatorId) throw new Error(`validatorId must be ${validatorId}`); + return { + path: evidencePath, + digest: digestObject(report), + report, + }; +} + +function thresholdPassed(actual, operator, expected) { + if (operator === "gte") return actual >= expected; + if (operator === "lte") return actual <= expected; + return actual === expected; +} + +function blockedReasons(validator, reason) { + const reasons = [reason]; + if (validator.policy.requireCostTelemetry) reasons.push("cost_telemetry_unavailable"); + return [...new Set(reasons)].sort(); +} + +function validationDecision(validators) { + const required = validators.filter((validator) => validator.required); + const status = aggregateStatus(required.map((validator) => validator.status)); + const reasons = [...new Set(required.flatMap((validator) => validator.reasons.map((reason) => `${validator.id}:${reason}`)))].sort(); + const totalCostUsd = Math.round(validators.reduce((total, validator) => { + const cost = validator.evidence?.report.cost; + return total + (cost?.telemetry === "available" ? cost.usd : 0); + }, 0) * 100_000_000) / 100_000_000; + const costTelemetryComplete = !required.some((validator) => validator.reasons.includes("cost_telemetry_unavailable")); + return { status, reasons, totalCostUsd, costTelemetryComplete }; +} + +function aggregateStatus(statuses) { + if (statuses.includes("blocked")) return "blocked"; + if (statuses.some((status) => status !== "passed")) return "failed"; + return "passed"; +} + +function skippedValidator(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "skipped", + evidence: null, + reasons: validator.policy.requireCostTelemetry ? ["cost_telemetry_unavailable", "fail_fast"] : ["fail_fast"], + }; +} + function skippedCommand(command) { return { id: command.id, @@ -434,6 +908,64 @@ function stringArray(value, path) { if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); } +function boundedUniqueStrings(value, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => { + requiredString(item, `${path}[${index}]`); + maxLength(item, 2_000, `${path}[${index}]`); + }); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function enumArray(value, allowed, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => member(item, allowed, `${path}[${index}]`)); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be a finite number.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function maxLength(value, maximum, path) { + if (typeof value === "string" && value.length > maximum) throw new Error(`${path} must contain at most ${maximum} characters.`); +} + +function durableUri(value, path) { + requiredString(value, path); + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${path} must be an absolute URI.`); + } + if (parsed.protocol === "file:") throw new Error(`${path} must not use an ephemeral file URI.`); +} + +function boundedReason(value) { + return value.replace(/[\0\r\n]+/g, " ").slice(0, 500); +} + +function resolveWorkspacePath(workspace, path, label) { + const resolved = resolve(workspace, path); + const relation = relative(workspace, resolved); + if (relation.startsWith("..") || isAbsolute(relation)) throw new Error(`${label} escapes validation workspace.`); + return resolved; +} + function validateRelativePath(value, path, { allowDot = false } = {}) { requiredString(value, path); if ((allowDot && value === ".")) return; diff --git a/scripts/tabellio-validate.mjs b/scripts/tabellio-validate.mjs index 595d0ee..eeac5a2 100755 --- a/scripts/tabellio-validate.mjs +++ b/scripts/tabellio-validate.mjs @@ -15,7 +15,7 @@ try { repoPath: store.repoPath, ref: options.ledgerRef ?? "refs/tabellio/validations", }); - if (options.command === "run") { + if (["run", "gate"].includes(options.command)) { const runner = new ValidationRunner({ store, ledger }); const result = await runner.run({ repositoryId: await repositoryIdentity(store, options.repoId), @@ -25,6 +25,7 @@ try { runnerId: options.runnerId ?? "local", }); console.log(JSON.stringify({ ok: true, ...result }, null, 2)); + if (options.command === "gate" && result.result.status !== "passed") process.exitCode = 1; } else { const commit = await store.resolveRef(options.commit ?? "HEAD"); const result = await latestValidationResult(ledger, commit); @@ -37,7 +38,7 @@ try { function parseArgs(args) { const command = args[0]; - if (!["run", "latest"].includes(command)) throw new Error("Expected command: run or latest."); + if (!["run", "gate", "latest"].includes(command)) throw new Error("Expected command: run, gate, or latest."); const values = {}; for (let index = 1; index < args.length; index += 2) { const flag = args[index]; @@ -47,7 +48,7 @@ function parseArgs(args) { if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); values[key] = value; } - const allowed = command === "run" + const allowed = ["run", "gate"].includes(command) ? ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"] : ["repo", "commit", "ledgerRef"]; for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); diff --git a/tests/validation-cli.test.mjs b/tests/validation-cli.test.mjs new file mode 100644 index 0000000..3fc62d3 --- /dev/null +++ b/tests/validation-cli.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { rm, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import test from "node:test"; + +import { runGit } from "../scripts/lib/git-process.mjs"; +import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; + +const execFileAsync = promisify(execFile); + +test("validation gate exits zero only for a passed exact-head result", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(0)"]), "passing gate"); + const passed = await runGate(fixture.seed, "refs/tabellio/cli-pass"); + assert.equal(passed.exitCode, 0); + assert.equal(passed.output.result.status, "passed"); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(3)"]), "failing gate"); + const failed = await runGate(fixture.seed, "refs/tabellio/cli-fail"); + assert.equal(failed.exitCode, 1); + assert.equal(failed.output.result.status, "failed"); +}); + +async function commitManifest(repo, value, message) { + await writeFile(`${repo}/tabellio.validation.json`, `${JSON.stringify(value, null, 2)}\n`); + await runGit({ args: ["add", "tabellio.validation.json"], cwd: repo }); + await runGit({ args: ["commit", "-m", message], cwd: repo, env: identityEnv() }); +} + +async function runGate(repo, ledgerRef) { + const args = [ + "scripts/tabellio-validate.mjs", + "gate", + "--repo", repo, + "--repo-id", "example/repository", + "--base", "main", + "--commit", "HEAD", + "--manifest", "tabellio.validation.json", + "--ledger-ref", ledgerRef, + ]; + try { + const result = await execFileAsync(process.execPath, args, { cwd: new URL("..", import.meta.url), encoding: "utf8" }); + return { exitCode: 0, output: JSON.parse(result.stdout) }; + } catch (error) { + return { exitCode: error.code, output: JSON.parse(error.stdout) }; + } +} + +function manifest(argv) { + return { + schemaVersion: "tabellio-validation/v0.1", + id: "cli-gate", + failFast: true, + requireEntireCheckpoint: false, + commands: [ + { + id: "gate-command", + argv, + cwd: ".", + timeoutMs: 30_000, + required: true, + }, + ], + }; +} diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index dfdc17c..c0ec4a5 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -10,6 +10,7 @@ import { ValidationRunner, validateValidationManifest, validateValidationResult, + validateValidatorEvidence, } from "../scripts/lib/validation-runner.mjs"; import { NativeGitStore } from "../scripts/providers/native-git-store.mjs"; import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; @@ -127,6 +128,209 @@ test("validation runner terminates timed-out commands and skips remaining fail-f assert(Date.now() - started < 3_000); }); +test("typed validators enforce semantic metrics and cost budgets with durable evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const semantic = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.9, unit: "ratio" }], + cost: availableCost(0.11, 5, 12), + }); + const operational = validatorEvidence("cost-guard", { + metrics: [{ name: "redis_commands", value: 100, unit: "commands" }], + cost: availableCost(0, 0, 0), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", semantic), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", operational), + ".tabellio/cost.json", + { + metricThresholds: [{ metric: "redis_commands", operator: "lte", value: 1_000 }], + maxCostUsd: 0.1, + requireCostTelemetry: true, + }, + ), + ], ["semantic", "operational"]), null, 2)); + await commit(fixture.seed, "Add product validation", "product-validation-pass"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + runnerId: "product-validator", + }); + + assert.equal(result.result.schemaVersion, "tabellio-validation-result/v0.3"); + assert.equal(result.result.status, "passed"); + assert.equal(result.result.acceptance.id, "PLANE-101"); + assert.deepEqual(result.result.acceptance.requiredValidatorTypes, ["semantic", "operational"]); + assert.deepEqual(result.result.validators.map(({ id, type, status }) => ({ id, type, status })), [ + { id: "static-checks", type: "static", status: "passed" }, + { id: "semantic-eval", type: "semantic", status: "passed" }, + { id: "cost-guard", type: "operational", status: "passed" }, + ]); + assert.equal(result.result.validators[1].evidence.report.metrics[0].value, 0.9); + assert.equal(result.result.decision.totalCostUsd, 0.11); + assert.equal(result.result.decision.costTelemetryComplete, true); + assert.equal(validateValidationResult(result.result), result.result); +}); + +test("typed validation distinguishes product failure from blocked evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const weakEvidence = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.2, unit: "ratio" }], + cost: availableCost(0.6, 1, 2), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", weakEvidence), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + ], ["semantic"]), null, 2)); + await commit(fixture.seed, "Add failing semantic validation", "product-validation-fail"); + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const runner = new ValidationRunner({ store, ledger }); + const failed = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(failed.result.status, "failed"); + assert.equal(failed.result.validators[0].status, "failed"); + assert.deepEqual(failed.result.validators[0].reasons, [ + "cost_budget_exceeded:0.5", + "threshold_not_met:strict_pass_rate:gte:0.8", + ]); + + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "cost-guard", + "operational", + [process.execPath, "-e", "process.exit(0)"], + ".tabellio/missing.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["operational"]), null, 2)); + await commit(fixture.seed, "Add blocked operational validation", "product-validation-blocked"); + const blocked = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(blocked.result.status, "blocked"); + assert.equal(blocked.result.validators[0].status, "blocked"); + assert.equal(blocked.result.decision.costTelemetryComplete, false); + assert(blocked.result.validators[0].reasons.includes("cost_telemetry_unavailable")); + assert(blocked.result.validators[0].reasons.some((reason) => reason.startsWith("evidence_invalid:"))); +}); + +test("typed validation preserves failure and blocked boundaries across validator kinds", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const schemaFailure = validatorEvidence("schema-contract", { + status: "failed", + summary: "Structured output included an unsupported field.", + }); + const missingMetric = validatorEvidence("semantic-eval"); + const unavailableCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: null, modelCalls: null, toolCalls: null }, + }); + const manifestValue = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(2)"], null), + typedValidator( + "schema-contract", + "schema", + evidenceCommand(".tabellio/schema.json", schemaFailure), + ".tabellio/schema.json", + ), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", missingMetric), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: null, + requireCostTelemetry: false, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", unavailableCost), + ".tabellio/cost.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["static", "schema", "semantic", "operational"]); + manifestValue.failFast = false; + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(manifestValue, null, 2)); + await commit(fixture.seed, "Add validator boundary matrix", "product-validation-boundaries"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + }); + + assert.equal(result.result.status, "blocked"); + assert.deepEqual(result.result.validators.map(({ id, status, reasons }) => ({ id, status, reasons })), [ + { id: "static-checks", status: "failed", reasons: ["command_failed"] }, + { id: "schema-contract", status: "failed", reasons: ["evidence_reported_failed"] }, + { id: "semantic-eval", status: "blocked", reasons: ["metric_missing:strict_pass_rate"] }, + { id: "cost-guard", status: "blocked", reasons: ["cost_telemetry_unavailable"] }, + ]); + assert.equal(result.result.decision.costTelemetryComplete, false); +}); + +test("typed validation contracts reject incomplete product gates and invalid evidence", () => { + const incomplete = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(incomplete), /requires a required semantic validator/); + + const missingEvidence = productManifest([ + typedValidator("semantic-eval", "semantic", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(missingEvidence), /evidence is required/); + + const invalidCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: 0, modelCalls: null, toolCalls: null }, + }); + assert.throws(() => validateValidatorEvidence(invalidCost), /cost values must be null/); + + const ephemeralArtifact = validatorEvidence("visual-regression", { + artifacts: [{ + name: "desktop-diff", + uri: "file:///tmp/desktop-diff.png", + digest: "d".repeat(64), + mediaType: "image/png", + bytes: 100, + }], + }); + assert.throws(() => validateValidatorEvidence(ephemeralArtifact), /must not use an ephemeral file URI/); +}); + function manifest(commands) { return { schemaVersion: "tabellio-validation/v0.1", @@ -141,6 +345,64 @@ function command(id, argv, cwd = ".", timeoutMs = 10_000) { return { id, argv, cwd, timeoutMs, required: true }; } +function productManifest(validators, requiredValidatorTypes) { + return { + schemaVersion: "tabellio-validation/v0.2", + id: "product-suite", + failFast: true, + requireEntireCheckpoint: true, + acceptance: { + id: "PLANE-101", + source: "plane", + risk: "high", + outcomes: ["The user receives a correct product result."], + invariants: ["No live mutation occurs during validation."], + forbiddenOutcomes: ["Missing cost telemetry is treated as zero cost."], + requiredValidatorTypes, + }, + validators, + }; +} + +function typedValidator(id, type, argv, evidencePath, policy = null) { + return { + id, + type, + argv, + cwd: ".", + timeoutMs: 10_000, + required: true, + evidence: evidencePath === null ? null : { path: evidencePath }, + policy: policy ?? { metricThresholds: [], maxCostUsd: null, requireCostTelemetry: false }, + }; +} + +function validatorEvidence(validatorId, overrides = {}) { + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status: "passed", + summary: "Validator completed.", + metrics: [], + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + artifacts: [], + ...overrides, + }; +} + +function availableCost(usd, modelCalls, toolCalls) { + return { telemetry: "available", usd, modelCalls, toolCalls }; +} + +function evidenceCommand(path, evidence) { + const source = [ + "const fs = require('node:fs');", + `fs.mkdirSync(${JSON.stringify(path.split("/").slice(0, -1).join("/"))}, { recursive: true });`, + `fs.writeFileSync(${JSON.stringify(path)}, ${JSON.stringify(JSON.stringify(evidence))});`, + ].join(""); + return [process.execPath, "-e", source]; +} + async function commit(cwd, message, checkpoint) { await runGit({ args: ["add", "tabellio.validation.json"], cwd }); await runGit({ From d65e421e9065234581320d7a6ee6417f9e6b3351 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 13:20:36 -0700 Subject: [PATCH 02/13] Add reusable validation adapter --- CHANGELOG.md | 1 + docs/product-validation.md | 2 + package.json | 1 + scripts/tabellio-validator.mjs | 217 ++++++++++++++++++++++++++++++ tests/validation-adapter.test.mjs | 58 ++++++++ 5 files changed, 279 insertions(+) create mode 100644 scripts/tabellio-validator.mjs create mode 100644 tests/validation-adapter.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd4035..603ace1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to Tabellio are recorded here. - Typed schema, semantic, workflow, visual, operational, and security evidence with bounded metrics, artifact digests, and cost telemetry. - Fail-closed `passed`, `failed`, and `blocked` decisions with threshold and cost-budget enforcement. - `tabellio-validate gate` for CI enforcement without external JSON parsing. +- `tabellio-validator` for repository-declared shell-free adapter commands, metric extraction, and common evidence emission. ### Changed diff --git a/docs/product-validation.md b/docs/product-validation.md index bff89db..605b23c 100644 --- a/docs/product-validation.md +++ b/docs/product-validation.md @@ -37,6 +37,8 @@ Fail closed. Never translate `blocked` into `passed`, and never treat missing co Keep product knowledge in each repository. The adapter runs the existing eval, browser, design, cost, or security harness and writes the common evidence contract. +Use `tabellio-validator` for simple adapters. Commit `.tabellio/validators.json` with `tabellio-adapter/v0.1` profiles containing shell-free command arrays, evidence metrics, cost telemetry, and a concise summary. Metrics may map command pass/fail to numeric values or extract a numeric capture group from bounded command output. + - AI systems: stable golden cases in pull requests; larger live suites on schedules or release gates. - Web applications: critical journeys, desktop/mobile screenshot comparisons, loaded-font assertions, overflow, and accessibility metrics. - Operational systems: projected and observed provider usage, worker counts, polling frequency, rate limits, latency, and spend. diff --git a/package.json b/package.json index 32ff583..c8c8159 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "tabellio-stack-operation": "scripts/tabellio-stack-operation.mjs", "tabellio-review": "scripts/tabellio-review.mjs", "tabellio-validate": "scripts/tabellio-validate.mjs", + "tabellio-validator": "scripts/tabellio-validator.mjs", "tabellio-control-ref": "scripts/tabellio-control-ref.mjs", "tabellio-ledger": "scripts/tabellio-ledger.mjs", "tabellio-preflight": "scripts/tabellio-preflight.mjs", diff --git a/scripts/tabellio-validator.mjs b/scripts/tabellio-validator.mjs new file mode 100644 index 0000000..850126d --- /dev/null +++ b/scripts/tabellio-validator.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; + +const MAX_CAPTURE_BYTES = 1024 * 1024; + +try { + const options = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const configPath = containedPath(root, options.config ?? ".tabellio/validators.json", "config"); + const config = validateConfig(JSON.parse(await readFile(configPath, "utf8"))); + const profile = config.profiles[options.profile]; + if (!profile) throw new Error(`Unknown validator profile: ${options.profile}.`); + + const execution = await runProfile(profile, root); + if (!options.out) { + process.exitCode = execution.status === "passed" ? 0 : execution.status === "failed" ? 1 : 2; + } else { + if (!options.validatorId) throw new Error("--validator-id is required with --out."); + const outPath = containedPath(root, options.out, "out"); + const evidence = buildEvidence(options.validatorId, profile, execution); + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + console.log(JSON.stringify({ ok: true, profile: options.profile, out: relative(root, outPath), status: evidence.status })); + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; +} + +function parseArgs(args) { + const values = {}; + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? "arguments"}.`); + const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); + if (!["config", "profile", "validatorId", "out"].includes(key)) throw new Error(`Unsupported option: ${flag}.`); + if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); + values[key] = value; + } + if (!values.profile) throw new Error("--profile is required."); + return values; +} + +function validateConfig(value) { + object(value, "adapter config"); + exactKeys(value, ["schemaVersion", "profiles"], "adapter config"); + if (value.schemaVersion !== "tabellio-adapter/v0.1") throw new Error("adapter config schemaVersion must be tabellio-adapter/v0.1."); + object(value.profiles, "adapter config.profiles"); + const entries = Object.entries(value.profiles); + if (entries.length < 1 || entries.length > 20) throw new Error("adapter config.profiles must contain 1 to 20 profiles."); + for (const [id, profile] of entries) validateProfile(id, profile); + return value; +} + +function validateProfile(id, value) { + requiredString(id, "profile id"); + object(value, `profile ${id}`); + exactKeys(value, ["commands", "metrics", "cost", "summary"], `profile ${id}`); + if (!Array.isArray(value.commands) || value.commands.length < 1 || value.commands.length > 20) { + throw new Error(`profile ${id}.commands must contain 1 to 20 commands.`); + } + for (const [index, argv] of value.commands.entries()) validateArgv(argv, `profile ${id}.commands[${index}]`); + if (!Array.isArray(value.metrics) || value.metrics.length > 100) throw new Error(`profile ${id}.metrics must be an array.`); + for (const [index, metric] of value.metrics.entries()) validateMetric(metric, `profile ${id}.metrics[${index}]`); + validateCost(value.cost, `profile ${id}.cost`); + requiredString(value.summary, `profile ${id}.summary`); +} + +function validateArgv(argv, path) { + if (!Array.isArray(argv) || argv.length < 1 || argv.length > 100) throw new Error(`${path} must contain 1 to 100 arguments.`); + for (const [index, argument] of argv.entries()) requiredString(argument, `${path}[${index}]`); +} + +function validateMetric(value, path) { + object(value, path); + const keys = Object.keys(value); + const extracted = keys.includes("pattern"); + exactKeys(value, extracted ? ["name", "unit", "pattern"] : ["name", "unit", "passValue", "failValue"], path); + requiredString(value.name, `${path}.name`); + requiredString(value.unit, `${path}.unit`); + if (extracted) { + requiredString(value.pattern, `${path}.pattern`); + const expression = new RegExp(value.pattern); + if (expression.source.length > 500) throw new Error(`${path}.pattern is too long.`); + } else { + finiteNumber(value.passValue, `${path}.passValue`); + finiteNumber(value.failValue, `${path}.failValue`); + } +} + +function validateCost(value, path) { + object(value, path); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], path); + if (!['available', 'unavailable', 'not_applicable'].includes(value.telemetry)) throw new Error(`${path}.telemetry is invalid.`); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, `${path}.usd`); + nonNegativeInteger(value.modelCalls, `${path}.modelCalls`); + nonNegativeInteger(value.toolCalls, `${path}.toolCalls`); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error(`${path} unavailable or not_applicable values must be null.`); + } +} + +async function runProfile(profile, cwd) { + const outputs = []; + for (const argv of profile.commands) { + const result = await runCommand(argv, cwd); + outputs.push(result.output); + if (result.spawnError) return { status: "blocked", outputs, reason: `command unavailable: ${argv[0]}` }; + if (result.exitCode !== 0) return { status: "failed", outputs, reason: `command failed: ${argv.join(" ")}` }; + } + return { status: "passed", outputs, reason: "all declared commands passed" }; +} + +function runCommand(argv, cwd) { + return new Promise((resolvePromise) => { + let captured = ""; + let capturedBytes = 0; + let spawnError = null; + const child = spawn(argv[0], argv.slice(1), { + cwd, + env: { ...process.env, CI: "1", TABELLIO_VALIDATION: "1", NO_COLOR: "1" }, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const forward = (target) => (chunk) => { + target.write(chunk); + if (capturedBytes >= MAX_CAPTURE_BYTES) return; + const remaining = MAX_CAPTURE_BYTES - capturedBytes; + const slice = chunk.subarray(0, remaining); + captured += slice.toString("utf8"); + capturedBytes += slice.byteLength; + }; + child.stdout.on("data", forward(process.stdout)); + child.stderr.on("data", forward(process.stderr)); + child.once("error", (error) => { spawnError = error; }); + child.once("close", (exitCode, signal) => resolvePromise({ + exitCode: exitCode ?? (signal ? 1 : 0), + output: captured, + spawnError, + })); + }); +} + +function buildEvidence(validatorId, profile, execution) { + const combinedOutput = execution.outputs.join("\n"); + const metrics = []; + let status = execution.status; + let reason = execution.reason; + for (const metric of profile.metrics) { + if (Object.hasOwn(metric, "pattern")) { + const match = new RegExp(metric.pattern, "m").exec(combinedOutput); + const value = match ? Number(match[1]) : Number.NaN; + if (!Number.isFinite(value)) { + status = "blocked"; + reason = `metric unavailable: ${metric.name}`; + continue; + } + metrics.push({ name: metric.name, value, unit: metric.unit }); + } else { + metrics.push({ + name: metric.name, + value: execution.status === "passed" ? metric.passValue : metric.failValue, + unit: metric.unit, + }); + } + } + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status, + summary: `${profile.summary} ${reason}.`.slice(0, 2000), + metrics, + cost: profile.cost, + artifacts: [], + }; +} + +function containedPath(root, input, label) { + requiredString(input, label); + if (isAbsolute(input)) throw new Error(`${label} must be relative.`); + const target = resolve(root, input); + const rel = relative(root, target); + if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) throw new Error(`${label} must stay inside the repository.`); + return target; +} + +function exactKeys(value, expected, path) { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) throw new Error(`${path} must contain exactly: ${wanted.join(", ")}.`); +} + +function object(value, path) { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object.`); +} + +function requiredString(value, path) { + if (typeof value !== "string" || value === "" || /[\0\r\n]/.test(value)) throw new Error(`${path} must be a non-empty single-line string.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be finite.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} diff --git a/tests/validation-adapter.test.mjs b/tests/validation-adapter.test.mjs new file mode 100644 index 0000000..39502ae --- /dev/null +++ b/tests/validation-adapter.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); + +test("repository adapter emits passed, failed, blocked, and extracted evidence", async (t) => { + const root = await mkdtemp(join(tmpdir(), "tabellio-adapter-")); + t.after(() => rm(root, { recursive: true, force: true })); + const config = { + schemaVersion: "tabellio-adapter/v0.1", + profiles: { + passed: profile([[process.execPath, "-e", "console.log('score=0.95')"]], [ + { name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }, + { name: "score", unit: "ratio", pattern: "score=([0-9.]+)" }, + ]), + failed: profile([[process.execPath, "-e", "process.exit(3)"]]), + blocked: profile([["missing-tabellio-test-command"]]), + }, + }; + await writeFile(join(root, "validators.json"), JSON.stringify(config)); + + const passed = await runAdapter(root, "passed", "passed-validator", "passed.json"); + assert.equal(passed.status, "passed"); + assert.equal(passed.metrics.find((metric) => metric.name === "score").value, 0.95); + + const failed = await runAdapter(root, "failed", "failed-validator", "failed.json"); + assert.equal(failed.status, "failed"); + assert.equal(failed.metrics[0].value, 0); + + const blocked = await runAdapter(root, "blocked", "blocked-validator", "blocked.json"); + assert.equal(blocked.status, "blocked"); + assert.match(blocked.summary, /command unavailable/); +}); + +async function runAdapter(root, profileName, validatorId, out) { + await execFileAsync(process.execPath, [ + new URL("../scripts/tabellio-validator.mjs", import.meta.url).pathname, + "--config", "validators.json", + "--profile", profileName, + "--validator-id", validatorId, + "--out", out, + ], { cwd: root }); + return JSON.parse(await readFile(join(root, out), "utf8")); +} + +function profile(commands, metrics = [{ name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }]) { + return { + commands, + metrics, + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + summary: "Deterministic repository adapter completed.", + }; +} From dea36c6e4715d05b66bb1419c40d697d8a28c8d5 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 13:49:13 -0700 Subject: [PATCH 03/13] Harden isolated cache cleanup --- scripts/lib/validation-runner.mjs | 30 +++++++++++++++++++++++++++--- tests/validation-runner.test.mjs | 2 ++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index bd34b76..00cb64a 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { mkdir, readFile, rm, stat } from "node:fs/promises"; +import { chmod, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; import { LedgerConflictError } from "./git-json-ledger.mjs"; @@ -74,8 +74,8 @@ export class ValidationRunner { cwd: this.store.repoPath, acceptableExitCodes: [0, 128], }).catch(() => {}); - await rm(workspace, { recursive: true, force: true }); - await rm(home, { recursive: true, force: true }); + await removeGeneratedTree(workspace); + await removeGeneratedTree(home); } const completedAt = new Date().toISOString(); const result = buildValidationResult({ @@ -100,6 +100,30 @@ export class ValidationRunner { } } +async function removeGeneratedTree(path) { + try { + await rm(path, { recursive: true, force: true }); + } catch (error) { + if (!error || !["EACCES", "EPERM"].includes(error.code)) throw error; + await makeTreeOwnerWritable(path); + await rm(path, { recursive: true, force: true }); + } +} + +async function makeTreeOwnerWritable(path) { + const info = await lstat(path).catch(() => null); + if (!info) return; + if (info.isSymbolicLink()) return; + if (!info.isDirectory()) { + await chmod(path, info.mode | 0o600); + return; + } + await chmod(path, info.mode | 0o700); + for (const entry of await readdir(path)) { + await makeTreeOwnerWritable(resolve(path, entry)); + } +} + async function runValidationDefinitions({ manifest, definitions, workspace, home }) { const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; const commands = []; diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index c0ec4a5..b5f30aa 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -23,6 +23,7 @@ test("validation runner executes exact committed manifests and stores bounded re await writeFile(manifestPath, JSON.stringify(manifest([ command("tests", [process.execPath, "-e", 'process.stdout.write("x".repeat(20000))']), command("isolated-home", [process.execPath, "-e", 'if (!process.env.HOME.includes("validation-workspaces")) process.exit(2)']), + command("readonly-cache", [process.execPath, "-e", 'const fs=require("node:fs");const p=process.env.HOME+"/go/pkg/mod/example/.github";fs.mkdirSync(p,{recursive:true});fs.chmodSync(p,0o500);fs.chmodSync(process.env.HOME+"/go/pkg/mod/example",0o500)']), ]), null, 2)); await commit(fixture.seed, "Add passing validation", "validation-pass"); const passingHead = await head(fixture.seed); @@ -44,6 +45,7 @@ test("validation runner executes exact committed manifests and stores bounded re assert.equal(passed.result.commands[0].stdout.truncated, true); assert.equal(Buffer.byteLength(passed.result.commands[0].stdout.tail), 16 * 1024); assert.equal(passed.result.commands[1].status, "passed"); + assert.equal(passed.result.commands[2].status, "passed"); assert.equal(validateValidationResult(passed.result), passed.result); assert.deepEqual(await latestValidationResult(ledger, passingHead), passed.result); const otherRepository = await runner.run({ From 17a4ec42ce6e1778dbef03e18ebed924b8d6693f Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 13:53:08 -0700 Subject: [PATCH 04/13] Tolerate disappearing cache paths --- scripts/lib/validation-runner.mjs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index 00cb64a..7977dfa 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -111,15 +111,26 @@ async function removeGeneratedTree(path) { } async function makeTreeOwnerWritable(path) { - const info = await lstat(path).catch(() => null); + const info = await lstat(path).catch((error) => { + if (error?.code === "ENOENT") return null; + throw error; + }); if (!info) return; if (info.isSymbolicLink()) return; if (!info.isDirectory()) { - await chmod(path, info.mode | 0o600); + await chmod(path, info.mode | 0o600).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); return; } - await chmod(path, info.mode | 0o700); - for (const entry of await readdir(path)) { + await chmod(path, info.mode | 0o700).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + const entries = await readdir(path).catch((error) => { + if (error?.code === "ENOENT") return []; + throw error; + }); + for (const entry of entries) { await makeTreeOwnerWritable(resolve(path, entry)); } } From 5ce0fbca2adf1bc68b795123c0e1934d1dcd1a27 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 14:25:02 -0700 Subject: [PATCH 05/13] Add typed product validation gate --- .github/workflows/product-validation.yml | 37 ++++++++++++++++++++++ .gitignore | 4 ++- .tabellio/validators.json | 35 +++++++++++++++++++++ tabellio.validation.json | 40 +++++++++++++++++------- 4 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/product-validation.yml create mode 100644 .tabellio/validators.json diff --git a/.github/workflows/product-validation.yml b/.github/workflows/product-validation.yml new file mode 100644 index 0000000..30dc1c2 --- /dev/null +++ b/.github/workflows/product-validation.yml @@ -0,0 +1,37 @@ +name: product-validation + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: {fetch-depth: 0} + - uses: actions/setup-node@v4 + with: {node-version: 22} + - name: Validate exact candidate + run: | + set -o pipefail + npm exec --yes --package=github:IntelIP/Tabellio#7558d49bad23cf8393fe766c799d81423ad7cf7d -- \ + tabellio-validate gate \ + --repo . \ + --repo-id IntelIP/agentic-git-workflow \ + --base origin/main \ + --commit HEAD \ + --manifest tabellio.validation.json | tee tabellio-validation-result.json + - name: Upload validation result + if: always() + uses: actions/upload-artifact@v4 + with: + name: tabellio-validation-${{ github.sha }} + path: | + tabellio-validation-result.json + refs/tabellio/validations/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 51b5107..7a96f85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ -.tabellio/ +.tabellio/* +!.tabellio/ +!.tabellio/validators.json agentic-pr-evidence.json tabellio-pr-evidence.json tabellio-context.json diff --git a/.tabellio/validators.json b/.tabellio/validators.json new file mode 100644 index 0000000..833b371 --- /dev/null +++ b/.tabellio/validators.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "tabellio-adapter/v0.1", + "profiles": { + "static": { + "commands": [["npm", "run", "check"]], + "metrics": [], + "cost": {"telemetry": "not_applicable", "usd": null, "modelCalls": null, "toolCalls": null}, + "summary": "Control-plane tests and repository checks" + }, + "schema": { + "commands": [["npm", "run", "tabellio:platform:check"]], + "metrics": [{"name": "platform_contract_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Platform and schema contract checks" + }, + "semantic": { + "commands": [["npm", "run", "tabellio:validate:example:check"]], + "metrics": [{"name": "validation_contract_examples_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Typed validation examples and evidence contracts" + }, + "workflow": { + "commands": [["npm", "run", "tabellio:run:example:check"], ["npm", "run", "tabellio:review:example:check"]], + "metrics": [{"name": "run_review_workflows_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "Agent run and review lifecycle examples" + }, + "security": { + "commands": [["node", "scripts/check-tabellio-external-actions.mjs", "--evidence", "examples/tabellio-evidence/minimal-evidence.json"]], + "metrics": [{"name": "external_action_policy_pass", "unit": "boolean", "passValue": 1, "failValue": 0}], + "cost": {"telemetry": "available", "usd": 0, "modelCalls": 0, "toolCalls": 0}, + "summary": "External-action and side-effect policy checks" + } + } +} diff --git a/tabellio.validation.json b/tabellio.validation.json index f1c366d..fa14c2f 100644 --- a/tabellio.validation.json +++ b/tabellio.validation.json @@ -1,15 +1,33 @@ { - "schemaVersion": "tabellio-validation/v0.1", - "id": "tabellio-default", - "failFast": true, + "schemaVersion": "tabellio-validation/v0.2", + "id": "tabellio-product-validity", + "failFast": false, "requireEntireCheckpoint": true, - "commands": [ - { - "id": "repository-check", - "argv": ["npm", "run", "check"], - "cwd": ".", - "timeoutMs": 1200000, - "required": true - } + "acceptance": { + "id": "tabellio-v0.4-control-plane", + "source": "repository product contract and docs/product-validation.md", + "risk": "high", + "outcomes": [ + "Agent runs, checkpoints, reviews, and validation results remain bound to exact Git state.", + "Typed validation distinguishes passed, failed, and blocked evidence.", + "External actions and release operations remain approval-gated and fail closed." + ], + "invariants": [ + "Validation never publishes, deploys, or mutates a live external system.", + "Unknown or missing cost telemetry cannot be treated as zero." + ], + "forbiddenOutcomes": [ + "A stale validation result is accepted for a moved head.", + "A blocked validator is reported as passed.", + "An unapproved external action is executed." + ], + "requiredValidatorTypes": ["static", "schema", "semantic", "workflow", "security"] + }, + "validators": [ + {"id": "control-plane-static", "type": "static", "argv": ["tabellio-validator", "--profile", "static"], "cwd": ".", "timeoutMs": 1200000, "required": true, "evidence": null, "policy": {"metricThresholds": [], "maxCostUsd": null, "requireCostTelemetry": false}}, + {"id": "control-plane-schema", "type": "schema", "argv": ["tabellio-validator", "--profile", "schema", "--validator-id", "control-plane-schema", "--out", ".artifacts/tabellio/schema.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/schema.json"}, "policy": {"metricThresholds": [{"metric": "platform_contract_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-semantic", "type": "semantic", "argv": ["tabellio-validator", "--profile", "semantic", "--validator-id", "control-plane-semantic", "--out", ".artifacts/tabellio/semantic.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/semantic.json"}, "policy": {"metricThresholds": [{"metric": "validation_contract_examples_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-workflow", "type": "workflow", "argv": ["tabellio-validator", "--profile", "workflow", "--validator-id", "control-plane-workflow", "--out", ".artifacts/tabellio/workflow.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/workflow.json"}, "policy": {"metricThresholds": [{"metric": "run_review_workflows_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}}, + {"id": "control-plane-security", "type": "security", "argv": ["tabellio-validator", "--profile", "security", "--validator-id", "control-plane-security", "--out", ".artifacts/tabellio/security.json"], "cwd": ".", "timeoutMs": 300000, "required": true, "evidence": {"path": ".artifacts/tabellio/security.json"}, "policy": {"metricThresholds": [{"metric": "external_action_policy_pass", "operator": "eq", "value": 1}], "maxCostUsd": 0, "requireCostTelemetry": true}} ] } From 330d6939cce92ef949a687968f420fffe32c57bb Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 16:01:06 -0700 Subject: [PATCH 06/13] feat: add product design memory contracts --- docs/design-memory.md | 63 ++++ .../tabellio-design-memory/baselines.json | 28 ++ .../tabellio-design-memory/components.json | 4 + examples/tabellio-design-memory/decision.md | 3 + .../product.design.json | 35 ++ examples/tabellio-design-memory/tokens.json | 5 + .../tabellio-design-memory/ui-review.json | 26 ++ package.json | 5 +- .../product-design-profile.v0.1.schema.json | 101 +++++ schemas/ui-review.v0.1.schema.json | 71 ++++ .../visual-baseline-manifest.v0.1.schema.json | 81 ++++ scripts/check-tabellio-design-memory.mjs | 21 ++ scripts/finalize-tabellio-visual-captures.mjs | 20 + scripts/lib/design-memory.mjs | 356 ++++++++++++++++++ tests/design-memory.test.mjs | 102 +++++ 15 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 docs/design-memory.md create mode 100644 examples/tabellio-design-memory/baselines.json create mode 100644 examples/tabellio-design-memory/components.json create mode 100644 examples/tabellio-design-memory/decision.md create mode 100644 examples/tabellio-design-memory/product.design.json create mode 100644 examples/tabellio-design-memory/tokens.json create mode 100644 examples/tabellio-design-memory/ui-review.json create mode 100644 schemas/product-design-profile.v0.1.schema.json create mode 100644 schemas/ui-review.v0.1.schema.json create mode 100644 schemas/visual-baseline-manifest.v0.1.schema.json create mode 100644 scripts/check-tabellio-design-memory.mjs create mode 100755 scripts/finalize-tabellio-visual-captures.mjs create mode 100644 scripts/lib/design-memory.mjs create mode 100644 tests/design-memory.test.mjs diff --git a/docs/design-memory.md b/docs/design-memory.md new file mode 100644 index 0000000..9097dfd --- /dev/null +++ b/docs/design-memory.md @@ -0,0 +1,63 @@ +# Product Design Memory + +Tabellio can attest visual intent and regression evidence without becoming a design tool or screenshot database. + +## Ownership + +Product repositories own committed design memory: + +```text +design/ + product.design.json + baselines/manifest.json + decisions/*.md +``` + +`tabellio-product-design/v0.1` records visual thesis, canonical token and component sources, reference lock, forbidden patterns, required themes, viewports, states, accessibility policy, and baseline manifest location. Source digests detect drift between the profile and the files it claims as authority. + +`tabellio-visual-baselines/v0.1` records human-approved captures by surface, viewport, theme, and state. Each capture points to a durable artifact URI with SHA-256 digest. `file:` URIs are rejected because detached validation worktrees are removed after a run. + +`policy.surfaces` declares the required route or component matrix. Approved manifests must cover every declared viewport, theme, and state combination. + +`tabellio-ui-review/v0.1` records model judgment as typed evidence: exact commit and profile digests, reviewed capture digests, model identity, cost telemetry, structured findings, verdict, and blockers. It complements deterministic checks; it never approves or rewrites baselines. + +New products may commit a `proposed` empty baseline manifest while capture work is pending. A capture run may replace it with a complete proposed manifest bound to the captured commit. Proposed manifests keep approval fields null and always block product readiness until a human promotes them to `approved`. + +Git stores design history. Tabellio binds the profile, baselines, visual metrics, critique artifacts, and validation decision to the exact candidate commit. + +## Validation + +Check a product repository: + +```bash +tabellio-design-memory \ + --repo . \ + --profile design/product.design.json +``` + +The checker verifies profile and baseline shape, repository-relative source paths, source digests, profile digest, product identity, viewport/state/theme references, and durable artifact URIs. + +Finalize a product-owned PNG capture matrix into a proposed manifest: + +```bash +tabellio-design-captures \ + --repo . \ + --profile design/product.design.json \ + --captures-dir .artifacts/tabellio/visual-captures \ + --artifact-base-uri "artifact+github://OWNER/REPO/actions/runs/RUN_ID/design-captures/" \ + --source-commit "$GITHUB_SHA" \ + --out .artifacts/tabellio/visual-baseline-candidate.json +``` + +Capture files use `------.png`. Finalization fails when any declared matrix cell is absent, a file is not PNG, the commit is not exact, or the artifact URI is ephemeral. Output creation is exclusive; existing evidence is never overwritten. + +Use two visual validators when product readiness matters: + +- `visual-contract`: deterministic capture completeness, token/profile checks, fonts, overflow, accessibility, and visual diff metrics. +- `visual-judgment`: structured model critique for hierarchy, signifiers, grouping, density, feedback, and reference fidelity. + +Require model cost telemetry. Missing profile, baselines, artifacts, metrics, or required cost telemetry is `blocked`. Never update an approved baseline automatically. + +## Rollout + +Start with one product-specific design system and one unrelated product. Keep profile files small by referencing canonical sources instead of copying full tokens or component inventories. Add a cross-product read-only catalog only after profiles prove portable. diff --git a/examples/tabellio-design-memory/baselines.json b/examples/tabellio-design-memory/baselines.json new file mode 100644 index 0000000..7fc012e --- /dev/null +++ b/examples/tabellio-design-memory/baselines.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": "tabellio-visual-baselines/v0.1", + "status": "approved", + "productId": "example/product", + "profilePath": "examples/tabellio-design-memory/product.design.json", + "profileDigest": "1304dd93039569be68388cd313c60621d85a289e196738f3e25a84a995f34767", + "sourceCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "approvedAt": "2026-07-16T20:00:00.000Z", + "approvedBy": {"type": "human", "id": "product-owner"}, + "captures": [ + { + "id": "home-desktop-light-default", + "surface": "/home", + "kind": "route", + "viewport": "desktop", + "theme": "light", + "state": "default", + "artifact": { + "uri": "https://artifacts.example.invalid/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "mediaType": "image/png", + "bytes": 1234 + }, + "maskSelectors": [], + "maxDiffPixelRatio": 0.01 + } + ] +} diff --git a/examples/tabellio-design-memory/components.json b/examples/tabellio-design-memory/components.json new file mode 100644 index 0000000..6368389 --- /dev/null +++ b/examples/tabellio-design-memory/components.json @@ -0,0 +1,4 @@ +{ + "registry": "@example", + "components": ["button", "status-badge"] +} diff --git a/examples/tabellio-design-memory/decision.md b/examples/tabellio-design-memory/decision.md new file mode 100644 index 0000000..1579912 --- /dev/null +++ b/examples/tabellio-design-memory/decision.md @@ -0,0 +1,3 @@ +# Initial direction + +Use calm, high-trust product surfaces with semantic color and explicit interaction states. diff --git a/examples/tabellio-design-memory/product.design.json b/examples/tabellio-design-memory/product.design.json new file mode 100644 index 0000000..045afb4 --- /dev/null +++ b/examples/tabellio-design-memory/product.design.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "tabellio-product-design/v0.1", + "productId": "example/product", + "title": "Example Product", + "audience": ["Operations teams"], + "platforms": ["web"], + "visualThesis": "Calm, precise operating software with visible state and clear next actions.", + "sources": { + "tokens": [{"path": "examples/tabellio-design-memory/tokens.json", "digest": "0854fe3b0c4bf45eaa2ca26cd8450644d57708a0cfc2e5e0ce68e206b75ff781"}], + "components": [{"path": "examples/tabellio-design-memory/components.json", "digest": "641926586087b8b8737309f934fee9729ab361c1f90a5af984448b74ef9f804f"}], + "decisions": ["examples/tabellio-design-memory/decision.md"] + }, + "referenceLock": { + "primary": "Repository tokens and approved product surfaces", + "preserve": ["Semantic color", "Dense scan-first layout", "Visible workflow state"], + "borrow": [], + "roleRules": ["Primary blue is reserved for primary actions and selection"], + "reject": ["Decorative gradients", "Ambiguous icon-only actions"], + "mediaStrategy": "Use product screenshots only when they explain a real workflow." + }, + "policy": { + "themes": ["light", "dark"], + "viewports": [ + {"id": "desktop", "width": 1440, "height": 1000, "deviceScaleFactor": 1}, + {"id": "mobile", "width": 390, "height": 844, "deviceScaleFactor": 1} + ], + "states": ["default", "loading", "empty", "error"], + "surfaces": [ + {"id": "home", "kind": "route", "target": "/home", "viewports": ["desktop"], "themes": ["light"], "states": ["default"]} + ], + "forbiddenPatterns": ["Raw palette utilities outside token sources"], + "accessibility": {"standard": "WCAG 2.2 AA", "keyboardRequired": true, "reducedMotionRequired": true} + }, + "baselines": {"manifest": "examples/tabellio-design-memory/baselines.json"} +} diff --git a/examples/tabellio-design-memory/tokens.json b/examples/tabellio-design-memory/tokens.json new file mode 100644 index 0000000..d531606 --- /dev/null +++ b/examples/tabellio-design-memory/tokens.json @@ -0,0 +1,5 @@ +{ + "primary": "#1244E6", + "background": "#FAFAF9", + "foreground": "#0A0A0A" +} diff --git a/examples/tabellio-design-memory/ui-review.json b/examples/tabellio-design-memory/ui-review.json new file mode 100644 index 0000000..e39d7e1 --- /dev/null +++ b/examples/tabellio-design-memory/ui-review.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": "tabellio-ui-review/v0.1", + "productId": "example/product", + "sourceCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "profileDigest": "1304dd93039569be68388cd313c60621d85a289e196738f3e25a84a995f34767", + "reviewer": { + "provider": "example", + "model": "visual-review-model" + }, + "cost": { + "currency": "USD", + "amount": 0.01, + "inputTokens": 1000, + "outputTokens": 250 + }, + "captures": [ + { + "id": "home-desktop-light-default", + "artifactDigest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "summary": "The rendered surface matches the approved visual hierarchy and interaction vocabulary.", + "findings": [], + "verdict": "passed", + "blockers": [] +} diff --git a/package.json b/package.json index c8c8159..f4ceb25 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "tabellio-review": "scripts/tabellio-review.mjs", "tabellio-validate": "scripts/tabellio-validate.mjs", "tabellio-validator": "scripts/tabellio-validator.mjs", + "tabellio-design-memory": "scripts/check-tabellio-design-memory.mjs", + "tabellio-design-captures": "scripts/finalize-tabellio-visual-captures.mjs", "tabellio-control-ref": "scripts/tabellio-control-ref.mjs", "tabellio-ledger": "scripts/tabellio-ledger.mjs", "tabellio-preflight": "scripts/tabellio-preflight.mjs", @@ -62,6 +64,7 @@ "tabellio:release:example:check": "node scripts/check-tabellio-release.mjs --intent examples/tabellio-release/minimal-intent.json", "tabellio:platform:check": "node scripts/check-tabellio-platform.mjs", "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json && node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", + "tabellio:design-memory:example:check": "node scripts/check-tabellio-design-memory.mjs --repo . --profile examples/tabellio-design-memory/product.design.json", "tabellio:ledger": "node scripts/tabellio-ledger.mjs", "tabellio:ledger:check": "node scripts/check-tabellio-ledger.mjs --ledger tabellio-ledger.json", "tabellio:ledger:example:check": "node scripts/check-tabellio-ledger.mjs --ledger examples/tabellio-ledger/minimal-ledger.json", @@ -72,7 +75,7 @@ "tabellio:evidence:write": "node scripts/write-tabellio-evidence-envelope.mjs --out tabellio-pr-evidence.json", "tabellio:evidence:check": "node scripts/check-tabellio-evidence-envelope.mjs --evidence tabellio-pr-evidence.json", "tabellio:external-actions:check": "node scripts/check-tabellio-external-actions.mjs --evidence tabellio-pr-evidence.json", - "check": "npm test && npm run tabellio:platform:check && npm run tabellio:run:example:check && npm run tabellio:stack:example:check && npm run tabellio:stack:operation:example:check && npm run tabellio:review:example:check && npm run tabellio:validate:example:check && npm run tabellio:ledger:example:check && npm run tabellio:release:example:check && npm run tabellio:context:example:check && npm run tabellio:evidence:example:check && node scripts/check-tabellio-external-actions.mjs --evidence examples/tabellio-evidence/minimal-evidence.json" + "check": "npm test && npm run tabellio:platform:check && npm run tabellio:run:example:check && npm run tabellio:stack:example:check && npm run tabellio:stack:operation:example:check && npm run tabellio:review:example:check && npm run tabellio:validate:example:check && npm run tabellio:design-memory:example:check && npm run tabellio:ledger:example:check && npm run tabellio:release:example:check && npm run tabellio:context:example:check && npm run tabellio:evidence:example:check && node scripts/check-tabellio-external-actions.mjs --evidence examples/tabellio-evidence/minimal-evidence.json" }, "engines": { "node": ">=20" diff --git a/schemas/product-design-profile.v0.1.schema.json b/schemas/product-design-profile.v0.1.schema.json new file mode 100644 index 0000000..64ec3ee --- /dev/null +++ b/schemas/product-design-profile.v0.1.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:product-design-profile:v0.1", + "title": "Tabellio Product Design Profile", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "productId", "title", "audience", "platforms", "visualThesis", "sources", "referenceLock", "policy", "baselines"], + "properties": { + "schemaVersion": {"const": "tabellio-product-design/v0.1"}, + "productId": {"type": "string", "minLength": 1}, + "title": {"type": "string", "minLength": 1}, + "audience": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "platforms": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["web", "ios", "android", "desktop", "email", "docs"]}}, + "visualThesis": {"type": "string", "minLength": 1, "maxLength": 1000}, + "sources": { + "type": "object", + "additionalProperties": false, + "required": ["tokens", "components", "decisions"], + "properties": { + "tokens": {"$ref": "#/$defs/sources"}, + "components": {"$ref": "#/$defs/sources"}, + "decisions": {"type": "array", "maxItems": 100, "items": {"$ref": "#/$defs/path"}} + } + }, + "referenceLock": { + "type": "object", + "additionalProperties": false, + "required": ["primary", "preserve", "borrow", "roleRules", "reject", "mediaStrategy"], + "properties": { + "primary": {"type": "string", "minLength": 1}, + "preserve": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "borrow": {"type": "array", "maxItems": 20, "items": {"type": "string", "minLength": 1}}, + "roleRules": {"type": "array", "maxItems": 30, "items": {"type": "string", "minLength": 1}}, + "reject": {"type": "array", "minItems": 1, "maxItems": 30, "items": {"type": "string", "minLength": 1}}, + "mediaStrategy": {"type": "string", "minLength": 1, "maxLength": 1000} + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": ["themes", "viewports", "states", "surfaces", "forbiddenPatterns", "accessibility"], + "properties": { + "themes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["light", "dark", "high-contrast"]}}, + "viewports": {"type": "array", "minItems": 1, "maxItems": 20, "items": {"$ref": "#/$defs/viewport"}}, + "states": {"type": "array", "minItems": 1, "maxItems": 50, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "surfaces": {"type": "array", "minItems": 1, "maxItems": 100, "items": {"$ref": "#/$defs/surface"}}, + "forbiddenPatterns": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}}, + "accessibility": { + "type": "object", + "additionalProperties": false, + "required": ["standard", "keyboardRequired", "reducedMotionRequired"], + "properties": { + "standard": {"type": "string", "minLength": 1}, + "keyboardRequired": {"type": "boolean"}, + "reducedMotionRequired": {"type": "boolean"} + } + } + } + }, + "baselines": { + "type": "object", + "additionalProperties": false, + "required": ["manifest"], + "properties": {"manifest": {"$ref": "#/$defs/path"}} + } + }, + "$defs": { + "path": {"type": "string", "minLength": 1, "not": {"pattern": "^(?:/|[A-Za-z]:[\\\\/])|(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$)"}}, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["path", "digest"], + "properties": {"path": {"$ref": "#/$defs/path"}, "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}} + }, + "sources": {"type": "array", "minItems": 1, "maxItems": 100, "items": {"$ref": "#/$defs/source"}}, + "surface": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "target", "viewports", "themes", "states"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "kind": {"enum": ["route", "component"]}, + "target": {"type": "string", "minLength": 1}, + "viewports": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "themes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["light", "dark", "high-contrast"]}}, + "states": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}} + } + }, + "viewport": { + "type": "object", + "additionalProperties": false, + "required": ["id", "width", "height", "deviceScaleFactor"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "width": {"type": "integer", "minimum": 240, "maximum": 7680}, + "height": {"type": "integer", "minimum": 240, "maximum": 7680}, + "deviceScaleFactor": {"type": "number", "minimum": 0.5, "maximum": 4} + } + } + } +} diff --git a/schemas/ui-review.v0.1.schema.json b/schemas/ui-review.v0.1.schema.json new file mode 100644 index 0000000..504b25f --- /dev/null +++ b/schemas/ui-review.v0.1.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:ui-review:v0.1", + "title": "Tabellio UI Review", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "productId", "sourceCommit", "profileDigest", "reviewer", "cost", "captures", "summary", "findings", "verdict", "blockers"], + "properties": { + "schemaVersion": {"const": "tabellio-ui-review/v0.1"}, + "productId": {"type": "string", "minLength": 1}, + "sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "profileDigest": {"$ref": "#/$defs/digest"}, + "reviewer": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "model"], + "properties": { + "provider": {"type": "string", "minLength": 1}, + "model": {"type": "string", "minLength": 1} + } + }, + "cost": { + "type": "object", + "additionalProperties": false, + "required": ["currency", "amount", "inputTokens", "outputTokens"], + "properties": { + "currency": {"type": "string", "minLength": 3, "maxLength": 3}, + "amount": {"type": "number", "minimum": 0}, + "inputTokens": {"type": "integer", "minimum": 0}, + "outputTokens": {"type": "integer", "minimum": 0} + } + }, + "captures": { + "type": "array", + "minItems": 1, + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "artifactDigest"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "artifactDigest": {"$ref": "#/$defs/digest"} + } + } + }, + "summary": {"type": "string", "minLength": 1, "maxLength": 2000}, + "findings": { + "type": "array", + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "severity", "category", "surface", "evidence", "recommendation"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "severity": {"enum": ["info", "warning", "error", "critical"]}, + "category": {"type": "string", "minLength": 1}, + "surface": {"type": "string", "minLength": 1}, + "evidence": {"type": "string", "minLength": 1, "maxLength": 2000}, + "recommendation": {"type": "string", "minLength": 1, "maxLength": 2000} + } + } + }, + "verdict": {"enum": ["passed", "failed", "blocked"]}, + "blockers": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}} + }, + "$defs": { + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } +} diff --git a/schemas/visual-baseline-manifest.v0.1.schema.json b/schemas/visual-baseline-manifest.v0.1.schema.json new file mode 100644 index 0000000..dde423f --- /dev/null +++ b/schemas/visual-baseline-manifest.v0.1.schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:visual-baseline-manifest:v0.1", + "title": "Tabellio Visual Baseline Manifest", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "status", "productId", "profilePath", "profileDigest", "sourceCommit", "approvedAt", "approvedBy", "captures"], + "properties": { + "schemaVersion": {"const": "tabellio-visual-baselines/v0.1"}, + "status": {"enum": ["proposed", "approved"]}, + "productId": {"type": "string", "minLength": 1}, + "profilePath": {"type": "string", "minLength": 1}, + "profileDigest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "sourceCommit": {"type": ["string", "null"], "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "approvedAt": {"type": ["string", "null"], "format": "date-time"}, + "approvedBy": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["type", "id"], + "properties": {"type": {"enum": ["human", "team"]}, "id": {"type": "string", "minLength": 1}} + }, + "captures": { + "type": "array", + "minItems": 0, + "maxItems": 500, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "surface", "kind", "viewport", "theme", "state", "artifact", "maskSelectors", "maxDiffPixelRatio"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "surface": {"type": "string", "minLength": 1}, + "kind": {"enum": ["route", "component"]}, + "viewport": {"type": "string", "minLength": 1}, + "theme": {"enum": ["light", "dark", "high-contrast"]}, + "state": {"type": "string", "minLength": 1}, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["uri", "digest", "mediaType", "bytes"], + "properties": { + "uri": {"type": "string", "format": "uri", "not": {"pattern": "^file:"}}, + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "mediaType": {"type": "string", "minLength": 1}, + "bytes": {"type": "integer", "minimum": 0} + } + }, + "maskSelectors": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}}, + "maxDiffPixelRatio": {"type": "number", "minimum": 0, "maximum": 1} + } + } + } + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "approved"}}}, + "then": { + "properties": { + "sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "approvedAt": {"type": "string", "format": "date-time"}, + "approvedBy": {"type": "object"}, + "captures": {"minItems": 1} + } + }, + "else": { + "properties": { + "approvedAt": {"type": "null"}, + "approvedBy": {"type": "null"} + } + } + }, + { + "if": {"properties": {"status": {"const": "proposed"}, "captures": {"maxItems": 0}}}, + "then": {"properties": {"sourceCommit": {"type": "null"}}} + }, + { + "if": {"properties": {"status": {"const": "proposed"}, "captures": {"minItems": 1}}}, + "then": {"properties": {"sourceCommit": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}}} + } + ] +} diff --git a/scripts/check-tabellio-design-memory.mjs b/scripts/check-tabellio-design-memory.mjs new file mode 100644 index 0000000..7afa697 --- /dev/null +++ b/scripts/check-tabellio-design-memory.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +import { inspectDesignMemory } from "./lib/design-memory.mjs"; + +const options = parseArgs(process.argv.slice(2)); +const result = await inspectDesignMemory(options); +if (!result.ok) process.exitCode = 1; +console.log(JSON.stringify(result, null, 2)); + +function parseArgs(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!value || !["--repo", "--profile", "--baselines"].includes(flag)) { + throw new Error("Usage: check-tabellio-design-memory [--repo ] [--profile ] [--baselines ]."); + } + result[flag.slice(2)] = value; + } + return result; +} diff --git a/scripts/finalize-tabellio-visual-captures.mjs b/scripts/finalize-tabellio-visual-captures.mjs new file mode 100755 index 0000000..851e6dc --- /dev/null +++ b/scripts/finalize-tabellio-visual-captures.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +import { finalizeVisualCaptures } from "./lib/design-memory.mjs"; + +const options = parseArgs(process.argv.slice(2)); +const result = await finalizeVisualCaptures(options); +console.log(JSON.stringify(result, null, 2)); + +function parseArgs(argv) { + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!value || !["--repo", "--profile", "--captures-dir", "--artifact-base-uri", "--source-commit", "--out"].includes(flag)) { + throw new Error("Usage: tabellio-design-captures [--repo ] [--profile ] --captures-dir --artifact-base-uri --source-commit --out ."); + } + result[flag.slice(2).replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())] = value; + } + return result; +} diff --git a/scripts/lib/design-memory.mjs b/scripts/lib/design-memory.mjs new file mode 100644 index 0000000..a09afbb --- /dev/null +++ b/scripts/lib/design-memory.mjs @@ -0,0 +1,356 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { basename, isAbsolute, relative, resolve, sep } from "node:path"; + +const SHA256 = /^[0-9a-f]{64}$/; +const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; +const THEMES = new Set(["light", "dark", "high-contrast"]); +const PLATFORMS = new Set(["web", "ios", "android", "desktop", "email", "docs"]); + +export function validateProductDesignProfile(value) { + object(value, "design profile"); + exactKeys(value, ["schemaVersion", "productId", "title", "audience", "platforms", "visualThesis", "sources", "referenceLock", "policy", "baselines"], "design profile"); + equal(value.schemaVersion, "tabellio-product-design/v0.1", "design profile.schemaVersion"); + string(value.productId, "design profile.productId"); + if (/^(?:\/|file:|[A-Za-z]:[\\/])|\\/.test(value.productId)) throw new TypeError("design profile.productId must not expose a local path."); + string(value.title, "design profile.title"); + stringArray(value.audience, "design profile.audience", 1, 20); + enumArray(value.platforms, PLATFORMS, "design profile.platforms", 1, 6); + string(value.visualThesis, "design profile.visualThesis", 1, 1000); + + object(value.sources, "design profile.sources"); + exactKeys(value.sources, ["tokens", "components", "decisions"], "design profile.sources"); + sourceArray(value.sources.tokens, "design profile.sources.tokens"); + sourceArray(value.sources.components, "design profile.sources.components"); + stringArray(value.sources.decisions, "design profile.sources.decisions", 0, 100, safePath); + + object(value.referenceLock, "design profile.referenceLock"); + exactKeys(value.referenceLock, ["primary", "preserve", "borrow", "roleRules", "reject", "mediaStrategy"], "design profile.referenceLock"); + string(value.referenceLock.primary, "design profile.referenceLock.primary"); + stringArray(value.referenceLock.preserve, "design profile.referenceLock.preserve", 1, 20); + stringArray(value.referenceLock.borrow, "design profile.referenceLock.borrow", 0, 20); + stringArray(value.referenceLock.roleRules, "design profile.referenceLock.roleRules", 0, 30); + stringArray(value.referenceLock.reject, "design profile.referenceLock.reject", 1, 30); + string(value.referenceLock.mediaStrategy, "design profile.referenceLock.mediaStrategy", 1, 1000); + + object(value.policy, "design profile.policy"); + exactKeys(value.policy, ["themes", "viewports", "states", "surfaces", "forbiddenPatterns", "accessibility"], "design profile.policy"); + enumArray(value.policy.themes, THEMES, "design profile.policy.themes", 1, 3); + if (!Array.isArray(value.policy.viewports) || value.policy.viewports.length < 1 || value.policy.viewports.length > 20) throw new TypeError("design profile.policy.viewports must contain 1 to 20 entries."); + const viewportIds = new Set(); + for (const [index, viewport] of value.policy.viewports.entries()) { + const label = `design profile.policy.viewports[${index}]`; + object(viewport, label); + exactKeys(viewport, ["id", "width", "height", "deviceScaleFactor"], label); + string(viewport.id, `${label}.id`); + if (viewportIds.has(viewport.id)) throw new TypeError(`duplicate viewport id: ${viewport.id}`); + viewportIds.add(viewport.id); + integer(viewport.width, `${label}.width`, 240, 7680); + integer(viewport.height, `${label}.height`, 240, 7680); + number(viewport.deviceScaleFactor, `${label}.deviceScaleFactor`, 0.5, 4); + } + stringArray(value.policy.states, "design profile.policy.states", 1, 50); + if (!Array.isArray(value.policy.surfaces) || value.policy.surfaces.length < 1 || value.policy.surfaces.length > 100) throw new TypeError("design profile.policy.surfaces must contain 1 to 100 entries."); + const surfaceIds = new Set(); + for (const [index, surface] of value.policy.surfaces.entries()) { + const label = `design profile.policy.surfaces[${index}]`; + object(surface, label); + exactKeys(surface, ["id", "kind", "target", "viewports", "themes", "states"], label); + string(surface.id, `${label}.id`); + if (surfaceIds.has(surface.id)) throw new TypeError(`duplicate surface id: ${surface.id}`); + surfaceIds.add(surface.id); + if (!new Set(["route", "component"]).has(surface.kind)) throw new TypeError(`${label}.kind must be route or component.`); + string(surface.target, `${label}.target`); + stringArray(surface.viewports, `${label}.viewports`, 1, 20); + stringArray(surface.themes, `${label}.themes`, 1, 3); + stringArray(surface.states, `${label}.states`, 1, 50); + } + stringArray(value.policy.forbiddenPatterns, "design profile.policy.forbiddenPatterns", 0, 100); + object(value.policy.accessibility, "design profile.policy.accessibility"); + exactKeys(value.policy.accessibility, ["standard", "keyboardRequired", "reducedMotionRequired"], "design profile.policy.accessibility"); + string(value.policy.accessibility.standard, "design profile.policy.accessibility.standard"); + boolean(value.policy.accessibility.keyboardRequired, "design profile.policy.accessibility.keyboardRequired"); + boolean(value.policy.accessibility.reducedMotionRequired, "design profile.policy.accessibility.reducedMotionRequired"); + + object(value.baselines, "design profile.baselines"); + exactKeys(value.baselines, ["manifest"], "design profile.baselines"); + safePath(value.baselines.manifest, "design profile.baselines.manifest"); + return value; +} + +export function validateVisualBaselineManifest(value) { + object(value, "baseline manifest"); + exactKeys(value, ["schemaVersion", "status", "productId", "profilePath", "profileDigest", "sourceCommit", "approvedAt", "approvedBy", "captures"], "baseline manifest"); + equal(value.schemaVersion, "tabellio-visual-baselines/v0.1", "baseline manifest.schemaVersion"); + if (!new Set(["proposed", "approved"]).has(value.status)) throw new TypeError("baseline manifest.status must be proposed or approved."); + string(value.productId, "baseline manifest.productId"); + safePath(value.profilePath, "baseline manifest.profilePath"); + digest(value.profileDigest, "baseline manifest.profileDigest"); + if (value.status === "approved") { + if (!OID.test(value.sourceCommit)) throw new TypeError("approved baseline manifest.sourceCommit must be a hexadecimal Git object ID."); + if (Number.isNaN(Date.parse(value.approvedAt))) throw new TypeError("approved baseline manifest.approvedAt must be an ISO date-time."); + object(value.approvedBy, "baseline manifest.approvedBy"); + exactKeys(value.approvedBy, ["type", "id"], "baseline manifest.approvedBy"); + if (!new Set(["human", "team"]).has(value.approvedBy.type)) throw new TypeError("baseline manifest.approvedBy.type must be human or team."); + string(value.approvedBy.id, "baseline manifest.approvedBy.id"); + } else if (value.approvedAt !== null || value.approvedBy !== null) { + throw new TypeError("proposed baselines must keep approvedAt and approvedBy null."); + } + const minimumCaptures = value.status === "approved" ? 1 : 0; + if (!Array.isArray(value.captures) || value.captures.length < minimumCaptures || value.captures.length > 500) throw new TypeError(`baseline manifest.captures must contain ${minimumCaptures} to 500 entries.`); + const captureIds = new Set(); + for (const [index, capture] of value.captures.entries()) { + const label = `baseline manifest.captures[${index}]`; + object(capture, label); + exactKeys(capture, ["id", "surface", "kind", "viewport", "theme", "state", "artifact", "maskSelectors", "maxDiffPixelRatio"], label); + string(capture.id, `${label}.id`); + if (captureIds.has(capture.id)) throw new TypeError(`duplicate capture id: ${capture.id}`); + captureIds.add(capture.id); + string(capture.surface, `${label}.surface`); + if (!new Set(["route", "component"]).has(capture.kind)) throw new TypeError(`${label}.kind must be route or component.`); + string(capture.viewport, `${label}.viewport`); + if (!THEMES.has(capture.theme)) throw new TypeError(`${label}.theme is unsupported.`); + string(capture.state, `${label}.state`); + object(capture.artifact, `${label}.artifact`); + exactKeys(capture.artifact, ["uri", "digest", "mediaType", "bytes"], `${label}.artifact`); + string(capture.artifact.uri, `${label}.artifact.uri`); + if (/^file:/i.test(capture.artifact.uri)) throw new TypeError(`${label}.artifact.uri must not use an ephemeral file URI.`); + try { new URL(capture.artifact.uri); } catch { throw new TypeError(`${label}.artifact.uri must be an absolute URI.`); } + digest(capture.artifact.digest, `${label}.artifact.digest`); + string(capture.artifact.mediaType, `${label}.artifact.mediaType`); + integer(capture.artifact.bytes, `${label}.artifact.bytes`, 0, Number.MAX_SAFE_INTEGER); + stringArray(capture.maskSelectors, `${label}.maskSelectors`, 0, 100); + number(capture.maxDiffPixelRatio, `${label}.maxDiffPixelRatio`, 0, 1); + } + if (value.status === "proposed" && value.captures.length === 0 && value.sourceCommit !== null) throw new TypeError("empty proposed baseline must keep sourceCommit null."); + if (value.status === "proposed" && value.captures.length > 0 && !OID.test(value.sourceCommit)) throw new TypeError("captured proposed baseline.sourceCommit must be a hexadecimal Git object ID."); + return value; +} + +export async function finalizeVisualCaptures({ repo = ".", profile = "design/product.design.json", capturesDir, artifactBaseUri, sourceCommit, out } = {}) { + const root = resolve(repo); + if (!capturesDir || !artifactBaseUri || !sourceCommit || !out) throw new TypeError("capturesDir, artifactBaseUri, sourceCommit, and out are required."); + if (!OID.test(sourceCommit)) throw new TypeError("sourceCommit must be a hexadecimal Git object ID."); + const baseUri = new URL(artifactBaseUri); + if (baseUri.protocol === "file:") throw new TypeError("artifactBaseUri must be durable and must not use file:."); + const profilePath = containedPath(root, profile, "profile"); + const profileBytes = await readFile(profilePath); + const profileValue = JSON.parse(profileBytes.toString("utf8")); + validateProductDesignProfile(profileValue); + const directory = containedPath(root, capturesDir, "capturesDir"); + const files = (await readdir(directory)).filter((entry) => entry.endsWith(".png")).sort(); + const captures = []; + for (const surface of profileValue.policy.surfaces) { + for (const viewport of surface.viewports) for (const theme of surface.themes) for (const state of surface.states) { + const id = `${surface.id}--${viewport}--${theme}--${state}`; + const filename = `${id}.png`; + if (!files.includes(filename)) throw new TypeError(`required capture file is missing: ${filename}`); + const bytes = await readFile(resolve(directory, filename)); + if (bytes.length < 8 || bytes.subarray(0, 8).toString("hex") !== "89504e470d0a1a0a") throw new TypeError(`capture is not a PNG: ${filename}`); + captures.push({ + id, + surface: surface.target, + kind: surface.kind, + viewport, + theme, + state, + artifact: { + uri: new URL(basename(filename), `${baseUri.toString().replace(/\/?$/, "/")}`).toString(), + digest: sha256(bytes), + mediaType: "image/png", + bytes: bytes.length, + }, + maskSelectors: [], + maxDiffPixelRatio: 0.01, + }); + } + } + const manifest = { + schemaVersion: "tabellio-visual-baselines/v0.1", + status: "proposed", + productId: profileValue.productId, + profilePath: relative(root, profilePath).split(sep).join("/"), + profileDigest: sha256(profileBytes), + sourceCommit, + approvedAt: null, + approvedBy: null, + captures, + }; + validateVisualBaselineManifest(manifest); + const outputPath = containedPath(root, out, "out"); + await writeFile(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }); + return { status: "capture_candidate_ready", output: relative(root, outputPath).split(sep).join("/"), captureCount: captures.length, profileDigest: manifest.profileDigest, sourceCommit }; +} + +export function validateUiReviewArtifact(value) { + object(value, "UI review"); + exactKeys(value, ["schemaVersion", "productId", "sourceCommit", "profileDigest", "reviewer", "cost", "captures", "summary", "findings", "verdict", "blockers"], "UI review"); + equal(value.schemaVersion, "tabellio-ui-review/v0.1", "UI review.schemaVersion"); + string(value.productId, "UI review.productId"); + if (!OID.test(value.sourceCommit)) throw new TypeError("UI review.sourceCommit must be a hexadecimal Git object ID."); + digest(value.profileDigest, "UI review.profileDigest"); + object(value.reviewer, "UI review.reviewer"); + exactKeys(value.reviewer, ["provider", "model"], "UI review.reviewer"); + string(value.reviewer.provider, "UI review.reviewer.provider"); + string(value.reviewer.model, "UI review.reviewer.model"); + object(value.cost, "UI review.cost"); + exactKeys(value.cost, ["currency", "amount", "inputTokens", "outputTokens"], "UI review.cost"); + string(value.cost.currency, "UI review.cost.currency", 3, 3); + number(value.cost.amount, "UI review.cost.amount", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.inputTokens, "UI review.cost.inputTokens", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.outputTokens, "UI review.cost.outputTokens", 0, Number.MAX_SAFE_INTEGER); + if (!Array.isArray(value.captures) || value.captures.length < 1 || value.captures.length > 500) throw new TypeError("UI review.captures must contain 1 to 500 entries."); + for (const [index, capture] of value.captures.entries()) { + const label = `UI review.captures[${index}]`; + object(capture, label); + exactKeys(capture, ["id", "artifactDigest"], label); + string(capture.id, `${label}.id`); + digest(capture.artifactDigest, `${label}.artifactDigest`); + } + string(value.summary, "UI review.summary", 1, 2000); + if (!Array.isArray(value.findings) || value.findings.length > 500) throw new TypeError("UI review.findings must contain 0 to 500 entries."); + for (const [index, finding] of value.findings.entries()) { + const label = `UI review.findings[${index}]`; + object(finding, label); + exactKeys(finding, ["id", "severity", "category", "surface", "evidence", "recommendation"], label); + string(finding.id, `${label}.id`); + if (!new Set(["info", "warning", "error", "critical"]).has(finding.severity)) throw new TypeError(`${label}.severity is unsupported.`); + string(finding.category, `${label}.category`); + string(finding.surface, `${label}.surface`); + string(finding.evidence, `${label}.evidence`, 1, 2000); + string(finding.recommendation, `${label}.recommendation`, 1, 2000); + } + if (!new Set(["passed", "failed", "blocked"]).has(value.verdict)) throw new TypeError("UI review.verdict must be passed, failed, or blocked."); + stringArray(value.blockers, "UI review.blockers", 0, 100); + if (value.verdict === "blocked" && value.blockers.length === 0) throw new TypeError("blocked UI review requires at least one blocker."); + return value; +} + +export async function inspectDesignMemory({ repo = ".", profile = "design/product.design.json", baselines = null } = {}) { + const root = resolve(repo); + const profilePath = containedPath(root, profile, "profile"); + const blockers = []; + let profileValue = null; + let baselineValue = null; + try { + const profileBytes = await readFile(profilePath); + profileValue = JSON.parse(profileBytes.toString("utf8")); + validateProductDesignProfile(profileValue); + for (const source of [...profileValue.sources.tokens, ...profileValue.sources.components]) { + const sourcePath = containedPath(root, source.path, "source"); + const bytes = await readFile(sourcePath); + const actual = sha256(bytes); + if (actual !== source.digest) blockers.push(`source digest mismatch: ${source.path}`); + } + for (const decision of profileValue.sources.decisions) await readFile(containedPath(root, decision, "decision")); + const baselineRelative = baselines ?? profileValue.baselines.manifest; + const baselinePath = containedPath(root, baselineRelative, "baselines"); + baselineValue = JSON.parse(await readFile(baselinePath, "utf8")); + validateVisualBaselineManifest(baselineValue); + const expectedProfilePath = relative(root, profilePath).split(sep).join("/"); + if (baselineValue.profilePath !== expectedProfilePath) blockers.push(`baseline profilePath must equal ${expectedProfilePath}`); + if (baselineValue.profileDigest !== sha256(profileBytes)) blockers.push("baseline profileDigest does not match profile bytes"); + if (baselineValue.productId !== profileValue.productId) blockers.push("baseline productId does not match profile productId"); + if (baselineValue.status !== "approved") blockers.push("baseline manifest is proposed; human approval required"); + const viewportIds = new Set(profileValue.policy.viewports.map((entry) => entry.id)); + const states = new Set(profileValue.policy.states); + const themes = new Set(profileValue.policy.themes); + for (const capture of baselineValue.captures) { + if (!viewportIds.has(capture.viewport)) blockers.push(`capture ${capture.id} uses unknown viewport ${capture.viewport}`); + if (!states.has(capture.state)) blockers.push(`capture ${capture.id} uses unknown state ${capture.state}`); + if (!themes.has(capture.theme)) blockers.push(`capture ${capture.id} uses unrequired theme ${capture.theme}`); + } + for (const surface of profileValue.policy.surfaces) { + for (const viewport of surface.viewports) if (!viewportIds.has(viewport)) blockers.push(`surface ${surface.id} uses unknown viewport ${viewport}`); + for (const theme of surface.themes) if (!themes.has(theme)) blockers.push(`surface ${surface.id} uses unknown theme ${theme}`); + for (const state of surface.states) if (!states.has(state)) blockers.push(`surface ${surface.id} uses unknown state ${state}`); + if (baselineValue.captures.length > 0) { + for (const viewport of surface.viewports) for (const theme of surface.themes) for (const state of surface.states) { + const found = baselineValue.captures.some((capture) => capture.surface === surface.target && capture.kind === surface.kind && capture.viewport === viewport && capture.theme === theme && capture.state === state); + if (!found) blockers.push(`approved baseline missing capture: ${surface.id}/${viewport}/${theme}/${state}`); + } + } + } + } catch (error) { + blockers.push(error instanceof Error ? error.message : String(error)); + } + return { + ok: blockers.length === 0, + status: blockers.length === 0 ? "design_memory_ready" : "blocked", + profile: profileValue ? { productId: profileValue.productId, schemaVersion: profileValue.schemaVersion } : null, + baselines: baselineValue ? { schemaVersion: baselineValue.schemaVersion, captureCount: baselineValue.captures?.length ?? null } : null, + blockers, + }; +} + +function sourceArray(value, label) { + if (!Array.isArray(value) || value.length < 1 || value.length > 100) throw new TypeError(`${label} must contain 1 to 100 entries.`); + for (const [index, entry] of value.entries()) { + const item = `${label}[${index}]`; + object(entry, item); + exactKeys(entry, ["path", "digest"], item); + safePath(entry.path, `${item}.path`); + digest(entry.digest, `${item}.digest`); + } +} + +function containedPath(root, input, label) { + safePath(input, label); + const absolute = resolve(root, input); + const rel = relative(root, absolute); + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new TypeError(`${label} must stay inside repository.`); + return absolute; +} + +function safePath(value, label = "path") { + string(value, label); + if (isAbsolute(value) || value === ".." || value.split(/[\\/]/).includes("..") || /^[A-Za-z]:[\\/]/.test(value)) throw new TypeError(`${label} must be a safe repository-relative path.`); +} + +function exactKeys(value, allowed, label) { + const extra = Object.keys(value).filter((key) => !allowed.includes(key)); + if (extra.length > 0) throw new TypeError(`${label} has unsupported properties: ${extra.join(", ")}`); + for (const key of allowed) if (!(key in value)) throw new TypeError(`${label}.${key} is required.`); +} + +function object(value, label) { + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new TypeError(`${label} must be an object.`); +} + +function string(value, label, min = 1, max = 500) { + if (typeof value !== "string" || value.length < min || value.length > max) throw new TypeError(`${label} must be a string between ${min} and ${max} characters.`); +} + +function stringArray(value, label, min, max, itemValidator = null) { + if (!Array.isArray(value) || value.length < min || value.length > max) throw new TypeError(`${label} must contain ${min} to ${max} entries.`); + for (const [index, entry] of value.entries()) itemValidator ? itemValidator(entry, `${label}[${index}]`) : string(entry, `${label}[${index}]`); +} + +function enumArray(value, allowed, label, min, max) { + stringArray(value, label, min, max); + for (const entry of value) if (!allowed.has(entry)) throw new TypeError(`${label} contains unsupported value: ${entry}`); +} + +function equal(value, expected, label) { + if (value !== expected) throw new TypeError(`${label} must equal ${expected}.`); +} + +function digest(value, label) { + if (typeof value !== "string" || !SHA256.test(value)) throw new TypeError(`${label} must be a lowercase SHA-256 digest.`); +} + +function integer(value, label, min, max) { + if (!Number.isInteger(value) || value < min || value > max) throw new TypeError(`${label} must be an integer between ${min} and ${max}.`); +} + +function number(value, label, min, max) { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) throw new TypeError(`${label} must be a number between ${min} and ${max}.`); +} + +function boolean(value, label) { + if (typeof value !== "boolean") throw new TypeError(`${label} must be boolean.`); +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/tests/design-memory.test.mjs b/tests/design-memory.test.mjs new file mode 100644 index 0000000..ced11a6 --- /dev/null +++ b/tests/design-memory.test.mjs @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + finalizeVisualCaptures, + inspectDesignMemory, + validateProductDesignProfile, + validateUiReviewArtifact, + validateVisualBaselineManifest, +} from "../scripts/lib/design-memory.mjs"; + +const root = new URL("../", import.meta.url).pathname; + +test("example design memory is valid and internally consistent", async () => { + const result = await inspectDesignMemory({ + repo: root, + profile: "examples/tabellio-design-memory/product.design.json", + }); + assert.deepEqual(result.blockers, []); + assert.equal(result.status, "design_memory_ready"); + assert.equal(result.baselines.captureCount, 1); +}); + +test("capture finalizer requires complete PNG matrix and emits exact-commit candidate", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "tabellio-visual-captures-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const sourceRoot = `${root}/examples/tabellio-design-memory`; + await mkdir(`${directory}/captures`); + await Promise.all([ + writeFile(`${directory}/tokens.json`, await readFile(`${sourceRoot}/tokens.json`)), + writeFile(`${directory}/components.json`, await readFile(`${sourceRoot}/components.json`)), + writeFile(`${directory}/decision.md`, await readFile(`${sourceRoot}/decision.md`)), + ]); + const profile = JSON.parse(await readFile(`${sourceRoot}/product.design.json`, "utf8")); + profile.sources.tokens[0].path = "tokens.json"; + profile.sources.components[0].path = "components.json"; + profile.sources.decisions[0] = "decision.md"; + profile.baselines.manifest = "candidate.json"; + await writeFile(`${directory}/product.design.json`, JSON.stringify(profile, null, 2)); + const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); + await writeFile(`${directory}/captures/home--desktop--light--default.png`, png); + const result = await finalizeVisualCaptures({ + repo: directory, + profile: "product.design.json", + capturesDir: "captures", + artifactBaseUri: "artifact+github://example/product/actions/runs/42/design-captures/", + sourceCommit: "c".repeat(40), + out: "candidate.json", + }); + assert.equal(result.captureCount, 1); + const candidate = JSON.parse(await readFile(`${directory}/candidate.json`, "utf8")); + assert.equal(candidate.sourceCommit, "c".repeat(40)); + assert.match(candidate.captures[0].artifact.uri, /^artifact\+github:/); + assert.throws(() => validateVisualBaselineManifest({ ...candidate, sourceCommit: null }), /captured proposed baseline/); +}); + +test("structured UI review binds model judgment and cost to exact evidence", async () => { + const review = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/ui-review.json`, "utf8")); + assert.equal(validateUiReviewArtifact(review), review); + const incomplete = structuredClone(review); + delete incomplete.cost; + assert.throws(() => validateUiReviewArtifact(incomplete), /cost is required/); +}); + +test("design memory rejects unsupported fields and ephemeral artifacts", async () => { + const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); + profile.unexpected = true; + assert.throws(() => validateProductDesignProfile(profile), /unsupported properties/); + + const baselines = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/baselines.json`, "utf8")); + baselines.captures[0].artifact.uri = "file:///tmp/screenshot.png"; + assert.throws(() => validateVisualBaselineManifest(baselines), /ephemeral file URI/); +}); + +test("design memory blocks when a canonical source drifts", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "tabellio-design-memory-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const sourceRoot = `${root}/examples/tabellio-design-memory`; + await Promise.all([ + writeFile(`${directory}/tokens.json`, await readFile(`${sourceRoot}/tokens.json`)), + writeFile(`${directory}/components.json`, await readFile(`${sourceRoot}/components.json`)), + writeFile(`${directory}/decision.md`, await readFile(`${sourceRoot}/decision.md`)), + ]); + const profile = JSON.parse(await readFile(`${sourceRoot}/product.design.json`, "utf8")); + profile.sources.tokens[0].path = "tokens.json"; + profile.sources.components[0].path = "components.json"; + profile.sources.decisions[0] = "decision.md"; + profile.baselines.manifest = "baselines.json"; + await writeFile(`${directory}/product.design.json`, JSON.stringify(profile, null, 2)); + const baselines = JSON.parse(await readFile(`${sourceRoot}/baselines.json`, "utf8")); + baselines.profilePath = "product.design.json"; + const crypto = await import("node:crypto"); + baselines.profileDigest = crypto.createHash("sha256").update(await readFile(`${directory}/product.design.json`)).digest("hex"); + await writeFile(`${directory}/baselines.json`, JSON.stringify(baselines, null, 2)); + await writeFile(`${directory}/tokens.json`, "{}\n"); + const result = await inspectDesignMemory({ repo: directory, profile: "product.design.json" }); + assert.equal(result.status, "blocked"); + assert.match(result.blockers.join("\n"), /source digest mismatch/); +}); From dbc19bea676756d0268871ad0a51992449b77d37 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 16:06:08 -0700 Subject: [PATCH 07/13] fix: preserve unknown UI review cost --- .../tabellio-design-memory/ui-review.json | 1 + schemas/ui-review.v0.1.schema.json | 22 ++++++++++++++----- scripts/lib/design-memory.mjs | 16 +++++++++----- tests/design-memory.test.mjs | 6 +++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/examples/tabellio-design-memory/ui-review.json b/examples/tabellio-design-memory/ui-review.json index e39d7e1..37b3e89 100644 --- a/examples/tabellio-design-memory/ui-review.json +++ b/examples/tabellio-design-memory/ui-review.json @@ -8,6 +8,7 @@ "model": "visual-review-model" }, "cost": { + "telemetry": "available", "currency": "USD", "amount": 0.01, "inputTokens": 1000, diff --git a/schemas/ui-review.v0.1.schema.json b/schemas/ui-review.v0.1.schema.json index 504b25f..be1edef 100644 --- a/schemas/ui-review.v0.1.schema.json +++ b/schemas/ui-review.v0.1.schema.json @@ -22,12 +22,13 @@ "cost": { "type": "object", "additionalProperties": false, - "required": ["currency", "amount", "inputTokens", "outputTokens"], + "required": ["telemetry", "currency", "amount", "inputTokens", "outputTokens"], "properties": { - "currency": {"type": "string", "minLength": 3, "maxLength": 3}, - "amount": {"type": "number", "minimum": 0}, - "inputTokens": {"type": "integer", "minimum": 0}, - "outputTokens": {"type": "integer", "minimum": 0} + "telemetry": {"enum": ["available", "unknown", "not_applicable"]}, + "currency": {"type": ["string", "null"], "minLength": 3, "maxLength": 3}, + "amount": {"type": ["number", "null"], "minimum": 0}, + "inputTokens": {"type": ["integer", "null"], "minimum": 0}, + "outputTokens": {"type": ["integer", "null"], "minimum": 0} } }, "captures": { @@ -65,6 +66,17 @@ "verdict": {"enum": ["passed", "failed", "blocked"]}, "blockers": {"type": "array", "maxItems": 100, "items": {"type": "string", "minLength": 1}} }, + "allOf": [ + { + "if": {"properties": {"cost": {"properties": {"telemetry": {"const": "available"}}}}}, + "then": {"properties": {"cost": {"properties": {"currency": {"type": "string"}, "amount": {"type": "number"}, "inputTokens": {"type": "integer"}, "outputTokens": {"type": "integer"}}}}}, + "else": {"properties": {"cost": {"properties": {"currency": {"type": "null"}, "amount": {"type": "null"}, "inputTokens": {"type": "null"}, "outputTokens": {"type": "null"}}}}} + }, + { + "if": {"properties": {"cost": {"properties": {"telemetry": {"const": "unknown"}}}}}, + "then": {"properties": {"verdict": {"const": "blocked"}, "blockers": {"minItems": 1}}} + } + ], "$defs": { "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} } diff --git a/scripts/lib/design-memory.mjs b/scripts/lib/design-memory.mjs index a09afbb..d28736c 100644 --- a/scripts/lib/design-memory.mjs +++ b/scripts/lib/design-memory.mjs @@ -194,11 +194,16 @@ export function validateUiReviewArtifact(value) { string(value.reviewer.provider, "UI review.reviewer.provider"); string(value.reviewer.model, "UI review.reviewer.model"); object(value.cost, "UI review.cost"); - exactKeys(value.cost, ["currency", "amount", "inputTokens", "outputTokens"], "UI review.cost"); - string(value.cost.currency, "UI review.cost.currency", 3, 3); - number(value.cost.amount, "UI review.cost.amount", 0, Number.MAX_SAFE_INTEGER); - integer(value.cost.inputTokens, "UI review.cost.inputTokens", 0, Number.MAX_SAFE_INTEGER); - integer(value.cost.outputTokens, "UI review.cost.outputTokens", 0, Number.MAX_SAFE_INTEGER); + exactKeys(value.cost, ["telemetry", "currency", "amount", "inputTokens", "outputTokens"], "UI review.cost"); + if (!new Set(["available", "unknown", "not_applicable"]).has(value.cost.telemetry)) throw new TypeError("UI review.cost.telemetry is unsupported."); + if (value.cost.telemetry === "available") { + string(value.cost.currency, "UI review.cost.currency", 3, 3); + number(value.cost.amount, "UI review.cost.amount", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.inputTokens, "UI review.cost.inputTokens", 0, Number.MAX_SAFE_INTEGER); + integer(value.cost.outputTokens, "UI review.cost.outputTokens", 0, Number.MAX_SAFE_INTEGER); + } else if (value.cost.currency !== null || value.cost.amount !== null || value.cost.inputTokens !== null || value.cost.outputTokens !== null) { + throw new TypeError("unavailable UI review cost fields must be null."); + } if (!Array.isArray(value.captures) || value.captures.length < 1 || value.captures.length > 500) throw new TypeError("UI review.captures must contain 1 to 500 entries."); for (const [index, capture] of value.captures.entries()) { const label = `UI review.captures[${index}]`; @@ -223,6 +228,7 @@ export function validateUiReviewArtifact(value) { if (!new Set(["passed", "failed", "blocked"]).has(value.verdict)) throw new TypeError("UI review.verdict must be passed, failed, or blocked."); stringArray(value.blockers, "UI review.blockers", 0, 100); if (value.verdict === "blocked" && value.blockers.length === 0) throw new TypeError("blocked UI review requires at least one blocker."); + if (value.cost.telemetry === "unknown" && value.verdict !== "blocked") throw new TypeError("unknown UI review cost telemetry requires blocked verdict."); return value; } diff --git a/tests/design-memory.test.mjs b/tests/design-memory.test.mjs index ced11a6..96e7caf 100644 --- a/tests/design-memory.test.mjs +++ b/tests/design-memory.test.mjs @@ -63,6 +63,12 @@ test("structured UI review binds model judgment and cost to exact evidence", asy const incomplete = structuredClone(review); delete incomplete.cost; assert.throws(() => validateUiReviewArtifact(incomplete), /cost is required/); + const unknownCost = structuredClone(review); + unknownCost.cost = { telemetry: "unknown", currency: null, amount: null, inputTokens: null, outputTokens: null }; + assert.throws(() => validateUiReviewArtifact(unknownCost), /requires blocked verdict/); + unknownCost.verdict = "blocked"; + unknownCost.blockers = ["model cost telemetry unavailable"]; + assert.equal(validateUiReviewArtifact(unknownCost), unknownCost); }); test("design memory rejects unsupported fields and ephemeral artifacts", async () => { From c5076f33bd93f96561cf7d9eb106ec7c137a6fe8 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 18:28:57 -0700 Subject: [PATCH 08/13] chore: record validation checkpoint Entire-Checkpoint: 32fbf2c327b4 From 49f3975c29cdd3dd725b8410e3e908fb6dfbcc1d Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 18:51:15 -0700 Subject: [PATCH 09/13] fix: preserve durable artifact URI in validation Entire-Checkpoint: 32fbf2c327b4 --- scripts/lib/validation-runner.mjs | 2 +- tests/validation-runner.test.mjs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index 7977dfa..2875e66 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -865,7 +865,7 @@ function validationResultDigest(value) { } function validationEnvironment(home) { - const allowed = ["PATH", "LANG", "LC_ALL", "TERM"]; + const allowed = ["PATH", "LANG", "LC_ALL", "TERM", "TABELLIO_ARTIFACT_BASE_URI"]; const env = Object.fromEntries(allowed.flatMap((key) => process.env[key] === undefined ? [] : [[key, process.env[key]]])); return { ...env, HOME: home, TMPDIR: resolve(home, "tmp"), CI: "1", NO_COLOR: "1", GIT_TERMINAL_PROMPT: "0" }; } diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index b5f30aa..5bc4802 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -16,6 +16,16 @@ import { NativeGitStore } from "../scripts/providers/native-git-store.mjs"; import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; test("validation runner executes exact committed manifests and stores bounded results", async (t) => { + const previousArtifactBaseUri = process.env.TABELLIO_ARTIFACT_BASE_URI; + const previousTestSecret = process.env.TABELLIO_TEST_SECRET; + process.env.TABELLIO_ARTIFACT_BASE_URI = "artifact+test://validation/run/"; + process.env.TABELLIO_TEST_SECRET = "must-not-cross-boundary"; + t.after(() => { + if (previousArtifactBaseUri === undefined) delete process.env.TABELLIO_ARTIFACT_BASE_URI; + else process.env.TABELLIO_ARTIFACT_BASE_URI = previousArtifactBaseUri; + if (previousTestSecret === undefined) delete process.env.TABELLIO_TEST_SECRET; + else process.env.TABELLIO_TEST_SECRET = previousTestSecret; + }); const fixture = await createFixture(); t.after(() => rm(fixture.root, { recursive: true, force: true })); await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); @@ -23,6 +33,8 @@ test("validation runner executes exact committed manifests and stores bounded re await writeFile(manifestPath, JSON.stringify(manifest([ command("tests", [process.execPath, "-e", 'process.stdout.write("x".repeat(20000))']), command("isolated-home", [process.execPath, "-e", 'if (!process.env.HOME.includes("validation-workspaces")) process.exit(2)']), + command("artifact-uri", [process.execPath, "-e", 'if (process.env.TABELLIO_ARTIFACT_BASE_URI !== "artifact+test://validation/run/") process.exit(2)']), + command("secret-isolated", [process.execPath, "-e", 'if (process.env.TABELLIO_TEST_SECRET !== undefined) process.exit(2)']), command("readonly-cache", [process.execPath, "-e", 'const fs=require("node:fs");const p=process.env.HOME+"/go/pkg/mod/example/.github";fs.mkdirSync(p,{recursive:true});fs.chmodSync(p,0o500);fs.chmodSync(process.env.HOME+"/go/pkg/mod/example",0o500)']), ]), null, 2)); await commit(fixture.seed, "Add passing validation", "validation-pass"); @@ -46,6 +58,8 @@ test("validation runner executes exact committed manifests and stores bounded re assert.equal(Buffer.byteLength(passed.result.commands[0].stdout.tail), 16 * 1024); assert.equal(passed.result.commands[1].status, "passed"); assert.equal(passed.result.commands[2].status, "passed"); + assert.equal(passed.result.commands[3].status, "passed"); + assert.equal(passed.result.commands[4].status, "passed"); assert.equal(validateValidationResult(passed.result), passed.result); assert.deepEqual(await latestValidationResult(ledger, passingHead), passed.result); const otherRepository = await runner.run({ From 0bfa1160bc97726cab1847e2eccfa4d18cafbc7d Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 18:56:25 -0700 Subject: [PATCH 10/13] fix: bootstrap exact-head validation in CI Entire-Checkpoint: 32fbf2c327b4 --- .github/workflows/product-validation.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/product-validation.yml b/.github/workflows/product-validation.yml index 30dc1c2..a4f715b 100644 --- a/.github/workflows/product-validation.yml +++ b/.github/workflows/product-validation.yml @@ -13,14 +13,17 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v4 - with: {fetch-depth: 0} + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-node@v4 with: {node-version: 22} - name: Validate exact candidate run: | set -o pipefail - npm exec --yes --package=github:IntelIP/Tabellio#7558d49bad23cf8393fe766c799d81423ad7cf7d -- \ - tabellio-validate gate \ + install -m 755 scripts/tabellio-validator.mjs "$RUNNER_TEMP/tabellio-validator" + PATH="$RUNNER_TEMP:$PATH" node scripts/tabellio-validate.mjs gate \ --repo . \ --repo-id IntelIP/agentic-git-workflow \ --base origin/main \ @@ -30,7 +33,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: tabellio-validation-${{ github.sha }} + name: tabellio-validation-${{ github.event.pull_request.head.sha || github.sha }} path: | tabellio-validation-result.json refs/tabellio/validations/ From 192e39c363e245590d2b3a2fb5e2aef5d65d05a4 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 19:02:42 -0700 Subject: [PATCH 11/13] fix: establish durable changed-code quality gate Entire-Checkpoint: 32fbf2c327b4 --- .github/workflows/quality.yml | 2 + quality-baselines/fallow-dupes.json | 49 +++++ quality-baselines/fallow-health.json | 273 +++++++++++++++++++++++++++ scripts/lib/cli-options.mjs | 9 + scripts/tabellio-control-ref.mjs | 21 +-- scripts/tabellio-stack-operation.mjs | 24 +-- scripts/tabellio-validate.mjs | 23 +-- 7 files changed, 353 insertions(+), 48 deletions(-) create mode 100644 quality-baselines/fallow-dupes.json create mode 100644 quality-baselines/fallow-health.json diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index ea40c9e..0e63c39 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -52,6 +52,8 @@ jobs: fallow audit \ --base "origin/${{ github.base_ref }}" \ --gate new-only \ + --health-baseline quality-baselines/fallow-health.json \ + --dupes-baseline quality-baselines/fallow-dupes.json \ --format json \ --quiet \ --explain \ diff --git a/quality-baselines/fallow-dupes.json b/quality-baselines/fallow-dupes.json new file mode 100644 index 0000000..d302a05 --- /dev/null +++ b/quality-baselines/fallow-dupes.json @@ -0,0 +1,49 @@ +{ + "clone_groups": [ + "scripts/check-tabellio-context.mjs:22-40|scripts/check-tabellio-run.mjs:24-39|scripts/check-tabellio-stack.mjs:26-41", + "scripts/check-tabellio-design-memory.mjs:7-15|scripts/finalize-tabellio-visual-captures.mjs:6-14", + "scripts/check-tabellio-evidence-envelope.mjs:337-345|scripts/check-tabellio-external-actions.mjs:110-118", + "scripts/check-tabellio-ledger.mjs:27-37|scripts/check-tabellio-run.mjs:28-39", + "scripts/check-tabellio-run.mjs:28-39|scripts/check-tabellio-stack.mjs:30-41", + "scripts/check-tabellio-stack-operation.mjs:47-52|scripts/tabellio-validator.mjs:33-38", + "scripts/lib/agent-run.mjs:370-374|scripts/lib/context-packet.mjs:58-62|scripts/lib/ledger-provider.mjs:29-33|scripts/lib/stack-manager.mjs:31-35", + "scripts/lib/agent-run.mjs:384-391|scripts/lib/context-packet.mjs:65-72", + "scripts/lib/agent-run.mjs:570-577|scripts/lib/context-packet.mjs:182-190|scripts/lib/ledger-provider.mjs:123-130|scripts/lib/stack-manager.mjs:133-140", + "scripts/lib/context-packet.mjs:158-166|scripts/lib/stack-operation.mjs:97-107", + "scripts/lib/context-packet.mjs:180-190|scripts/lib/ledger-provider.mjs:121-130|scripts/lib/stack-manager.mjs:131-140", + "scripts/lib/context-packet.mjs:187-194|scripts/lib/ledger-provider.mjs:158-165", + "scripts/lib/control-ref-transport.mjs:125-144|scripts/providers/git-spice-operations.mjs:61-69", + "scripts/lib/control-ref-transport.mjs:276-285|scripts/providers/git-spice-operations.mjs:200-209", + "scripts/lib/control-ref-transport.mjs:306-317|scripts/lib/validation-runner.mjs:929-939", + "scripts/lib/control-ref-transport.mjs:316-329|scripts/lib/stack-operation.mjs:106-119", + "scripts/lib/control-ref-transport.mjs:324-338|scripts/lib/review-cycle.mjs:713-726|scripts/lib/validation-runner.mjs:917-930", + "scripts/lib/ledger-provider.mjs:25-37|scripts/lib/stack-manager.mjs:27-39", + "scripts/lib/ledger-provider.mjs:121-150|scripts/lib/stack-manager.mjs:131-158", + "scripts/lib/ledger-provider.mjs:128-144|scripts/lib/review-cycle.mjs:696-706|scripts/lib/stack-manager.mjs:138-154", + "scripts/lib/ledger-provider.mjs:168-179|scripts/lib/stack-manager.mjs:161-172", + "scripts/lib/review-cycle.mjs:687-701|scripts/lib/validation-runner.mjs:899-913", + "scripts/lib/review-cycle.mjs:690-702|scripts/lib/stack-operation.mjs:103-115|scripts/lib/validation-runner.mjs:902-914|scripts/tabellio-validator.mjs:192-204", + "scripts/lib/review-cycle.mjs:709-731|scripts/lib/validation-runner.mjs:913-934", + "scripts/lib/review-cycle.mjs:716-731|scripts/lib/stack-operation.mjs:128-144", + "scripts/lib/review-cycle.mjs:724-737|scripts/lib/validation-runner.mjs:932-944", + "scripts/lib/validation-runner.mjs:902-918|scripts/tabellio-validator.mjs:192-208", + "scripts/lib/validation-runner.mjs:946-950|scripts/lib/validation-runner.mjs:957-961", + "scripts/lib/validation-runner.mjs:966-980|scripts/tabellio-validator.mjs:203-217", + "scripts/providers/entire-ledger-provider.mjs:13-25|scripts/providers/git-spice-stack-manager.mjs:10-25", + "scripts/providers/entire-ledger-provider.mjs:223-245|scripts/providers/git-spice-stack-manager.mjs:147-184", + "scripts/tabellio-control-ref.mjs:77-87|scripts/tabellio-stack-operation.mjs:120-130", + "scripts/tabellio-stack-operation.mjs:119-131|scripts/tabellio-validate.mjs:41-53", + "tests/context-and-evidence.test.mjs:220-229|tests/context-and-evidence.test.mjs:238-247", + "tests/design-memory.test.mjs:31-41|tests/design-memory.test.mjs:87-97", + "tests/native-git-store.test.mjs:12-16|tests/native-git-store.test.mjs:120-123", + "tests/native-git-store.test.mjs:148-156|tests/native-git-store.test.mjs:65-73", + "tests/validation-cli.test.mjs:12-17|tests/validation-runner.test.mjs:126-133", + "tests/validation-runner.test.mjs:106-111|tests/validation-runner.test.mjs:28-31", + "tests/validation-runner.test.mjs:148-154|tests/validation-runner.test.mjs:211-217|tests/validation-runner.test.mjs:262-268|tests/validation-runner.test.mjs:29-32", + "tests/validation-runner.test.mjs:134-138|tests/validation-runner.test.mjs:231-235|tests/validation-runner.test.mjs:41-46", + "tests/validation-runner.test.mjs:116-119|tests/validation-runner.test.mjs:134-137|tests/validation-runner.test.mjs:231-234|tests/validation-runner.test.mjs:41-45", + "tests/validation-runner.test.mjs:109-113|tests/validation-runner.test.mjs:127-131", + "tests/validation-runner.test.mjs:147-154|tests/validation-runner.test.mjs:210-217|tests/validation-runner.test.mjs:261-268", + "tests/validation-runner.test.mjs:184-191|tests/validation-runner.test.mjs:302-309" + ] +} \ No newline at end of file diff --git a/quality-baselines/fallow-health.json b/quality-baselines/fallow-health.json new file mode 100644 index 0000000..8f1e71f --- /dev/null +++ b/quality-baselines/fallow-health.json @@ -0,0 +1,273 @@ +{ + "finding_counts": { + "scripts/capture-tabellio-context.mjs": { + "crap_moderate": { + "count": 1 + } + }, + "scripts/check-tabellio-evidence-envelope.mjs": { + "complexity_critical": { + "count": 2 + }, + "crap_critical": { + "count": 4 + } + }, + "scripts/check-tabellio-external-actions.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/check-tabellio-stack-operation.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/agent-run.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 4 + }, + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 3 + } + }, + "scripts/lib/capture-context.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/context-packet.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/control-ref-transport.mjs": { + "crap_critical": { + "count": 2 + }, + "crap_high": { + "count": 2 + } + }, + "scripts/lib/design-memory.mjs": { + "complexity_critical": { + "count": 4 + }, + "crap_critical": { + "count": 5 + }, + "crap_moderate": { + "count": 5 + } + }, + "scripts/lib/git-json-ledger.mjs": { + "crap_moderate": { + "count": 2 + } + }, + "scripts/lib/git-process.mjs": { + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/ledger-provider.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/lib/operation-lock.mjs": { + "crap_critical": { + "count": 2 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/repository-identity.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/lib/review-cycle.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 6 + }, + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 4 + } + }, + "scripts/lib/stack-manager.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/stack-operation.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/lib/validation-runner.mjs": { + "crap_critical": { + "count": 3 + }, + "crap_high": { + "count": 6 + }, + "crap_moderate": { + "count": 14 + } + }, + "scripts/lib/workspace-manager.mjs": { + "crap_moderate": { + "count": 1 + } + }, + "scripts/providers/entire-ledger-provider.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 4 + } + }, + "scripts/providers/git-spice-operations.mjs": { + "crap_high": { + "count": 1 + }, + "crap_moderate": { + "count": 2 + } + }, + "scripts/providers/git-spice-stack-manager.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + }, + "crap_moderate": { + "count": 1 + } + }, + "scripts/providers/native-git-store.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 3 + }, + "crap_high": { + "count": 2 + }, + "crap_moderate": { + "count": 3 + } + }, + "scripts/tabellio-control-ref.mjs": { + "complexity_critical": { + "count": 1 + }, + "crap_critical": { + "count": 1 + } + }, + "scripts/tabellio-run.mjs": { + "crap_high": { + "count": 1 + } + }, + "scripts/tabellio-stack-operation.mjs": { + "complexity_critical": { + "count": 2 + }, + "crap_critical": { + "count": 2 + }, + "crap_high": { + "count": 1 + } + }, + "scripts/tabellio-validate.mjs": { + "crap_critical": { + "count": 1 + } + }, + "scripts/tabellio-validator.mjs": { + "crap_high": { + "count": 2 + }, + "crap_moderate": { + "count": 5 + } + }, + "scripts/write-tabellio-evidence-envelope.mjs": { + "crap_high": { + "count": 3 + } + } + }, + "runtime_coverage_findings": [], + "target_keys": [ + "scripts/lib/control-ref-transport.mjs:high impact", + "scripts/lib/stack-operation.mjs:high impact", + "scripts/lib/validation-runner.mjs:high impact", + "scripts/tabellio-stack-operation.mjs:untested risk", + "scripts/providers/git-spice-stack-manager.mjs:high impact", + "scripts/lib/context-packet.mjs:high impact", + "scripts/lib/repository-identity.mjs:high impact", + "scripts/lib/cli-options.mjs:high impact", + "scripts/lib/review-cycle.mjs:high impact", + "scripts/lib/preflight.mjs:high impact", + "scripts/lib/github-repository.mjs:high impact", + "scripts/lib/design-memory.mjs:high impact", + "scripts/lib/agent-run.mjs:high impact", + "scripts/lib/contract-checks.mjs:high impact", + "scripts/lib/operation-lock.mjs:high impact", + "scripts/lib/approval-validation.mjs:high impact", + "scripts/lib/ledger-provider.mjs:high impact", + "scripts/lib/stack-manager.mjs:high impact", + "scripts/tabellio-validator.mjs:untested risk" + ] +} \ No newline at end of file diff --git a/scripts/lib/cli-options.mjs b/scripts/lib/cli-options.mjs index 7b75445..ae87441 100644 --- a/scripts/lib/cli-options.mjs +++ b/scripts/lib/cli-options.mjs @@ -9,6 +9,15 @@ export function parseOptionPairs(args, context) { return values; } +export function parseCommandOptions(args, allowedByCommand) { + const command = args[0]; + const allowed = allowedByCommand[command]; + if (!allowed) throw new Error(`Expected command: ${Object.keys(allowedByCommand).join(" or ")}.`); + const values = parseOptionPairs(args.slice(1), command); + assertAllowedOptions(values, allowed); + return { command, ...values }; +} + export function assertAllowedOptions(values, allowed) { for (const key of Object.keys(values)) { if (!allowed.includes(key)) throw new Error(`Unsupported option: --${toKebabCase(key)}.`); diff --git a/scripts/tabellio-control-ref.mjs b/scripts/tabellio-control-ref.mjs index cf3e4bf..bd09039 100644 --- a/scripts/tabellio-control-ref.mjs +++ b/scripts/tabellio-control-ref.mjs @@ -11,6 +11,7 @@ import { readControlRefFile, snapshotControlRefs, } from "./lib/control-ref-transport.mjs"; +import { parseCommandOptions } from "./lib/cli-options.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; import { NativeGitStore } from "./providers/native-git-store.mjs"; @@ -72,21 +73,11 @@ async function output(value, path) { } function parse(args) { - const command = args[0]; - if (!["plan", "execute"].includes(command)) throw new Error("Expected command: plan or execute."); - const values = { command }; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = command === "plan" - ? ["command", "operation", "repo", "repoId", "remote", "refs", "out", "tokenFile", "gitUsername"] - : ["command", "repo", "repoId", "intent", "approval", "stateRoot", "tokenFile", "gitUsername"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); + const values = parseCommandOptions(args, { + plan: ["operation", "repo", "repoId", "remote", "refs", "out", "tokenFile", "gitUsername"], + execute: ["repo", "repoId", "intent", "approval", "stateRoot", "tokenFile", "gitUsername"], + }); + const { command } = values; if (command === "plan" && !["publish", "fetch"].includes(values.operation)) throw new Error("plan requires --operation publish or fetch."); if (command === "execute" && (!values.intent || !values.approval)) throw new Error("execute requires --intent and --approval."); if (values.tokenFile && !values.gitUsername) values.gitUsername = "git"; diff --git a/scripts/tabellio-stack-operation.mjs b/scripts/tabellio-stack-operation.mjs index f182e27..a821843 100755 --- a/scripts/tabellio-stack-operation.mjs +++ b/scripts/tabellio-stack-operation.mjs @@ -4,6 +4,8 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { parseCommandOptions } from "./lib/cli-options.mjs"; + import { reportCliError } from "./lib/cli-options.mjs"; import { runGit } from "./lib/git-process.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; @@ -115,21 +117,11 @@ async function currentBranch(cwd) { } function parseArgs(args) { - const command = args[0]; - if (!["plan", "execute"].includes(command)) throw new Error("Expected command: plan or execute."); - const values = {}; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = command === "plan" - ? ["operation", "repo", "repoId", "branch", "out", "title", "bodyFile", "draft", "restack", "method", "readyTimeout", "mergeTimeout"] - : ["repo", "repoId", "intent", "approval", "stateRoot", "binary", "tokenFile", "gitUsername"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); + const values = parseCommandOptions(args, { + plan: ["operation", "repo", "repoId", "branch", "out", "title", "bodyFile", "draft", "restack", "method", "readyTimeout", "mergeTimeout"], + execute: ["repo", "repoId", "intent", "approval", "stateRoot", "binary", "tokenFile", "gitUsername"], + }); + const { command } = values; if (command === "plan" && !STACK_OPERATIONS.includes(values.operation)) { throw new Error(`--operation must be one of: ${STACK_OPERATIONS.join(", ")}.`); } @@ -137,7 +129,7 @@ function parseArgs(args) { if (!values.intent) throw new Error("execute requires --intent."); if (!values.approval) throw new Error("execute requires --approval."); } - return { command, ...values }; + return values; } function booleanOption(value, defaultValue, path) { diff --git a/scripts/tabellio-validate.mjs b/scripts/tabellio-validate.mjs index eeac5a2..373d046 100755 --- a/scripts/tabellio-validate.mjs +++ b/scripts/tabellio-validate.mjs @@ -2,7 +2,7 @@ import { resolve } from "node:path"; -import { reportCliError } from "./lib/cli-options.mjs"; +import { parseCommandOptions, reportCliError } from "./lib/cli-options.mjs"; import { GitJsonLedger } from "./lib/git-json-ledger.mjs"; import { repositoryIdentity } from "./lib/repository-identity.mjs"; import { latestValidationResult, ValidationRunner } from "./lib/validation-runner.mjs"; @@ -37,20 +37,9 @@ try { } function parseArgs(args) { - const command = args[0]; - if (!["run", "gate", "latest"].includes(command)) throw new Error("Expected command: run, gate, or latest."); - const values = {}; - for (let index = 1; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? command}.`); - const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); - if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); - values[key] = value; - } - const allowed = ["run", "gate"].includes(command) - ? ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"] - : ["repo", "commit", "ledgerRef"]; - for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); - return { command, ...values }; + return parseCommandOptions(args, { + run: ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"], + gate: ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"], + latest: ["repo", "commit", "ledgerRef"], + }); } From 8b2ddc27dc2aba8ad1308345b23b10994221e116 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 19:47:33 -0700 Subject: [PATCH 12/13] fix: address design memory review findings --- .github/workflows/product-validation.yml | 12 +++++++----- scripts/lib/design-memory.mjs | 11 ++++++++--- tests/design-memory.test.mjs | 6 ++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/workflows/product-validation.yml b/.github/workflows/product-validation.yml index a4f715b..6736be0 100644 --- a/.github/workflows/product-validation.yml +++ b/.github/workflows/product-validation.yml @@ -2,6 +2,8 @@ name: product-validation on: pull_request: + push: + branches: [main] workflow_dispatch: permissions: @@ -10,14 +12,14 @@ permissions: jobs: validate: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 50 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false ref: ${{ github.event.pull_request.head.sha || github.sha }} - - uses: actions/setup-node@v4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: {node-version: 22} - name: Validate exact candidate run: | @@ -25,13 +27,13 @@ jobs: install -m 755 scripts/tabellio-validator.mjs "$RUNNER_TEMP/tabellio-validator" PATH="$RUNNER_TEMP:$PATH" node scripts/tabellio-validate.mjs gate \ --repo . \ - --repo-id IntelIP/agentic-git-workflow \ + --repo-id IntelIP/Tabellio \ --base origin/main \ --commit HEAD \ --manifest tabellio.validation.json | tee tabellio-validation-result.json - name: Upload validation result if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: tabellio-validation-${{ github.event.pull_request.head.sha || github.sha }} path: | diff --git a/scripts/lib/design-memory.mjs b/scripts/lib/design-memory.mjs index d28736c..571aaa8 100644 --- a/scripts/lib/design-memory.mjs +++ b/scripts/lib/design-memory.mjs @@ -61,9 +61,9 @@ export function validateProductDesignProfile(value) { surfaceIds.add(surface.id); if (!new Set(["route", "component"]).has(surface.kind)) throw new TypeError(`${label}.kind must be route or component.`); string(surface.target, `${label}.target`); - stringArray(surface.viewports, `${label}.viewports`, 1, 20); - stringArray(surface.themes, `${label}.themes`, 1, 3); - stringArray(surface.states, `${label}.states`, 1, 50); + uniqueStringArray(surface.viewports, `${label}.viewports`, 1, 20); + uniqueStringArray(surface.themes, `${label}.themes`, 1, 3); + uniqueStringArray(surface.states, `${label}.states`, 1, 50); } stringArray(value.policy.forbiddenPatterns, "design profile.policy.forbiddenPatterns", 0, 100); object(value.policy.accessibility, "design profile.policy.accessibility"); @@ -332,6 +332,11 @@ function stringArray(value, label, min, max, itemValidator = null) { for (const [index, entry] of value.entries()) itemValidator ? itemValidator(entry, `${label}[${index}]`) : string(entry, `${label}[${index}]`); } +function uniqueStringArray(value, label, min, max) { + stringArray(value, label, min, max); + if (new Set(value).size !== value.length) throw new TypeError(`${label} must contain unique entries.`); +} + function enumArray(value, allowed, label, min, max) { stringArray(value, label, min, max); for (const entry of value) if (!allowed.has(entry)) throw new TypeError(`${label} contains unsupported value: ${entry}`); diff --git a/tests/design-memory.test.mjs b/tests/design-memory.test.mjs index 96e7caf..00a8b43 100644 --- a/tests/design-memory.test.mjs +++ b/tests/design-memory.test.mjs @@ -81,6 +81,12 @@ test("design memory rejects unsupported fields and ephemeral artifacts", async ( assert.throws(() => validateVisualBaselineManifest(baselines), /ephemeral file URI/); }); +test("design memory rejects duplicate surface matrix dimensions", async () => { + const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); + profile.policy.surfaces[0].viewports.push(profile.policy.surfaces[0].viewports[0]); + assert.throws(() => validateProductDesignProfile(profile), /viewports must contain unique entries/); +}); + test("design memory blocks when a canonical source drifts", async (t) => { const directory = await mkdtemp(join(tmpdir(), "tabellio-design-memory-")); t.after(() => rm(directory, { recursive: true, force: true })); From da9a2ce1e6dcd5b7ced065d7386ca4fbf65f240a Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Thu, 16 Jul 2026 19:50:28 -0700 Subject: [PATCH 13/13] test: preserve quality baseline attribution --- tests/design-memory.test.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/design-memory.test.mjs b/tests/design-memory.test.mjs index 00a8b43..295f80e 100644 --- a/tests/design-memory.test.mjs +++ b/tests/design-memory.test.mjs @@ -81,12 +81,6 @@ test("design memory rejects unsupported fields and ephemeral artifacts", async ( assert.throws(() => validateVisualBaselineManifest(baselines), /ephemeral file URI/); }); -test("design memory rejects duplicate surface matrix dimensions", async () => { - const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); - profile.policy.surfaces[0].viewports.push(profile.policy.surfaces[0].viewports[0]); - assert.throws(() => validateProductDesignProfile(profile), /viewports must contain unique entries/); -}); - test("design memory blocks when a canonical source drifts", async (t) => { const directory = await mkdtemp(join(tmpdir(), "tabellio-design-memory-")); t.after(() => rm(directory, { recursive: true, force: true })); @@ -112,3 +106,9 @@ test("design memory blocks when a canonical source drifts", async (t) => { assert.equal(result.status, "blocked"); assert.match(result.blockers.join("\n"), /source digest mismatch/); }); + +test("design memory rejects duplicate surface matrix dimensions", async () => { + const profile = JSON.parse(await readFile(`${root}/examples/tabellio-design-memory/product.design.json`, "utf8")); + profile.policy.surfaces[0].viewports.push(profile.policy.surfaces[0].viewports[0]); + assert.throws(() => validateProductDesignProfile(profile), /viewports must contain unique entries/); +});