diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe8308..603ace1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to Tabellio are recorded here. ## Unreleased +## 0.4.0 - 2026-07-16 + +### Added + +- Product-validity manifests that bind acceptance outcomes, invariants, forbidden outcomes, risk, and required validator types to an exact candidate commit. +- Typed schema, semantic, workflow, visual, operational, and security evidence with bounded metrics, artifact digests, and cost telemetry. +- Fail-closed `passed`, `failed`, and `blocked` decisions with threshold and cost-budget enforcement. +- `tabellio-validate gate` for CI enforcement without external JSON parsing. +- `tabellio-validator` for repository-declared shell-free adapter commands, metric extraction, and common evidence emission. + +### Changed + +- Product-changing work now requires evidence beyond command exit status when the manifest declares typed validators. +- Missing required evidence, missing metrics, command errors, timeouts, and unknown required cost telemetry block readiness. +- Existing v0.1 manifests and v0.1/v0.2 result readers remain supported during migration. + +### Release Gates + +- `tabellio-preflight --profile release` +- `npm run check` +- Fallow whole-repository and changed-code scans +- `npm pack --dry-run --json` +- Exact merged-head Tabellio validation + ## 0.3.0 - 2026-07-15 ### Added diff --git a/README.md b/README.md index 2ff8f8c..dfc6c74 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Tabellio attaches a structured evidence packet to a pull request. The packet is | Git | Repository, base branch, head branch, commit SHA, and PR metadata | | Runtime | Human, CI, or agent runtime that produced the change | | Diff | Changed files | -| Validation | Commands run and check results | +| Validation | Exact-head commands plus structural, semantic, workflow, visual, operational, and security results | | Approvals | Required, granted, denied, or skipped approvals | | Side effects | Deployment, migration, infra, billing, secret, provider, and destructive-action policy | | Artifacts | Evidence files generated by the run | @@ -58,7 +58,7 @@ AI-assisted pull requests should not depend on reviewer trust alone. Tabellio gi | Layer | Tooling | Role | | --- | --- | --- | | Runtime | [Node.js 20+](https://nodejs.org/) | Runs the local writer and validators | -| Validation | `tabellio-validate` on any trusted worker | Runs an exact committed command manifest and stores results on a Git ref | +| Validation | `tabellio-validate` on any trusted worker | Runs exact committed command or typed product validators and stores results on a Git ref | | Evidence contract | [JSON Schema](https://json-schema.org/) | Validates the evidence envelope and external-action policy | | Code storage | [GitHub](https://github.com/) | Stores code refs and tags; provides a thin pull-request shell | | Stacked review | [git-spice](https://abhinav.github.io/git-spice/) | Stack engine for small dependent GitHub pull requests | @@ -103,6 +103,12 @@ npm run tabellio:platform:check node scripts/tabellio-validate.mjs run --repo . --commit HEAD --manifest tabellio.validation.json ``` +Use `gate` in CI. It persists the same exact-head result but exits non-zero unless the final decision is `passed`: + +```bash +tabellio-validate gate --repo . --repo-id owner/repository --base main --commit HEAD --manifest tabellio.validation.json +``` + Keep `origin` limited to ordinary code branches and tags. Configure a separate private GitHub repository under another remote name before publishing control refs. The transport refuses to target `origin`. Before agent or release work, run: @@ -162,6 +168,7 @@ node scripts/tabellio-run.mjs promote --run-id run-42 --repo . ``` See [Agent run lifecycle](docs/agent-run-lifecycle.md) for state and failure behavior. +See [Product validation](docs/product-validation.md) for acceptance contracts, typed evidence, cost gates, and repository adapters. Package scripts: diff --git a/docs/codex-review.md b/docs/codex-review.md index 7edc9af..96272fc 100644 --- a/docs/codex-review.md +++ b/docs/codex-review.md @@ -36,3 +36,5 @@ Codex review is not: - deployment approval - substitute for human ownership - proof that agent output is secure + +When the repository commits a product-validation manifest, review begins only after the exact pull-request head has a `passed` v0.3 decision. A `blocked` decision means the evidence is incomplete or untrustworthy; it is not a product pass and should not be converted into reviewer guesswork. diff --git a/docs/operations-hardening.md b/docs/operations-hardening.md index 1a63af2..d46d1aa 100644 --- a/docs/operations-hardening.md +++ b/docs/operations-hardening.md @@ -11,9 +11,10 @@ Tabellio keeps code and control state in standard Git objects, but production sa ## Bounded Work -- Validation manifests allow at most 50 commands, 100 arguments per command, and one-hour command timeouts. +- Validation manifests allow at most 50 commands or validators, 100 arguments per entry, and one-hour command timeouts. - Timed-out commands receive `SIGTERM`, then `SIGKILL` after one second. Fail-fast suites mark remaining commands skipped. - Full stdout and stderr are hashed; only the newest 16 KiB of each stream is retained. +- Typed evidence files are limited to 1 MiB, 100 metrics, and 50 immutable artifact references. Required missing cost telemetry blocks validation instead of being treated as zero spend. - Agent reviews allow at most 1,000 findings. Review cycles bound feedback, fixes, check statuses, titles, bodies, summaries, event details, and retained event history. - Remote control-ref reads and atomic pushes use a 15-minute timeout. Local atomic ref updates use a 30-second timeout. diff --git a/docs/product-validation.md b/docs/product-validation.md new file mode 100644 index 0000000..605b23c --- /dev/null +++ b/docs/product-validation.md @@ -0,0 +1,84 @@ +# Product Validation + +Tabellio product validation answers a stronger question than “did tests exit zero?”: did the exact committed candidate satisfy its declared user outcomes while staying inside semantic, visual, operational, cost, and safety boundaries? + +## Contracts + +`tabellio-validation/v0.2` binds three layers in one committed manifest: + +1. `acceptance` records the source request, risk, required outcomes, invariants, forbidden outcomes, and required validator types. +2. `validators` run shell-free argument arrays in the exact-head detached worktree. +3. `policy` applies metric thresholds and cost requirements to a bounded evidence report. + +Validator types are `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security`. Static validators can use command exit status alone. Every other type must emit `tabellio-validator-evidence/v0.1` at its declared relative path. + +## Evidence + +An evidence report contains: + +- `status`: the adapter’s passed, failed, or blocked observation; +- numeric metrics with explicit units; +- cost telemetry state, USD cost, model calls, and tool calls; +- immutable artifact metadata for screenshots, traces, reports, or source snapshots. + +The runner validates the evidence shape and validator identity, embeds the bounded report in the durable result, and evaluates policy itself. Adapter-reported `passed` cannot override a failed metric threshold or exceeded cost cap. + +## Decision Semantics + +| Decision | Meaning | +| --- | --- | +| `passed` | Every required validator completed with trustworthy evidence inside all thresholds. | +| `failed` | Trustworthy evidence proves a required product or budget threshold was missed. | +| `blocked` | Required proof is missing or untrustworthy because of invalid evidence, missing metrics or cost telemetry, command errors, or timeouts. | + +Fail closed. Never translate `blocked` into `passed`, and never treat missing cost telemetry as zero. + +## Repository Adapters + +Keep product knowledge in each repository. The adapter runs the existing eval, browser, design, cost, or security harness and writes the common evidence contract. + +Use `tabellio-validator` for simple adapters. Commit `.tabellio/validators.json` with `tabellio-adapter/v0.1` profiles containing shell-free command arrays, evidence metrics, cost telemetry, and a concise summary. Metrics may map command pass/fail to numeric values or extract a numeric capture group from bounded command output. + +- AI systems: stable golden cases in pull requests; larger live suites on schedules or release gates. +- Web applications: critical journeys, desktop/mobile screenshot comparisons, loaded-font assertions, overflow, and accessibility metrics. +- Operational systems: projected and observed provider usage, worker counts, polling frequency, rate limits, latency, and spend. +- Structured outputs: strict schema validation plus semantic oracles that catch plausible but wrong values. + +Use fixtures and isolated namespaces in pull requests. Live external mutation remains separately approval-gated. + +## Review Integration + +Review sync reads only the newest validation result stored under the exact pull-request head. `passed` becomes a successful local check; `failed` or `blocked` prevents merge readiness. A new head requires new validation. + +Run contract checks with: + +```bash +node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json +node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json +``` + +Run the exact-head suite with: + +```bash +node scripts/tabellio-validate.mjs run \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +Use the fail-closed command in CI: + +```bash +tabellio-validate gate \ + --repo . \ + --repo-id example/project \ + --base main \ + --commit HEAD \ + --manifest tabellio.validation.json +``` + +`gate` still records the result. It exits non-zero when the decision is `failed` or `blocked`. + +Artifact upload and path-based automatic validator selection remain scheduler or repository-policy responsibilities. The first contract keeps evaluation portable without creating a second CI or deployment authority. diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md new file mode 100644 index 0000000..b510b73 --- /dev/null +++ b/docs/releases/v0.4.0.md @@ -0,0 +1,12 @@ +Tabellio v0.4.0 adds product-validity and cost-aware validation. + +## Highlights + +- Acceptance contracts declare user outcomes, invariants, forbidden outcomes, risk, and required evidence types. +- Typed validators cover schema, semantics, workflows, visuals, operations, cost, and security. +- Evidence carries numeric metrics, explicit units, cost telemetry, tool and model call counts, and immutable artifact metadata. +- Policy evaluation belongs to the runner. Adapter-reported success cannot override a missed threshold or cost cap. +- Missing or untrustworthy required proof is `blocked`, never an implied pass. +- `tabellio-validate gate` records exact-head results and exits non-zero unless the decision is `passed`. + +Existing v0.1 validation manifests remain readable. Live provider calls, deployments, infrastructure changes, and publication remain separately approval-gated. diff --git a/docs/validation-runner.md b/docs/validation-runner.md index a814a40..b042f22 100644 --- a/docs/validation-runner.md +++ b/docs/validation-runner.md @@ -24,6 +24,12 @@ Tabellio validation replaces the GitHub Actions runtime dependency with a small Commands are executable-plus-argument arrays. Tabellio never invokes a shell, interpolates command strings, or reads a newer working-tree manifest while validating an older commit. +Manifest v0.1 remains supported. Product-facing repositories can use `tabellio-validation/v0.2`, which adds an immutable acceptance contract and typed `static`, `schema`, `semantic`, `workflow`, `visual`, `operational`, and `security` validators. Every required validator type must have a required validator. Non-static validators must write a bounded `tabellio-validator-evidence/v0.1` report. + +Validator policy can require numeric metric thresholds, cost telemetry, and a maximum validation cost. Missing or invalid evidence, missing required metrics, unavailable required cost telemetry, command errors, and timeouts produce `blocked`. Valid evidence that misses a product threshold or cost budget produces `failed`. Only complete required evidence inside every boundary produces `passed`. + +See [Product validation](product-validation.md) and [the example manifest](../examples/tabellio-validation/product-manifest.json). + ## Run ```bash @@ -47,7 +53,7 @@ The runner: 7. Removes the worktree even after failure. 8. Writes an integrity-protected result to `refs/tabellio/validations` with compare-and-swap retries. -New results use `tabellio-validation-result/v0.2`. They require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 results. +Command-manifest results use `tabellio-validation-result/v0.2`. Product-validation results use v0.3 and embed the acceptance digest, typed validator results, bounded evidence reports, total observed validation cost, and final policy decision. Both require `checkpointRevision` so checkpoint proof remains bound to the pull-request head when the validated revision is a later squash-merge commit. Runtime readers continue to accept legacy v0.1 and v0.2 results. Read the newest result for a commit: @@ -64,3 +70,5 @@ Publishing `refs/tabellio/validations` to a remote is a separate approved Git wr ## Security Boundary Detached worktrees and isolated home directories protect host source and common credential paths; they are not a hostile-code sandbox. Validation commands can execute arbitrary repository code and use the network. Run untrusted contributions in a disposable VM, container, or sandbox with scoped credentials and network policy. Never place secrets in manifest arguments or print them: argv and bounded output tails are retained as durable evidence. + +Typed validation does not authorize live writes. Repository adapters should use fixtures, isolated namespaces, read-only credentials, and explicit cost caps. Artifact metadata must point to durable content-addressed storage when a reviewer needs the screenshot, trace, or report after the temporary worktree is removed. diff --git a/docs/workflow-model.md b/docs/workflow-model.md index 19bef23..dfd6dc5 100644 --- a/docs/workflow-model.md +++ b/docs/workflow-model.md @@ -12,6 +12,8 @@ Tabellio turns an agentic coding run into a GitHub-bound context packet and, whe | Workspace | Isolated Git worktree assigned to one agent run | | Context packet | Integrity-protected binding between task, actor, exact commits, changed files, checkpoints, and merge preview | | Evidence envelope | JSON record of Git state, changed files, commands, checks, approvals, side-effect policy, and artifacts | +| Acceptance contract | Committed outcomes, invariants, forbidden outcomes, risk, and required validator types | +| Validator evidence | Bounded metrics, cost telemetry, and immutable artifact references emitted by one product validator | | External action policy | Default-deny policy for deploys, migrations, infra, DNS, billing, secrets, provider reads, and destructive actions | | Pull request | Human review surface for the diff plus evidence summary | | Stack | Ordered PR chain for larger changes | @@ -34,7 +36,7 @@ task -> resolve immutable base commit -> isolated worktree and agent branch -> small commits and mandatory Entire checkpoint metadata - -> deterministic checks + -> deterministic and product-validity checks -> context packet -> read-only merge preview -> evidence envelope @@ -60,7 +62,7 @@ Each PR should expose: The context packet is usable without a pull request. GitHub is the review and code-distribution surface; Tabellio's evidence remains independently verifiable. -Evidence is not a claim that the work is correct. Evidence is the record reviewers inspect before trusting the work. +Raw evidence is not automatically a correctness claim. A typed validation decision proves that the exact committed evidence met the committed thresholds. Human review still owns product judgment outside the declared contract. ## Stack Discipline @@ -112,6 +114,6 @@ Future versions can add: - SLSA provenance export - in-toto link metadata - OpenTelemetry spans -- model/tool eval suites +- path-based validator policy selection - automated git-spice submission and merge orchestration for GitHub - GitHub merge queue metadata diff --git a/examples/tabellio-validation/minimal-validator-evidence.json b/examples/tabellio-validation/minimal-validator-evidence.json new file mode 100644 index 0000000..882a907 --- /dev/null +++ b/examples/tabellio-validation/minimal-validator-evidence.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": "tabellio-validator-evidence/v0.1", + "validatorId": "semantic-golden-set", + "status": "passed", + "summary": "Eight of ten required scenarios passed the strict semantic rubric.", + "metrics": [ + { + "name": "strict_pass_rate", + "value": 0.8, + "unit": "ratio" + } + ], + "cost": { + "telemetry": "available", + "usd": 0.18, + "modelCalls": 10, + "toolCalls": 24 + }, + "artifacts": [] +} diff --git a/examples/tabellio-validation/product-manifest.json b/examples/tabellio-validation/product-manifest.json new file mode 100644 index 0000000..4a7761a --- /dev/null +++ b/examples/tabellio-validation/product-manifest.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": "tabellio-validation/v0.2", + "id": "product-validity", + "failFast": true, + "requireEntireCheckpoint": true, + "acceptance": { + "id": "PLANE-101", + "source": "plane", + "risk": "high", + "outcomes": [ + "The user receives a correct result for the requested workflow." + ], + "invariants": [ + "Validation cannot mutate production resources." + ], + "forbiddenOutcomes": [ + "Unknown cost telemetry is treated as zero cost." + ], + "requiredValidatorTypes": [ + "semantic", + "operational" + ] + }, + "validators": [ + { + "id": "semantic-golden-set", + "type": "semantic", + "argv": [ + "npm", + "run", + "eval:semantic:ci" + ], + "cwd": ".", + "timeoutMs": 600000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/semantic-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "strict_pass_rate", + "operator": "gte", + "value": 0.8 + } + ], + "maxCostUsd": 1, + "requireCostTelemetry": true + } + }, + { + "id": "runtime-cost-guard", + "type": "operational", + "argv": [ + "npm", + "run", + "cost:check" + ], + "cwd": ".", + "timeoutMs": 120000, + "required": true, + "evidence": { + "path": ".artifacts/tabellio/cost-evidence.json" + }, + "policy": { + "metricThresholds": [ + { + "metric": "projected_monthly_cost_usd", + "operator": "lte", + "value": 20 + } + ], + "maxCostUsd": 0.25, + "requireCostTelemetry": true + } + } + ] +} diff --git a/package.json b/package.json index e4698e2..c8c8159 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@intelip/tabellio", - "version": "0.3.0", + "version": "0.4.0", "description": "Tabellio: GitHub-native context and evidence for agentic development.", "type": "module", "license": "Apache-2.0", @@ -16,6 +16,7 @@ "tabellio-stack-operation": "scripts/tabellio-stack-operation.mjs", "tabellio-review": "scripts/tabellio-review.mjs", "tabellio-validate": "scripts/tabellio-validate.mjs", + "tabellio-validator": "scripts/tabellio-validator.mjs", "tabellio-control-ref": "scripts/tabellio-control-ref.mjs", "tabellio-ledger": "scripts/tabellio-ledger.mjs", "tabellio-preflight": "scripts/tabellio-preflight.mjs", @@ -60,7 +61,7 @@ "tabellio:release": "node scripts/tabellio-release.mjs", "tabellio:release:example:check": "node scripts/check-tabellio-release.mjs --intent examples/tabellio-release/minimal-intent.json", "tabellio:platform:check": "node scripts/check-tabellio-platform.mjs", - "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", + "tabellio:validate:example:check": "node scripts/check-tabellio-validation.mjs --manifest tabellio.validation.json && node scripts/check-tabellio-validation.mjs --manifest examples/tabellio-validation/product-manifest.json && node scripts/check-tabellio-validation.mjs --evidence examples/tabellio-validation/minimal-validator-evidence.json && node scripts/check-tabellio-validation.mjs --result examples/tabellio-validation/minimal-result.json", "tabellio:ledger": "node scripts/tabellio-ledger.mjs", "tabellio:ledger:check": "node scripts/check-tabellio-ledger.mjs --ledger tabellio-ledger.json", "tabellio:ledger:example:check": "node scripts/check-tabellio-ledger.mjs --ledger examples/tabellio-ledger/minimal-ledger.json", diff --git a/schemas/validation-manifest.v0.1.schema.json b/schemas/validation-manifest.v0.1.schema.json new file mode 100644 index 0000000..b250820 --- /dev/null +++ b/schemas/validation-manifest.v0.1.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.1", + "title": "Tabellio Validation Manifest v0.1", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.1" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "commands": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "type": "object", + "required": ["id", "argv", "cwd", "timeoutMs", "required"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" } + } + } + } + } +} diff --git a/schemas/validation-manifest.v0.2.schema.json b/schemas/validation-manifest.v0.2.schema.json new file mode 100644 index 0000000..2ce8d2d --- /dev/null +++ b/schemas/validation-manifest.v0.2.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-manifest:v0.2", + "title": "Tabellio Product Validation Manifest", + "type": "object", + "required": ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation/v0.2" }, + "id": { "type": "string", "minLength": 1 }, + "failFast": { "type": "boolean" }, + "requireEntireCheckpoint": { "type": "boolean" }, + "acceptance": { "$ref": "#/$defs/acceptance" }, + "validators": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { "$ref": "#/$defs/validator" } + } + }, + "$defs": { + "validatorType": { + "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] + }, + "boundedText": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "acceptance": { + "type": "object", + "required": ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "outcomes": { "type": "array", "minItems": 1, "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "invariants": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "forbiddenOutcomes": { "type": "array", "maxItems": 50, "uniqueItems": true, "items": { "$ref": "#/$defs/boundedText" } }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validator": { + "type": "object", + "required": ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "argv": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "timeoutMs": { "type": "integer", "minimum": 100, "maximum": 3600000 }, + "required": { "type": "boolean" }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { "path": { "type": "string", "minLength": 1 } } + } + ] + }, + "policy": { "$ref": "#/$defs/policy" } + }, + "allOf": [ + { + "if": { "properties": { "type": { "not": { "const": "static" } } }, "required": ["type"] }, + "then": { "properties": { "evidence": { "type": "object" } } } + } + ] + }, + "policy": { + "type": "object", + "required": ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], + "additionalProperties": false, + "properties": { + "metricThresholds": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["metric", "operator", "value"], + "additionalProperties": false, + "properties": { + "metric": { "type": "string", "minLength": 1 }, + "operator": { "enum": ["gte", "lte", "eq"] }, + "value": { "type": "number" } + } + } + }, + "maxCostUsd": { "type": ["number", "null"], "minimum": 0 }, + "requireCostTelemetry": { "type": "boolean" } + }, + "allOf": [ + { + "if": { "properties": { "maxCostUsd": { "type": "number" } }, "required": ["maxCostUsd"] }, + "then": { "properties": { "requireCostTelemetry": { "const": true } } } + } + ] + } + } +} diff --git a/schemas/validation-result.v0.3.schema.json b/schemas/validation-result.v0.3.schema.json new file mode 100644 index 0000000..83756f1 --- /dev/null +++ b/schemas/validation-result.v0.3.schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validation-result:v0.3", + "title": "Tabellio Product Validation Result", + "type": "object", + "required": ["schemaVersion", "runId", "repository", "revision", "checkpointRevision", "suite", "runner", "status", "checkpoints", "commands", "acceptance", "validators", "decision", "startedAt", "completedAt", "integrity"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validation-result/v0.3" }, + "runId": { "type": "string", "minLength": 1 }, + "repository": { + "type": "object", + "required": ["id"], + "additionalProperties": false, + "properties": { "id": { "type": "string", "minLength": 1 } } + }, + "revision": { "$ref": "#/$defs/revision" }, + "checkpointRevision": { "$ref": "#/$defs/revision" }, + "suite": { + "type": "object", + "required": ["id", "manifestPath", "manifestDigest"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "manifestPath": { "type": "string", "minLength": 1 }, + "manifestDigest": { "$ref": "#/$defs/sha256" } + } + }, + "runner": { + "type": "object", + "required": ["id", "runtime"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "runtime": { "type": "string", "minLength": 1 } + } + }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "checkpoints": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, + "commands": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/commandResult" } }, + "acceptance": { "$ref": "#/$defs/acceptanceResult" }, + "validators": { "type": "array", "minItems": 1, "maxItems": 50, "items": { "$ref": "#/$defs/validatorResult" } }, + "decision": { "$ref": "#/$defs/decision" }, + "startedAt": { "type": "string", "format": "date-time" }, + "completedAt": { "type": "string", "format": "date-time" }, + "integrity": { + "type": "object", + "required": ["algorithm", "digest"], + "additionalProperties": false, + "properties": { + "algorithm": { "const": "sha256" }, + "digest": { "$ref": "#/$defs/sha256" } + } + } + }, + "$defs": { + "oid": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "validatorType": { "enum": ["static", "schema", "semantic", "workflow", "visual", "operational", "security"] }, + "revision": { + "type": "object", + "required": ["baseCommit", "mergeBase", "headCommit"], + "additionalProperties": false, + "properties": { + "baseCommit": { "$ref": "#/$defs/oid" }, + "mergeBase": { "$ref": "#/$defs/oid" }, + "headCommit": { "$ref": "#/$defs/oid" } + } + }, + "output": { + "type": "object", + "required": ["bytes", "digest", "tail", "truncated"], + "additionalProperties": false, + "properties": { + "bytes": { "type": "integer", "minimum": 0 }, + "digest": { "$ref": "#/$defs/sha256" }, + "tail": { "type": "string" }, + "truncated": { "type": "boolean" } + } + }, + "commandResult": { + "type": "object", + "required": ["id", "argv", "cwd", "required", "status", "exitCode", "signal", "durationMs", "stdout", "stderr", "startedAt", "completedAt", "error"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "argv": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "cwd": { "type": "string", "minLength": 1 }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "error", "timed_out", "skipped"] }, + "exitCode": { "type": ["integer", "null"] }, + "signal": { "type": ["string", "null"] }, + "durationMs": { "type": "integer", "minimum": 0 }, + "stdout": { "$ref": "#/$defs/output" }, + "stderr": { "$ref": "#/$defs/output" }, + "startedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "completedAt": { "oneOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "error": { "type": ["string", "null"] } + } + }, + "acceptanceResult": { + "type": "object", + "required": ["id", "source", "risk", "digest", "requiredValidatorTypes"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "risk": { "enum": ["low", "medium", "high", "critical"] }, + "digest": { "$ref": "#/$defs/sha256" }, + "requiredValidatorTypes": { "type": "array", "minItems": 1, "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/validatorType" } } + } + }, + "validatorResult": { + "type": "object", + "required": ["id", "type", "required", "status", "evidence", "reasons"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "$ref": "#/$defs/validatorType" }, + "required": { "type": "boolean" }, + "status": { "enum": ["passed", "failed", "blocked", "skipped"] }, + "evidence": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "required": ["path", "digest", "report"], + "additionalProperties": false, + "properties": { + "path": { "type": "string", "minLength": 1 }, + "digest": { "$ref": "#/$defs/sha256" }, + "report": { "$ref": "#/$defs/evidenceReport" } + } + } + ] + }, + "reasons": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } } + } + }, + "evidenceReport": { + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { "type": "array", "maxItems": 100, "items": { "$ref": "#/$defs/metric" } }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { "type": "array", "maxItems": 50, "items": { "$ref": "#/$defs/artifact" } } + } + }, + "metric": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + }, + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "$ref": "#/$defs/sha256" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + }, + "decision": { + "type": "object", + "required": ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], + "additionalProperties": false, + "properties": { + "status": { "enum": ["passed", "failed", "blocked"] }, + "reasons": { "type": "array", "maxItems": 6400, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 2000 } }, + "totalCostUsd": { "type": "number", "minimum": 0 }, + "costTelemetryComplete": { "type": "boolean" } + } + } + } +} diff --git a/schemas/validator-evidence.v0.1.schema.json b/schemas/validator-evidence.v0.1.schema.json new file mode 100644 index 0000000..14fec11 --- /dev/null +++ b/schemas/validator-evidence.v0.1.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:tabellio:schema:validator-evidence:v0.1", + "title": "Tabellio Validator Evidence", + "type": "object", + "required": ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "tabellio-validator-evidence/v0.1" }, + "validatorId": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "blocked"] }, + "summary": { "type": "string", "minLength": 1, "maxLength": 2000 }, + "metrics": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "required": ["name", "value", "unit"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 } + } + } + }, + "cost": { "$ref": "#/$defs/cost" }, + "artifacts": { + "type": "array", + "maxItems": 50, + "items": { "$ref": "#/$defs/artifact" } + } + }, + "$defs": { + "cost": { + "type": "object", + "required": ["telemetry", "usd", "modelCalls", "toolCalls"], + "additionalProperties": false, + "properties": { + "telemetry": { "enum": ["available", "unavailable", "not_applicable"] }, + "usd": { "type": ["number", "null"], "minimum": 0 }, + "modelCalls": { "type": ["integer", "null"], "minimum": 0 }, + "toolCalls": { "type": ["integer", "null"], "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "telemetry": { "const": "available" } }, "required": ["telemetry"] }, + "then": { + "properties": { + "usd": { "type": "number" }, + "modelCalls": { "type": "integer" }, + "toolCalls": { "type": "integer" } + } + }, + "else": { + "properties": { + "usd": { "type": "null" }, + "modelCalls": { "type": "null" }, + "toolCalls": { "type": "null" } + } + } + } + ] + }, + "artifact": { + "type": "object", + "required": ["name", "uri", "digest", "mediaType", "bytes"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "format": "uri", "not": { "pattern": "^file:" } }, + "digest": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "mediaType": { "type": "string", "minLength": 1 }, + "bytes": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/scripts/check-tabellio-validation.mjs b/scripts/check-tabellio-validation.mjs index 228bc90..be3f1ae 100755 --- a/scripts/check-tabellio-validation.mjs +++ b/scripts/check-tabellio-validation.mjs @@ -3,19 +3,24 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; -import { validateValidationManifest, validateValidationResult } from "./lib/validation-runner.mjs"; +import { + validateValidationManifest, + validateValidationResult, + validateValidatorEvidence, +} from "./lib/validation-runner.mjs"; const type = process.argv[2]; const path = process.argv[3]; const blockers = []; let value = null; try { - if (!path || process.argv.length !== 4 || !["--manifest", "--result"].includes(type)) { - throw new Error("Usage: check-tabellio-validation (--manifest|--result) ."); + if (!path || process.argv.length !== 4 || !["--manifest", "--result", "--evidence"].includes(type)) { + throw new Error("Usage: check-tabellio-validation (--manifest|--result|--evidence) ."); } value = JSON.parse(await readFile(resolve(path), "utf8")); if (type === "--manifest") validateValidationManifest(value); - else validateValidationResult(value); + else if (type === "--result") validateValidationResult(value); + else validateValidatorEvidence(value); } catch (error) { blockers.push(error instanceof Error ? error.message : String(error)); } @@ -23,12 +28,14 @@ const result = { ok: blockers.length === 0, status: blockers.length === 0 ? "validation_contract_ready" : "blocked", checkedAt: new Date().toISOString(), - type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : null, + type: type === "--manifest" ? "manifest" : type === "--result" ? "result" : type === "--evidence" ? "evidence" : null, path, summary: value ? { schemaVersion: value.schemaVersion, - id: value.id ?? value.runId ?? null, + id: value.id ?? value.runId ?? value.validatorId ?? null, commandCount: Array.isArray(value.commands) ? value.commands.length : null, + validatorCount: Array.isArray(value.validators) ? value.validators.length : null, + metricCount: Array.isArray(value.metrics) ? value.metrics.length : null, } : null, blockers, }; diff --git a/scripts/lib/validation-runner.mjs b/scripts/lib/validation-runner.mjs index 1986498..7977dfa 100644 --- a/scripts/lib/validation-runner.mjs +++ b/scripts/lib/validation-runner.mjs @@ -1,16 +1,21 @@ import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { mkdir, rm, stat } from "node:fs/promises"; +import { chmod, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve } from "node:path"; import { LedgerConflictError } from "./git-json-ledger.mjs"; import { runGit } from "./git-process.mjs"; import { digestObject } from "./stack-operation.mjs"; -const VALIDATION_MANIFEST_SCHEMA_VERSION = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V1 = "tabellio-validation/v0.1"; +const VALIDATION_MANIFEST_SCHEMA_VERSION_V2 = "tabellio-validation/v0.2"; const VALIDATION_RESULT_SCHEMA_VERSION_V1 = "tabellio-validation-result/v0.1"; const VALIDATION_RESULT_SCHEMA_VERSION_V2 = "tabellio-validation-result/v0.2"; +const VALIDATION_RESULT_SCHEMA_VERSION_V3 = "tabellio-validation-result/v0.3"; +const VALIDATOR_EVIDENCE_SCHEMA_VERSION = "tabellio-validator-evidence/v0.1"; +const VALIDATOR_TYPES = ["static", "schema", "semantic", "workflow", "visual", "operational", "security"]; const MAX_OUTPUT_TAIL_BYTES = 16 * 1024; +const MAX_EVIDENCE_BYTES = 1024 * 1024; export class ValidationRunner { constructor({ store, ledger }) { @@ -55,50 +60,38 @@ export class ValidationRunner { await mkdir(workspaceRoot, { recursive: true }); if (await stat(workspace).catch(() => null)) throw new Error(`Validation workspace already exists: ${workspace}.`); const startedAt = now.toISOString(); - const commands = []; + const definitions = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V1 + ? manifest.commands + : manifest.validators; + let execution = null; try { await runGit({ args: ["worktree", "add", "--detach", workspace, revision.headCommit], cwd: this.store.repoPath }); await mkdir(resolve(home, "tmp"), { recursive: true }); - let stopped = false; - for (const command of manifest.commands) { - if (stopped) { - commands.push(skippedCommand(command)); - continue; - } - const result = await runValidationCommand(command, workspace, home); - commands.push(result); - if (manifest.failFast && command.required && result.status !== "passed") stopped = true; - } + execution = await runValidationDefinitions({ manifest, definitions, workspace, home }); } finally { await runGit({ args: ["worktree", "remove", "--force", workspace], cwd: this.store.repoPath, acceptableExitCodes: [0, 128], }).catch(() => {}); - await rm(workspace, { recursive: true, force: true }); - await rm(home, { recursive: true, force: true }); + await removeGeneratedTree(workspace); + await removeGeneratedTree(home); } const completedAt = new Date().toISOString(); - const requiredFailed = commands.some((command, index) => manifest.commands[index].required && command.status !== "passed"); - const result = { - schemaVersion: VALIDATION_RESULT_SCHEMA_VERSION_V2, + const result = buildValidationResult({ + manifest, + definitions, + execution, runId, - repository: { id: repositoryId }, + repositoryId, revision, checkpointRevision, - suite: { - id: manifest.id, - manifestPath, - manifestDigest: digestObject(manifest), - }, - runner: { id: runnerId, runtime: `node-${process.version}` }, - status: requiredFailed ? "failed" : "passed", + manifestPath, + runnerId, checkpoints, - commands, startedAt, completedAt, - integrity: { algorithm: "sha256", digest: "0".repeat(64) }, - }; + }); result.integrity.digest = validationResultDigest(result); validateValidationResult(result); const path = validationPath(revision.headCommit, runId); @@ -107,6 +100,121 @@ export class ValidationRunner { } } +async function removeGeneratedTree(path) { + try { + await rm(path, { recursive: true, force: true }); + } catch (error) { + if (!error || !["EACCES", "EPERM"].includes(error.code)) throw error; + await makeTreeOwnerWritable(path); + await rm(path, { recursive: true, force: true }); + } +} + +async function makeTreeOwnerWritable(path) { + const info = await lstat(path).catch((error) => { + if (error?.code === "ENOENT") return null; + throw error; + }); + if (!info) return; + if (info.isSymbolicLink()) return; + if (!info.isDirectory()) { + await chmod(path, info.mode | 0o600).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + return; + } + await chmod(path, info.mode | 0o700).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + const entries = await readdir(path).catch((error) => { + if (error?.code === "ENOENT") return []; + throw error; + }); + for (const entry of entries) { + await makeTreeOwnerWritable(resolve(path, entry)); + } +} + +async function runValidationDefinitions({ manifest, definitions, workspace, home }) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const commands = []; + const validators = []; + let stopped = false; + for (const definition of definitions) { + if (stopped) { + commands.push(skippedCommand(definition)); + if (typed) validators.push(skippedValidator(definition)); + continue; + } + const commandResult = await runValidationCommand(definition, workspace, home); + commands.push(commandResult); + const gateStatus = typed + ? await appendValidatorResult(validators, definition, commandResult, workspace) + : commandResult.status; + stopped = manifest.failFast && definition.required && gateStatus !== "passed"; + } + return { commands, validators }; +} + +async function appendValidatorResult(validators, definition, commandResult, workspace) { + const result = await evaluateValidator(definition, commandResult, workspace); + validators.push(result); + return result.status; +} + +function buildValidationResult({ + manifest, + definitions, + execution, + runId, + repositoryId, + revision, + checkpointRevision, + manifestPath, + runnerId, + checkpoints, + startedAt, + completedAt, +}) { + const typed = manifest.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2; + const decision = typed ? validationDecision(execution.validators) : null; + const requiredFailed = execution.commands.some((command, index) => definitions[index].required && command.status !== "passed"); + const result = { + schemaVersion: typed ? VALIDATION_RESULT_SCHEMA_VERSION_V3 : VALIDATION_RESULT_SCHEMA_VERSION_V2, + runId, + repository: { id: repositoryId }, + revision, + checkpointRevision, + suite: { + id: manifest.id, + manifestPath, + manifestDigest: digestObject(manifest), + }, + runner: { id: runnerId, runtime: `node-${process.version}` }, + status: decision?.status ?? (requiredFailed ? "failed" : "passed"), + checkpoints, + commands: execution.commands, + startedAt, + completedAt, + integrity: { algorithm: "sha256", digest: "0".repeat(64) }, + }; + if (!typed) return result; + result.acceptance = acceptanceResult(manifest.acceptance); + result.validators = execution.validators; + result.decision = decision; + return result; +} + +function acceptanceResult(acceptance) { + return { + id: acceptance.id, + source: acceptance.source, + risk: acceptance.risk, + digest: digestObject(acceptance), + requiredValidatorTypes: [...acceptance.requiredValidatorTypes], + }; +} + export async function latestValidationResult(ledger, commit, repositoryId = null) { oid(commit, "commit"); if (repositoryId !== null) requiredString(repositoryId, "repositoryId"); @@ -126,11 +234,25 @@ export async function latestValidationResult(ledger, commit, repositoryId = null export function validateValidationManifest(value) { object(value, "validation manifest"); - exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); - equals(value.schemaVersion, VALIDATION_MANIFEST_SCHEMA_VERSION, "validation manifest.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_MANIFEST_SCHEMA_VERSION_V1, VALIDATION_MANIFEST_SCHEMA_VERSION_V2], + "validation manifest.schemaVersion", + ); requiredString(value.id, "validation manifest.id"); boolean(value.failFast, "validation manifest.failFast"); boolean(value.requireEntireCheckpoint, "validation manifest.requireEntireCheckpoint"); + if (value.schemaVersion === VALIDATION_MANIFEST_SCHEMA_VERSION_V2) { + exactKeys( + value, + ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "acceptance", "validators"], + "validation manifest", + ); + validateAcceptanceContract(value.acceptance); + validateValidators(value.validators, value.acceptance.requiredValidatorTypes); + return value; + } + exactKeys(value, ["schemaVersion", "id", "failFast", "requireEntireCheckpoint", "commands"], "validation manifest"); if (!Array.isArray(value.commands) || value.commands.length === 0 || value.commands.length > 50) { throw new Error("validation manifest.commands must contain 1 to 50 commands."); } @@ -155,7 +277,11 @@ export function validateValidationManifest(value) { export function validateValidationResult(value) { object(value, "validation result"); - member(value.schemaVersion, [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2], "validation result.schemaVersion"); + member( + value.schemaVersion, + [VALIDATION_RESULT_SCHEMA_VERSION_V1, VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3], + "validation result.schemaVersion", + ); exactKeys(value, validationResultKeys(value.schemaVersion), "validation result"); requiredString(value.runId, "validation result.runId"); object(value.repository, "validation result.repository"); @@ -172,7 +298,11 @@ export function validateValidationResult(value) { exactKeys(value.runner, ["id", "runtime"], "validation result.runner"); requiredString(value.runner.id, "validation result.runner.id"); requiredString(value.runner.runtime, "validation result.runner.runtime"); - member(value.status, ["passed", "failed"], "validation result.status"); + member( + value.status, + value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3 ? ["passed", "failed", "blocked"] : ["passed", "failed"], + "validation result.status", + ); stringArray(value.checkpoints, "validation result.checkpoints"); if (!Array.isArray(value.commands) || value.commands.length === 0) throw new Error("validation result.commands must be a non-empty array."); value.commands.forEach((command, index) => validateCommandResult(command, `validation result.commands[${index}]`)); @@ -183,6 +313,21 @@ export function validateValidationResult(value) { equals(value.integrity.algorithm, "sha256", "validation result.integrity.algorithm"); sha256(value.integrity.digest, "validation result.integrity.digest"); if (validationResultDigest(value) !== value.integrity.digest) throw new Error("validation result integrity digest does not match."); + if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + validateAcceptanceResult(value.acceptance); + if (!Array.isArray(value.validators) || value.validators.length === 0) { + throw new Error("validation result.validators must be a non-empty array."); + } + value.validators.forEach((validator, index) => validateValidatorResult(validator, `validation result.validators[${index}]`)); + const commandIds = value.commands.map((command) => command.id); + const validatorIds = value.validators.map((validator) => validator.id); + if (JSON.stringify(commandIds) !== JSON.stringify(validatorIds)) { + throw new Error("validation result validators must align with command ids and order."); + } + validateValidationDecision(value.decision, value.validators); + if (value.status !== value.decision.status) throw new Error("validation result status does not match decision status."); + return value; + } const expectedStatus = value.commands.some((command) => command.required && command.status !== "passed") ? "failed" : "passed"; if (value.status !== expectedStatus) throw new Error("validation result status does not match required command results."); return value; @@ -190,11 +335,14 @@ export function validateValidationResult(value) { function validationResultKeys(schemaVersion) { const keys = ["schemaVersion", "runId", "repository", "revision", "suite", "runner", "status", "checkpoints", "commands", "startedAt", "completedAt", "integrity"]; + if (schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V3) { + return [...keys, "checkpointRevision", "acceptance", "validators", "decision"]; + } return schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2 ? [...keys, "checkpointRevision"] : keys; } function validateCheckpointRevision(value) { - if (value.schemaVersion === VALIDATION_RESULT_SCHEMA_VERSION_V2) { + if ([VALIDATION_RESULT_SCHEMA_VERSION_V2, VALIDATION_RESULT_SCHEMA_VERSION_V3].includes(value.schemaVersion)) { validateRevision(value.checkpointRevision, "validation result.checkpointRevision"); } } @@ -207,9 +355,220 @@ function validateRevision(value, path) { oid(value.headCommit, `${path}.headCommit`); } +function validateAcceptanceContract(value) { + object(value, "validation manifest.acceptance"); + exactKeys( + value, + ["id", "source", "risk", "outcomes", "invariants", "forbiddenOutcomes", "requiredValidatorTypes"], + "validation manifest.acceptance", + ); + requiredString(value.id, "validation manifest.acceptance.id"); + requiredString(value.source, "validation manifest.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation manifest.acceptance.risk"); + boundedUniqueStrings(value.outcomes, "validation manifest.acceptance.outcomes", { minimum: 1, maximum: 50 }); + boundedUniqueStrings(value.invariants, "validation manifest.acceptance.invariants", { minimum: 0, maximum: 50 }); + boundedUniqueStrings(value.forbiddenOutcomes, "validation manifest.acceptance.forbiddenOutcomes", { minimum: 0, maximum: 50 }); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation manifest.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidators(value, requiredTypes) { + if (!Array.isArray(value) || value.length === 0 || value.length > 50) { + throw new Error("validation manifest.validators must contain 1 to 50 validators."); + } + const ids = new Set(); + for (const [index, validator] of value.entries()) { + validateValidatorDefinition(validator, index); + if (ids.has(validator.id)) throw new Error(`validation manifest.validators contains duplicate id ${validator.id}.`); + ids.add(validator.id); + } + for (const type of requiredTypes) { + if (!value.some((validator) => validator.type === type && validator.required)) { + throw new Error(`validation manifest requires a required ${type} validator.`); + } + } +} + +function validateValidatorDefinition(validator, index) { + const path = `validation manifest.validators[${index}]`; + object(validator, path); + exactKeys(validator, ["id", "type", "argv", "cwd", "timeoutMs", "required", "evidence", "policy"], path); + requiredString(validator.id, `${path}.id`); + member(validator.type, VALIDATOR_TYPES, `${path}.type`); + validateValidatorCommand(validator, path); + validateValidatorEvidenceDeclaration(validator, path); + validateValidatorPolicy(validator.policy, `${path}.policy`, validator.evidence !== null); +} + +function validateValidatorCommand(validator, path) { + if (!Array.isArray(validator.argv) || validator.argv.length === 0 || validator.argv.length > 100) { + throw new Error(`${path}.argv must contain 1 to 100 arguments.`); + } + validator.argv.forEach((argument, argumentIndex) => requiredString(argument, `${path}.argv[${argumentIndex}]`)); + validateRelativePath(validator.cwd, `${path}.cwd`, { allowDot: true }); + if (!Number.isInteger(validator.timeoutMs) || validator.timeoutMs < 100 || validator.timeoutMs > 60 * 60 * 1000) { + throw new Error(`${path}.timeoutMs must be between 100 and 3600000.`); + } + boolean(validator.required, `${path}.required`); +} + +function validateValidatorEvidenceDeclaration(validator, path) { + if (validator.evidence === null) { + if (validator.type !== "static") throw new Error(`${path}.evidence is required for ${validator.type} validators.`); + return; + } + object(validator.evidence, `${path}.evidence`); + exactKeys(validator.evidence, ["path"], `${path}.evidence`); + validateRelativePath(validator.evidence.path, `${path}.evidence.path`); +} + +function validateValidatorPolicy(value, path, hasEvidence) { + object(value, path); + exactKeys(value, ["metricThresholds", "maxCostUsd", "requireCostTelemetry"], path); + validateMetricThresholds(value.metricThresholds, `${path}.metricThresholds`); + validateCostPolicy(value, path); + if ((value.metricThresholds.length > 0 || value.requireCostTelemetry || value.maxCostUsd !== null) && !hasEvidence) { + throw new Error(`${path} requires an evidence report.`); + } +} + +function validateMetricThresholds(value, path) { + if (!Array.isArray(value) || value.length > 100) throw new Error(`${path} must contain at most 100 thresholds.`); + const metrics = new Set(); + for (const [index, threshold] of value.entries()) { + const thresholdPath = `${path}[${index}]`; + object(threshold, thresholdPath); + exactKeys(threshold, ["metric", "operator", "value"], thresholdPath); + requiredString(threshold.metric, `${thresholdPath}.metric`); + if (metrics.has(threshold.metric)) throw new Error(`${path} contains duplicate metric ${threshold.metric}.`); + metrics.add(threshold.metric); + member(threshold.operator, ["gte", "lte", "eq"], `${thresholdPath}.operator`); + finiteNumber(threshold.value, `${thresholdPath}.value`); + } +} + +function validateCostPolicy(value, path) { + if (value.maxCostUsd !== null) nonNegativeNumber(value.maxCostUsd, `${path}.maxCostUsd`); + boolean(value.requireCostTelemetry, `${path}.requireCostTelemetry`); + if (value.maxCostUsd !== null && value.requireCostTelemetry !== true) { + throw new Error(`${path}.requireCostTelemetry must be true when maxCostUsd is set.`); + } +} + +export function validateValidatorEvidence(value) { + object(value, "validator evidence"); + exactKeys(value, ["schemaVersion", "validatorId", "status", "summary", "metrics", "cost", "artifacts"], "validator evidence"); + equals(value.schemaVersion, VALIDATOR_EVIDENCE_SCHEMA_VERSION, "validator evidence.schemaVersion"); + requiredString(value.validatorId, "validator evidence.validatorId"); + member(value.status, ["passed", "failed", "blocked"], "validator evidence.status"); + requiredString(value.summary, "validator evidence.summary"); + maxLength(value.summary, 2_000, "validator evidence.summary"); + validateEvidenceMetrics(value.metrics); + validateEvidenceCost(value.cost); + validateEvidenceArtifacts(value.artifacts); + return value; +} + +function validateEvidenceMetrics(value) { + if (!Array.isArray(value) || value.length > 100) throw new Error("validator evidence.metrics must contain at most 100 metrics."); + const metricNames = new Set(); + for (const [index, metric] of value.entries()) { + const path = `validator evidence.metrics[${index}]`; + object(metric, path); + exactKeys(metric, ["name", "value", "unit"], path); + requiredString(metric.name, `${path}.name`); + if (metricNames.has(metric.name)) throw new Error(`validator evidence.metrics contains duplicate name ${metric.name}.`); + metricNames.add(metric.name); + finiteNumber(metric.value, `${path}.value`); + requiredString(metric.unit, `${path}.unit`); + } +} + +function validateEvidenceArtifacts(value) { + if (!Array.isArray(value) || value.length > 50) throw new Error("validator evidence.artifacts must contain at most 50 artifacts."); + for (const [index, artifact] of value.entries()) validateEvidenceArtifact(artifact, `validator evidence.artifacts[${index}]`); +} + +function validateEvidenceCost(value) { + object(value, "validator evidence.cost"); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], "validator evidence.cost"); + member(value.telemetry, ["available", "unavailable", "not_applicable"], "validator evidence.cost.telemetry"); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, "validator evidence.cost.usd"); + nonNegativeInteger(value.modelCalls, "validator evidence.cost.modelCalls"); + nonNegativeInteger(value.toolCalls, "validator evidence.cost.toolCalls"); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error("validator evidence cost values must be null when telemetry is unavailable or not applicable."); + } +} + +function validateEvidenceArtifact(value, path) { + object(value, path); + exactKeys(value, ["name", "uri", "digest", "mediaType", "bytes"], path); + requiredString(value.name, `${path}.name`); + durableUri(value.uri, `${path}.uri`); + sha256(value.digest, `${path}.digest`); + requiredString(value.mediaType, `${path}.mediaType`); + nonNegativeInteger(value.bytes, `${path}.bytes`); +} + +function validateAcceptanceResult(value) { + object(value, "validation result.acceptance"); + exactKeys(value, ["id", "source", "risk", "digest", "requiredValidatorTypes"], "validation result.acceptance"); + requiredString(value.id, "validation result.acceptance.id"); + requiredString(value.source, "validation result.acceptance.source"); + member(value.risk, ["low", "medium", "high", "critical"], "validation result.acceptance.risk"); + sha256(value.digest, "validation result.acceptance.digest"); + enumArray( + value.requiredValidatorTypes, + VALIDATOR_TYPES, + "validation result.acceptance.requiredValidatorTypes", + { minimum: 1, maximum: VALIDATOR_TYPES.length }, + ); +} + +function validateValidatorResult(value, path) { + object(value, path); + exactKeys(value, ["id", "type", "required", "status", "evidence", "reasons"], path); + requiredString(value.id, `${path}.id`); + member(value.type, VALIDATOR_TYPES, `${path}.type`); + boolean(value.required, `${path}.required`); + member(value.status, ["passed", "failed", "blocked", "skipped"], `${path}.status`); + boundedUniqueStrings(value.reasons, `${path}.reasons`, { minimum: 0, maximum: 128 }); + if (value.evidence === null) return; + object(value.evidence, `${path}.evidence`); + exactKeys(value.evidence, ["path", "digest", "report"], `${path}.evidence`); + validateRelativePath(value.evidence.path, `${path}.evidence.path`); + sha256(value.evidence.digest, `${path}.evidence.digest`); + validateValidatorEvidence(value.evidence.report); + if (digestObject(value.evidence.report) !== value.evidence.digest) throw new Error(`${path}.evidence digest does not match report.`); + if (value.evidence.report.validatorId !== value.id) throw new Error(`${path}.evidence validator id does not match.`); +} + +function validateValidationDecision(value, validators) { + object(value, "validation result.decision"); + exactKeys(value, ["status", "reasons", "totalCostUsd", "costTelemetryComplete"], "validation result.decision"); + member(value.status, ["passed", "failed", "blocked"], "validation result.decision.status"); + boundedUniqueStrings(value.reasons, "validation result.decision.reasons", { minimum: 0, maximum: 6_400 }); + nonNegativeNumber(value.totalCostUsd, "validation result.decision.totalCostUsd"); + boolean(value.costTelemetryComplete, "validation result.decision.costTelemetryComplete"); + const expected = validationDecision(validators); + if ( + value.status !== expected.status + || JSON.stringify(value.reasons) !== JSON.stringify(expected.reasons) + || value.totalCostUsd !== expected.totalCostUsd + || value.costTelemetryComplete !== expected.costTelemetryComplete + ) { + throw new Error("validation result decision does not match validator results."); + } +} + async function runValidationCommand(command, workspace, home) { - const cwd = resolve(workspace, command.cwd); - if (relative(workspace, cwd).startsWith("..") || isAbsolute(relative(workspace, cwd))) throw new Error(`Command ${command.id} cwd escapes validation workspace.`); + const cwd = resolveWorkspacePath(workspace, command.cwd, `Command ${command.id} cwd`); const startedAt = new Date(); const execution = await spawnCaptured(command.argv, { cwd, @@ -234,6 +593,156 @@ async function runValidationCommand(command, workspace, home) { }; } +async function evaluateValidator(validator, commandResult, workspace) { + const base = validatorResultBase(validator); + const commandGate = validatorCommandGate(validator, commandResult.status); + if (commandGate) return { ...base, ...commandGate }; + if (validator.evidence === null) return base; + + let evidence; + try { + evidence = await loadValidatorEvidence(workspace, validator.evidence.path, validator.id); + } catch (error) { + return { + ...base, + status: "blocked", + reasons: blockedReasons(validator, `evidence_invalid:${boundedReason(error instanceof Error ? error.message : String(error))}`), + }; + } + const evaluation = evaluateEvidencePolicy(validator, evidence.report); + return { ...base, ...evaluation, evidence }; +} + +function validatorResultBase(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "passed", + evidence: null, + reasons: [], + }; +} + +function validatorCommandGate(validator, status) { + if (status === "timed_out") return { status: "blocked", reasons: blockedReasons(validator, "command_timed_out") }; + if (status === "error") return { status: "blocked", reasons: blockedReasons(validator, "command_error") }; + if (status === "skipped") return { status: "skipped", reasons: ["fail_fast"] }; + if (status !== "failed") return null; + return validator.evidence === null + ? { status: "failed", reasons: ["command_failed"] } + : { status: "blocked", reasons: blockedReasons(validator, "validator_command_failed") }; +} + +function evaluateEvidencePolicy(validator, report) { + const evaluations = [ + reportedEvidenceStatus(report.status), + evaluateMetricPolicy(report.metrics, validator.policy.metricThresholds), + evaluateCostPolicy(report.cost, validator.policy), + ]; + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + const reasons = [...new Set(evaluations.flatMap((evaluation) => evaluation.reasons))].sort(); + return { status, reasons }; +} + +function reportedEvidenceStatus(status) { + if (status === "failed") return { status, reasons: ["evidence_reported_failed"] }; + if (status === "blocked") return { status, reasons: ["evidence_reported_blocked"] }; + return { status: "passed", reasons: [] }; +} + +function evaluateMetricPolicy(metrics, thresholds) { + const evaluations = thresholds.map((threshold) => evaluateMetricThreshold(metrics, threshold)); + const status = aggregateStatus(evaluations.map((evaluation) => evaluation.status)); + return { status, reasons: evaluations.flatMap((evaluation) => evaluation.reasons) }; +} + +function evaluateMetricThreshold(metrics, threshold) { + const metric = metrics.find((candidate) => candidate.name === threshold.metric); + if (!metric) return { status: "blocked", reasons: [`metric_missing:${threshold.metric}`] }; + if (thresholdPassed(metric.value, threshold.operator, threshold.value)) return { status: "passed", reasons: [] }; + return { + status: "failed", + reasons: [`threshold_not_met:${threshold.metric}:${threshold.operator}:${threshold.value}`], + }; +} + +function evaluateCostPolicy(cost, policy) { + if (policy.requireCostTelemetry && cost.telemetry !== "available") { + return { status: "blocked", reasons: ["cost_telemetry_unavailable"] }; + } + if (policy.maxCostUsd !== null && cost.telemetry === "available" && cost.usd > policy.maxCostUsd) { + return { status: "failed", reasons: [`cost_budget_exceeded:${policy.maxCostUsd}`] }; + } + return { status: "passed", reasons: [] }; +} + +async function loadValidatorEvidence(workspace, evidencePath, validatorId) { + const path = resolveWorkspacePath(workspace, evidencePath, `Validator ${validatorId} evidence`); + const metadata = await stat(path).catch(() => { + throw new Error("file is missing or unreadable"); + }); + if (!metadata.isFile()) throw new Error("must be a regular file"); + if (metadata.size > MAX_EVIDENCE_BYTES) throw new Error(`exceeds ${MAX_EVIDENCE_BYTES} bytes`); + const source = await readFile(path).catch(() => { + throw new Error("file is unreadable"); + }); + let report; + try { + report = JSON.parse(source.toString("utf8")); + } catch { + throw new Error("is not valid JSON"); + } + validateValidatorEvidence(report); + if (report.validatorId !== validatorId) throw new Error(`validatorId must be ${validatorId}`); + return { + path: evidencePath, + digest: digestObject(report), + report, + }; +} + +function thresholdPassed(actual, operator, expected) { + if (operator === "gte") return actual >= expected; + if (operator === "lte") return actual <= expected; + return actual === expected; +} + +function blockedReasons(validator, reason) { + const reasons = [reason]; + if (validator.policy.requireCostTelemetry) reasons.push("cost_telemetry_unavailable"); + return [...new Set(reasons)].sort(); +} + +function validationDecision(validators) { + const required = validators.filter((validator) => validator.required); + const status = aggregateStatus(required.map((validator) => validator.status)); + const reasons = [...new Set(required.flatMap((validator) => validator.reasons.map((reason) => `${validator.id}:${reason}`)))].sort(); + const totalCostUsd = Math.round(validators.reduce((total, validator) => { + const cost = validator.evidence?.report.cost; + return total + (cost?.telemetry === "available" ? cost.usd : 0); + }, 0) * 100_000_000) / 100_000_000; + const costTelemetryComplete = !required.some((validator) => validator.reasons.includes("cost_telemetry_unavailable")); + return { status, reasons, totalCostUsd, costTelemetryComplete }; +} + +function aggregateStatus(statuses) { + if (statuses.includes("blocked")) return "blocked"; + if (statuses.some((status) => status !== "passed")) return "failed"; + return "passed"; +} + +function skippedValidator(validator) { + return { + id: validator.id, + type: validator.type, + required: validator.required, + status: "skipped", + evidence: null, + reasons: validator.policy.requireCostTelemetry ? ["cost_telemetry_unavailable", "fail_fast"] : ["fail_fast"], + }; +} + function skippedCommand(command) { return { id: command.id, @@ -434,6 +943,64 @@ function stringArray(value, path) { if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); } +function boundedUniqueStrings(value, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => { + requiredString(item, `${path}[${index}]`); + maxLength(item, 2_000, `${path}[${index}]`); + }); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function enumArray(value, allowed, path, { minimum, maximum }) { + if (!Array.isArray(value) || value.length < minimum || value.length > maximum) { + throw new Error(`${path} must contain ${minimum} to ${maximum} values.`); + } + value.forEach((item, index) => member(item, allowed, `${path}[${index}]`)); + if (new Set(value).size !== value.length) throw new Error(`${path} must contain unique values.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be a finite number.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function maxLength(value, maximum, path) { + if (typeof value === "string" && value.length > maximum) throw new Error(`${path} must contain at most ${maximum} characters.`); +} + +function durableUri(value, path) { + requiredString(value, path); + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${path} must be an absolute URI.`); + } + if (parsed.protocol === "file:") throw new Error(`${path} must not use an ephemeral file URI.`); +} + +function boundedReason(value) { + return value.replace(/[\0\r\n]+/g, " ").slice(0, 500); +} + +function resolveWorkspacePath(workspace, path, label) { + const resolved = resolve(workspace, path); + const relation = relative(workspace, resolved); + if (relation.startsWith("..") || isAbsolute(relation)) throw new Error(`${label} escapes validation workspace.`); + return resolved; +} + function validateRelativePath(value, path, { allowDot = false } = {}) { requiredString(value, path); if ((allowDot && value === ".")) return; diff --git a/scripts/tabellio-validate.mjs b/scripts/tabellio-validate.mjs index 595d0ee..eeac5a2 100755 --- a/scripts/tabellio-validate.mjs +++ b/scripts/tabellio-validate.mjs @@ -15,7 +15,7 @@ try { repoPath: store.repoPath, ref: options.ledgerRef ?? "refs/tabellio/validations", }); - if (options.command === "run") { + if (["run", "gate"].includes(options.command)) { const runner = new ValidationRunner({ store, ledger }); const result = await runner.run({ repositoryId: await repositoryIdentity(store, options.repoId), @@ -25,6 +25,7 @@ try { runnerId: options.runnerId ?? "local", }); console.log(JSON.stringify({ ok: true, ...result }, null, 2)); + if (options.command === "gate" && result.result.status !== "passed") process.exitCode = 1; } else { const commit = await store.resolveRef(options.commit ?? "HEAD"); const result = await latestValidationResult(ledger, commit); @@ -37,7 +38,7 @@ try { function parseArgs(args) { const command = args[0]; - if (!["run", "latest"].includes(command)) throw new Error("Expected command: run or latest."); + if (!["run", "gate", "latest"].includes(command)) throw new Error("Expected command: run, gate, or latest."); const values = {}; for (let index = 1; index < args.length; index += 2) { const flag = args[index]; @@ -47,7 +48,7 @@ function parseArgs(args) { if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); values[key] = value; } - const allowed = command === "run" + const allowed = ["run", "gate"].includes(command) ? ["repo", "repoId", "commit", "base", "manifest", "runnerId", "ledgerRef"] : ["repo", "commit", "ledgerRef"]; for (const key of Object.keys(values)) if (!allowed.includes(key)) throw new Error(`Unsupported option: --${key}.`); diff --git a/scripts/tabellio-validator.mjs b/scripts/tabellio-validator.mjs new file mode 100644 index 0000000..850126d --- /dev/null +++ b/scripts/tabellio-validator.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; + +const MAX_CAPTURE_BYTES = 1024 * 1024; + +try { + const options = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const configPath = containedPath(root, options.config ?? ".tabellio/validators.json", "config"); + const config = validateConfig(JSON.parse(await readFile(configPath, "utf8"))); + const profile = config.profiles[options.profile]; + if (!profile) throw new Error(`Unknown validator profile: ${options.profile}.`); + + const execution = await runProfile(profile, root); + if (!options.out) { + process.exitCode = execution.status === "passed" ? 0 : execution.status === "failed" ? 1 : 2; + } else { + if (!options.validatorId) throw new Error("--validator-id is required with --out."); + const outPath = containedPath(root, options.out, "out"); + const evidence = buildEvidence(options.validatorId, profile, execution); + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + console.log(JSON.stringify({ ok: true, profile: options.profile, out: relative(root, outPath), status: evidence.status })); + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; +} + +function parseArgs(args) { + const values = {}; + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith("--") || value === undefined) throw new Error(`Expected a value after ${flag ?? "arguments"}.`); + const key = flag.slice(2).replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase()); + if (!["config", "profile", "validatorId", "out"].includes(key)) throw new Error(`Unsupported option: ${flag}.`); + if (Object.hasOwn(values, key)) throw new Error(`Duplicate option: ${flag}.`); + values[key] = value; + } + if (!values.profile) throw new Error("--profile is required."); + return values; +} + +function validateConfig(value) { + object(value, "adapter config"); + exactKeys(value, ["schemaVersion", "profiles"], "adapter config"); + if (value.schemaVersion !== "tabellio-adapter/v0.1") throw new Error("adapter config schemaVersion must be tabellio-adapter/v0.1."); + object(value.profiles, "adapter config.profiles"); + const entries = Object.entries(value.profiles); + if (entries.length < 1 || entries.length > 20) throw new Error("adapter config.profiles must contain 1 to 20 profiles."); + for (const [id, profile] of entries) validateProfile(id, profile); + return value; +} + +function validateProfile(id, value) { + requiredString(id, "profile id"); + object(value, `profile ${id}`); + exactKeys(value, ["commands", "metrics", "cost", "summary"], `profile ${id}`); + if (!Array.isArray(value.commands) || value.commands.length < 1 || value.commands.length > 20) { + throw new Error(`profile ${id}.commands must contain 1 to 20 commands.`); + } + for (const [index, argv] of value.commands.entries()) validateArgv(argv, `profile ${id}.commands[${index}]`); + if (!Array.isArray(value.metrics) || value.metrics.length > 100) throw new Error(`profile ${id}.metrics must be an array.`); + for (const [index, metric] of value.metrics.entries()) validateMetric(metric, `profile ${id}.metrics[${index}]`); + validateCost(value.cost, `profile ${id}.cost`); + requiredString(value.summary, `profile ${id}.summary`); +} + +function validateArgv(argv, path) { + if (!Array.isArray(argv) || argv.length < 1 || argv.length > 100) throw new Error(`${path} must contain 1 to 100 arguments.`); + for (const [index, argument] of argv.entries()) requiredString(argument, `${path}[${index}]`); +} + +function validateMetric(value, path) { + object(value, path); + const keys = Object.keys(value); + const extracted = keys.includes("pattern"); + exactKeys(value, extracted ? ["name", "unit", "pattern"] : ["name", "unit", "passValue", "failValue"], path); + requiredString(value.name, `${path}.name`); + requiredString(value.unit, `${path}.unit`); + if (extracted) { + requiredString(value.pattern, `${path}.pattern`); + const expression = new RegExp(value.pattern); + if (expression.source.length > 500) throw new Error(`${path}.pattern is too long.`); + } else { + finiteNumber(value.passValue, `${path}.passValue`); + finiteNumber(value.failValue, `${path}.failValue`); + } +} + +function validateCost(value, path) { + object(value, path); + exactKeys(value, ["telemetry", "usd", "modelCalls", "toolCalls"], path); + if (!['available', 'unavailable', 'not_applicable'].includes(value.telemetry)) throw new Error(`${path}.telemetry is invalid.`); + if (value.telemetry === "available") { + nonNegativeNumber(value.usd, `${path}.usd`); + nonNegativeInteger(value.modelCalls, `${path}.modelCalls`); + nonNegativeInteger(value.toolCalls, `${path}.toolCalls`); + } else if (value.usd !== null || value.modelCalls !== null || value.toolCalls !== null) { + throw new Error(`${path} unavailable or not_applicable values must be null.`); + } +} + +async function runProfile(profile, cwd) { + const outputs = []; + for (const argv of profile.commands) { + const result = await runCommand(argv, cwd); + outputs.push(result.output); + if (result.spawnError) return { status: "blocked", outputs, reason: `command unavailable: ${argv[0]}` }; + if (result.exitCode !== 0) return { status: "failed", outputs, reason: `command failed: ${argv.join(" ")}` }; + } + return { status: "passed", outputs, reason: "all declared commands passed" }; +} + +function runCommand(argv, cwd) { + return new Promise((resolvePromise) => { + let captured = ""; + let capturedBytes = 0; + let spawnError = null; + const child = spawn(argv[0], argv.slice(1), { + cwd, + env: { ...process.env, CI: "1", TABELLIO_VALIDATION: "1", NO_COLOR: "1" }, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const forward = (target) => (chunk) => { + target.write(chunk); + if (capturedBytes >= MAX_CAPTURE_BYTES) return; + const remaining = MAX_CAPTURE_BYTES - capturedBytes; + const slice = chunk.subarray(0, remaining); + captured += slice.toString("utf8"); + capturedBytes += slice.byteLength; + }; + child.stdout.on("data", forward(process.stdout)); + child.stderr.on("data", forward(process.stderr)); + child.once("error", (error) => { spawnError = error; }); + child.once("close", (exitCode, signal) => resolvePromise({ + exitCode: exitCode ?? (signal ? 1 : 0), + output: captured, + spawnError, + })); + }); +} + +function buildEvidence(validatorId, profile, execution) { + const combinedOutput = execution.outputs.join("\n"); + const metrics = []; + let status = execution.status; + let reason = execution.reason; + for (const metric of profile.metrics) { + if (Object.hasOwn(metric, "pattern")) { + const match = new RegExp(metric.pattern, "m").exec(combinedOutput); + const value = match ? Number(match[1]) : Number.NaN; + if (!Number.isFinite(value)) { + status = "blocked"; + reason = `metric unavailable: ${metric.name}`; + continue; + } + metrics.push({ name: metric.name, value, unit: metric.unit }); + } else { + metrics.push({ + name: metric.name, + value: execution.status === "passed" ? metric.passValue : metric.failValue, + unit: metric.unit, + }); + } + } + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status, + summary: `${profile.summary} ${reason}.`.slice(0, 2000), + metrics, + cost: profile.cost, + artifacts: [], + }; +} + +function containedPath(root, input, label) { + requiredString(input, label); + if (isAbsolute(input)) throw new Error(`${label} must be relative.`); + const target = resolve(root, input); + const rel = relative(root, target); + if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) throw new Error(`${label} must stay inside the repository.`); + return target; +} + +function exactKeys(value, expected, path) { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) throw new Error(`${path} must contain exactly: ${wanted.join(", ")}.`); +} + +function object(value, path) { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object.`); +} + +function requiredString(value, path) { + if (typeof value !== "string" || value === "" || /[\0\r\n]/.test(value)) throw new Error(`${path} must be a non-empty single-line string.`); +} + +function finiteNumber(value, path) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be finite.`); +} + +function nonNegativeNumber(value, path) { + finiteNumber(value, path); + if (value < 0) throw new Error(`${path} must be non-negative.`); +} + +function nonNegativeInteger(value, path) { + if (!Number.isInteger(value) || value < 0) throw new Error(`${path} must be a non-negative integer.`); +} diff --git a/tests/validation-adapter.test.mjs b/tests/validation-adapter.test.mjs new file mode 100644 index 0000000..39502ae --- /dev/null +++ b/tests/validation-adapter.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); + +test("repository adapter emits passed, failed, blocked, and extracted evidence", async (t) => { + const root = await mkdtemp(join(tmpdir(), "tabellio-adapter-")); + t.after(() => rm(root, { recursive: true, force: true })); + const config = { + schemaVersion: "tabellio-adapter/v0.1", + profiles: { + passed: profile([[process.execPath, "-e", "console.log('score=0.95')"]], [ + { name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }, + { name: "score", unit: "ratio", pattern: "score=([0-9.]+)" }, + ]), + failed: profile([[process.execPath, "-e", "process.exit(3)"]]), + blocked: profile([["missing-tabellio-test-command"]]), + }, + }; + await writeFile(join(root, "validators.json"), JSON.stringify(config)); + + const passed = await runAdapter(root, "passed", "passed-validator", "passed.json"); + assert.equal(passed.status, "passed"); + assert.equal(passed.metrics.find((metric) => metric.name === "score").value, 0.95); + + const failed = await runAdapter(root, "failed", "failed-validator", "failed.json"); + assert.equal(failed.status, "failed"); + assert.equal(failed.metrics[0].value, 0); + + const blocked = await runAdapter(root, "blocked", "blocked-validator", "blocked.json"); + assert.equal(blocked.status, "blocked"); + assert.match(blocked.summary, /command unavailable/); +}); + +async function runAdapter(root, profileName, validatorId, out) { + await execFileAsync(process.execPath, [ + new URL("../scripts/tabellio-validator.mjs", import.meta.url).pathname, + "--config", "validators.json", + "--profile", profileName, + "--validator-id", validatorId, + "--out", out, + ], { cwd: root }); + return JSON.parse(await readFile(join(root, out), "utf8")); +} + +function profile(commands, metrics = [{ name: "suite_pass", unit: "ratio", passValue: 1, failValue: 0 }]) { + return { + commands, + metrics, + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + summary: "Deterministic repository adapter completed.", + }; +} diff --git a/tests/validation-cli.test.mjs b/tests/validation-cli.test.mjs new file mode 100644 index 0000000..3fc62d3 --- /dev/null +++ b/tests/validation-cli.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { rm, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import test from "node:test"; + +import { runGit } from "../scripts/lib/git-process.mjs"; +import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; + +const execFileAsync = promisify(execFile); + +test("validation gate exits zero only for a passed exact-head result", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(0)"]), "passing gate"); + const passed = await runGate(fixture.seed, "refs/tabellio/cli-pass"); + assert.equal(passed.exitCode, 0); + assert.equal(passed.output.result.status, "passed"); + + await commitManifest(fixture.seed, manifest([process.execPath, "-e", "process.exit(3)"]), "failing gate"); + const failed = await runGate(fixture.seed, "refs/tabellio/cli-fail"); + assert.equal(failed.exitCode, 1); + assert.equal(failed.output.result.status, "failed"); +}); + +async function commitManifest(repo, value, message) { + await writeFile(`${repo}/tabellio.validation.json`, `${JSON.stringify(value, null, 2)}\n`); + await runGit({ args: ["add", "tabellio.validation.json"], cwd: repo }); + await runGit({ args: ["commit", "-m", message], cwd: repo, env: identityEnv() }); +} + +async function runGate(repo, ledgerRef) { + const args = [ + "scripts/tabellio-validate.mjs", + "gate", + "--repo", repo, + "--repo-id", "example/repository", + "--base", "main", + "--commit", "HEAD", + "--manifest", "tabellio.validation.json", + "--ledger-ref", ledgerRef, + ]; + try { + const result = await execFileAsync(process.execPath, args, { cwd: new URL("..", import.meta.url), encoding: "utf8" }); + return { exitCode: 0, output: JSON.parse(result.stdout) }; + } catch (error) { + return { exitCode: error.code, output: JSON.parse(error.stdout) }; + } +} + +function manifest(argv) { + return { + schemaVersion: "tabellio-validation/v0.1", + id: "cli-gate", + failFast: true, + requireEntireCheckpoint: false, + commands: [ + { + id: "gate-command", + argv, + cwd: ".", + timeoutMs: 30_000, + required: true, + }, + ], + }; +} diff --git a/tests/validation-runner.test.mjs b/tests/validation-runner.test.mjs index dfdc17c..b5f30aa 100644 --- a/tests/validation-runner.test.mjs +++ b/tests/validation-runner.test.mjs @@ -10,6 +10,7 @@ import { ValidationRunner, validateValidationManifest, validateValidationResult, + validateValidatorEvidence, } from "../scripts/lib/validation-runner.mjs"; import { NativeGitStore } from "../scripts/providers/native-git-store.mjs"; import { createFixture, identityEnv } from "./helpers/git-fixture.mjs"; @@ -22,6 +23,7 @@ test("validation runner executes exact committed manifests and stores bounded re await writeFile(manifestPath, JSON.stringify(manifest([ command("tests", [process.execPath, "-e", 'process.stdout.write("x".repeat(20000))']), command("isolated-home", [process.execPath, "-e", 'if (!process.env.HOME.includes("validation-workspaces")) process.exit(2)']), + command("readonly-cache", [process.execPath, "-e", 'const fs=require("node:fs");const p=process.env.HOME+"/go/pkg/mod/example/.github";fs.mkdirSync(p,{recursive:true});fs.chmodSync(p,0o500);fs.chmodSync(process.env.HOME+"/go/pkg/mod/example",0o500)']), ]), null, 2)); await commit(fixture.seed, "Add passing validation", "validation-pass"); const passingHead = await head(fixture.seed); @@ -43,6 +45,7 @@ test("validation runner executes exact committed manifests and stores bounded re assert.equal(passed.result.commands[0].stdout.truncated, true); assert.equal(Buffer.byteLength(passed.result.commands[0].stdout.tail), 16 * 1024); assert.equal(passed.result.commands[1].status, "passed"); + assert.equal(passed.result.commands[2].status, "passed"); assert.equal(validateValidationResult(passed.result), passed.result); assert.deepEqual(await latestValidationResult(ledger, passingHead), passed.result); const otherRepository = await runner.run({ @@ -127,6 +130,209 @@ test("validation runner terminates timed-out commands and skips remaining fail-f assert(Date.now() - started < 3_000); }); +test("typed validators enforce semantic metrics and cost budgets with durable evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const semantic = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.9, unit: "ratio" }], + cost: availableCost(0.11, 5, 12), + }); + const operational = validatorEvidence("cost-guard", { + metrics: [{ name: "redis_commands", value: 100, unit: "commands" }], + cost: availableCost(0, 0, 0), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", semantic), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", operational), + ".tabellio/cost.json", + { + metricThresholds: [{ metric: "redis_commands", operator: "lte", value: 1_000 }], + maxCostUsd: 0.1, + requireCostTelemetry: true, + }, + ), + ], ["semantic", "operational"]), null, 2)); + await commit(fixture.seed, "Add product validation", "product-validation-pass"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + runnerId: "product-validator", + }); + + assert.equal(result.result.schemaVersion, "tabellio-validation-result/v0.3"); + assert.equal(result.result.status, "passed"); + assert.equal(result.result.acceptance.id, "PLANE-101"); + assert.deepEqual(result.result.acceptance.requiredValidatorTypes, ["semantic", "operational"]); + assert.deepEqual(result.result.validators.map(({ id, type, status }) => ({ id, type, status })), [ + { id: "static-checks", type: "static", status: "passed" }, + { id: "semantic-eval", type: "semantic", status: "passed" }, + { id: "cost-guard", type: "operational", status: "passed" }, + ]); + assert.equal(result.result.validators[1].evidence.report.metrics[0].value, 0.9); + assert.equal(result.result.decision.totalCostUsd, 0.11); + assert.equal(result.result.decision.costTelemetryComplete, true); + assert.equal(validateValidationResult(result.result), result.result); +}); + +test("typed validation distinguishes product failure from blocked evidence", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const weakEvidence = validatorEvidence("semantic-eval", { + metrics: [{ name: "strict_pass_rate", value: 0.2, unit: "ratio" }], + cost: availableCost(0.6, 1, 2), + }); + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", weakEvidence), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: 0.5, + requireCostTelemetry: true, + }, + ), + ], ["semantic"]), null, 2)); + await commit(fixture.seed, "Add failing semantic validation", "product-validation-fail"); + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const runner = new ValidationRunner({ store, ledger }); + const failed = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(failed.result.status, "failed"); + assert.equal(failed.result.validators[0].status, "failed"); + assert.deepEqual(failed.result.validators[0].reasons, [ + "cost_budget_exceeded:0.5", + "threshold_not_met:strict_pass_rate:gte:0.8", + ]); + + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(productManifest([ + typedValidator( + "cost-guard", + "operational", + [process.execPath, "-e", "process.exit(0)"], + ".tabellio/missing.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["operational"]), null, 2)); + await commit(fixture.seed, "Add blocked operational validation", "product-validation-blocked"); + const blocked = await runner.run({ repositoryId: "example/repository", commit: "HEAD", base: "main" }); + assert.equal(blocked.result.status, "blocked"); + assert.equal(blocked.result.validators[0].status, "blocked"); + assert.equal(blocked.result.decision.costTelemetryComplete, false); + assert(blocked.result.validators[0].reasons.includes("cost_telemetry_unavailable")); + assert(blocked.result.validators[0].reasons.some((reason) => reason.startsWith("evidence_invalid:"))); +}); + +test("typed validation preserves failure and blocked boundaries across validator kinds", async (t) => { + const fixture = await createFixture(); + t.after(() => rm(fixture.root, { recursive: true, force: true })); + await runGit({ args: ["switch", "feature"], cwd: fixture.seed }); + const schemaFailure = validatorEvidence("schema-contract", { + status: "failed", + summary: "Structured output included an unsupported field.", + }); + const missingMetric = validatorEvidence("semantic-eval"); + const unavailableCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: null, modelCalls: null, toolCalls: null }, + }); + const manifestValue = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(2)"], null), + typedValidator( + "schema-contract", + "schema", + evidenceCommand(".tabellio/schema.json", schemaFailure), + ".tabellio/schema.json", + ), + typedValidator( + "semantic-eval", + "semantic", + evidenceCommand(".tabellio/semantic.json", missingMetric), + ".tabellio/semantic.json", + { + metricThresholds: [{ metric: "strict_pass_rate", operator: "gte", value: 0.8 }], + maxCostUsd: null, + requireCostTelemetry: false, + }, + ), + typedValidator( + "cost-guard", + "operational", + evidenceCommand(".tabellio/cost.json", unavailableCost), + ".tabellio/cost.json", + { metricThresholds: [], maxCostUsd: 0.1, requireCostTelemetry: true }, + ), + ], ["static", "schema", "semantic", "operational"]); + manifestValue.failFast = false; + await writeFile(`${fixture.seed}/tabellio.validation.json`, JSON.stringify(manifestValue, null, 2)); + await commit(fixture.seed, "Add validator boundary matrix", "product-validation-boundaries"); + + const store = await NativeGitStore.open(fixture.seed); + const ledger = await GitJsonLedger.open({ repoPath: fixture.seed, ref: "refs/tabellio/validations" }); + const result = await new ValidationRunner({ store, ledger }).run({ + repositoryId: "example/repository", + commit: "HEAD", + base: "main", + }); + + assert.equal(result.result.status, "blocked"); + assert.deepEqual(result.result.validators.map(({ id, status, reasons }) => ({ id, status, reasons })), [ + { id: "static-checks", status: "failed", reasons: ["command_failed"] }, + { id: "schema-contract", status: "failed", reasons: ["evidence_reported_failed"] }, + { id: "semantic-eval", status: "blocked", reasons: ["metric_missing:strict_pass_rate"] }, + { id: "cost-guard", status: "blocked", reasons: ["cost_telemetry_unavailable"] }, + ]); + assert.equal(result.result.decision.costTelemetryComplete, false); +}); + +test("typed validation contracts reject incomplete product gates and invalid evidence", () => { + const incomplete = productManifest([ + typedValidator("static-checks", "static", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(incomplete), /requires a required semantic validator/); + + const missingEvidence = productManifest([ + typedValidator("semantic-eval", "semantic", [process.execPath, "-e", "process.exit(0)"], null), + ], ["semantic"]); + assert.throws(() => validateValidationManifest(missingEvidence), /evidence is required/); + + const invalidCost = validatorEvidence("cost-guard", { + cost: { telemetry: "unavailable", usd: 0, modelCalls: null, toolCalls: null }, + }); + assert.throws(() => validateValidatorEvidence(invalidCost), /cost values must be null/); + + const ephemeralArtifact = validatorEvidence("visual-regression", { + artifacts: [{ + name: "desktop-diff", + uri: "file:///tmp/desktop-diff.png", + digest: "d".repeat(64), + mediaType: "image/png", + bytes: 100, + }], + }); + assert.throws(() => validateValidatorEvidence(ephemeralArtifact), /must not use an ephemeral file URI/); +}); + function manifest(commands) { return { schemaVersion: "tabellio-validation/v0.1", @@ -141,6 +347,64 @@ function command(id, argv, cwd = ".", timeoutMs = 10_000) { return { id, argv, cwd, timeoutMs, required: true }; } +function productManifest(validators, requiredValidatorTypes) { + return { + schemaVersion: "tabellio-validation/v0.2", + id: "product-suite", + failFast: true, + requireEntireCheckpoint: true, + acceptance: { + id: "PLANE-101", + source: "plane", + risk: "high", + outcomes: ["The user receives a correct product result."], + invariants: ["No live mutation occurs during validation."], + forbiddenOutcomes: ["Missing cost telemetry is treated as zero cost."], + requiredValidatorTypes, + }, + validators, + }; +} + +function typedValidator(id, type, argv, evidencePath, policy = null) { + return { + id, + type, + argv, + cwd: ".", + timeoutMs: 10_000, + required: true, + evidence: evidencePath === null ? null : { path: evidencePath }, + policy: policy ?? { metricThresholds: [], maxCostUsd: null, requireCostTelemetry: false }, + }; +} + +function validatorEvidence(validatorId, overrides = {}) { + return { + schemaVersion: "tabellio-validator-evidence/v0.1", + validatorId, + status: "passed", + summary: "Validator completed.", + metrics: [], + cost: { telemetry: "not_applicable", usd: null, modelCalls: null, toolCalls: null }, + artifacts: [], + ...overrides, + }; +} + +function availableCost(usd, modelCalls, toolCalls) { + return { telemetry: "available", usd, modelCalls, toolCalls }; +} + +function evidenceCommand(path, evidence) { + const source = [ + "const fs = require('node:fs');", + `fs.mkdirSync(${JSON.stringify(path.split("/").slice(0, -1).join("/"))}, { recursive: true });`, + `fs.writeFileSync(${JSON.stringify(path)}, ${JSON.stringify(JSON.stringify(evidence))});`, + ].join(""); + return [process.execPath, "-e", source]; +} + async function commit(cwd, message, checkpoint) { await runGit({ args: ["add", "tabellio.validation.json"], cwd }); await runGit({