From 7dbc7b177cb0f80e8787643b6524f06114430dcd Mon Sep 17 00:00:00 2001 From: thossullivan Date: Tue, 18 Aug 2026 09:48:38 -0500 Subject: [PATCH 1/2] Prepare 0.5 public contracts and migration hardening --- .github/workflows/ci.yml | 5 + .github/workflows/feed-refresh.yml | 36 ++ .github/workflows/npm-release.yml | 234 +++++-- .github/workflows/public-contract.yml | 277 ++++++++ .github/workflows/published-consumer-uat.yml | 275 ++++++++ CONTRIBUTING.md | 10 +- README.md | 185 +++++- SECURITY.md | 10 +- SPEC.md | 27 +- action.yml | 39 +- bot.yml.example | 31 +- bot/bot.mjs | 324 +++++----- bot/test/package.mjs | 109 +++- bot/test/run.mjs | 349 +++++++++-- check.mjs | 59 +- docs/CONTEXT.md | 67 +- docs/DESIGN_BOT.md | 51 +- docs/PRODUCT_PLAN.md | 22 +- examples/model-eol-eval.mjs | 84 +++ examples/workflows/model-eol.yml | 8 + feeds/amazon.json | 6 +- feeds/anthropic.json | 5 +- lib/apply.mjs | 624 ++++++++++++------- lib/feeds.mjs | 3 +- lib/json-schema.mjs | 291 +++++++++ lib/reports.mjs | 117 +++- lib/scanner.mjs | 164 ++++- lib/validate-document.mjs | 320 ++++++++++ lib/validate-feed.mjs | 8 +- package.json | 5 +- refresh/diff.mjs | 14 +- refresh/distributors.mjs | 75 ++- refresh/refresh.mjs | 3 + refresh/test/run.mjs | 315 +++++++++- schema/model-eol.alert.schema.json | 56 +- schema/model-eol.bot-config.schema.json | 12 +- schema/model-eol.check.schema.json | 56 ++ schema/model-eol.inventory.schema.json | 163 ++--- schema/model-eol.plan.schema.json | 10 +- schema/model-eol.schedule.schema.json | 20 +- schema/model-eol.schema.json | 32 +- scripts/build-public-site.mjs | 235 +++++++ scripts/feed-changelog.mjs | 4 + scripts/feed-refresh-receipt.mjs | 249 ++++++++ scripts/github-release-state.mjs | 74 +++ scripts/package-integrity.mjs | 76 +++ scripts/published-consumer-uat.mjs | 243 ++++++++ scripts/release-receipt.mjs | 153 +++++ scripts/release-state.mjs | 235 +++++++ scripts/test-action-contract.mjs | 31 +- scripts/test-document-validation.mjs | 431 +++++++++++++ scripts/test-eval-harness.mjs | 148 +++++ scripts/test-feed-changelog.mjs | 2 + scripts/test-public-site.mjs | 342 ++++++++++ scripts/verify-public-site.mjs | 210 +++++++ test/run.mjs | 435 ++++++++++++- 56 files changed, 6568 insertions(+), 801 deletions(-) create mode 100644 .github/workflows/public-contract.yml create mode 100644 .github/workflows/published-consumer-uat.yml create mode 100644 examples/model-eol-eval.mjs create mode 100644 lib/json-schema.mjs create mode 100644 lib/validate-document.mjs create mode 100644 schema/model-eol.check.schema.json create mode 100644 scripts/build-public-site.mjs create mode 100644 scripts/feed-refresh-receipt.mjs create mode 100644 scripts/github-release-state.mjs create mode 100644 scripts/package-integrity.mjs create mode 100644 scripts/published-consumer-uat.mjs create mode 100644 scripts/release-receipt.mjs create mode 100644 scripts/release-state.mjs create mode 100644 scripts/test-document-validation.mjs create mode 100644 scripts/test-eval-harness.mjs create mode 100644 scripts/test-public-site.mjs create mode 100644 scripts/verify-public-site.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9df85d3..3e5353f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,11 @@ jobs: if (!matched.has(expected)) throw new Error(`inventory did not detect UAT reference: ${expected}`) } NODE + - name: UAT public document validation + uses: ./ + with: + command: validate + paths: action-uat-inventory.json - name: UAT auto-discovered config and ignores uses: ./ with: diff --git a/.github/workflows/feed-refresh.yml b/.github/workflows/feed-refresh.yml index bde4a96..abb93ac 100644 --- a/.github/workflows/feed-refresh.yml +++ b/.github/workflows/feed-refresh.yml @@ -61,6 +61,7 @@ jobs: else echo "changed=false" >> "$GITHUB_OUTPUT" fi + echo "checked_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" echo "providers=$providers" >> "$GITHUB_OUTPUT" echo "distributors=$distributors" >> "$GITHUB_OUTPUT" @@ -101,6 +102,39 @@ jobs: fi echo "url=$pr_url" >> "$GITHUB_OUTPUT" + - name: Create byte-exact refresh receipt + if: success() + env: + CHECKED_AT: ${{ steps.check.outputs.checked_at }} + MATERIAL_CHANGES: ${{ steps.check.outputs.changed }} + PR_URL: ${{ steps.pr.outputs.url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REFRESH_SHA: ${{ github.sha }} + run: | + args=( + create + --repo-dir . + --out _refresh-receipt/feed-refresh-receipt.json + --checked-at "$CHECKED_AT" + --refresh-run-url "$RUN_URL" + --refresh-sha "$REFRESH_SHA" + ) + if [ "$MATERIAL_CHANGES" = 'true' ]; then + args+=(--state pending --pending-pr-url "$PR_URL") + else + args+=(--state clean) + fi + node scripts/feed-refresh-receipt.mjs "${args[@]}" + + - name: Upload refresh receipt + if: success() + uses: actions/upload-artifact@v7 + with: + name: feed-refresh-receipt + path: _refresh-receipt/feed-refresh-receipt.json + if-no-files-found: error + overwrite: true + - name: Record successful refresh check if: success() env: @@ -112,9 +146,11 @@ jobs: if [ "$MATERIAL_CHANGES" = "true" ]; then echo "Material feed changes were regenerated and validated." echo "Review PR: $PR_URL" + echo "The pending receipt hashes the regenerated PR feeds; it cannot mark the older main feeds clean." else echo "All configured live sources were checked successfully; no material feed changes were found." echo "The committed feed-generated date is intentionally unchanged because no feed data changed." + echo "The clean receipt hashes the exact feeds checked on main." fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml index 154f0a5..339921b 100644 --- a/.github/workflows/npm-release.yml +++ b/.github/workflows/npm-release.yml @@ -7,13 +7,13 @@ # GitHub Actions, repository thossullivan/model-eol, workflow npm-release.yml. # Provenance attestations come with it. # -# No loop: the version-bump commit touches package.json only, which never -# matches the feeds/** path filter. -# -# workflow_dispatch releases regardless of the feeds guard - the path for code -# releases (fixes, features) to reach npm with the same provenance and an -# explicit stable version. It still refuses a no-op: dispatch with zero commits -# since the last tag does nothing. +# The immutable version tag is the recovery journal. A rerun can resume after +# the atomic Git push or after npm publish, but only after proving the version +# tag, moving v0 tag, release commit, source commit, registry, and GitHub release +# form one valid state. +# Recovery means rerunning the original failed workflow run: GitHub preserves +# that event's pre-release GITHUB_SHA. A fresh dispatch from the already tagged +# release commit remains a zero-commit no-op instead of reopening publication. name: npm-release @@ -24,11 +24,13 @@ on: workflow_dispatch: inputs: release_version: - description: Exact stable semver for this code release (for example, 0.3.0) + description: Exact stable semver for this code release (for example, 0.5.0) required: true type: string -concurrency: npm-release +concurrency: + group: model-eol-release-and-moving-uat + queue: max permissions: contents: write @@ -46,25 +48,25 @@ jobs: node-version: 24 package-manager-cache: false registry-url: https://registry.npmjs.org - - name: npm >= 11.5 for trusted publishing + - name: Pinned npm for trusted publishing run: | - npm install -g npm@latest - npm --version + npm install -g npm@11.6.4 + test "$(npm --version)" = '11.6.4' - name: Guard - release code explicitly and feed-only changes automatically id: guard run: | last=$(git describe --tags --abbrev=0 --match 'v*.*.*') - echo "last tag: $last" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - if [ "$GITHUB_REF" != "refs/heads/main" ]; then - echo "changed=false" >> "$GITHUB_OUTPUT" + echo "last tag reachable from source: $last" + if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ]; then + if [ "$GITHUB_REF" != 'refs/heads/main' ]; then + echo 'changed=false' >> "$GITHUB_OUTPUT" echo "dispatch ref $GITHUB_REF is not main - refusing to release it" elif [ "$(git rev-list "$last"..HEAD --count)" -gt 0 ]; then - echo "changed=true" >> "$GITHUB_OUTPUT" + echo 'changed=true' >> "$GITHUB_OUTPUT" echo "manual dispatch - releasing $(git rev-list "$last"..HEAD --count) commit(s) since $last" else - echo "changed=false" >> "$GITHUB_OUTPUT" + echo 'changed=false' >> "$GITHUB_OUTPUT" echo "nothing since $last - refusing a no-op release" fi else @@ -75,56 +77,190 @@ jobs: if: steps.guard.outputs.changed == 'true' run: npm test - - name: Validate requested code release - if: github.event_name == 'workflow_dispatch' && steps.guard.outputs.changed == 'true' + - name: Resolve new or resumable release state + if: steps.guard.outputs.changed == 'true' + id: state env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} RELEASE_VERSION: ${{ inputs.release_version }} + run: | + set -euo pipefail + target_args=(target --event-name "$GITHUB_EVENT_NAME") + resolve_args=(resolve --event-name "$GITHUB_EVENT_NAME") + if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ]; then + target_args+=(--requested-version "$RELEASE_VERSION") + resolve_args+=(--requested-version "$RELEASE_VERSION") + fi + target=$(node scripts/release-state.mjs "${target_args[@]}") + release_pages="$RUNNER_TEMP/model-eol-github-releases.json" + gh api --paginate --slurp \ + "/repos/$GITHUB_REPOSITORY/releases?per_page=100" > "$release_pages" + github_release_exists=$(node scripts/github-release-state.mjs --input "$release_pages" --tag "v$target") + published=$(npm view model-eol versions --json --prefer-online) + git fetch --force origin \ + '+refs/heads/main:refs/remotes/origin/main' \ + '+refs/tags/*:refs/tags/*' + node scripts/release-state.mjs "${resolve_args[@]}" \ + --source-sha "$GITHUB_SHA" \ + --published-json "$published" \ + --github-release-exists "$github_release_exists" \ + --github-output "$GITHUB_OUTPUT" + + - name: Validate requested code release exactly + if: github.event_name == 'workflow_dispatch' && steps.guard.outputs.changed == 'true' && steps.state.outputs.mode == 'create' + env: + RELEASE_VERSION: ${{ inputs.release_version }} run: | current=$(node -p "require('./package.json').version") - published=$(npm view model-eol versions --json) + published=$(npm view model-eol versions --json --prefer-online) node scripts/validate-release-version.mjs "$current" "$RELEASE_VERSION" "$published" - if git show-ref --verify --quiet "refs/tags/v$RELEASE_VERSION"; then - echo "tag v$RELEASE_VERSION already exists" >&2 - exit 1 - fi - if gh release view "v$RELEASE_VERSION" >/dev/null 2>&1; then - echo "GitHub release v$RELEASE_VERSION already exists" >&2 - exit 1 - fi - - name: Bump release version, tag, push + - name: Create or recover exact release commit if: steps.guard.outputs.changed == 'true' - id: bump + id: exact env: - RELEASE_VERSION: ${{ inputs.release_version }} + RELEASE_VERSION: ${{ steps.state.outputs.version }} + RELEASE_TAG: ${{ steps.state.outputs.tag }} + RELEASE_MODE: ${{ steps.state.outputs.mode }} + RECOVERED_COMMIT: ${{ steps.state.outputs.release_commit }} run: | - git config user.name "model-eol release" - git config user.email "actions@users.noreply.github.com" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - version=$(npm version "$RELEASE_VERSION" -m "model-eol v%s - dispatched code release") + set -euo pipefail + if [ "$RELEASE_MODE" = 'create' ]; then + git config user.name 'model-eol release' + git config user.email 'actions@users.noreply.github.com' + if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ]; then + version=$(npm version "$RELEASE_VERSION" -m 'model-eol v%s - dispatched code release') + else + version=$(npm version patch -m 'model-eol v%s - automated feed-data release') + fi + if [ "$version" != "$RELEASE_TAG" ]; then + echo "npm created $version, expected exact tag $RELEASE_TAG" >&2 + exit 1 + fi + git tag -f v0 "$version" + git push --atomic origin main "$version" +refs/tags/v0:refs/tags/v0 + release_commit=$(git rev-parse "$version^{commit}") else - version=$(npm version patch -m "model-eol v%s - automated feed-data release") + version="$RELEASE_TAG" + release_commit="$RECOVERED_COMMIT" fi - echo "version=$version" >> "$GITHUB_OUTPUT" - git tag -f v0 "$version" - git push --atomic origin main "$version" +refs/tags/v0:refs/tags/v0 + git checkout --detach "$version" + actual_version=$(node -p "require('./package.json').version") + actual_commit=$(git rev-parse HEAD) + if [ "$actual_version" != "$RELEASE_VERSION" ] || [ "$actual_commit" != "$release_commit" ]; then + echo "release checkout is not the resolved exact version and commit" >&2 + exit 1 + fi + echo "version=$RELEASE_VERSION" >> "$GITHUB_OUTPUT" + echo "tag=$version" >> "$GITHUB_OUTPUT" + echo "release_commit=$release_commit" >> "$GITHUB_OUTPUT" - - name: Publish to npm + - name: Pack and hash the exact release commit if: steps.guard.outputs.changed == 'true' - run: npm publish + id: package + env: + RELEASE_VERSION: ${{ steps.exact.outputs.version }} + PACK_DIR: ${{ runner.temp }}/model-eol-release-package + PACK_MANIFEST: ${{ runner.temp }}/model-eol-release-package.json + run: | + set -euo pipefail + mkdir -p "$PACK_DIR" + npm pack --json --ignore-scripts --pack-destination "$PACK_DIR" > "$PACK_MANIFEST" + node --input-type=module <<'NODE' + import fs from 'node:fs' + import path from 'node:path' + import { assertSha512Integrity, verifyPackageIntegrity } from './scripts/package-integrity.mjs' + const entries = JSON.parse(fs.readFileSync(process.env.PACK_MANIFEST, 'utf8')) + if (!Array.isArray(entries) || entries.length !== 1) throw new Error('npm pack must emit exactly one package') + const entry = entries[0] + if (entry.name !== 'model-eol' || entry.version !== process.env.RELEASE_VERSION) { + throw new Error(`npm pack emitted ${entry.name}@${entry.version}, expected model-eol@${process.env.RELEASE_VERSION}`) + } + if (typeof entry.filename !== 'string' || path.basename(entry.filename) !== entry.filename) { + throw new Error(`npm pack emitted an unsafe filename ${entry.filename}`) + } + const integrity = assertSha512Integrity(entry.integrity, 'npm pack integrity') + const tarball = path.resolve(process.env.PACK_DIR, entry.filename) + verifyPackageIntegrity({ tarball, expectedIntegrity: integrity }) + fs.appendFileSync(process.env.GITHUB_OUTPUT, `tarball=${tarball}\nintegrity=${integrity}\n`) + NODE - - name: GitHub release + - name: Publish missing exact version to npm with OIDC + if: steps.guard.outputs.changed == 'true' && steps.state.outputs.publish == 'true' + env: + RELEASE_TARBALL: ${{ steps.package.outputs.tarball }} + run: npm publish "$RELEASE_TARBALL" --ignore-scripts + + - name: Smoke-test the exact published consumer artifact if: steps.guard.outputs.changed == 'true' + env: + RELEASE_VERSION: ${{ steps.exact.outputs.version }} + RELEASE_INTEGRITY: ${{ steps.package.outputs.integrity }} + run: | + node scripts/published-consumer-uat.mjs \ + --package "model-eol@$RELEASE_VERSION" \ + --expected-version "$RELEASE_VERSION" \ + --expected-integrity "$RELEASE_INTEGRITY" \ + --expected-engine '>=22' + + - name: Create missing GitHub release + if: steps.guard.outputs.changed == 'true' && steps.state.outputs.create_github_release == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ steps.exact.outputs.tag }} run: | - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - gh release create "${{ steps.bump.outputs.version }}" \ - --title "${{ steps.bump.outputs.version }}" \ - --generate-notes + if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ]; then + gh release create "$RELEASE_TAG" --title "$RELEASE_TAG" --generate-notes else - gh release create "${{ steps.bump.outputs.version }}" \ - --title "${{ steps.bump.outputs.version }} - feed data" \ + gh release create "$RELEASE_TAG" \ + --title "$RELEASE_TAG - feed data" \ --notes "Automated feed-data release: the weekly refresh landed material feed changes, republished so \`npx model-eol\` always checks against current retirement dates. The semantic diff is in the merged feed-refresh PR. No code changes." fi + + - name: Verify remote release refs and GitHub release + if: steps.guard.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ steps.exact.outputs.tag }} + RELEASE_COMMIT: ${{ steps.exact.outputs.release_commit }} + run: | + set -euo pipefail + git fetch --force origin \ + "+refs/tags/$RELEASE_TAG:refs/tags/$RELEASE_TAG" \ + '+refs/tags/v0:refs/tags/v0' + immutable_commit=$(git rev-parse "refs/tags/$RELEASE_TAG^{commit}") + moving_commit=$(git rev-parse 'refs/tags/v0^{commit}') + if [ "$immutable_commit" != "$RELEASE_COMMIT" ] || [ "$moving_commit" != "$RELEASE_COMMIT" ]; then + echo "remote $RELEASE_TAG and v0 must both resolve to $RELEASE_COMMIT" >&2 + exit 1 + fi + release_json="$RUNNER_TEMP/model-eol-github-release.json" + gh release view "$RELEASE_TAG" --json tagName,isDraft,isPrerelease > "$release_json" + RELEASE_JSON="$release_json" node --input-type=module <<'NODE' + import fs from 'node:fs' + const release = JSON.parse(fs.readFileSync(process.env.RELEASE_JSON, 'utf8')) + if (release.tagName !== process.env.RELEASE_TAG || release.isDraft !== false || release.isPrerelease !== false) { + throw new Error(`${process.env.RELEASE_TAG} is not a published stable GitHub release`) + } + NODE + + - name: Write exact release result + env: + RELEASED: ${{ steps.guard.outputs.changed }} + RELEASE_VERSION: ${{ steps.exact.outputs.version }} + RELEASE_COMMIT: ${{ steps.exact.outputs.release_commit }} + RELEASE_INTEGRITY: ${{ steps.package.outputs.integrity }} + run: | + args=(create --out _release-result/npm-release-result.json --source-sha "$GITHUB_SHA") + if [ "$RELEASED" = 'true' ]; then + args+=(--version "$RELEASE_VERSION" --release-sha "$RELEASE_COMMIT" --registry-integrity "$RELEASE_INTEGRITY") + fi + node scripts/release-receipt.mjs "${args[@]}" + + - name: Upload exact release result + uses: actions/upload-artifact@v7 + with: + name: npm-release-result + path: _release-result/npm-release-result.json + if-no-files-found: error + overwrite: true diff --git a/.github/workflows/public-contract.yml b/.github/workflows/public-contract.yml new file mode 100644 index 0000000..f9d6141 --- /dev/null +++ b/.github/workflows/public-contract.yml @@ -0,0 +1,277 @@ +name: public-contract + +on: + push: + branches: [main] + paths: + - 'feeds/**' + - 'schema/**' + - 'scripts/build-public-site.mjs' + - 'scripts/feed-changelog.mjs' + - 'scripts/feed-refresh-receipt.mjs' + - 'scripts/verify-public-site.mjs' + - 'lib/cli.mjs' + - 'lib/validate-feed.mjs' + - 'refresh/diff.mjs' + - '.github/workflows/public-contract.yml' + workflow_run: + workflows: [feed-refresh] + branches: [main] + types: [completed] + workflow_dispatch: + +concurrency: + group: public-contract + cancel-in-progress: false + +jobs: + build: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + publish: ${{ steps.receipt.outputs.publish }} + published_sha: ${{ steps.source.outputs.sha }} + permissions: + actions: read + contents: read + pages: write + pull-requests: read + steps: + - uses: actions/checkout@v7 + with: + ref: main + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + + - name: Detect unresolved material feed work + id: pending + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + pending_url=$(gh pr list --state open --limit 1000 --json headRefName,url,isCrossRepository \ + --jq '[.[] | select(.isCrossRepository == false and (.headRefName | startswith("feed-refresh/")))][0].url // ""') + echo "url=$pending_url" >> "$GITHUB_OUTPUT" + + - name: Refuse manual or push publication while a feed PR is unresolved + if: github.event_name != 'workflow_run' && steps.pending.outputs.url != '' + env: + PENDING_URL: ${{ steps.pending.outputs.url }} + run: | + echo "material feed work is still pending at $PENDING_URL; refusing to label main clean" >&2 + exit 1 + + - name: Confirm this refresh event has not been superseded + if: github.event_name == 'workflow_run' + id: current + env: + GH_TOKEN: ${{ github.token }} + EVENT_RUN_ID: ${{ github.event.workflow_run.id }} + run: | + set -euo pipefail + latest_file="$RUNNER_TEMP/model-eol-latest-refresh.json" + gh run list --workflow feed-refresh.yml --branch main --limit 1 \ + --json databaseId,status,conclusion > "$latest_file" + LATEST_FILE="$latest_file" node --input-type=module <<'NODE' + import fs from 'node:fs' + const runs = JSON.parse(fs.readFileSync(process.env.LATEST_FILE, 'utf8')) + if (!Array.isArray(runs) || runs.length !== 1) throw new Error('unable to resolve the newest feed-refresh run') + const latest = runs[0] + const current = String(latest.databaseId) === process.env.EVENT_RUN_ID + fs.appendFileSync(process.env.GITHUB_OUTPUT, `current=${current}\n`) + if (!current) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `This feed-refresh event was superseded by run ${latest.databaseId} (${latest.status}/${latest.conclusion ?? 'pending'}); it cannot roll the public contract back.\n`) + } + NODE + + - name: Download this refresh run's exact receipt + if: github.event_name == 'workflow_run' && steps.current.outputs.current == 'true' + uses: actions/download-artifact@v8 + with: + name: feed-refresh-receipt + path: _refresh-receipt + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Resolve newest receipt matching main feeds + if: github.event_name != 'workflow_run' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + candidate_root="$RUNNER_TEMP/model-eol-refresh-receipts" + mkdir -p "$candidate_root" _refresh-receipt + found=false + runs_file="$candidate_root/successful-runs.tsv" + gh run list \ + --workflow feed-refresh.yml \ + --branch main \ + --limit 100 \ + --json databaseId,headSha,url,status,conclusion \ + --jq '.[] | [.databaseId, .headSha, .url, .status, .conclusion] | @tsv' > "$runs_file" + while IFS=$'\t' read -r run_id refresh_sha run_url run_status run_conclusion; do + [ -n "$run_id" ] || continue + if ! git cat-file -e "$refresh_sha^{commit}" 2>/dev/null; then + echo "refresh run $run_id names unavailable commit $refresh_sha; refusing receipt fallback" >&2 + exit 1 + fi + if ! git cat-file -e "$refresh_sha:scripts/feed-refresh-receipt.mjs" 2>/dev/null; then + continue + fi + if [ "$run_status" != completed ] || [ "$run_conclusion" != success ]; then + echo "newest receipt-era refresh run $run_id is $run_status/$run_conclusion; refusing fallback until a newer refresh succeeds" >&2 + exit 1 + fi + candidate="$candidate_root/$run_id" + mkdir -p "$candidate" + if ! gh run download "$run_id" --name feed-refresh-receipt --dir "$candidate" >/dev/null 2>&1; then + echo "successful receipt-era refresh run $run_id has no downloadable receipt; refusing fallback" >&2 + exit 1 + fi + set +e + node scripts/feed-refresh-receipt.mjs verify \ + --repo-dir . \ + --receipt "$candidate/feed-refresh-receipt.json" \ + --expected-run-url "$run_url" \ + --expected-refresh-sha "$refresh_sha" \ + --require-match + receipt_status=$? + set -e + if [ "$receipt_status" -eq 0 ]; then + cp "$candidate/feed-refresh-receipt.json" _refresh-receipt/feed-refresh-receipt.json + found=true + break + fi + if [ "$receipt_status" -eq 3 ]; then + state=$(node -e "const r=require(process.argv[1]); process.stdout.write(r.state)" "$candidate/feed-refresh-receipt.json") + if [ "$state" = 'pending' ]; then + echo "newest receipt-era run $run_id has pending feed changes that are not on main; refusing stale receipt fallback" >&2 + else + echo "newest receipt-era run $run_id verified different feeds; refusing stale receipt fallback until feed-refresh runs again" >&2 + fi + exit 1 + fi + if [ "$receipt_status" -ne 3 ]; then + echo "invalid feed refresh receipt from run $run_id" >&2 + exit "$receipt_status" + fi + done < "$runs_file" + if [ "$found" != true ]; then + echo 'no successful feed-refresh receipt matches every feed currently on main' >&2 + exit 1 + fi + + - name: Verify selected receipt against the published inputs + if: github.event_name != 'workflow_run' || steps.current.outputs.current == 'true' + id: receipt + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_RUN_URL: ${{ github.event.workflow_run.html_url }} + EVENT_REFRESH_SHA: ${{ github.event.workflow_run.head_sha }} + PENDING_REFRESH_PR: ${{ steps.pending.outputs.url }} + run: | + set -euo pipefail + args=( + verify + --repo-dir . + --receipt _refresh-receipt/feed-refresh-receipt.json + --github-output "$GITHUB_OUTPUT" + --require-match + ) + if [ "$EVENT_NAME" = 'workflow_run' ]; then + args+=(--expected-run-url "$EVENT_RUN_URL" --expected-refresh-sha "$EVENT_REFRESH_SHA") + fi + set +e + node scripts/feed-refresh-receipt.mjs "${args[@]}" + receipt_status=$? + set -e + if [ "$receipt_status" -eq 0 ]; then + if [ -n "$PENDING_REFRESH_PR" ]; then + echo "material feed work is still pending at $PENDING_REFRESH_PR; refusing to advance last_checked" >&2 + exit 1 + fi + echo 'publish=true' >> "$GITHUB_OUTPUT" + elif [ "$receipt_status" -eq 3 ] && [ "$EVENT_NAME" = 'workflow_run' ]; then + state=$(node -e "const r=require('./_refresh-receipt/feed-refresh-receipt.json'); process.stdout.write(r.state)") + if [ "$state" != 'pending' ]; then + echo 'a clean receipt does not match the feeds on main' >&2 + exit 1 + fi + echo 'publish=false' >> "$GITHUB_OUTPUT" + echo 'The refresh found material changes, so its pending receipt cannot advance last_checked for the older main feeds.' >> "$GITHUB_STEP_SUMMARY" + else + exit "$receipt_status" + fi + + - name: Bind publication to main HEAD + if: steps.receipt.outputs.publish == 'true' + id: source + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Build versioned public contract + if: steps.receipt.outputs.publish == 'true' + run: | + node scripts/build-public-site.mjs \ + --out-dir _site \ + --receipt _refresh-receipt/feed-refresh-receipt.json \ + --source-sha "${{ steps.source.outputs.sha }}" + + - name: Validate generated public contract + if: steps.receipt.outputs.publish == 'true' + run: | + node check.mjs validate _site/schema/0.1/*.json _site/feeds/*.json + node -e 'const fs=require("node:fs"); const h=require("./_site/health.json"); const i=require("./_site/index.json"); const schemas=fs.readdirSync("./_site/schema/0.1").filter(name=>name.endsWith(".json")); if (h.schema !== "model-eol/health@0.1" || !schemas.length || i.schemas.length !== schemas.length || h.published_commit !== i.published_commit) process.exit(1)' + + - name: Upload exact deployment expectation + if: steps.receipt.outputs.publish == 'true' + uses: actions/upload-artifact@v7 + with: + name: public-contract-expectation + path: _site + if-no-files-found: error + overwrite: true + + - if: steps.receipt.outputs.publish == 'true' + uses: actions/configure-pages@v6 + - if: steps.receipt.outputs.publish == 'true' + uses: actions/upload-pages-artifact@v5 + with: + path: _site + + deploy: + if: needs.build.outputs.publish == 'true' + needs: build + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Publish public contract + id: deployment + uses: actions/deploy-pages@v5 + - uses: actions/checkout@v7 + with: + ref: ${{ needs.build.outputs.published_sha }} + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + - name: Download exact deployment expectation + uses: actions/download-artifact@v8 + with: + name: public-contract-expectation + path: _expected-site + - name: Verify every live public contract asset + env: + PAGE_URL: ${{ steps.deployment.outputs.page_url }} + run: node scripts/verify-public-site.mjs --base-url "$PAGE_URL" --expected-dir _expected-site diff --git a/.github/workflows/published-consumer-uat.yml b/.github/workflows/published-consumer-uat.yml new file mode 100644 index 0000000..3822718 --- /dev/null +++ b/.github/workflows/published-consumer-uat.yml @@ -0,0 +1,275 @@ +name: published-consumer-uat + +on: + workflow_run: + workflows: [npm-release] + branches: [main] + types: [completed] + workflow_dispatch: + inputs: + version: + description: Exact published stable 0.x version to test; v0 must currently point to it + required: true + type: string + +permissions: + actions: read + contents: read + +concurrency: + group: model-eol-release-and-moving-uat + queue: max + +jobs: + resolve: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + run: ${{ steps.release.outputs.released }} + version: ${{ steps.release.outputs.version }} + release_commit: ${{ steps.refs.outputs.release_commit }} + registry_integrity: ${{ steps.refs.outputs.registry_integrity }} + moving_current: ${{ steps.refs.outputs.moving_current }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || 'main' }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + - name: Download npm-release's exact result + if: github.event_name == 'workflow_run' + uses: actions/download-artifact@v8 + with: + name: npm-release-result + path: _release-result + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Resolve exact tested release + id: release + env: + EVENT_NAME: ${{ github.event_name }} + EXPECTED_SOURCE_SHA: ${{ github.event.workflow_run.head_sha }} + REQUESTED_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = 'workflow_run' ]; then + node scripts/release-receipt.mjs verify \ + --receipt _release-result/npm-release-result.json \ + --expected-source-sha "$EXPECTED_SOURCE_SHA" \ + --github-output "$GITHUB_OUTPUT" + else + if [[ ! "$REQUESTED_VERSION" =~ ^0\.[0-9]+\.[0-9]+$ ]]; then + echo "invalid stable 0.x version: $REQUESTED_VERSION" >&2 + exit 2 + fi + echo 'released=true' >> "$GITHUB_OUTPUT" + echo "version=$REQUESTED_VERSION" >> "$GITHUB_OUTPUT" + echo "tag=v$REQUESTED_VERSION" >> "$GITHUB_OUTPUT" + fi + - name: Verify remote moving and immutable Action refs + if: steps.release.outputs.released == 'true' + id: refs + env: + VERSION: ${{ steps.release.outputs.version }} + EXPECTED_RELEASE_COMMIT: ${{ steps.release.outputs.release_commit }} + EXPECTED_REGISTRY_INTEGRITY: ${{ steps.release.outputs.registry_integrity }} + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + git fetch --force origin \ + "+refs/tags/v$VERSION:refs/tags/v$VERSION" \ + '+refs/tags/v0:refs/tags/v0' + immutable_commit=$(git rev-parse "refs/tags/v$VERSION^{commit}") + moving_commit=$(git rev-parse 'refs/tags/v0^{commit}') + if [ "$moving_commit" = "$immutable_commit" ]; then + echo 'moving_current=true' >> "$GITHUB_OUTPUT" + elif [ "$EVENT_NAME" = 'workflow_dispatch' ]; then + echo "remote v0 is $moving_commit, but manually requested immutable v$VERSION is $immutable_commit" >&2 + exit 1 + else + echo 'moving_current=false' >> "$GITHUB_OUTPUT" + echo "v0 now resolves to $moving_commit; exact v$VERSION UAT remains authoritative and this superseded moving-line monitor will be skipped" >> "$GITHUB_STEP_SUMMARY" + fi + if [ -n "$EXPECTED_RELEASE_COMMIT" ] && [ "$immutable_commit" != "$EXPECTED_RELEASE_COMMIT" ]; then + echo "release receipt names $EXPECTED_RELEASE_COMMIT, but v$VERSION is $immutable_commit" >&2 + exit 1 + fi + tagged_version=$(git show "$immutable_commit:package.json" | node -e 'let s=""; process.stdin.on("data", c => s += c).on("end", () => process.stdout.write(JSON.parse(s).version))') + if [ "$tagged_version" != "$VERSION" ]; then + echo "v$VERSION contains package version $tagged_version" >&2 + exit 1 + fi + registry_metadata="$RUNNER_TEMP/model-eol-registry-metadata.json" + npm view "model-eol@$VERSION" version dist.integrity --json --prefer-online > "$registry_metadata" + REGISTRY_METADATA="$registry_metadata" node --input-type=module <<'NODE' + import fs from 'node:fs' + import { assertSha512Integrity } from './scripts/package-integrity.mjs' + const metadata = JSON.parse(fs.readFileSync(process.env.REGISTRY_METADATA, 'utf8')) + if (metadata.version !== process.env.VERSION) throw new Error(`registry returned model-eol@${metadata.version}, expected ${process.env.VERSION}`) + const integrity = assertSha512Integrity(metadata['dist.integrity'], 'registry integrity') + if (process.env.EXPECTED_REGISTRY_INTEGRITY && integrity !== process.env.EXPECTED_REGISTRY_INTEGRITY) { + throw new Error(`registry integrity ${integrity} does not match release receipt ${process.env.EXPECTED_REGISTRY_INTEGRITY}`) + } + fs.appendFileSync(process.env.GITHUB_OUTPUT, `registry_integrity=${integrity}\n`) + NODE + echo "release_commit=$immutable_commit" >> "$GITHUB_OUTPUT" + + package-consumer: + needs: resolve + if: needs.resolve.outputs.run == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.resolve.outputs.release_commit }} + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + registry-url: https://registry.npmjs.org + - name: Exact published package UAT + env: + VERSION: ${{ needs.resolve.outputs.version }} + INTEGRITY: ${{ needs.resolve.outputs.registry_integrity }} + run: node scripts/published-consumer-uat.mjs --package "model-eol@$VERSION" --expected-version "$VERSION" --expected-integrity "$INTEGRITY" --expected-engine '>=22' + moving-package-consumer: + needs: resolve + if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.moving_current == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.resolve.outputs.release_commit }} + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + registry-url: https://registry.npmjs.org + - name: Moving npm 0.x line UAT + env: + VERSION: ${{ needs.resolve.outputs.version }} + INTEGRITY: ${{ needs.resolve.outputs.registry_integrity }} + run: | + set +e + node scripts/published-consumer-uat.mjs --package model-eol@0 --expected-version "$VERSION" --expected-integrity "$INTEGRITY" --expected-engine '>=22' + uat_status=$? + set -e + current=$(npm view model-eol@0 version --json --prefer-online | node -e 'let s=""; process.stdin.on("data", c => s += c).on("end", () => { const v=JSON.parse(s); const r=Array.isArray(v)?v.at(-1):v; if(typeof r!=="string")process.exit(1); process.stdout.write(r) })') + if [ "$current" != "$VERSION" ]; then + echo "npm 0.x moved to $current while model-eol@$VERSION was being monitored" >&2 + exit 1 + fi + exit "$uat_status" + + exact-action-consumer: + needs: resolve + if: needs.resolve.outputs.run == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.resolve.outputs.release_commit }} + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + - name: Create isolated exact Action consumer + run: | + mkdir consumer + printf '%s\n' \ + 'import OpenAI from "openai"' \ + 'const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })' \ + 'export const model = "o3-deep-research"' \ + > consumer/app.mjs + - name: Immutable release Action inventory UAT + uses: ./ + with: + command: inventory + paths: consumer + output-file: exact-action-inventory.json + - name: Immutable release Action validate round-trip UAT + uses: ./ + with: + command: validate + paths: exact-action-inventory.json + output-file: exact-action-validation.txt + - name: Verify immutable release Action result + run: | + node --input-type=module <<'NODE' + import fs from 'node:fs' + const inventory = JSON.parse(fs.readFileSync('exact-action-inventory.json', 'utf8')) + if (inventory.schema !== 'model-eol/inventory@0.1') throw new Error(`unexpected schema ${inventory.schema}`) + if (!inventory.model_references.some(item => item.matched === 'o3-deep-research')) { + throw new Error('immutable release Action did not find the consumer model') + } + const validation = fs.readFileSync('exact-action-validation.txt', 'utf8') + if (!validation.includes('valid inventory document')) throw new Error('immutable release Action did not validate its own inventory artifact') + NODE + + action-consumer: + needs: resolve + if: needs.resolve.outputs.run == 'true' && needs.resolve.outputs.moving_current == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/setup-node@v7 + with: + node-version: 24 + package-manager-cache: false + - name: Create isolated consumer + run: | + mkdir consumer + printf '%s\n' \ + 'import OpenAI from "openai"' \ + 'const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })' \ + 'export const model = "o3-deep-research"' \ + > consumer/app.mjs + - name: Moving v0 Action inventory UAT + uses: thossullivan/model-eol@v0 + with: + command: inventory + paths: consumer + output-file: action-inventory.json + - name: Moving v0 Action validate round-trip UAT + uses: thossullivan/model-eol@v0 + with: + command: validate + paths: action-inventory.json + output-file: action-validation.txt + - name: Verify moving v0 Action result + run: | + node --input-type=module <<'NODE' + import fs from 'node:fs' + const inventory = JSON.parse(fs.readFileSync('action-inventory.json', 'utf8')) + if (inventory.schema !== 'model-eol/inventory@0.1') throw new Error(`unexpected schema ${inventory.schema}`) + if (!inventory.model_references.some(item => item.matched === 'o3-deep-research')) { + throw new Error('moving v0 Action did not find the consumer model') + } + const validation = fs.readFileSync('action-validation.txt', 'utf8') + if (!validation.includes('valid inventory document')) throw new Error('moving v0 Action did not validate its own inventory artifact') + NODE + - name: Reverify remote v0 after the Action ran + env: + VERSION: ${{ needs.resolve.outputs.version }} + EXPECTED_RELEASE_COMMIT: ${{ needs.resolve.outputs.release_commit }} + run: | + set -euo pipefail + remote="https://github.com/$GITHUB_REPOSITORY.git" + immutable_commit=$(git ls-remote "$remote" "refs/tags/v$VERSION^{}" | cut -f1) + if [ -z "$immutable_commit" ]; then + immutable_commit=$(git ls-remote "$remote" "refs/tags/v$VERSION" | cut -f1) + fi + moving_commit=$(git ls-remote "$remote" 'refs/tags/v0^{}' | cut -f1) + if [ -z "$moving_commit" ]; then + moving_commit=$(git ls-remote "$remote" 'refs/tags/v0' | cut -f1) + fi + if [ "$immutable_commit" != "$EXPECTED_RELEASE_COMMIT" ] || [ "$moving_commit" != "$EXPECTED_RELEASE_COMMIT" ]; then + echo "v0 or immutable v$VERSION moved while the moving Action UAT was running" >&2 + exit 1 + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e4740f..10dcad8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ purpose. Open an issue - that conversation is the whole point of the project. ## Development setup -Node 20+, nothing else. There are no dependencies to install - not for the +Node 22+, nothing else. There are no dependencies to install - not for the tool, not for development. ```bash @@ -48,10 +48,10 @@ npm test ## Quality gates -`npm test` composes all four suites (checker, refresh, bot, changelog) plus -feed validation - it must pass, offline, with no API keys. CI runs exactly -this. Every behavior change needs a regression test that fails on the old -code. +`npm test` composes the checker, refresh, bot, Action, schema validation, +example eval-harness, packed-consumer, changelog, and public-site suites - it +must pass offline, with no API keys. CI runs exactly this. Every behavior change +needs a regression test that fails on the old code. Two testing conventions that bite newcomers: diff --git a/README.md b/README.md index f3f5721..8038361 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![feed refresh](https://github.com/thossullivan/model-eol/actions/workflows/feed-refresh.yml/badge.svg)](https://github.com/thossullivan/model-eol/actions/workflows/feed-refresh.yml) ![spec](https://img.shields.io/badge/spec-model--eol%2F0.1-blue) ![deps](https://img.shields.io/badge/runtime%20deps-zero-brightgreen) -![node](https://img.shields.io/badge/node-%3E%3D20-brightgreen) +![node](https://img.shields.io/badge/node-%3E%3D22-brightgreen) ![license](https://img.shields.io/badge/license-MIT-blue) **A machine-readable deprecation feed format for AI models, plus the reference @@ -14,9 +14,34 @@ tooling that turns it into a Dependabot for models.** ![model-eol demo: retired models, distributor clocks, worst-case date](docs/img/demo.gif) +## Release line + +**0.4.1 - trusted migrations.** The 0.4 line made bot migrations independently +evaluable and bound each result to the exact default-branch commit, plan, eval +configuration, and feed digests. The workflow resolves the package version once +and reuses that exact version across planning, evaluation, and publication. It added +path-scoped policy and lifecycle routing for mixed-provider repositories, +trusted PR/issue ownership, stale-work reconciliation, and fresh-work behavior +when a retired model returns. The 0.4.1 patch moved the copy-ready workflows to +the current Node-backed Actions and fixed Google models-endpoint pagination. The +bundled feeds contain 313 entries across Amazon, Anthropic, Google, and OpenAI. + +**0.5.0 - public contract and trust.** This milestone makes the data contract public: +canonical hosted schemas, a zero-dependency `validate` command, hosted feeds, +an Atom changelog, and byte-exact refresh receipts whose digests must match the +published data. It adds status-aware distributor refresh including Bedrock Public +Extended Access, preserves structured replacement guidance across JSON reports +and preserves channel-specific lifecycle clocks in CycloneDX, +preflights multi-file apply plan-wide and rolls back later commit failures, +fails explicitly on unapproved symlink coverage, and keeps configured eval +failures as distinct durable issues. It also +ships a repository-owned eval-harness starter, permanent exact-package consumer +UAT, and a Node 22 support floor. The hosted URLs become authoritative only after +the first green Pages deployment. + ## The problem -On July 23, 2026, OpenAI shut down 15 model snapshots on schedule. One of my +On July 23, 2026, OpenAI shut down a scheduled wave of model snapshots. One of my tuned research workflows returned a model-not-found error that morning even though the retirement had been documented for months. A model ID is the one dependency my toolchain could not see. When an npm package is deprecated, the warning prints @@ -36,14 +61,17 @@ second, so the existing trackers can converge instead of each scraping alone. - **`feeds/`** - Amazon (4 entries), Anthropic (29 entries), Google (86 entries) and OpenAI (194 entries), generated from the providers' live deprecation pages plus the AWS Bedrock and Google Vertex AI lifecycle pages, feed data generated 2026-08-18. Every dated entry carries a source URL. - **`check.mjs`** - zero-dependency CLI: CI gate, PR diff gate, inventory, CycloneDX - ML-BOM export, retirement schedule, alerts and badges, migration plan/apply. + ML-BOM export, retirement schedule, alerts and badges, migration plan/apply, + and public document validation. - **`refresh/`** - regenerates the feeds from provider pages, models endpoints, and distributor lifecycle pages, with a semantic diff for human review. - **`bot/`** - the Dependabot part: a cron GitHub workflow that maintains one migration PR or issue per retiring model, published as the `model-eol-bot` binary in the same npm package. -- **`scripts/feed-changelog.mjs`** - the feeds' git history as Atom/markdown, so - "model retirements as they are announced" is a feed you can subscribe to. +- **Public contract rollout** - the 0.5 Pages workflow publishes versioned + schemas, hosted feeds, refresh health, and an Atom changelog. Those URLs + become authoritative only after Pages is enabled and the exact-byte deployment + check passes. ## Try it @@ -72,6 +100,9 @@ node check.mjs path/to/your/repo --via aws-bedrock # inventory, and a CycloneDX 1.6 ML-BOM for your SBOM pipeline node check.mjs inventory path/to/your/repo node check.mjs inventory path/to/your/repo --format cyclonedx > model-bom.json +# the primary CI gate is also a schema-valid public report +node check.mjs check path/to/your/repo --json > model-eol-check.json +node check.mjs validate model-eol-check.json # retirement schedule with the repo-level worst case node check.mjs schedule path/to/your/repo # GitHub Actions annotations, Markdown, or a shields.io badge @@ -80,6 +111,8 @@ node check.mjs alert path/to/your/repo --format badge > model-eol-badge.json # migration plan (only high-confidence direct API refs are patchable) and safe apply node check.mjs plan path/to/your/repo --days 90 > plan.json node check.mjs apply --plan plan.json --dry-run +# validate provider feeds, repository policy, or model-eol machine reports +node check.mjs validate feeds/openai.json .model-eol.json plan.json ``` @@ -118,6 +151,10 @@ the `v0` line: - uses: actions/checkout@v7 with: fetch-depth: 0 # --changed needs history to diff against the base ref +- uses: actions/setup-node@v7 + with: + node-version: 22 + package-manager-cache: false - uses: thossullivan/model-eol@v0 with: command: check @@ -131,10 +168,16 @@ or retiring within the threshold. Add `via: aws-bedrock` (or `azure-ai-foundry`) to judge by a distributor's clock. The copy-paste version with both gates - PR diff plus a weekly full-repository check - is [`examples/workflows/model-eol.yml`](examples/workflows/model-eol.yml). The -Action also exposes `inventory`, `schedule`, `alert`, and `plan`. Use +Action also exposes `inventory`, `schedule`, `alert`, `plan`, and `validate`. Use `format: cyclonedx` for an ML-BOM, or `output-file` to retain any report as a workflow artifact while keeping the same output in the job log. +CycloneDX exports use one machine-learning component per canonical model and +lifecycle channel. Direct publisher (`publisher-direct`), Azure, Bedrock, and +other routed references therefore retain their own status and shutdown clock +under deterministic, channel-qualified `bom-ref` values; occurrences are sorted +and scoped to the component whose clock they used. + ## One repository policy The CLI, Action, and bot all discover a strict `.model-eol.json` at the target @@ -151,7 +194,7 @@ Git repository root. CLI flags or Action inputs override configured values, and "paths": ["test/fixtures/**", "vendor"] }, "issues": { "enabled": true }, - "eval": { "command": "npm test" } + "eval": { "command": "node scripts/model-eol-eval.mjs" } } ``` @@ -160,8 +203,23 @@ in `ignore.models` suppresses the whole alias family; `ignore.paths` uses repo-relative `*`, `**`, and `?` globs and excludes matching files before scan coverage limits are counted. With no config the CLI keeps its historical `scope: all`; once a config exists, omitted values use the shared bot defaults -(`days: 90`, `scope: direct`, and the publisher clock). The complete contract is -[`schema/model-eol.bot-config.schema.json`](schema/model-eol.bot-config.schema.json). +(`days: 90`, `scope: direct`, and the publisher clock). Portable structure is +defined by +[`schema/model-eol.bot-config.schema.json`](schema/model-eol.bot-config.schema.json), +while `model-eol validate` also enforces runtime semantic and safety checks that +Draft-07 cannot express cleanly. + +For content-heavy repositories, start with `inventory`, then exclude caches, +generated catalogs, archived fixtures, and documentation that are not live model +dependencies. Published-repository UAT deliberately verifies both outcomes: the +unconfigured scan reports evidence, while an explicit path policy produces a +clean check and empty bot decision table without changing repository content. + +Parent-repository scans do not recurse into tracked Git submodules. A detected +gitlink is reported as incomplete coverage, so `check` and `plan` fail closed +unless the path is explicitly ignored or `--allow-incomplete` is chosen. To +inspect submodule contents, run model-eol against the checked-out submodule as a +separate target/repository policy. Mixed-provider monorepos can keep those top-level values as repository defaults, then apply path policies and a lifecycle channel per reference: @@ -250,8 +308,9 @@ human. The workflow resolves `model-eol@0` from npm once and reuses that exact version across its plan, read-only evaluation, and write-token publication jobs; consumer repositories do not copy `check.mjs` or the `bot/` source directory. Dismissals are respected, reintroduced retired models create fresh work, and -bot-owned PRs/issues close when their finding disappears. Preview everything with -no GitHub calls and without adding a project dependency: +bot-owned PRs/issues close when their finding disappears. In a repository without +a configured eval, preview everything with no GitHub calls and without adding a +project dependency: ```sh npx --yes --package=model-eol@0 model-eol-bot --dry-run --target-dir . --repo OWNER/REPO @@ -262,31 +321,106 @@ it uses the bot defaults of 90 days and direct references. `eval.command` also comes from the config unless the `MODEL_EOL_EVAL_COMMAND` repository variable is set as an explicit workflow override. Each patchable model is applied in an isolated checkout and evaluated independently with its own old/new IDs, timeout, -bounded report, and explicitly allowed environment. The content-bound result +bounded report, and explicitly forwarded environment names. The content-bound result manifest is also pinned to the evaluated Git commit, letting passing migrations proceed while a failing peer remains blocked without publishing against a newer, unevaluated default branch. +### Prove the swap works in your repository + +Model behavior is repository-specific, so model-eol never guesses which test is +good enough to authorize a migration. Check a harness into the repository and +name it explicitly in `.model-eol.json`. A non-empty +`MODEL_EOL_EVAL_COMMAND` repository variable overrides that command; otherwise +the config is authoritative. With neither configured, the result manifest says +that evaluation was not configured. + +Start by copying [`examples/model-eol-eval.mjs`](examples/model-eol-eval.mjs) to +`scripts/model-eol-eval.mjs`, then replace its `VERIFY` command with the smallest +deterministic test that proves the behavior your application relies on. A useful +harness checks response or tool-call shape, structured-output parsing, required +capabilities, and a small golden task set. A generic build passing is supporting +evidence, but it does not by itself prove model behavior. The starter invokes a +dedicated `npm run eval:model-eol` script so adopting it requires an explicit +repository test rather than accidentally treating an unrelated unit suite as an +LLM-behavior eval. +Point `eval:model-eol` at the underlying behavior test, not back at the harness +itself. + +The harness runs after the proposed migration in an isolated checkout and gets: + +- `MODEL_EOL_OLD_ID` and `MODEL_EOL_NEW_ID` for the current migration. +- `MODEL_EOL_PLAN`, a one-model plan containing the exact changed references. +- `MODEL_EOL_REPORT`, the required path for a bounded, publishable Markdown receipt. + +Exit zero plus a regular report file means pass; any other exit, timeout, missing +report, tracked workspace mutation, or checkout drift fails that migration. Test +stdout and stderr are intentionally not published. Put only non-secret evidence +in the report, and forward only credentials required by trusted eval code through +`eval.pass_env`. That allowlist controls normal subprocess environment forwarding; +it is not an OS sandbox and does not isolate code running as the same user. The +workflow's security boundary is that this job has no repository write token and +the publication job never executes repository-owned eval code. + +Before enabling the scheduled bot, commit the harness and policy on a clean +branch and exercise the same isolated evaluator locally: + +```bash +set -euo pipefail +repo="$(pwd -P)" +version="$(npm view model-eol@0 version --json | node -e 'let s="";process.stdin.on("data",c=>s+=c).on("end",()=>{const v=JSON.parse(s);const r=Array.isArray(v)?v.at(-1):v;if(typeof r!=="string")process.exit(1);process.stdout.write(r)})')" +tmp="$(mktemp -d)" +trap 'rm -rf -- "$tmp"' EXIT +cd "$tmp" +if [ -f "$repo/.model-eol.json" ]; then + MODEL_EOL_UAT_REPO="$repo" npx --yes --package="model-eol@$version" \ + -c 'cd "$MODEL_EOL_UAT_REPO" && model-eol plan .' > "$tmp/plan.json" +else + MODEL_EOL_UAT_REPO="$repo" npx --yes --package="model-eol@$version" \ + -c 'cd "$MODEL_EOL_UAT_REPO" && model-eol plan . --days 90 --scope direct' > "$tmp/plan.json" +fi +npx --yes --package="model-eol@$version" model-eol-bot evaluate \ + --target-dir "$repo" \ + --plan-file "$tmp/plan.json" \ + --output-file "$tmp/eval.json" +node -e 'const r=require(process.argv[1]); console.log(r.results)' "$tmp/eval.json" +MODEL_EOL_EVAL_RESULTS_FILE="$tmp/eval.json" \ + npx --yes --package="model-eol@$version" model-eol-bot \ + --dry-run --target-dir "$repo" --repo OWNER/REPO +``` + +Each planned model is patched and evaluated independently. The cleanup trap +removes the temporary artifacts when this shell exits; copy `eval.json` elsewhere +after the dry run if you want to retain the receipt as UAT evidence. Then let +`bot.yml.example` run the same contract with any explicitly opted-in provider +keys available to its read-only evaluate job, not its write-capable publish job. + +Starting in 0.5, inline `model-eol-bot --eval` and the legacy unbound report/status +artifacts are refused. A configured publication must consume the commit-bound +manifest from `model-eol-bot evaluate`; missing results fail before GitHub API +access. + On its first actionable run, the bot creates the `model-eol` label. If repository policy prevents label creation or assignment, it fails closed rather than publish work whose ownership cannot be authenticated. Existing work is trusted only when the label, metadata, deterministic branch, repository, base, and Git lease agree. -For HTTPS remotes—including private repositories—the bot passes `GITHUB_TOKEN` to +For HTTPS remotes - including private repositories - the bot passes `GITHUB_TOKEN` to temporary Git processes as a host-scoped authorization header and never places it in a remote URL. Two operational notes the hard way teaches: PRs created with `GITHUB_TOKEN` do not trigger `pull_request` workflows (use a fine-grained PAT or GitHub App token when checks must run, stored as the optional `MODEL_EOL_BOT_TOKEN` secret), and the -workflow splits privileges—provider keys live only in the read-only evaluation -job, while write tokens live only in the reconciliation job. +workflow splits privileges - only provider secrets explicitly added for the +trusted eval live in the read-only evaluation job, while write tokens live only +in the reconciliation job. ## Keeping the feeds honest ```sh node refresh/refresh.mjs --check # semantic diff vs live pages; exit 3 = PR-worthy node refresh/refresh.mjs --distributor aws-bedrock,vertex-ai # distributor lifecycle clocks -node scripts/feed-changelog.mjs # feeds' git history as an Atom feed +node scripts/feed-changelog.mjs # local rendering of the hosted Atom feed ``` Parse failures fail loudly and never emit a guessed feed. This runs automatically: @@ -297,6 +431,14 @@ their keep: they caught the hand-compiled feeds drifting from Anthropic's recommendations within one week, and corrected a hand-compiled Bedrock date that was three months wrong. +Every successful live refresh emits a byte-exact receipt. The Pages workflow +publishes the [public contract](https://thossullivan.github.io/model-eol/) only +when that receipt hashes the exact feeds on `main`: a no-change check can advance +`last_checked` immediately, while a material-change run waits for its generated +feed PR to merge. Feed `generated` remains semantic - it changes only when source +data changes. The hosted contract contains canonical Draft-07 schemas, feeds +with SHA-256 receipts, and the Atom changelog. + The refresh also travels in the other direction. On August 3, Google removed the previously listed October 16 earliest shutdown dates for the Gemini 2.5 Pro, Flash, and Flash-Lite GA models. The pipeline opened @@ -322,6 +464,15 @@ move any of us can make here. CI runs the full suite on every push and PR. The copy-ready bot workflow ships as `bot.yml.example` and consumes the published package rather than local copies of the tooling. +- The 0.5 Pages workflow publishes canonical schemas, feeds, refresh health, and + Atom at `thossullivan.github.io/model-eol`. Treat that host as authoritative + only after repository Pages is enabled and its first deployment passes the + exact-byte live check. `model-eol validate` checks feeds, repository policy, + check reports, inventories, schedules, alerts, and plans against the same zero-dependency + runtime contracts. + Maintainer rollout: set Pages source to GitHub Actions, dispatch `feed-refresh` + from `main`, and require the resulting `public-contract` deployment to pass + before releasing 0.5. - Current-model entries (and therefore policy-floor horizons) populate only when refresh runs with provider API keys for the models endpoints. - The checker matches known IDs only - it will not discover models absent from the @@ -333,7 +484,7 @@ move any of us can make here. changes republish automatically as patch versions (trusted publishing with provenance), while manual code releases require an explicit stable version, so `npx model-eol` always checks against current dates. -- Not yet: feed signing, gateway route resolvers. +- Not yet: feed signing, authenticated account-level gateway/cloud resolvers. ## Contributing diff --git a/SECURITY.md b/SECURITY.md index 8965490..2af7a0a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -21,9 +21,13 @@ The surface that matters, given zero runtime dependencies: - Feed integrity: `feeds/*.json` are generated from provider pages, and a poisoned feed changes CI verdicts downstream. Feed signing is on the roadmap, not shipped. -- The bot workflow's privilege split: provider keys live only in the - read-only plan/eval job, write tokens only in the publish job. Anything - that lets one side reach the other's credentials is a finding. +- The bot workflow's privilege split: the plan job is read-only, the evaluate + job receives only provider secrets explicitly needed by trusted eval code and + has no write token, and the publish job receives the write token but never + executes repository-owned eval code. `eval.pass_env` controls normal + subprocess forwarding; it is not an OS sandbox or same-user secret-isolation + boundary. Anything that bridges the evaluate/publish privilege split is a + finding. The checker itself needs no credentials: scanning is static analysis, so a report that assumes an API key inside `check.mjs` is out of scope by diff --git a/SPEC.md b/SPEC.md index e9c1b21..95aaaa7 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,6 +1,6 @@ -# model-eol - a machine-readable model deprecation feed (draft spec v0.1) +# model-eol - a machine-readable model deprecation feed (public draft v0.1) -> Status: sketch, 2026-07-25. The problem: model retirement dates live in HTML docs +> Status: public draft, updated 2026-08-18. The problem: model retirement dates live in HTML docs > and emails, so every deprecation checker is a scraper with a private registry > format. Ordinary software solved this layer - endoflife.date for EOL data, OSV for > vulnerability feeds. This is the model version: small enough that a provider could @@ -57,10 +57,20 @@ is data, which is exactly what scraping HTML can never give you. `distributions[].status`, when present, is one of `active`, `legacy`, `extended-access`, or `retired`. `legacy` means no new consumers but existing -usage works; `extended-access` means availability continues past the stated -`shutdown` under a paid or opt-in program (Bedrock offers this for some models) - -the `shutdown` date stays the base retirement, and the status flags that a -negotiated tail exists. +usage works. `extended-access` means the distributor has announced or entered a +distinct extended-access portion of that lifecycle; it does not imply availability +after `shutdown`. On Amazon Bedrock, public extended access is a potentially +higher-priced portion of the Legacy period for active users and ends at EOL, so +`shutdown` remains the Bedrock EOL date. Bedrock still calls the encompassing +state Legacy; `extended-access` is this feed's finer-grained channel signal. +Private arrangements after EOL are not represented by this status. `retired` +means the distributor explicitly reports the model as no longer generally +available even when it does not publish an exact shutdown date. + +A model may contain at most one distribution for each `via`. Duplicate channel +records are ambiguous and fail runtime semantic validation. Draft-07 can enforce +whole-object uniqueness, but cannot express uniqueness keyed by the `via` field, +so `model-eol validate` and feed loading enforce this constraint. ### Structured replacement contract @@ -119,6 +129,11 @@ months early - or tell an OpenAI-direct shop they're fine because Azure says so. - **Consumers**: anything that can read JSON - a CI step, a dashboard, a Dependabot clone that opens migration PRs when `shutdown - today < threshold`. +Repository tools that export CycloneDX SHOULD represent a canonical model once +per lifecycle channel, with a deterministic channel-qualified `bom-ref`. This +keeps direct publisher and distributor clocks distinct and binds each source +occurrence to the lifecycle decision that evaluated it. + ## Non-goals (v0.1) Pricing, capabilities, context windows (models.dev and friends already do this); diff --git a/action.yml b/action.yml index 8c6096b..68b4039 100644 --- a/action.yml +++ b/action.yml @@ -5,7 +5,7 @@ branding: color: 'red' inputs: command: - description: 'Command to run: check, inventory, schedule, alert, or plan' + description: 'Command to run: check, inventory, schedule, alert, plan, or validate' required: false default: 'check' paths: @@ -40,6 +40,10 @@ inputs: description: 'Path to a model-eol JSON config (defaults to repository config discovery)' required: false default: '' + document-type: + description: 'Optional validate document type: feed, config, check, inventory, schedule, alert, or plan' + required: false + default: '' include-docs: description: 'Scan documentation files in addition to the default source and config files' required: false @@ -72,6 +76,7 @@ runs: MODEL_EOL_FORMAT: ${{ inputs.format }} MODEL_EOL_FEEDS: ${{ inputs.feeds }} MODEL_EOL_CONFIG: ${{ inputs.config }} + MODEL_EOL_DOCUMENT_TYPE: ${{ inputs['document-type'] }} MODEL_EOL_INCLUDE_DOCS: ${{ inputs['include-docs'] }} MODEL_EOL_ALLOW_INCOMPLETE: ${{ inputs['allow-incomplete'] }} MODEL_EOL_JSON: ${{ inputs.json }} @@ -79,6 +84,12 @@ runs: run: | set -euo pipefail + NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]')" + if ! [[ "$NODE_MAJOR" =~ ^[0-9]+$ ]] || [ "$NODE_MAJOR" -lt 22 ]; then + echo "model-eol action: Node 22 or newer is required; found $(node --version 2>/dev/null || echo unknown)" >&2 + exit 2 + fi + split_model_eol_paths() { PATH_ARGS=() if [[ "$MODEL_EOL_PATHS" == *$'\n'* ]]; then @@ -92,9 +103,9 @@ runs: split_model_eol_paths case "$MODEL_EOL_COMMAND" in - check|inventory|schedule|alert|plan) ;; + check|inventory|schedule|alert|plan|validate) ;; *) - echo "model-eol action: command must be check, inventory, schedule, alert, or plan" >&2 + echo "model-eol action: command must be check, inventory, schedule, alert, plan, or validate" >&2 exit 2 ;; esac @@ -106,8 +117,12 @@ runs: echo "model-eol action: format is only supported by the inventory and alert commands" >&2 exit 2 fi + if [ -n "$MODEL_EOL_DOCUMENT_TYPE" ] && [ "$MODEL_EOL_COMMAND" != 'validate' ]; then + echo "model-eol action: document-type is only supported by the validate command" >&2 + exit 2 + fi - ARGS=("$MODEL_EOL_COMMAND" "${PATH_ARGS[@]}") + ARGS=("$MODEL_EOL_COMMAND") if [ -n "$MODEL_EOL_DAYS" ]; then ARGS+=(--days "$MODEL_EOL_DAYS"); fi if [ -n "$MODEL_EOL_VIA" ]; then ARGS+=(--via "$MODEL_EOL_VIA"); fi if [ -n "$MODEL_EOL_CHANGED" ]; then ARGS+=(--changed "$MODEL_EOL_CHANGED"); fi @@ -115,6 +130,7 @@ runs: if [ -n "$MODEL_EOL_FORMAT" ]; then ARGS+=(--format "$MODEL_EOL_FORMAT"); fi if [ -n "$MODEL_EOL_FEEDS" ]; then ARGS+=(--feeds "$MODEL_EOL_FEEDS"); fi if [ -n "$MODEL_EOL_CONFIG" ]; then ARGS+=(--config "$MODEL_EOL_CONFIG"); fi + if [ -n "$MODEL_EOL_DOCUMENT_TYPE" ]; then ARGS+=(--type "$MODEL_EOL_DOCUMENT_TYPE"); fi append_boolean_flag() { local value="$1" @@ -146,11 +162,13 @@ runs: esac case "$MODEL_EOL_JSON" in true) - if [ "$MODEL_EOL_COMMAND" = 'plan' ]; then - echo "model-eol action: json is only supported by the check, inventory, schedule, and alert commands" >&2 - exit 2 - fi - ARGS+=(--json) + case "$MODEL_EOL_COMMAND" in + check|inventory|schedule|alert) ARGS+=(--json) ;; + *) + echo "model-eol action: json is only supported by the check, inventory, schedule, and alert commands" >&2 + exit 2 + ;; + esac ;; false|'') ;; *) @@ -159,6 +177,9 @@ runs: ;; esac + # Keep every consumer-controlled path after the option terminator. + ARGS+=(-- "${PATH_ARGS[@]}") + if [ -z "$MODEL_EOL_OUTPUT_FILE" ]; then exec node "$MODEL_EOL_ACTION_PATH/check.mjs" "${ARGS[@]}" fi diff --git a/bot.yml.example b/bot.yml.example index 6d4a04e..4d366bc 100644 --- a/bot.yml.example +++ b/bot.yml.example @@ -36,9 +36,12 @@ jobs: exit 1 fi printf '%s\n' "$MODEL_EOL_VERSION" > "$RUNNER_TEMP/model-eol-plan/model-eol-version" - PLAN_ARGS=(plan .) - if [ ! -f .model-eol.json ]; then PLAN_ARGS+=(--days 90 --scope direct); fi - npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol "${PLAN_ARGS[@]}" > "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" + # Resolve npx outside the consumer checkout so a repository whose own + # package is named model-eol cannot shadow the requested published bin. + cd "$RUNNER_TEMP/model-eol-plan" + npx --yes --package="model-eol@$MODEL_EOL_VERSION" \ + -c 'cd "$GITHUB_WORKSPACE" && if [ -f .model-eol.json ]; then model-eol plan .; else model-eol plan . --days 90 --scope direct; fi' \ + > "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" - name: Upload plan uses: actions/upload-artifact@v7 with: @@ -55,8 +58,9 @@ jobs: permissions: contents: read steps: - # Provider credentials exist only in this read-only job. The evaluator - # forwards only names explicitly allowed by eval.pass_env. + # Add only the provider secrets required by trusted default-branch eval + # code, and list the same names in eval.pass_env. Environment filtering is + # not an OS sandbox; this job's hard boundary is that it has no write token. - uses: actions/checkout@v7 with: persist-credentials: false @@ -71,10 +75,8 @@ jobs: path: ${{ runner.temp }}/model-eol-plan - name: Apply and evaluate each migration independently env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # Example (opt in only when needed by eval.pass_env): + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} # A non-empty repository variable overrides eval.command. Otherwise # the strictly validated command in .model-eol.json is used. MODEL_EOL_EVAL_COMMAND: ${{ vars.MODEL_EOL_EVAL_COMMAND }} @@ -85,8 +87,9 @@ jobs: echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2 exit 1 fi + cd "$RUNNER_TEMP/model-eol-eval" npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot evaluate \ - --target-dir . \ + --target-dir "$GITHUB_WORKSPACE" \ --plan-file "$RUNNER_TEMP/model-eol-plan/model-eol-plan.json" \ --output-file "$RUNNER_TEMP/model-eol-eval/model-eol-eval-results.json" - name: Upload bounded eval results @@ -134,15 +137,21 @@ jobs: MODEL_EOL_EVAL_COMMAND: ${{ vars.MODEL_EOL_EVAL_COMMAND }} MODEL_EOL_TOKEN_KIND: ${{ secrets.MODEL_EOL_BOT_TOKEN && 'external-token' || 'github-token' }} run: | + mkdir -p "$RUNNER_TEMP/model-eol-publish" MODEL_EOL_VERSION="$(cat "$RUNNER_TEMP/model-eol-plan/model-eol-version")" if [[ ! "$MODEL_EOL_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "refusing invalid resolved model-eol version: $MODEL_EOL_VERSION" >&2 exit 1 fi - npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot --repo "${{ github.repository }}" --target-dir . + cd "$RUNNER_TEMP/model-eol-publish" + npx --yes --package="model-eol@$MODEL_EOL_VERSION" model-eol-bot \ + --repo "${{ github.repository }}" \ + --target-dir "$GITHUB_WORKSPACE" # A PR made with GITHUB_TOKEN does not trigger pull_request workflows. For # required checks, store a narrowly scoped fine-grained PAT or GitHub App token # as MODEL_EOL_BOT_TOKEN. Keep provider keys out of the publish job. +# For migration evidence, copy examples/model-eol-eval.mjs into the consumer, +# customize its dedicated eval command, and set eval.command in .model-eol.json. # Set feeds.allow_vendored_fallback: true only for degraded report-only runs. # Independent concurrent invocations need external mutual exclusion. diff --git a/bot/bot.mjs b/bot/bot.mjs index 61389a3..bb8c684 100755 --- a/bot/bot.mjs +++ b/bot/bot.mjs @@ -22,7 +22,7 @@ import { sha256, stableJson, } from './lib/common.mjs' -import { validatePlanItems } from '../lib/apply.mjs' +import { assertValidPlanDocument } from '../lib/validate-document.mjs' import { parseCliArgs } from '../lib/cli.mjs' import { loadConfig } from './lib/config.mjs' import { downloadFeeds } from './lib/feeds.mjs' @@ -46,6 +46,7 @@ const VENDORED_FEEDS = path.join(ROOT, 'feeds') const PLAN_SCHEMA = 'model-eol.plan/0.1' const BOT_SCHEMA = 'model-eol.bot/0.1' const EVAL_SCHEMA = 'model-eol.eval/0.1' +const EVAL_FAILURE_CHANNEL = 'configured-eval' const EVAL_ARTIFACT_MAX_BYTES = 64 * 1024 * 1024 const ISSUE_REASONS = new Set([ 'not-direct-api', @@ -69,6 +70,8 @@ export const parseArgs = argv => { 'target-dir': { type: 'string' }, config: { type: 'string' }, 'dry-run': { type: 'boolean' }, + // Retained only to emit an actionable migration error. Publication must + // never execute repository-owned eval code in its write-capable process. eval: { type: 'boolean' }, 'feeds-url': { type: 'string', multiple: true }, help: { type: 'boolean', short: 'h' }, @@ -80,7 +83,7 @@ export const parseArgs = argv => { targetDir: values['target-dir'] ?? process.cwd(), configPath: values.config ?? null, dryRun: values['dry-run'] ?? false, - evalEnabled: values.eval ?? false, + deprecatedInlineEval: values.eval ?? false, feedsUrls: [], help: values.help ?? false, } @@ -120,14 +123,15 @@ export const parseEvaluateArgs = argv => { export const helpText = () => `model-eol bot Usage: - node bot/bot.mjs [--repo owner/name] [--target-dir PATH] [--config PATH] [--dry-run] [--eval] [--feeds-url URL[,URL...]] + node bot/bot.mjs [--repo owner/name] [--target-dir PATH] [--config PATH] [--dry-run] [--feeds-url URL[,URL...]] node bot/bot.mjs evaluate --target-dir PATH --plan-file PATH --output-file PATH [--config PATH] [--feeds-url URL[,URL...]] Environment: GITHUB_TOKEN Required unless --dry-run; authenticates the API and HTTPS Git remotes. GITHUB_API_URL GitHub API root, default https://api.github.com. MODEL_EOL_TOKEN_KIND Set to github-token to add the checks warning to PRs. - MODEL_EOL_EVAL_COMMAND Optional command override used by both evaluate and publish. + MODEL_EOL_EVAL_COMMAND Optional override executed by evaluate and digest-checked by publish. + MODEL_EOL_EVAL_RESULTS_FILE Commit-bound result manifest produced by evaluate. ` const validateRepo = repo => { @@ -167,11 +171,8 @@ const validatePlan = plan => { if (plan?.plan_schema !== PLAN_SCHEMA) { throw new Error(`refusing plan schema ${plan?.plan_schema ?? 'missing'}; expected ${PLAN_SCHEMA}`) } - if (!Array.isArray(plan.items) || !Array.isArray(plan.issues)) { - throw new Error('refusing malformed plan document: items and issues arrays are required') - } try { - validatePlanItems(plan.items) + assertValidPlanDocument(plan) } catch (error) { throw new Error(`refusing malformed plan document: ${error.message}`) } @@ -199,57 +200,6 @@ const verifiedPlanArtifact = (file, generatedPlan) => { return artifact } -const readEvalArtifact = (file, maxBytes, statusFile = null) => { - if (!file && !statusFile) return null - let exitCode = null - let statusError = null - if (!statusFile) { - statusError = 'eval status artifact is missing' - } else { - try { - const statusArtifact = readReportCapped(statusFile, 1024) - if (statusArtifact.missing) statusError = 'eval status artifact is missing' - else { - const statusText = statusArtifact.report.trim() - if (!/^-?\d+$/.test(statusText)) statusError = 'eval status artifact is malformed' - else { - const parsed = Number(statusText) - if (!Number.isSafeInteger(parsed)) statusError = 'eval status artifact is outside the safe integer range' - else exitCode = parsed - } - } - } catch (error) { - statusError = `eval status artifact is unreadable: ${error.message}` - } - } - let artifact - try { - artifact = file ? readReportCapped(file, maxBytes) : { missing: true, report: null } - } catch (error) { - return { - status: 'fail', - exit_code: exitCode, - report: `eval report artifact is unreadable: ${error.message}`, - } - } - if (statusError) { - return { - status: 'fail', - exit_code: exitCode, - report: artifact.report ? `${statusError}; report: ${artifact.report}` : statusError, - } - } - if (artifact.missing) { - return { - status: 'fail', - exit_code: exitCode, - report: exitCode === 0 ? 'eval report artifact is missing' : null, - } - } - if (artifact.report === '' && exitCode === 0) return null - return { status: exitCode === 0 ? 'pass' : 'fail', exit_code: exitCode, report: artifact.report || null } -} - const withoutGenerated = plan => { const { generated, ...stable } = plan return stable @@ -318,6 +268,20 @@ const readEvalResults = ({ file, config, commandOverride, plan, groups, baseHead return results } +const boundEvalResults = ({ file, config, commandOverride, plan, groups, baseHead }) => { + const settings = effectiveEval(config, commandOverride) + if (groups.length && settings.command && !file) { + throw new Error('configured eval requires MODEL_EOL_EVAL_RESULTS_FILE from `model-eol-bot evaluate`; publication never executes eval.command') + } + const results = file + ? readEvalResults({ file, config, commandOverride, plan, groups, baseHead }) + : null + return { + results, + configDigest: results && settings.command ? evalConfigDigest(settings) : null, + } +} + const feedContext = feedsDir => { const records = new Map() let files = [] @@ -431,6 +395,34 @@ const buildIssueGroups = (plan, config, root, records) => { return [...groups.values()].sort((a, b) => `${a.publisher}/${a.subject}/${a.shutdown ?? ''}`.localeCompare(`${b.publisher}/${b.subject}/${b.shutdown ?? ''}`)) } +const evalIssueGroupFor = (group, evalResult, root) => ({ + kind: 'issue', + id: group.id, + subject: group.id, + channel: EVAL_FAILURE_CHANNEL, + publisher: group.publisher, + shutdown: group.items[0]?.shutdown ?? null, + via: group.via ?? null, + issues: group.items.map(item => ({ + file: item.file, + line: item.line, + matched: item.matched, + id: item.id, + publisher: item.publisher, + status: item.status, + shutdown: item.shutdown, + requested_via: group.via ?? null, + replacement: item.replacement, + reason: 'eval-failed', + sources: item.sources ?? [], + notes: item.notes ?? null, + })), + root, + context: group.context, + evalResult, + feedDigest: sha256(stableJson({ feed_digest: group.feedDigest, eval_result: evalResult })), +}) + const sourcesFor = group => { const values = group.kind === 'model' ? group.items.flatMap(item => item.sources ?? []) @@ -484,7 +476,7 @@ const replacementSection = (item, now) => [ : []), ] -export const buildPullBody = ({ group, headSha, baseSha = null, now = new Date(), tokenKind = null, evalResult = null }) => { +export const buildPullBody = ({ group, headSha, baseSha = null, now = new Date(), tokenKind = null, evalResult = null, evalConfigHash = null }) => { const item = group.items[0] const announced = markdownText(group.context?.announced ?? 'not specified') const days = daysRemaining(item.shutdown, now) @@ -500,6 +492,7 @@ export const buildPullBody = ({ group, headSha, baseSha = null, now = new Date() base_sha: baseSha, head_sha: headSha, feed_digest: group.feedDigest, + ...(evalConfigHash ? { eval_config_digest: evalConfigHash } : {}), } const sections = [ metadataLine(metadata), @@ -551,7 +544,7 @@ export const buildIssueBody = ({ group, now = new Date() }) => { const evidence = group.issues .map(issue => `- ${markdownText(`${repoPath(issue.file, group.root || '.')}:${issue.line} - ${issue.reason}${issue.matched ? ` (${issue.matched})` : ''}`)}`) .join('\n') - return [ + const sections = [ metadataLine(metadata), '', '## Finding', @@ -578,7 +571,10 @@ export const buildIssueBody = ({ group, now = new Date() }) => { '', '## Feed notes', notesSection(group), - ].join('\n') + ] + const evaluation = evalSection(group.evalResult) + if (evaluation) sections.push('', evaluation) + return sections.join('\n') } const pullTitle = group => `model-eol: migrate ${group.id} before ${group.items[0].shutdown}` @@ -618,7 +614,10 @@ const matchingIssues = (issues, group) => issues if (record.metadata.id !== (group.id || group.subject)) return false if (record.metadata.publisher !== group.publisher) return false if ((record.metadata.via ?? null) !== (group.via ?? null)) return false - return !group.channel || record.metadata.channel === group.channel || (record.metadata.id === group.channel && !record.metadata.channel) + const expectedChannel = group.channel ?? null + const actualChannel = record.metadata.channel ?? null + if (actualChannel === expectedChannel) return true + return expectedChannel !== null && actualChannel === null && record.metadata.id === expectedChannel }) const modelIdentity = (publisher, id, via) => `${publisher}\0${id}\0${via ?? ''}` @@ -656,10 +655,12 @@ const groupFromMetadata = (kind, metadata) => ({ const reconcileStaleWork = async ({ api, pulls, issues, models, issueGroups }) => { const activeModels = new Set(models.map(group => modelIdentity(group.publisher, group.id, group.via))) - const activeIssues = new Set(issueGroups.flatMap(group => [ - issueIdentity(group.publisher, group.id || group.subject, group.via, group.channel), - issueIdentity(group.publisher, group.id || group.subject, group.via, null), - ])) + const activeIssues = new Set(issueGroups.flatMap(group => { + const exact = issueIdentity(group.publisher, group.id || group.subject, group.via, group.channel) + return group.channel === EVAL_FAILURE_CHANNEL + ? [exact] + : [exact, issueIdentity(group.publisher, group.id || group.subject, group.via, null)] + })) const decisions = [] for (const record of ownedPullRecords(pulls, api.repo)) { if (!isOpen(record.item)) continue @@ -676,7 +677,8 @@ const reconcileStaleWork = async ({ api, pulls, issues, models, issueGroups }) = if (!isOpen(record.item)) continue const key = issueIdentity(record.metadata.publisher, record.metadata.id, record.metadata.via, record.metadata.channel) const legacyKey = issueIdentity(record.metadata.publisher, record.metadata.id, record.metadata.via, null) - if (activeIssues.has(key) || activeIssues.has(legacyKey)) continue + const evalFailure = record.metadata.channel === EVAL_FAILURE_CHANNEL + if (activeIssues.has(key) || (!evalFailure && activeIssues.has(legacyKey))) continue await api.comment(record.item.number, staleWorkComment('issue')) await api.updateIssue(record.item.number, { state: 'closed', @@ -780,7 +782,11 @@ export const evaluatePlan = async ({ warn, }) try { + const baseHead = gitHead(targetPath) const generatedPlan = runPlan({ workDir: targetPath, config, configPath: planConfigPath, feedsDir: feedSet.dir, checkerPath, warn }) + if (gitHead(targetPath) !== baseHead) { + throw new Error('refusing eval: target commit changed while generating the migration plan') + } const plan = verifiedPlanArtifact(planFile, generatedPlan) const groups = buildModelGroups(plan, config, targetPath, feedContext(feedSet.dir)) const results = [] @@ -795,6 +801,9 @@ export const evaluatePlan = async ({ let result try { cloneRepository(targetPath, clone) + if (gitHead(clone) !== baseHead) { + throw new Error(`refusing eval: isolated checkout for ${group.publisher}/${group.id} does not match the captured base commit`) + } const applied = spawnSync(process.execPath, [checkerPath, 'apply', '--plan', selectedPlanPath], { cwd: clone, encoding: 'utf8', @@ -849,10 +858,13 @@ export const evaluatePlan = async ({ }) } } + if (gitHead(targetPath) !== baseHead) { + throw new Error('refusing eval: target commit changed during migration evaluation') + } const artifact = { schema: EVAL_SCHEMA, generated: now.toISOString(), - base_sha: gitHead(targetPath), + base_sha: baseHead, plan_digest: planDigest(plan), eval_config_digest: evalConfigDigest(settings), configured: Boolean(settings.command), @@ -865,18 +877,19 @@ export const evaluatePlan = async ({ } } -const makePatch = ({ source, base, branch, expectedHead, allowMissingBranch = false, group, plan, config, checkerPath, evalEnabled, root, warn, gitAuth }) => { +const makePatch = ({ source, base, expectedBaseHead, branch, expectedHead, allowMissingBranch = false, group, plan, checkerPath, gitAuth }) => { const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-bot-work-')) const planRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-plan-')) const clone = path.join(workRoot, 'repo') - const fullPlanPath = path.join(planRoot, 'plan.json') const selectedPlanPath = path.join(planRoot, 'selected-plan.json') - const reportPath = path.join(planRoot, 'eval-report.md') - writeJson(fullPlanPath, plan) writeJson(selectedPlanPath, { ...plan, items: group.items, issues: [] }) try { cloneRepository(source, clone, gitAuth) prepareBranch(clone, branch, base, gitAuth) + const preparedBaseHead = gitHead(clone) + if (preparedBaseHead !== expectedBaseHead) { + throw new Error(`refusing patch: prepared base commit ${preparedBaseHead} does not match evaluated base commit ${expectedBaseHead}`) + } const applied = spawnSync(process.execPath, [checkerPath, 'apply', '--plan', selectedPlanPath], { cwd: clone, encoding: 'utf8', @@ -886,51 +899,12 @@ const makePatch = ({ source, base, branch, expectedHead, allowMissingBranch = fa throw new Error(`apply subprocess failed: ${applied.error?.message || applied.stderr?.trim() || `exit ${applied.status}`}`) } const planFiles = [...new Set(group.items.map(item => repoPath(item.file, clone)))] - const postApplyHashes = hashFiles(clone, planFiles) - const postApplyHead = gitHead(clone) - - let evalResult = null - if (evalEnabled && config.eval.command) { - try { - evalResult = runEvalHook({ - command: config.eval.command, - timeoutMs: config.eval.timeout_ms, - maxReportBytes: config.eval.max_report_bytes, - passEnv: config.eval.pass_env, - cwd: clone, - oldId: group.id, - newId: group.items[0].replacement, - planPath: fullPlanPath, - reportPath, - }) - } catch (error) { - warn(`model-eol: warning: eval runner failed for ${group.id}: ${error.message}`) - evalResult = { status: 'fail', exit_code: null, report: `eval runner error: ${error.message}` } - } - } - if (evalResult) { - const drift = evalWorkspaceDrift(clone, planFiles, postApplyHashes, postApplyHead) - if (drift) { - evalResult = { - ...evalResult, - status: 'fail', - report: capReport([evalResult.report, drift].filter(Boolean).join('\n'), config.eval.max_report_bytes), - } - } - } - if (evalResult && evalResult.status !== 'pass') { - const error = new Error(`eval hook did not pass: ${evalResult.status}`) - error.code = 'MODEL_EOL_EVAL_FAILED' - error.evalResult = evalResult - throw error - } - configureIdentity(clone) const files = planFiles const message = `model-eol: migrate ${group.id} to ${group.items[0].replacement} (${group.feedDigest.slice(0, 8)})` const headSha = commitAll(clone, files, message) pushBranch(clone, branch, expectedHead, gitAuth, { allowMissing: allowMissingBranch }) - return { headSha, evalResult } + return { headSha } } finally { fs.rmSync(workRoot, { recursive: true, force: true }) fs.rmSync(planRoot, { recursive: true, force: true }) @@ -939,7 +913,30 @@ const makePatch = ({ source, base, branch, expectedHead, allowMissingBranch = fa const decision = (group, action, extra = {}) => ({ group, action, ...extra }) -const processModel = async ({ api, pulls, group, source, base, baseHead, plan, config, checkerPath, evalEnabled, externalEval, root, now, tokenKind, warn, gitAuth }) => { +const closePullForEvalFailure = async ({ api, open, evalResult }) => { + await api.comment(open.item.number, `model-eol is closing this bot-owned pull request because its configured migration eval no longer passes (${markdownCode(evalResult.status)}). The migration remains blocked and can be regenerated after the eval clears.`) + await api.updatePull(open.item.number, { + state: 'closed', + body: staleClosedBody(open.item.body, open.metadata), + }) +} + +const processEvalFailure = async ({ api, issueRecords, group, evalResult, root, now, issuesEnabled, open = null }) => { + const evalIssueGroup = evalIssueGroupFor(group, evalResult, root) + const issueDecision = issuesEnabled + ? await processIssue({ api, issues: issueRecords, group: evalIssueGroup, now }) + : null + if (open) await closePullForEvalFailure({ api, open, evalResult }) + return decision(group, 'eval-failed', { + evalResult, + evalIssueGroup: issuesEnabled ? evalIssueGroup : null, + issueAction: issueDecision?.action ?? null, + issueNumber: issueDecision?.number ?? null, + closedPullNumber: open?.item.number ?? null, + }) +} + +const processModel = async ({ api, pulls, issueRecords, group, source, base, baseHead, plan, config, checkerPath, externalEval, evalConfigHash, root, now, tokenKind, gitAuth }) => { const matches = matchingPulls(pulls, group, api.repo) const conflict = matches.find(record => record.conflict) if (conflict) return decision(group, 'conflict', { number: conflict.item.number }) @@ -956,9 +953,9 @@ const processModel = async ({ api, pulls, group, source, base, baseHead, plan, c return decision(group, 'stand-down', { number: open.item.number }) } if (externalEval && externalEval.status !== 'pass') { - return decision(group, 'eval-failed', { number: open.item.number, evalResult: externalEval }) + return processEvalFailure({ api, issueRecords, group, evalResult: externalEval, root, now, issuesEnabled: config.issues.enabled, open }) } - if (open.metadata.feed_digest === currentDigest && open.metadata.base_sha === baseHead) { + if (open.metadata.feed_digest === currentDigest && open.metadata.base_sha === baseHead && (!evalConfigHash || open.metadata.eval_config_digest === evalConfigHash)) { return decision(group, 'skip-unchanged', { number: open.item.number }) } let patch @@ -966,36 +963,26 @@ const processModel = async ({ api, pulls, group, source, base, baseHead, plan, c patch = makePatch({ source, base, + expectedBaseHead: baseHead, branch: open.item.head?.ref || group.branch, expectedHead: open.metadata.head_sha, group, plan, - config, checkerPath, - evalEnabled, - root, - warn, gitAuth, }) } catch (error) { - if (error.code === 'MODEL_EOL_EVAL_FAILED') { - return decision(group, 'eval-failed', { number: open.item.number, evalResult: error.evalResult }) - } if (error.code !== 'MODEL_EOL_BRANCH_STAND_DOWN') throw error await api.comment(open.item.number, standingDownComment(group, open.metadata, error.currentHead)) return decision(group, 'stand-down', { number: open.item.number }) } - if (patch.evalResult && patch.evalResult.status !== 'pass') { - return decision(group, 'eval-failed', { number: open.item.number, evalResult: patch.evalResult }) - } - if (!patch.evalResult && externalEval) patch.evalResult = externalEval - const body = buildPullBody({ group, headSha: patch.headSha, baseSha: baseHead, now, tokenKind, evalResult: patch.evalResult }) + const body = buildPullBody({ group, headSha: patch.headSha, baseSha: baseHead, now, tokenKind, evalResult: externalEval, evalConfigHash }) await api.updatePull(open.item.number, { title: pullTitle(group), body }) return decision(group, 'update', { number: open.item.number, body, headSha: patch.headSha }) } if (externalEval && externalEval.status !== 'pass') { - return decision(group, 'eval-failed', { evalResult: externalEval }) + return processEvalFailure({ api, issueRecords, group, evalResult: externalEval, root, now, issuesEnabled: config.issues.enabled }) } const replacement = group.items[0].replacement @@ -1017,32 +1004,22 @@ const processModel = async ({ api, pulls, group, source, base, baseHead, plan, c patch = makePatch({ source, base, + expectedBaseHead: baseHead, branch: group.branch, expectedHead: previousBotHead, allowMissingBranch: Boolean(previousBotHead && !isOpen(previous.item)), group, plan, - config, checkerPath, - evalEnabled, - root, - warn, gitAuth, }) } catch (error) { - if (error.code === 'MODEL_EOL_EVAL_FAILED') { - return decision(group, 'eval-failed', { evalResult: error.evalResult }) - } if (error.code !== 'MODEL_EOL_BRANCH_STAND_DOWN') throw error const prior = matches.find(record => record.metadata?.head_sha === previousBotHead) if (prior) await api.comment(prior.item.number, standingDownComment(group, prior.metadata, error.currentHead)) return decision(group, 'stand-down', { number: prior?.item.number }) } - if (patch.evalResult && patch.evalResult.status !== 'pass') { - return decision(group, 'eval-failed', { evalResult: patch.evalResult }) - } - if (!patch.evalResult && externalEval) patch.evalResult = externalEval - const body = buildPullBody({ group, headSha: patch.headSha, baseSha: baseHead, now, tokenKind, evalResult: patch.evalResult }) + const body = buildPullBody({ group, headSha: patch.headSha, baseSha: baseHead, now, tokenKind, evalResult: externalEval, evalConfigHash }) const latestPulls = await api.listPullsByHead(group.branch) const latestMatches = matchingPulls(latestPulls, group, api.repo) const latestConflict = latestMatches.find(record => record.conflict) @@ -1093,9 +1070,9 @@ const processIssue = async ({ api, issues, group, now }) => { return decision(group, 'create', { number: created.number, body }) } -const dryDecision = (group, now, tokenKind) => group.kind === 'model' +const dryDecision = (group, now, tokenKind, evalResult = null, evalConfigHash = null) => group.kind === 'model' ? decision(group, 'create', { - body: buildPullBody({ group, headSha: 'dry-run', now, tokenKind }), + body: buildPullBody({ group, headSha: 'dry-run', now, tokenKind, evalResult, evalConfigHash }), branch: group.branch, }) : decision(group, 'create', { body: buildIssueBody({ group, now }) }) @@ -1178,6 +1155,9 @@ export const runBot = async ({ targetDir = process.cwd(), configPath = null, dryRun = false, + deprecatedInlineEval = false, + // Programmatic compatibility aliases are accepted only to fail with the + // same migration guidance as the removed CLI/environment contracts. evalEnabled = false, feedsUrls = [], token = process.env.GITHUB_TOKEN, @@ -1195,6 +1175,12 @@ export const runBot = async ({ now = new Date(), warn = message => console.error(message), } = {}) => { + if (deprecatedInlineEval || evalEnabled) { + throw new Error('inline --eval was removed because publication is write-capable; run `model-eol-bot evaluate` in a read-only job and pass its manifest with MODEL_EOL_EVAL_RESULTS_FILE') + } + if (evalReportFile || evalStatusFile) { + throw new Error('MODEL_EOL_EVAL_REPORT_FILE and MODEL_EOL_EVAL_STATUS_FILE were removed because they are not commit-bound; use MODEL_EOL_EVAL_RESULTS_FILE from `model-eol-bot evaluate`') + } if (!dryRun) { validateRepo(repo) if (!token) throw new Error('GITHUB_TOKEN is required unless --dry-run is used') @@ -1226,8 +1212,25 @@ export const runBot = async ({ const records = feedContext(feedSet.dir) const models = buildModelGroups(plan, config, scan.path, records) const issues = config.issues.enabled ? buildIssueGroups(plan, config, scan.path, records) : [] - const report = group => feedSet.degraded ? reportOnlyDecision(group, now, tokenKind) : dryDecision(group, now, tokenKind) - const decisions = [...models.map(report), ...issues.map(report)] + if (feedSet.degraded) { + const report = group => reportOnlyDecision(group, now, tokenKind) + const decisions = [...models.map(report), ...issues.map(report)] + return { plan, config, decisions, feedsDir: feedSet.dir, degraded: true } + } + const boundEval = boundEvalResults({ + file: evalResultsFile, + config, + commandOverride: evalCommandOverride, + plan, + groups: models, + baseHead: evalResultsFile ? gitHead(scan.path) : null, + }) + const decisions = models.map(group => { + const evalResult = boundEval.results?.get(evalIdentity(group.publisher, group.id, group.via)) ?? null + if (evalResult && evalResult.status !== 'pass') return decision(group, 'eval-failed', { evalResult }) + return dryDecision(group, now, tokenKind, evalResult, boundEval.configDigest) + }) + decisions.push(...issues.map(group => dryDecision(group, now, tokenKind))) return { plan, config, decisions, feedsDir: feedSet.dir, degraded: Boolean(feedSet.degraded) } } @@ -1257,27 +1260,29 @@ export const runBot = async ({ const decisions = [...models.map(report), ...issues.map(report)] return { plan, config, decisions, feedsDir: feedSet.dir, degraded: true } } - const externalEvalResults = evalResultsFile - ? readEvalResults({ file: evalResultsFile, config, commandOverride: evalCommandOverride, plan, groups: models, baseHead }) - : null - const legacyExternalEval = externalEvalResults ? null : readEvalArtifact( - evalReportFile ? asAbsolute(evalReportFile) : null, - config.eval.max_report_bytes, - evalStatusFile ? asAbsolute(evalStatusFile) : null, - ) + const boundEval = boundEvalResults({ + file: evalResultsFile, + config, + commandOverride: evalCommandOverride, + plan, + groups: models, + baseHead, + }) const api = new GitHubClient({ repo, apiUrl, token, transport, warn }) const labelReady = models.length || issues.length ? await api.ensureModelEolLabel() : true const pulls = await api.listPulls() const issueRecords = await api.listIssues() const decisions = [] + const evalIssueGroups = [] if (!labelReady) { for (const group of [...models, ...issues]) decisions.push(decision(group, 'label-unavailable')) } else { for (const group of models) { - const externalEval = externalEvalResults?.get(evalIdentity(group.publisher, group.id, group.via)) ?? legacyExternalEval + const externalEval = boundEval.results?.get(evalIdentity(group.publisher, group.id, group.via)) ?? null decisions.push(await processModel({ api, pulls, + issueRecords, group, source, base, @@ -1285,18 +1290,21 @@ export const runBot = async ({ plan, config, checkerPath, - evalEnabled, externalEval, + evalConfigHash: boundEval.configDigest, root: baseClone, now, tokenKind, - warn, gitAuth, })) + const latestDecision = decisions.at(-1) + if (latestDecision?.evalIssueGroup) evalIssueGroups.push(latestDecision.evalIssueGroup) } for (const group of issues) decisions.push(await processIssue({ api, issues: issueRecords, group, now })) } - decisions.push(...await reconcileStaleWork({ api, pulls, issues: issueRecords, models, issueGroups: issues })) + if (labelReady) { + decisions.push(...await reconcileStaleWork({ api, pulls, issues: issueRecords, models, issueGroups: [...issues, ...evalIssueGroups] })) + } return { plan, config, decisions, feedsDir: feedSet.dir, degraded: false } } finally { scan?.cleanup() @@ -1334,7 +1342,7 @@ export const main = async (argv = process.argv.slice(2), env = process.env) => { targetDir: options.targetDir, configPath: options.configPath, dryRun: options.dryRun, - evalEnabled: options.evalEnabled, + deprecatedInlineEval: options.deprecatedInlineEval, feedsUrls: options.feedsUrls, planFile: env.MODEL_EOL_PLAN_FILE || null, evalReportFile: env.MODEL_EOL_EVAL_REPORT_FILE || null, diff --git a/bot/test/package.mjs b/bot/test/package.mjs index c50e9f8..ab6e39d 100644 --- a/bot/test/package.mjs +++ b/bot/test/package.mjs @@ -5,6 +5,8 @@ import os from 'node:os' import path from 'node:path' import { spawnSync } from 'node:child_process' +import { verifyPackageIntegrity } from '../../scripts/package-integrity.mjs' + const root = path.resolve(import.meta.dirname, '../..') const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-package-test-')) const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm' @@ -34,6 +36,7 @@ try { } assert(manifest.bin?.['model-eol'] === 'check.mjs', 'package exposes the model-eol checker bin') assert(manifest.bin?.['model-eol-bot'] === 'bot/bot.mjs', 'package exposes the model-eol-bot bin') + assert(manifest.engines?.node === '>=22', 'package declares the supported Node 22 floor') const workflow = fs.readFileSync(path.join(root, 'bot.yml.example'), 'utf8') for (const [action, major, count] of [ ['checkout', 'v7', 3], @@ -51,18 +54,49 @@ try { const source = fs.readFileSync(path.join(root, file), 'utf8') const references = [...source.matchAll(/actions\/checkout@(v\d+)/g)].map(match => match[1]) assert(references.length === count && references.every(reference => reference === 'v7'), `${file} uses the Node 24 checkout@v7 action`) + const setupReferences = [...source.matchAll(/actions\/setup-node@(v\d+)/g)].map(match => match[1]) + assert(setupReferences.length === count && setupReferences.every(reference => reference === 'v7'), `${file} sets up a supported Node runtime before every composite Action use`) + assert(source.match(/node-version: 22/g)?.length === count, `${file} pins every composite Action job to Node 22`) + assert(source.match(/package-manager-cache: false/g)?.length === count, `${file} disables package-manager caching for every composite Action job`) } + const readme = fs.readFileSync(path.join(root, 'README.md'), 'utf8') + assert( + readme.includes('repo="$(pwd -P)"') + && readme.includes('cd "$tmp"') + && readme.match(/MODEL_EOL_UAT_REPO="\$repo"/g)?.length === 2 + && readme.includes(`-c 'cd "$MODEL_EOL_UAT_REPO" && model-eol plan .'`) + && readme.includes(`-c 'cd "$MODEL_EOL_UAT_REPO" && model-eol plan . --days 90 --scope direct'`) + && readme.match(/--target-dir "\$repo"/g)?.length >= 2, + 'README resolves published bins externally while planning from the consumer repository for clone-portable paths', + ) assert(workflow.match(/node-version: 22/g)?.length === 3, 'consumer workflow runs every model-eol job on supported Node 22') assert(workflow.match(/package-manager-cache: false/g)?.length === 3, 'consumer workflow disables automatic package-manager caching in every job') - const repositoryWorkflows = [ - '.github/workflows/ci.yml', - '.github/workflows/feed-refresh.yml', - '.github/workflows/npm-release.yml', - ].map(file => fs.readFileSync(path.join(root, file), 'utf8')).join('\n') - assert(repositoryWorkflows.match(/actions\/setup-node@v7/g)?.length === 4, 'repository workflows use the Node 24 setup-node@v7 action') - assert(repositoryWorkflows.match(/package-manager-cache: false/g)?.length === 4, 'repository workflows disable automatic package-manager caching in every job') + const repositoryWorkflows = fs.readdirSync(path.join(root, '.github', 'workflows')) + .filter(file => file.endsWith('.yml') || file.endsWith('.yaml')) + .map(file => fs.readFileSync(path.join(root, '.github', 'workflows', file), 'utf8')) + .join('\n') + for (const [action, major] of [ + ['checkout', 'v7'], + ['setup-node', 'v7'], + ['upload-artifact', 'v7'], + ['download-artifact', 'v8'], + ['configure-pages', 'v6'], + ['upload-pages-artifact', 'v5'], + ['deploy-pages', 'v5'], + ]) { + const references = [...repositoryWorkflows.matchAll(new RegExp(`actions/${action}@(v\\d+)`, 'g'))].map(match => match[1]) + if (references.length) assert(references.every(reference => reference === major), `repository workflows use actions/${action}@${major}`) + } + const setupNodeCount = repositoryWorkflows.match(/actions\/setup-node@v7/g)?.length ?? 0 + assert(setupNodeCount > 0 && repositoryWorkflows.match(/package-manager-cache: false/g)?.length === setupNodeCount, 'repository workflows disable automatic package-manager caching in every Node job') + assert(repositoryWorkflows.includes('node scripts/published-consumer-uat.mjs'), 'release automation smoke-tests the exact published package') + assert(repositoryWorkflows.includes('uses: thossullivan/model-eol@v0'), 'hosted consumer UAT exercises the moving v0 Action') + assert(repositoryWorkflows.includes('name: npm-release-result') && repositoryWorkflows.includes('run-id: ${{ github.event.workflow_run.id }}'), 'hosted consumer UAT receives the exact release version artifact') + assert(repositoryWorkflows.includes('Moving v0 Action validate round-trip UAT'), 'hosted moving v0 Action validates its emitted inventory') + assert(repositoryWorkflows.includes('Immutable release Action validate round-trip UAT'), 'hosted UAT validates the immutable release Action before monitoring moving v0') + assert(repositoryWorkflows.includes('--expected-integrity "$INTEGRITY"') && repositoryWorkflows.includes('npm publish "$RELEASE_TARBALL" --ignore-scripts'), 'hosted release UAT binds the installed package to the exact published tarball') assert(workflow.includes('MODEL_EOL_PACKAGE: model-eol@0'), 'consumer workflow uses the model-eol v0 package line') - assert(workflow.includes('model-eol "${PLAN_ARGS[@]}"') && workflow.includes('model-eol-bot --repo'), 'consumer workflow invokes both published bins') + assert(workflow.includes(`-c 'cd "$GITHUB_WORKSPACE" && if [ -f .model-eol.json ]; then model-eol plan .`) && workflow.includes('--target-dir "$GITHUB_WORKSPACE"'), 'consumer workflow invokes both published bins outside self-shadowing consumer package resolution') assert(!workflow.includes('node check.mjs') && !workflow.includes('node bot/bot.mjs'), 'consumer workflow has no repository-local tool assumption') assert(workflow.match(/persist-credentials: false/g)?.length === 3, 'all workflow jobs disable persisted checkout credentials') assert(workflow.match(/npm view "\$MODEL_EOL_PACKAGE" version/g)?.length === 1, 'workflow resolves the moving major package line exactly once') @@ -72,6 +106,7 @@ try { assert(workflow.includes('model-eol-version'), 'resolved exact package version is carried as a workflow artifact') const publishJob = workflow.slice(workflow.indexOf('\n publish:')) assert(!publishJob.includes('OPENAI_API_KEY') && !publishJob.includes('ANTHROPIC_API_KEY') && !publishJob.includes('GOOGLE_API_KEY') && !publishJob.includes('GEMINI_API_KEY'), 'write-capable publish job receives no provider API keys') + assert(!/^\s+(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY|GEMINI_API_KEY):/m.test(workflow), 'copy-ready workflow injects no provider secret until the consumer explicitly opts in') assert(workflow.includes('model-eol-bot evaluate') && workflow.includes('MODEL_EOL_EVAL_RESULTS_FILE'), 'consumer workflow uses the isolated evaluator manifest contract') const packResult = run(npm, ['pack', root, '--json', '--ignore-scripts'], { cwd: tempRoot }) @@ -93,6 +128,8 @@ try { 'bot/lib/feeds.mjs', 'bot/lib/git.mjs', 'bot/lib/github.mjs', + 'examples/model-eol-eval.mjs', + 'schema/model-eol.check.schema.json', ]) { assert(packedFiles.has(file), `packed artifact contains ${file}`) } @@ -105,13 +142,19 @@ try { const consumer = path.join(tempRoot, 'consumer') const npmCache = path.join(tempRoot, 'npm-cache') fs.mkdirSync(consumer) - fs.writeFileSync(path.join(consumer, 'app.py'), 'model = "o3-deep-research-2025-06-26"\n') + fs.writeFileSync(path.join(consumer, 'app.py'), [ + 'from openai import OpenAI', + 'client = OpenAI()', + 'result = client.responses.create(model="o3-deep-research-2025-06-26", input="test")', + '', + ].join('\n')) run('git', ['init', '-b', 'main'], { cwd: consumer }) run('git', ['config', 'user.name', 'package-contract'], { cwd: consumer }) run('git', ['config', 'user.email', 'package-contract@example.invalid'], { cwd: consumer }) run('git', ['add', 'app.py'], { cwd: consumer }) run('git', ['commit', '-m', 'consumer fixture'], { cwd: consumer }) const tarball = path.join(tempRoot, packed.filename) + assert(verifyPackageIntegrity({ tarball, expectedIntegrity: packed.integrity }) === packed.integrity, 'release integrity verifier matches npm pack bytes') run(npm, [ 'install', '--ignore-scripts', @@ -122,20 +165,62 @@ try { tarball, ], { cwd: consumer }) + const installedRoot = path.join(consumer, 'node_modules', 'model-eol') + const consumerManifestFile = path.join(consumer, 'package.json') + const consumerManifest = JSON.parse(fs.readFileSync(consumerManifestFile, 'utf8')) + consumerManifest.scripts = { 'eval:model-eol': 'node verify-model-swap.mjs' } + fs.writeFileSync(consumerManifestFile, `${JSON.stringify(consumerManifest, null, 2)}\n`) + fs.copyFileSync(path.join(installedRoot, 'examples', 'model-eol-eval.mjs'), path.join(consumer, 'model-eol-eval.mjs')) + fs.writeFileSync(path.join(consumer, '.model-eol.json'), '{"eval":{"command":"node model-eol-eval.mjs"}}\n') + fs.writeFileSync(path.join(consumer, 'verify-model-swap.mjs'), [ + "import fs from 'node:fs'", + "const source = fs.readFileSync(new URL('./app.py', import.meta.url), 'utf8')", + "const oldId = process.env.MODEL_EOL_OLD_ID", + "const newId = process.env.MODEL_EOL_NEW_ID", + "if (!oldId || !newId || source.includes(oldId) || !source.includes(newId)) process.exit(1)", + '', + ].join('\n')) + run('git', ['add', '.model-eol.json', 'model-eol-eval.mjs', 'package.json', 'package-lock.json', 'verify-model-swap.mjs'], { cwd: consumer }) + run('git', ['commit', '-m', 'configure packed eval harness'], { cwd: consumer }) const binSuffix = process.platform === 'win32' ? '.cmd' : '' const checkerBin = path.join(consumer, 'node_modules', '.bin', `model-eol${binSuffix}`) const botBin = path.join(consumer, 'node_modules', '.bin', `model-eol-bot${binSuffix}`) const checker = run(checkerBin, ['inventory', consumer], { cwd: consumer }) assert(checker.stdout.includes('o3-deep-research'), 'clean consumer runs the packed model-eol checker bin') - const bot = run(botBin, ['--dry-run', '--target-dir', consumer, '--repo', 'example/consumer'], { cwd: consumer }) - assert(bot.stdout.includes('model-eol decision table'), 'clean consumer runs the packed model-eol-bot bin') + const packedCheck = spawnSync(checkerBin, ['check', consumer, '--json'], { + cwd: consumer, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }) + assert(!packedCheck.error && packedCheck.status === 1, 'packed check --json preserves its finding exit code') + const packedCheckDocument = JSON.parse(packedCheck.stdout) + assert(packedCheckDocument.schema === 'model-eol/check@0.1' && packedCheckDocument.findings.length > 0, 'packed check emits the public check discriminator and findings') + const packedCheckFile = path.join(tempRoot, 'packed-check.json') + fs.writeFileSync(packedCheckFile, packedCheck.stdout) + const packedCheckValidation = run(checkerBin, ['validate', packedCheckFile], { cwd: consumer }) + assert(packedCheckValidation.stdout.includes('valid check document'), 'packed consumer validates its emitted check artifact') + const validation = run(checkerBin, ['validate', path.join(installedRoot, 'feeds', 'openai.json')], { cwd: consumer }) + assert(validation.stdout.includes('valid feed document'), 'clean consumer runs the packed public document validator') const packedPlan = run(checkerBin, ['plan', '.', '--days', '90', '--scope', 'direct'], { cwd: consumer }) + const packedPlanDocument = JSON.parse(packedPlan.stdout) const packedPlanFile = path.join(tempRoot, 'packed-plan.json') const packedEvalFile = path.join(tempRoot, 'packed-eval.json') fs.writeFileSync(packedPlanFile, packedPlan.stdout) run(botBin, ['evaluate', '--target-dir', consumer, '--plan-file', packedPlanFile, '--output-file', packedEvalFile], { cwd: consumer }) const packedEval = JSON.parse(fs.readFileSync(packedEvalFile, 'utf8')) - assert(packedEval.schema === 'model-eol.eval/0.1' && packedEval.configured === false && /^[0-9a-f]{40,64}$/.test(packedEval.base_sha), 'clean consumer runs the packed commit-bound isolated-evaluator entrypoint') + assert(packedEval.schema === 'model-eol.eval/0.1' && packedEval.configured === true && /^[0-9a-f]{40,64}$/.test(packedEval.base_sha), 'clean consumer runs the packed configured commit-bound evaluator') + assert(packedEval.results.length === 1 && packedEval.results[0].status === 'pass' && packedEval.results[0].report?.includes('verification command passed'), 'packed eval starter emits one bounded passing report') + const bot = run(botBin, ['--dry-run', '--target-dir', consumer, '--repo', 'example/consumer'], { + cwd: consumer, + env: { ...process.env, MODEL_EOL_EVAL_RESULTS_FILE: packedEvalFile }, + }) + const migration = packedPlanDocument.items[0] + assert( + bot.stdout.includes(`- create ${migration.publisher}/${migration.id}`) + && bot.stdout.includes('Result: pass (exit code 0).') + && bot.stdout.includes(migration.replacement), + 'clean consumer authorizes the exact planned migration only from the packed passing evaluator manifest', + ) } finally { fs.rmSync(tempRoot, { recursive: true, force: true }) } diff --git a/bot/test/run.mjs b/bot/test/run.mjs index 4a515c6..5f07886 100644 --- a/bot/test/run.mjs +++ b/bot/test/run.mjs @@ -38,6 +38,15 @@ const assert = (condition, message) => { const unknownFlag = spawnSync(process.execPath, [path.join(root, 'bot', 'bot.mjs'), '--dyas', '90'], { encoding: 'utf8' }) assert(unknownFlag.status === 2 && unknownFlag.stderr.includes('--dyas') && unknownFlag.stderr.includes('--help'), 'unknown bot flags exit 2 with the bad flag and help hint') +const deprecatedInlineEval = spawnSync(process.execPath, [path.join(root, 'bot', 'bot.mjs'), '--eval', '--dry-run'], { encoding: 'utf8' }) +assert(deprecatedInlineEval.status === 2 && deprecatedInlineEval.stderr.includes('inline --eval was removed') && deprecatedInlineEval.stderr.includes('MODEL_EOL_EVAL_RESULTS_FILE'), 'removed inline --eval fails with read-only evaluate migration guidance') + +const deprecatedEvalArtifacts = spawnSync(process.execPath, [path.join(root, 'bot', 'bot.mjs'), '--dry-run'], { + encoding: 'utf8', + env: { ...process.env, MODEL_EOL_EVAL_REPORT_FILE: 'legacy-report.md' }, +}) +assert(deprecatedEvalArtifacts.status === 2 && deprecatedEvalArtifacts.stderr.includes('not commit-bound') && deprecatedEvalArtifacts.stderr.includes('MODEL_EOL_EVAL_RESULTS_FILE'), 'removed unbound eval report/status contract fails with a bound-manifest migration hint') + const git = (cwd, args) => { const result = spawnSync('git', args, { cwd, encoding: 'utf8' }) if (result.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${result.stderr}`) @@ -72,6 +81,44 @@ const makeRepo = ({ name = 'repo', files = {}, config = null } = {}) => { return { bare, work } } +let boundArtifactSequence = 0 +const evaluateForPublish = async ({ + targetDir, + configPath = null, + vendoredFeeds = path.join(root, 'feeds'), + checkerPath, + commandOverride = null, + now = new Date('2026-08-01T00:00:00Z'), +} = {}) => { + const selectedConfig = configPath ?? path.join(targetDir, '.model-eol.json') + const config = loadConfig(selectedConfig) + const planConfigPath = fs.existsSync(selectedConfig) ? selectedConfig : null + const plan = runPlan({ + workDir: targetDir, + config, + configPath: planConfigPath, + feedsDir: vendoredFeeds, + checkerPath, + warn: () => {}, + }) + const suffix = ++boundArtifactSequence + const planFile = path.join(tempRoot, `bound-plan-${suffix}.json`) + const evalResultsFile = path.join(tempRoot, `bound-eval-${suffix}.json`) + write(planFile, JSON.stringify(plan, null, 2)) + const evaluation = await evaluatePlan({ + targetDir, + configPath, + planFile, + outputFile: evalResultsFile, + vendoredFeeds, + checkerPath, + commandOverride, + now, + warn: () => {}, + }) + return { plan, planFile, evalResultsFile, evaluation } +} + const bareBranchFile = (repo, branch, file) => git(repo.bare, ['show', `${branch}:${file}`]) const headOf = (repo, branch) => git(repo.bare, ['rev-parse', `refs/heads/${branch}`]) @@ -174,6 +221,9 @@ const config = { days: 90, scope: 'direct', issues: { enabled: false }, +} +const lifecycleConfig = { + ...config, eval: { command: 'node -e \'require("fs").writeFileSync(process.env.MODEL_EOL_REPORT, "eval report with ```fence```")\'', timeout_ms: 1000, @@ -181,19 +231,25 @@ const config = { pass_env: [], }, } -const repo = makeRepo({ name: 'lifecycle', files: baseFiles, config }) +const repo = makeRepo({ name: 'lifecycle', files: baseFiles, config: lifecycleConfig }) const github = new FakeGitHub() -const run = options => runBot({ - repo: 'example/app', - targetDir: repo.work, - token: 'test-token', - transport: github.transport.bind(github), - vendoredFeeds: path.join(root, 'feeds'), - now: new Date('2026-08-01T00:00:00Z'), - ...options, -}) +const run = async (options = {}) => { + const vendoredFeeds = options.vendoredFeeds ?? path.join(root, 'feeds') + const artifacts = await evaluateForPublish({ targetDir: repo.work, vendoredFeeds }) + return runBot({ + repo: 'example/app', + targetDir: repo.work, + token: 'test-token', + transport: github.transport.bind(github), + vendoredFeeds, + planFile: artifacts.planFile, + evalResultsFile: artifacts.evalResultsFile, + now: new Date('2026-08-01T00:00:00Z'), + ...options, + }) +} -const first = await run({ evalEnabled: true, tokenKind: 'github-token' }) +const first = await run({ tokenKind: 'github-token' }) const firstDecision = first.decisions.find(item => item.group.kind === 'model') const branch = branchFor('openai', 'o3-deep-research-2025-06-26') const createCall = github.callsFor('POST', '/pulls')[0] @@ -302,7 +358,7 @@ assert(clockPublisherDecision?.action === 'create' && clockDistributorDecision?. assert(clockGithub.callsFor('POST', '/pulls').length === 2 && clockGithub.pulls[0].head.ref !== clockGithub.pulls[1].head.ref, 'clock-specific PR matching does not reuse the publisher-clock PR') const callsAfterCreate = github.calls.length -const unchanged = await run({ evalEnabled: true }) +const unchanged = await run() assert(unchanged.decisions.find(item => item.group.kind === 'model')?.action === 'skip-unchanged', 'unchanged feed digest does nothing') assert(github.calls.length > callsAfterCreate && github.callsFor('PATCH', '/pulls').length === 0, 'unchanged state only performs discovery') @@ -329,6 +385,31 @@ const baseFreshSecondDecision = baseFreshSecond.decisions.find(item => item.grou assert(baseFreshSecondDecision?.action === 'update', 'unchanged finding is regenerated when its recorded default-branch base is stale') assert(parseMetadata(baseFreshSecondDecision?.body)?.base_sha === headOf(baseFreshRepo, 'main'), 'refreshed PR metadata records the independently cloned default-branch head') +const publishWindowRepo = makeRepo({ name: 'publish-window-drift', files: baseFiles, config: { issues: { enabled: false } } }) +const publishWindowGithub = new FakeGitHub() +publishWindowGithub.beforeListPulls = (_api, count) => { + if (count !== 1) return + write(path.join(publishWindowRepo.work, 'README.md'), 'default branch moved after the publisher captured its base\n') + git(publishWindowRepo.work, ['add', 'README.md']) + git(publishWindowRepo.work, ['commit', '-m', 'move base during publish']) + git(publishWindowRepo.work, ['push', 'origin', 'main']) +} +let publishWindowError = null +try { + await runBot({ + repo: 'example/publish-window-drift', + targetDir: publishWindowRepo.work, + token: 'test-token', + transport: publishWindowGithub.transport.bind(publishWindowGithub), + vendoredFeeds: path.join(root, 'feeds'), + now: new Date('2026-08-01T00:00:00Z'), + }) +} catch (error) { + publishWindowError = error +} +assert(publishWindowError?.message.includes('prepared base commit') && publishWindowError.message.includes('does not match evaluated base commit'), 'publisher refuses a patch checkout when the default branch moves after plan and eval verification') +assert(publishWindowGithub.callsFor('POST', '/pulls').length === 0 && !gitTry(publishWindowRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'publish-window base drift fails before patch branch push or pull-request creation') + const staleRepo = makeRepo({ name: 'stale-reconciliation', files: baseFiles, config: { issues: { enabled: false } } }) const staleGithub = new FakeGitHub() const staleRun = () => runBot({ @@ -417,7 +498,7 @@ const replacementChanged = await replacementRun({ vendoredFeeds: replacementYFee assert(replacementChanged.decisions.find(item => item.group.kind === 'model')?.action === 'create' && replacementGithub.callsFor('POST', '/pulls').length === 2, 'changed replacement creates a fresh PR instead of reusing dismissal') const beforeUpdateHead = headOf(repo, branch) -const updated = await run({ evalEnabled: true, vendoredFeeds: changedFeeds }) +const updated = await run({ vendoredFeeds: changedFeeds }) const updateDecision = updated.decisions.find(item => item.group.kind === 'model') assert(updateDecision?.action === 'update', 'changed digest with intact bot head updates the PR') assert(headOf(repo, branch) !== beforeUpdateHead, 'changed digest force-pushes a regenerated branch') @@ -732,30 +813,30 @@ if (fifoAvailable) { assert(true, 'FIFO eval test skipped because mkfifo is unavailable') } -const externalEvalRepo = makeRepo({ name: 'external-eval-status', files: baseFiles, config: { issues: { enabled: false } } }) -const externalEvalGithub = new FakeGitHub() -const externalReport = path.join(tempRoot, 'external-eval-report.md') -const externalStatus = path.join(tempRoot, 'external-eval-status.txt') -const externalRun = () => runBot({ - repo: 'example/external-eval-status', - targetDir: externalEvalRepo.work, - token: 'test-token', - transport: externalEvalGithub.transport.bind(externalEvalGithub), - vendoredFeeds: path.join(root, 'feeds'), - evalReportFile: externalReport, - evalStatusFile: externalStatus, - now: new Date('2026-08-01T00:00:00Z'), +const missingBoundRepo = makeRepo({ + name: 'missing-bound-eval', + files: baseFiles, + config: { + issues: { enabled: false }, + eval: { command: 'node scripts/model-eol-eval.mjs' }, + }, }) -write(externalReport, 'external report') -write(externalStatus, '0garbage') -const malformedExternal = await externalRun() -assert(malformedExternal.decisions.find(item => item.group.kind === 'model')?.action === 'eval-failed', 'malformed external eval status fails closed') -fs.rmSync(externalStatus, { force: true }) -const missingExternal = await externalRun() -assert(missingExternal.decisions.find(item => item.group.kind === 'model')?.action === 'eval-failed', 'missing external eval status fails even with a report') -write(externalStatus, '0') -const cleanExternal = await externalRun() -assert(cleanExternal.decisions.find(item => item.group.kind === 'model')?.action === 'create' && externalEvalGithub.callsFor('POST', '/pulls').length === 1, 'strict zero external eval status permits the patch') +const missingBoundGithub = new FakeGitHub() +let missingBoundError = null +try { + await runBot({ + repo: 'example/missing-bound-eval', + targetDir: missingBoundRepo.work, + token: 'test-token', + transport: missingBoundGithub.transport.bind(missingBoundGithub), + vendoredFeeds: path.join(root, 'feeds'), + now: new Date('2026-08-01T00:00:00Z'), + }) +} catch (error) { + missingBoundError = error +} +assert(missingBoundError?.message.includes('configured eval requires MODEL_EOL_EVAL_RESULTS_FILE'), 'configured publication refuses to bypass the read-only evaluator') +assert(missingBoundGithub.calls.length === 0 && !gitTry(missingBoundRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'missing bound eval results fail before GitHub API calls or patch pushes') const evalMissingRepo = makeRepo({ name: 'eval-missing-report', @@ -766,13 +847,15 @@ const evalMissingRepo = makeRepo({ }, }) const evalMissingGithub = new FakeGitHub() +const evalMissingArtifacts = await evaluateForPublish({ targetDir: evalMissingRepo.work }) const evalMissingResult = await runBot({ repo: 'example/eval-missing-report', targetDir: evalMissingRepo.work, token: 'test-token', transport: evalMissingGithub.transport.bind(evalMissingGithub), vendoredFeeds: path.join(root, 'feeds'), - evalEnabled: true, + planFile: evalMissingArtifacts.planFile, + evalResultsFile: evalMissingArtifacts.evalResultsFile, now: new Date('2026-08-01T00:00:00Z'), }) const evalMissingDecision = evalMissingResult.decisions.find(item => item.group.kind === 'model') @@ -788,13 +871,15 @@ const extraEvalRepo = makeRepo({ }, }) const extraEvalGithub = new FakeGitHub() +const extraEvalArtifacts = await evaluateForPublish({ targetDir: extraEvalRepo.work }) const extraEvalResult = await runBot({ repo: 'example/eval-extra-file', targetDir: extraEvalRepo.work, token: 'test-token', transport: extraEvalGithub.transport.bind(extraEvalGithub), vendoredFeeds: path.join(root, 'feeds'), - evalEnabled: true, + planFile: extraEvalArtifacts.planFile, + evalResultsFile: extraEvalArtifacts.evalResultsFile, now: new Date('2026-08-01T00:00:00Z'), }) const extraEvalDecision = extraEvalResult.decisions.find(item => item.group.kind === 'model') @@ -810,13 +895,15 @@ const mutatedEvalRepo = makeRepo({ }, }) const mutatedEvalGithub = new FakeGitHub() +const mutatedEvalArtifacts = await evaluateForPublish({ targetDir: mutatedEvalRepo.work }) const mutatedEvalResult = await runBot({ repo: 'example/eval-mutated-plan-file', targetDir: mutatedEvalRepo.work, token: 'test-token', transport: mutatedEvalGithub.transport.bind(mutatedEvalGithub), vendoredFeeds: path.join(root, 'feeds'), - evalEnabled: true, + planFile: mutatedEvalArtifacts.planFile, + evalResultsFile: mutatedEvalArtifacts.evalResultsFile, now: new Date('2026-08-01T00:00:00Z'), }) const mutatedEvalDecision = mutatedEvalResult.decisions.find(item => item.group.kind === 'model') @@ -831,16 +918,141 @@ const cleanRideRepo = makeRepo({ }, }) const cleanRideGithub = new FakeGitHub() +const cleanRideArtifacts = await evaluateForPublish({ targetDir: cleanRideRepo.work }) const cleanRideResult = await runBot({ repo: 'example/eval-clean-ride', targetDir: cleanRideRepo.work, token: 'test-token', transport: cleanRideGithub.transport.bind(cleanRideGithub), vendoredFeeds: path.join(root, 'feeds'), - evalEnabled: true, + planFile: cleanRideArtifacts.planFile, + evalResultsFile: cleanRideArtifacts.evalResultsFile, now: new Date('2026-08-01T00:00:00Z'), }) -assert(cleanRideResult.decisions.find(item => item.group.kind === 'model')?.action === 'create', 'clean eval ride-along still commits the patch') +const cleanRideDecision = cleanRideResult.decisions.find(item => item.group.kind === 'model') +const cleanRideMetadata = parseMetadata(cleanRideDecision?.body) +assert(cleanRideDecision?.action === 'create', 'bound passing eval result permits the write-only publisher to commit the patch') +assert(cleanRideDecision?.body.includes('Result: pass') && cleanRideDecision.body.includes('pass') && cleanRideMetadata?.eval_config_digest === cleanRideArtifacts.evaluation.artifact.eval_config_digest, 'published PR records the passing report and exact eval configuration digest') + +const durableEvalRepo = makeRepo({ name: 'durable-eval-failure', files: baseFiles }) +const durableEvalGithub = new FakeGitHub() +const durableEvalConfigFile = path.join(tempRoot, 'durable-eval-config.json') +const writeDurableEvalConfig = passing => write(durableEvalConfigFile, JSON.stringify({ + issues: { enabled: true }, + eval: { + command: passing + ? 'node -e \'require("fs").writeFileSync(process.env.MODEL_EOL_REPORT, "durable eval passed")\'' + : 'node -e \'require("fs").writeFileSync(process.env.MODEL_EOL_REPORT, "durable eval failed with ```untrusted``` output");process.exit(7)\'', + timeout_ms: 1000, + max_report_bytes: 1024, + pass_env: [], + }, +})) +const durableEvalRun = async () => { + const artifacts = await evaluateForPublish({ + targetDir: durableEvalRepo.work, + configPath: durableEvalConfigFile, + }) + return runBot({ + repo: 'example/durable-eval-failure', + targetDir: durableEvalRepo.work, + configPath: durableEvalConfigFile, + token: 'test-token', + transport: durableEvalGithub.transport.bind(durableEvalGithub), + vendoredFeeds: path.join(root, 'feeds'), + planFile: artifacts.planFile, + evalResultsFile: artifacts.evalResultsFile, + now: new Date('2026-08-01T00:00:00Z'), + }) +} +writeDurableEvalConfig(false) +const durableEvalFailed = await durableEvalRun() +const durableEvalFailedDecision = durableEvalFailed.decisions.find(item => item.group.kind === 'model') +const firstEvalIssue = durableEvalGithub.issues.find(item => item.state === 'open') +const firstEvalIssueMetadata = parseMetadata(firstEvalIssue?.body) +assert(durableEvalFailedDecision?.action === 'eval-failed' && durableEvalFailedDecision.issueAction === 'create', 'failed configured eval creates a durable issue decision') +assert(durableEvalGithub.callsFor('POST', '/pulls').length === 0 && durableEvalGithub.callsFor('POST', '/issues').length === 1, 'failed configured eval opens an issue and never a pull request') +assert(firstEvalIssue?.labels?.some(label => label === 'model-eol' || label.name === 'model-eol') && firstEvalIssueMetadata?.channel === 'configured-eval', 'eval failure issue is labelled and carries a distinct trusted metadata identity') +assert(firstEvalIssue?.body.includes('Result: fail') && firstEvalIssue.body.includes('durable eval failed') && !firstEvalIssue.body.includes('```untrusted```'), 'eval failure issue records a safely fenced bounded report') +const durableEvalStillFailed = await durableEvalRun() +assert(durableEvalStillFailed.decisions.find(item => item.group.kind === 'model')?.issueAction === 'skip-unchanged' && durableEvalGithub.callsFor('POST', '/issues').length === 1, 'persistent eval failure maintains one unchanged durable issue') + +const durablePatchCallsBeforeLabelFailure = durableEvalGithub.calls.filter(call => call.method === 'PATCH').length +durableEvalGithub.labelExists = false +durableEvalGithub.labelCreateStatus = 403 +const durableLabelUnavailable = await durableEvalRun() +assert(durableLabelUnavailable.decisions.some(item => item.action === 'label-unavailable'), 'transient label denial fails closed before processing configured eval work') +assert(firstEvalIssue?.state === 'open' && durableEvalGithub.calls.filter(call => call.method === 'PATCH').length === durablePatchCallsBeforeLabelFailure, 'label denial preserves an active configured-eval issue without stale reconciliation writes') +durableEvalGithub.labelExists = true +durableEvalGithub.labelCreateStatus = 201 + +writeDurableEvalConfig(true) +const durableEvalCleared = await durableEvalRun() +const durableEvalPull = durableEvalGithub.pulls.find(item => item.state === 'open') +assert(durableEvalCleared.decisions.some(item => item.group.kind === 'model' && item.action === 'create') && durableEvalPull, 'cleared eval opens the migration pull request') +assert(durableEvalCleared.decisions.some(item => item.group.kind === 'issue' && item.action === 'close-stale') && firstEvalIssue?.state === 'closed' && parseMetadata(firstEvalIssue?.body)?.stale_closed === true, 'cleared eval reconciles its durable issue as stale') + +writeDurableEvalConfig(false) +const durableEvalReappeared = await durableEvalRun() +const secondEvalIssue = durableEvalGithub.issues.find(item => item.state === 'open') +assert(durableEvalReappeared.decisions.some(item => item.group.kind === 'model' && item.action === 'eval-failed') && secondEvalIssue?.number !== firstEvalIssue?.number, 'reappearing eval failure creates fresh durable work after automated closure') +assert(durableEvalGithub.callsFor('POST', '/pulls').length === 1 && durableEvalPull?.state === 'closed' && parseMetadata(durableEvalPull?.body)?.stale_closed === true, 'reappearing eval failure closes the trusted bot PR and never opens a failing replacement PR') + +writeDurableEvalConfig(true) +const durableEvalClearedAgain = await durableEvalRun() +assert(durableEvalClearedAgain.decisions.some(item => item.group.kind === 'model' && item.action === 'create') && durableEvalGithub.callsFor('POST', '/pulls').length === 2, 'a second cleared eval safely regenerates a fresh migration PR') +assert(secondEvalIssue?.state === 'closed' && parseMetadata(secondEvalIssue?.body)?.stale_closed === true, 'the reappeared eval issue reconciles when the eval clears again') + +const channelCollisionRepo = makeRepo({ + name: 'eval-issue-channel-collision', + files: { + ...baseFiles, + 'generic.py': 'MODEL = "o3-deep-research"\n', + }, +}) +const channelCollisionGithub = new FakeGitHub() +const channelCollisionConfigFile = path.join(tempRoot, 'eval-issue-channel-collision.json') +const writeChannelCollisionConfig = passing => write(channelCollisionConfigFile, JSON.stringify({ + issues: { enabled: true }, + eval: { + command: passing + ? 'node -e \'require("fs").writeFileSync(process.env.MODEL_EOL_REPORT, "pass")\'' + : 'node -e \'require("fs").writeFileSync(process.env.MODEL_EOL_REPORT, "fail");process.exit(7)\'', + timeout_ms: 1000, + max_report_bytes: 1024, + pass_env: [], + }, +})) +const channelCollisionRun = async () => { + const artifacts = await evaluateForPublish({ + targetDir: channelCollisionRepo.work, + configPath: channelCollisionConfigFile, + }) + return runBot({ + repo: 'example/eval-issue-channel-collision', + targetDir: channelCollisionRepo.work, + configPath: channelCollisionConfigFile, + token: 'test-token', + transport: channelCollisionGithub.transport.bind(channelCollisionGithub), + vendoredFeeds: path.join(root, 'feeds'), + planFile: artifacts.planFile, + evalResultsFile: artifacts.evalResultsFile, + now: new Date('2026-08-01T00:00:00Z'), + }) +} +writeChannelCollisionConfig(false) +await channelCollisionRun() +const collisionEvalIssues = () => channelCollisionGithub.issues.filter(issue => parseMetadata(issue.body)?.channel === 'configured-eval') +const collisionOrdinaryIssues = () => channelCollisionGithub.issues.filter(issue => (parseMetadata(issue.body)?.channel ?? null) === null) +assert(collisionEvalIssues().length === 1 && collisionOrdinaryIssues().length === 1 && channelCollisionGithub.issues.every(issue => issue.state === 'open'), 'a failed eval and ordinary finding for the same model keep distinct issue identities') +await channelCollisionRun() +assert(channelCollisionGithub.issues.length === 2 && parseMetadata(collisionEvalIssues()[0].body)?.channel === 'configured-eval', 'persistent failure maintains both issue channels without overwriting metadata') +writeChannelCollisionConfig(true) +await channelCollisionRun() +assert(collisionEvalIssues()[0].state === 'closed' && collisionOrdinaryIssues()[0].state === 'open', 'clearing eval closes only configured-eval work and preserves the ordinary finding') +writeChannelCollisionConfig(false) +await channelCollisionRun() +assert(collisionEvalIssues().filter(issue => issue.state === 'open').length === 1 && collisionOrdinaryIssues().length === 1 && collisionOrdinaryIssues()[0].state === 'open', 'reappearing eval failure creates fresh eval work without duplicating or suppressing its ordinary sibling') const maliciousFeeds = path.join(tempRoot, 'malicious-feeds') fs.cpSync(path.join(root, 'feeds'), maliciousFeeds, { recursive: true }) @@ -952,14 +1164,18 @@ for (const name of envNames) { assert(reportForBody('```untrusted```') === 'untrusted', 'report embedding strips backticks') const transportCallsBeforeDry = github.calls.length +const dryArtifacts = await evaluateForPublish({ targetDir: repo.work }) const dry = await runBot({ repo: 'example/app', targetDir: repo.work, dryRun: true, transport: async () => { throw new Error('dry-run must not call transport') }, vendoredFeeds: path.join(root, 'feeds'), + planFile: dryArtifacts.planFile, + evalResultsFile: dryArtifacts.evalResultsFile, }) assert(dry.decisions.some(item => item.action === 'create'), 'dry-run prints would-be create decisions') +assert(dry.decisions.find(item => item.group.kind === 'model')?.body.includes('Result: pass'), 'dry-run consumes the same bound eval result instead of bypassing configured evaluation') assert(github.calls.length === transportCallsBeforeDry, 'dry-run makes zero GitHub transport calls') assert(!gitTry(repo.bare, ['show-ref', '--verify', `refs/heads/${branch}-dry-run`]), 'dry-run makes no push') @@ -1019,7 +1235,7 @@ assert(tamperedArtifactError?.message.includes('does not match the independently assert(artifactGithub.calls.length === 0, 'tampered plan artifact is rejected before any GitHub reads or writes') const policyArgsChecker = path.join(tempRoot, 'policy-args-checker.mjs') -write(policyArgsChecker, `const forbidden = process.argv.slice(2).filter(value => ["--days", "--scope", "--via"].includes(value)); if (forbidden.length) { console.error("config masked by " + forbidden.join(",")); process.exit(9) } console.log(JSON.stringify({plan_schema:"model-eol.plan/0.1",generated:new Date().toISOString(),threshold_days:90,via:null,scan_notes:["config-only-policy"],items:[],issues:[]}))\n`) +write(policyArgsChecker, `const forbidden = process.argv.slice(2).filter(value => ["--days", "--scope", "--via"].includes(value)); if (forbidden.length) { console.error("config masked by " + forbidden.join(",")); process.exit(9) } console.log(JSON.stringify({plan_schema:"model-eol.plan/0.1",generated:new Date().toISOString(),threshold_days:90,via:null,scan_notes:[{reason:"config-only-policy"}],items:[],issues:[]}))\n`) const policyArgsRepo = makeRepo({ name: 'policy-args', files: baseFiles, @@ -1035,7 +1251,7 @@ const policyArgsResult = await runBot({ checkerPath: policyArgsChecker, vendoredFeeds: path.join(root, 'feeds'), }) -assert(policyArgsResult.plan.scan_notes.includes('config-only-policy'), 'bot forwards config without top-level days/scope/via flags that would mask path routes and overrides') +assert(policyArgsResult.plan.scan_notes.some(note => note.reason === 'config-only-policy'), 'bot forwards config without top-level days/scope/via flags that would mask path routes and overrides') const nullViaRepo = makeRepo({ name: 'null-via', @@ -1113,6 +1329,51 @@ assert(partitionEvaluation.artifact.base_sha === headOf(partitionRepo, 'main'), assert(partitionPass.report.includes('true|explicit-pass-env|undefined|1'), 'per-group eval sees the patched migration, selected one-item plan, and explicit pass_env only') assert(Buffer.byteLength(partitionFail.report) <= 128 && partitionFail.report.includes('truncated'), 'per-group eval report honors the configured byte cap') +const movingSourceCommand = 'node -e \'const fs=require("fs");const {spawnSync}=require("child_process");fs.writeFileSync(process.env.MODEL_EOL_REPORT,"pass");const r=spawnSync("git",["-C",process.env.MODEL_EOL_TEST_SOURCE,"commit","--allow-empty","-m","move source during eval"],{stdio:"ignore"});process.exit(r.status??1)\'' +const movingSourceRepo = makeRepo({ + name: 'moving-eval-source', + files: baseFiles, + config: { + issues: { enabled: false }, + eval: { + command: movingSourceCommand, + timeout_ms: 1000, + max_report_bytes: 128, + pass_env: ['MODEL_EOL_TEST_SOURCE'], + }, + }, +}) +const movingSourceConfigFile = path.join(movingSourceRepo.work, '.model-eol.json') +const movingSourcePlan = runPlan({ + workDir: movingSourceRepo.work, + config: loadConfig(movingSourceConfigFile), + configPath: movingSourceConfigFile, + feedsDir: path.join(root, 'feeds'), + warn: () => {}, +}) +const movingSourcePlanFile = path.join(tempRoot, 'moving-source-plan.json') +const movingSourceEvalFile = path.join(tempRoot, 'moving-source-eval.json') +write(movingSourcePlanFile, JSON.stringify(movingSourcePlan, null, 2)) +const savedEvalSource = process.env.MODEL_EOL_TEST_SOURCE +process.env.MODEL_EOL_TEST_SOURCE = movingSourceRepo.work +let movingSourceError = null +try { + await evaluatePlan({ + targetDir: movingSourceRepo.work, + configPath: movingSourceConfigFile, + planFile: movingSourcePlanFile, + outputFile: movingSourceEvalFile, + vendoredFeeds: path.join(root, 'feeds'), + commandOverride: null, + }) +} catch (error) { + movingSourceError = error +} finally { + if (savedEvalSource === undefined) delete process.env.MODEL_EOL_TEST_SOURCE + else process.env.MODEL_EOL_TEST_SOURCE = savedEvalSource +} +assert(movingSourceError?.message.includes('target commit changed during migration evaluation') && !fs.existsSync(movingSourceEvalFile), 'evaluator never labels earlier results with a base commit that moved during evaluation') + const timeoutEvalFile = path.join(tempRoot, 'partition-timeout-eval.json') const timeoutEvaluation = await evaluatePlan({ targetDir: partitionRepo.work, @@ -1318,7 +1579,7 @@ assert(headOf(leaseRepo, leaseBranch) === leaseHeadBefore, 'committer identity m assert(leaseGithub.callsFor('POST', '/comments').some(call => call.body.body?.includes('forge the configured identity')), 'lease stand-down comment records the residual identity-forgery risk') const malformedChecker = path.join(tempRoot, 'malformed-plan-checker.mjs') -write(malformedChecker, 'console.log(JSON.stringify({ plan_schema: "model-eol.plan/0.1", items: [{ file: "direct.py", line: 0, occurrence: 0, matched: "old", replacement: "new", expected_line_sha256: "' + '0'.repeat(64) + '" }], issues: [], scan_notes: [] }))\n') +write(malformedChecker, 'console.log(JSON.stringify({ plan_schema: "model-eol.plan/0.1", generated: new Date().toISOString(), threshold_days: 90, via: null, items: [{ file: "direct.py", line: 0, occurrence: 0, matched: "old", replacement: "new", expected_line_sha256: "' + '0'.repeat(64) + '", id: "old", publisher: "openai", shutdown: "2026-07-01", days: -31, status: "retired", sources: [], notes: null }], issues: [], scan_notes: [] }))\n') let malformedPlanRefused = false try { await runBot({ repo: 'example/app', targetDir: repo.work, dryRun: true, checkerPath: malformedChecker, vendoredFeeds: path.join(root, 'feeds') }) diff --git a/check.mjs b/check.mjs index b59f0c6..d04c44e 100755 --- a/check.mjs +++ b/check.mjs @@ -10,9 +10,11 @@ // node check.mjs alert [paths...] [--format github|markdown|badge] [--json] // node check.mjs plan [paths...] [--days N] [--scope all|direct] [--via DISTRIBUTOR] [--feeds DIR] // node check.mjs apply --plan plan.json [--dry-run] +// node check.mjs validate document.json [...] [--type feed|config|check|inventory|schedule|alert|plan] // // exit codes: check/alert 0 = clear, 1 = finding, 2 = usage error; -// apply 0 = all items applied, 1 = item refused, 2 = usage or plan error. +// apply 0 = all items applied, 1 = item refused, 2 = usage or plan error; +// validate 0 = every document valid, 2 = usage, parse, schema, or semantic error. import fs from 'node:fs' import path from 'node:path' @@ -45,6 +47,7 @@ import { formatSchedule, } from './lib/reports.mjs' import { addedLinesForTargets, filterFindingsToChanged, gitRootFor, incompleteScanNotes, scanTargets } from './lib/scanner.mjs' +import { DOCUMENT_TYPES, formatDocumentErrors, validateDocumentFile } from './lib/validate-document.mjs' const rootForTarget = target => { const absolute = path.resolve(target) @@ -93,7 +96,7 @@ const repoPathForFile = (file, roots) => { } const main = () => { -const COMMANDS = new Set(['check', 'inventory', 'schedule', 'alert', 'plan', 'apply', 'help']) +const COMMANDS = new Set(['check', 'inventory', 'schedule', 'alert', 'plan', 'apply', 'validate', 'help']) const args = process.argv.slice(2) if (args[0] === '--help' || args[0] === '-h') args[0] = 'help' @@ -115,6 +118,7 @@ try { 'allow-incomplete': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, changed: { type: 'string' }, + type: { type: 'string' }, help: { type: 'boolean', short: 'h' }, }, allowPositionals: true, @@ -148,6 +152,7 @@ Usage: node check.mjs alert [paths...] [--config FILE] [--days N] [--scope all|direct] [--format github|markdown|badge] [--json] node check.mjs plan [paths...] [--config FILE] [--days N] [--scope all|direct] [--via DISTRIBUTOR] [--feeds DIR] [--allow-incomplete] node check.mjs apply --plan plan.json [--dry-run] + node check.mjs validate document.json [...] [--type feed|config|check|inventory|schedule|alert|plan] Commands: check Fail when tracked model IDs are retired or retiring within --days. @@ -159,6 +164,8 @@ Commands: alert Emit GitHub Actions annotations, Markdown, or a Shields badge JSON and fail on errors. plan Emit a JSON migration plan with only safely patchable direct API references. apply Apply a plan with line-hash and occurrence checks; use --dry-run to preview. + validate Validate a feed, config, check, inventory, schedule, alert, or plan document. + The type is inferred from its discriminator or .model-eol.json filename unless --type is set. Scopes: all Check every tracked model ID found. This preserves the original behavior. @@ -171,12 +178,21 @@ Configuration: Exit codes: 0 Clean, or report generated successfully. 1 Findings, alert errors, or refused apply items. - 2 Usage, feed, scan, or plan errors. + 2 Usage, feed, scan, plan, or validation errors. 3 Output stream failure other than EPIPE. `) return 0 } +if (positionals.some(value => value.length === 0)) { + console.error('positional paths must be non-empty') + return 2 +} +if (values.via === '') { + console.error('--via must be a non-empty lifecycle channel') + return 2 +} + if (command === 'apply') { if (!PLAN_FILE || positionals.length) { console.error('apply requires --plan plan.json and accepts no target paths') @@ -191,6 +207,41 @@ if (command === 'apply') { } } +if (command === 'validate') { + const allowedOptions = new Set(['type']) + const unsupported = Object.keys(values).find(option => !allowedOptions.has(option)) + if (positionals.length === 0 || unsupported) { + const detail = unsupported ? `; --${unsupported} is not supported by validate` : '' + console.error(`validate requires one or more JSON documents${detail}`) + return 2 + } + if (values.type !== undefined && !DOCUMENT_TYPES.includes(values.type) && values.type !== 'bot-config' && values.type !== 'model-eol-config') { + console.error(`unknown document type ${JSON.stringify(values.type)}; expected ${DOCUMENT_TYPES.join(', ')}`) + return 2 + } + let failed = false + for (const file of positionals) { + try { + const result = validateDocumentFile(file, { type: values.type ?? null }) + if (result.errors.length) { + for (const error of formatDocumentErrors(result.errors)) console.error(`${file}: ${error}`) + failed = true + continue + } + console.log(`${file}: valid ${result.type} document`) + } catch (error) { + console.error(`${file}: validation failed: ${error.message}`) + failed = true + } + } + return failed ? 2 : 0 +} + +if (values.type !== undefined) { + console.error('--type is only supported by the validate command') + return 2 +} + let configLocation let repositoryConfig try { @@ -355,7 +406,7 @@ if (command === 'plan') { if (command === 'check') { if (AS_JSON) { - console.log(JSON.stringify({ threshold_days: DAYS, distributor: VIA, scope: SCOPE, scan_notes: scan.notes, findings: changedFindings }, null, 2)) + console.log(JSON.stringify({ schema: 'model-eol/check@0.1', threshold_days: DAYS, distributor: VIA, scope: SCOPE, scan_notes: scan.notes, findings: changedFindings }, null, 2)) } else { console.log(formatCheck({ findings: changedFindings, bad, scannedFiles: scan.files.length, days: DAYS, scope: SCOPE })) } diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index e56c5c1..83c19d3 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -1,11 +1,11 @@ # Context - read this first if you're picking the project up cold -*Written 2026-07-25, the week the problem bit. Everything in this file is -public-safe; the repo is private pre-publish but flips public later.* +*Written 2026-07-25, the week the problem bit; updated 2026-08-18 after the +public release and field UAT. Everything in this file is public-safe.* ## Origin -On July 23, 2026, OpenAI shut down 18 models in one scheduled wave, including +On July 23, 2026, OpenAI shut down a scheduled wave of models, including `o3-deep-research` - a model the author had a tuned research workflow on. The companion essay ("The Model BOM," forthcoming on tomsullivan.dev) argues that the model line in a software bill of materials is categorically different from every @@ -101,9 +101,11 @@ competition - see the outreach issue. 12. **Policy floors are stated policy, not contracts** - the wording in SPEC.md and the schedule output says exactly that, and feeds without a stated floor make no forward claim. -13. **The eval hook is a security boundary.** Config-supplied command, so: separate - least-privilege job, scrubbed env namespace, bounded runtime and report size, - report treated as untrusted content in PR bodies. +13. **The eval hook crosses a trust boundary.** Repository-owned code runs only in + the read-only evaluate job, with explicit normal environment forwarding, + bounded runtime/report size, and reports treated as untrusted content. The + forwarding allowlist is not an OS sandbox; the hard privilege boundary is no + repository write token, and publication never executes the eval command. 14. **No model-attribution trailers on commits.** Implementation is a mix of Claude and Codex delegation; per-commit honesty lives in commit bodies ("Implemented by Codex (gpt-5.6-luna); reviewed by Claude"), never in Co-Authored-By trailers. @@ -130,30 +132,35 @@ competition - see the outreach issue. 4. Sweep this repo for anything non-public-safe (should be nothing; keep it that way - write every commit as if the repo were already public). -## State as of 2026-08-01 (v0.1.0 tagged, v0.1.1 in progress) - -Shipped and field-tested: hardened scanner (git ls-files traversal), plan/apply -with strict gating, bot adapter (PR/issue lifecycle, sandboxed eval hook), -refresh tooling (OpenAI + Anthropic pages, models endpoints, aws-bedrock -distributor clocks), policy floors + safe_until/earliest-risk, --changed PR -gate, CycloneDX export, badge + Atom changelog outputs, strict parseArgs CLIs -with TTY colors, VHS demo, weekly feed-refresh workflow LIVE in this repo -(verified green on GitHub runners). - -Field-test receipts: found a model dead 47 days in a private repo's promptfoo -config; a sweep across the maintainer's local repos found ~230 retired refs -(most are pricing-table noise in one project - ignore-config material); -feed drift caught within one week of hand compilation; a hand-compiled Bedrock -date corrected by three months. - -DONE since: npm publish (v0.2.2 live; npm-release.yml auto-publishes patch -versions on material feed changes - caniuse-lite pattern, trusted publishing), -CI workflow activated, provider API key secrets set, the public flip. -Remaining, maintainer-side: configure the npm trusted publisher (package -Settings -> Trusted publisher -> GitHub Actions, workflow npm-release.yml), -activating the bot workflow from bot.yml.example, the essay/post. -Remaining, backlog: Google/Vertex/Azure fetchers, gateway resolvers, monorepo -ownership groups, feed signing, feeds-dir self-scan exclusion. +## State as of 2026-08-18 + +The direct-first product is shipped on npm and as a moving `v0` Action: bounded +scanner, strict repository policy, mixed-repository overrides/routes, lifecycle +reports, CycloneDX, safe plan/apply, and a stateless GitHub bot with trusted +ownership, isolated per-migration evals, and stale-work reconciliation. Provider +refresh covers OpenAI, Anthropic, and Google; distributor refresh covers Bedrock +and Vertex. Weekly refresh, failure receipts, semantic PRs, trusted npm publishing, +and exact-package consumer tests are live. + +The staged public-contract milestone gives every public schema a canonical ID, exposes +`model-eol validate`, and adds a Pages publisher for hosted feeds, Atom, and +`last_checked` health. The host becomes authoritative only after Pages is enabled +and the exact-byte deployment UAT passes. The same milestone treats distributor +status changes as material, preserves structured replacement guidance in +machine-readable reports, +and closes the remaining symlink false-clean edge. Node 22 is the supported floor; +runtime and development dependencies remain zero. + +Field-test receipts include a hosted plan/evaluate/publish run that created a +one-line migration PR, owned-repository sweeps, a packed offline consumer, and an +independent published `0.4.1` run against a content-heavy repository. The latter +correctly found eight raw retired references, then went clean with an explicit +policy excluding cached data, draft fixtures, and a reference catalog. That is +the intended operating model: inventory first, then auditable repository policy. + +Expansion backlog: authenticated Azure/Bedrock/Vertex/gateway resolvers, more +publisher feeds, waiver ownership/expiry, remote-feed trust/signing and overlay +composition, and broader file-format coverage. ## Conventions diff --git a/docs/DESIGN_BOT.md b/docs/DESIGN_BOT.md index 182c13a..dbac8e1 100644 --- a/docs/DESIGN_BOT.md +++ b/docs/DESIGN_BOT.md @@ -1,11 +1,10 @@ # Design - the model-eol bot ("Dependabot for models") -*Status: agreed 2026-08-01 after two adversarial design reviews (Claude + Codex) -and a first field test of the core against real repos. SHIPPED same day in -v0.1.0: all tracks below are implemented (see docs/CONTEXT.md "State as of -2026-08-01"). One deviation earned by field testing: retired generic -model-references file issues too (reason not-direct-api) - the original issue -allowlist accidentally made them invisible.* +*Status 2026-08-18: the architecture below is shipped, adversarially reviewed, +and proven through a hosted plan/evaluate/publish UAT. Current releases add +commit-bound evaluator artifacts, trusted ownership leases, stale-work +reconciliation, per-reference routing, and permanent published-consumer tests. +Retired generic model references remain issue-only (`not-direct-api`).* ## Decisions locked @@ -97,24 +96,41 @@ bot adapter consuming versioned `plan --json`. Nothing stateful in the core. - **Token reality**: PRs created with `GITHUB_TOKEN` do not trigger `on: pull_request` workflows. Document a fine-grained PAT / GitHub App token as the recommended path; degrade gracefully (PR still opens, body warns). -- **Feed freshness**: the adapter fetches feeds from this repo's main branch at - runtime (schema-validated), with the vendored copy as offline fallback and - `--feeds` as override. A pinned action must not mean pinned data. +- **Feed freshness**: the copy-ready workflow resolves the moving npm `0.x` line + once and carries that exact published package through all three jobs. Its + schema-validated bundled feeds are the default. Explicit remote feed URLs are + opt-in, fail closed, and may use a configured vendored fallback in degraded + report-only mode. ### A4. Eval hook (security boundary, per Codex review - critical finding) Contract: `{"eval": {"command": "..."}}` in `.model-eol.json`; env vars `MODEL_EOL_OLD_ID`, `MODEL_EOL_NEW_ID`, `MODEL_EOL_PLAN`; exit 0 = pass; -optional markdown report via `MODEL_EOL_REPORT` path. +a regular bounded Markdown report is required at the `MODEL_EOL_REPORT` path for +an exit-zero run to count as passing. + +The command is intentionally explicit rather than inferred from package files: +model-eol cannot know whether a unit suite proves tool calling, structured output, +retrieval quality, or another repository-specific behavior. The checked-in +[`examples/model-eol-eval.mjs`](../examples/model-eol-eval.mjs) is the +dependency-free starter. Consumers should customize its verification command and +semantic assertions, always write a non-secret receipt to `MODEL_EOL_REPORT`, and +run the evaluator locally against a clean committed checkout before enabling the +hosted workflow. A non-empty `MODEL_EOL_EVAL_COMMAND` workflow variable takes +precedence over `eval.command`; an absent command is recorded as unconfigured, +not silently treated as a passing eval. Execution rules: the workflow resolves the moving npm major once, validates and records the exact version, and reuses it in all three jobs. The plan job emits a versioned plan. A separate least-privilege evaluate job independently verifies that plan, applies each migration in its own temporary checkout, then invokes the configured command with its own old/new IDs and selected plan. It has no -write token; provider keys are forwarded only when named by `eval.pass_env`. -Timeout and report-size limits apply per migration, and eval workspace/history -drift turns that migration into a failure. +write token; consumers add only the provider secrets their trusted eval needs, +and those names must also appear in `eval.pass_env`. Environment filtering controls +normal child-process forwarding, not same-user OS isolation; repository code and +its dependencies remain trusted inside the evaluate job. Timeout and report-size +limits apply per migration, and tracked eval workspace/history drift turns that +migration into a failure. The evaluate job always uploads one bounded result manifest containing an exact pass/fail/timeout record per migration. The manifest is bound to the evaluated @@ -124,7 +140,14 @@ independently regenerates the plan, verifies both artifacts before any GitHub operation, publishes passing migrations, records failed peers as blocked, and returns non-zero if any conflict, eval failure, lease stand-down, label failure, or degraded report-only decision remains. Reports are untrusted content: capped -and fenced when embedded in PR bodies. +and fenced when embedded in PR bodies. The publish process never executes +repository-owned eval code. When `eval.command` is configured, a missing bound +result manifest fails before GitHub API access. + +The former inline `model-eol-bot --eval` mode and the unbound report/status +artifacts are intentionally refused: process-level environment scrubbing is not a +privilege boundary when evaluation and publication share a user and write token. +Consumers migrate to the plan/evaluate/publish workflow in `bot.yml.example`. ### Config - `.model-eol.json` in the consuming repo diff --git a/docs/PRODUCT_PLAN.md b/docs/PRODUCT_PLAN.md index 82241f9..0c646a8 100644 --- a/docs/PRODUCT_PLAN.md +++ b/docs/PRODUCT_PLAN.md @@ -1,10 +1,12 @@ # Product plan - direct-first model retirement inventory -*Status 2026-08-01: the MVP scope below shipped in v0.1.0, plus the bot adapter, -refresh automation, and the aws-bedrock distributor fetcher (feed-side clocks -from AWS's lifecycle page - account-level deployment resolvers remain future -work as described under "Next resolvers"). Current state lives in -docs/CONTEXT.md.* +*Status 2026-08-18: the direct-first product is shipped and live-UAT proven. +The public-contract milestone stages canonical schema IDs and Pages publication, +plus zero-dependency document validation, explicit refresh receipts, status-aware +distributor diffs, and permanent published-consumer UAT. Hosted URLs become +authoritative only after Pages is enabled and its first exact-byte deployment is +green. Account-level resolvers remain deliberate expansion work under "Next +resolvers." Current state lives in docs/CONTEXT.md.* ## Thesis @@ -47,6 +49,7 @@ node check.mjs inventory . --json node check.mjs schedule . --days 90 node check.mjs alert . --days 90 --scope direct node check.mjs . --days 90 --via azure-ai-foundry +node check.mjs validate feeds/openai.json plan.json ``` ## Mixed repositories and generated code @@ -84,8 +87,15 @@ CLI flags still override repository and path policy. Machine reports carry the effective threshold, scope, requested channel, and matching rule indexes when a path rule or route changed behavior. +The primary `check --json` report has its own public discriminator and Draft-07 +schema. CycloneDX exports identify components by canonical model plus requested +lifecycle channel, using `publisher-direct` for the un-routed publisher clock so +it cannot collide with a custom distributor named `publisher`. Azure, Bedrock, +and other clocks for one model cannot overwrite each other; occurrences remain +attached to the channel used. + The scanner reads `.baml` source but skips conservative generated-code headers, -BAML-generated clients, and model-eol inventory/schedule/alert/plan artifacts. +BAML-generated clients, and model-eol check/inventory/schedule/alert/plan artifacts. CycloneDX is skipped only when its metadata carries model-eol generator provenance, so third-party BOMs stay visible. Intentional artifact skips are diagnostics, not incomplete-coverage warnings. diff --git a/examples/model-eol-eval.mjs b/examples/model-eol-eval.mjs new file mode 100644 index 0000000..316e652 --- /dev/null +++ b/examples/model-eol-eval.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node + +// Copy this file into the consuming repository and replace VERIFY with the +// smallest command that proves the model-dependent behavior you rely on. +// model-eol runs it from an isolated checkout after applying one migration. + +import fs from 'node:fs' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm' +const VERIFY = Object.freeze({ command: npm, args: ['run', 'eval:model-eol'] }) + +const required = name => { + const value = process.env[name] + if (!value) throw new Error(`${name} is required`) + return value +} + +const singleLine = value => String(value).replace(/[\r\n]+/g, ' ').trim() + +let reportPath = process.env.MODEL_EOL_REPORT || null +let exitCode = 1 +const report = ['## Repository migration eval', ''] + +try { + const oldId = required('MODEL_EOL_OLD_ID') + const newId = required('MODEL_EOL_NEW_ID') + const planPath = required('MODEL_EOL_PLAN') + reportPath = required('MODEL_EOL_REPORT') + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + if (plan.plan_schema !== 'model-eol.plan/0.1' || !Array.isArray(plan.items) || plan.items.length === 0) { + throw new Error('MODEL_EOL_PLAN is not a non-empty model-eol plan') + } + if (plan.items.some(item => item.id !== oldId || item.replacement !== newId)) { + throw new Error('MODEL_EOL_PLAN contains a migration outside MODEL_EOL_OLD_ID/MODEL_EOL_NEW_ID') + } + + const files = [...new Set(plan.items.map(item => item.file))].sort() + report.push( + `- Migration: \`${singleLine(oldId)}\` to \`${singleLine(newId)}\``, + `- Planned references: ${plan.items.length} across ${files.length} file(s)`, + `- Verification: \`${VERIFY.command} ${VERIFY.args.join(' ')}\``, + ) + + const started = Date.now() + const result = spawnSync(VERIFY.command, VERIFY.args, { + cwd: process.cwd(), + env: process.env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 8 * 1024 * 1024, + }) + const elapsedMs = Date.now() - started + const passed = !result.error && result.status === 0 + report.push( + `- Result: ${passed ? 'pass' : 'fail'}`, + `- Exit code: ${Number.isInteger(result.status) ? result.status : 'unavailable'}`, + `- Duration: ${elapsedMs} ms`, + ) + if (result.signal) report.push(`- Signal: ${singleLine(result.signal)}`) + if (result.error) report.push(`- Runner error: ${singleLine(result.error.message)}`) + report.push('', passed + ? 'The repository verification command passed in the isolated patched checkout.' + : 'The repository verification command failed. Reproduce it locally for full logs; command output is intentionally not copied into the publishable report.') + exitCode = passed ? 0 : 1 +} catch (error) { + report.push(`- Result: fail`, '', `Harness error: ${singleLine(error.message)}`) + exitCode = 1 +} finally { + if (reportPath) { + try { + fs.mkdirSync(path.dirname(path.resolve(reportPath)), { recursive: true }) + fs.writeFileSync(reportPath, `${report.join('\n')}\n`, { encoding: 'utf8', mode: 0o600 }) + } catch (error) { + console.error(`could not write MODEL_EOL_REPORT: ${error.message}`) + exitCode = 1 + } + } else { + console.error('MODEL_EOL_REPORT is required') + } +} + +process.exitCode = exitCode diff --git a/examples/workflows/model-eol.yml b/examples/workflows/model-eol.yml index 9732ed8..43decf5 100644 --- a/examples/workflows/model-eol.yml +++ b/examples/workflows/model-eol.yml @@ -24,6 +24,10 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 # --changed needs history to diff against the base ref + - uses: actions/setup-node@v7 + with: + node-version: 22 + package-manager-cache: false - uses: thossullivan/model-eol@v0 with: command: check @@ -36,6 +40,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 22 + package-manager-cache: false - uses: thossullivan/model-eol@v0 with: command: alert diff --git a/feeds/amazon.json b/feeds/amazon.json index f213592..3dcae1e 100644 --- a/feeds/amazon.json +++ b/feeds/amazon.json @@ -1,7 +1,7 @@ { "spec": "model-eol/0.1", "publisher": "amazon", - "generated": "2026-08-18T00:47:44.931Z", + "generated": "2026-08-18T12:06:03.442Z", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", "models": [ { @@ -14,6 +14,7 @@ "via": "aws-bedrock", "announced": "2026-03-30", "shutdown": "2026-09-30", + "status": "legacy", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ] @@ -28,6 +29,7 @@ "via": "aws-bedrock", "announced": "2026-03-30", "shutdown": "2026-09-30", + "status": "legacy", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ] @@ -42,6 +44,7 @@ "via": "aws-bedrock", "announced": "2026-03-13", "shutdown": "2026-09-14", + "status": "legacy", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ] @@ -56,6 +59,7 @@ "via": "aws-bedrock", "announced": "2026-03-13", "shutdown": "2026-09-14", + "status": "legacy", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ] diff --git a/feeds/anthropic.json b/feeds/anthropic.json index e44cee6..50c6f0c 100644 --- a/feeds/anthropic.json +++ b/feeds/anthropic.json @@ -1,7 +1,7 @@ { "spec": "model-eol/0.1", "publisher": "anthropic", - "generated": "2026-08-18T00:47:44.931Z", + "generated": "2026-08-18T12:06:03.442Z", "source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", "policy": { "min_notice_days": 60, @@ -45,6 +45,7 @@ "via": "aws-bedrock", "announced": "2026-03-10", "shutdown": "2026-09-10", + "status": "extended-access", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ], @@ -74,6 +75,7 @@ "via": "aws-bedrock", "announced": "2026-04-14", "shutdown": "2026-10-14", + "status": "extended-access", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ], @@ -103,6 +105,7 @@ "via": "aws-bedrock", "announced": "2026-07-08", "shutdown": "2027-01-08", + "status": "extended-access", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html" } ], diff --git a/lib/apply.mjs b/lib/apply.mjs index 4ffab29..3915821 100644 --- a/lib/apply.mjs +++ b/lib/apply.mjs @@ -2,7 +2,12 @@ import crypto from 'node:crypto' import fs from 'node:fs' import path from 'node:path' +import { assertValidPlanDocument, readJsonDocument, validatePlanItems } from './validate-document.mjs' + +export { validatePlanItems } + const sha256 = value => crypto.createHash('sha256').update(value).digest('hex') +const utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }) const occurrenceIndex = (line, value, occurrence) => { if (!value || !Number.isInteger(occurrence) || occurrence < 0) return -1 @@ -22,63 +27,93 @@ const printError = (item, message) => { console.error(`model-eol: apply refused ${itemLabel(item)}: ${message}`) } -export const validatePlanItems = items => { - if (!Array.isArray(items)) throw new Error('plan must contain an items array') - for (const [index, item] of items.entries()) { - if (!item || typeof item !== 'object' || Array.isArray(item)) { - throw new Error(`plan item ${index} must be an object`) - } - if (typeof item.file !== 'string' || item.file.length === 0) { - throw new Error(`plan item ${index} file must be a non-empty string`) - } - if (!Number.isInteger(item.line) || item.line < 1) { - throw new Error(`plan item ${index} line must be an integer >= 1`) - } - if (!Number.isInteger(item.occurrence) || item.occurrence < 0) { - throw new Error(`plan item ${index} occurrence must be an integer >= 0`) - } - if (typeof item.matched !== 'string' || item.matched.length === 0) { - throw new Error(`plan item ${index} matched must be a non-empty string`) - } - if (typeof item.replacement !== 'string' || item.replacement.length === 0) { - throw new Error(`plan item ${index} replacement must be a non-empty string`) - } - if (typeof item.expected_line_sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(item.expected_line_sha256)) { - throw new Error(`plan item ${index} expected_line_sha256 must be 64 lowercase hexadecimal characters`) - } - } - return items -} - const isInside = (root, candidate) => { const relative = path.relative(root, candidate) return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) } -const realPathForContainment = value => { - let current = value - const suffix = [] +const resolvePlanFile = (rootDir, file) => { + const lexicalRoot = path.resolve(rootDir) + if (file.split(/[\\/]/).includes('..')) throw new Error('file path contains .. traversal') + const resolved = path.resolve(lexicalRoot, file) + if (isInside(lexicalRoot, resolved)) return resolved + if (!path.isAbsolute(file)) throw new Error('file resolves outside rootDir') + + // Remap only system-level root aliases such as macOS /var -> /private/var. + const physicalRoot = fs.realpathSync(lexicalRoot) + let candidateRoot = path.dirname(resolved) while (true) { try { - return path.join(fs.realpathSync(current), ...suffix) + if (fs.realpathSync(candidateRoot) === physicalRoot) { + const relative = path.relative(candidateRoot, resolved) + const remapped = path.resolve(lexicalRoot, relative) + if (isInside(lexicalRoot, remapped)) return remapped + } } catch { - const parent = path.dirname(current) - if (parent === current) return value - suffix.unshift(path.basename(current)) - current = parent } + const parent = path.dirname(candidateRoot) + if (parent === candidateRoot) break + candidateRoot = parent } + throw new Error('file resolves outside rootDir') } -const resolvePlanFile = (rootDir, file) => { - const lexicalRoot = path.resolve(rootDir) - const root = fs.realpathSync(lexicalRoot) - if (file.split(/[\\/]/).includes('..')) throw new Error('file path contains .. traversal') - const resolved = path.resolve(lexicalRoot, file) - if (!path.isAbsolute(file) && !isInside(lexicalRoot, resolved)) throw new Error('file resolves outside rootDir') - const realFile = realPathForContainment(resolved) - if (!isInside(root, realFile)) throw new Error('file resolves outside rootDir') - return resolved +const assertSafePath = (rootDir, file) => { + const root = path.resolve(rootDir) + if (!isInside(root, file)) throw new Error('file resolves outside rootDir') + + const rootStat = fs.lstatSync(root) + if (rootStat.isSymbolicLink()) throw new Error(`file path contains symlink component: ${root}`) + if (!rootStat.isDirectory()) throw new Error(`rootDir is not a directory: ${root}`) + + const relative = path.relative(root, file) + const components = relative === '' ? [] : relative.split(path.sep) + let current = root + let currentStat = rootStat + for (const [index, component] of components.entries()) { + current = path.join(current, component) + currentStat = fs.lstatSync(current) + if (currentStat.isSymbolicLink()) throw new Error(`file path contains symlink component: ${current}`) + if (index < components.length - 1 && !currentStat.isDirectory()) { + throw new Error(`parent path component is not a directory: ${current}`) + } + } + if (!currentStat.isFile()) throw new Error(`target is not a regular file: ${file}`) + return currentStat +} + +const readTarget = (rootDir, file) => { + assertSafePath(rootDir, file) + const noFollow = fs.constants.O_NOFOLLOW ?? 0 + let descriptor + let stat + let buffer + try { + descriptor = fs.openSync(file, fs.constants.O_RDONLY | noFollow) + stat = fs.fstatSync(descriptor) + if (!stat.isFile()) throw new Error(`target is not a regular file: ${file}`) + buffer = fs.readFileSync(descriptor) + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor) + } + + const pathStat = assertSafePath(rootDir, file) + if (pathStat.dev !== stat.dev || pathStat.ino !== stat.ino) { + throw new Error(`target changed while it was being read: ${file}`) + } + let text + try { + text = utf8Decoder.decode(buffer) + } catch { + throw new Error(`target is not valid UTF-8: ${file}`) + } + return { + buffer, + text, + dev: stat.dev, + ino: stat.ino, + mode: stat.mode & 0o7777, + } } const groupItems = items => { @@ -120,16 +155,21 @@ const locateOccurrences = (line, items) => { const locateReplacements = (line, items) => { const located = locateOccurrences(line, items) if (located.failures.length) return located - const seen = new Set() const failures = [] const locations = [] for (const location of located.locations) { - const key = `${location.item.line}:${location.index}` - if (seen.has(key)) { - failures.push({ item: location.item, message: 'duplicate occurrence in plan' }) + const end = location.index + location.item.matched.length + const conflict = locations.find(existing => ( + location.index < existing.index + existing.item.matched.length && + existing.index < end + )) + if (conflict) { + const message = conflict.index === location.index + ? 'duplicate occurrence in plan' + : 'replacement span overlaps another plan item' + failures.push({ item: location.item, message }) continue } - seen.add(key) locations.push(location) } return { locations, failures } @@ -186,224 +226,346 @@ const restoreGroup = (postImage, items) => { return search(postImage, descriptors) } -const temporaryFileFor = file => path.join( +const temporaryFileFor = (file, purpose) => path.join( path.dirname(file), - `.model-eol-${path.basename(file)}-${process.pid}-${crypto.randomBytes(12).toString('hex')}.tmp`, + `.model-eol-${path.basename(file)}-${process.pid}-${purpose}-${crypto.randomBytes(12).toString('hex')}.tmp`, ) -const writeAtomically = (file, content) => { - const temporaryFile = temporaryFileFor(file) - try { - const mode = fs.statSync(file).mode & 0o7777 - fs.writeFileSync(temporaryFile, content, { encoding: 'utf8', flag: 'wx', mode }) - fs.chmodSync(temporaryFile, mode) - fs.renameSync(temporaryFile, file) - } catch (error) { +const sortedItems = items => items.toSorted((a, b) => a.planIndex - b.planIndex) + +const analyzeFile = (file, items, snapshot) => { + const lines = snapshot.text.split('\n') + const ready = [] + const already = [] + const failures = [] + const readyGroups = [] + + for (const [lineNumber, lineItems] of groupItemsByLine(items)) { + const line = lines[lineNumber - 1] + if (line === undefined) { + for (const item of lineItems) failures.push({ item, message: 'line is not present' }) + continue + } + + if (lineItems.length > MAX_RESTORE_ITEMS) { + for (const item of lineItems) failures.push({ item, message: 'line hash does not match expected_line_sha256' }) + continue + } + + if (hasExpectedPreImage(line, lineItems)) { + const located = locateReplacements(line, lineItems) + failures.push(...located.failures) + for (const location of located.locations) { + ready.push(location.item) + } + readyGroups.push({ + lineNumber, + items: located.locations.map(location => location.item), + locations: located.locations, + }) + continue + } + + if (restoreGroup(line, lineItems) !== null) already.push(...lineItems) + else { + for (const item of lineItems) failures.push({ item, message: 'line hash does not match expected_line_sha256' }) + } + } + + const updatedLines = lines.slice() + for (const { lineNumber, locations } of readyGroups) { + updatedLines[lineNumber - 1] = applyLocatedReplacements(lines[lineNumber - 1], locations) + } + + return { + file, + items, + snapshot, + ready: sortedItems(ready), + already: sortedItems(already), + failures, + updatedContent: updatedLines.join('\n'), + } +} + +const inspectFiles = (rootDir, prepared) => { + const analyses = [] + const failures = [] + for (const [file, items] of groupItems(prepared)) { try { - fs.unlinkSync(temporaryFile) - } catch { + const analysis = analyzeFile(file, items, readTarget(rootDir, file)) + analyses.push(analysis) + failures.push(...analysis.failures) + } catch (error) { + for (const item of items) failures.push({ item, message: `unreadable file: ${error.message}` }) } - throw error } + return { analyses, failures } } -export const applyPlan = ({ planPath, dryRun = false, rootDir }) => { - if (typeof rootDir !== 'string' || rootDir.length === 0) throw new Error('applyPlan requires rootDir') - const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) - if (!plan || typeof plan !== 'object' || Array.isArray(plan)) throw new Error('plan must be an object') - validatePlanItems(plan.items) +const uniqueFailures = failures => { + const seen = new Set() + return failures + .toSorted((a, b) => a.item.planIndex - b.item.planIndex) + .filter(failure => { + if (seen.has(failure.item)) return false + seen.add(failure.item) + return true + }) +} - let failed = 0 - let applied = 0 +const reportAlready = (analyses, excluded = new Set()) => { let alreadyApplied = 0 + const items = sortedItems(analyses.flatMap(analysis => analysis.already)) + for (const item of items) { + if (excluded.has(item)) continue + console.log(`already-applied ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) + alreadyApplied++ + } + return alreadyApplied +} - const prepared = [] - for (const [planIndex, item] of plan.items.entries()) { +const reportRefusal = (analyses, failures, cascadeMessage) => { + const specific = uniqueFailures(failures) + const refused = new Set(specific.map(failure => failure.item)) + for (const failure of specific) printError(failure.item, failure.message) + + let failed = specific.length + const ready = sortedItems(analyses.flatMap(analysis => analysis.ready)) + for (const item of ready) { + if (refused.has(item)) continue + printError(item, cascadeMessage) + refused.add(item) + failed++ + } + const alreadyApplied = reportAlready(analyses, refused) + return { failed, applied: 0, alreadyApplied } +} + +const snapshotsMatch = (left, right) => ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.buffer.equals(right.buffer) +) + +const verifySnapshots = (rootDir, analyses) => { + const failures = [] + for (const analysis of analyses) { try { - prepared.push({ ...item, planIndex, resolvedFile: resolvePlanFile(rootDir, item.file) }) + const latest = readTarget(rootDir, analysis.file) + if (!snapshotsMatch(latest, analysis.snapshot)) { + for (const item of analysis.items) failures.push({ item, message: 'target changed after validation' }) + } } catch (error) { - printError(item, error.message) - failed++ + for (const item of analysis.items) failures.push({ item, message: `write failed: ${error.message}` }) } } + return failures +} - for (const [file, items] of groupItems(prepared)) { - let text +const writeStagedFile = (file, content, mode, created) => { + const descriptor = fs.openSync( + file, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, + mode, + ) + created.push(file) + try { + fs.writeFileSync(descriptor, content) + } finally { + fs.closeSync(descriptor) + } + fs.chmodSync(file, mode) +} + +const removeTemporaryFiles = (temporaryFiles, unlinkSync) => { + const failures = [] + for (const temporaryFile of temporaryFiles) { + if (!temporaryFile) continue try { - text = fs.readFileSync(file, 'utf8') - } catch (e) { - for (const item of items) { - printError(item, `unreadable file: ${e.message}`) - failed++ - } - continue + unlinkSync(temporaryFile) + } catch (error) { + if (error?.code !== 'ENOENT') failures.push({ temporaryFile, error }) } + } + return failures +} - const lines = text.split('\n') - let ready = [] - const already = [] - const failures = [] - - const readyGroups = [] - for (const [lineNumber, lineItems] of groupItemsByLine(items)) { - const line = lines[lineNumber - 1] - if (line === undefined) { - for (const item of lineItems) failures.push({ item, message: 'line is not present' }) - continue - } +const cleanupFailureDetail = failures => failures.length + ? `; temporary cleanup failed (${failures.map(({ temporaryFile, error }) => `${temporaryFile}: ${error.message}`).join('; ')})` + : '' - if (lineItems.length > MAX_RESTORE_ITEMS) { - for (const item of lineItems) failures.push({ item, message: 'line hash does not match expected_line_sha256' }) - continue - } +const reportCleanupFailures = failures => { + for (const { temporaryFile, error } of failures) { + console.error(`model-eol: apply cleanup failed ${temporaryFile}: ${error.message}`) + } +} - if (hasExpectedPreImage(line, lineItems)) { - const located = locateOccurrences(line, lineItems) - failures.push(...located.failures) - const failedItems = new Set(located.failures.map(failure => failure.item)) - const groupReady = lineItems.filter(item => !failedItems.has(item)) - ready.push(...groupReady) - readyGroups.push({ lineNumber, items: groupReady, locations: located.locations }) - continue - } +const stageAnalysis = (rootDir, analysis, unlinkSync) => { + assertSafePath(rootDir, analysis.file) + const nextPath = temporaryFileFor(analysis.file, 'next') + const backupPath = temporaryFileFor(analysis.file, 'backup') + const created = [] + try { + writeStagedFile(nextPath, Buffer.from(analysis.updatedContent, 'utf8'), analysis.snapshot.mode, created) + writeStagedFile(backupPath, analysis.snapshot.buffer, analysis.snapshot.mode, created) + return { analysis, nextPath, backupPath, keepBackup: false } + } catch (error) { + const cleanupFailures = removeTemporaryFiles(created, unlinkSync) + throw new Error(`${error.message}${cleanupFailureDetail(cleanupFailures)}`) + } +} - if (restoreGroup(line, lineItems) !== null) already.push(...lineItems) - else { - for (const item of lineItems) failures.push({ item, message: 'line hash does not match expected_line_sha256' }) - } - } +const cleanupStages = (stages, unlinkSync) => removeTemporaryFiles( + stages.flatMap(stage => [stage.nextPath, stage.keepBackup ? null : stage.backupPath]), + unlinkSync, +) - if (failures.length) { - ready.sort((a, b) => a.planIndex - b.planIndex) - already.sort((a, b) => a.planIndex - b.planIndex) - failures.sort((a, b) => a.item.planIndex - b.item.planIndex) - for (const failure of failures) { - printError(failure.item, failure.message) - failed++ - } - for (const item of ready) { - printError(item, 'another item for this file failed validation; file was not changed') - failed++ - } - for (const item of already) { - console.log(`already-applied ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) - alreadyApplied++ - } - continue - } +export const applyPlan = ({ planPath, dryRun = false, rootDir, _test } = {}) => { + if (typeof rootDir !== 'string' || rootDir.length === 0) throw new Error('applyPlan requires rootDir') + const plan = readJsonDocument(planPath) + assertValidPlanDocument(plan) - const duplicateFailures = [] - const duplicateItems = new Set() - for (const readyGroup of readyGroups) { - const located = locateReplacements(lines[readyGroup.lineNumber - 1], readyGroup.items) - duplicateFailures.push(...located.failures) - for (const failure of located.failures) duplicateItems.add(failure.item) - readyGroup.locations = located.locations - readyGroup.items = readyGroup.items.filter(item => !duplicateItems.has(item)) - } - if (duplicateFailures.length) { - ready = ready.filter(item => !duplicateItems.has(item)) - failures.push(...duplicateFailures) - ready.sort((a, b) => a.planIndex - b.planIndex) - already.sort((a, b) => a.planIndex - b.planIndex) - failures.sort((a, b) => a.item.planIndex - b.item.planIndex) - for (const failure of failures) { - printError(failure.item, failure.message) - failed++ - } - for (const item of ready) { - printError(item, 'another item for this file failed validation; file was not changed') - failed++ - } - continue + const prepared = [] + const pathFailures = [] + for (const [planIndex, item] of plan.items.entries()) { + try { + prepared.push({ ...item, planIndex, resolvedFile: resolvePlanFile(rootDir, item.file) }) + } catch (error) { + pathFailures.push({ item: { ...item, planIndex }, message: error.message }) } + } - ready.sort((a, b) => a.planIndex - b.planIndex) - already.sort((a, b) => a.planIndex - b.planIndex) + const initial = inspectFiles(rootDir, prepared) + const initialFailures = pathFailures.concat(initial.failures) + if (initialFailures.length) { + return reportRefusal( + initial.analyses, + initialFailures, + 'another item in this plan failed validation; no files were changed', + ) + } - const updatedLines = lines.slice() - for (const { lineNumber, locations } of readyGroups) { - updatedLines[lineNumber - 1] = applyLocatedReplacements(lines[lineNumber - 1], locations) + const initiallyReady = initial.analyses.flatMap(analysis => analysis.ready) + if (dryRun) { + const alreadyApplied = reportAlready(initial.analyses) + for (const item of sortedItems(initiallyReady)) { + console.log(`would change ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) } - const updatedContent = updatedLines.join('\n') + return { failed: 0, applied: 0, alreadyApplied } + } + if (!initiallyReady.length) { + return { failed: 0, applied: 0, alreadyApplied: reportAlready(initial.analyses) } + } - for (const item of already) { - console.log(`already-applied ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) - alreadyApplied++ - } - if (!ready.length) continue + // Re-read every plan target together before staging output. + const current = inspectFiles(rootDir, prepared) + if (current.failures.length) { + return reportRefusal( + current.analyses, + current.failures, + 'another item in this plan failed revalidation; no files were changed', + ) + } - if (dryRun) { - for (const item of ready) { - console.log(`would change ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) - } - continue - } + const currentReady = current.analyses.flatMap(analysis => analysis.ready) + if (!currentReady.length) { + return { failed: 0, applied: 0, alreadyApplied: reportAlready(current.analyses) } + } - let latestText + const unlinkSync = typeof _test?.unlinkSync === 'function' + ? temporaryFile => _test.unlinkSync(temporaryFile) + : temporaryFile => fs.unlinkSync(temporaryFile) + + const stages = [] + let stagingFailure = null + for (const analysis of current.analyses) { + if (!analysis.ready.length) continue try { - latestText = fs.readFileSync(file, 'utf8') - } catch (e) { - for (const item of ready) { - printError(item, `write failed: ${e.message}`) - failed++ - } - continue - } - const latestLines = latestText.split('\n') - const latestReadyGroups = [] - const verificationFailures = [] - for (const { lineNumber, items: lineItems } of readyGroups) { - const line = latestLines[lineNumber - 1] - if (line === undefined) { - for (const item of lineItems) verificationFailures.push({ item, message: 'line is not present' }) - continue - } - if (!hasExpectedPreImage(line, lineItems)) { - for (const item of lineItems) verificationFailures.push({ item, message: 'line hash does not match expected_line_sha256' }) - continue - } - const located = locateReplacements(line, lineItems) - if (located.failures.length) { - verificationFailures.push(...located.failures) - continue - } - latestReadyGroups.push({ lineNumber, locations: located.locations }) - } - verificationFailures.sort((a, b) => a.item.planIndex - b.item.planIndex) - if (verificationFailures.length) { - for (const failure of verificationFailures) { - printError(failure.item, failure.message) - failed++ - } - const verificationFailedItems = new Set(verificationFailures.map(failure => failure.item)) - for (const item of ready) { - if (!verificationFailedItems.has(item)) { - printError(item, 'another item for this file failed validation; file was not changed') - failed++ - } - } - continue + stages.push(stageAnalysis(rootDir, analysis, unlinkSync)) + } catch (error) { + stagingFailure = { analysis, error } + break } + } + if (stagingFailure) { + reportCleanupFailures(cleanupStages(stages, unlinkSync)) + return reportRefusal( + current.analyses, + stagingFailure.analysis.ready.map(item => ({ item, message: `write failed: ${stagingFailure.error.message}` })), + 'another file could not be staged; no files were changed', + ) + } - const latestUpdatedLines = latestLines.slice() - for (const { lineNumber, locations } of latestReadyGroups) { - latestUpdatedLines[lineNumber - 1] = applyLocatedReplacements(latestLines[lineNumber - 1], locations) - } + // Verify every snapshot again before the first commit rename. + const precommitFailures = verifySnapshots(rootDir, current.analyses) + if (precommitFailures.length) { + reportCleanupFailures(cleanupStages(stages, unlinkSync)) + return reportRefusal( + current.analyses, + precommitFailures, + 'another target changed before commit; no files were changed', + ) + } + + const renameSync = typeof _test?.renameSync === 'function' + ? (source, target, context) => _test.renameSync(source, target, context) + : (source, target) => fs.renameSync(source, target) + const committed = [] + let commitFailure = null + for (const [index, stage] of stages.entries()) { try { - writeAtomically(file, latestText === text ? updatedContent : latestUpdatedLines.join('\n')) - } catch (e) { - for (const item of ready) { - printError(item, `write failed: ${e.message}`) - failed++ - } - continue + const latest = readTarget(rootDir, stage.analysis.file) + if (!snapshotsMatch(latest, stage.analysis.snapshot)) throw new Error('target changed after pre-commit verification') + renameSync(stage.nextPath, stage.analysis.file, { phase: 'commit', index, file: stage.analysis.file }) + stage.nextPath = null + committed.push(stage) + } catch (error) { + commitFailure = { stage, error } + break } - for (const item of ready) { - console.log(`applied ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) - applied++ + } + + if (commitFailure) { + const rollbackFailures = [] + for (const [rollbackIndex, stage] of committed.toReversed().entries()) { + try { + assertSafePath(rootDir, stage.analysis.file) + renameSync(stage.backupPath, stage.analysis.file, { + phase: 'rollback', + index: rollbackIndex, + file: stage.analysis.file, + }) + stage.backupPath = null + } catch (error) { + stage.keepBackup = true + rollbackFailures.push({ stage, error }) + } } + reportCleanupFailures(cleanupStages(stages, unlinkSync)) + + const rollbackDetail = rollbackFailures.length + ? `; rollback incomplete (${rollbackFailures.map(({ stage, error }) => `${stage.analysis.file}: ${error.message}; backup retained at ${stage.backupPath}`).join('; ')})` + : committed.length ? '; earlier file commits were rolled back' : '' + const cascadeMessage = rollbackFailures.length + ? 'another file failed during plan commit and rollback was incomplete' + : 'another file failed during plan commit; earlier commits were rolled back' + return reportRefusal( + current.analyses, + commitFailure.stage.analysis.ready.map(item => ({ + item, + message: `write failed: ${commitFailure.error.message}${rollbackDetail}`, + })), + cascadeMessage, + ) } - return { failed, applied, alreadyApplied } + const cleanupFailures = cleanupStages(stages, unlinkSync) + reportCleanupFailures(cleanupFailures) + const alreadyApplied = reportAlready(current.analyses) + for (const item of sortedItems(currentReady)) { + console.log(`applied ${itemLabel(item)}: ${item.matched} -> ${item.replacement}`) + } + return { failed: cleanupFailures.length, applied: currentReady.length, alreadyApplied } } diff --git a/lib/feeds.mjs b/lib/feeds.mjs index 780f1bb..0c94ada 100644 --- a/lib/feeds.mjs +++ b/lib/feeds.mjs @@ -27,8 +27,7 @@ export const loadFeeds = feedsDir => { const file = path.join(feedsDir, f) const feed = JSON.parse(fs.readFileSync(file, 'utf8')) if (feed.spec !== 'model-eol/0.1') { - console.error(`skipping ${f}: unknown spec ${feed.spec}`) - continue + throw new Error(`${file}: unsupported feed spec ${JSON.stringify(feed.spec)}; expected "model-eol/0.1"`) } assertValidFeed(feed, file) feeds.push({ ...feed, file }) diff --git a/lib/json-schema.mjs b/lib/json-schema.mjs new file mode 100644 index 0000000..b02b880 --- /dev/null +++ b/lib/json-schema.mjs @@ -0,0 +1,291 @@ +const isObject = value => value !== null && typeof value === 'object' && !Array.isArray(value) + +const sameJsonValue = (left, right) => { + if (Object.is(left, right)) return true + if (Array.isArray(left) && Array.isArray(right)) { + return left.length === right.length && left.every((value, index) => sameJsonValue(value, right[index])) + } + if (isObject(left) && isObject(right)) { + const leftKeys = Object.keys(left) + const rightKeys = Object.keys(right) + return leftKeys.length === rightKeys.length && leftKeys.every(key => + Object.hasOwn(right, key) && sameJsonValue(left[key], right[key])) + } + return false +} + +const jsonType = value => { + if (value === null) return 'null' + if (Array.isArray(value)) return 'array' + if (Number.isInteger(value)) return 'integer' + return typeof value +} + +const matchesType = (value, type) => { + if (type === 'null') return value === null + if (type === 'array') return Array.isArray(value) + if (type === 'object') return isObject(value) + if (type === 'integer') return Number.isFinite(value) && Number.isInteger(value) + if (type === 'number') return Number.isFinite(value) && typeof value === 'number' + return typeof value === type +} + +const propertyPath = (base, property) => + /^[A-Za-z_$][A-Za-z0-9_$-]*$/.test(property) + ? `${base}.${property}` + : `${base}[${JSON.stringify(property)}]` + +const isCalendarDate = value => { + if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false + const [year, month, day] = value.split('-').map(Number) + const date = new Date(Date.UTC(year, month - 1, day)) + if (year >= 0 && year <= 99) date.setUTCFullYear(year) + return date.getUTCFullYear() === year && date.getUTCMonth() + 1 === month && date.getUTCDate() === day +} + +const isDateTime = value => { + if (typeof value !== 'string') return false + const match = value.match(/^(\d{4}-\d{2}-\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/) + if (!match || !isCalendarDate(match[1])) return false + const hour = Number(match[2]) + const minute = Number(match[3]) + const second = Number(match[4]) + const offsetHour = match[6] === undefined ? 0 : Number(match[6]) + const offsetMinute = match[7] === undefined ? 0 : Number(match[7]) + return hour <= 23 && minute <= 59 && second <= 59 && offsetHour <= 23 && offsetMinute <= 59 && !Number.isNaN(Date.parse(value)) +} + +const isUri = value => { + if (typeof value !== 'string' || !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(value)) return false + try { + return Boolean(new URL(value).protocol) + } catch { + return false + } +} + +const FORMAT_VALIDATORS = new Map([ + ['date', isCalendarDate], + ['date-time', isDateTime], + ['uri', isUri], +]) + +const withoutFragment = value => value.split('#', 1)[0] + +const decodePointer = fragment => { + if (!fragment) return [] + const decoded = decodeURIComponent(fragment) + if (!decoded.startsWith('/')) throw new Error(`unsupported JSON Schema fragment #${fragment}`) + return decoded.slice(1).split('/').map(token => token.replaceAll('~1', '/').replaceAll('~0', '~')) +} + +const atPointer = (root, fragment, reference) => { + let value = root + for (const token of decodePointer(fragment)) { + if ((isObject(value) || Array.isArray(value)) && Object.hasOwn(value, token)) value = value[token] + else throw new Error(`unresolved JSON Schema reference ${reference}`) + } + return value +} + +export class JsonSchemaRegistry { + constructor(schemas) { + this.schemas = new Map() + this.ids = new WeakMap() + for (const schema of schemas) this.add(schema) + } + + add(schema) { + if (!isObject(schema) || typeof schema.$id !== 'string' || !schema.$id) { + throw new Error('each registered JSON Schema must have a non-empty $id') + } + let id + try { + id = withoutFragment(new URL(schema.$id).href) + } catch { + throw new Error(`JSON Schema has an invalid $id: ${schema.$id}`) + } + const previous = this.schemas.get(id) + if (previous && previous !== schema) throw new Error(`duplicate JSON Schema $id: ${id}`) + this.schemas.set(id, schema) + this.ids.set(schema, id) + return this + } + + idFor(schema) { + const id = this.ids.get(schema) + if (!id) throw new Error('JSON Schema is not registered') + return id + } + + resolve(reference, fromRoot) { + const base = this.idFor(fromRoot) + let resolved + try { + resolved = new URL(reference, base) + } catch { + throw new Error(`invalid JSON Schema reference ${reference}`) + } + const rootId = withoutFragment(resolved.href) + const root = this.schemas.get(rootId) + if (!root) throw new Error(`unresolved JSON Schema reference ${reference} from ${base}`) + const fragment = resolved.hash ? resolved.hash.slice(1) : '' + return { schema: atPointer(root, fragment, reference), root } + } + + assertReferences() { + const seen = new Set() + const visit = (value, root) => { + if (!value || typeof value !== 'object' || seen.has(value)) return + seen.add(value) + if (typeof value.$ref === 'string') this.resolve(value.$ref, root) + if (Array.isArray(value)) { + for (const item of value) visit(item, root) + } else { + for (const item of Object.values(value)) visit(item, root) + } + } + for (const root of this.schemas.values()) visit(root, root) + return true + } +} + +const describeTypes = types => types.length === 1 ? types[0] : `one of: ${types.join(', ')}` + +export const validateJsonSchema = (value, schema, { registry = new JsonSchemaRegistry([schema]) } = {}) => { + const errors = [] + const fail = (path, keyword, message) => errors.push({ path, keyword, message }) + + const validate = (instance, rule, path, root) => { + if (typeof rule === 'boolean') { + if (!rule) fail(path, 'falseSchema', 'is not allowed') + return + } + if (!isObject(rule)) throw new Error(`invalid JSON Schema rule at ${path}`) + + if (typeof rule.$ref === 'string') { + const resolved = registry.resolve(rule.$ref, root) + validate(instance, resolved.schema, path, resolved.root) + return + } + + if (Array.isArray(rule.allOf)) { + for (const member of rule.allOf) validate(instance, member, path, root) + } + if (Array.isArray(rule.oneOf)) { + let matches = 0 + for (const member of rule.oneOf) { + const errorCount = errors.length + validate(instance, member, path, root) + if (errors.length === errorCount) matches++ + else errors.splice(errorCount) + } + if (matches !== 1) fail(path, 'oneOf', `must match exactly one schema; matched ${matches}`) + } + + const declaredTypes = rule.type === undefined + ? [] + : Array.isArray(rule.type) ? rule.type : [rule.type] + if (declaredTypes.length && !declaredTypes.some(type => matchesType(instance, type))) { + fail(path, 'type', `must be ${describeTypes(declaredTypes)}; got ${jsonType(instance)}`) + return + } + + if (Object.hasOwn(rule, 'const') && !sameJsonValue(instance, rule.const)) { + fail(path, 'const', `must equal ${JSON.stringify(rule.const)}`) + } + if (Array.isArray(rule.enum) && !rule.enum.some(candidate => sameJsonValue(instance, candidate))) { + fail(path, 'enum', `must be one of ${rule.enum.map(candidate => JSON.stringify(candidate)).join(', ')}`) + } + + if (typeof instance === 'string') { + if (Number.isInteger(rule.minLength) && instance.length < rule.minLength) { + fail(path, 'minLength', `must have at least ${rule.minLength} character(s)`) + } + if (Number.isInteger(rule.maxLength) && instance.length > rule.maxLength) { + fail(path, 'maxLength', `must have at most ${rule.maxLength} character(s)`) + } + if (typeof rule.pattern === 'string' && !(new RegExp(rule.pattern)).test(instance)) { + fail(path, 'pattern', `must match pattern ${rule.pattern}`) + } + if (typeof rule.format === 'string') { + const formatValidator = FORMAT_VALIDATORS.get(rule.format) + if (!formatValidator) throw new Error(`unsupported JSON Schema format ${rule.format}`) + if (!formatValidator(instance)) fail(path, 'format', `must match format ${rule.format}`) + } + } + + if (typeof instance === 'number' && Number.isFinite(instance)) { + if (typeof rule.minimum === 'number' && instance < rule.minimum) { + fail(path, 'minimum', `must be >= ${rule.minimum}`) + } + if (typeof rule.maximum === 'number' && instance > rule.maximum) { + fail(path, 'maximum', `must be <= ${rule.maximum}`) + } + } + + if (Array.isArray(instance)) { + if (Number.isInteger(rule.minItems) && instance.length < rule.minItems) { + fail(path, 'minItems', `must contain at least ${rule.minItems} item(s)`) + } + if (Number.isInteger(rule.maxItems) && instance.length > rule.maxItems) { + fail(path, 'maxItems', `must contain at most ${rule.maxItems} item(s)`) + } + if (rule.uniqueItems === true) { + for (let index = 0; index < instance.length; index++) { + if (instance.slice(0, index).some(item => sameJsonValue(item, instance[index]))) { + fail(`${path}[${index}]`, 'uniqueItems', 'must not duplicate another array item') + } + } + } + if (rule.items !== undefined) { + instance.forEach((item, index) => validate(item, rule.items, `${path}[${index}]`, root)) + } + } + + if (isObject(instance)) { + const properties = isObject(rule.properties) ? rule.properties : {} + if (Array.isArray(rule.required)) { + for (const property of rule.required) { + if (!Object.hasOwn(instance, property)) fail(propertyPath(path, property), 'required', 'is required') + } + } + for (const [property, propertySchema] of Object.entries(properties)) { + if (Object.hasOwn(instance, property)) validate(instance[property], propertySchema, propertyPath(path, property), root) + } + if (isObject(rule.dependencies)) { + for (const [property, dependencies] of Object.entries(rule.dependencies)) { + if (!Object.hasOwn(instance, property)) continue + if (Array.isArray(dependencies)) { + for (const dependency of dependencies) { + if (!Object.hasOwn(instance, dependency)) { + fail(propertyPath(path, dependency), 'dependencies', `is required when ${property} is present`) + } + } + } else { + validate(instance, dependencies, path, root) + } + } + } + for (const property of Object.keys(instance)) { + if (Object.hasOwn(properties, property)) continue + if (rule.additionalProperties === false) { + fail(propertyPath(path, property), 'additionalProperties', 'is not allowed') + } else if (isObject(rule.additionalProperties) || typeof rule.additionalProperties === 'boolean') { + validate(instance[property], rule.additionalProperties, propertyPath(path, property), root) + } + } + } + } + + const root = schema + registry.idFor(root) + validate(value, schema, '$', root) + return errors +} + +export const assertJsonSchema = (value, schema, options = {}) => { + const errors = validateJsonSchema(value, schema, options) + if (errors.length) throw new Error(errors.map(error => `${error.path}: ${error.message}`).join('\n')) + return value +} diff --git a/lib/reports.mjs b/lib/reports.mjs index 3d73862..91c7829 100644 --- a/lib/reports.mjs +++ b/lib/reports.mjs @@ -65,6 +65,8 @@ export const buildInventory = ({ scan, findings, days, via, scope, targets }) => ...(ref.mapped_from ? { mapped_from: ref.mapped_from } : {}), safe_until: lifecycle.safe_until, replacement: lifecycle.replacement, + ...(lifecycle.replacement_options?.length ? { replacement_options: lifecycle.replacement_options } : {}), + ...(lifecycle.replacement_note ? { replacement_note: lifecycle.replacement_note } : {}), source: ref.source, policy: ref.policy ?? null, feed_generated: ref.generated ?? null, @@ -229,30 +231,101 @@ export const formatSchedule = (schedule, days) => { } export const formatInventoryCycloneDX = inventory => { - const components = new Map() - for (const item of inventory.model_references) { - let component = components.get(item.id) - if (!component) { - component = { - type: 'machine-learning-model', - name: item.id, - group: item.publisher, - properties: [ - { name: 'model-eol:status', value: String(item.status) }, - { name: 'model-eol:shutdown', value: item.shutdown ?? '' }, - { name: 'model-eol:replacement', value: item.replacement ?? '' }, - { name: 'model-eol:usage', value: item.usage }, - ], - evidence: { occurrences: [] }, + const statusSeverity = new Map([ + ['retired', 0], + ['retiring', 1], + ['scheduled', 2], + ['watch', 3], + ['ok', 4], + ]) + const lifecycleChannel = item => { + if (typeof item.requested_via === 'string') { + return { requested: item.requested_via, label: item.requested_via } + } + if (item.requested_via === null) return { requested: null, label: 'publisher-direct' } + if (item.via && item.via !== 'publisher' && item.via !== 'publisher-fallback') { + return { requested: item.via, label: item.via } + } + return { requested: null, label: 'publisher-direct' } + } + const encodeIdentityPart = value => encodeURIComponent(String(value)) + const compareText = (a, b) => a < b ? -1 : a > b ? 1 : 0 + const identityFor = item => { + const channel = lifecycleChannel(item) + return { + channel, + key: JSON.stringify([item.publisher, item.id, channel.requested]), + bomRef: `model-eol:model:${encodeIdentityPart(item.publisher)}:${encodeIdentityPart(item.id)}:via:${encodeIdentityPart(channel.label)}`, + } + } + const compareReferences = (a, b) => { + const aIdentity = identityFor(a) + const bIdentity = identityFor(b) + return compareText(a.publisher, b.publisher) || + compareText(a.id, b.id) || + compareText(aIdentity.channel.label, bIdentity.channel.label) || + compareText(a.file, b.file) || + a.line - b.line || + compareText(a.usage, b.usage) || + compareText(a.matched, b.matched) + } + + const groups = new Map() + for (const item of [...inventory.model_references].sort(compareReferences)) { + const identity = identityFor(item) + let group = groups.get(identity.key) + if (!group) { + group = { + identity, + item, + status: item.status, + usages: new Set(), + occurrences: new Set(), } - components.set(item.id, component) + groups.set(identity.key, group) } - const usage = component.properties.find(property => property.name === 'model-eol:usage') - const usages = new Set(usage.value ? usage.value.split(',') : []) - usages.add(item.usage) - usage.value = [...usages].sort().join(',') - component.evidence.occurrences.push({ location: `${item.file}#${item.line}` }) + if ((statusSeverity.get(item.status) ?? 9) < (statusSeverity.get(group.status) ?? 9)) { + group.status = item.status + } + group.usages.add(item.usage) + group.occurrences.add(`${item.file}#${item.line}`) } + + const components = [...groups.values()].map(group => { + const { item, identity } = group + return { + type: 'machine-learning-model', + 'bom-ref': identity.bomRef, + name: item.id, + group: item.publisher, + properties: [ + { name: 'model-eol:lifecycle_channel', value: identity.channel.label }, + { name: 'model-eol:applied_clock', value: item.via ?? 'unavailable' }, + { name: 'model-eol:status', value: String(group.status) }, + { name: 'model-eol:shutdown', value: item.shutdown ?? '' }, + ...(item.date_precision + ? [{ name: 'model-eol:date_precision', value: item.date_precision }] + : []), + ...(item.distribution_status + ? [{ name: 'model-eol:distribution_status', value: item.distribution_status }] + : []), + ...(item.safe_until + ? [{ name: 'model-eol:safe_until', value: item.safe_until }] + : []), + { name: 'model-eol:replacement', value: item.replacement ?? '' }, + ...(item.replacement_options?.length + ? [{ name: 'model-eol:replacement_options', value: JSON.stringify(item.replacement_options) }] + : []), + ...(item.replacement_note + ? [{ name: 'model-eol:replacement_note', value: item.replacement_note }] + : []), + { name: 'model-eol:usage', value: [...group.usages].sort().join(',') }, + ], + evidence: { + occurrences: [...group.occurrences].sort().map(location => ({ location })), + }, + } + }) return { bomFormat: 'CycloneDX', specVersion: '1.6', @@ -263,7 +336,7 @@ export const formatInventoryCycloneDX = inventory => { { name: 'model-eol:generator', value: 'model-eol/inventory-cyclonedx@0.1' }, ], }, - components: [...components.values()], + components, } } diff --git a/lib/scanner.mjs b/lib/scanner.mjs index 918241b..55e76f9 100644 --- a/lib/scanner.mjs +++ b/lib/scanner.mjs @@ -1,6 +1,7 @@ import fs from 'node:fs' import path from 'node:path' import { spawnSync } from 'node:child_process' +import { TextDecoder } from 'node:util' import { buildModelPattern } from './feeds.mjs' import { matchesAnyGlob, matchesGlob, normalizeRepoPath } from './glob.mjs' @@ -25,10 +26,15 @@ export const INCOMPLETE_SCAN_REASONS = new Set([ 'unreadable-file', 'unreadable-path', 'unreadable-directory', + 'invalid-utf8', + 'symlink-skipped', + 'submodule-skipped', 'file-count-cap', 'git-listing-failure', ]) +const utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }) + export const incompleteScanNotes = notes => notes.filter(note => INCOMPLETE_SCAN_REASONS.has(note.reason)) export const scanIsIncomplete = notes => incompleteScanNotes(notes).length > 0 @@ -167,17 +173,63 @@ export const gitRootFor = target => { const changedKey = (file, line) => `${path.resolve(file)}:${line}` -const parseDiffPath = value => { - const raw = value.trim() - if (raw === '/dev/null') return null - let decoded = raw - if (raw.startsWith('"')) { - try { - decoded = JSON.parse(raw) - } catch { - decoded = raw.slice(1, raw.endsWith('"') ? -1 : undefined) +const gitPathEscapes = new Map([ + ['a', 0x07], + ['b', 0x08], + ['t', 0x09], + ['n', 0x0a], + ['v', 0x0b], + ['f', 0x0c], + ['r', 0x0d], + ['\\', 0x5c], + ['"', 0x22], +]) + +const decodeGitQuotedPath = raw => { + if (!raw.endsWith('"')) throw new Error(`malformed quoted Git diff path: ${raw}`) + const bytes = [] + let literalStart = 1 + const appendLiteral = end => { + if (end > literalStart) bytes.push(...Buffer.from(raw.slice(literalStart, end), 'utf8')) + } + for (let index = 1; index < raw.length - 1; index++) { + const character = raw[index] + if (character === '"') throw new Error(`malformed quoted Git diff path: ${raw}`) + if (character !== '\\') continue + appendLiteral(index) + index++ + if (index >= raw.length - 1) throw new Error(`malformed quoted Git diff path: ${raw}`) + const escaped = raw[index] + if (/[0-7]/.test(escaped)) { + let octal = escaped + while (octal.length < 3 && index + 1 < raw.length - 1 && /[0-7]/.test(raw[index + 1])) { + octal += raw[++index] + } + const value = Number.parseInt(octal, 8) + if (value > 0xff) throw new Error(`invalid octal escape in Git diff path: ${raw}`) + bytes.push(value) + } else if (gitPathEscapes.has(escaped)) { + bytes.push(gitPathEscapes.get(escaped)) + } else { + throw new Error(`unsupported escape in Git diff path: ${raw}`) } + literalStart = index + 1 + } + appendLiteral(raw.length - 1) + let decoded + try { + decoded = utf8Decoder.decode(Buffer.from(bytes)) + } catch { + throw new Error(`Git diff path is not valid UTF-8: ${raw}`) } + if (decoded.includes('\0')) throw new Error(`Git diff path contains a NUL byte: ${raw}`) + return decoded +} + +export const parseDiffPath = value => { + const raw = value.endsWith('\r') ? value.slice(0, -1) : value + const decoded = raw.startsWith('"') ? decodeGitQuotedPath(raw) : raw + if (decoded === '/dev/null') return null return decoded.startsWith('b/') ? decoded.slice(2) : decoded } @@ -211,7 +263,11 @@ export const addedLinesForTargets = (targets, baseRef) => { } return relative || '.' }) - const result = spawnSync('git', ['diff', '--unified=0', baseRef, '--', ...relativeTargets], { + const result = spawnSync('git', [ + '-c', 'core.quotePath=true', + 'diff', '--no-color', '--src-prefix=a/', '--dst-prefix=b/', + '--text', '--no-ext-diff', '--no-textconv', '--unified=0', baseRef, '--', ...relativeTargets, + ], { cwd: gitRoot, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], @@ -245,6 +301,9 @@ export const addedLinesForTargets = (targets, baseRef) => { newLinesRemaining = hunk[2] === undefined ? 1 : Number(hunk[2]) continue } + // Unified-diff metadata such as "\\ No newline at end of file" does not + // consume a line on either side of the hunk. + if (line.startsWith('\\ ')) continue if (!inHunk || !file || newLinesRemaining === 0) continue if (line.startsWith('+')) added.add(changedKey(file, newLine)) if (!line.startsWith('-')) { @@ -278,20 +337,22 @@ const collectFilesDetailed = (targets, { } } const excludedFiles = new Set(ignoredFiles.map(canonicalPath)) - const roots = ignoreRoots.map(canonicalPath) + const roots = [...new Set(ignoreRoots.flatMap(root => [path.resolve(root), canonicalPath(root)]))] let fileLimitWarned = false let stopped = false const note = value => notes.push(value) const repoPathsFor = absolute => { - const resolved = canonicalPath(absolute) - const repoPaths = [] - for (const root of roots) { - const relative = path.relative(root, resolved) - if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) continue - repoPaths.push(normalizeRepoPath(relative)) + const candidates = [...new Set([path.resolve(absolute), canonicalPath(absolute)])] + const repoPaths = new Set() + for (const candidate of candidates) { + for (const root of roots) { + const relative = path.relative(root, candidate) + if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) continue + repoPaths.add(normalizeRepoPath(relative)) + } } - return repoPaths + return [...repoPaths] } const matchesIgnoredRepoPath = repoPath => matchesAnyGlob(repoPath, ignorePaths) || ignorePaths.some(pattern => { @@ -300,9 +361,12 @@ const collectFilesDetailed = (targets, { }) const isIgnoredFile = absolute => { const resolved = canonicalPath(absolute) - if (excludedFiles.has(resolved) || repoPathsFor(resolved).some(matchesIgnoredRepoPath)) return true - const policy = policyForFile?.(resolved) - return Boolean(policy?.repoPath && matchesIgnoredRepoPathWithPatterns(policy.repoPath, policy.ignore?.paths ?? [])) + if (excludedFiles.has(resolved) || repoPathsFor(absolute).some(matchesIgnoredRepoPath)) return true + const policyPaths = [...new Set([path.resolve(absolute), resolved])] + return policyPaths.some(candidate => { + const policy = policyForFile?.(candidate) + return Boolean(policy?.repoPath && matchesIgnoredRepoPathWithPatterns(policy.repoPath, policy.ignore?.paths ?? [])) + }) } const matchesIgnoredRepoPathWithPatterns = (repoPath, patterns) => matchesAnyGlob(repoPath, patterns) || patterns.some(pattern => { @@ -327,7 +391,11 @@ const collectFilesDetailed = (targets, { note({ reason: 'unreadable-file', file: displayPath(absolute), message: e.message }) return } - if (!st.isFile() || st.isSymbolicLink() || !isScannable(absolute, includeDocs)) return + if (st.isSymbolicLink()) { + if (!isIgnoredFile(absolute)) note({ reason: 'symlink-skipped', file: displayPath(absolute) }) + return + } + if (!st.isFile() || !isScannable(absolute, includeDocs)) return if (st.size > MAX_FILE_BYTES) { note({ reason: 'file-too-large', @@ -362,7 +430,10 @@ const collectFilesDetailed = (targets, { note({ reason: 'unreadable-path', file: displayPath(absolute), message: e.message }) return } - if (st.isSymbolicLink()) return + if (st.isSymbolicLink()) { + if (!isIgnoredFile(absolute)) note({ reason: 'symlink-skipped', file: displayPath(absolute) }) + return + } if (!st.isDirectory()) { addFile(absolute) return @@ -400,10 +471,46 @@ const collectFilesDetailed = (targets, { }) return } + const tracked = spawnSync('git', [ + 'ls-files', '-z', '--cached', '--stage', '--', relativeTarget, + ], { + cwd: gitRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }) + if (tracked.error || tracked.status !== 0) { + const detail = tracked.error?.message || tracked.stderr?.trim() || `exit ${tracked.status}` + note({ + reason: 'git-listing-failure', + file: displayPath(absoluteTarget), + message: `git index metadata listing failed: ${detail}; no recursive fallback used`, + }) + return + } + const gitlinks = new Set() + for (const record of tracked.stdout.split('\0')) { + if (!record) continue + const separator = record.indexOf('\t') + const metadata = separator === -1 ? '' : record.slice(0, separator) + const relativeFile = separator === -1 ? '' : record.slice(separator + 1) + if (!/^[0-7]{6} [0-9a-f]+ [0-3]$/.test(metadata) || !relativeFile) { + note({ + reason: 'git-listing-failure', + file: displayPath(absoluteTarget), + message: 'git index metadata listing returned a malformed record; no recursive fallback used', + }) + return + } + if (metadata.startsWith('160000 ')) gitlinks.add(relativeFile) + } for (const relativeFile of result.stdout.split('\0')) { if (stopped) break if (!relativeFile) continue const absoluteFile = path.resolve(gitRoot, relativeFile) + if (gitlinks.has(relativeFile)) { + if (!isIgnoredFile(absoluteFile)) note({ reason: 'submodule-skipped', file: displayPath(absoluteFile) }) + continue + } if (path.relative(gitRoot, absoluteFile).split(path.sep).some(part => SKIP_DIRS.has(part))) continue addFile(absoluteFile) } @@ -536,13 +643,20 @@ export const scanTargets = ({ } for (const file of files) { - let text + let bytes try { - text = fs.readFileSync(file, 'utf8') + bytes = fs.readFileSync(file) } catch (e) { collected.notes.push({ reason: 'unreadable-file', file, message: e.message }) continue } + let text + try { + text = utf8Decoder.decode(bytes) + } catch (e) { + collected.notes.push({ reason: 'invalid-utf8', file, message: e.message }) + continue + } // Product artifacts and generated clients describe model IDs but are not // authoritative usage or safe migration targets. if (modelEolDocument(file, text)) { diff --git a/lib/validate-document.mjs b/lib/validate-document.mjs new file mode 100644 index 0000000..a654f1b --- /dev/null +++ b/lib/validate-document.mjs @@ -0,0 +1,320 @@ +import fs from 'node:fs' +import path from 'node:path' +import { TextDecoder } from 'node:util' + +import { normalizeConfig } from './config.mjs' +import { JsonSchemaRegistry, validateJsonSchema } from './json-schema.mjs' +import { validateFeed } from './validate-feed.mjs' + +export const DOCUMENT_TYPES = Object.freeze(['feed', 'config', 'check', 'inventory', 'schedule', 'alert', 'plan']) + +const SCHEMA_FILES = Object.freeze({ + feed: 'model-eol.schema.json', + config: 'model-eol.bot-config.schema.json', + check: 'model-eol.check.schema.json', + inventory: 'model-eol.inventory.schema.json', + schedule: 'model-eol.schedule.schema.json', + alert: 'model-eol.alert.schema.json', + plan: 'model-eol.plan.schema.json', +}) + +const TYPE_ALIASES = new Map([ + ['bot-config', 'config'], + ['model-eol-config', 'config'], +]) + +const utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }) + +const SCHEMA_DISCRIMINATORS = new Map([ + ['model-eol/check@0.1', 'check'], + ['model-eol/inventory@0.1', 'inventory'], + ['model-eol/schedule@0.1', 'schedule'], + ['model-eol/alert@0.1', 'alert'], +]) + +let catalogCache + +export const loadDocumentSchemaCatalog = () => { + if (catalogCache) return catalogCache + const directory = new URL('../schema/', import.meta.url) + const byType = new Map() + for (const type of DOCUMENT_TYPES) { + const file = new URL(SCHEMA_FILES[type], directory) + let schema + try { + schema = JSON.parse(fs.readFileSync(file, 'utf8')) + } catch (error) { + throw new Error(`failed to load ${type} schema: ${error.message}`) + } + byType.set(type, schema) + } + const registry = new JsonSchemaRegistry([...byType.values()]) + registry.assertReferences() + const byId = new Map([...byType.entries()].map(([type, schema]) => [schema.$id, { type, schema }])) + catalogCache = { byType, byId, registry } + return catalogCache +} + +export const normalizeDocumentType = value => { + if (typeof value !== 'string' || !value) return null + const normalized = TYPE_ALIASES.get(value) ?? value + return DOCUMENT_TYPES.includes(normalized) ? normalized : null +} + +const isObject = value => value !== null && typeof value === 'object' && !Array.isArray(value) + +const configKeys = new Set([ + 'days', 'scope', 'via', 'feeds', 'ignore', 'overrides', 'routes', 'issues', 'eval', +]) + +export const detectDocumentType = (document, { file = null } = {}) => { + if (!isObject(document)) return null + if (typeof document.$schema === 'string' && typeof document.$id === 'string') return 'schema' + if (Object.hasOwn(document, 'spec')) return 'feed' + if (Object.hasOwn(document, 'plan_schema')) return 'plan' + if (typeof document.schema === 'string') return SCHEMA_DISCRIMINATORS.get(document.schema) ?? null + if (file && path.basename(file) === '.model-eol.json') return 'config' + const keys = Object.keys(document) + if (keys.length && keys.every(key => configKeys.has(key))) return 'config' + return null +} + +const schemaDefinitionErrors = (document, catalog) => { + const errors = [] + const fail = (path, keyword, message) => errors.push({ path, keyword, message }) + if (!isObject(document)) return [{ path: '$', keyword: 'type', message: 'must be an object' }] + if (document.$schema !== 'http://json-schema.org/draft-07/schema#') { + fail('$.$schema', '$schema', 'must equal "http://json-schema.org/draft-07/schema#"') + } + const registered = catalog.byId.get(document.$id) + if (!registered) fail('$.$id', '$id', `must identify one of the ${catalog.byType.size} published model-eol schemas`) + + const validTypes = new Set(['null', 'boolean', 'object', 'array', 'number', 'integer', 'string']) + const validFormats = new Set(['date', 'date-time', 'uri']) + const supportedKeywords = new Set([ + '$schema', '$id', '$ref', '$comment', + 'title', 'description', 'default', 'examples', 'readOnly', 'writeOnly', + 'type', 'const', 'enum', 'format', 'pattern', + 'minLength', 'maxLength', 'minimum', 'maximum', + 'minItems', 'maxItems', 'uniqueItems', 'items', + 'required', 'properties', 'additionalProperties', 'dependencies', + 'definitions', 'allOf', 'oneOf', + ]) + const childPath = (base, key) => `${base}.${key}` + const lint = (rule, at) => { + if (typeof rule === 'boolean') return + if (!isObject(rule)) { + fail(at, 'schema', 'must be an object or boolean schema') + return + } + for (const keyword of Object.keys(rule)) { + if (!supportedKeywords.has(keyword)) { + fail(childPath(at, keyword), keyword, `unsupported JSON Schema keyword ${keyword}`) + } + } + if (rule.$ref !== undefined && typeof rule.$ref !== 'string') fail(childPath(at, '$ref'), '$ref', 'must be a string') + if (rule.type !== undefined) { + const types = Array.isArray(rule.type) ? rule.type : [rule.type] + if (!types.length || types.some(type => typeof type !== 'string' || !validTypes.has(type))) { + fail(childPath(at, 'type'), 'type', 'must name one or more JSON types') + } + } + if (rule.required !== undefined && (!Array.isArray(rule.required) || rule.required.some(value => typeof value !== 'string'))) { + fail(childPath(at, 'required'), 'required', 'must be an array of property names') + } + if (rule.enum !== undefined && (!Array.isArray(rule.enum) || rule.enum.length === 0)) { + fail(childPath(at, 'enum'), 'enum', 'must be a non-empty array') + } + if (rule.format !== undefined && (typeof rule.format !== 'string' || !validFormats.has(rule.format))) { + fail(childPath(at, 'format'), 'format', 'must be date, date-time, or uri') + } + if (rule.pattern !== undefined) { + if (typeof rule.pattern !== 'string') fail(childPath(at, 'pattern'), 'pattern', 'must be a string') + else { + try { + new RegExp(rule.pattern) + } catch { + fail(childPath(at, 'pattern'), 'pattern', 'must be a valid regular expression') + } + } + } + for (const keyword of ['minLength', 'maxLength', 'minItems', 'maxItems']) { + if (rule[keyword] !== undefined && (!Number.isInteger(rule[keyword]) || rule[keyword] < 0)) { + fail(childPath(at, keyword), keyword, 'must be a non-negative integer') + } + } + if (rule.uniqueItems !== undefined && typeof rule.uniqueItems !== 'boolean') { + fail(childPath(at, 'uniqueItems'), 'uniqueItems', 'must be a boolean') + } + for (const keyword of ['minimum', 'maximum']) { + if (rule[keyword] !== undefined && (typeof rule[keyword] !== 'number' || !Number.isFinite(rule[keyword]))) { + fail(childPath(at, keyword), keyword, 'must be a finite number') + } + } + for (const keyword of ['properties', 'definitions']) { + if (rule[keyword] === undefined) continue + if (!isObject(rule[keyword])) { + fail(childPath(at, keyword), keyword, 'must be an object of schemas') + continue + } + for (const [name, child] of Object.entries(rule[keyword])) lint(child, `${at}.${keyword}[${JSON.stringify(name)}]`) + } + if (rule.items !== undefined) lint(rule.items, childPath(at, 'items')) + if (rule.additionalProperties !== undefined) lint(rule.additionalProperties, childPath(at, 'additionalProperties')) + for (const keyword of ['allOf', 'oneOf']) { + if (rule[keyword] === undefined) continue + if (!Array.isArray(rule[keyword]) || rule[keyword].length === 0) { + fail(childPath(at, keyword), keyword, 'must be a non-empty array of schemas') + continue + } + rule[keyword].forEach((child, index) => lint(child, `${at}.${keyword}[${index}]`)) + } + if (rule.dependencies !== undefined) { + if (!isObject(rule.dependencies)) fail(childPath(at, 'dependencies'), 'dependencies', 'must be an object') + else { + for (const [name, dependency] of Object.entries(rule.dependencies)) { + const dependencyPath = `${at}.dependencies[${JSON.stringify(name)}]` + if (Array.isArray(dependency)) { + if (dependency.some(value => typeof value !== 'string')) fail(dependencyPath, 'dependencies', 'must contain property names') + } else lint(dependency, dependencyPath) + } + } + } + } + lint(document, '$') + + if (registered) { + try { + const companionSchemas = [...catalog.byType.values()].filter(schema => schema.$id !== document.$id) + const registry = new JsonSchemaRegistry([document, ...companionSchemas]) + registry.assertReferences() + } catch (error) { + fail('$', '$ref', error.message) + } + } + return errors +} + +const semanticErrors = (document, type) => { + if (type === 'feed') { + return validateFeed(document).map(error => ({ + path: error.path === 'feed' ? '$' : `$.${error.path}`, + keyword: 'model-eol-feed', + message: error.message, + })) + } + if (type === 'config') { + try { + normalizeConfig(document) + return [] + } catch (error) { + return [{ path: '$', keyword: 'model-eol-config', message: error.message }] + } + } + if (type === 'plan') { + try { + validatePlanItems(document.items) + return [] + } catch (error) { + return [{ path: '$.items', keyword: 'model-eol-plan', message: error.message }] + } + } + return [] +} + +export const validateDocument = (document, { type, file = null } = {}) => { + const selectedType = type === undefined || type === null + ? detectDocumentType(document, { file }) + : normalizeDocumentType(type) + if (!selectedType) { + const detail = type + ? `unknown document type ${JSON.stringify(type)}` + : typeof document?.schema === 'string' + ? `unknown document schema ${JSON.stringify(document.schema)}` + : 'could not determine the document type' + return { + type: null, + errors: [{ path: '$', keyword: 'documentType', message: `${detail}; use --type ${DOCUMENT_TYPES.join('|')}` }], + } + } + + const catalog = loadDocumentSchemaCatalog() + if (selectedType === 'schema') return { type: selectedType, errors: schemaDefinitionErrors(document, catalog) } + const { byType, registry } = catalog + const schemaErrors = validateJsonSchema(document, byType.get(selectedType), { registry }) + return { + type: selectedType, + errors: schemaErrors.length ? schemaErrors : semanticErrors(document, selectedType), + } +} + +export const formatDocumentErrors = errors => errors.map(error => `${error.path}: ${error.message}`) + +export const assertValidDocument = (document, options = {}) => { + const result = validateDocument(document, options) + if (result.errors.length) throw new Error(formatDocumentErrors(result.errors).join('\n')) + return { document, type: result.type } +} + +export const validatePlanItems = items => { + if (!Array.isArray(items)) throw new Error('plan must contain an items array') + for (const [index, item] of items.entries()) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + throw new Error(`plan item ${index} must be an object`) + } + if (typeof item.file !== 'string' || item.file.length === 0) { + throw new Error(`plan item ${index} file must be a non-empty string`) + } + if (!Number.isInteger(item.line) || item.line < 1) { + throw new Error(`plan item ${index} line must be an integer >= 1`) + } + if (!Number.isInteger(item.occurrence) || item.occurrence < 0) { + throw new Error(`plan item ${index} occurrence must be an integer >= 0`) + } + if (typeof item.matched !== 'string' || item.matched.length === 0) { + throw new Error(`plan item ${index} matched must be a non-empty string`) + } + if (typeof item.replacement !== 'string' || item.replacement.length === 0) { + throw new Error(`plan item ${index} replacement must be a non-empty string`) + } + if (item.replacement === item.matched) { + throw new Error(`plan item ${index} replacement must differ from matched`) + } + if (typeof item.expected_line_sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(item.expected_line_sha256)) { + throw new Error(`plan item ${index} expected_line_sha256 must be 64 lowercase hexadecimal characters`) + } + } + return items +} + +export const assertValidPlanDocument = plan => { + validatePlanItems(plan?.items) + assertValidDocument(plan, { type: 'plan' }) + return plan +} + +export const readJsonDocument = file => { + let bytes + try { + bytes = fs.readFileSync(file) + } catch (error) { + throw new Error(`could not read ${file}: ${error.message}`) + } + let source + try { + source = utf8Decoder.decode(bytes) + } catch { + throw new Error(`invalid UTF-8 in ${file}`) + } + try { + return JSON.parse(source) + } catch (error) { + throw new Error(`invalid JSON in ${file}: ${error.message}`) + } +} + +export const validateDocumentFile = (file, { type = null } = {}) => { + const document = readJsonDocument(file) + const result = validateDocument(document, { type, file }) + return { ...result, document } +} diff --git a/lib/validate-feed.mjs b/lib/validate-feed.mjs index be1eb6e..d62bc32 100644 --- a/lib/validate-feed.mjs +++ b/lib/validate-feed.mjs @@ -204,17 +204,23 @@ export const validateFeed = feed => { if (!Array.isArray(model.distributions)) { fail(`${at}.distributions`, 'must be an array') } else { + const seenDistributionVias = new Map() for (const [distributionIndex, distribution] of model.distributions.entries()) { const dAt = `${at}.distributions[${distributionIndex}]` if (!checkObjectKeys(distribution, dAt, new Set(['via', 'announced', 'shutdown', 'date_precision', 'status', 'source']))) continue if (!checkString(distribution.via, `${dAt}.via`, MAX_IDENTIFIER_LENGTH) || !distribution.via) fail(`${dAt}.via`, 'must be a non-empty string') + if (typeof distribution.via === 'string' && distribution.via.length > 0) { + const previous = seenDistributionVias.get(distribution.via) + if (previous) fail(`${dAt}.via`, `duplicate distributor via "${distribution.via}" also used at ${previous}`) + else seenDistributionVias.set(distribution.via, `${dAt}.via`) + } const distributionAnnounced = checkDate(distribution.announced, `${dAt}.announced`) const distributionShutdown = checkDate(distribution.shutdown, `${dAt}.shutdown`) if (distributionAnnounced && distributionShutdown && distribution.shutdown < distribution.announced) fail(dAt, 'shutdown precedes announced') if (distribution.date_precision !== undefined && (typeof distribution.date_precision !== 'string' || !DATE_PRECISIONS.has(distribution.date_precision))) fail(`${dAt}.date_precision`, 'must be exact or earliest') if (distribution.status !== undefined && (typeof distribution.status !== 'string' || !DISTRIBUTION_STATUSES.has(distribution.status))) fail(`${dAt}.status`, 'has an unsupported status') checkUrl(distribution.source, `${dAt}.source`) - if (distribution.shutdown && !(distribution.source || model.source || feed.source)) fail(dAt, 'dated distribution needs a source') + if ((distribution.announced || distribution.shutdown) && !(distribution.source || model.source || feed.source)) fail(dAt, 'dated distribution needs a source') } } } diff --git a/package.json b/package.json index de1c412..7bbe586 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "model-eol-bot": "bot/bot.mjs" }, "scripts": { - "test": "node test/run.mjs && node refresh/test/run.mjs && node bot/test/run.mjs && node scripts/test-action-contract.mjs && node bot/test/package.mjs && node scripts/test-feed-changelog.mjs" + "test": "node test/run.mjs && node refresh/test/run.mjs && node bot/test/run.mjs && node scripts/test-action-contract.mjs && node scripts/test-document-validation.mjs && node scripts/test-eval-harness.mjs && node bot/test/package.mjs && node scripts/test-feed-changelog.mjs && node scripts/test-public-site.mjs" }, "repository": { "type": "git", @@ -38,12 +38,13 @@ "bot/lib/", "feeds/", "schema/", + "examples/", "SPEC.md", "README.md", "LICENSE", "action.yml" ], "engines": { - "node": ">=20" + "node": ">=22" } } diff --git a/refresh/diff.mjs b/refresh/diff.mjs index 7d019ee..9c82147 100644 --- a/refresh/diff.mjs +++ b/refresh/diff.mjs @@ -97,7 +97,12 @@ function distributionChanges(oldModel, model, publisher) { changes.push({ publisher, id: model.id, via, kind: 'removed', old, next }) continue } - if (old.announced !== next.announced || old.shutdown !== next.shutdown || old.date_precision !== next.date_precision) { + if ( + old.announced !== next.announced || + old.shutdown !== next.shutdown || + old.date_precision !== next.date_precision || + old.status !== next.status + ) { changes.push({ publisher, id: model.id, via, kind: 'changed', old, next }) } } @@ -212,7 +217,7 @@ const dateValue = (date, precision) => `${code(date)}${precision === 'earliest' const dateLine = model => `announced: ${code(model.announced)}; shutdown: ${dateValue(model.shutdown, model.date_precision)}` function distributionDateLine(distribution) { - return `announced: ${code(distribution?.announced)}; EOL: ${dateValue(distribution?.shutdown, distribution?.date_precision)}` + return `announced: ${code(distribution?.announced)}; EOL: ${dateValue(distribution?.shutdown, distribution?.date_precision)}; status: ${code(distribution?.status)}` } function distributionModelLabel(change) { @@ -224,7 +229,7 @@ function renderDistributionChanges(result) { for (const change of result.distributionChanges) { const label = code(distributionModelLabel(change)) if (change.kind === 'added') { - lines.push(`- ${label} - ${code(change.via)} dates added; ${distributionDateLine(change.next)}`) + lines.push(`- ${label} - ${code(change.via)} distribution added; ${distributionDateLine(change.next)}`) } else if (change.kind === 'removed') { lines.push(`- ${label} - ${code(change.via)} distribution removed`) } else { @@ -236,6 +241,9 @@ function renderDistributionChanges(result) { } else if (change.old.date_precision !== change.next.date_precision) { lines.push(`- ${label} - ${code(change.via)} EOL date precision changed ${dateValue(change.old.shutdown, change.old.date_precision)} -> ${dateValue(change.next.shutdown, change.next.date_precision)}`) } + if (change.old.status !== change.next.status) { + lines.push(`- ${label} - ${code(change.via)} status changed ${code(change.old.status)} -> ${code(change.next.status)}`) + } } } for (const item of result.unconfirmedDistributions) { diff --git a/refresh/distributors.mjs b/refresh/distributors.mjs index a51723e..40b3aaa 100644 --- a/refresh/distributors.mjs +++ b/refresh/distributors.mjs @@ -97,7 +97,7 @@ function expandRows(rows) { }) } -function headerIndexes(rows) { +function bedrockHeaderIndexes(rows) { for (const [row, candidate] of rows.entries()) { const labels = candidate.cells.map(cell => cell.text.toLowerCase()) const model = labels.findIndex(label => /\bmodel\s+(?:id|identifier)\b/i.test(label)) @@ -107,7 +107,15 @@ function headerIndexes(rows) { /end\s*[- ]?of\s*[- ]?life.*\bdate\b|\bdate\b.*end\s*[- ]?of\s*[- ]?life/i.test(label) || /\b(?:retirement|discontinuation)\b.*\bdate\b|\bdate\b.*\b(?:retirement|discontinuation)\b/i.test(label) )) - if (model >= 0 && legacy >= 0 && eol >= 0) return { row, model, legacy, eol } + if (model < 0 || legacy < 0 || eol < 0) continue + const extendedAccess = labels.findIndex(label => ( + /\bpublic\s+extended\s+access\b.*\bdate\b|\bdate\b.*\bpublic\s+extended\s+access\b/i.test(label) + )) + if (extendedAccess < 0) { + throw new Error('aws-bedrock lifecycle table is missing the Public extended access start date column') + } + const status = labels.findIndex(label => /^(?:(?:model|lifecycle)\s+)*status$/i.test(label.trim())) + return { row, model, legacy, eol, extendedAccess, status } } return undefined } @@ -123,6 +131,25 @@ function lifecycleDate(cell, field, bedrockId) { return parsed } +function bedrockLifecycleStatus(cell, bedrockId) { + const value = cell?.text.trim().toLowerCase().replace(/[‐‑‒–\u2014−]/g, '-').replace(/\s+/g, ' ') + const statuses = new Map([ + ['active', 'active'], + ['legacy', 'legacy'], + ['extended access', 'extended-access'], + ['public extended access', 'extended-access'], + ['end-of-life', 'retired'], + ['end of life', 'retired'], + ['end-of-life (eol)', 'retired'], + ['end of life (eol)', 'retired'], + ['eol', 'retired'], + ['retired', 'retired'], + ]) + const status = statuses.get(value) + if (!status) throw new Error(`aws-bedrock lifecycle entry ${bedrockId} has an unsupported lifecycle status: ${cell?.text || '(empty)'}`) + return status +} + function vertexHeaderIndexes(rows) { for (const [row, candidate] of rows.entries()) { const labels = candidate.cells.map(cell => cell.text.toLowerCase()) @@ -133,10 +160,14 @@ function vertexHeaderIndexes(rows) { return undefined } -function sameRecord(left, right) { - return left.bedrockId === right.bedrockId && - left.legacy === right.legacy && - left.eol === right.eol +function mergeRegionalBedrockRecord(left, right) { + if (left.legacy !== right.legacy || left.eol !== right.eol) return undefined + if (left.status === right.status) return left + const statuses = new Set([left.status, right.status]) + if (statuses.size === 2 && statuses.has('legacy') && statuses.has('extended-access')) { + return { ...left, status: 'extended-access' } + } + return undefined } const isBedrockModelId = value => /^[a-z0-9][a-z0-9_-]*\.[a-z0-9][a-z0-9._:+-]*$/i.test(value) @@ -170,7 +201,7 @@ export function parseBedrockLifecycleHtml(html) { for (const table of tables) { const rows = expandRows(tableRows(table[1])) - const headers = headerIndexes(rows) + const headers = bedrockHeaderIndexes(rows) if (!headers) continue recognisedTables++ @@ -184,16 +215,34 @@ export function parseBedrockLifecycleHtml(html) { const offset = model.index - headers.model const legacyCell = row.cells[headers.legacy + offset] const eolCell = row.cells[headers.eol + offset] - if (!legacyCell || !eolCell) { + const extendedAccessCell = row.cells[headers.extendedAccess + offset] + const statusCell = headers.status >= 0 ? row.cells[headers.status + offset] : undefined + if (!legacyCell || !eolCell || !extendedAccessCell || (headers.status >= 0 && !statusCell)) { throw new Error(`aws-bedrock lifecycle entry ${bedrockId} is missing lifecycle columns`) } const legacy = lifecycleDate(legacyCell, 'legacy', bedrockId) const eol = lifecycleDate(eolCell, 'EOL', bedrockId) + const extendedAccess = lifecycleDate(extendedAccessCell, 'public extended access', bedrockId) if (legacy && eol && eol < legacy) { throw new Error(`aws-bedrock lifecycle entry ${bedrockId} has EOL before legacy date`) } - const record = { bedrockId } + if (extendedAccess && legacy && extendedAccess < legacy) { + throw new Error(`aws-bedrock lifecycle entry ${bedrockId} has public extended access before legacy date`) + } + if (extendedAccess && eol && extendedAccess > eol) { + throw new Error(`aws-bedrock lifecycle entry ${bedrockId} has public extended access after EOL date`) + } + // The table schedules Extended Access, so preserve that signal without comparing against the runner clock. + const status = statusCell + ? bedrockLifecycleStatus(statusCell, bedrockId) + : extendedAccess + ? 'extended-access' + : 'legacy' + if (status === 'extended-access' && !extendedAccess) { + throw new Error(`aws-bedrock lifecycle entry ${bedrockId} reports extended access without a start date`) + } + const record = { bedrockId, status } if (legacy !== undefined) record.legacy = legacy if (eol !== undefined) record.eol = eol records.push(record) @@ -209,8 +258,10 @@ export function parseBedrockLifecycleHtml(html) { const previous = unique.get(record.bedrockId) if (!previous) { unique.set(record.bedrockId, record) - } else if (!sameRecord(previous, record)) { - throw new Error(`aws-bedrock lifecycle page has conflicting rows for ${record.bedrockId}`) + } else { + const merged = mergeRegionalBedrockRecord(previous, record) + if (!merged) throw new Error(`aws-bedrock lifecycle page has conflicting rows for ${record.bedrockId}`) + unique.set(record.bedrockId, merged) } } return [...unique.values()] @@ -497,7 +548,7 @@ export function mergeDistributions(feeds, { priorRecord.date_precision !== record.date_precision || priorRecord.status !== record.status )) { - throw new Error(`${via} records map to ${target.model.id} with conflicting dates`) + throw new Error(`${via} records map to ${target.model.id} with conflicting lifecycle data`) } matchedRecords.set(targetKey, record) diff --git a/refresh/refresh.mjs b/refresh/refresh.mjs index 7bb2430..67f1024 100644 --- a/refresh/refresh.mjs +++ b/refresh/refresh.mjs @@ -214,6 +214,9 @@ export async function run(options) { markdown: renderSemanticDiff(item.committed, item.feed, diffOptions), } }) + for (const [index, diff] of diffs.entries()) { + if (!diff.result.changed) generated[index].feed.generated = generated[index].committed.generated + } const changed = diffs.some(item => item.result.changed) const markdown = combinedDiff(diffs) process.stdout.write(markdown) diff --git a/refresh/test/run.mjs b/refresh/test/run.mjs index 1c635e7..4759dfd 100644 --- a/refresh/test/run.mjs +++ b/refresh/test/run.mjs @@ -35,16 +35,20 @@ import { compareFeeds, renderSemanticDiff } from '../diff.mjs' import { validateGeneratedFeeds } from '../refresh.mjs' import { classifyFeedReleasePaths } from '../../scripts/feed-release-guard.mjs' import { validateReleaseVersion } from '../../scripts/validate-release-version.mjs' +import { resolveReleaseState, targetReleaseVersion } from '../../scripts/release-state.mjs' +import { createReleaseReceipt, validateReleaseReceipt } from '../../scripts/release-receipt.mjs' +import { assertSha512Integrity, sha512Integrity } from '../../scripts/package-integrity.mjs' +import { stableGithubReleaseExists } from '../../scripts/github-release-state.mjs' const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '../..') const refresh = path.join(root, 'refresh', 'refresh.mjs') const fixtures = path.join(root, 'refresh', 'test', 'fixture') -function run(args) { +function run(args, { env = process.env } = {}) { try { return { code: 0, - out: execFileSync(process.execPath, [refresh, ...args], { encoding: 'utf8' }), + out: execFileSync(process.execPath, [refresh, ...args], { encoding: 'utf8', env }), } } catch (error) { return { @@ -136,6 +140,7 @@ const endpointLookalikeHtml = fs.readFileSync(path.join(fixtures, 'anthropic-end const openaiEntries = parseOpenAIDeprecations(openaiHtml) const anthropicEntries = parseAnthropicDeprecations(anthropicHtml) const bedrockEntries = parseBedrockLifecycleHtml(bedrockHtml) +const bedrockById = new Map(bedrockEntries.map(entry => [entry.bedrockId, entry])) const googleEntries = parseGoogleDeprecations(googleHtml) const vertexEntries = parseVertexModelVersionsHtml(vertexHtml) const openaiIds = parseOpenAIModels(fs.readFileSync(path.join(fixtures, 'openai-models.json'), 'utf8')) @@ -220,14 +225,64 @@ assert(normalizeVertexId('publishers/google/models/gemini-2.5-pro') === 'gemini- assert(bedrockEntries.length === 17, 'Bedrock lifecycle fixture parses and deduplicates logical model rows') assert(bedrockEntries.find(entry => entry.bedrockId === 'anthropic.claude-3-haiku-20240307-v1:0')?.eol === '2026-09-10', 'Bedrock parser handles rowspan model rows') assert(bedrockEntries.find(entry => entry.bedrockId === 'amazon.nova-canvas-v1:0')?.legacy === '2026-03-30', 'Bedrock parser reads human legacy dates') -assert(bedrockEntries.every(entry => entry.legacy && entry.eol), 'Bedrock lifecycle records contain only parsed lifecycle dates') +assert(bedrockEntries.every(entry => entry.legacy && entry.eol), 'Bedrock lifecycle records preserve parsed Legacy and EOL dates') +assert(bedrockById.get('amazon.nova-canvas-v1:0')?.status === 'legacy', 'Bedrock parser marks rows without a public Extended Access date as Legacy') +assert(bedrockById.get('anthropic.claude-3-haiku-20240307-v1:0')?.status === 'extended-access', 'Bedrock parser ingests the public Extended Access lifecycle phase') +assert(bedrockById.get('anthropic.claude-3-sonnet-20240229-v1:0')?.status === 'extended-access', 'Bedrock parser conservatively combines regional Legacy and Extended Access rows') const shiftedBedrockRow = parseBedrockLifecycleHtml(`
ProviderModelModel IDRegionsLegacy dateEOL datePublic extended access date
Command Rcohere.command-r-v1:0us-east-1, us-west-2February 19, 2026August 19, 2026May 19, 2026
`) -assert(shiftedBedrockRow[0]?.bedrockId === 'cohere.command-r-v1:0' && shiftedBedrockRow[0]?.eol === '2026-08-19', 'Bedrock parser realigns rows whose provider cell is omitted') +assert(shiftedBedrockRow[0]?.bedrockId === 'cohere.command-r-v1:0' && shiftedBedrockRow[0]?.eol === '2026-08-19' && shiftedBedrockRow[0]?.status === 'extended-access', 'Bedrock parser realigns rows whose provider cell is omitted') +const explicitBedrockStatuses = parseBedrockLifecycleHtml(` + + + + + +
Model IDLegacy dateEOL datePublic extended access start dateLifecycle status
amazon.legacy-model-v1:0March 1, 2026September 1, 2026—Legacy
amazon.extended-model-v1:0March 1, 2026September 1, 2026June 1, 2026Public Extended Access
amazon.eol-model-v1:0March 1, 2026September 1, 2026June 1, 2026End-of-Life (EOL)
+`) +assert(JSON.stringify(explicitBedrockStatuses.map(entry => entry.status)) === JSON.stringify(['legacy', 'extended-access', 'retired']), 'Bedrock parser normalizes explicit official lifecycle statuses') +for (const [label, value] of [['unknown', 'Deprecated'], ['empty', '']]) { + let reason = '' + try { + parseBedrockLifecycleHtml(` + + + +
Model IDLegacy dateEOL datePublic extended access start dateStatus
amazon.bad-status-v1:0March 1, 2026September 1, 2026—${value}
+ `) + } catch (error) { + reason = error.message + } + assert(reason.includes('unsupported lifecycle status'), `Bedrock parser fails closed on ${label} lifecycle status values`) +} +let malformedExtendedAccessReason = '' +try { + parseBedrockLifecycleHtml(` + + + +
Model IDLegacy dateEOL datePublic extended access start date
amazon.bad-extended-date-v1:0March 1, 2026September 1, 2026Eventually
+ `) +} catch (error) { + malformedExtendedAccessReason = error.message +} +assert(malformedExtendedAccessReason.includes('unrecognised public extended access date'), 'Bedrock parser fails closed on malformed public Extended Access dates') +let missingExtendedAccessColumnReason = '' +try { + parseBedrockLifecycleHtml(` + + + +
Model IDLegacy dateEOL date
amazon.schema-drift-v1:0March 1, 2026September 1, 2026
+ `) +} catch (error) { + missingExtendedAccessColumnReason = error.message +} +assert(missingExtendedAccessColumnReason.includes('missing the Public extended access start date column'), 'Bedrock parser fails closed when the official table schema drops the Extended Access column') assert(openaiEntries.length === 43, 'OpenAI real-structure fixture parses all selected model entries') assert(openaiEntries.filter(entry => entry.announced === '2026-04-22').length >= 20, 'OpenAI announcement date is inherited across a section') assert(openaiById.get('o3-deep-research-2025-06-26')?.announced === '2026-04-22', 'OpenAI July wave inherits its April announcement date') @@ -421,7 +476,7 @@ const distributorMerge = mergeBedrockDistributions([distributorCommitted], { sourceUrl: BEDROCK_LIFECYCLE_URL, records: [ { bedrockId: 'anthropic.publisher-alias-v1:0', legacy: '2026-08-01', eol: '2027-02-01' }, - { bedrockId: 'anthropic.existing-model-20250101-v2:0', legacy: '2026-02-01', eol: '2027-01-01' }, + { bedrockId: 'anthropic.existing-model-20250101-v2:0', legacy: '2026-02-01', eol: '2027-01-01', status: 'extended-access' }, { bedrockId: 'meta.llama3-1-405b-instruct-v1:0', legacy: '2026-08-01', eol: '2027-02-01' }, ], }) @@ -430,11 +485,12 @@ const publisherModel = distributorFeed.models.find(model => model.id === 'publis assert(publisherModel.distributions?.[0]?.via === 'aws-bedrock' && publisherModel.distributions[0].shutdown === '2027-02-01', 'Bedrock merge upserts a distribution through a publisher alias') const existingModel = distributorFeed.models.find(model => model.id === 'existing-model-20250101') assert(existingModel.distributions[1].shutdown === '2027-01-01', 'Bedrock merge updates a changed EOL date in place') +assert(existingModel.distributions[1].status === 'extended-access', 'Bedrock merge carries a normalized Extended Access status') assert(JSON.stringify(existingModel.distributions[0]) === existingBeforeForeign, 'Bedrock merge preserves foreign-via distributions') const existingAfterFields = { ...existingModel } delete existingAfterFields.distributions assert(JSON.stringify(existingAfterFields) === JSON.stringify(existingBeforeFields), 'Bedrock merge preserves entry-level lifecycle and replacement fields') -assert(JSON.stringify(Object.keys(existingModel.distributions[1])) === JSON.stringify(['via', 'announced', 'shutdown', 'source']), 'Bedrock distribution fields retain canonical order') +assert(JSON.stringify(Object.keys(existingModel.distributions[1])) === JSON.stringify(['via', 'announced', 'shutdown', 'status', 'source']), 'Bedrock distribution fields retain canonical order') assert(distributorMerge.unconfirmedDistributions.some(item => item.id === 'stale-model'), 'Bedrock merge reports an unconfirmed existing distribution') assert(distributorMerge.noPublisherFeed.some(item => item.bedrockId === 'meta.llama3-1-405b-instruct-v1:0'), 'Bedrock merge reports unmatched models without inventing entries') assert(!distributorFeed.models.some(model => model.id === 'llama3-1-405b-instruct'), 'Bedrock merge does not create an unmatched publisher entry') @@ -455,6 +511,16 @@ const unknownNamespaceMerge = mergeBedrockDistributions([{ publisher: 'openai', records: [{ bedrockId: 'future-provider.unknown-model:0', legacy: '2026-08-01', eol: '2027-02-01' }], }) assert(unknownNamespaceMerge.noPublisherFeed.some(item => item.bedrockId === 'future-provider.unknown-model:0') && !unknownNamespaceMerge.feeds[0].models[0].distributions, 'unknown Bedrock namespaces remain skipped with a note') +let invalidBedrockRecordStatus = '' +try { + mergeBedrockDistributions([{ publisher: 'anthropic', models: [{ id: 'invalid-status-model' }] }], { + sourceUrl: BEDROCK_LIFECYCLE_URL, + records: [{ bedrockId: 'anthropic.invalid-status-model-v1:0', legacy: '2026-08-01', eol: '2027-02-01', status: 'deprecated' }], + }) +} catch (error) { + invalidBedrockRecordStatus = error.message +} +assert(invalidBedrockRecordStatus.includes('invalid status'), 'Bedrock merge refuses unknown parser status values before feed generation') const vertexMerge = mergeVertexDistributions([googleFeed, anthropicFeed], { records: vertexEntries, @@ -525,6 +591,41 @@ const precisionFeed = { ...oldFeed, models: [{ id: 'precision', shutdown: '2026- const precisionDiff = renderSemanticDiff({ ...oldFeed, models: [{ id: 'precision', shutdown: '2026-10-01' }] }, precisionFeed) assert(precisionDiff.includes('2026-10-01') && precisionDiff.includes('(earliest)'), 'semantic diff renders earliest date precision') +const statusOld = { ...oldFeed, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', announced: '2026-03-01', shutdown: '2026-09-01', status: 'legacy' }] }] } +const statusNew = { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', announced: '2026-03-01', shutdown: '2026-09-01', status: 'extended-access' }] }] } +const statusDiff = compareFeeds(statusOld, statusNew) +const statusMarkdown = renderSemanticDiff(statusOld, statusNew) +assert(statusDiff.changed && statusDiff.distributionChanges.length === 1, 'semantic diff treats a distribution status-only transition as material') +assert(statusMarkdown.includes('status changed `legacy` -> `extended-access`'), 'semantic diff renders status-only transitions for refresh PR and Atom inputs') +const statusAdded = compareFeeds( + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01' }] }] }, + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01', status: 'legacy' }] }] }, +) +const statusAddedMarkdown = renderSemanticDiff( + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01' }] }] }, + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01', status: 'legacy' }] }] }, +) +const statusRemovedMarkdown = renderSemanticDiff( + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01', status: 'legacy' }] }] }, + { ...statusOld, models: [{ id: 'bedrock-status', distributions: [{ via: 'aws-bedrock', shutdown: '2026-09-01' }] }] }, +) +assert(statusAdded.changed && statusAdded.distributionChanges.length === 1 && statusAddedMarkdown.includes('status changed `not set` -> `legacy`'), 'semantic diff treats adding a distribution status as material and rendered') +assert(statusRemovedMarkdown.includes('status changed `legacy` -> `not set`'), 'semantic diff treats removing a distribution status as material and rendered') +const distributionAdded = compareFeeds( + { ...oldFeed, models: [{ id: 'distribution-membership' }] }, + { ...oldFeed, models: [{ id: 'distribution-membership', distributions: [{ via: 'aws-bedrock', status: 'legacy' }] }] }, +) +const distributionAddedMarkdown = renderSemanticDiff( + { ...oldFeed, models: [{ id: 'distribution-membership' }] }, + { ...oldFeed, models: [{ id: 'distribution-membership', distributions: [{ via: 'aws-bedrock', status: 'legacy' }] }] }, +) +const distributionRemovedMarkdown = renderSemanticDiff( + { ...oldFeed, models: [{ id: 'distribution-membership', distributions: [{ via: 'aws-bedrock', status: 'legacy' }] }] }, + { ...oldFeed, models: [{ id: 'distribution-membership' }] }, +) +assert(distributionAdded.changed && distributionAdded.distributionChanges[0]?.kind === 'added' && distributionAddedMarkdown.includes('distribution added') && distributionAddedMarkdown.includes('status: `legacy`'), 'semantic diff keeps distribution additions material and renders status') +assert(distributionRemovedMarkdown.includes('distribution removed'), 'semantic diff keeps distribution removals material and rendered') + const aliasDiffOld = { ...oldFeed, models: [{ id: 'alias-model', aliases: ['old-alias'], shutdown: '2026-10-01' }] } const aliasDiffNew = { ...aliasDiffOld, models: [{ id: 'alias-model', aliases: ['new-alias'], shutdown: '2026-10-01' }] } const aliasDiff = compareFeeds(aliasDiffOld, aliasDiffNew) @@ -761,6 +862,25 @@ assert(vertexCheck.out.includes('no publisher feed'), 'Vertex --check reports un const bothDistributorsCheck = run(['--distributor', 'aws-bedrock,vertex-ai', '--check', '--fixtures', fixtures]) assert(bothDistributorsCheck.code === 3 && bothDistributorsCheck.out.includes('vertex-ai'), 'refresh accepts comma-separated distributors') +const mixedDistributorOut = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-refresh-mixed-distributors-')) +const mixedGenerated = '2099-01-02T03:04:05Z' +const mixedDistributorWrite = run([ + '--distributor', 'aws-bedrock,vertex-ai', + '--out', mixedDistributorOut, + '--fixtures', fixtures, +], { env: { ...process.env, MODEL_EOL_GENERATED: mixedGenerated } }) +assert(mixedDistributorWrite.code === 0, 'mixed distributor refresh writes successfully') +const mixedOutputs = Object.fromEntries(['amazon', 'anthropic', 'google', 'openai'].map(publisher => [ + publisher, + JSON.parse(fs.readFileSync(path.join(mixedDistributorOut, `${publisher}.json`), 'utf8')), +])) +const mixedCommitted = Object.fromEntries(['amazon', 'anthropic', 'google', 'openai'].map(publisher => [ + publisher, + JSON.parse(fs.readFileSync(path.join(root, 'feeds', `${publisher}.json`), 'utf8')), +])) +assert(mixedOutputs.anthropic.generated === mixedGenerated && mixedOutputs.anthropic.generated !== mixedCommitted.anthropic.generated, 'mixed distributor write advances generated for the publisher with material distribution changes') +assert(['amazon', 'google', 'openai'].every(publisher => mixedOutputs[publisher].generated === mixedCommitted[publisher].generated), 'mixed distributor write preserves generated for every semantically unchanged publisher') + const refreshWorkflow = fs.readFileSync(path.join(root, '.github/workflows/feed-refresh.yml'), 'utf8') assert(refreshWorkflow.includes('[ "$providers" -ne 0 ] && [ "$providers" -ne 3 ]') && refreshWorkflow.includes('exit code $providers'), 'workflow fails explicitly on unexpected provider refresh exit codes') assert(refreshWorkflow.includes('[ "$distributors" -ne 0 ] && [ "$distributors" -ne 3 ]') && refreshWorkflow.includes('exit code $distributors'), 'workflow fails explicitly on unexpected distributor refresh exit codes') @@ -770,6 +890,8 @@ assert(refreshWorkflow.includes('issues: write') && refreshWorkflow.includes('if assert(refreshWorkflow.includes('gh issue comment "$issue" --body "$body"') && refreshWorkflow.includes('gh issue create --title "$title"'), 'workflow updates one failure issue instead of silently repeating failures') assert(refreshWorkflow.includes('Resolve prior feed refresh failure') && refreshWorkflow.includes('gh issue close "$issue" --reason completed'), 'a successful refresh resolves the prior failure issue') assert(refreshWorkflow.includes('$GITHUB_STEP_SUMMARY') && refreshWorkflow.includes('no material feed changes were found') && refreshWorkflow.includes('feed-generated date is intentionally unchanged'), 'a successful no-change refresh records an honest result without changing feed freshness') +assert(refreshWorkflow.includes('node scripts/feed-refresh-receipt.mjs "${args[@]}"') && refreshWorkflow.includes('name: feed-refresh-receipt'), 'every successful refresh uploads a dependency-free receipt artifact') +assert(refreshWorkflow.includes('--state pending --pending-pr-url "$PR_URL"') && refreshWorkflow.includes('--state clean'), 'refresh receipts distinguish pending material changes from clean committed feeds') const readme = fs.readFileSync(path.join(root, 'README.md'), 'utf8') assert(readme.includes('actions/workflows/feed-refresh.yml/badge.svg'), 'README exposes feed-refresh workflow health') const releaseWorkflow = fs.readFileSync(path.join(root, '.github/workflows/npm-release.yml'), 'utf8') @@ -791,10 +913,161 @@ try { publishedReleaseRejected = true } assert(publishedReleaseRejected, 'release validation refuses a version already published to npm') -assert(releaseWorkflow.includes('npm view model-eol versions --json') && releaseWorkflow.includes('tag v$RELEASE_VERSION already exists') && releaseWorkflow.includes('GitHub release v$RELEASE_VERSION already exists'), 'manual releases refuse existing npm versions, tags, and GitHub releases') +assert(targetReleaseVersion({ eventName: 'push', sourceVersion: '0.4.1' }) === '0.4.2', 'automatic feed release state resolves the exact next patch') +assert(targetReleaseVersion({ eventName: 'workflow_dispatch', sourceVersion: '0.4.1', requestedVersion: '0.5.0' }) === '0.5.0', 'manual release state preserves the exact requested version') +const releaseSource = 'a'.repeat(40) +const releaseCommit = 'b'.repeat(40) +const releaseIntegrity = sha512Integrity(Buffer.from('model-eol@0.5.0')) +assert(assertSha512Integrity(releaseIntegrity) === releaseIntegrity, 'release package integrity accepts one canonical SHA-512 SRI digest') +const stableReleaseApiEntry = { tag_name: 'v0.5.0', draft: false, prerelease: false } +assert(stableGithubReleaseExists({ pages: [[stableReleaseApiEntry]], tag: 'v0.5.0' }), 'release recovery recognizes an exact published stable GitHub release from paginated API state') +assert(!stableGithubReleaseExists({ pages: [[{ tag_name: 'v0.4.1', draft: false, prerelease: false }]], tag: 'v0.5.0' }), 'release recovery recognizes an absent exact GitHub release') +for (const candidate of [ + { ...stableReleaseApiEntry, draft: true }, + { ...stableReleaseApiEntry, prerelease: true }, +]) { + let unstableReleaseRejected = false + try { + stableGithubReleaseExists({ pages: [[candidate]], tag: 'v0.5.0' }) + } catch { + unstableReleaseRejected = true + } + assert(unstableReleaseRejected, `release recovery rejects a matching ${candidate.draft ? 'draft' : 'prerelease'} GitHub release`) +} +let malformedReleaseApiRejected = false +try { + stableGithubReleaseExists({ pages: [{ tag_name: 'v0.5.0', draft: false, prerelease: false }], tag: 'v0.5.0' }) +} catch { + malformedReleaseApiRejected = true +} +assert(malformedReleaseApiRejected, 'release recovery fails closed on malformed paginated GitHub API state') +const newReleaseState = resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseSource, + publishedVersions: ['0.4.1'], + githubReleaseExists: false, +}) +assert(newReleaseState.mode === 'create' && newReleaseState.publish && newReleaseState.createGithubRelease, 'release state creates a wholly missing exact release') +const resumableTag = { + commit: releaseCommit, + parent: releaseSource, + version: '0.5.0', + movingCommit: releaseCommit, + mainContainsRelease: true, + changedPaths: ['package.json'], +} +const tagOnlyState = resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseCommit, + publishedVersions: ['0.4.1'], + githubReleaseExists: false, + tag: resumableTag, +}) +assert(tagOnlyState.mode === 'resume' && tagOnlyState.publish && tagOnlyState.createGithubRelease, 'same-event workflow rerun resumes after tags were pushed but npm is missing') +const registryOnlyState = resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseCommit, + publishedVersions: ['0.4.1', '0.5.0'], + githubReleaseExists: false, + tag: resumableTag, +}) +assert(registryOnlyState.mode === 'resume' && !registryOnlyState.publish && registryOnlyState.createGithubRelease, 'same-event workflow rerun resumes after npm publish when the GitHub release is missing') +const completeReleaseState = resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseCommit, + publishedVersions: ['0.4.1', '0.5.0'], + githubReleaseExists: true, + tag: resumableTag, +}) +assert(completeReleaseState.mode === 'resume' && !completeReleaseState.publish && !completeReleaseState.createGithubRelease, 'release state recognizes an already complete release without repeating publication') +let orphanRegistryRejected = false +try { + resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseSource, + publishedVersions: ['0.5.0'], + githubReleaseExists: false, + }) +} catch { + orphanRegistryRejected = true +} +assert(orphanRegistryRejected, 'release state refuses an npm version without an immutable release tag') +let movingTagMismatchRejected = false +try { + resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseCommit, + publishedVersions: ['0.4.1'], + githubReleaseExists: false, + tag: { ...resumableTag, movingCommit: 'c'.repeat(40) }, + }) +} catch { + movingTagMismatchRejected = true +} +assert(movingTagMismatchRejected, 'release state refuses recovery when moving v0 and the immutable tag diverge') +let wrongReleaseParentRejected = false +try { + resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: releaseCommit, + publishedVersions: ['0.4.1'], + githubReleaseExists: false, + tag: { ...resumableTag, parent: 'c'.repeat(40) }, + }) +} catch { + wrongReleaseParentRejected = true +} +assert(wrongReleaseParentRejected, 'release state refuses a version tag whose release commit has the wrong source parent') +let mainDriftRejected = false +try { + resolveReleaseState({ + eventName: 'workflow_dispatch', + sourceVersion: '0.4.1', + requestedVersion: '0.5.0', + sourceCommit: releaseSource, + mainCommit: 'c'.repeat(40), + publishedVersions: ['0.4.1'], + githubReleaseExists: false, + }) +} catch { + mainDriftRejected = true +} +assert(mainDriftRejected, 'new release state refuses to push after origin/main drifts from the workflow source') +const releasedReceipt = createReleaseReceipt({ sourceCommit: releaseSource, version: '0.5.0', releaseCommit, registryIntegrity: releaseIntegrity }) +const validatedReleaseReceipt = validateReleaseReceipt(releasedReceipt, { expectedSourceCommit: releaseSource }) +assert(validatedReleaseReceipt.release_commit === releaseCommit && validatedReleaseReceipt.registry_integrity === releaseIntegrity, 'release receipt binds the workflow source, exact version, release commit, and registry tarball integrity') +let invalidReleaseIntegrityRejected = false +try { + validateReleaseReceipt({ ...releasedReceipt, registry_integrity: 'sha512-not-base64' }) +} catch { + invalidReleaseIntegrityRejected = true +} +assert(invalidReleaseIntegrityRejected, 'release receipt rejects malformed registry integrity') +assert(!validateReleaseReceipt(createReleaseReceipt({ sourceCommit: releaseSource })).released, 'release receipt represents intentional no-op runs without guessing a version') assert(releaseWorkflow.includes('node scripts/validate-release-version.mjs "$current" "$RELEASE_VERSION" "$published"'), 'manual releases run the tested release-version validator') assert(releaseWorkflow.includes('npm version "$RELEASE_VERSION"'), 'manual releases apply the exact requested version') -assert(releaseWorkflow.includes('npm version patch -m "model-eol v%s - automated feed-data release"'), 'automated feed releases remain patch-only') +assert(releaseWorkflow.includes("npm version patch -m 'model-eol v%s - automated feed-data release'"), 'automated feed releases remain patch-only') const feedOnlyRelease = classifyFeedReleasePaths(['feeds/openai.json', 'feeds/google.json', 'README.md']) assert(feedOnlyRelease.changed && feedOnlyRelease.blockedPaths.length === 0, 'feed and generated README metadata changes may publish an automatic patch') const mixedRelease = classifyFeedReleasePaths(['feeds/openai.json', 'README.md', 'lib/scanner.mjs', '.github/workflows/ci.yml']) @@ -802,6 +1075,31 @@ assert(!mixedRelease.changed && JSON.stringify(mixedRelease.blockedPaths) === JS assert(!classifyFeedReleasePaths(['README.md']).changed, 'README-only changes do not publish an automatic feed patch') assert(releaseWorkflow.includes('node scripts/feed-release-guard.mjs "$last"') && !releaseWorkflow.includes('git diff --quiet "$last"..HEAD -- feeds/'), 'npm release workflow uses the tested mixed-change guard') assert(releaseWorkflow.includes('git push --atomic origin main "$version" +refs/tags/v0:refs/tags/v0') && !releaseWorkflow.includes('git push -f origin v0'), 'release commit, immutable version tag, and moving v0 tag push atomically') +assert(releaseWorkflow.includes("steps.state.outputs.publish == 'true'") && releaseWorkflow.includes("steps.state.outputs.create_github_release == 'true'"), 'release workflow independently resumes missing npm publication and GitHub release phases') +assert(releaseWorkflow.includes("pre-release GITHUB_SHA") && releaseWorkflow.includes('--source-sha "$GITHUB_SHA"'), 'release recovery is explicitly bound to rerunning the original event and its pre-release source commit') +assert(releaseWorkflow.includes('name: npm-release-result') && releaseWorkflow.includes('node scripts/release-receipt.mjs "${args[@]}"') && releaseWorkflow.includes('--registry-integrity "$RELEASE_INTEGRITY"'), 'release workflow exports its exact version, commit, and registry integrity through an artifact') +assert(releaseWorkflow.includes('id-token: write') && releaseWorkflow.includes('npm publish "$RELEASE_TARBALL" --ignore-scripts'), 'release workflow preserves npm trusted publishing through OIDC') +assert(releaseWorkflow.includes('npm install -g npm@11.6.4') && releaseWorkflow.includes(`test "$(npm --version)" = '11.6.4'`) && !releaseWorkflow.includes('npm@latest'), 'OIDC release execution pins and verifies its npm CLI instead of running a mutable latest version') +assert(releaseWorkflow.includes('npm pack --json --ignore-scripts') && releaseWorkflow.includes('verifyPackageIntegrity') && releaseWorkflow.includes('--expected-integrity "$RELEASE_INTEGRITY"'), 'release publication hashes exact tag bytes, publishes that tarball, and verifies registry integrity') +assert(releaseWorkflow.includes('gh api --paginate --slurp') && releaseWorkflow.includes('node scripts/github-release-state.mjs') && releaseWorkflow.includes('isDraft !== false') && !releaseWorkflow.includes('if gh release view'), 'release existence lookup fails closed and accepts only a published stable GitHub release') +const publicWorkflow = fs.readFileSync(path.join(root, '.github/workflows/public-contract.yml'), 'utf8') +assert(publicWorkflow.includes('no successful feed-refresh receipt matches every feed currently on main') && publicWorkflow.includes('successful receipt-era refresh run $run_id has no downloadable receipt') && publicWorkflow.includes('refusing stale receipt fallback'), 'public contract advances only from the newest valid receipt and fails closed on missing or mismatching receipt-era artifacts') +assert(publicWorkflow.includes('.status, .conclusion') && !publicWorkflow.includes('--status success') && publicWorkflow.includes('refusing fallback until a newer refresh succeeds'), 'newer failed, queued, or in-progress receipt-era refreshes block fallback to older clean receipts') +assert(publicWorkflow.includes('.isCrossRepository == false') && publicWorkflow.includes('startswith("feed-refresh/")') && publicWorkflow.includes('refusing to advance last_checked') && publicWorkflow.includes('pull-requests: read'), 'an unresolved same-repository material-change PR prevents later clean receipts without trusting fork branch names') +assert(publicWorkflow.includes('Confirm this refresh event has not been superseded') && publicWorkflow.includes('it cannot roll the public contract back'), 'a delayed workflow_run event cannot deploy an older matching receipt after a newer refresh') +assert(['lib/cli.mjs', 'lib/validate-feed.mjs', 'refresh/diff.mjs'].every(file => publicWorkflow.includes(`- '${file}'`)), 'public contract redeploys when any transitive build dependency changes') +assert(publicWorkflow.includes('node scripts/verify-public-site.mjs') && publicWorkflow.includes('name: public-contract-expectation'), 'Pages verification compares the live deployment with the exact built artifact') +const publishedUatWorkflow = fs.readFileSync(path.join(root, '.github/workflows/published-consumer-uat.yml'), 'utf8') +assert(publishedUatWorkflow.includes('name: npm-release-result') && publishedUatWorkflow.includes('run-id: ${{ github.event.workflow_run.id }}') && !publishedUatWorkflow.includes('dist-tags.latest'), 'workflow-run UAT consumes the exact release result instead of npm latest') +assert(publishedUatWorkflow.includes('moving_commit') && publishedUatWorkflow.includes('immutable_commit') && publishedUatWorkflow.includes('Moving v0 Action validate round-trip UAT') && publishedUatWorkflow.includes('Reverify remote v0 after the Action ran'), 'published UAT brackets the moving v0 Action with immutable-ref checks and validates its output round-trip') +assert(publishedUatWorkflow.includes('Immutable release Action inventory UAT') && publishedUatWorkflow.includes('Immutable release Action validate round-trip UAT') && publishedUatWorkflow.match(/uses: \.\//g)?.length === 2, 'published UAT validates the exact immutable release Action locally before treating v0 as a moving-line monitor') +assert(publishedUatWorkflow.includes("git ls-remote \"$remote\" 'refs/tags/v0^{}'") && publishedUatWorkflow.includes('--expected-integrity "$INTEGRITY"'), 'published UAT peels the moving Action tag and binds installed package bytes to the registry digest') +assert(publishedUatWorkflow.includes("needs.resolve.outputs.moving_current == 'true'") && publishedUatWorkflow.includes('exact v$VERSION UAT remains authoritative'), 'superseded moving aliases are monitoring results and never invalidate immutable exact-version UAT') +assert(publishedUatWorkflow.includes('v0 or immutable v$VERSION moved while the moving Action UAT was running') && publishedUatWorkflow.includes('exit 1'), 'moving Action monitoring fails if either bound ref changes during its two-step round-trip') +assert(publishedUatWorkflow.includes('const r=Array.isArray(v)?v.at(-1):v') && publishedUatWorkflow.includes('if(typeof r!=="string")process.exit(1)'), 'moving npm-line recheck normalizes npm view arrays to the resolved latest version') +assert(releaseWorkflow.includes('group: model-eol-release-and-moving-uat') && releaseWorkflow.includes('queue: max') && publishedUatWorkflow.includes('group: model-eol-release-and-moving-uat') && publishedUatWorkflow.includes('queue: max'), 'release and moving-alias UAT serialize through one non-cancelling queued concurrency group') +assert(publishedUatWorkflow.includes("ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || 'main' }}"), 'workflow-run UAT resolves its receipt with the triggering release commit\'s own verifier') +assert(publishedUatWorkflow.includes('ref: ${{ needs.resolve.outputs.release_commit }}'), 'package UAT runs the exact release commit\'s own consumer harness') const freshnessScript = fs.readFileSync(path.join(root, 'scripts/update-readme-freshness.mjs'), 'utf8') assert(freshnessScript.includes('AWS Bedrock and Google Vertex AI lifecycle pages'), 'README freshness metadata names every automated distributor source') @@ -848,6 +1146,7 @@ assert(corruptRun.code === 1, 'corrupted fixture exits nonzero') assert(!fs.existsSync(corruptOut), 'corrupted fixture produces no output directory') fs.rmSync(outputDir, { recursive: true, force: true }) +fs.rmSync(mixedDistributorOut, { recursive: true, force: true }) fs.rmSync(corruptBedrockDir, { recursive: true, force: true }) fs.rmSync(corruptOpenAIDir, { recursive: true, force: true }) fs.rmSync(corruptDir, { recursive: true, force: true }) diff --git a/schema/model-eol.alert.schema.json b/schema/model-eol.alert.schema.json index 9454440..66f4b92 100644 --- a/schema/model-eol.alert.schema.json +++ b/schema/model-eol.alert.schema.json @@ -1,28 +1,64 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/alert.schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.alert.schema.json", "title": "model-eol alert output", - "description": "Machine-readable alert payload emitted by `model-eol alert --json`. Draft 0.1 - the $id is a placeholder until the repo is public.", + "description": "Machine-readable alert payload emitted by `model-eol alert --json`.", "type": "object", "required": ["schema", "generated", "threshold_days", "scope", "scan_notes", "errors", "warnings"], "properties": { "schema": { "const": "model-eol/alert@0.1" }, "generated": { "type": "string", "format": "date-time" }, - "threshold_days": { "type": "number" }, - "distributor": { "type": ["string", "null"] }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "distributor": { "type": ["string", "null"], "minLength": 1 }, "scope": { "enum": ["all", "direct"] }, - "scan_notes": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/scanNote" } }, - "errors": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/modelReference" } }, + "scan_notes": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/scanNote" } }, + "errors": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/modelReference" } }, "warnings": { "type": "array", "items": { "oneOf": [ - { "$ref": "model-eol.inventory.schema.json#/definitions/modelReference" }, - { "$ref": "model-eol.inventory.schema.json#/definitions/candidateReference" }, - { "$ref": "model-eol.inventory.schema.json#/definitions/integrationHint" } + { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/modelReference" }, + { "$ref": "#/definitions/candidateWarning" }, + { "$ref": "#/definitions/integrationWarning" } ] } } }, - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "candidateWarning": { + "type": "object", + "required": ["file", "line", "kind", "matched", "usage", "resolved_provider", "confidence", "evidence", "reason", "status"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "kind": { "const": "candidate-model-reference" }, + "matched": { "type": "string", "minLength": 1 }, + "usage": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/usage" }, + "resolved_provider": { "type": "string", "minLength": 1 }, + "confidence": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/confidence" }, + "evidence": { "type": "string", "minLength": 1 }, + "effective_scope": { "enum": ["all", "direct"] }, + "policy_provenance": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/policyProvenance" }, + "reason": { "type": "string", "minLength": 1 }, + "status": { "const": "unknown" } + }, + "additionalProperties": false + }, + "integrationWarning": { + "type": "object", + "required": ["file", "line", "kind", "usage", "provider", "matched", "evidence", "status"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "kind": { "const": "integration-hint" }, + "usage": { "enum": ["direct-api", "cloud-provider", "gateway"] }, + "provider": { "type": "string", "minLength": 1 }, + "matched": { "type": "string", "minLength": 1 }, + "evidence": { "type": "string", "minLength": 1 }, + "status": { "const": "unresolved" } + }, + "additionalProperties": false + } + } } diff --git a/schema/model-eol.bot-config.schema.json b/schema/model-eol.bot-config.schema.json index 1299d71..49cda2f 100644 --- a/schema/model-eol.bot-config.schema.json +++ b/schema/model-eol.bot-config.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/bot-config.schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.bot-config.schema.json", "title": "model-eol repository configuration", "description": "Optional strict .model-eol.json configuration shared by the zero-dependency CLI, Action, and GitHub bot adapter.", "type": "object", @@ -86,7 +86,7 @@ "type": ["string", "null"], "minLength": 1, "default": null, - "description": "Opt-in evaluation command. Direct bot --eval runs it per changed patch in a temporary clone; the split example workflow runs it once as a read-only plan preflight." + "description": "Opt-in repository-owned command run independently for each patchable migration only in the read-only evaluate phase. Write-capable publication consumes its commit-bound result manifest and never executes this command." }, "timeout_ms": { "type": "integer", @@ -102,9 +102,13 @@ }, "pass_env": { "type": "array", - "items": { "type": "string", "minLength": 1 }, + "items": { + "type": "string", + "minLength": 1, + "pattern": "^(?!(?:[Gg][Ii][Tt][Hh][Uu][Bb]_|[Gg][Hh]_|[Aa][Cc][Tt][Ii][Oo][Nn][Ss]_|[Ss][Ss][Hh]_|[Aa][Ww][Ss]_[Ss][Ee][Cc][Rr][Ee][Tt]|[\\s\\S]*(?:[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]|[Cc][Rr][Ee][Dd][Ee][Nn][Tt][Ii][Aa][Ll])))[\\s\\S]+$" + }, "default": [], - "description": "Environment variable names explicitly passed to the eval command." + "description": "Environment variable names explicitly forwarded to trusted eval code. GitHub, Actions, SSH, AWS secret, token, secret, password, and credential name patterns are refused. This controls normal subprocess forwarding and is not an OS sandbox or same-user secret-isolation boundary." } }, "default": { "command": null, "timeout_ms": 600000, "max_report_bytes": 65536, "pass_env": [] } diff --git a/schema/model-eol.check.schema.json b/schema/model-eol.check.schema.json new file mode 100644 index 0000000..bf73b39 --- /dev/null +++ b/schema/model-eol.check.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.check.schema.json", + "title": "model-eol check output", + "description": "Machine-readable CI gate output emitted by `model-eol check --json`.", + "type": "object", + "required": ["schema", "threshold_days", "distributor", "scope", "scan_notes", "findings"], + "properties": { + "schema": { "const": "model-eol/check@0.1" }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "distributor": { "type": ["string", "null"], "minLength": 1 }, + "scope": { "enum": ["all", "direct"] }, + "scan_notes": { + "type": "array", + "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/scanNote" } + }, + "findings": { "type": "array", "items": { "$ref": "#/definitions/finding" } } + }, + "additionalProperties": false, + "definitions": { + "finding": { + "type": "object", + "required": ["file", "line", "matched", "id", "publisher", "usage", "resolved_provider", "confidence", "status", "shutdown", "date_precision", "distribution_status", "via", "days", "safe_until", "replacement", "replacement_options", "replacement_note", "threshold_days", "effective_scope", "requested_via"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "matched": { "type": "string", "minLength": 1 }, + "id": { "type": "string", "minLength": 1 }, + "publisher": { "type": "string", "minLength": 1 }, + "usage": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/usage" }, + "resolved_provider": { "type": "string", "minLength": 1 }, + "confidence": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/confidence" }, + "status": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/status" }, + "shutdown": { "type": ["string", "null"], "format": "date" }, + "date_precision": { "enum": ["exact", "earliest", null] }, + "distribution_status": { "enum": ["active", "legacy", "extended-access", "retired", null] }, + "via": { "type": ["string", "null"], "minLength": 1 }, + "days": { "type": ["integer", "null"] }, + "safe_until": { "type": ["string", "null"], "format": "date" }, + "replacement": { "type": ["string", "null"], "minLength": 1 }, + "replacement_options": { + "type": ["array", "null"], + "minItems": 1, + "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" } + }, + "replacement_note": { "type": ["string", "null"], "minLength": 1 }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "effective_scope": { "enum": ["all", "direct"] }, + "requested_via": { "type": ["string", "null"], "minLength": 1 }, + "policy_provenance": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/policyProvenance" }, + "mapped_from": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + } +} diff --git a/schema/model-eol.inventory.schema.json b/schema/model-eol.inventory.schema.json index 1d3886f..8297936 100644 --- a/schema/model-eol.inventory.schema.json +++ b/schema/model-eol.inventory.schema.json @@ -1,17 +1,17 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/inventory.schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json", "title": "model-eol inventory output", - "description": "Machine-readable repository inventory emitted by `model-eol inventory --json`. Draft 0.1 - the $id is a placeholder until the repo is public.", + "description": "Machine-readable repository inventory emitted by `model-eol inventory --json`.", "type": "object", "required": ["schema", "generated", "threshold_days", "scope", "targets", "scanned_files", "scan_notes", "summary", "model_references", "candidate_model_references", "integration_hints"], "properties": { "schema": { "const": "model-eol/inventory@0.1" }, "generated": { "type": "string", "format": "date-time" }, - "threshold_days": { "type": "number" }, - "distributor": { "type": ["string", "null"] }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "distributor": { "type": ["string", "null"], "minLength": 1 }, "scope": { "enum": ["all", "direct"] }, - "targets": { "type": "array", "items": { "type": "string" } }, + "targets": { "type": "array", "items": { "type": "string", "minLength": 1 } }, "scanned_files": { "type": "integer", "minimum": 0 }, "scan_notes": { "type": "array", "items": { "$ref": "#/definitions/scanNote" } }, "summary": { @@ -39,96 +39,97 @@ "type": "object", "required": ["file", "line"], "properties": { - "file": { "type": "string" }, + "file": { "type": "string", "minLength": 1 }, "line": { "type": "integer", "minimum": 1 } - } + }, + "additionalProperties": false }, "modelReference": { - "allOf": [ - { "$ref": "#/definitions/location" }, - { - "type": "object", - "required": ["kind", "matched", "id", "publisher", "usage", "resolved_provider", "confidence", "evidence", "status", "shutdown", "via", "days", "safe_until", "replacement", "source"], + "type": "object", + "required": ["file", "line", "kind", "matched", "id", "publisher", "usage", "resolved_provider", "confidence", "evidence", "status", "shutdown", "via", "days", "safe_until", "replacement", "source"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "kind": { "const": "model-reference" }, + "matched": { "type": "string", "minLength": 1 }, + "id": { "type": "string", "minLength": 1 }, + "publisher": { "type": "string", "minLength": 1 }, + "usage": { "$ref": "#/definitions/usage" }, + "resolved_provider": { "type": "string", "minLength": 1 }, + "confidence": { "$ref": "#/definitions/confidence" }, + "evidence": { "type": "string", "minLength": 1 }, + "status": { "$ref": "#/definitions/status" }, + "shutdown": { "type": ["string", "null"], "format": "date" }, + "date_precision": { "enum": ["exact", "earliest", null] }, + "distribution_status": { "enum": ["active", "legacy", "extended-access", "retired", null] }, + "via": { "type": ["string", "null"], "minLength": 1 }, + "requested_via": { "type": ["string", "null"], "minLength": 1 }, + "days": { "type": ["integer", "null"] }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "effective_scope": { "enum": ["all", "direct"] }, + "policy_provenance": { "$ref": "#/definitions/policyProvenance" }, + "mapped_from": { "type": "string", "minLength": 1 }, + "safe_until": { "type": ["string", "null"], "format": "date" }, + "replacement": { "type": ["string", "null"], "minLength": 1 }, + "replacement_options": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" } + }, + "replacement_note": { "type": "string", "minLength": 1 }, + "source": { "type": ["string", "null"], "format": "uri" }, + "policy": { + "type": ["object", "null"], + "required": ["min_notice_days", "source"], "properties": { - "kind": { "const": "model-reference" }, - "matched": { "type": "string" }, - "id": { "type": "string" }, - "publisher": { "type": "string" }, - "usage": { "$ref": "#/definitions/usage" }, - "resolved_provider": { "type": "string" }, - "confidence": { "$ref": "#/definitions/confidence" }, - "evidence": { "type": "string" }, - "status": { "$ref": "#/definitions/status" }, - "shutdown": { "type": ["string", "null"] }, - "date_precision": { "enum": ["exact", "earliest", null] }, - "distribution_status": { "enum": ["active", "legacy", "extended-access", "retired", null] }, - "via": { "type": ["string", "null"] }, - "requested_via": { "type": ["string", "null"] }, - "days": { "type": ["integer", "null"] }, - "threshold_days": { "type": "integer", "minimum": 0 }, - "effective_scope": { "enum": ["all", "direct"] }, - "policy_provenance": { "$ref": "#/definitions/policyProvenance" }, - "mapped_from": { "type": "string", "minLength": 1 }, - "safe_until": { "type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, - "replacement": { "type": ["string", "null"] }, - "source": { "type": ["string", "null"] }, - "policy": { - "type": ["object", "null"], - "required": ["min_notice_days", "source"], - "properties": { - "min_notice_days": { "type": "integer", "minimum": 0 }, - "source": { "type": "string", "format": "uri" } - }, - "additionalProperties": false - }, - "feed_generated": { "type": ["string", "null"], "format": "date-time" } - } - } - ] + "min_notice_days": { "type": "integer", "minimum": 0 }, + "source": { "type": "string", "format": "uri" } + }, + "additionalProperties": false + }, + "feed_generated": { "type": ["string", "null"], "format": "date-time" } + }, + "additionalProperties": false }, "candidateReference": { - "allOf": [ - { "$ref": "#/definitions/location" }, - { - "type": "object", - "required": ["kind", "matched", "usage", "resolved_provider", "confidence", "evidence", "reason"], - "properties": { - "kind": { "const": "candidate-model-reference" }, - "matched": { "type": "string" }, - "usage": { "$ref": "#/definitions/usage" }, - "resolved_provider": { "type": "string" }, - "confidence": { "$ref": "#/definitions/confidence" }, - "evidence": { "type": "string" }, - "effective_scope": { "enum": ["all", "direct"] }, - "policy_provenance": { "$ref": "#/definitions/policyProvenance" }, - "reason": { "type": "string" } - } - } - ] + "type": "object", + "required": ["file", "line", "kind", "matched", "usage", "resolved_provider", "confidence", "evidence", "reason"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "kind": { "const": "candidate-model-reference" }, + "matched": { "type": "string", "minLength": 1 }, + "usage": { "$ref": "#/definitions/usage" }, + "resolved_provider": { "type": "string", "minLength": 1 }, + "confidence": { "$ref": "#/definitions/confidence" }, + "evidence": { "type": "string", "minLength": 1 }, + "effective_scope": { "enum": ["all", "direct"] }, + "policy_provenance": { "$ref": "#/definitions/policyProvenance" }, + "reason": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false }, "integrationHint": { - "allOf": [ - { "$ref": "#/definitions/location" }, - { - "type": "object", - "required": ["kind", "usage", "provider", "matched", "evidence"], - "properties": { - "kind": { "const": "integration-hint" }, - "usage": { "enum": ["direct-api", "cloud-provider", "gateway"] }, - "provider": { "type": "string" }, - "matched": { "type": "string" }, - "evidence": { "type": "string" } - } - } - ] + "type": "object", + "required": ["file", "line", "kind", "usage", "provider", "matched", "evidence"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "line": { "type": "integer", "minimum": 1 }, + "kind": { "const": "integration-hint" }, + "usage": { "enum": ["direct-api", "cloud-provider", "gateway"] }, + "provider": { "type": "string", "minLength": 1 }, + "matched": { "type": "string", "minLength": 1 }, + "evidence": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false }, "scanNote": { "type": "object", "required": ["reason"], "properties": { "reason": { "type": "string", "minLength": 1 }, - "file": { "type": "string" }, - "message": { "type": "string" }, + "file": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 }, "bytes": { "type": "integer", "minimum": 0 }, "limit_bytes": { "type": "integer", "minimum": 0 }, "limit_files": { "type": "integer", "minimum": 0 } diff --git a/schema/model-eol.plan.schema.json b/schema/model-eol.plan.schema.json index 21b8330..322525e 100644 --- a/schema/model-eol.plan.schema.json +++ b/schema/model-eol.plan.schema.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/plan.schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.plan.schema.json", "title": "model-eol migration plan", - "description": "Deterministic, hash-guarded migration plan emitted by `model-eol plan`. Draft 0.1 - the $id is a placeholder until the repo is public.", + "description": "Deterministic, hash-guarded migration plan emitted by `model-eol plan`.", "type": "object", "required": ["plan_schema", "generated", "threshold_days", "via", "scan_notes", "items", "issues"], "properties": { "plan_schema": { "const": "model-eol.plan/0.1" }, "generated": { "type": "string", "format": "date-time" }, - "threshold_days": { "type": "number" }, + "threshold_days": { "type": "integer", "minimum": 0 }, "via": { "type": ["string", "null"] }, "scan_notes": { "type": "array", "items": { "$ref": "#/definitions/scanNote" } }, "items": { "type": "array", "items": { "$ref": "#/definitions/item" } }, @@ -28,7 +28,7 @@ "id": { "type": "string", "minLength": 1 }, "publisher": { "type": "string", "minLength": 1 }, "replacement": { "type": "string", "minLength": 1 }, - "shutdown": { "type": "string" }, + "shutdown": { "type": "string", "format": "date" }, "days": { "type": "integer" }, "status": { "enum": ["retired", "retiring"] }, "requested_via": { "type": ["string", "null"] }, @@ -53,7 +53,7 @@ "usage": { "enum": ["direct-api", "cloud-provider", "gateway", "model-reference"] }, "confidence": { "enum": ["high", "medium", "low"] }, "status": { "enum": ["retired", "retiring", "scheduled", "watch", "ok"] }, - "shutdown": { "type": ["string", "null"] }, + "shutdown": { "type": ["string", "null"], "format": "date" }, "via": { "type": ["string", "null"] }, "requested_via": { "type": ["string", "null"] }, "days": { "type": ["integer", "null"] }, diff --git a/schema/model-eol.schedule.schema.json b/schema/model-eol.schedule.schema.json index 2f1c88e..87135b2 100644 --- a/schema/model-eol.schedule.schema.json +++ b/schema/model-eol.schedule.schema.json @@ -1,31 +1,31 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/schedule.schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.schedule.schema.json", "title": "model-eol schedule output", - "description": "Machine-readable deprecation schedule emitted by `model-eol schedule --json`. Draft 0.1 - the $id is a placeholder until the repo is public.", + "description": "Machine-readable deprecation schedule emitted by `model-eol schedule --json`.", "type": "object", "required": ["schema", "generated", "threshold_days", "scope", "targets", "scanned_files", "scan_notes", "earliest_risk", "items", "candidate_model_references", "unresolved_integrations"], "properties": { "schema": { "const": "model-eol/schedule@0.1" }, "generated": { "type": "string", "format": "date-time" }, - "threshold_days": { "type": "number" }, - "distributor": { "type": ["string", "null"] }, + "threshold_days": { "type": "integer", "minimum": 0 }, + "distributor": { "type": ["string", "null"], "minLength": 1 }, "scope": { "enum": ["all", "direct"] }, - "targets": { "type": "array", "items": { "type": "string" } }, + "targets": { "type": "array", "items": { "type": "string", "minLength": 1 } }, "scanned_files": { "type": "integer", "minimum": 0 }, - "scan_notes": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/scanNote" } }, + "scan_notes": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/scanNote" } }, "earliest_risk": { "type": ["object", "null"], "required": ["safe_until", "id"], "properties": { - "safe_until": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "safe_until": { "type": "string", "format": "date" }, "id": { "type": "string", "minLength": 1 } }, "additionalProperties": false }, - "items": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/modelReference" } }, - "candidate_model_references": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/candidateReference" } }, - "unresolved_integrations": { "type": "array", "items": { "$ref": "model-eol.inventory.schema.json#/definitions/integrationHint" } } + "items": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/modelReference" } }, + "candidate_model_references": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/candidateReference" } }, + "unresolved_integrations": { "type": "array", "items": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/integrationHint" } } }, "additionalProperties": false } diff --git a/schema/model-eol.schema.json b/schema/model-eol.schema.json index ded64b3..5e57668 100644 --- a/schema/model-eol.schema.json +++ b/schema/model-eol.schema.json @@ -1,13 +1,13 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://example.invalid/model-eol/0.1/schema.json", + "$id": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.schema.json", "title": "model-eol feed", - "description": "A machine-readable deprecation feed for AI models. Draft 0.1 - the $id is a placeholder until the repo is public.", + "description": "A machine-readable deprecation feed for AI models, version model-eol/0.1.", "type": "object", "required": ["spec", "publisher", "generated", "models"], "properties": { "spec": { "const": "model-eol/0.1" }, - "publisher": { "type": "string", "minLength": 1 }, + "publisher": { "type": "string", "minLength": 1, "maxLength": 256 }, "generated": { "type": "string", "format": "date-time" }, "source": { "type": "string", "format": "uri" }, "note": { "type": "string" }, @@ -27,21 +27,25 @@ }, "additionalProperties": false, "definitions": { - "isoDate": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "isoDate": { "type": "string", "format": "date" }, "model": { "type": "object", "required": ["id"], "properties": { - "id": { "type": "string", "minLength": 1 }, - "aliases": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "id": { "type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, + "aliases": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" } + }, "announced": { "$ref": "#/definitions/isoDate" }, "shutdown": { "$ref": "#/definitions/isoDate" }, "date_precision": { "enum": ["exact", "earliest"] }, - "replacement": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, + "replacement": { "type": "string", "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, "replacement_options": { "type": "array", "minItems": 1, - "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" } + "items": { "type": "string", "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" } }, "replacement_note": { "type": "string" }, "notes": { "type": "string" }, @@ -63,6 +67,18 @@ } } }, + "dependencies": { + "replacement": { + "properties": { + "replacement_options": false + } + }, + "replacement_options": { + "properties": { + "replacement": false + } + } + }, "additionalProperties": false } } diff --git a/scripts/build-public-site.mjs b/scripts/build-public-site.mjs new file mode 100644 index 0000000..733c0f7 --- /dev/null +++ b/scripts/build-public-site.mjs @@ -0,0 +1,235 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process' +import crypto from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' +import { assertValidFeed } from '../lib/validate-feed.mjs' +import { readFeedRefreshReceipt } from './feed-refresh-receipt.mjs' + +const PUBLIC_BASE = 'https://thossullivan.github.io/model-eol' +const SCHEMA_VERSION = '0.1' +const defaultRoot = path.resolve(import.meta.dirname, '..') + +const usage = () => 'Usage: node scripts/build-public-site.mjs --out-dir DIR --receipt FILE [--source-sha SHA] [--repo-dir DIR]' + +const fail = message => { + const error = new Error(`${message}\n${usage()}`) + error.exitCode = 2 + throw error +} + +const sha256 = value => crypto.createHash('sha256').update(value).digest('hex') + +const gitHead = repoDir => { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repoDir, encoding: 'utf8' }) + if (result.error || result.status !== 0) { + throw new Error(`unable to resolve site source commit: ${(result.stderr || result.error?.message || '').trim()}`) + } + return result.stdout.trim() +} + +const git = (repoDir, args, { encoding = 'utf8' } = {}) => { + const result = spawnSync('git', args, { + cwd: repoDir, + encoding, + maxBuffer: 64 * 1024 * 1024, + }) + if (result.error || result.status !== 0) { + const detail = encoding === null + ? result.error?.message + : (result.stderr || result.error?.message || '').trim() + throw new Error(`git ${args[0]} failed${detail ? `: ${detail}` : ''}`) + } + return result.stdout +} + +const assertSha = (value, name) => { + if (!/^[0-9a-f]{40,64}$/i.test(value)) fail(`${name} must be a 40-64 character hexadecimal Git commit`) + return value.toLowerCase() +} + +const writeJson = (file, value) => fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`) + +const publishedJsonPaths = root => ['schema', 'feeds'].flatMap(directory => + fs.readdirSync(path.join(root, directory)) + .filter(name => name.endsWith('.json')) + .sort() + .map(name => `${directory}/${name}`)).sort() + +const assertPublishedInputsBound = ({ root, sourceCommit }) => { + const status = git(root, [ + 'status', '--porcelain=v1', '--untracked-files=all', '--', + ':(glob)schema/*.json', ':(glob)feeds/*.json', + ]) + if (status.trim()) throw new Error(`published schema/feed inputs are dirty relative to ${sourceCommit}: ${status.trim()}`) + + const committed = git(root, ['ls-tree', '-r', '--name-only', '-z', sourceCommit, '--', 'schema', 'feeds'], { encoding: null }) + .toString('utf8') + .split('\0') + .filter(name => /^(?:schema|feeds)\/[^/]+\.json$/.test(name)) + .sort() + const working = publishedJsonPaths(root) + if (JSON.stringify(committed) !== JSON.stringify(working)) { + throw new Error(`published schema/feed file set does not match source commit ${sourceCommit}`) + } + + for (const relative of working) { + const file = path.join(root, ...relative.split('/')) + if (!fs.lstatSync(file).isFile()) throw new Error(`published input must be a regular file: ${relative}`) + const committedBytes = git(root, ['show', `${sourceCommit}:${relative}`], { encoding: null }) + const workingBytes = fs.readFileSync(file) + if (!workingBytes.equals(committedBytes)) { + throw new Error(`published input ${relative} does not match source commit ${sourceCommit}`) + } + } +} + +const copyJsonDirectory = ({ from, to, validate = null }) => { + const names = fs.readdirSync(from).filter(name => name.endsWith('.json')).sort() + if (!names.length) throw new Error(`no JSON files found in ${from}`) + fs.mkdirSync(to, { recursive: true }) + return names.map(name => { + const source = path.join(from, name) + const bytes = fs.readFileSync(source) + const document = JSON.parse(bytes.toString('utf8')) + if (validate) validate(document, source) + fs.writeFileSync(path.join(to, name), bytes) + return { name, document, sha256: sha256(bytes) } + }) +} + +const html = ({ schemas, feeds }) => ` + + + + + model-eol public data + + +
+

model-eol public data

+

Versioned, machine-readable lifecycle contracts from model-eol.

+

Schemas

+ +

Feeds

+ +

Atom changelog · refresh health · publication manifest

+
+ + +` + +export function buildPublicSite({ + repoDir = defaultRoot, + outDir, + receiptFile, + sourceSha, +}) { + const root = path.resolve(repoDir) + const output = path.resolve(outDir) + if (!receiptFile) fail('--receipt is required') + const receiptResult = readFeedRefreshReceipt(path.resolve(receiptFile), { repoDir: root }) + if (!receiptResult.matches) { + throw new Error(`feed refresh receipt does not match the feeds being published: ${receiptResult.mismatches.join('; ')}`) + } + const receipt = receiptResult.receipt + const headCommit = assertSha(gitHead(root), 'repository HEAD') + const sourceCommit = assertSha(sourceSha ?? headCommit, '--source-sha') + if (sourceCommit !== headCommit) throw new Error(`--source-sha ${sourceCommit} does not match repository HEAD ${headCommit}`) + assertPublishedInputsBound({ root, sourceCommit }) + + if (fs.existsSync(output) && fs.readdirSync(output).length) { + throw new Error(`output directory must be empty: ${output}`) + } + fs.mkdirSync(output, { recursive: true }) + + const schemas = copyJsonDirectory({ + from: path.join(root, 'schema'), + to: path.join(output, 'schema', SCHEMA_VERSION), + }) + const feeds = copyJsonDirectory({ + from: path.join(root, 'feeds'), + to: path.join(output, 'feeds'), + validate: (feed, source) => assertValidFeed(feed, source), + }) + + const changelog = spawnSync(process.execPath, [ + path.join(root, 'scripts', 'feed-changelog.mjs'), + '--repo-dir', root, + '--out', path.join(output, 'changelog.atom'), + ], { cwd: root, encoding: 'utf8' }) + if (changelog.error || changelog.status !== 0) { + throw new Error(`failed to build Atom changelog: ${(changelog.stderr || changelog.error?.message || '').trim()}`) + } + + const feedRecords = feeds.map(({ name, document, sha256: digest }) => ({ + publisher: document.publisher, + generated: document.generated, + url: `${PUBLIC_BASE}/feeds/${name}`, + sha256: digest, + })) + writeJson(path.join(output, 'health.json'), { + schema: 'model-eol/health@0.1', + last_checked: receipt.checked_at, + refresh_run: receipt.refresh_run, + refresh_commit: receipt.refresh_commit, + published_commit: sourceCommit, + feeds: feedRecords, + }) + writeJson(path.join(output, 'index.json'), { + schema: 'model-eol/publication@0.1', + published_commit: sourceCommit, + schemas: schemas.map(({ name }) => ({ + id: `${PUBLIC_BASE}/schema/${SCHEMA_VERSION}/${name}`, + url: `${PUBLIC_BASE}/schema/${SCHEMA_VERSION}/${name}`, + })), + feeds: feedRecords, + atom: `${PUBLIC_BASE}/changelog.atom`, + health: `${PUBLIC_BASE}/health.json`, + }) + fs.writeFileSync(path.join(output, 'index.html'), html({ schemas, feeds })) + + return { output, schemas: schemas.length, feeds: feeds.length } +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + options: { + 'out-dir': { type: 'string' }, + receipt: { type: 'string' }, + 'source-sha': { type: 'string' }, + 'repo-dir': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage(), + }) + if (values.help) { + console.log(usage()) + return 0 + } + if (positionals.length) fail(`unexpected positional argument: ${positionals[0]}`) + if (!values['out-dir']) fail('--out-dir is required') + if (!values.receipt) fail('--receipt is required') + + const result = buildPublicSite({ + repoDir: values['repo-dir'] ?? defaultRoot, + outDir: values['out-dir'], + receiptFile: values.receipt, + sourceSha: values['source-sha'], + }) + console.log(`built ${result.schemas} schemas and ${result.feeds} feeds in ${result.output}`) + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`public site build failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/feed-changelog.mjs b/scripts/feed-changelog.mjs index 11c3641..a09c261 100644 --- a/scripts/feed-changelog.mjs +++ b/scripts/feed-changelog.mjs @@ -9,6 +9,7 @@ import { compareFeeds, renderSemanticDiff } from '../refresh/diff.mjs' const DEFAULT_LIMIT = 50 const EMPTY_UPDATED = '1970-01-01T00:00:00Z' +const PUBLIC_ATOM_URL = 'https://thossullivan.github.io/model-eol/changelog.atom' const repoRoot = path.resolve(import.meta.dirname, '..') function usageError(message) { @@ -237,6 +238,9 @@ function renderAtom(commits, entries) { tag:model-eol,${year}:feed-changelog model-eol feed changelog + model-eol maintainers + + ${escapeXml(updated)}${entryBlock} ` diff --git a/scripts/feed-refresh-receipt.mjs b/scripts/feed-refresh-receipt.mjs new file mode 100644 index 0000000..99bb60b --- /dev/null +++ b/scripts/feed-refresh-receipt.mjs @@ -0,0 +1,249 @@ +#!/usr/bin/env node + +import crypto from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' + +export const FEED_REFRESH_RECEIPT_SCHEMA = 'model-eol/feed-refresh-receipt@0.1' + +const shaPattern = /^[0-9a-f]{40,64}$/ +const digestPattern = /^[0-9a-f]{64}$/ +const feedNamePattern = /^[a-z0-9][a-z0-9-]*\.json$/ + +const usage = `Usage: + node scripts/feed-refresh-receipt.mjs create --repo-dir DIR --out FILE --state clean|pending --checked-at ISO_DATE --refresh-run-url HTTPS_URL --refresh-sha SHA [--pending-pr-url HTTPS_URL] + node scripts/feed-refresh-receipt.mjs verify --repo-dir DIR --receipt FILE [--expected-run-url HTTPS_URL] [--expected-refresh-sha SHA] [--github-output FILE] [--require-match]` + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +const isPlainObject = value => value !== null && typeof value === 'object' && !Array.isArray(value) + +const assertExactKeys = (value, expected, label) => { + const actual = Object.keys(value).sort() + const wanted = [...expected].sort() + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + throw new Error(`${label} fields must be exactly ${wanted.join(', ')}; got ${actual.join(', ')}`) + } +} + +const assertInstant = (value, label) => { + const match = typeof value === 'string' + ? /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z$/.exec(value) + : null + const parsed = match ? new Date(value) : null + if (!match || !Number.isFinite(parsed.getTime()) + || parsed.getUTCFullYear() !== Number(match[1]) + || parsed.getUTCMonth() + 1 !== Number(match[2]) + || parsed.getUTCDate() !== Number(match[3]) + || parsed.getUTCHours() !== Number(match[4]) + || parsed.getUTCMinutes() !== Number(match[5]) + || parsed.getUTCSeconds() !== Number(match[6])) { + throw new Error(`${label} must be a valid UTC ISO 8601 instant`) + } + return value +} + +const assertSha = (value, label) => { + if (typeof value !== 'string' || !shaPattern.test(value)) throw new Error(`${label} must be a 40-64 character lowercase hexadecimal Git commit`) + return value +} + +const assertHttpsUrl = (value, label) => { + let parsed + try { + parsed = new URL(value) + } catch { + throw new Error(`${label} must be a valid HTTPS URL`) + } + if (parsed.protocol !== 'https:' || parsed.username || parsed.password) throw new Error(`${label} must be a valid HTTPS URL`) + return parsed.toString() +} + +const sha256 = bytes => crypto.createHash('sha256').update(bytes).digest('hex') + +export const hashFeedDirectory = repoDir => { + const feedDir = path.join(path.resolve(repoDir), 'feeds') + const names = fs.readdirSync(feedDir).filter(name => name.endsWith('.json')).sort() + if (!names.length) throw new Error(`no JSON feeds found in ${feedDir}`) + return names.map(name => ({ + path: `feeds/${name}`, + sha256: sha256(fs.readFileSync(path.join(feedDir, name))), + })) +} + +export function createFeedRefreshReceipt({ + repoDir, + state, + checkedAt, + refreshRunUrl, + refreshSha, + pendingPrUrl = null, +}) { + if (state !== 'clean' && state !== 'pending') throw new Error('receipt state must be clean or pending') + const receipt = { + schema: FEED_REFRESH_RECEIPT_SCHEMA, + state, + changed: state === 'pending', + pending: state === 'pending', + checked_at: assertInstant(checkedAt, 'checked_at'), + refresh_run: assertHttpsUrl(refreshRunUrl, 'refresh_run'), + refresh_commit: assertSha(refreshSha, 'refresh_commit'), + feeds: hashFeedDirectory(repoDir), + } + if (state === 'pending') receipt.pending_pr = assertHttpsUrl(pendingPrUrl, 'pending_pr') + else if (pendingPrUrl !== null) throw new Error('a clean receipt cannot include pending_pr') + return receipt +} + +export function validateFeedRefreshReceipt(receipt, { + repoDir, + expectedRunUrl = null, + expectedRefreshSha = null, +} = {}) { + if (!isPlainObject(receipt)) throw new Error('feed refresh receipt must be a JSON object') + const keys = ['schema', 'state', 'changed', 'pending', 'checked_at', 'refresh_run', 'refresh_commit', 'feeds'] + if (receipt.state === 'pending') keys.push('pending_pr') + assertExactKeys(receipt, keys, 'feed refresh receipt') + if (receipt.schema !== FEED_REFRESH_RECEIPT_SCHEMA) throw new Error(`unsupported feed refresh receipt schema ${receipt.schema}`) + if (receipt.state !== 'clean' && receipt.state !== 'pending') throw new Error('receipt state must be clean or pending') + if (typeof receipt.changed !== 'boolean' || typeof receipt.pending !== 'boolean') throw new Error('receipt changed and pending fields must be boolean') + if (receipt.changed !== (receipt.state === 'pending') || receipt.pending !== (receipt.state === 'pending')) { + throw new Error('receipt changed/pending fields do not agree with receipt state') + } + assertInstant(receipt.checked_at, 'checked_at') + const runUrl = assertHttpsUrl(receipt.refresh_run, 'refresh_run') + const refreshSha = assertSha(receipt.refresh_commit, 'refresh_commit') + if (receipt.state === 'pending') assertHttpsUrl(receipt.pending_pr, 'pending_pr') + if (expectedRunUrl !== null && runUrl !== assertHttpsUrl(expectedRunUrl, 'expected refresh run URL')) { + throw new Error(`receipt refresh_run ${runUrl} does not match ${expectedRunUrl}`) + } + if (expectedRefreshSha !== null && refreshSha !== assertSha(expectedRefreshSha, 'expected refresh SHA')) { + throw new Error(`receipt refresh_commit ${refreshSha} does not match ${expectedRefreshSha}`) + } + if (!Array.isArray(receipt.feeds) || !receipt.feeds.length) throw new Error('receipt feeds must be a non-empty array') + const priorPaths = new Set() + for (const [index, feed] of receipt.feeds.entries()) { + if (!isPlainObject(feed)) throw new Error(`receipt feeds[${index}] must be an object`) + assertExactKeys(feed, ['path', 'sha256'], `receipt feeds[${index}]`) + if (typeof feed.path !== 'string' || !feed.path.startsWith('feeds/') || !feedNamePattern.test(feed.path.slice('feeds/'.length))) { + throw new Error(`receipt feeds[${index}].path must identify one feeds/*.json file`) + } + if (priorPaths.has(feed.path)) throw new Error(`receipt contains duplicate feed path ${feed.path}`) + priorPaths.add(feed.path) + if (typeof feed.sha256 !== 'string' || !digestPattern.test(feed.sha256)) throw new Error(`receipt ${feed.path} has an invalid SHA-256 digest`) + } + const sorted = [...receipt.feeds].sort((left, right) => left.path.localeCompare(right.path)) + if (JSON.stringify(sorted) !== JSON.stringify(receipt.feeds)) throw new Error('receipt feeds must be sorted by path') + + const actualFeeds = hashFeedDirectory(repoDir) + const actualByPath = new Map(actualFeeds.map(feed => [feed.path, feed.sha256])) + const receiptByPath = new Map(receipt.feeds.map(feed => [feed.path, feed.sha256])) + const mismatches = [] + for (const feed of actualFeeds) { + const expected = receiptByPath.get(feed.path) + if (expected === undefined) mismatches.push(`${feed.path}: missing from receipt`) + else if (expected !== feed.sha256) mismatches.push(`${feed.path}: expected ${expected}, got ${feed.sha256}`) + } + for (const feed of receipt.feeds) { + if (!actualByPath.has(feed.path)) mismatches.push(`${feed.path}: not present in repository`) + } + return { receipt, matches: mismatches.length === 0, mismatches } +} + +export const readFeedRefreshReceipt = (receiptFile, options) => { + let receipt + try { + receipt = JSON.parse(fs.readFileSync(receiptFile, 'utf8')) + } catch (error) { + throw new Error(`unable to read feed refresh receipt ${receiptFile}: ${error.message}`) + } + return validateFeedRefreshReceipt(receipt, options) +} + +const appendGithubOutput = (file, result) => { + fs.appendFileSync(file, [ + `matches=${result.matches}`, + `state=${result.receipt.state}`, + `checked_at=${result.receipt.checked_at}`, + `refresh_run=${result.receipt.refresh_run}`, + `refresh_sha=${result.receipt.refresh_commit}`, + '', + ].join('\n')) +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + allowPositionals: true, + options: { + 'repo-dir': { type: 'string' }, + out: { type: 'string' }, + state: { type: 'string' }, + 'checked-at': { type: 'string' }, + 'refresh-run-url': { type: 'string' }, + 'refresh-sha': { type: 'string' }, + 'pending-pr-url': { type: 'string' }, + receipt: { type: 'string' }, + 'expected-run-url': { type: 'string' }, + 'expected-refresh-sha': { type: 'string' }, + 'github-output': { type: 'string' }, + 'require-match': { type: 'boolean' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + const [command, ...extra] = positionals + if (!command || extra.length || !['create', 'verify'].includes(command)) fail('command must be create or verify') + const repoDir = values['repo-dir'] ?? process.cwd() + if (command === 'create') { + for (const name of ['out', 'state', 'checked-at', 'refresh-run-url', 'refresh-sha']) { + if (!values[name]) fail(`--${name} is required for create`) + } + const receipt = createFeedRefreshReceipt({ + repoDir, + state: values.state, + checkedAt: values['checked-at'], + refreshRunUrl: values['refresh-run-url'], + refreshSha: values['refresh-sha'], + pendingPrUrl: values['pending-pr-url'] ?? null, + }) + fs.mkdirSync(path.dirname(path.resolve(values.out)), { recursive: true }) + fs.writeFileSync(values.out, `${JSON.stringify(receipt, null, 2)}\n`) + console.log(`wrote ${receipt.state} feed refresh receipt for ${receipt.feeds.length} feeds to ${values.out}`) + return 0 + } + + if (!values.receipt) fail('--receipt is required for verify') + const result = readFeedRefreshReceipt(values.receipt, { + repoDir, + expectedRunUrl: values['expected-run-url'] ?? null, + expectedRefreshSha: values['expected-refresh-sha'] ?? null, + }) + if (values['github-output']) appendGithubOutput(values['github-output'], result) + if (!result.matches) { + console.error(`receipt feed hashes do not match repository feeds:\n${result.mismatches.map(item => `- ${item}`).join('\n')}`) + if (values['require-match']) return 3 + } else { + console.log(`${result.receipt.state} receipt matches all ${result.receipt.feeds.length} repository feeds`) + } + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`feed refresh receipt failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/github-release-state.mjs b/scripts/github-release-state.mjs new file mode 100644 index 0000000..33ee81f --- /dev/null +++ b/scripts/github-release-state.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' + +const stableTagPattern = /^v0\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/ +const usage = 'Usage: node scripts/github-release-state.mjs --input PAGINATED_JSON --tag vVERSION' + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +export function stableGithubReleaseExists({ pages, tag }) { + if (typeof tag !== 'string' || !stableTagPattern.test(tag)) throw new Error(`GitHub release tag must be an exact stable v0.x version, got ${tag}`) + if (!Array.isArray(pages) || pages.some(page => !Array.isArray(page))) { + throw new Error('paginated GitHub releases response must be an outer array of page arrays') + } + const matches = [] + for (const page of pages) { + for (const release of page) { + if (release === null || typeof release !== 'object' || Array.isArray(release)) throw new Error('GitHub release entry must be an object') + if (typeof release.tag_name !== 'string' || typeof release.draft !== 'boolean' || typeof release.prerelease !== 'boolean') { + throw new Error('GitHub release entry is missing tag_name, draft, or prerelease state') + } + if (release.tag_name === tag) matches.push(release) + } + } + if (matches.length > 1) throw new Error(`GitHub releases API returned duplicate entries for ${tag}`) + if (!matches.length) return false + if (matches[0].draft || matches[0].prerelease) { + throw new Error(`${tag} exists as a draft or prerelease; refusing to treat it as the stable release`) + } + return true +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + options: { + input: { type: 'string' }, + tag: { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + if (positionals.length) fail(`unexpected positional argument: ${positionals[0]}`) + if (!values.input) fail('--input is required') + if (!values.tag) fail('--tag is required') + let pages + try { + pages = JSON.parse(fs.readFileSync(values.input, 'utf8')) + } catch (error) { + throw new Error(`unable to read paginated GitHub releases response: ${error.message}`) + } + console.log(stableGithubReleaseExists({ pages, tag: values.tag })) + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`GitHub release state failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/package-integrity.mjs b/scripts/package-integrity.mjs new file mode 100644 index 0000000..b6019c6 --- /dev/null +++ b/scripts/package-integrity.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +import crypto from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' + +const usage = 'Usage: node scripts/package-integrity.mjs verify --tarball FILE --expected-integrity SHA512_SRI' + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +export const assertSha512Integrity = (value, label = 'integrity') => { + if (typeof value !== 'string' || !value.startsWith('sha512-')) { + throw new Error(`${label} must be an npm sha512 Subresource Integrity value`) + } + const encoded = value.slice('sha512-'.length) + let digest + try { + digest = Buffer.from(encoded, 'base64') + } catch { + digest = Buffer.alloc(0) + } + if (digest.length !== 64 || digest.toString('base64') !== encoded) { + throw new Error(`${label} must contain one canonical 64-byte SHA-512 digest`) + } + return value +} + +export const sha512Integrity = bytes => `sha512-${crypto.createHash('sha512').update(bytes).digest('base64')}` + +export const verifyPackageIntegrity = ({ tarball, expectedIntegrity }) => { + const expected = assertSha512Integrity(expectedIntegrity, 'expected integrity') + const actual = sha512Integrity(fs.readFileSync(tarball)) + if (actual !== expected) throw new Error(`package integrity mismatch for ${tarball}: expected ${expected}, got ${actual}`) + return actual +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + allowPositionals: true, + options: { + tarball: { type: 'string' }, + 'expected-integrity': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + const [command, ...extra] = positionals + if (command !== 'verify' || extra.length) fail('command must be verify') + if (!values.tarball) fail('--tarball is required') + if (!values['expected-integrity']) fail('--expected-integrity is required') + console.log(verifyPackageIntegrity({ + tarball: path.resolve(values.tarball), + expectedIntegrity: values['expected-integrity'], + })) + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`package integrity failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/published-consumer-uat.mjs b/scripts/published-consumer-uat.mjs new file mode 100644 index 0000000..9f6f11e --- /dev/null +++ b/scripts/published-consumer-uat.mjs @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' +import { assertSha512Integrity } from './package-integrity.mjs' + +const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm' +const packagePattern = /^model-eol@(0|0\.\d+\.\d+)$/ +const versionPattern = /^0\.\d+\.\d+$/ + +const usage = 'Usage: node scripts/published-consumer-uat.mjs --package model-eol@VERSION --expected-version VERSION --expected-integrity SHA512_SRI [--expected-engine RANGE]' + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 10 * 1024 * 1024, + ...options, + }) + if (result.error || result.status !== 0) { + const detail = result.error?.message || result.stderr?.trim() || result.stdout?.trim() || `exit ${result.status}` + throw new Error(`${command} ${args.join(' ')} failed: ${detail}`) + } + return result +} + +const parseJson = (text, label) => { + try { + return JSON.parse(text) + } catch (error) { + throw new Error(`${label} was not valid JSON: ${error.message}`) + } +} + +const waitForPublishedVersion = ({ version, expectedIntegrity, cache }) => { + let detail = '' + for (let attempt = 1; attempt <= 12; attempt++) { + const result = spawnSync(npm, ['view', `model-eol@${version}`, 'version', 'dist.integrity', '--json', '--cache', cache, '--prefer-online'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }) + if (!result.error && result.status === 0) { + let metadata = null + try { + metadata = JSON.parse(result.stdout) + } catch { + } + if (metadata?.version === version) { + const registryIntegrity = assertSha512Integrity(metadata['dist.integrity'], `model-eol@${version} registry integrity`) + if (registryIntegrity !== expectedIntegrity) { + throw new Error(`model-eol@${version} registry integrity ${registryIntegrity} does not match release integrity ${expectedIntegrity}`) + } + return registryIntegrity + } + } + detail = result.error?.message || result.stderr?.trim() || result.stdout?.trim() || `exit ${result.status}` + if (attempt < 12) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000) + } + throw new Error(`npm did not expose model-eol@${version} after 60 seconds: ${detail}`) +} + +export function runPublishedConsumerUat({ packageSpec, expectedVersion, expectedIntegrity, expectedEngine = null }) { + if (!packagePattern.test(packageSpec)) fail('--package must be model-eol@0 or an exact stable 0.x version') + if (!versionPattern.test(expectedVersion)) fail('--expected-version must be an exact stable 0.x version') + const releaseIntegrity = assertSha512Integrity(expectedIntegrity, '--expected-integrity') + if (expectedEngine !== null && (!expectedEngine || /[\r\n]/.test(expectedEngine))) fail('--expected-engine must be a non-empty single-line range') + + const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-published-uat-')) + const consumer = path.join(temp, 'consumer') + const cache = path.join(temp, 'npm-cache') + try { + waitForPublishedVersion({ version: expectedVersion, expectedIntegrity: releaseIntegrity, cache }) + fs.mkdirSync(consumer) + fs.writeFileSync(path.join(consumer, 'package.json'), `${JSON.stringify({ + name: 'model-eol-published-uat', + private: true, + scripts: { 'eval:model-eol': 'node eval-model-swap.mjs' }, + }, null, 2)}\n`) + fs.writeFileSync(path.join(consumer, 'app.mjs'), [ + 'import OpenAI from "openai"', + 'const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })', + 'export const model = "o3-deep-research"', + '', + ].join('\n')) + fs.writeFileSync(path.join(consumer, 'eval-model-swap.mjs'), [ + "import fs from 'node:fs'", + "const source = fs.readFileSync(new URL('./app.mjs', import.meta.url), 'utf8')", + "const oldId = process.env.MODEL_EOL_OLD_ID", + "const newId = process.env.MODEL_EOL_NEW_ID", + "if (!oldId || !newId || !source.includes(newId) || source.includes(oldId)) throw new Error('planned model swap was not applied')", + '', + ].join('\n')) + run('git', ['init', '-q', '-b', 'main'], { cwd: consumer }) + run('git', ['config', 'user.name', 'model-eol published UAT'], { cwd: consumer }) + run('git', ['config', 'user.email', 'published-uat@example.invalid'], { cwd: consumer }) + run(npm, [ + 'install', '--ignore-scripts', '--no-audit', '--no-fund', '--prefer-online', + '--cache', cache, packageSpec, + ], { cwd: consumer }) + + const installedRoot = path.join(consumer, 'node_modules', 'model-eol') + const manifest = parseJson(fs.readFileSync(path.join(installedRoot, 'package.json'), 'utf8'), 'installed manifest') + if (manifest.version !== expectedVersion) { + throw new Error(`${packageSpec} installed ${manifest.version}; expected ${expectedVersion}`) + } + const lock = parseJson(fs.readFileSync(path.join(consumer, 'package-lock.json'), 'utf8'), 'consumer package lock') + const lockedPackage = lock.packages?.['node_modules/model-eol'] + if (lockedPackage?.version !== expectedVersion || lockedPackage?.integrity !== releaseIntegrity) { + throw new Error(`${packageSpec} lock entry is not bound to model-eol@${expectedVersion} with integrity ${releaseIntegrity}`) + } + if (expectedEngine !== null && manifest.engines?.node !== expectedEngine) { + throw new Error(`${packageSpec} declares Node ${manifest.engines?.node}; expected ${expectedEngine}`) + } + for (const field of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) { + if (manifest[field] && Object.keys(manifest[field]).length) throw new Error(`${packageSpec} unexpectedly has ${field}`) + } + const harnessSource = path.join(installedRoot, 'examples', 'model-eol-eval.mjs') + if (!fs.existsSync(harnessSource)) throw new Error(`${packageSpec} does not contain examples/model-eol-eval.mjs`) + fs.mkdirSync(path.join(consumer, 'scripts')) + fs.copyFileSync(harnessSource, path.join(consumer, 'scripts', 'model-eol-eval.mjs')) + fs.writeFileSync(path.join(consumer, '.model-eol.json'), `${JSON.stringify({ + eval: { command: 'node scripts/model-eol-eval.mjs' }, + }, null, 2)}\n`) + run('git', ['add', '.model-eol.json', 'app.mjs', 'eval-model-swap.mjs', 'package.json', 'package-lock.json', 'scripts/model-eol-eval.mjs'], { cwd: consumer }) + run('git', ['commit', '-q', '-m', 'consumer fixture with eval harness'], { cwd: consumer }) + + const binSuffix = process.platform === 'win32' ? '.cmd' : '' + const checker = path.join(consumer, 'node_modules', '.bin', `model-eol${binSuffix}`) + const bot = path.join(consumer, 'node_modules', '.bin', `model-eol-bot${binSuffix}`) + const inventory = parseJson(run(checker, ['inventory', '.'], { cwd: consumer }).stdout, 'published inventory') + if (inventory.schema !== 'model-eol/inventory@0.1' || !inventory.model_references.some(item => item.matched === 'o3-deep-research')) { + throw new Error('published checker did not inventory the consumer model') + } + const inventoryFile = path.join(temp, 'inventory.json') + fs.writeFileSync(inventoryFile, `${JSON.stringify(inventory, null, 2)}\n`) + const validation = run(checker, ['validate', inventoryFile], { cwd: consumer }) + if (!validation.stdout.includes('valid inventory document')) throw new Error('published checker did not validate its own inventory artifact') + + const checkResult = spawnSync(checker, ['check', '.', '--json'], { + cwd: consumer, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + maxBuffer: 10 * 1024 * 1024, + }) + if (checkResult.error || checkResult.status !== 1) { + const detail = checkResult.error?.message || checkResult.stderr?.trim() || checkResult.stdout?.trim() || `exit ${checkResult.status}` + throw new Error(`published check --json did not preserve its finding exit code: ${detail}`) + } + const checkDocument = parseJson(checkResult.stdout, 'published check report') + if (checkDocument.schema !== 'model-eol/check@0.1' || !checkDocument.findings.length) { + throw new Error('published checker did not emit its public check discriminator and finding') + } + const checkFile = path.join(temp, 'check.json') + fs.writeFileSync(checkFile, checkResult.stdout) + const checkValidation = run(checker, ['validate', checkFile], { cwd: consumer }) + if (!checkValidation.stdout.includes('valid check document')) throw new Error('published checker did not validate its own check artifact') + + const planResult = run(checker, ['plan', '.', '--days', '90', '--scope', 'direct'], { cwd: consumer }) + const plan = parseJson(planResult.stdout, 'published plan') + if (plan.plan_schema !== 'model-eol.plan/0.1' || plan.items.length !== 1) { + throw new Error('published checker did not produce the expected safe migration plan') + } + const planFile = path.join(temp, 'plan.json') + const evalFile = path.join(temp, 'eval.json') + fs.writeFileSync(planFile, planResult.stdout) + run(bot, ['evaluate', '--target-dir', consumer, '--plan-file', planFile, '--output-file', evalFile], { cwd: consumer }) + const evaluation = parseJson(fs.readFileSync(evalFile, 'utf8'), 'published evaluation') + if (evaluation.schema !== 'model-eol.eval/0.1' || evaluation.configured !== true || !/^[0-9a-f]{40,64}$/.test(evaluation.base_sha)) { + throw new Error('published bot did not produce a configured commit-bound evaluator manifest') + } + if (evaluation.results.length !== 1 || evaluation.results[0]?.status !== 'pass' || evaluation.results[0]?.exit_code !== 0) { + throw new Error('published eval harness did not return exactly one passing result') + } + if (!evaluation.results[0]?.report?.includes('The repository verification command passed')) { + throw new Error('published eval harness did not emit its bounded passing report') + } + const dryRun = run(bot, ['--dry-run', '--target-dir', consumer, '--repo', 'example/published-uat'], { + cwd: consumer, + env: { ...process.env, MODEL_EOL_EVAL_RESULTS_FILE: evalFile }, + }) + const planned = plan.items[0] + const expectedDecision = `- create ${planned.publisher}/${planned.id}` + if (!dryRun.stdout.includes(expectedDecision) || !dryRun.stdout.includes('Result: pass (exit code 0).')) { + throw new Error(`published bot dry-run did not authorize the evaluated migration ${planned.publisher}/${planned.id}`) + } + if (!dryRun.stdout.includes(planned.replacement) || !dryRun.stdout.includes('The repository verification command passed')) { + throw new Error('published bot dry-run did not preserve the evaluated old/new migration identity and report') + } + + return { version: manifest.version, integrity: releaseIntegrity, references: inventory.model_references.length, items: plan.items.length } + } finally { + fs.rmSync(temp, { recursive: true, force: true }) + } +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + options: { + package: { type: 'string' }, + 'expected-version': { type: 'string' }, + 'expected-integrity': { type: 'string' }, + 'expected-engine': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + if (positionals.length) fail(`unexpected positional argument: ${positionals[0]}`) + if (!values.package) fail('--package is required') + if (!values['expected-version']) fail('--expected-version is required') + if (!values['expected-integrity']) fail('--expected-integrity is required') + const result = runPublishedConsumerUat({ + packageSpec: values.package, + expectedVersion: values['expected-version'], + expectedIntegrity: values['expected-integrity'], + expectedEngine: values['expected-engine'] ?? null, + }) + console.log(`published consumer UAT passed for model-eol@${result.version} (${result.references} reference(s), ${result.items} migration item)`) + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`published consumer UAT failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/release-receipt.mjs b/scripts/release-receipt.mjs new file mode 100644 index 0000000..4236eab --- /dev/null +++ b/scripts/release-receipt.mjs @@ -0,0 +1,153 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' +import { assertSha512Integrity } from './package-integrity.mjs' + +export const RELEASE_RECEIPT_SCHEMA = 'model-eol/npm-release-result@0.2' + +const versionPattern = /^0\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/ +const shaPattern = /^[0-9a-f]{40,64}$/ + +const usage = `Usage: + node scripts/release-receipt.mjs create --out FILE --source-sha SHA [--version VERSION --release-sha SHA --registry-integrity SHA512_SRI] + node scripts/release-receipt.mjs verify --receipt FILE [--expected-source-sha SHA] [--github-output FILE]` + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +const assertSha = (value, label) => { + if (typeof value !== 'string' || !shaPattern.test(value)) throw new Error(`${label} must be a 40-64 character lowercase hexadecimal Git commit`) + return value +} + +const assertVersion = value => { + if (typeof value !== 'string' || !versionPattern.test(value)) throw new Error(`release version must be an exact stable 0.x version, got ${value}`) + return value +} + +const exactKeys = (value, keys) => { + const actual = Object.keys(value).sort() + const expected = [...keys].sort() + if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error(`release receipt fields must be exactly ${expected.join(', ')}; got ${actual.join(', ')}`) +} + +export function createReleaseReceipt({ sourceCommit, version = null, releaseCommit = null, registryIntegrity = null }) { + const receipt = { + schema: RELEASE_RECEIPT_SCHEMA, + released: version !== null, + source_commit: assertSha(sourceCommit, 'source_commit'), + } + if (version === null) { + if (releaseCommit !== null || registryIntegrity !== null) throw new Error('release_commit and registry_integrity cannot be set for a no-release result') + return receipt + } + receipt.version = assertVersion(version) + receipt.tag = `v${version}` + receipt.release_commit = assertSha(releaseCommit, 'release_commit') + receipt.registry_integrity = assertSha512Integrity(registryIntegrity, 'registry_integrity') + return receipt +} + +export function validateReleaseReceipt(receipt, { expectedSourceCommit = null } = {}) { + if (receipt === null || typeof receipt !== 'object' || Array.isArray(receipt)) throw new Error('release receipt must be a JSON object') + if (typeof receipt.released !== 'boolean') throw new Error('release receipt released must be boolean') + exactKeys(receipt, receipt.released + ? ['schema', 'released', 'source_commit', 'version', 'tag', 'release_commit', 'registry_integrity'] + : ['schema', 'released', 'source_commit']) + if (receipt.schema !== RELEASE_RECEIPT_SCHEMA) throw new Error(`unsupported release receipt schema ${receipt.schema}`) + const sourceCommit = assertSha(receipt.source_commit, 'source_commit') + if (expectedSourceCommit !== null && sourceCommit !== assertSha(expectedSourceCommit, 'expected source commit')) { + throw new Error(`receipt source_commit ${sourceCommit} does not match workflow source ${expectedSourceCommit}`) + } + if (receipt.released) { + assertVersion(receipt.version) + if (receipt.tag !== `v${receipt.version}`) throw new Error(`release receipt tag ${receipt.tag} does not match version ${receipt.version}`) + assertSha(receipt.release_commit, 'release_commit') + assertSha512Integrity(receipt.registry_integrity, 'registry_integrity') + } + return receipt +} + +const readReceipt = (file, options) => { + let receipt + try { + receipt = JSON.parse(fs.readFileSync(file, 'utf8')) + } catch (error) { + throw new Error(`unable to read release receipt ${file}: ${error.message}`) + } + return validateReleaseReceipt(receipt, options) +} + +const appendGithubOutput = (file, receipt) => { + fs.appendFileSync(file, [ + `released=${receipt.released}`, + `version=${receipt.version ?? ''}`, + `tag=${receipt.tag ?? ''}`, + `release_commit=${receipt.release_commit ?? ''}`, + `registry_integrity=${receipt.registry_integrity ?? ''}`, + '', + ].join('\n')) +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + allowPositionals: true, + options: { + out: { type: 'string' }, + 'source-sha': { type: 'string' }, + version: { type: 'string' }, + 'release-sha': { type: 'string' }, + 'registry-integrity': { type: 'string' }, + receipt: { type: 'string' }, + 'expected-source-sha': { type: 'string' }, + 'github-output': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + const [command, ...extra] = positionals + if (!command || extra.length || !['create', 'verify'].includes(command)) fail('command must be create or verify') + if (command === 'create') { + if (!values.out) fail('--out is required for create') + if (!values['source-sha']) fail('--source-sha is required for create') + const releaseFields = [values.version, values['release-sha'], values['registry-integrity']] + if (releaseFields.some(value => value === undefined) && releaseFields.some(value => value !== undefined)) { + fail('--version, --release-sha, and --registry-integrity must be supplied together') + } + const receipt = createReleaseReceipt({ + sourceCommit: values['source-sha'], + version: values.version ?? null, + releaseCommit: values['release-sha'] ?? null, + registryIntegrity: values['registry-integrity'] ?? null, + }) + fs.mkdirSync(path.dirname(path.resolve(values.out)), { recursive: true }) + fs.writeFileSync(values.out, `${JSON.stringify(receipt, null, 2)}\n`) + console.log(receipt.released ? `wrote release receipt for ${receipt.tag}` : 'wrote no-release receipt') + return 0 + } + if (!values.receipt) fail('--receipt is required for verify') + const receipt = readReceipt(values.receipt, { expectedSourceCommit: values['expected-source-sha'] ?? null }) + if (values['github-output']) appendGithubOutput(values['github-output'], receipt) + console.log(receipt.released ? `verified release receipt for ${receipt.tag}` : 'verified no-release receipt') + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`release receipt failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/release-state.mjs b/scripts/release-state.mjs new file mode 100644 index 0000000..bd068b8 --- /dev/null +++ b/scripts/release-state.mjs @@ -0,0 +1,235 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' + +import { parseCliArgs } from '../lib/cli.mjs' + +const versionPattern = /^0\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/ +const shaPattern = /^[0-9a-f]{40,64}$/ + +const usage = `Usage: + node scripts/release-state.mjs target --event-name push|workflow_dispatch [--requested-version VERSION] + node scripts/release-state.mjs resolve --event-name push|workflow_dispatch --source-sha SHA --published-json JSON --github-release-exists true|false [--requested-version VERSION] [--github-output FILE]` + +const fail = message => { + const error = new Error(`${message}\n${usage}`) + error.exitCode = 2 + throw error +} + +const versionParts = (value, label) => { + const match = typeof value === 'string' ? versionPattern.exec(value) : null + if (!match) throw new Error(`${label} must be an exact stable 0.x version, got ${value}`) + return [0, Number(match[1]), Number(match[2])] +} + +const compareVersions = (left, right) => { + for (let index = 0; index < left.length; index++) { + if (left[index] !== right[index]) return left[index] > right[index] ? 1 : -1 + } + return 0 +} + +export const targetReleaseVersion = ({ eventName, sourceVersion, requestedVersion = null }) => { + const source = versionParts(sourceVersion, 'source package version') + if (eventName === 'push') return `${source[0]}.${source[1]}.${source[2] + 1}` + if (eventName !== 'workflow_dispatch') throw new Error(`unsupported release event ${eventName}`) + const requested = versionParts(requestedVersion, 'requested release version') + if (compareVersions(requested, source) <= 0) throw new Error(`release version ${requestedVersion} must be greater than package version ${sourceVersion}`) + return requestedVersion +} + +export function resolveReleaseState({ + eventName, + sourceVersion, + requestedVersion = null, + sourceCommit, + mainCommit, + publishedVersions = [], + githubReleaseExists, + tag = null, +}) { + if (!shaPattern.test(sourceCommit ?? '')) throw new Error('source commit must be a 40-64 character lowercase hexadecimal Git commit') + if (!shaPattern.test(mainCommit ?? '')) throw new Error('origin/main commit must be a 40-64 character lowercase hexadecimal Git commit') + if (!Array.isArray(publishedVersions) || publishedVersions.some(version => typeof version !== 'string')) throw new Error('published versions must be an array of strings') + if (typeof githubReleaseExists !== 'boolean') throw new Error('GitHub release state must be boolean') + const version = targetReleaseVersion({ eventName, sourceVersion, requestedVersion }) + const releaseTag = `v${version}` + const published = publishedVersions.includes(version) + + if (tag === null) { + if (published) throw new Error(`${version} exists on npm without immutable tag ${releaseTag}; refusing to invent a release commit`) + if (githubReleaseExists) throw new Error(`GitHub release ${releaseTag} exists without its immutable Git tag`) + if (mainCommit !== sourceCommit) throw new Error(`origin/main moved from release source ${sourceCommit} to ${mainCommit}; refusing a non-atomic release push`) + return { + version, + tag: releaseTag, + mode: 'create', + publish: true, + createGithubRelease: true, + releaseCommit: null, + } + } + + if (!tag || typeof tag !== 'object' || Array.isArray(tag)) throw new Error('tag state must be an object or null') + for (const [label, value] of [ + ['release tag commit', tag.commit], + ['release tag parent', tag.parent], + ['moving v0 commit', tag.movingCommit], + ]) { + if (!shaPattern.test(value ?? '')) throw new Error(`${label} must be a Git commit`) + } + if (tag.parent !== sourceCommit) throw new Error(`${releaseTag} is not the release commit directly above source ${sourceCommit}`) + if (tag.version !== version) throw new Error(`${releaseTag} contains package version ${tag.version}, expected ${version}`) + if (tag.movingCommit !== tag.commit) throw new Error(`moving v0 tag ${tag.movingCommit} does not match immutable ${releaseTag} commit ${tag.commit}`) + if (tag.mainContainsRelease !== true) throw new Error(`origin/main does not contain immutable release commit ${tag.commit}`) + if (JSON.stringify(tag.changedPaths) !== JSON.stringify(['package.json'])) { + throw new Error(`${releaseTag} release commit must change only package.json; got ${(tag.changedPaths ?? []).join(', ')}`) + } + if (githubReleaseExists && !published) throw new Error(`${releaseTag} has a GitHub release but model-eol@${version} is missing from npm`) + return { + version, + tag: releaseTag, + mode: 'resume', + publish: !published, + createGithubRelease: !githubReleaseExists, + releaseCommit: tag.commit, + } +} + +const runGit = (args, { allowStatus = [] } = {}) => { + const result = spawnSync('git', args, { encoding: 'utf8' }) + if (result.error) throw new Error(`git ${args.join(' ')} failed: ${result.error.message}`) + if (result.status !== 0 && !allowStatus.includes(result.status)) { + throw new Error(`git ${args.join(' ')} failed: ${(result.stderr || result.stdout).trim() || `exit ${result.status}`}`) + } + return result +} + +const revParse = ref => runGit(['rev-parse', '--verify', `${ref}^{commit}`]).stdout.trim() + +const maybeRevParse = ref => { + const result = runGit(['rev-parse', '--verify', `${ref}^{commit}`], { allowStatus: [1, 128] }) + return result.status === 0 ? result.stdout.trim() : null +} + +const packageVersionAt = commit => { + const result = runGit(['show', `${commit}:package.json`]) + try { + return JSON.parse(result.stdout).version + } catch (error) { + throw new Error(`package.json at ${commit} is invalid: ${error.message}`) + } +} + +const currentPackageVersion = () => { + try { + return JSON.parse(fs.readFileSync('package.json', 'utf8')).version + } catch (error) { + throw new Error(`unable to read source package.json: ${error.message}`) + } +} + +const parseBoolean = (value, label) => { + if (value === 'true') return true + if (value === 'false') return false + throw new Error(`${label} must be true or false`) +} + +const appendGithubOutput = (file, state) => { + fs.appendFileSync(file, [ + `version=${state.version}`, + `tag=${state.tag}`, + `mode=${state.mode}`, + `publish=${state.publish}`, + `create_github_release=${state.createGithubRelease}`, + `release_commit=${state.releaseCommit ?? ''}`, + '', + ].join('\n')) +} + +function main(argv) { + const { values, positionals } = parseCliArgs({ + args: argv, + allowPositionals: true, + options: { + 'event-name': { type: 'string' }, + 'requested-version': { type: 'string' }, + 'source-sha': { type: 'string' }, + 'published-json': { type: 'string' }, + 'github-release-exists': { type: 'string' }, + 'github-output': { type: 'string' }, + help: { type: 'boolean', short: 'h' }, + }, + help: usage, + }) + if (values.help) { + console.log(usage) + return 0 + } + const [command, ...extra] = positionals + if (!command || extra.length || !['target', 'resolve'].includes(command)) fail('command must be target or resolve') + if (!values['event-name']) fail('--event-name is required') + const sourceVersion = currentPackageVersion() + const requestedVersion = values['requested-version'] ?? null + if (command === 'target') { + console.log(targetReleaseVersion({ eventName: values['event-name'], sourceVersion, requestedVersion })) + return 0 + } + for (const name of ['source-sha', 'published-json', 'github-release-exists']) { + if (values[name] === undefined) fail(`--${name} is required for resolve`) + } + const sourceCommit = revParse('HEAD') + if (sourceCommit !== values['source-sha']) throw new Error(`checked out source ${sourceCommit} does not match workflow source ${values['source-sha']}`) + let publishedVersions + try { + const parsed = JSON.parse(values['published-json']) + publishedVersions = Array.isArray(parsed) ? parsed : [parsed] + } catch (error) { + throw new Error(`--published-json is invalid: ${error.message}`) + } + const version = targetReleaseVersion({ eventName: values['event-name'], sourceVersion, requestedVersion }) + const releaseTag = `v${version}` + const releaseCommit = maybeRevParse(`refs/tags/${releaseTag}`) + let tag = null + const mainCommit = revParse('refs/remotes/origin/main') + if (releaseCommit !== null) { + const parentLine = runGit(['rev-list', '--parents', '-n', '1', releaseCommit]).stdout.trim().split(/\s+/) + if (parentLine.length !== 2) throw new Error(`${releaseTag} must identify a single-parent release commit`) + const parent = parentLine[1] + const changedPaths = runGit(['diff', '--name-only', `${parent}..${releaseCommit}`]).stdout.split('\n').filter(Boolean).sort() + const ancestry = runGit(['merge-base', '--is-ancestor', releaseCommit, mainCommit], { allowStatus: [1] }) + tag = { + commit: releaseCommit, + parent, + version: packageVersionAt(releaseCommit), + movingCommit: maybeRevParse('refs/tags/v0'), + mainContainsRelease: ancestry.status === 0, + changedPaths, + } + } + const state = resolveReleaseState({ + eventName: values['event-name'], + sourceVersion, + requestedVersion, + sourceCommit, + mainCommit, + publishedVersions, + githubReleaseExists: parseBoolean(values['github-release-exists'], '--github-release-exists'), + tag, + }) + if (values['github-output']) appendGithubOutput(values['github-output'], state) + console.log(JSON.stringify(state, null, 2)) + return 0 +} + +if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) { + try { + process.exitCode = main(process.argv.slice(2)) + } catch (error) { + console.error(`release state failed: ${error.message}`) + process.exitCode = error.exitCode ?? 1 + } +} diff --git a/scripts/test-action-contract.mjs b/scripts/test-action-contract.mjs index cbf5c3a..0b5050f 100644 --- a/scripts/test-action-contract.mjs +++ b/scripts/test-action-contract.mjs @@ -10,6 +10,7 @@ import { fileURLToPath } from 'node:url' const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const actionPath = path.join(root, 'action.yml') const actionSource = fs.readFileSync(actionPath, 'utf8') +assert.match(actionSource, /Node 22 or newer is required/, 'Action fails clearly when an ambient self-hosted runner is below the supported Node floor') const inputDefaults = new Map() let inInputs = false @@ -53,6 +54,7 @@ const requiredDefaults = [ 'format', 'feeds', 'config', + 'document-type', 'include-docs', 'allow-incomplete', 'json', @@ -73,6 +75,7 @@ const envNameByInput = { format: 'MODEL_EOL_FORMAT', feeds: 'MODEL_EOL_FEEDS', config: 'MODEL_EOL_CONFIG', + 'document-type': 'MODEL_EOL_DOCUMENT_TYPE', 'include-docs': 'MODEL_EOL_INCLUDE_DOCS', 'allow-incomplete': 'MODEL_EOL_ALLOW_INCOMPLETE', json: 'MODEL_EOL_JSON', @@ -131,6 +134,10 @@ const plan = runAction({ command: 'plan', paths: fixture }) assert.equal(plan.code, 0, `plan works with Action defaults: ${plan.err}`) assert.equal(parseJson(plan.out, 'plan output is JSON').plan_schema, 'model-eol.plan/0.1') +const validation = runAction({ command: 'validate', paths: 'feeds/openai.json\nfeeds/anthropic.json' }) +assert.equal(validation.code, 0, `validate accepts multiple public documents: ${validation.err}`) +assert.match(validation.out, /openai\.json: valid feed document/, 'validate reports the selected document type') + const cyclonedx = runAction({ command: 'inventory', paths: fixture, format: 'cyclonedx' }) assert.equal(cyclonedx.code, 0, `inventory accepts its command-specific format: ${cyclonedx.err}`) assert.equal(parseJson(cyclonedx.out, 'CycloneDX output is JSON').bomFormat, 'CycloneDX') @@ -141,12 +148,16 @@ assert.match(markdown.out, /^# model-eol alert/m, 'alert accepts its command-spe const jsonCheck = runAction({ command: 'check', paths: fixture, json: 'true' }) assert.equal(jsonCheck.code, 1, 'JSON check preserves finding exit code') -assert(Array.isArray(parseJson(jsonCheck.out, 'check --json emits JSON').findings), 'json input reaches the CLI') +assert.equal(parseJson(jsonCheck.out, 'check --json emits JSON').schema, 'model-eol/check@0.1', 'json input emits the public check discriminator') const invalidCommand = runAction({ command: 'apply', paths: fixture }) assert.equal(invalidCommand.code, 2, 'Action rejects commands outside its scanning contract') assert.match(invalidCommand.err, /command must be/, 'invalid command has an actionable diagnostic') +const wrongDocumentTypeCommand = runAction({ command: 'check', paths: fixture, 'document-type': 'feed' }) +assert.equal(wrongDocumentTypeCommand.code, 2, 'Action rejects document-type outside validate') +assert.match(wrongDocumentTypeCommand.err, /only supported by the validate command/, 'document-type error names its command') + const wrongFormatCommand = runAction({ command: 'schedule', paths: fixture, format: 'json' }) assert.equal(wrongFormatCommand.code, 2, 'Action rejects format for commands that do not accept it') @@ -169,6 +180,10 @@ const wrongJsonCommand = runAction({ command: 'plan', paths: fixture, json: 'tru assert.equal(wrongJsonCommand.code, 2, 'Action rejects redundant json input for plan') assert.match(wrongJsonCommand.err, /only supported by the check, inventory, schedule, and alert/, 'json error names supported commands') +const validateJsonCommand = runAction({ command: 'validate', paths: 'feeds/openai.json', json: 'true' }) +assert.equal(validateJsonCommand.code, 2, 'Action rejects json input for validate') +assert.match(validateJsonCommand.err, /only supported by the check, inventory, schedule, and alert/, 'validate json error names supported commands') + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-action-contract-')) try { const docsOnly = path.join(tempRoot, 'docs only') @@ -189,6 +204,13 @@ try { assert.equal(multilinePaths.code, 0, `newline-separated paths preserve spaces: ${multilinePaths.err}`) assert.deepEqual(parseJson(multilinePaths.out, 'multiline path inventory is JSON').targets, ['first path', 'second path']) + const dashPath = path.join(tempRoot, '--models') + fs.mkdirSync(dashPath) + fs.writeFileSync(path.join(dashPath, 'app.py'), 'MODEL = "o3-deep-research"\n') + const dashPathInventory = runAction({ command: 'inventory', paths: '--models\n' }, { cwd: tempRoot }) + assert.equal(dashPathInventory.code, 0, `dash-prefixed paths are separated from CLI options: ${dashPathInventory.err}`) + assert.equal(parseJson(dashPathInventory.out, 'dash path inventory is JSON').targets[0], '--models') + const outputPath = path.join(tempRoot, 'inventory.json') const outputInventory = runAction({ command: 'inventory', paths: 'first path\n', 'output-file': outputPath }, { cwd: tempRoot }) assert.equal(outputInventory.code, 0, `output-file succeeds for reports: ${outputInventory.err}`) @@ -199,6 +221,13 @@ try { assert.equal(outputCheck.code, 1, 'output-file does not mask the CLI finding exit code') assert.equal(fs.readFileSync(findingOutputPath, 'utf8'), outputCheck.out, 'finding output is still captured') + const checkJsonOutputPath = path.join(tempRoot, 'check.json') + const outputJsonCheck = runAction({ command: 'check', paths: 'first path\n', json: 'true', 'output-file': checkJsonOutputPath }, { cwd: tempRoot }) + assert.equal(outputJsonCheck.code, 1, 'Action captures a finding check JSON artifact without masking its exit code') + const validateCheck = runAction({ command: 'validate', paths: `${checkJsonOutputPath}\n`, 'document-type': 'check' }, { cwd: tempRoot }) + assert.equal(validateCheck.code, 0, `Action validates the emitted check artifact through explicit document-type selection: ${validateCheck.err}`) + assert.match(validateCheck.out, /valid check document/, 'Action reports the selected check document type') + const impossibleOutput = runAction({ command: 'inventory', paths: 'first path\n', diff --git a/scripts/test-document-validation.mjs b/scripts/test-document-validation.mjs new file mode 100644 index 0000000..cd1002c --- /dev/null +++ b/scripts/test-document-validation.mjs @@ -0,0 +1,431 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +import { + DOCUMENT_TYPES, + loadDocumentSchemaCatalog, + validateDocument, +} from '../lib/validate-document.mjs' +import { normalizeConfig } from '../lib/config.mjs' +import { loadFeeds } from '../lib/feeds.mjs' +import { validateJsonSchema } from '../lib/json-schema.mjs' +import { formatInventoryCycloneDX } from '../lib/reports.mjs' +import { validateFeed } from '../lib/validate-feed.mjs' + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const cli = path.join(root, 'check.mjs') +const fixture = path.join(root, 'test/fixture') +const canonicalBase = 'https://thossullivan.github.io/model-eol/schema/0.1/' + +const run = (args, { cwd = root } = {}) => { + const result = spawnSync(process.execPath, [cli, ...args], { + cwd, + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + }) + if (result.error) throw result.error + return { code: result.status, out: result.stdout, err: result.stderr } +} + +const writeJson = (file, value) => fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`) + +const validFeed = () => ({ + spec: 'model-eol/0.1', + publisher: 'test', + generated: '2026-08-18T00:00:00Z', + source: 'https://example.test/deprecations', + models: [{ id: 'test-model' }], +}) + +const referencesIn = value => { + if (!value || typeof value !== 'object') return [] + if (Array.isArray(value)) return value.flatMap(referencesIn) + return [ + ...(typeof value.$ref === 'string' ? [value.$ref] : []), + ...Object.values(value).flatMap(referencesIn), + ] +} + +const catalog = loadDocumentSchemaCatalog() +assert.equal(catalog.byType.size, DOCUMENT_TYPES.length, 'every public document schema loads') +assert.deepEqual([...catalog.byType.keys()], DOCUMENT_TYPES, 'the schema catalog covers every public document type') +for (const [type, schema] of catalog.byType) { + assert(schema.$id.startsWith(canonicalBase), `${type} has a canonical public schema ID`) + assert.equal(schema.$schema, 'http://json-schema.org/draft-07/schema#', `${type} uses the canonical Draft-07 dialect URI`) + for (const reference of referencesIn(schema)) { + assert(reference.startsWith('#/') || reference.startsWith(canonicalBase), `${type} uses only local or canonical cross-references`) + } +} +assert.equal(catalog.registry.assertReferences(), true, 'all local and cross-document references resolve') + +const conflictingReplacement = validFeed() +conflictingReplacement.models = [ + { id: 'old-model', replacement: 'new-model', replacement_options: ['other-model'] }, + { id: 'new-model' }, +] +const conflictingReplacementErrors = validateJsonSchema(conflictingReplacement, catalog.byType.get('feed')) +assert(conflictingReplacementErrors.some(error => error.path === '$.models[0].replacement_options' && error.keyword === 'falseSchema'), 'the standalone Draft-07 feed schema rejects replacement plus replacement_options') +assert(validateFeed(conflictingReplacement).some(error => error.message.includes('mutually exclusive')), 'the runtime feed validator independently rejects conflicting replacement guidance') +const replacementChoice = validFeed() +replacementChoice.models[0].replacement_options = ['external-choice'] +assert.equal(validateJsonSchema(replacementChoice, catalog.byType.get('feed')).length, 0, 'the standalone feed schema permits issue-only replacement options without a replacement') + +const duplicateDistribution = validFeed() +duplicateDistribution.models[0].distributions = [ + { via: 'aws-bedrock', shutdown: '2026-09-01' }, + { via: 'aws-bedrock', shutdown: '2027-09-01' }, +] +assert.equal(validateJsonSchema(duplicateDistribution, catalog.byType.get('feed')).length, 0, 'the portable Draft-07 schema leaves distributor-key uniqueness to semantic validation') +assert(validateFeed(duplicateDistribution).some(error => error.path === 'models[0].distributions[1].via' && error.message.includes('duplicate distributor via')), 'the runtime feed validator rejects ambiguous duplicate distributor clocks') + +const unprovenDistributionAnnouncement = validFeed() +delete unprovenDistributionAnnouncement.source +unprovenDistributionAnnouncement.models[0].distributions = [ + { via: 'aws-bedrock', announced: '2026-08-01' }, +] +assert(validateFeed(unprovenDistributionAnnouncement).some(error => error.path === 'models[0].distributions[0]' && error.message.includes('dated distribution needs a source')), 'the runtime feed validator requires provenance for announced-only distributor dates') + +const credentialEnvironmentNames = [ + 'GITHUB_TOKEN', 'gh_auth', 'Actions_Runtime_URL', 'ssh_auth_sock', + 'AWS_SECRET_ACCESS_KEY', 'MODEL_EOL_SECRET', 'database_password', 'credential_file', +] +for (const name of credentialEnvironmentNames) { + const errors = validateJsonSchema({ eval: { pass_env: [name] } }, catalog.byType.get('config')) + assert(errors.some(error => error.path === '$.eval.pass_env[0]' && error.keyword === 'pattern'), `the standalone config schema rejects credential-like pass_env name ${name}`) + assert.throws(() => normalizeConfig({ eval: { pass_env: [name] } }), new RegExp(name, 'i'), `the runtime config validator rejects credential-like pass_env name ${name}`) +} +assert.equal(validateJsonSchema({ eval: { pass_env: ['OPENAI_API_KEY', 'MODEL_EOL_REPORT', 'AWS_REGION'] } }, catalog.byType.get('config')).length, 0, 'the standalone config schema permits explicit non-credential eval inputs') + +const cycloneDxReference = ({ + file, + line, + usage, + requestedVia, + via, + status, + shutdown, + distributionStatus = null, +}) => ({ + file, + line, + matched: 'shared-model', + id: 'shared-model', + publisher: 'test-publisher', + usage, + requested_via: requestedVia, + via, + status, + shutdown, + date_precision: null, + distribution_status: distributionStatus, + safe_until: shutdown, + replacement: 'next-model', + replacement_options: null, + replacement_note: null, +}) +const mixedClockReferences = [ + cycloneDxReference({ file: 'z-direct.py', line: 9, usage: 'direct-api', requestedVia: null, via: 'publisher', status: 'retired', shutdown: '2026-08-01' }), + cycloneDxReference({ file: 'a-direct.py', line: 2, usage: 'model-reference', requestedVia: null, via: 'publisher', status: 'retiring', shutdown: '2026-08-01' }), + cycloneDxReference({ file: 'a-direct.py', line: 2, usage: 'model-reference', requestedVia: null, via: 'publisher', status: 'retiring', shutdown: '2026-08-01' }), + cycloneDxReference({ file: 'custom-publisher/service.ts', line: 3, usage: 'gateway', requestedVia: 'publisher', via: 'publisher', status: 'scheduled', shutdown: '2027-02-01' }), + cycloneDxReference({ file: 'azure/service.ts', line: 4, usage: 'cloud-provider', requestedVia: 'azure-ai-foundry', via: 'azure-ai-foundry', status: 'scheduled', shutdown: '2027-01-01' }), + cycloneDxReference({ file: 'bedrock/service.ts', line: 7, usage: 'cloud-provider', requestedVia: 'aws-bedrock', via: 'aws-bedrock', status: 'ok', shutdown: null, distributionStatus: 'extended-access' }), +] +const cycloneDxInventory = references => ({ + generated: '2026-08-18T00:00:00Z', + model_references: references, +}) +const mixedClockCycloneDx = formatInventoryCycloneDX(cycloneDxInventory(mixedClockReferences)) +const cycloneDxProperty = (component, name) => component.properties.find(property => property.name === name)?.value +const componentForChannel = channel => mixedClockCycloneDx.components.find(component => cycloneDxProperty(component, 'model-eol:lifecycle_channel') === channel) +assert.equal(mixedClockCycloneDx.bomFormat, 'CycloneDX', 'channel-qualified inventory remains a CycloneDX BOM') +assert.equal(mixedClockCycloneDx.specVersion, '1.6', 'channel-qualified inventory preserves the official CycloneDX 1.6 contract') +assert.equal(mixedClockCycloneDx.components.length, 4, 'one canonical model used through four lifecycle channels emits four components') +assert.equal(new Set(mixedClockCycloneDx.components.map(component => component['bom-ref'])).size, 4, 'channel-qualified component bom-refs are unique') +assert(mixedClockCycloneDx.components.every(component => component['bom-ref'].includes(`:${encodeURIComponent(cycloneDxProperty(component, 'model-eol:lifecycle_channel'))}`)), 'each component bom-ref carries its lifecycle channel') +assert.equal(cycloneDxProperty(componentForChannel('publisher-direct'), 'model-eol:status'), 'retired', 'direct publisher lifecycle does not inherit a distributor status') +assert.equal(cycloneDxProperty(componentForChannel('publisher-direct'), 'model-eol:shutdown'), '2026-08-01', 'direct publisher lifecycle keeps its own shutdown clock') +assert.equal(cycloneDxProperty(componentForChannel('publisher'), 'model-eol:shutdown'), '2027-02-01', 'a custom distribution literally named publisher does not collide with the direct publisher clock') +assert.equal(cycloneDxProperty(componentForChannel('azure-ai-foundry'), 'model-eol:shutdown'), '2027-01-01', 'Azure lifecycle keeps its separate shutdown clock') +assert.equal(cycloneDxProperty(componentForChannel('aws-bedrock'), 'model-eol:distribution_status'), 'extended-access', 'Bedrock lifecycle keeps its channel-specific distribution status') +assert.deepEqual(componentForChannel('publisher-direct').evidence.occurrences, [ + { location: 'a-direct.py#2' }, + { location: 'z-direct.py#9' }, +], 'component occurrences are sorted and deduplicated within their lifecycle channel') +assert.deepEqual( + formatInventoryCycloneDX(cycloneDxInventory([...mixedClockReferences].reverse())), + mixedClockCycloneDx, + 'CycloneDX component identities, lifecycle properties, and occurrences are independent of input order', +) + +const oneOfSchema = { + $id: 'https://example.test/one-of.schema.json', + oneOf: [{ type: 'string' }, { const: 'matches-both' }], +} +assert.equal(validateJsonSchema('matches-one', oneOfSchema).length, 0, 'oneOf accepts exactly one matching branch') +const noOneOfMatch = validateJsonSchema(42, oneOfSchema) +assert(noOneOfMatch.some(error => error.keyword === 'oneOf' && error.message.includes('matched 0')), 'oneOf rejects a value matching no branches') +const twoOneOfMatches = validateJsonSchema('matches-both', oneOfSchema) +assert(twoOneOfMatches.some(error => error.keyword === 'oneOf' && error.message.includes('matched 2')), 'oneOf rejects a value matching multiple branches') + +const schemaDependencySchema = { + $id: 'https://example.test/schema-dependency.schema.json', + type: 'object', + dependencies: { + trigger: { + required: ['peer'], + properties: { peer: { const: 'expected' } }, + }, + }, +} +const missingSchemaDependency = validateJsonSchema({ trigger: true }, schemaDependencySchema) +assert(missingSchemaDependency.some(error => error.path === '$.peer' && error.keyword === 'required'), 'schema-valued dependencies validate the whole containing object') +const invalidSchemaDependency = validateJsonSchema({ trigger: true, peer: 'wrong' }, schemaDependencySchema) +assert(invalidSchemaDependency.some(error => error.path === '$.peer' && error.keyword === 'const'), 'schema-valued dependencies enforce their nested assertions') +assert.equal(validateJsonSchema({ trigger: true, peer: 'expected' }, schemaDependencySchema).length, 0, 'schema-valued dependencies accept a conforming object') + +const schemaDependencyDocument = structuredClone(catalog.byType.get('config')) +schemaDependencyDocument.definitions.referenceRoute.dependencies.match = { required: ['model'] } +assert.equal(validateDocument(schemaDependencyDocument).errors.length, 0, 'the public schema linter accepts dependency schemas enforced by the runtime') +const unsupportedAssertionDocument = structuredClone(catalog.byType.get('inventory')) +unsupportedAssertionDocument.anyOf = [{ required: ['schema'] }] +const unsupportedAssertion = validateDocument(unsupportedAssertionDocument) +assert(unsupportedAssertion.errors.some(error => error.path === '$.anyOf' && error.keyword === 'anyOf'), 'the public schema linter rejects assertion keywords the runtime cannot enforce') + +const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-validation-')) +try { + const schemaPaths = [...catalog.byType.values()].map(schema => { + const name = new URL(schema.$id).pathname.split('/').at(-1) + return path.join(root, 'schema', name) + }) + for (const schemaPath of schemaPaths) { + const result = run(['validate', schemaPath]) + assert.equal(result.code, 0, `${path.basename(schemaPath)} passes public schema-definition validation: ${result.err}`) + assert.match(result.out, /valid schema document/, 'schema definition auto-selection is reported') + } + + const feedPaths = [] + for (const feedName of fs.readdirSync(path.join(root, 'feeds')).filter(file => file.endsWith('.json'))) { + const feedPath = path.join(root, 'feeds', feedName) + feedPaths.push(feedPath) + const result = run(['validate', feedPath]) + assert.equal(result.code, 0, `${feedName} validates through automatic feed selection: ${result.err}`) + assert.match(result.out, /valid feed document/, `${feedName} reports its selected type`) + } + const batch = run(['validate', ...schemaPaths, ...feedPaths]) + assert.equal(batch.code, 0, `one validate invocation accepts all schema and feed documents: ${batch.err}`) + assert.equal(batch.out.trim().split('\n').length, schemaPaths.length + feedPaths.length, 'batch validation reports every valid document') + + const mixedSpecFeeds = path.join(tempRoot, 'mixed-spec-feeds') + fs.mkdirSync(mixedSpecFeeds) + writeJson(path.join(mixedSpecFeeds, 'current.json'), validFeed()) + writeJson(path.join(mixedSpecFeeds, 'future.json'), { + ...validFeed(), + spec: 'model-eol/0.2', + publisher: 'future', + models: [{ id: 'future-model' }], + }) + assert.throws( + () => loadFeeds(mixedSpecFeeds), + error => error.message.includes('future.json') && error.message.includes('unsupported feed spec "model-eol/0.2"'), + 'feed loading fails closed when one file in a mixed directory uses an unsupported spec', + ) + const mixedSpecCheck = run([fixture, '--feeds', mixedSpecFeeds]) + assert.equal(mixedSpecCheck.code, 2, 'check exits 2 instead of scanning with a partially loaded mixed-spec feed directory') + assert.match(mixedSpecCheck.err, /failed to load feeds.*unsupported feed spec "model-eol\/0\.2"/s, 'check names the unsupported feed spec instead of silently skipping it') + + const configPath = path.join(tempRoot, '.model-eol.json') + writeJson(configPath, { + days: 90, + scope: 'direct', + routes: [{ paths: ['services/**'], via: 'aws-bedrock' }], + }) + const autoConfig = run(['validate', configPath]) + assert.equal(autoConfig.code, 0, `the canonical config filename selects the config schema: ${autoConfig.err}`) + assert.match(autoConfig.out, /valid config document/, 'config validation identifies the document type') + + const namedConfigPath = path.join(tempRoot, 'policy.json') + writeJson(namedConfigPath, {}) + const explicitConfig = run(['validate', namedConfigPath, '--type', 'config']) + assert.equal(explicitConfig.code, 0, `explicit schema selection validates an empty config: ${explicitConfig.err}`) + + const generated = new Map() + for (const [type, args, expectedCode] of [ + ['check', ['check', fixture, '--json'], 1], + ['inventory', ['inventory', fixture, '--json'], 0], + ['schedule', ['schedule', fixture, '--json'], 0], + ['alert', ['alert', fixture, '--json'], 1], + ['plan', ['plan', fixture], 0], + ]) { + const report = run(args) + assert.equal(report.code, expectedCode, `${type} fixture report is emitted with its established exit code: ${report.err}`) + const file = path.join(tempRoot, `${type}.json`) + fs.writeFileSync(file, report.out) + generated.set(type, JSON.parse(report.out)) + const validation = run(['validate', file]) + assert.equal(validation.code, 0, `emitted ${type} report conforms to its public schema: ${validation.err}`) + assert.match(validation.out, new RegExp(`valid ${type} document`), `${type} is automatically selected by discriminator`) + } + + assert.equal(generated.get('check').schema, 'model-eol/check@0.1', 'check --json emits its stable public discriminator') + const strictCheckFinding = structuredClone(generated.get('check')) + strictCheckFinding.findings[0].unexpected = true + assert(validateDocument(strictCheckFinding, { type: 'check' }).errors.some(error => error.path === '$.findings[0].unexpected' && error.keyword === 'additionalProperties'), 'check findings reject unknown nested fields') + const artifactInventory = JSON.parse(run(['inventory', tempRoot, '--json']).out) + assert(!artifactInventory.model_references.some(reference => reference.file.endsWith('/check.json')), 'a generated check report is not re-ingested as repository model usage') + assert(artifactInventory.scan_notes.some(note => note.reason === 'model-eol-document-skipped' && note.file.endsWith('/check.json')), 'scanner records the generated check report as an intentional product artifact skip') + + const nestedExtraCases = [ + ['model reference', 'model_references', generated.get('inventory').model_references, 0], + ['candidate reference', 'candidate_model_references', generated.get('inventory').candidate_model_references, 0], + ['integration hint', 'integration_hints', generated.get('inventory').integration_hints, 0], + ] + for (const [label, property, values, index] of nestedExtraCases) { + assert(values.length > index, `fixture emits a ${label} for strictness testing`) + const report = structuredClone(generated.get('inventory')) + report[property][index].unexpected = true + const result = validateDocument(report, { type: 'inventory' }) + assert(result.errors.some(error => error.path === `$.${property}[${index}].unexpected` && error.keyword === 'additionalProperties'), `${label} rejects unknown nested fields`) + } + const nestedScanNote = structuredClone(generated.get('inventory')) + nestedScanNote.scan_notes = [{ reason: 'test-note', unexpected: true }] + const nestedScanNoteResult = validateDocument(nestedScanNote, { type: 'inventory' }) + assert(nestedScanNoteResult.errors.some(error => error.path === '$.scan_notes[0].unexpected' && error.keyword === 'additionalProperties'), 'scan notes reject unknown nested fields') + + const emptyModelIdentifier = structuredClone(generated.get('inventory')) + emptyModelIdentifier.model_references[0].id = '' + assert(validateDocument(emptyModelIdentifier, { type: 'inventory' }).errors.some(error => error.path === '$.model_references[0].id' && error.keyword === 'minLength'), 'emitted model identifiers must be non-empty') + const emptyIntegrationPath = structuredClone(generated.get('inventory')) + emptyIntegrationPath.integration_hints[0].file = '' + assert(validateDocument(emptyIntegrationPath, { type: 'inventory' }).errors.some(error => error.path === '$.integration_hints[0].file' && error.keyword === 'minLength'), 'emitted integration paths must be non-empty') + + const strictScheduleItem = structuredClone(generated.get('schedule')) + strictScheduleItem.items[0].unexpected = true + assert(validateDocument(strictScheduleItem, { type: 'schedule' }).errors.some(error => error.path === '$.items[0].unexpected' && error.keyword === 'additionalProperties'), 'schedule cross-references retain strict model-reference validation') + const strictScheduleCandidate = structuredClone(generated.get('schedule')) + strictScheduleCandidate.candidate_model_references[0].unexpected = true + assert(validateDocument(strictScheduleCandidate, { type: 'schedule' }).errors.some(error => error.path === '$.candidate_model_references[0].unexpected' && error.keyword === 'additionalProperties'), 'schedule cross-references retain strict candidate validation') + + for (const kind of ['candidate-model-reference', 'integration-hint']) { + const strictAlertWarning = structuredClone(generated.get('alert')) + const index = strictAlertWarning.warnings.findIndex(item => item.kind === kind) + assert(index >= 0, `fixture alert emits a ${kind} warning for strictness testing`) + strictAlertWarning.warnings[index].unexpected = true + const result = validateDocument(strictAlertWarning, { type: 'alert' }) + assert(result.errors.some(error => error.path === `$.warnings[${index}]` && error.keyword === 'oneOf'), `alert ${kind} warnings reject unknown nested fields`) + } + + const malformedPath = path.join(tempRoot, 'malformed.json') + fs.writeFileSync(malformedPath, '{"spec":') + const malformed = run(['validate', malformedPath]) + assert.equal(malformed.code, 2, 'malformed JSON exits 2') + assert.match(malformed.err, /invalid JSON/, 'malformed JSON has a precise parse diagnostic') + + const invalidUtf8Path = path.join(tempRoot, 'invalid-utf8.json') + fs.writeFileSync(invalidUtf8Path, Buffer.from([0x7b, 0x22, 0xff, 0x22, 0x3a, 0x31, 0x7d])) + const invalidUtf8 = run(['validate', invalidUtf8Path]) + assert.equal(invalidUtf8.code, 2, 'invalid UTF-8 exits 2') + assert.match(invalidUtf8.err, /invalid UTF-8/, 'invalid UTF-8 is refused without replacement decoding') + + const unknownPath = path.join(tempRoot, 'unknown.json') + writeJson(unknownPath, { hello: 'world' }) + const unknown = run(['validate', unknownPath]) + assert.equal(unknown.code, 2, 'an unknown automatic document type exits 2') + assert.match(unknown.err, /could not determine the document type/, 'unknown automatic type recommends explicit selection') + const explicitUnknown = run(['validate', unknownPath, '--type', 'mystery']) + assert.equal(explicitUnknown.code, 2, 'an unknown explicit type exits 2') + assert.match(explicitUnknown.err, /unknown document type/, 'unknown explicit type is named') + + const extraFieldPath = path.join(tempRoot, 'extra-field.json') + writeJson(extraFieldPath, { ...validFeed(), unexpected: true }) + const extraField = run(['validate', extraFieldPath]) + assert.equal(extraField.code, 2, 'additional properties are rejected with exit 2') + assert.match(extraField.err, /\$\.unexpected: is not allowed/, 'additionalProperties reports the exact field path') + + const invalidDatePath = path.join(tempRoot, 'invalid-date.json') + const invalidDate = validFeed() + invalidDate.models[0].shutdown = '2026-02-30' + writeJson(invalidDatePath, invalidDate) + const invalidDateResult = run(['validate', invalidDatePath]) + assert.equal(invalidDateResult.code, 2, 'an impossible date exits 2') + assert.match(invalidDateResult.err, /\$\.models\[0\]\.shutdown: must match format date/, 'date validation reports the exact model field') + + const invalidDateTimePath = path.join(tempRoot, 'invalid-date-time.json') + writeJson(invalidDateTimePath, { ...validFeed(), generated: '2026-02-30T00:00:00Z' }) + const invalidDateTime = run(['validate', invalidDateTimePath]) + assert.equal(invalidDateTime.code, 2, 'an impossible date-time exits 2') + assert.match(invalidDateTime.err, /\$\.generated: must match format date-time/, 'date-time validation reports the exact metadata field') + + const aggregated = run(['validate', feedPaths[0], malformedPath, invalidDateTimePath]) + assert.equal(aggregated.code, 2, 'batch validation exits 2 when any document fails') + assert.match(aggregated.out, /valid feed document/, 'batch validation still reports valid peers') + assert(aggregated.err.includes(malformedPath) && aggregated.err.includes(invalidDateTimePath), 'batch validation aggregates every document failure') + + const unresolvedReplacementPath = path.join(tempRoot, 'unresolved-replacement.json') + const unresolvedReplacement = validFeed() + unresolvedReplacement.models[0].replacement = 'missing-model' + writeJson(unresolvedReplacementPath, unresolvedReplacement) + const unresolved = run(['validate', unresolvedReplacementPath]) + assert.equal(unresolved.code, 2, 'feed semantic errors exit 2 after schema conformance') + assert.match(unresolved.err, /does not resolve to an id or alias in this feed/, 'the public command invokes the strict runtime feed validator') + + const duplicateDistributionPath = path.join(tempRoot, 'duplicate-distribution.json') + writeJson(duplicateDistributionPath, duplicateDistribution) + const duplicateDistributionResult = run(['validate', duplicateDistributionPath]) + assert.equal(duplicateDistributionResult.code, 2, 'duplicate distributor clocks fail public CLI validation') + assert.match(duplicateDistributionResult.err, /duplicate distributor via "aws-bedrock"/, 'CLI validation names the ambiguous distributor clock') + + const unprovenDistributionPath = path.join(tempRoot, 'unproven-distribution-announcement.json') + writeJson(unprovenDistributionPath, unprovenDistributionAnnouncement) + const unprovenDistributionResult = run(['validate', unprovenDistributionPath]) + assert.equal(unprovenDistributionResult.code, 2, 'announced-only distributor dates without provenance fail public CLI validation') + assert.match(unprovenDistributionResult.err, /dated distribution needs a source/, 'CLI validation explains missing distributor date provenance') + + const unsafeConfigPath = path.join(tempRoot, 'unsafe-config.json') + writeJson(unsafeConfigPath, { eval: { pass_env: ['GITHUB_TOKEN'] } }) + const unsafeConfig = run(['validate', unsafeConfigPath, '--type', 'config']) + assert.equal(unsafeConfig.code, 2, 'runtime config restrictions are part of public validation') + assert.match(unsafeConfig.err, /\$\.eval\.pass_env\[0\]: must match pattern/, 'public validation enforces credential-name refusal in the portable schema') + + const missingRouteModelPath = path.join(tempRoot, 'missing-route-model.json') + writeJson(missingRouteModelPath, { routes: [{ paths: ['**'], via: 'aws-bedrock', match: 'chat-prod' }] }) + const missingRouteModel = run(['validate', missingRouteModelPath, '--type', 'config']) + assert.equal(missingRouteModel.code, 2, 'Draft-07 property dependencies are enforced') + assert.match(missingRouteModel.err, /\.model: is required when match is present/, 'dependency errors identify the missing peer') + + const crossedSchedule = structuredClone(generated.get('schedule')) + crossedSchedule.scan_notes = [{ reason: 'test', unexpected: true }] + const crossed = validateDocument(crossedSchedule, { type: 'schedule' }) + assert.equal(crossed.errors.length, 1, 'external inventory references validate nested schedule values') + assert.equal(crossed.errors[0].path, '$.scan_notes[0].unexpected', 'cross-reference errors preserve the nested instance path') + + const invalidAlert = structuredClone(generated.get('alert')) + invalidAlert.warnings = [{}] + const alertOneOf = validateDocument(invalidAlert, { type: 'alert' }) + assert(alertOneOf.errors.some(error => error.path === '$.warnings[0]' && error.keyword === 'oneOf'), 'alert warning unions enforce oneOf through public cross-references') + + const wrongPlanType = structuredClone(generated.get('plan')) + wrongPlanType.threshold_days = '90' + const wrongPlan = validateDocument(wrongPlanType, { type: 'plan' }) + assert(wrongPlan.errors.some(error => error.path === '$.threshold_days' && error.keyword === 'type'), 'strict plan validation rejects the wrong root field type') + + const noOpPlan = structuredClone(generated.get('plan')) + assert(noOpPlan.items.length > 0, 'fixture emits a migration item for semantic plan validation') + noOpPlan.items[0].replacement = noOpPlan.items[0].matched + const noOpPlanResult = validateDocument(noOpPlan, { type: 'plan' }) + assert(noOpPlanResult.errors.some(error => error.keyword === 'model-eol-plan' && error.message.includes('must differ')), 'public plan validation rejects semantic no-op replacements') + + const missing = run(['validate', path.join(tempRoot, 'missing.json')]) + assert.equal(missing.code, 2, 'an unreadable document exits 2') + assert.match(missing.err, /could not read/, 'an unreadable document names the read failure') +} finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) +} + +console.log('document validation assertions passed') diff --git a/scripts/test-eval-harness.mjs b/scripts/test-eval-harness.mjs new file mode 100644 index 0000000..481444a --- /dev/null +++ b/scripts/test-eval-harness.mjs @@ -0,0 +1,148 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +const root = path.resolve(import.meta.dirname, '..') +const harness = path.join(root, 'examples', 'model-eol-eval.mjs') +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-eval-harness-')) +const planPath = path.join(temp, 'plan.json') +const reportPath = path.join(temp, 'report.md') +const packagePath = path.join(temp, 'package.json') +const checker = path.join(root, 'check.mjs') +const bot = path.join(root, 'bot', 'bot.mjs') + +const plan = { + plan_schema: 'model-eol.plan/0.1', + items: [{ + file: 'client.mjs', + id: 'old-model', + replacement: 'new-model', + }], +} + +const runHarness = () => spawnSync(process.execPath, [harness], { + cwd: temp, + env: { + ...process.env, + MODEL_EOL_OLD_ID: 'old-model', + MODEL_EOL_NEW_ID: 'new-model', + MODEL_EOL_PLAN: planPath, + MODEL_EOL_REPORT: reportPath, + }, + encoding: 'utf8', +}) + +try { + fs.writeFileSync(planPath, `${JSON.stringify(plan)}\n`) + fs.writeFileSync(path.join(temp, 'verify.mjs'), 'console.log("private test output")\n') + fs.writeFileSync(packagePath, JSON.stringify({ + name: 'model-eol-eval-harness-test', + private: true, + scripts: { 'eval:model-eol': 'node verify.mjs' }, + })) + + const passing = runHarness() + assert.equal(passing.status, 0, passing.stderr) + const passingReport = fs.readFileSync(reportPath, 'utf8') + assert.match(passingReport, /Result: pass/) + assert.match(passingReport, /old-model.*new-model/) + assert(!passingReport.includes('private test output'), 'publishable report excludes arbitrary test output') + + fs.writeFileSync(path.join(temp, 'verify.mjs'), 'console.error("secret-like failure output"); process.exit(7)\n') + const failing = runHarness() + assert.equal(failing.status, 1) + const failingReport = fs.readFileSync(reportPath, 'utf8') + assert.match(failingReport, /Result: fail/) + assert.match(failingReport, /Exit code: 7/) + assert(!failingReport.includes('secret-like failure output'), 'failure report excludes arbitrary stderr') + + plan.items[0].id = 'different-model' + fs.writeFileSync(planPath, `${JSON.stringify(plan)}\n`) + const crossed = runHarness() + assert.equal(crossed.status, 1) + assert.match(fs.readFileSync(reportPath, 'utf8'), /outside MODEL_EOL_OLD_ID/) + + const uatRepo = path.join(temp, 'self-named-model-eol') + const invocationDir = path.join(temp, 'external-invocation') + const uatPlanPath = path.join(invocationDir, 'plan.json') + const uatEvalPath = path.join(invocationDir, 'eval.json') + fs.mkdirSync(path.join(uatRepo, 'scripts'), { recursive: true }) + fs.mkdirSync(invocationDir) + fs.copyFileSync(harness, path.join(uatRepo, 'scripts', 'model-eol-eval.mjs')) + fs.writeFileSync(path.join(uatRepo, 'package.json'), `${JSON.stringify({ + name: 'model-eol', + private: true, + scripts: { 'eval:model-eol': 'node verify-model-swap.mjs' }, + }, null, 2)}\n`) + fs.writeFileSync(path.join(uatRepo, '.model-eol.json'), '{"eval":{"command":"node scripts/model-eol-eval.mjs"}}\n') + fs.writeFileSync(path.join(uatRepo, 'app.mjs'), [ + 'import OpenAI from "openai"', + 'const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })', + 'export const model = "o3-deep-research"', + '', + ].join('\n')) + fs.writeFileSync(path.join(uatRepo, 'verify-model-swap.mjs'), [ + "import fs from 'node:fs'", + "const source = fs.readFileSync(new URL('./app.mjs', import.meta.url), 'utf8')", + "if (source.includes(process.env.MODEL_EOL_OLD_ID) || !source.includes(process.env.MODEL_EOL_NEW_ID)) process.exit(1)", + '', + ].join('\n')) + for (const args of [ + ['init', '-q', '-b', 'main'], + ['config', 'user.name', 'model-eol local UAT'], + ['config', 'user.email', 'local-uat@example.invalid'], + ['add', '.'], + ['commit', '-q', '-m', 'self-named local UAT fixture'], + ]) { + const result = spawnSync('git', args, { cwd: uatRepo, encoding: 'utf8' }) + assert.equal(result.status, 0, result.stderr) + } + + const planned = spawnSync('bash', ['-c', + 'cd "$MODEL_EOL_UAT_REPO" && "$MODEL_EOL_UAT_NODE" "$MODEL_EOL_UAT_CHECKER" plan . --days 90 --scope direct', + ], { + cwd: invocationDir, + env: { + ...process.env, + MODEL_EOL_UAT_REPO: uatRepo, + MODEL_EOL_UAT_NODE: process.execPath, + MODEL_EOL_UAT_CHECKER: checker, + }, + encoding: 'utf8', + }) + assert.equal(planned.status, 0, planned.stderr) + fs.writeFileSync(uatPlanPath, planned.stdout) + const uatPlan = JSON.parse(planned.stdout) + assert.equal(uatPlan.items.length, 1) + assert.equal(path.isAbsolute(uatPlan.items[0].file), false, 'external invocation plans clone-portable repository-relative paths') + + const evaluated = spawnSync(process.execPath, [ + bot, 'evaluate', + '--target-dir', uatRepo, + '--plan-file', uatPlanPath, + '--output-file', uatEvalPath, + ], { cwd: invocationDir, encoding: 'utf8' }) + assert.equal(evaluated.status, 0, evaluated.stderr) + const uatEval = JSON.parse(fs.readFileSync(uatEvalPath, 'utf8')) + assert.equal(uatEval.results.length, 1) + assert.equal(uatEval.results[0].status, 'pass') + + const authorized = spawnSync(process.execPath, [ + bot, '--dry-run', '--target-dir', uatRepo, '--repo', 'example/local-uat', + ], { + cwd: invocationDir, + env: { ...process.env, MODEL_EOL_EVAL_RESULTS_FILE: uatEvalPath }, + encoding: 'utf8', + }) + assert.equal(authorized.status, 0, authorized.stderr) + assert.match(authorized.stdout, /- create openai\/o3-deep-research/) + assert.match(authorized.stdout, /Result: pass \(exit code 0\)/) +} finally { + fs.rmSync(temp, { recursive: true, force: true }) +} + +console.log('eval harness contract tests passed') diff --git a/scripts/test-feed-changelog.mjs b/scripts/test-feed-changelog.mjs index b25fffe..3059bbc 100644 --- a/scripts/test-feed-changelog.mjs +++ b/scripts/test-feed-changelog.mjs @@ -103,6 +103,8 @@ try { assert(atom.includes('test: 1 shutdown change, 1 model added'), 'entry title includes shutdown and added-model counts') assert(atom.includes('initial import'), 'initial feed import is represented') assert(atom.includes('<pre>') && !atom.includes('
'), 'Atom content wrapper is XML-escaped')
+  assert(atom.includes('model-eol maintainers'), 'Atom feed identifies its author')
+  assert(atom.includes(''), 'Atom feed publishes its canonical subscription URL')
 
   for (const tag of ['feed', 'entry', 'title', 'updated']) {
     const opening = (atom.match(new RegExp(`<${tag}(?:\\s|>)`, 'g')) ?? []).length
diff --git a/scripts/test-public-site.mjs b/scripts/test-public-site.mjs
new file mode 100644
index 0000000..7ee833e
--- /dev/null
+++ b/scripts/test-public-site.mjs
@@ -0,0 +1,342 @@
+#!/usr/bin/env node
+
+import assert from 'node:assert/strict'
+import crypto from 'node:crypto'
+import fs from 'node:fs'
+import http from 'node:http'
+import os from 'node:os'
+import path from 'node:path'
+import { spawnSync } from 'node:child_process'
+
+import {
+  createFeedRefreshReceipt,
+  validateFeedRefreshReceipt,
+} from './feed-refresh-receipt.mjs'
+import { verifyPublicSite } from './verify-public-site.mjs'
+
+const root = path.resolve(import.meta.dirname, '..')
+const builder = path.join(root, 'scripts', 'build-public-site.mjs')
+const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-public-site-'))
+const repo = path.join(temp, 'repo')
+const output = path.join(temp, 'site')
+const receiptFile = path.join(temp, 'feed-refresh-receipt.json')
+const checkedAt = '2026-08-18T12:34:56Z'
+const refreshRun = 'https://github.com/thossullivan/model-eol/actions/runs/123456'
+let server = null
+
+const runGit = args => {
+  const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8' })
+  assert.equal(result.status, 0, result.stderr || result.stdout)
+  return result.stdout.trim()
+}
+
+fs.cpSync(root, repo, {
+  recursive: true,
+  filter: source => {
+    const relative = path.relative(root, source)
+    const first = relative.split(path.sep)[0]
+    return first !== '.git' && first !== 'node_modules'
+  },
+})
+runGit(['init', '--quiet'])
+runGit(['add', '--all'])
+runGit(['-c', 'user.name=model-eol test', '-c', 'user.email=model-eol@example.test', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'public contract fixture'])
+const sourceSha = runGit(['rev-parse', 'HEAD'])
+
+const runBuilder = args => spawnSync(process.execPath, [builder, ...args, '--repo-dir', repo], {
+  cwd: repo,
+  encoding: 'utf8',
+})
+
+try {
+  const receipt = createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt,
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+  })
+  assert.equal(receipt.changed, false)
+  assert.equal(receipt.pending, false)
+  fs.writeFileSync(receiptFile, `${JSON.stringify(receipt, null, 2)}\n`)
+  const receiptCheck = validateFeedRefreshReceipt(receipt, {
+    repoDir: repo,
+    expectedRunUrl: refreshRun,
+    expectedRefreshSha: sourceSha,
+  })
+  assert.equal(receiptCheck.matches, true)
+  assert.deepEqual(receipt.feeds.map(feed => feed.path), ['feeds/amazon.json', 'feeds/anthropic.json', 'feeds/google.json', 'feeds/openai.json'])
+
+  const pending = createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'pending',
+    checkedAt,
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+    pendingPrUrl: 'https://github.com/thossullivan/model-eol/pull/123',
+  })
+  assert.equal(pending.changed, true)
+  assert.equal(pending.pending, true)
+  assert.equal(validateFeedRefreshReceipt(pending, { repoDir: repo }).matches, true)
+
+  const result = runBuilder([
+    '--out-dir', output,
+    '--receipt', receiptFile,
+    '--source-sha', sourceSha,
+  ])
+  assert.equal(result.status, 0, result.stderr)
+
+  const schemaNames = fs.readdirSync(path.join(repo, 'schema')).filter(name => name.endsWith('.json')).sort()
+  const feedNames = fs.readdirSync(path.join(repo, 'feeds')).filter(name => name.endsWith('.json')).sort()
+  assert(schemaNames.includes('model-eol.check.schema.json'))
+  assert.deepEqual(fs.readdirSync(path.join(output, 'schema', '0.1')).sort(), schemaNames)
+  assert.deepEqual(fs.readdirSync(path.join(output, 'feeds')).sort(), feedNames)
+
+  const health = JSON.parse(fs.readFileSync(path.join(output, 'health.json'), 'utf8'))
+  assert.equal(health.schema, 'model-eol/health@0.1')
+  assert.equal(health.last_checked, checkedAt)
+  assert.equal(health.refresh_run, refreshRun)
+  assert.equal(health.refresh_commit, sourceSha)
+  assert.equal(health.published_commit, sourceSha)
+  assert.deepEqual(health.feeds.map(feed => feed.publisher).sort(), ['amazon', 'anthropic', 'google', 'openai'])
+  for (const feed of health.feeds) {
+    const name = `${feed.publisher}.json`
+    const bytes = fs.readFileSync(path.join(output, 'feeds', name))
+    assert.equal(feed.sha256, crypto.createHash('sha256').update(bytes).digest('hex'))
+    assert.equal(feed.url, `https://thossullivan.github.io/model-eol/feeds/${name}`)
+    assert.equal(feed.sha256, receipt.feeds.find(item => item.path === `feeds/${name}`).sha256)
+  }
+
+  const publication = JSON.parse(fs.readFileSync(path.join(output, 'index.json'), 'utf8'))
+  assert.equal(publication.schema, 'model-eol/publication@0.1')
+  assert.equal(publication.published_commit, sourceSha)
+  assert.equal(publication.schemas.length, schemaNames.length)
+  assert(publication.schemas.every(item => item.id === item.url && item.url.startsWith('https://thossullivan.github.io/model-eol/schema/0.1/')))
+  assert.equal(publication.atom, 'https://thossullivan.github.io/model-eol/changelog.atom')
+  assert.equal(publication.health, 'https://thossullivan.github.io/model-eol/health.json')
+
+  const atom = fs.readFileSync(path.join(output, 'changelog.atom'), 'utf8')
+  assert.match(atom, //)
+  assert.match(fs.readFileSync(path.join(output, 'index.html'), 'utf8'), /model-eol public data/)
+
+  const dirtyOutput = runBuilder(['--out-dir', output, '--receipt', receiptFile])
+  assert.equal(dirtyOutput.status, 1)
+  assert.match(dirtyOutput.stderr, /output directory must be empty/)
+
+  const mismatched = structuredClone(receipt)
+  mismatched.feeds[0].sha256 = '0'.repeat(64)
+  assert.equal(validateFeedRefreshReceipt(mismatched, { repoDir: repo }).matches, false)
+  const mismatchedFile = path.join(temp, 'mismatched-receipt.json')
+  fs.writeFileSync(mismatchedFile, `${JSON.stringify(mismatched, null, 2)}\n`)
+  const mismatchedBuild = runBuilder([
+    '--out-dir', path.join(temp, 'mismatched-site'),
+    '--receipt', mismatchedFile,
+  ])
+  assert.equal(mismatchedBuild.status, 1)
+  assert.match(mismatchedBuild.stderr, /receipt does not match the feeds being published/)
+
+  const wrongSourceBuild = runBuilder([
+    '--out-dir', path.join(temp, 'wrong-source-site'),
+    '--receipt', receiptFile,
+    '--source-sha', 'f'.repeat(40),
+  ])
+  assert.equal(wrongSourceBuild.status, 1)
+  assert.match(wrongSourceBuild.stderr, /does not match repository HEAD/)
+
+  const schemaInput = path.join(repo, 'schema', 'model-eol.schema.json')
+  const schemaBytes = fs.readFileSync(schemaInput)
+  fs.writeFileSync(schemaInput, Buffer.concat([schemaBytes, Buffer.from('\n')]))
+  const dirtySchemaBuild = runBuilder([
+    '--out-dir', path.join(temp, 'dirty-schema-site'),
+    '--receipt', receiptFile,
+    '--source-sha', sourceSha,
+  ])
+  fs.writeFileSync(schemaInput, schemaBytes)
+  assert.equal(dirtySchemaBuild.status, 1)
+  assert.match(dirtySchemaBuild.stderr, /published schema\/feed inputs are dirty relative to/)
+
+  const feedInput = path.join(repo, 'feeds', 'amazon.json')
+  const feedBytes = fs.readFileSync(feedInput)
+  fs.writeFileSync(feedInput, Buffer.concat([feedBytes, Buffer.from('\n')]))
+  const dirtyFeedReceipt = createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt,
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+  })
+  const dirtyFeedReceiptFile = path.join(temp, 'dirty-feed-receipt.json')
+  fs.writeFileSync(dirtyFeedReceiptFile, `${JSON.stringify(dirtyFeedReceipt, null, 2)}\n`)
+  const dirtyFeedBuild = runBuilder([
+    '--out-dir', path.join(temp, 'dirty-feed-site'),
+    '--receipt', dirtyFeedReceiptFile,
+    '--source-sha', sourceSha,
+  ])
+  fs.writeFileSync(feedInput, feedBytes)
+  assert.equal(dirtyFeedBuild.status, 1)
+  assert.match(dirtyFeedBuild.stderr, /published schema\/feed inputs are dirty relative to/)
+
+  const untrackedFeed = path.join(repo, 'feeds', 'untracked.json')
+  fs.copyFileSync(feedInput, untrackedFeed)
+  const untrackedReceipt = createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt,
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+  })
+  const untrackedReceiptFile = path.join(temp, 'untracked-feed-receipt.json')
+  fs.writeFileSync(untrackedReceiptFile, `${JSON.stringify(untrackedReceipt, null, 2)}\n`)
+  const untrackedFeedBuild = runBuilder([
+    '--out-dir', path.join(temp, 'untracked-feed-site'),
+    '--receipt', untrackedReceiptFile,
+    '--source-sha', sourceSha,
+  ])
+  fs.unlinkSync(untrackedFeed)
+  assert.equal(untrackedFeedBuild.status, 1)
+  assert.match(untrackedFeedBuild.stderr, /published schema\/feed inputs are dirty relative to/)
+
+  assert.throws(() => createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt: 'yesterday',
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+  }), /valid UTC ISO 8601 instant/)
+  assert.throws(() => createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt: '2026-02-30T12:00:00Z',
+    refreshRunUrl: refreshRun,
+    refreshSha: sourceSha,
+  }), /valid UTC ISO 8601 instant/)
+  assert.throws(() => createFeedRefreshReceipt({
+    repoDir: repo,
+    state: 'clean',
+    checkedAt,
+    refreshRunUrl: 'http://example.test/run',
+    refreshSha: sourceSha,
+  }), /valid HTTPS URL/)
+
+  let tamperHealth = false
+  let redirectHealth = false
+  server = http.createServer((request, response) => {
+    const pathname = new URL(request.url, 'http://127.0.0.1').pathname
+    const relative = decodeURIComponent(pathname).replace(/^\/+/, '')
+    const file = path.resolve(output, relative)
+    if (!file.startsWith(`${path.resolve(output)}${path.sep}`) || !fs.existsSync(file) || !fs.statSync(file).isFile()) {
+      response.writeHead(404).end('not found')
+      return
+    }
+    if (redirectHealth && relative === 'health.json') {
+      response.writeHead(302, { location: '/index.json' }).end()
+      return
+    }
+    if (tamperHealth && relative === 'health.json') {
+      response.writeHead(200).end(`${JSON.stringify({ ...health, last_checked: '2026-08-18T12:34:57Z' }, null, 2)}\n`)
+      return
+    }
+    response.writeHead(200).end(fs.readFileSync(file))
+  })
+  await new Promise((resolve, reject) => {
+    server.once('error', reject)
+    server.listen(0, '127.0.0.1', resolve)
+  })
+  const address = server.address()
+  const canonicalBase = 'https://thossullivan.github.io/model-eol/'
+  const fetchLocalContract = async (url, options) => {
+    const requested = new URL(url)
+    const relative = requested.pathname.slice('/model-eol/'.length)
+    const response = await fetch(`http://127.0.0.1:${address.port}/${relative}`, options)
+    return {
+      ok: response.ok,
+      status: response.status,
+      redirected: response.redirected,
+      url: requested.href,
+      arrayBuffer: () => response.arrayBuffer(),
+    }
+  }
+  const live = await verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: output,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  })
+  assert.deepEqual({ feeds: live.feeds, schemas: live.schemas }, { feeds: feedNames.length, schemas: schemaNames.length })
+  assert.equal(live.lastChecked, checkedAt)
+  assert.equal(live.refreshCommit, sourceSha)
+  assert.equal(live.publishedCommit, sourceSha)
+
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: output,
+    fetchImpl: async (url, options) => ({
+      ...await fetchLocalContract(url, options),
+      url: 'https://example.test/redirected.json',
+    }),
+    attempts: 1,
+    retryMs: 0,
+  }), /health\.json did not match the exact built artifact/)
+
+  tamperHealth = true
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: output,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  }), /health\.json did not match the exact built artifact/)
+  tamperHealth = false
+  redirectHealth = true
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: output,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  }), /health\.json did not match the exact built artifact/)
+  redirectHealth = false
+
+  const omittedFeedSite = path.join(temp, 'omitted-feed-site')
+  fs.cpSync(output, omittedFeedSite, { recursive: true })
+  const omittedHealth = JSON.parse(fs.readFileSync(path.join(omittedFeedSite, 'health.json'), 'utf8'))
+  const omittedIndex = JSON.parse(fs.readFileSync(path.join(omittedFeedSite, 'index.json'), 'utf8'))
+  omittedHealth.feeds.pop()
+  omittedIndex.feeds.pop()
+  fs.writeFileSync(path.join(omittedFeedSite, 'health.json'), `${JSON.stringify(omittedHealth, null, 2)}\n`)
+  fs.writeFileSync(path.join(omittedFeedSite, 'index.json'), `${JSON.stringify(omittedIndex, null, 2)}\n`)
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: omittedFeedSite,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  }), /health does not enumerate every built feed exactly once/)
+
+  const wrongHealthSite = path.join(temp, 'wrong-health-site')
+  fs.cpSync(output, wrongHealthSite, { recursive: true })
+  const wrongHealthIndex = JSON.parse(fs.readFileSync(path.join(wrongHealthSite, 'index.json'), 'utf8'))
+  wrongHealthIndex.health = 'https://thossullivan.github.io/model-eol/index.json'
+  fs.writeFileSync(path.join(wrongHealthSite, 'index.json'), `${JSON.stringify(wrongHealthIndex, null, 2)}\n`)
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: canonicalBase,
+    expectedDir: wrongHealthSite,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  }), /publication health URL does not identify health\.json/)
+
+  await assert.rejects(() => verifyPublicSite({
+    baseUrl: 'https://example.test/model-eol/',
+    expectedDir: output,
+    fetchImpl: fetchLocalContract,
+    attempts: 1,
+    retryMs: 0,
+  }), /baseUrl must be the canonical public contract root/)
+} finally {
+  if (server) await new Promise(resolve => server.close(resolve))
+  fs.rmSync(temp, { recursive: true, force: true })
+}
+
+console.log('public site and receipt contract tests passed')
diff --git a/scripts/verify-public-site.mjs b/scripts/verify-public-site.mjs
new file mode 100644
index 0000000..6532b43
--- /dev/null
+++ b/scripts/verify-public-site.mjs
@@ -0,0 +1,210 @@
+#!/usr/bin/env node
+
+import crypto from 'node:crypto'
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { parseCliArgs } from '../lib/cli.mjs'
+
+const CONTRACT_BASE = new URL('https://thossullivan.github.io/model-eol/')
+const digestPattern = /^[0-9a-f]{64}$/
+const shaPattern = /^[0-9a-f]{40,64}$/
+
+const usage = 'Usage: node scripts/verify-public-site.mjs --base-url HTTPS_URL --expected-dir DIR [--attempts COUNT] [--retry-ms MILLISECONDS]'
+
+const fail = message => {
+  const error = new Error(`${message}\n${usage}`)
+  error.exitCode = 2
+  throw error
+}
+
+const sha256 = bytes => crypto.createHash('sha256').update(bytes).digest('hex')
+const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds))
+
+const readExpected = (root, relativePath) => {
+  const fullPath = path.join(root, ...relativePath.split('/'))
+  if (!fs.existsSync(fullPath) || !fs.statSync(fullPath).isFile()) throw new Error(`expected build artifact is missing ${relativePath}`)
+  return fs.readFileSync(fullPath)
+}
+
+const parseExpectedJson = (root, relativePath) => {
+  try {
+    return JSON.parse(readExpected(root, relativePath).toString('utf8'))
+  } catch (error) {
+    throw new Error(`expected ${relativePath} is not valid JSON: ${error.message}`)
+  }
+}
+
+const contractPath = (value, label) => {
+  let parsed
+  try {
+    parsed = new URL(value)
+  } catch {
+    throw new Error(`${label} is not a valid URL`)
+  }
+  if (parsed.origin !== CONTRACT_BASE.origin || !parsed.pathname.startsWith(CONTRACT_BASE.pathname) || parsed.search || parsed.hash) {
+    throw new Error(`${label} must be beneath ${CONTRACT_BASE.href}`)
+  }
+  const relative = decodeURIComponent(parsed.pathname.slice(CONTRACT_BASE.pathname.length))
+  if (!relative || relative.split('/').some(part => !part || part === '.' || part === '..')) throw new Error(`${label} has an unsafe public path`)
+  return relative
+}
+
+const assertExpectedContract = ({ expectedDir, health, publication }) => {
+  if (health.schema !== 'model-eol/health@0.1') throw new Error(`expected health schema is ${health.schema}`)
+  if (!shaPattern.test(health.refresh_commit ?? '')) throw new Error('expected health has an invalid refresh_commit')
+  if (!shaPattern.test(health.published_commit ?? '')) throw new Error('expected health has an invalid published_commit')
+  if (!/^\d{4}-\d{2}-\d{2}T/.test(health.last_checked ?? '')) throw new Error('expected health has an invalid last_checked')
+  if (publication.schema !== 'model-eol/publication@0.1') throw new Error(`expected publication schema is ${publication.schema}`)
+  if (publication.published_commit !== health.published_commit) throw new Error('expected index and health published commits differ')
+  if (JSON.stringify(publication.feeds) !== JSON.stringify(health.feeds)) throw new Error('expected index and health feed records differ')
+  if (contractPath(publication.health, 'health') !== 'health.json') throw new Error('publication health URL does not identify health.json')
+  if (!Array.isArray(health.feeds) || !health.feeds.length) throw new Error('expected health has no feeds')
+  if (!Array.isArray(publication.schemas) || !publication.schemas.length) throw new Error('expected publication has no schema URLs')
+
+  const assets = [
+    { relativePath: 'health.json', kind: 'health' },
+    { relativePath: 'index.json', kind: 'index' },
+    { relativePath: 'index.html', kind: 'index-page' },
+  ]
+  const localFeeds = fs.readdirSync(path.join(expectedDir, 'feeds')).filter(name => name.endsWith('.json')).sort()
+  const indexedFeeds = []
+  for (const [index, feed] of health.feeds.entries()) {
+    if (!digestPattern.test(feed.sha256 ?? '')) throw new Error(`expected feed ${index} has an invalid SHA-256 digest`)
+    const relativePath = contractPath(feed.url, `feeds[${index}].url`)
+    if (!relativePath.startsWith('feeds/')) throw new Error(`feeds[${index}].url is not a feed URL`)
+    const expectedBytes = readExpected(expectedDir, relativePath)
+    if (sha256(expectedBytes) !== feed.sha256) throw new Error(`${relativePath} does not match its expected health digest`)
+    indexedFeeds.push(path.basename(relativePath))
+    assets.push({ relativePath, kind: 'feed' })
+  }
+  if (JSON.stringify(indexedFeeds.sort()) !== JSON.stringify(localFeeds)) throw new Error('health does not enumerate every built feed exactly once')
+  const atomPath = contractPath(publication.atom, 'atom')
+  if (atomPath !== 'changelog.atom') throw new Error('publication atom URL does not identify changelog.atom')
+  assets.push({ relativePath: atomPath, kind: 'atom' })
+
+  const localSchemas = fs.readdirSync(path.join(expectedDir, 'schema', '0.1')).filter(name => name.endsWith('.json')).sort()
+  if (publication.schemas.length !== localSchemas.length) throw new Error(`expected publication must contain all ${localSchemas.length} built schema URLs`)
+  const indexedSchemas = []
+  for (const [index, schema] of publication.schemas.entries()) {
+    if (schema.id !== schema.url) throw new Error(`schemas[${index}] id and URL differ`)
+    const relativePath = contractPath(schema.url, `schemas[${index}].url`)
+    if (!relativePath.startsWith('schema/0.1/')) throw new Error(`schemas[${index}].url is not a versioned schema URL`)
+    indexedSchemas.push(path.basename(relativePath))
+    readExpected(expectedDir, relativePath)
+    assets.push({ relativePath, kind: 'schema' })
+  }
+  if (JSON.stringify(indexedSchemas.sort()) !== JSON.stringify(localSchemas)) throw new Error('publication index does not enumerate every built schema exactly once')
+  const paths = assets.map(asset => asset.relativePath)
+  if (new Set(paths).size !== paths.length) throw new Error('publication contract contains duplicate asset URLs')
+  return assets
+}
+
+const fetchExact = async ({ fetchImpl, url, expectedBytes, attempts, retryMs, label }) => {
+  let detail = 'no response'
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      const response = await fetchImpl(url, { cache: 'no-store', redirect: 'error', headers: { accept: '*/*' } })
+      if (!response.ok) detail = `HTTP ${response.status}`
+      else if (response.redirected) detail = 'unexpected redirect'
+      else if (response.url && new URL(response.url).href !== new URL(url).href) detail = `unexpected final URL ${response.url}`
+      else {
+        const actual = Buffer.from(await response.arrayBuffer())
+        if (actual.equals(expectedBytes)) return
+        detail = `SHA-256 ${sha256(actual)} (expected ${sha256(expectedBytes)})`
+      }
+    } catch (error) {
+      detail = error.message
+    }
+    if (attempt < attempts) await delay(retryMs)
+  }
+  throw new Error(`${label} did not match the exact built artifact after ${attempts} attempt(s): ${detail}`)
+}
+
+export async function verifyPublicSite({
+  baseUrl,
+  expectedDir,
+  fetchImpl = globalThis.fetch,
+  attempts = 6,
+  retryMs = 5000,
+}) {
+  if (typeof fetchImpl !== 'function') throw new Error('fetch is unavailable')
+  if (!Number.isSafeInteger(attempts) || attempts < 1 || attempts > 20) throw new Error('attempts must be an integer from 1 to 20')
+  if (!Number.isSafeInteger(retryMs) || retryMs < 0 || retryMs > 60000) throw new Error('retryMs must be an integer from 0 to 60000')
+  const expectedRoot = path.resolve(expectedDir)
+  const health = parseExpectedJson(expectedRoot, 'health.json')
+  const publication = parseExpectedJson(expectedRoot, 'index.json')
+  const assets = assertExpectedContract({ expectedDir: expectedRoot, health, publication })
+  const suppliedBase = new URL(baseUrl)
+  const normalizedBase = `${suppliedBase.origin}${suppliedBase.pathname.replace(/\/?$/, '/')}`
+  if (suppliedBase.username || suppliedBase.password || suppliedBase.search || suppliedBase.hash || normalizedBase !== CONTRACT_BASE.href) {
+    throw new Error(`baseUrl must be the canonical public contract root ${CONTRACT_BASE.href}`)
+  }
+  const deployedBase = CONTRACT_BASE
+
+  for (const asset of assets) {
+    const expectedBytes = readExpected(expectedRoot, asset.relativePath)
+    await fetchExact({
+      fetchImpl,
+      url: new URL(asset.relativePath, deployedBase),
+      expectedBytes,
+      attempts,
+      retryMs,
+      label: asset.relativePath,
+    })
+  }
+  return {
+    lastChecked: health.last_checked,
+    refreshCommit: health.refresh_commit,
+    publishedCommit: health.published_commit,
+    feeds: assets.filter(asset => asset.kind === 'feed').length,
+    schemas: assets.filter(asset => asset.kind === 'schema').length,
+  }
+}
+
+async function main(argv) {
+  const { values, positionals } = parseCliArgs({
+    args: argv,
+    options: {
+      'base-url': { type: 'string' },
+      'expected-dir': { type: 'string' },
+      attempts: { type: 'string' },
+      'retry-ms': { type: 'string' },
+      help: { type: 'boolean', short: 'h' },
+    },
+    help: usage,
+  })
+  if (values.help) {
+    console.log(usage)
+    return 0
+  }
+  if (positionals.length) fail(`unexpected positional argument: ${positionals[0]}`)
+  if (!values['base-url']) fail('--base-url is required')
+  if (!values['expected-dir']) fail('--expected-dir is required')
+  let base
+  try {
+    base = new URL(values['base-url'])
+  } catch {
+    fail('--base-url must be a valid HTTPS URL')
+  }
+  if (base.protocol !== 'https:') fail('--base-url must be a valid HTTPS URL')
+  const attempts = values.attempts === undefined ? 6 : Number(values.attempts)
+  const retryMs = values['retry-ms'] === undefined ? 5000 : Number(values['retry-ms'])
+  const result = await verifyPublicSite({
+    baseUrl: base.href,
+    expectedDir: values['expected-dir'],
+    attempts,
+    retryMs,
+  })
+  console.log(`verified exact deployed contract from ${result.publishedCommit}: ${result.feeds} feeds, ${result.schemas} schemas, checked ${result.lastChecked}`)
+  return 0
+}
+
+if (path.resolve(process.argv[1] ?? '') === path.resolve(import.meta.filename)) {
+  try {
+    process.exitCode = await main(process.argv.slice(2))
+  } catch (error) {
+    console.error(`public site verification failed: ${error.message}`)
+    process.exitCode = error.exitCode ?? 1
+  }
+}
diff --git a/test/run.mjs b/test/run.mjs
index 7ff4f09..5d2d759 100644
--- a/test/run.mjs
+++ b/test/run.mjs
@@ -8,10 +8,12 @@ import fs from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
 
+import { applyPlan } from '../lib/apply.mjs'
 import { color, colorEnabled } from '../lib/color.mjs'
 import { assertIsoDate, buildModelPattern, findingFromRef, lifecycleFor, loadFeeds } from '../lib/feeds.mjs'
 import { buildPlan } from '../lib/plan.mjs'
 import { formatCheck, formatSchedule } from '../lib/reports.mjs'
+import { parseDiffPath } from '../lib/scanner.mjs'
 
 const root = path.join(import.meta.dirname, '..')
 const run = (args, options = {}) => {
@@ -64,6 +66,11 @@ assert(help.code === 0 && help.out.includes('3  Output stream failure other than
 const unknownFlag = run([path.join(root, 'test/fixture'), '--dyas', '90'])
 assert(unknownFlag.code === 2 && unknownFlag.err.includes('--dyas') && unknownFlag.err.includes('--help'), 'unknown check flags exit 2 with the bad flag and help hint')
 
+const emptyTarget = run(['inventory', '', '--json'])
+assert(emptyTarget.code === 2 && emptyTarget.out === '' && emptyTarget.err.includes('paths must be non-empty'), 'empty positional targets fail before emitting a schema-invalid report')
+const emptyVia = run(['inventory', path.join(root, 'test/fixture'), '--via=', '--json'])
+assert(emptyVia.code === 2 && emptyVia.out === '' && emptyVia.err.includes('--via must be a non-empty'), 'an explicitly empty lifecycle channel fails before emitting a schema-invalid report')
+
 const invalidDays = run([path.join(root, 'test/fixture'), '--days', 'banana'])
 assert(invalidDays.code === 2 && invalidDays.err.includes('finite non-negative integer'), 'non-numeric --days exits 2')
 const fractionalDays = run([path.join(root, 'test/fixture'), '--days', '1.5'])
@@ -204,7 +211,10 @@ const cyclonedxComponent = cyclonedx.components.find(item => item.name === 'o3-d
 const property = (component, name) => component?.properties.find(item => item.name === name)?.value
 assert(cyclonedxRun.code === 0, 'CycloneDX inventory exits 0')
 assert(cyclonedx.bomFormat === 'CycloneDX' && cyclonedx.specVersion === '1.6' && cyclonedx.version === 1, 'CycloneDX required BOM keys emitted')
-assert(cyclonedx.components.length === new Set(cj.model_references.map(item => item.id)).size, 'CycloneDX has one component per unique tracked model id')
+assert(
+  cyclonedx.components.length === new Set(cj.model_references.map(item => JSON.stringify([item.publisher, item.id, item.requested_via]))).size,
+  'CycloneDX has one component per unique publisher, canonical model, and requested lifecycle channel',
+)
 assert(property(cyclonedxComponent, 'model-eol:status') === 'retired', 'CycloneDX carries model-eol status property')
 assert(cyclonedxComponent?.evidence?.occurrences.some(item => item.location.endsWith('direct.py#8')), 'CycloneDX carries model reference occurrences')
 assert(!cyclonedx.components.some(item => item.name === 'gpt-9-ultra-20990101'), 'CycloneDX omits candidate model references')
@@ -554,6 +564,100 @@ assert(allowedPlan.code === 0 && JSON.parse(allowedPlan.out).scan_notes.length >
 const incompleteInventoryText = run(['inventory', incompleteDir, '--format', 'text'])
 assert(incompleteInventoryText.code === 0 && incompleteInventoryText.out.includes('WARNING: scan incomplete'), 'inventory human output warns about scan notes without failing')
 
+const invalidUtf8Dir = path.join(tempRoot, 'invalid-utf8-scan')
+fs.mkdirSync(invalidUtf8Dir)
+fs.writeFileSync(path.join(invalidUtf8Dir, 'app.py'), Buffer.concat([
+  Buffer.from('MODEL = "o3-deep-research"\n'),
+  Buffer.from([0xff, 0x0a]),
+]))
+const invalidUtf8Check = run([invalidUtf8Dir, '--json'])
+assert(invalidUtf8Check.code === 2 && invalidUtf8Check.err.includes('invalid-utf8'), 'invalid UTF-8 is coverage loss instead of being scanned through replacement characters')
+const invalidUtf8Allowed = run([invalidUtf8Dir, '--allow-incomplete', '--json'])
+const invalidUtf8AllowedJson = JSON.parse(invalidUtf8Allowed.out)
+assert(invalidUtf8Allowed.code === 0 && invalidUtf8AllowedJson.findings.length === 0 && invalidUtf8AllowedJson.scan_notes.some(note => note.reason === 'invalid-utf8'), 'allow-incomplete records invalid UTF-8 without emitting findings from lossy text')
+
+const symlinkTarget = path.join(tempRoot, 'symlink-outside.py')
+const explicitSymlink = path.join(tempRoot, 'explicit-link.py')
+fs.writeFileSync(symlinkTarget, 'MODEL = "o3-deep-research"\n')
+fs.symlinkSync(symlinkTarget, explicitSymlink)
+const explicitSymlinkCheck = run([explicitSymlink, '--json'])
+assert(explicitSymlinkCheck.code === 2 && explicitSymlinkCheck.err.includes('symlink-skipped'), 'an explicit symlink-only target fails check as incomplete coverage')
+const explicitSymlinkAllowed = run([explicitSymlink, '--allow-incomplete', '--json'])
+const explicitSymlinkAllowedJson = JSON.parse(explicitSymlinkAllowed.out)
+assert(explicitSymlinkAllowed.code === 0 && explicitSymlinkAllowedJson.findings.length === 0 && explicitSymlinkAllowedJson.scan_notes.some(note => note.reason === 'symlink-skipped'), 'allow-incomplete records an explicit symlink without following its target')
+const explicitSymlinkPlan = run(['plan', explicitSymlink])
+assert(explicitSymlinkPlan.code === 2 && explicitSymlinkPlan.err.includes('symlink-skipped'), 'an explicit symlink-only target fails plan as incomplete coverage')
+
+const trackedSymlinkRepo = path.join(tempRoot, 'tracked-symlink-repo')
+fs.mkdirSync(trackedSymlinkRepo)
+const trackedSymlinkGit = args => spawnSync('git', args, { cwd: trackedSymlinkRepo, encoding: 'utf8' })
+assert(trackedSymlinkGit(['init', '-q']).status === 0, 'tracked-symlink fixture initializes a git root')
+assert(trackedSymlinkGit(['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'tracked-symlink fixture configures git email')
+assert(trackedSymlinkGit(['config', 'user.name', 'model-eol test']).status === 0, 'tracked-symlink fixture configures git name')
+fs.symlinkSync(symlinkTarget, path.join(trackedSymlinkRepo, 'tracked-link.py'))
+assert(trackedSymlinkGit(['add', 'tracked-link.py']).status === 0 && trackedSymlinkGit(['commit', '-qm', 'track symlink']).status === 0, 'tracked-symlink fixture commits the link itself')
+const trackedSymlinkCheck = run([trackedSymlinkRepo, '--json'])
+assert(trackedSymlinkCheck.code === 2 && trackedSymlinkCheck.err.includes('symlink-skipped'), 'a tracked symlink fails check as incomplete coverage')
+const trackedSymlinkAllowed = run([trackedSymlinkRepo, '--allow-incomplete', '--json'])
+const trackedSymlinkAllowedJson = JSON.parse(trackedSymlinkAllowed.out)
+assert(trackedSymlinkAllowed.code === 0 && trackedSymlinkAllowedJson.findings.length === 0 && trackedSymlinkAllowedJson.scan_notes.some(note => note.reason === 'symlink-skipped' && note.file.endsWith('tracked-link.py')), 'tracked symlink coverage loss is explicit and its outside target is never scanned')
+fs.writeFileSync(path.join(trackedSymlinkRepo, '.model-eol.json'), '{"ignore":{"paths":["tracked-link.py"]}}\n')
+assert(trackedSymlinkGit(['add', '.model-eol.json']).status === 0 && trackedSymlinkGit(['commit', '-qm', 'ignore intentional symlink']).status === 0, 'tracked-symlink fixture commits an explicit path policy')
+const ignoredTrackedSymlink = run([trackedSymlinkRepo, '--json'])
+assert(ignoredTrackedSymlink.code === 0 && JSON.parse(ignoredTrackedSymlink.out).scan_notes.every(note => note.reason !== 'symlink-skipped'), 'repository path policy can explicitly accept an intentional tracked symlink')
+
+const trackedSubmoduleRepo = path.join(tempRoot, 'tracked-submodule-repo')
+const trackedSubmodulePath = path.join(trackedSubmoduleRepo, 'sub')
+fs.mkdirSync(trackedSubmodulePath, { recursive: true })
+const trackedSubmoduleGit = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8' })
+assert(trackedSubmoduleGit(trackedSubmodulePath, ['init', '-q']).status === 0, 'tracked-submodule fixture initializes the nested repository')
+assert(trackedSubmoduleGit(trackedSubmodulePath, ['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'tracked-submodule fixture configures nested git email')
+assert(trackedSubmoduleGit(trackedSubmodulePath, ['config', 'user.name', 'model-eol test']).status === 0, 'tracked-submodule fixture configures nested git name')
+fs.writeFileSync(path.join(trackedSubmodulePath, 'app.py'), 'MODEL = "o3-deep-research"\n')
+assert(trackedSubmoduleGit(trackedSubmodulePath, ['add', 'app.py']).status === 0 && trackedSubmoduleGit(trackedSubmodulePath, ['commit', '-qm', 'submodule fixture']).status === 0, 'tracked-submodule fixture commits a retired reference in the nested repository')
+assert(trackedSubmoduleGit(trackedSubmoduleRepo, ['init', '-q']).status === 0, 'tracked-submodule fixture initializes the parent repository')
+assert(trackedSubmoduleGit(trackedSubmoduleRepo, ['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'tracked-submodule fixture configures parent git email')
+assert(trackedSubmoduleGit(trackedSubmoduleRepo, ['config', 'user.name', 'model-eol test']).status === 0, 'tracked-submodule fixture configures parent git name')
+assert(trackedSubmoduleGit(trackedSubmoduleRepo, ['add', 'sub']).status === 0 && trackedSubmoduleGit(trackedSubmoduleRepo, ['commit', '-qm', 'track submodule']).status === 0, 'tracked-submodule fixture commits the nested repository as a gitlink')
+
+const initializedSubmoduleCheck = run([trackedSubmoduleRepo, '--json'])
+assert(initializedSubmoduleCheck.code === 2 && initializedSubmoduleCheck.err.includes('submodule-skipped'), 'a checked-out tracked submodule fails check as incomplete coverage')
+const initializedSubmodulePlan = run(['plan', trackedSubmoduleRepo])
+assert(initializedSubmodulePlan.code === 2 && initializedSubmodulePlan.err.includes('submodule-skipped'), 'a checked-out tracked submodule fails plan as incomplete coverage')
+const allowedInitializedSubmodule = run([trackedSubmoduleRepo, '--allow-incomplete', '--json'])
+const allowedInitializedSubmoduleJson = JSON.parse(allowedInitializedSubmodule.out)
+assert(
+  allowedInitializedSubmodule.code === 0 &&
+    allowedInitializedSubmoduleJson.findings.length === 0 &&
+    allowedInitializedSubmoduleJson.scan_notes.some(note => note.reason === 'submodule-skipped' && note.file.endsWith('/sub')),
+  'allow-incomplete records a checked-out submodule without recursively scanning its retired reference',
+)
+const allowedInitializedSubmodulePlan = run(['plan', trackedSubmoduleRepo, '--allow-incomplete'])
+const allowedInitializedSubmodulePlanJson = JSON.parse(allowedInitializedSubmodulePlan.out)
+assert(
+  allowedInitializedSubmodulePlan.code === 0 &&
+    allowedInitializedSubmodulePlanJson.items.length === 0 &&
+    allowedInitializedSubmodulePlanJson.scan_notes.some(note => note.reason === 'submodule-skipped'),
+  'allow-incomplete emits a plan receipt for a checked-out submodule without recursing into it',
+)
+
+const trackedSubmoduleConfig = path.join(trackedSubmoduleRepo, '.model-eol.json')
+fs.writeFileSync(trackedSubmoduleConfig, '{"ignore":{"paths":["sub"]}}\n')
+const ignoredTrackedSubmodule = run([trackedSubmoduleRepo, '--json'])
+assert(ignoredTrackedSubmodule.code === 0 && JSON.parse(ignoredTrackedSubmodule.out).scan_notes.every(note => note.reason !== 'submodule-skipped'), 'repository path policy can explicitly accept an intentional tracked submodule')
+const ignoredTrackedSubmodulePlan = run(['plan', trackedSubmoduleRepo])
+assert(ignoredTrackedSubmodulePlan.code === 0 && JSON.parse(ignoredTrackedSubmodulePlan.out).scan_notes.every(note => note.reason !== 'submodule-skipped'), 'an explicitly ignored tracked submodule does not block plan generation')
+fs.rmSync(trackedSubmoduleConfig)
+
+fs.rmSync(trackedSubmodulePath, { recursive: true, force: true })
+const uninitializedSubmoduleCheck = run([trackedSubmoduleRepo, '--json'])
+assert(uninitializedSubmoduleCheck.code === 2 && uninitializedSubmoduleCheck.err.includes('submodule-skipped') && !uninitializedSubmoduleCheck.err.includes('unreadable-file'), 'an uninitialized tracked submodule fails check with a precise incomplete-coverage reason')
+const uninitializedSubmodulePlan = run(['plan', trackedSubmoduleRepo])
+assert(uninitializedSubmodulePlan.code === 2 && uninitializedSubmodulePlan.err.includes('submodule-skipped'), 'an uninitialized tracked submodule fails plan as incomplete coverage')
+const allowedUninitializedSubmodule = run([trackedSubmoduleRepo, '--allow-incomplete', '--json'])
+const allowedUninitializedSubmoduleJson = JSON.parse(allowedUninitializedSubmodule.out)
+assert(allowedUninitializedSubmodule.code === 0 && allowedUninitializedSubmoduleJson.scan_notes.some(note => note.reason === 'submodule-skipped'), 'allow-incomplete records an uninitialized tracked submodule')
+
 const orangeBadgeDir = path.join(tempRoot, 'orange-badge')
 fs.mkdirSync(orangeBadgeDir)
 fs.writeFileSync(path.join(orangeBadgeDir, 'app.py'), 'MODEL = "badge-retiring-model"\n')
@@ -693,6 +797,9 @@ const git = args => spawnSync('git', args, { cwd: changedRepo, encoding: 'utf8'
 assert(git(['init', '-q']).status === 0, 'diff test initializes a git repository')
 assert(git(['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'diff test configures git email')
 assert(git(['config', 'user.name', 'model-eol test']).status === 0, 'diff test configures git name')
+assert(git(['config', 'diff.mnemonicPrefix', 'true']).status === 0, 'diff test enables hostile mnemonic prefixes')
+assert(git(['config', 'diff.noprefix', 'true']).status === 0, 'diff test enables hostile no-prefix output')
+assert(git(['config', 'color.diff', 'always']).status === 0, 'diff test enables hostile forced color output')
 fs.writeFileSync(changedFile, 'MODEL = "o3-deep-research"\n')
 assert(git(['add', 'app.py']).status === 0 && git(['commit', '-qm', 'base']).status === 0, 'diff test creates the base commit')
 fs.writeFileSync(changedFile, 'MODEL = "o3-deep-research"\nNEW_MODEL = "claude-opus-4-1-20250805"\n')
@@ -700,7 +807,65 @@ const changedRun = run(['check', changedRepo, '--days', '30', '--changed', 'HEAD
 assert(changedRun.out.trim().startsWith('{'), `--changed emits JSON (stderr: ${changedRun.err.trim()})`)
 const changedJson = JSON.parse(changedRun.out)
 assert(changedRun.code === 1, '--changed fails for an added bad model')
-assert(changedJson.findings.length === 1 && changedJson.findings[0].id === 'claude-opus-4-1-20250805', '--changed filters out unchanged bad model lines')
+assert(changedJson.findings.length === 1 && changedJson.findings[0].id === 'claude-opus-4-1-20250805', '--changed pins parseable prefixes and color despite hostile Git config')
+assert(parseDiffPath('"b/path with spaces.py"') === 'path with spaces.py', 'Git diff path parsing preserves ordinary quoted paths with spaces')
+assert(parseDiffPath('"b/\\303\\251.py"') === 'é.py', 'Git diff path parsing decodes octal UTF-8 bytes')
+let malformedDiffPathFailed = false
+try {
+  parseDiffPath('"b/unsupported\\q.py"')
+} catch {
+  malformedDiffPathFailed = true
+}
+assert(malformedDiffPathFailed, 'Git diff path parsing fails closed on unsupported quoted escapes')
+
+const noFinalNewlineRepo = path.join(tempRoot, 'changed-no-final-newline-repo')
+fs.mkdirSync(noFinalNewlineRepo)
+const noFinalNewlineGit = args => spawnSync('git', args, { cwd: noFinalNewlineRepo, encoding: 'utf8' })
+assert(noFinalNewlineGit(['init', '-q']).status === 0, 'no-final-newline diff test initializes a git repository')
+assert(noFinalNewlineGit(['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'no-final-newline diff test configures git email')
+assert(noFinalNewlineGit(['config', 'user.name', 'model-eol test']).status === 0, 'no-final-newline diff test configures git name')
+const removedMarkerFile = path.join(noFinalNewlineRepo, 'removed-marker.py')
+const addedMarkerFile = path.join(noFinalNewlineRepo, 'added-marker.py')
+fs.writeFileSync(removedMarkerFile, 'MODEL = "gpt-5.6-sol"')
+fs.writeFileSync(addedMarkerFile, 'MODEL = "gpt-5.6-sol"\n')
+assert(noFinalNewlineGit(['add', '.']).status === 0 && noFinalNewlineGit(['commit', '-qm', 'base']).status === 0, 'no-final-newline diff test creates the base commit')
+fs.writeFileSync(removedMarkerFile, 'MODEL = "o3-deep-research"\n')
+fs.writeFileSync(addedMarkerFile, 'MODEL = "o3-deep-research"')
+const noFinalNewlineChanged = run(['check', noFinalNewlineRepo, '--days', '30', '--changed', 'HEAD', '--json'])
+const noFinalNewlineChangedJson = JSON.parse(noFinalNewlineChanged.out)
+const noFinalNewlineFindingFiles = new Set(noFinalNewlineChangedJson.findings.map(finding => path.basename(finding.file)))
+assert(noFinalNewlineChanged.code === 1, '--changed fails when no-final-newline metadata surrounds added retired models')
+assert(noFinalNewlineFindingFiles.has('removed-marker.py'), '--changed ignores a no-final-newline marker after the removed line')
+assert(noFinalNewlineFindingFiles.has('added-marker.py'), '--changed ignores a no-final-newline marker after the added line')
+
+const unicodeChangedRepo = path.join(tempRoot, 'changed-unicode-path-repo')
+fs.mkdirSync(unicodeChangedRepo)
+const unicodeChangedGit = args => spawnSync('git', args, { cwd: unicodeChangedRepo, encoding: 'utf8' })
+assert(unicodeChangedGit(['init', '-q']).status === 0, 'Unicode-path diff test initializes a git repository')
+assert(unicodeChangedGit(['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'Unicode-path diff test configures git email')
+assert(unicodeChangedGit(['config', 'user.name', 'model-eol test']).status === 0, 'Unicode-path diff test configures git name')
+const unicodeChangedFile = path.join(unicodeChangedRepo, 'é.py')
+fs.writeFileSync(unicodeChangedFile, 'MODEL = "gpt-5.6-sol"\n')
+assert(unicodeChangedGit(['add', '.']).status === 0 && unicodeChangedGit(['commit', '-qm', 'base']).status === 0, 'Unicode-path diff test creates the base commit')
+fs.writeFileSync(unicodeChangedFile, 'MODEL = "o3-deep-research"\n')
+const unicodeChanged = run(['check', unicodeChangedRepo, '--days', '30', '--changed', 'HEAD', '--json'])
+const unicodeChangedJson = JSON.parse(unicodeChanged.out)
+assert(unicodeChanged.code === 1 && unicodeChangedJson.findings.some(finding => finding.file.endsWith('é.py')), '--changed maps Git octal UTF-8 paths back to scanned filenames')
+
+const disabledDiffRepo = path.join(tempRoot, 'changed-disabled-diff-repo')
+fs.mkdirSync(disabledDiffRepo)
+const disabledDiffGit = args => spawnSync('git', args, { cwd: disabledDiffRepo, encoding: 'utf8' })
+assert(disabledDiffGit(['init', '-q']).status === 0, 'disabled-diff test initializes a git repository')
+assert(disabledDiffGit(['config', 'user.email', 'model-eol-test@example.invalid']).status === 0, 'disabled-diff test configures git email')
+assert(disabledDiffGit(['config', 'user.name', 'model-eol test']).status === 0, 'disabled-diff test configures git name')
+const disabledDiffFile = path.join(disabledDiffRepo, 'app.py')
+fs.writeFileSync(disabledDiffFile, 'MODEL = "gpt-5.6-sol"\n')
+assert(disabledDiffGit(['add', '.']).status === 0 && disabledDiffGit(['commit', '-qm', 'base']).status === 0, 'disabled-diff test creates the base commit')
+fs.writeFileSync(path.join(disabledDiffRepo, '.gitattributes'), '*.py -diff\n')
+fs.writeFileSync(disabledDiffFile, 'MODEL = "o3-deep-research"\n')
+const disabledDiffChanged = run(['check', disabledDiffRepo, '--days', '30', '--changed', 'HEAD', '--json'])
+const disabledDiffChangedJson = JSON.parse(disabledDiffChanged.out)
+assert(disabledDiffChanged.code === 1 && disabledDiffChangedJson.findings.some(finding => finding.file.endsWith('app.py')), '--changed forces scannable files to text despite a PR-controlled -diff attribute')
 const nonGit = path.join(tempRoot, 'not-a-git-repo')
 fs.mkdirSync(nonGit)
 fs.writeFileSync(path.join(nonGit, 'app.py'), 'MODEL = "o3-deep-research"\n')
@@ -786,6 +951,29 @@ const duplicate = run(['check', path.join(root, 'test/fixture'), '--feeds', dupl
 assert(duplicate.code === 2, 'duplicate feed key rejects the feed set')
 assert(duplicate.err.includes('duplicate-model') && duplicate.err.includes('one.json') && duplicate.err.includes('two.json'), 'duplicate feed error names key and both files')
 
+const duplicateDistributionFeeds = path.join(tempRoot, 'duplicate-distribution-feeds')
+fs.mkdirSync(duplicateDistributionFeeds)
+fs.writeFileSync(path.join(duplicateDistributionFeeds, 'duplicate-distribution.json'), JSON.stringify({
+  spec: 'model-eol/0.1',
+  publisher: 'duplicate-distribution',
+  generated: '2026-08-01T00:00:00Z',
+  source: 'https://example.invalid/duplicate-distribution',
+  models: [{
+    id: 'duplicate-distribution-model',
+    distributions: [
+      { via: 'aws-bedrock', shutdown: '2026-09-01' },
+      { via: 'aws-bedrock', shutdown: '2027-09-01' },
+    ],
+  }],
+}))
+let duplicateDistributionLoadError = ''
+try {
+  loadFeeds(duplicateDistributionFeeds)
+} catch (error) {
+  duplicateDistributionLoadError = error.message
+}
+assert(duplicateDistributionLoadError.includes('duplicate distributor via "aws-bedrock"'), 'feed loading rejects duplicate lifecycle records for one distributor clock')
+
 const planRun = run(['plan', path.join(root, 'test/fixture'), '--days', '90'])
 const plan = JSON.parse(planRun.out)
 assert(planRun.code === 0, 'plan exits 0 and always emits JSON')
@@ -830,6 +1018,19 @@ const choicePlanRun = run(['plan', gateDir, '--feeds', gateFeeds, '--days', '90'
 const choicePlan = JSON.parse(choicePlanRun.out)
 const choiceIssue = choicePlan.issues.find(issue => issue.id === 'retired-with-options')
 assert(choiceIssue?.reason === 'replacement-choice' && JSON.stringify(choiceIssue.replacement_options) === JSON.stringify(['first-choice', 'second-choice']) && choiceIssue.replacement_note === 'verify the parameter profile', 'plan emits replacement-choice issues with options and notes')
+const choiceInventory = JSON.parse(run(['inventory', gateDir, '--feeds', gateFeeds, '--json']).out)
+const choiceInventoryRef = choiceInventory.model_references.find(item => item.id === 'retired-with-options')
+assert(JSON.stringify(choiceInventoryRef?.replacement_options) === JSON.stringify(['first-choice', 'second-choice']) && choiceInventoryRef?.replacement_note === 'verify the parameter profile', 'inventory preserves structured replacement options and notes')
+const choiceSchedule = JSON.parse(run(['schedule', gateDir, '--feeds', gateFeeds, '--json']).out)
+const choiceScheduleRef = choiceSchedule.items.find(item => item.id === 'retired-with-options')
+assert(JSON.stringify(choiceScheduleRef?.replacement_options) === JSON.stringify(['first-choice', 'second-choice']) && choiceScheduleRef?.replacement_note === 'verify the parameter profile', 'schedule preserves structured replacement options and notes')
+const choiceAlert = JSON.parse(run(['alert', gateDir, '--feeds', gateFeeds, '--json']).out)
+const choiceAlertRef = choiceAlert.errors.find(item => item.id === 'retired-with-options')
+assert(JSON.stringify(choiceAlertRef?.replacement_options) === JSON.stringify(['first-choice', 'second-choice']) && choiceAlertRef?.replacement_note === 'verify the parameter profile', 'alert preserves structured replacement options and notes')
+const choiceCycloneDx = JSON.parse(run(['inventory', gateDir, '--feeds', gateFeeds, '--format', 'cyclonedx']).out)
+const choiceCycloneDxComponent = choiceCycloneDx.components.find(item => item.name === 'retired-with-options')
+assert(property(choiceCycloneDxComponent, 'model-eol:replacement_options') === JSON.stringify(['first-choice', 'second-choice']), 'CycloneDX preserves ordered replacement options as JSON')
+assert(property(choiceCycloneDxComponent, 'model-eol:replacement_note') === 'verify the parameter profile', 'CycloneDX preserves replacement guidance notes')
 
 const ar6Dir = path.join(tempRoot, 'ar6-proximity')
 const ar6Feeds = path.join(tempRoot, 'ar6-feeds')
@@ -888,6 +1089,7 @@ const writeApplyPlan = item => fs.writeFileSync(applyPlanFile, JSON.stringify({
   generated: '2026-08-01T00:00:00Z',
   threshold_days: 90,
   via: null,
+  scan_notes: [],
   items: Array.isArray(item) ? item : [item],
   issues: [],
 }))
@@ -911,6 +1113,21 @@ assert(sharedApplied.code === 0 && fs.readFileSync(applyFile, 'utf8') === `${sha
 const sharedRerun = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
 assert(sharedRerun.code === 0 && sharedRerun.err === '' && (sharedRerun.out.match(/already-applied /g) ?? []).length === 2 && !sharedRerun.out.includes('failed'), 'grouped apply is idempotent for every item')
 
+const overlappingLine = 'abc'
+const overlappingItems = [
+  { ...applyItem, matched: 'abc', replacement: 'first', expected_line_sha256: hash(overlappingLine) },
+  { ...applyItem, matched: 'bc', replacement: 'second', expected_line_sha256: hash(overlappingLine) },
+]
+fs.writeFileSync(applyFile, `${overlappingLine}\n`)
+writeApplyPlan(overlappingItems)
+const overlappingApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  overlappingApply.code === 1 &&
+    overlappingApply.err.includes('replacement span overlaps another plan item') &&
+    fs.readFileSync(applyFile, 'utf8') === `${overlappingLine}\n`,
+  'overlapping same-line replacement spans refuse the whole plan without losing an item',
+)
+
 const mixedLine = 'FIRST = "new-one"; SECOND = "old-two"'
 fs.writeFileSync(applyFile, `${mixedLine}\n`)
 writeApplyPlan(sharedItems)
@@ -924,6 +1141,132 @@ const temporaryApplyFiles = fs.readdirSync(applyDir).filter(name => name.startsW
 assert(atomicApply.code === 0 && fs.readFileSync(applyFile, 'utf8') === `${sharedNewLine}\n`, 'atomic apply writes the complete final content')
 assert(temporaryApplyFiles.length === 0, 'atomic apply leaves no temporary file')
 
+const transactionFileA = path.join(applyDir, 'transaction-a.py')
+const transactionFileB = path.join(applyDir, 'transaction-b.py')
+const transactionOldA = 'MODEL = "old-a"'
+const transactionOldB = 'MODEL = "old-b"'
+const transactionNewA = 'MODEL = "new-a"'
+const transactionNewB = 'MODEL = "new-b"'
+const transactionItemA = {
+  ...applyItem,
+  file: transactionFileA,
+  matched: 'old-a',
+  replacement: 'new-a',
+  expected_line_sha256: hash(transactionOldA),
+}
+const transactionItemB = {
+  ...applyItem,
+  file: transactionFileB,
+  matched: 'old-b',
+  replacement: 'new-b',
+  expected_line_sha256: hash(transactionOldB),
+}
+const mismatchedTransactionItemB = {
+  ...transactionItemB,
+  expected_line_sha256: hash('MODEL = "not-old-b"'),
+}
+for (const [label, items] of [
+  ['good-then-bad', [transactionItemA, mismatchedTransactionItemB]],
+  ['bad-then-good', [mismatchedTransactionItemB, transactionItemA]],
+]) {
+  fs.writeFileSync(transactionFileA, `${transactionOldA}\n`)
+  fs.writeFileSync(transactionFileB, `${transactionOldB}\n`)
+  writeApplyPlan(items)
+  const result = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+  assert(
+    result.code === 1 &&
+      fs.readFileSync(transactionFileA, 'utf8') === `${transactionOldA}\n` &&
+      fs.readFileSync(transactionFileB, 'utf8') === `${transactionOldB}\n`,
+    `plan-wide preflight writes neither file for ${label}`,
+  )
+}
+
+fs.writeFileSync(transactionFileA, `${transactionOldA}\n`)
+writeApplyPlan([{ ...transactionItemB, file: '../escape.py' }, transactionItemA])
+const badPathTransaction = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  badPathTransaction.code === 1 && fs.readFileSync(transactionFileA, 'utf8') === `${transactionOldA}\n`,
+  'a bad plan path prevents a valid file from being written',
+)
+
+fs.writeFileSync(transactionFileA, `${transactionOldA}\n`)
+fs.writeFileSync(transactionFileB, `${transactionOldB}\n`)
+writeApplyPlan([transactionItemA, transactionItemB])
+const twoFileApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  twoFileApply.code === 0 &&
+    fs.readFileSync(transactionFileA, 'utf8') === `${transactionNewA}\n` &&
+    fs.readFileSync(transactionFileB, 'utf8') === `${transactionNewB}\n` &&
+    (twoFileApply.out.match(/applied /g) ?? []).length === 2,
+  'a valid two-file plan commits both staged outputs',
+)
+const twoFileRerun = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  twoFileRerun.code === 0 &&
+    (twoFileRerun.out.match(/already-applied /g) ?? []).length === 2 &&
+    fs.readFileSync(transactionFileA, 'utf8') === `${transactionNewA}\n` &&
+    fs.readFileSync(transactionFileB, 'utf8') === `${transactionNewB}\n`,
+  'a committed two-file plan is idempotent',
+)
+
+fs.writeFileSync(transactionFileA, `${transactionOldA}\n`)
+fs.writeFileSync(transactionFileB, `${transactionOldB}\n`)
+writeApplyPlan([transactionItemA, transactionItemB])
+let injectedCommitCount = 0
+const rollbackResult = applyPlan({
+  planPath: applyPlanFile,
+  rootDir: applyDir,
+  _test: {
+    renameSync: (source, target, context) => {
+      if (context.phase === 'commit') {
+        injectedCommitCount++
+        if (injectedCommitCount === 2) throw new Error('injected second commit failure')
+      }
+      fs.renameSync(source, target)
+    },
+  },
+})
+const rollbackTemporaryFiles = fs.readdirSync(applyDir).filter(name => name.startsWith('.model-eol-'))
+assert(
+  rollbackResult.failed > 0 &&
+    injectedCommitCount === 2 &&
+    fs.readFileSync(transactionFileA, 'utf8') === `${transactionOldA}\n` &&
+    fs.readFileSync(transactionFileB, 'utf8') === `${transactionOldB}\n` &&
+    rollbackTemporaryFiles.length === 0,
+  'a later commit rename failure rolls back earlier file commits and cleans its stages',
+)
+
+fs.writeFileSync(transactionFileA, `${transactionOldA}\n`)
+writeApplyPlan(transactionItemA)
+const cleanupErrors = []
+const originalConsoleError = console.error
+let cleanupResult
+try {
+  console.error = (...values) => cleanupErrors.push(values.join(' '))
+  cleanupResult = applyPlan({
+    planPath: applyPlanFile,
+    rootDir: applyDir,
+    _test: {
+      unlinkSync: temporaryFile => {
+        if (temporaryFile.includes('-backup-')) throw new Error('injected cleanup failure')
+        fs.unlinkSync(temporaryFile)
+      },
+    },
+  })
+} finally {
+  console.error = originalConsoleError
+}
+const retainedCleanupFiles = fs.readdirSync(applyDir).filter(name => name.startsWith('.model-eol-'))
+assert(
+  cleanupResult.failed > 0 &&
+    cleanupResult.applied === 1 &&
+    fs.readFileSync(transactionFileA, 'utf8') === `${transactionNewA}\n` &&
+    retainedCleanupFiles.length === 1 &&
+    cleanupErrors.some(message => message.includes('apply cleanup failed') && message.includes('injected cleanup failure')),
+  'a committed apply with an unlink failure reports non-success and names its retained temporary backup',
+)
+for (const temporaryFile of retainedCleanupFiles) fs.unlinkSync(path.join(applyDir, temporaryFile))
+
 const mismatchItem = { ...applyItem, expected_line_sha256: hash('MODEL = "different-model"') }
 fs.writeFileSync(applyFile, `${oldLine}\n`)
 writeApplyPlan(mismatchItem)
@@ -938,6 +1281,57 @@ const dryRun = run(['apply', '--plan', applyPlanFile, '--dry-run'], { cwd: apply
 assert(dryRun.code === 0 && dryRun.out.includes('would change'), 'apply dry-run reports the proposed change')
 assert(fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`, 'apply dry-run writes nothing')
 
+const invalidUtf8 = Buffer.concat([Buffer.from(`${oldLine}\n`), Buffer.from([0xff, 0x0a])])
+fs.writeFileSync(applyFile, invalidUtf8)
+writeApplyPlan(applyItem)
+const invalidUtf8Apply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  invalidUtf8Apply.code === 1 &&
+    invalidUtf8Apply.err.includes('target is not valid UTF-8') &&
+    fs.readFileSync(applyFile).equals(invalidUtf8),
+  'apply refuses invalid UTF-8 without re-encoding unrelated bytes',
+)
+
+fs.writeFileSync(applyFile, `${oldLine}\n`)
+writeApplyPlan({ ...applyItem, replacement: applyItem.matched })
+const noOpApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  noOpApply.code === 2 &&
+    noOpApply.err.includes('replacement must differ from matched') &&
+    fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`,
+  'apply rejects a semantic no-op plan instead of reporting it applied forever',
+)
+
+writeApplyPlan(applyItem)
+const invalidUtf8Plan = fs.readFileSync(applyPlanFile)
+const publisherMarker = invalidUtf8Plan.indexOf(Buffer.from('"publisher":"test"'))
+assert(publisherMarker >= 0, 'apply fixture contains its publisher marker')
+invalidUtf8Plan[publisherMarker + Buffer.byteLength('"publisher":"te')] = 0xff
+fs.writeFileSync(applyPlanFile, invalidUtf8Plan)
+const invalidUtf8PlanApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  invalidUtf8PlanApply.code === 2 &&
+    invalidUtf8PlanApply.err.includes('invalid UTF-8') &&
+    fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`,
+  'apply rejects an invalid UTF-8 plan before file use',
+)
+
+writeApplyPlan(applyItem)
+const invalidSchemaPlan = JSON.parse(fs.readFileSync(applyPlanFile, 'utf8'))
+invalidSchemaPlan.plan_schema = 'model-eol.plan/9.9'
+fs.writeFileSync(applyPlanFile, JSON.stringify(invalidSchemaPlan))
+const invalidSchemaApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(invalidSchemaApply.code === 2 && invalidSchemaApply.err.includes('plan_schema'), 'apply refuses an unsupported plan_schema before file use')
+assert(fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`, 'plan schema refusal writes nothing')
+
+writeApplyPlan(applyItem)
+const incompleteDocumentPlan = JSON.parse(fs.readFileSync(applyPlanFile, 'utf8'))
+delete incompleteDocumentPlan.scan_notes
+fs.writeFileSync(applyPlanFile, JSON.stringify(incompleteDocumentPlan))
+const incompleteDocumentApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(incompleteDocumentApply.code === 2 && incompleteDocumentApply.err.includes('scan_notes'), 'apply validates the full plan document against the public schema')
+assert(fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`, 'full-document schema refusal writes nothing')
+
 fs.writeFileSync(applyFile, `${oldLine}\n`)
 writeApplyPlan([applyItem, { ...applyItem, line: 0 }])
 const malformedPlan = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
@@ -949,6 +1343,36 @@ const escapedPlan = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
 assert(escapedPlan.code === 1 && escapedPlan.err.includes('file path contains .. traversal'), 'apply refuses plan paths that escape rootDir')
 assert(fs.readFileSync(applyFile, 'utf8') === `${oldLine}\n`, 'escaped plan does not write the in-root file')
 
+const applySymlinkTarget = path.join(applyDir, 'apply-symlink-target.py')
+const applyFinalSymlink = path.join(applyDir, 'apply-final-link.py')
+fs.writeFileSync(applySymlinkTarget, `${oldLine}\n`)
+fs.symlinkSync(applySymlinkTarget, applyFinalSymlink)
+writeApplyPlan({ ...applyItem, file: applyFinalSymlink })
+const finalSymlinkApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  finalSymlinkApply.code === 1 &&
+    finalSymlinkApply.err.includes('symlink') &&
+    fs.lstatSync(applyFinalSymlink).isSymbolicLink() &&
+    fs.readFileSync(applySymlinkTarget, 'utf8') === `${oldLine}\n`,
+  'apply rejects a final symlink without replacing the link or changing its target',
+)
+
+const applyParentTarget = path.join(applyDir, 'apply-parent-target')
+const applyParentSymlink = path.join(applyDir, 'apply-parent-link')
+const applyParentTargetFile = path.join(applyParentTarget, 'fixture.py')
+fs.mkdirSync(applyParentTarget)
+fs.writeFileSync(applyParentTargetFile, `${oldLine}\n`)
+fs.symlinkSync(applyParentTarget, applyParentSymlink)
+writeApplyPlan({ ...applyItem, file: path.join(applyParentSymlink, 'fixture.py') })
+const parentSymlinkApply = run(['apply', '--plan', applyPlanFile], { cwd: applyDir })
+assert(
+  parentSymlinkApply.code === 1 &&
+    parentSymlinkApply.err.includes('symlink') &&
+    fs.lstatSync(applyParentSymlink).isSymbolicLink() &&
+    fs.readFileSync(applyParentTargetFile, 'utf8') === `${oldLine}\n`,
+  'apply rejects a symlinked parent without replacing the link or changing its target',
+)
+
 const manyRestoreOriginal = Array.from({ length: 17 }, (_, index) => `old-${index}`).join(' ')
 const manyRestorePost = Array.from({ length: 17 }, (_, index) => `new-${index}`).join(' ')
 const manyRestoreItems = Array.from({ length: 17 }, (_, index) => ({
@@ -982,6 +1406,7 @@ assert(fs.readFileSync(applyFile, 'utf8') === `${boundedRestorePost}\n`, 'visite
 for (const schemaFile of [
   'schema/model-eol.schema.json',
   'schema/model-eol.bot-config.schema.json',
+  'schema/model-eol.check.schema.json',
   'schema/model-eol.inventory.schema.json',
   'schema/model-eol.schedule.schema.json',
   'schema/model-eol.alert.schema.json',
@@ -992,11 +1417,9 @@ for (const schemaFile of [
 }
 
 const inventorySchema = JSON.parse(fs.readFileSync(path.join(root, 'schema/model-eol.inventory.schema.json'), 'utf8'))
-const inventoryReferenceKeys = new Set([
-  ...Object.keys(inventorySchema.definitions.location.properties),
-  ...Object.keys(inventorySchema.definitions.modelReference.allOf[1].properties),
-])
+const inventoryReferenceKeys = new Set(Object.keys(inventorySchema.definitions.modelReference.properties))
 assert(bedrockRef && Object.keys(bedrockRef).every(key => inventoryReferenceKeys.has(key)), 'inventory schema recognizes every emitted routed-reference field')
+assert(inventorySchema.definitions.modelReference.additionalProperties === false && inventorySchema.definitions.candidateReference.additionalProperties === false && inventorySchema.definitions.integrationHint.additionalProperties === false, 'inventory schema makes every emitted nested reference shape strict')
 assert(Array.isArray(bedrockRef?.policy_provenance?.override_indexes) && Number.isInteger(bedrockRef?.policy_provenance?.route_index), 'emitted inventory policy provenance has the schema-defined shape')
 const planSchema = JSON.parse(fs.readFileSync(path.join(root, 'schema/model-eol.plan.schema.json'), 'utf8'))
 const routedPlanIssue = mixedPlan.issues.find(item => item.mapped_from === 'bedrock-prod')

From 7476d645771fbf36595728277ac620467014a3d4 Mon Sep 17 00:00:00 2001
From: thossullivan 
Date: Tue, 18 Aug 2026 13:10:53 -0500
Subject: [PATCH 2/2] Harden adversarial release boundaries

---
 .github/workflows/published-consumer-uat.yml |  37 +-
 README.md                                    |   5 +
 bot/bot.mjs                                  | 204 ++++++++-
 bot/lib/git.mjs                              |  49 ++
 bot/lib/github.mjs                           |   4 +
 bot/test/run.mjs                             | 453 ++++++++++++++++++-
 lib/config.mjs                               |  17 +-
 lib/feeds.mjs                                |  22 +-
 lib/scanner.mjs                              |  22 +-
 refresh/test/run.mjs                         |   5 +-
 test/run.mjs                                 |  70 +++
 11 files changed, 846 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/published-consumer-uat.yml b/.github/workflows/published-consumer-uat.yml
index 3822718..27e4f7a 100644
--- a/.github/workflows/published-consumer-uat.yml
+++ b/.github/workflows/published-consumer-uat.yml
@@ -1,16 +1,12 @@
 name: published-consumer-uat
 
+# Keep release-code execution on workflow_run's read-only cache scope. For a
+# manual recovery, re-run this UAT run or the original npm-release run in Actions.
 on:
   workflow_run:
     workflows: [npm-release]
     branches: [main]
     types: [completed]
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Exact published stable 0.x version to test; v0 must currently point to it
-        required: true
-        type: string
 
 permissions:
   actions: read
@@ -22,7 +18,7 @@ concurrency:
 
 jobs:
   resolve:
-    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    if: github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     outputs:
       run: ${{ steps.release.outputs.released }}
@@ -33,7 +29,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
         with:
-          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || 'main' }}
+          ref: ${{ github.event.workflow_run.head_sha }}
           fetch-depth: 0
           persist-credentials: false
       - uses: actions/setup-node@v7
@@ -41,7 +37,6 @@ jobs:
           node-version: 24
           package-manager-cache: false
       - name: Download npm-release's exact result
-        if: github.event_name == 'workflow_run'
         uses: actions/download-artifact@v8
         with:
           name: npm-release-result
@@ -51,25 +46,13 @@ jobs:
       - name: Resolve exact tested release
         id: release
         env:
-          EVENT_NAME: ${{ github.event_name }}
           EXPECTED_SOURCE_SHA: ${{ github.event.workflow_run.head_sha }}
-          REQUESTED_VERSION: ${{ inputs.version }}
         run: |
           set -euo pipefail
-          if [ "$EVENT_NAME" = 'workflow_run' ]; then
-            node scripts/release-receipt.mjs verify \
-              --receipt _release-result/npm-release-result.json \
-              --expected-source-sha "$EXPECTED_SOURCE_SHA" \
-              --github-output "$GITHUB_OUTPUT"
-          else
-            if [[ ! "$REQUESTED_VERSION" =~ ^0\.[0-9]+\.[0-9]+$ ]]; then
-              echo "invalid stable 0.x version: $REQUESTED_VERSION" >&2
-              exit 2
-            fi
-            echo 'released=true' >> "$GITHUB_OUTPUT"
-            echo "version=$REQUESTED_VERSION" >> "$GITHUB_OUTPUT"
-            echo "tag=v$REQUESTED_VERSION" >> "$GITHUB_OUTPUT"
-          fi
+          node scripts/release-receipt.mjs verify \
+            --receipt _release-result/npm-release-result.json \
+            --expected-source-sha "$EXPECTED_SOURCE_SHA" \
+            --github-output "$GITHUB_OUTPUT"
       - name: Verify remote moving and immutable Action refs
         if: steps.release.outputs.released == 'true'
         id: refs
@@ -77,7 +60,6 @@ jobs:
           VERSION: ${{ steps.release.outputs.version }}
           EXPECTED_RELEASE_COMMIT: ${{ steps.release.outputs.release_commit }}
           EXPECTED_REGISTRY_INTEGRITY: ${{ steps.release.outputs.registry_integrity }}
-          EVENT_NAME: ${{ github.event_name }}
         run: |
           set -euo pipefail
           git fetch --force origin \
@@ -87,9 +69,6 @@ jobs:
           moving_commit=$(git rev-parse 'refs/tags/v0^{commit}')
           if [ "$moving_commit" = "$immutable_commit" ]; then
             echo 'moving_current=true' >> "$GITHUB_OUTPUT"
-          elif [ "$EVENT_NAME" = 'workflow_dispatch' ]; then
-            echo "remote v0 is $moving_commit, but manually requested immutable v$VERSION is $immutable_commit" >&2
-            exit 1
           else
             echo 'moving_current=false' >> "$GITHUB_OUTPUT"
             echo "v0 now resolves to $moving_commit; exact v$VERSION UAT remains authoritative and this superseded moving-line monitor will be skipped" >> "$GITHUB_STEP_SUMMARY"
diff --git a/README.md b/README.md
index 8038361..6dc400b 100644
--- a/README.md
+++ b/README.md
@@ -221,6 +221,11 @@ unless the path is explicitly ignored or `--allow-incomplete` is chosen. To
 inspect submodule contents, run model-eol against the checked-out submodule as a
 separate target/repository policy.
 
+Untracked nested Git repositories are handled the same way: the parent reports
+`nested-repository-skipped` and never recurses into the nested checkout. Ignore
+an intentional nested repository explicitly, allow incomplete coverage, or scan
+it separately with its own repository policy.
+
 Mixed-provider monorepos can keep those top-level values as repository defaults,
 then apply path policies and a lifecycle channel per reference:
 
diff --git a/bot/bot.mjs b/bot/bot.mjs
index bb8c684..2578f33 100755
--- a/bot/bot.mjs
+++ b/bot/bot.mjs
@@ -33,11 +33,15 @@ import {
   commitAll,
   configureIdentity,
   defaultBranch,
+  deleteRemoteBranch,
   gitAuthentication,
+  isCommitAvailable,
   originFor,
   prepareBranch,
   pushBranch,
+  restoreRemoteBranch,
   verifyBotBranch,
+  verifyRemoteBranchHead,
 } from './lib/git.mjs'
 
 const ROOT = path.resolve(import.meta.dirname, '..')
@@ -913,6 +917,124 @@ const makePatch = ({ source, base, expectedBaseHead, branch, expectedHead, allow
 
 const decision = (group, action, extra = {}) => ({ group, action, ...extra })
 
+const publicationBaseDecision = ({ group, root, base, baseHead, gitAuth, number = undefined }) => {
+  const state = verifyRemoteBranchHead(root, base, baseHead, gitAuth)
+  if (state.safe) return null
+  return decision(group, 'stand-down', {
+    ...(number === undefined ? {} : { number }),
+    reason: state.error ? 'default-branch-unverifiable' : 'default-branch-moved',
+    expectedBaseHead: baseHead,
+    currentBaseHead: state.head,
+  })
+}
+
+const staleBodyFor = body => {
+  const metadata = parseMetadata(body)
+  if (!metadata) throw new Error('refusing to close bot pull request with malformed generated metadata')
+  return staleClosedBody(body, metadata)
+}
+
+const cleanupPushedBranchAndRethrow = ({ error, root, branch, headSha, gitAuth, context, relatedErrors = [] }) => {
+  try {
+    deleteRemoteBranch(root, branch, headSha, gitAuth)
+  } catch (cleanupError) {
+    throw new AggregateError(
+      [error, ...relatedErrors, cleanupError],
+      `${context} and exact-head cleanup of ${branch} also failed`,
+    )
+  }
+  if (relatedErrors.length) {
+    throw new AggregateError(
+      [error, ...relatedErrors],
+      `${context}; exact-head cleanup succeeded after pull-request state verification also failed`,
+    )
+  }
+  throw error
+}
+
+const rollbackPushedBranchAndRethrow = ({ error, root, branch, headSha, restoreHead, gitAuth, context }) => {
+  try {
+    restoreRemoteBranch(root, branch, headSha, restoreHead, gitAuth)
+  } catch (rollbackError) {
+    throw new AggregateError(
+      [error, rollbackError],
+      `${context} and exact-head rollback of ${branch} also failed`,
+    )
+  }
+  throw error
+}
+
+const recoverFailedPullUpdate = async ({ api, open, priorBody, error, root, group, headSha, restoreHead, rollbackTrusted, gitAuth, context }) => {
+  let current = null
+  let readError = null
+  try {
+    current = await api.getPull(open.item.number)
+  } catch (failure) {
+    readError = failure
+  }
+  const metadata = parseMetadata(current?.body)
+  const priorBodyProven = rollbackTrusted
+    && current?.number === open.item.number
+    && isOpen(current)
+    && hasModelEolLabel(current)
+    && current.head?.ref === group.branch
+    && current.head?.repo?.full_name === api.repo
+    && current.body === priorBody
+    && metadata?.head_sha === restoreHead
+  if (priorBodyProven) {
+    rollbackPushedBranchAndRethrow({
+      error,
+      root,
+      branch: group.branch,
+      headSha,
+      restoreHead,
+      gitAuth,
+      context,
+    })
+  }
+  cleanupPushedBranchAndRethrow({
+    error,
+    root,
+    branch: group.branch,
+    headSha,
+    gitAuth,
+    context: `${context} with ambiguous or changed pull-request state`,
+    relatedErrors: readError ? [readError] : [],
+  })
+}
+
+const closePullAtNewHead = async ({ api, open, body, root, branch, headSha, gitAuth }) => {
+  const closedBody = staleBodyFor(body)
+  try {
+    await api.updatePull(open.item.number, { state: 'closed', body: closedBody })
+  } catch (error) {
+    cleanupPushedBranchAndRethrow({
+      error,
+      root,
+      branch,
+      headSha,
+      gitAuth,
+      context: `pull request #${open.item.number} closure failed`,
+    })
+  }
+  return closedBody
+}
+
+const publicationBaseStandDownComment = (baseDecision, headSha) => baseDecision.reason === 'default-branch-unverifiable'
+  ? `model-eol is leaving this pull request closed because it could not re-verify the default branch after pushing evaluated migration head ${markdownCode(headSha)}. The new head is recorded as automated stale work and must be regenerated from a freshly evaluated base before reopening.`
+  : `model-eol is leaving this pull request closed because the default branch moved from evaluated commit ${markdownCode(baseDecision.expectedBaseHead)} to ${markdownCode(baseDecision.currentBaseHead || 'missing')} after pushing migration head ${markdownCode(headSha)}. The new head is recorded as automated stale work and must be reevaluated before reopening.`
+
+const closedBaseStandDownDecision = async ({ api, open, baseDecision, closedBody, headSha }) => {
+  await api.comment(open.item.number, publicationBaseStandDownComment(baseDecision, headSha))
+  return {
+    ...baseDecision,
+    number: open.item.number,
+    closedPullNumber: open.item.number,
+    body: closedBody,
+    headSha,
+  }
+}
+
 const closePullForEvalFailure = async ({ api, open, evalResult }) => {
   await api.comment(open.item.number, `model-eol is closing this bot-owned pull request because its configured migration eval no longer passes (${markdownCode(evalResult.status)}). The migration remains blocked and can be regenerated after the eval clears.`)
   await api.updatePull(open.item.number, {
@@ -977,7 +1099,29 @@ const processModel = async ({ api, pulls, issueRecords, group, source, base, bas
       return decision(group, 'stand-down', { number: open.item.number })
     }
     const body = buildPullBody({ group, headSha: patch.headSha, baseSha: baseHead, now, tokenKind, evalResult: externalEval, evalConfigHash })
-    await api.updatePull(open.item.number, { title: pullTitle(group), body })
+    const baseDecision = publicationBaseDecision({ group, root, base, baseHead, gitAuth, number: open.item.number })
+    if (baseDecision) {
+      const closedBody = await closePullAtNewHead({ api, open, body, root, branch: group.branch, headSha: patch.headSha, gitAuth })
+      return closedBaseStandDownDecision({ api, open, baseDecision, closedBody, headSha: patch.headSha })
+    }
+    const priorBody = open.item.body
+    try {
+      await api.updatePull(open.item.number, { title: pullTitle(group), body })
+    } catch (error) {
+      await recoverFailedPullUpdate({
+        api,
+        open,
+        priorBody,
+        error,
+        root,
+        group,
+        headSha: patch.headSha,
+        restoreHead: open.metadata.head_sha,
+        rollbackTrusted: true,
+        gitAuth,
+        context: `pull request #${open.item.number} update failed`,
+      })
+    }
     return decision(group, 'update', { number: open.item.number, body, headSha: patch.headSha })
   }
 
@@ -991,6 +1135,11 @@ const processModel = async ({ api, pulls, issueRecords, group, source, base, bas
 
   const previous = matches.find(record => record.metadata?.head_sha) ?? null
   const previousBotHead = previous?.metadata.head_sha ?? null
+  const previousRollbackHead = previousBotHead
+    && verifyBotBranch(root, group.branch, previousBotHead, gitAuth).safe
+    && isCommitAvailable(root, previousBotHead)
+    ? previousBotHead
+    : null
   if (!previousBotHead) {
     const occupied = verifyBotBranch(root, group.branch, null, gitAuth)
     if (occupied.error) throw new Error(occupied.error)
@@ -1026,19 +1175,58 @@ const processModel = async ({ api, pulls, issueRecords, group, source, base, bas
   if (latestConflict) return decision(group, 'conflict', { number: latestConflict.item.number })
   const latestOpen = latestMatches.find(record => isOpen(record.item))
   if (latestOpen) {
-    await api.updatePull(latestOpen.item.number, { title: pullTitle(group), body })
+    const baseDecision = publicationBaseDecision({ group, root, base, baseHead, gitAuth, number: latestOpen.item.number })
+    if (baseDecision) {
+      const closedBody = await closePullAtNewHead({ api, open: latestOpen, body, root, branch: group.branch, headSha: patch.headSha, gitAuth })
+      return closedBaseStandDownDecision({ api, open: latestOpen, baseDecision, closedBody, headSha: patch.headSha })
+    }
+    const priorBody = latestOpen.item.body
+    try {
+      await api.updatePull(latestOpen.item.number, { title: pullTitle(group), body })
+    } catch (error) {
+      await recoverFailedPullUpdate({
+        api,
+        open: latestOpen,
+        priorBody,
+        error,
+        root,
+        group,
+        headSha: patch.headSha,
+        restoreHead: previousRollbackHead,
+        rollbackTrusted: Boolean(previousRollbackHead && latestOpen.metadata.head_sha === previousRollbackHead),
+        gitAuth,
+        context: `concurrent pull request #${latestOpen.item.number} update failed`,
+      })
+    }
     return decision(group, 'update', { number: latestOpen.item.number, body, headSha: patch.headSha })
   }
   const latestUntrusted = latestPulls.find(item => pullOnExpectedBranch(item, group, api.repo) && !latestMatches.some(record => record.item.number === item.number))
   if (latestUntrusted) return decision(group, 'conflict', { number: latestUntrusted.number })
   const latestSuppressed = latestMatches.find(record => !isOpen(record.item) && !isMerged(record.item) && record.metadata.stale_closed !== true && record.metadata.shutdown === group.items[0].shutdown && (record.metadata.replacement === undefined || record.metadata.replacement === replacement))
   if (latestSuppressed) return decision(group, 'skip-dismissed', { number: latestSuppressed.item.number })
-  const created = await api.createPull({
-    title: pullTitle(group),
-    head: group.branch,
-    base,
-    body,
-  })
+  const baseDecision = publicationBaseDecision({ group, root, base, baseHead, gitAuth })
+  if (baseDecision) {
+    deleteRemoteBranch(root, group.branch, patch.headSha, gitAuth)
+    return { ...baseDecision, headSha: patch.headSha, deletedBranch: group.branch }
+  }
+  let created
+  try {
+    created = await api.createPull({
+      title: pullTitle(group),
+      head: group.branch,
+      base,
+      body,
+    })
+  } catch (error) {
+    cleanupPushedBranchAndRethrow({
+      error,
+      root,
+      branch: group.branch,
+      headSha: patch.headSha,
+      gitAuth,
+      context: 'pull request creation failed',
+    })
+  }
   return decision(group, 'create', { number: created.number, body, headSha: patch.headSha })
 }
 
diff --git a/bot/lib/git.mjs b/bot/lib/git.mjs
index afa835e..630e715 100644
--- a/bot/lib/git.mjs
+++ b/bot/lib/git.mjs
@@ -105,6 +105,55 @@ export const verifyBotBranch = (cwd, branch, expectedHead, auth = null) => {
   }
 }
 
+export const verifyRemoteBranchHead = (cwd, branch, expectedHead, auth = null) => {
+  try {
+    run(cwd, ['fetch', 'origin', '--prune'], false, auth)
+  } catch (error) {
+    return { head: null, safe: false, error: error.message }
+  }
+  const head = run(cwd, ['rev-parse', '--verify', `refs/remotes/origin/${branch}`], true)
+  return {
+    head,
+    safe: head === expectedHead,
+    error: null,
+  }
+}
+
+const exactObjectId = value => typeof value === 'string' && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i.test(value)
+
+const requireExactObjectId = (value, name) => {
+  if (!exactObjectId(value)) throw new Error(`refusing remote branch mutation without an exact ${name} commit`)
+}
+
+export const isCommitAvailable = (cwd, head) => exactObjectId(head)
+  && run(cwd, ['cat-file', '-e', `${head}^{commit}`], true) !== null
+
+export const deleteRemoteBranch = (cwd, branch, expectedHead, auth = null) => {
+  requireExactObjectId(expectedHead, 'expected')
+  const ref = `refs/heads/${branch}`
+  run(cwd, [
+    'push',
+    `--force-with-lease=${ref}:${expectedHead}`,
+    'origin',
+    `:${ref}`,
+  ], false, auth)
+}
+
+export const restoreRemoteBranch = (cwd, branch, expectedCurrentHead, restoreHead, auth = null) => {
+  requireExactObjectId(expectedCurrentHead, 'current')
+  requireExactObjectId(restoreHead, 'restore')
+  if (!isCommitAvailable(cwd, restoreHead)) {
+    throw new Error(`refusing remote branch rollback because restore commit ${restoreHead} is unavailable`)
+  }
+  const ref = `refs/heads/${branch}`
+  run(cwd, [
+    'push',
+    `--force-with-lease=${ref}:${expectedCurrentHead}`,
+    'origin',
+    `${restoreHead}:${ref}`,
+  ], false, auth)
+}
+
 export const pushBranch = (cwd, branch, expectedHead = null, auth = null, { allowMissing = false } = {}) => {
   const destination = `HEAD:refs/heads/${branch}`
   if (expectedHead !== null) {
diff --git a/bot/lib/github.mjs b/bot/lib/github.mjs
index 0a1acb9..18dbc23 100644
--- a/bot/lib/github.mjs
+++ b/bot/lib/github.mjs
@@ -149,6 +149,10 @@ export class GitHubClient {
     return data
   }
 
+  async getPull(number) {
+    return (await this.request('GET', `/repos/${this.repo}/pulls/${number}`)).data
+  }
+
   async listIssues() {
     const issues = await this.listAll(`/repos/${this.repo}/issues`)
     return issues.filter(issue => !issue.pull_request)
diff --git a/bot/test/run.mjs b/bot/test/run.mjs
index 5f07886..e92748f 100644
--- a/bot/test/run.mjs
+++ b/bot/test/run.mjs
@@ -20,7 +20,7 @@ import { branchFor, metadataLine, parseMetadata, slugFor } from '../lib/common.m
 import { loadConfig } from '../lib/config.mjs'
 import { downloadFeeds } from '../lib/feeds.mjs'
 import { reportForBody, runEvalHook } from '../lib/eval.mjs'
-import { cloneRepository, gitAuthentication, originFor } from '../lib/git.mjs'
+import { cloneRepository, deleteRemoteBranch, gitAuthentication, originFor } from '../lib/git.mjs'
 
 const root = path.resolve(import.meta.dirname, '../..')
 const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'model-eol-bot-test-'))
@@ -122,6 +122,58 @@ const evaluateForPublish = async ({
 const bareBranchFile = (repo, branch, file) => git(repo.bare, ['show', `${branch}:${file}`])
 const headOf = (repo, branch) => git(repo.bare, ['rev-parse', `refs/heads/${branch}`])
 
+const stageDefaultBranchAdvance = (repo, name) => {
+  write(path.join(repo.work, 'README.md'), `default branch advance for ${name}\n`)
+  git(repo.work, ['add', 'README.md'])
+  git(repo.work, ['commit', '-m', `stage ${name} default branch advance`])
+  const head = git(repo.work, ['rev-parse', 'HEAD'])
+  const ref = `refs/heads/model-eol-test-${name}`
+  git(repo.work, ['push', 'origin', `HEAD:${ref}`])
+  return { head, ref }
+}
+
+const advanceDefaultBranchAfterMigrationPush = (repo, branch, nextRef) => {
+  const hook = path.join(repo.bare, 'hooks', 'post-receive')
+  write(hook, [
+    '#!/bin/sh',
+    'while read -r _old _new ref',
+    'do',
+    `  if [ "$ref" = "refs/heads/${branch}" ]`,
+    '  then',
+    `    git update-ref refs/heads/main ${nextRef}`,
+    '  fi',
+    'done',
+    '',
+  ].join('\n'))
+  fs.chmodSync(hook, 0o755)
+}
+
+const deleteLeaseRepo = makeRepo({ name: 'delete-exact-head-lease', files: { 'README.md': 'first branch head\n' } })
+const deleteLeaseBranch = 'model-eol/test-delete-lease'
+git(deleteLeaseRepo.work, ['push', 'origin', `HEAD:refs/heads/${deleteLeaseBranch}`])
+const deleteLeaseExpectedHead = headOf(deleteLeaseRepo, deleteLeaseBranch)
+write(path.join(deleteLeaseRepo.work, 'README.md'), 'second branch head\n')
+git(deleteLeaseRepo.work, ['add', 'README.md'])
+git(deleteLeaseRepo.work, ['commit', '-m', 'advance leased branch'])
+git(deleteLeaseRepo.work, ['push', 'origin', `HEAD:refs/heads/${deleteLeaseBranch}`])
+const deleteLeaseCurrentHead = headOf(deleteLeaseRepo, deleteLeaseBranch)
+let deleteLeaseError = null
+try {
+  deleteRemoteBranch(deleteLeaseRepo.work, deleteLeaseBranch, deleteLeaseExpectedHead)
+} catch (error) {
+  deleteLeaseError = error
+}
+assert(deleteLeaseError && headOf(deleteLeaseRepo, deleteLeaseBranch) === deleteLeaseCurrentHead, 'remote branch deletion refuses a stale exact-head lease and preserves the newer head')
+let malformedDeleteLeaseError = null
+try {
+  deleteRemoteBranch(deleteLeaseRepo.work, deleteLeaseBranch, 'a'.repeat(41))
+} catch (error) {
+  malformedDeleteLeaseError = error
+}
+assert(malformedDeleteLeaseError?.message.includes('exact expected commit') && headOf(deleteLeaseRepo, deleteLeaseBranch) === deleteLeaseCurrentHead, 'remote branch deletion rejects malformed commit lengths before touching the current head')
+deleteRemoteBranch(deleteLeaseRepo.work, deleteLeaseBranch, deleteLeaseCurrentHead)
+assert(!gitTry(deleteLeaseRepo.bare, ['show-ref', '--verify', `refs/heads/${deleteLeaseBranch}`]), 'remote branch deletion succeeds only with the exact current head')
+
 class FakeGitHub {
   constructor() {
     this.calls = []
@@ -134,6 +186,10 @@ class FakeGitHub {
     this.beforeListIssues = null
     this.labelExists = false
     this.labelCreateStatus = 201
+    this.pullCreateStatus = 201
+    this.pullUpdateStatus = 200
+    this.pullUpdateApplyBeforeFailure = false
+    this.pullUpdateThrowAfterApply = false
   }
 
   response(data, status = 200) {
@@ -160,6 +216,11 @@ class FakeGitHub {
       this.beforeListPulls?.(this, this.listPullCount)
       return this.response(this.pulls)
     }
+    if (options.method === 'GET' && /\/pulls\/\d+$/.test(pathName)) {
+      const number = Number(pathName.split('/').at(-1))
+      const pull = this.pulls.find(item => item.number === number)
+      return pull ? this.response(pull) : this.response({ message: 'Not Found' }, 404)
+    }
     if (options.method === 'GET' && pathName.endsWith('/issues')) {
       this.listIssueCount++
       this.beforeListIssues?.(this, this.listIssueCount)
@@ -171,6 +232,7 @@ class FakeGitHub {
       return this.response({ object: { sha: pull?.head?.sha ?? null } })
     }
     if (options.method === 'POST' && pathName.endsWith('/pulls')) {
+      if (this.pullCreateStatus >= 400) return this.response({ message: 'pull creation failed' }, this.pullCreateStatus)
       const number = this.nextNumber++
       const created = {
         number,
@@ -188,9 +250,16 @@ class FakeGitHub {
     if (options.method === 'PATCH' && pathName.includes('/pulls/')) {
       const number = Number(pathName.split('/').at(-1))
       const pull = this.pulls.find(item => item.number === number)
+      if (this.pullUpdateStatus >= 400 && !this.pullUpdateApplyBeforeFailure) {
+        return this.response({ message: 'pull update failed' }, this.pullUpdateStatus)
+      }
       Object.assign(pull, body)
       const metadata = parseMetadata(body.body)
       if (metadata && pull.head) pull.head.sha = metadata.head_sha
+      if (this.pullUpdateStatus >= 400) {
+        if (this.pullUpdateThrowAfterApply) throw new Error('simulated pull update transport failure after apply')
+        return this.response({ message: 'pull update failed after apply' }, this.pullUpdateStatus)
+      }
       return this.response(pull)
     }
     if (options.method === 'POST' && pathName.includes('/comments')) return this.response({ id: this.nextNumber++, body: body.body }, 201)
@@ -410,6 +479,163 @@ try {
 assert(publishWindowError?.message.includes('prepared base commit') && publishWindowError.message.includes('does not match evaluated base commit'), 'publisher refuses a patch checkout when the default branch moves after plan and eval verification')
 assert(publishWindowGithub.callsFor('POST', '/pulls').length === 0 && !gitTry(publishWindowRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'publish-window base drift fails before patch branch push or pull-request creation')
 
+const postPushCreateRepo = makeRepo({ name: 'post-push-create-base-drift', files: baseFiles, config: { issues: { enabled: false } } })
+const postPushCreateGithub = new FakeGitHub()
+const postPushCreateBase = headOf(postPushCreateRepo, 'main')
+let postPushCreateHead = null
+postPushCreateGithub.beforeListPulls = (_api, count) => {
+  if (count !== 2) return
+  write(path.join(postPushCreateRepo.work, 'README.md'), 'default branch moved after the migration branch push\n')
+  git(postPushCreateRepo.work, ['add', 'README.md'])
+  git(postPushCreateRepo.work, ['commit', '-m', 'move base after patch push'])
+  git(postPushCreateRepo.work, ['push', 'origin', 'main'])
+  postPushCreateHead = headOf(postPushCreateRepo, 'main')
+}
+const postPushCreateResult = await runBot({
+  repo: 'example/post-push-create-base-drift',
+  targetDir: postPushCreateRepo.work,
+  token: 'test-token',
+  transport: postPushCreateGithub.transport.bind(postPushCreateGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+})
+const postPushCreateDecision = postPushCreateResult.decisions.find(item => item.group.kind === 'model')
+assert(
+  postPushCreateDecision?.action === 'stand-down'
+    && postPushCreateDecision.reason === 'default-branch-moved'
+    && postPushCreateDecision.expectedBaseHead === postPushCreateBase
+    && postPushCreateDecision.currentBaseHead === postPushCreateHead,
+  'new-PR publication stands down when the default branch moves after the migration branch push',
+)
+assert(postPushCreateGithub.callsFor('POST', '/pulls').length === 0, 'post-push default-branch drift creates no pull request')
+assert(!gitTry(postPushCreateRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'post-push refusal deletes the unpublished migration branch under its exact pushed-head lease')
+
+const postPushCreateRecovery = await runBot({
+  repo: 'example/post-push-create-base-drift',
+  targetDir: postPushCreateRepo.work,
+  token: 'test-token',
+  transport: postPushCreateGithub.transport.bind(postPushCreateGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+})
+const postPushCreateRecoveryDecision = postPushCreateRecovery.decisions.find(item => item.group.kind === 'model')
+const postPushCreateRecoveryMetadata = parseMetadata(postPushCreateRecoveryDecision?.body)
+assert(postPushCreateRecoveryDecision?.action === 'create' && postPushCreateGithub.callsFor('POST', '/pulls').length === 1, 'a fresh run after new-PR drift creates safe work instead of conflicting with an orphan branch')
+assert(postPushCreateRecoveryMetadata?.base_sha === postPushCreateHead && postPushCreateRecoveryMetadata?.head_sha === headOf(postPushCreateRepo, branch), 'new-PR drift recovery binds the receipt to the moved base and regenerated branch head')
+
+const createFailureRepo = makeRepo({ name: 'pull-create-failure-cleanup', files: baseFiles, config: { issues: { enabled: false } } })
+const createFailureGithub = new FakeGitHub()
+createFailureGithub.pullCreateStatus = 503
+const createFailureRun = () => runBot({
+  repo: 'example/pull-create-failure-cleanup',
+  targetDir: createFailureRepo.work,
+  token: 'test-token',
+  transport: createFailureGithub.transport.bind(createFailureGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+})
+let createFailureError = null
+try {
+  await createFailureRun()
+} catch (error) {
+  createFailureError = error
+}
+assert(createFailureError?.message.includes('GitHub POST') && createFailureGithub.pulls.length === 0, 'pull-request creation failure is surfaced without publishing a pull request')
+assert(!gitTry(createFailureRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'pull-request creation failure deletes its unpublished branch under the exact pushed-head lease')
+createFailureGithub.pullCreateStatus = 201
+const createFailureRecovery = await createFailureRun()
+const createFailureRecoveryDecision = createFailureRecovery.decisions.find(item => item.group.kind === 'model')
+assert(createFailureRecoveryDecision?.action === 'create' && parseMetadata(createFailureRecoveryDecision?.body)?.head_sha === headOf(createFailureRepo, branch), 'a retry after pull creation failure safely recreates the branch and pull request')
+
+const latestUpdateFailureRepo = makeRepo({ name: 'latest-open-update-failure', files: baseFiles, config: { issues: { enabled: false } } })
+const latestUpdateFailureGithub = new FakeGitHub()
+let latestUpdateFailurePull = null
+latestUpdateFailureGithub.pullUpdateStatus = 503
+latestUpdateFailureGithub.beforeListPulls = (client, count) => {
+  if (count !== 2) return
+  const pushedHead = headOf(latestUpdateFailureRepo, branch)
+  latestUpdateFailurePull = {
+    number: 93,
+    state: 'open',
+    labels: [{ name: 'model-eol' }],
+    body: metadataLine({
+      schema: 'model-eol.bot/0.1',
+      id: 'o3-deep-research-2025-06-26',
+      publisher: 'openai',
+      shutdown: '2026-07-23',
+      via: null,
+      replacement: 'gpt-5.6-sol',
+      base_sha: headOf(latestUpdateFailureRepo, 'main'),
+      head_sha: pushedHead,
+      feed_digest: 'concurrent-update-failure',
+    }),
+    head: { ref: branch, sha: pushedHead, repo: { full_name: 'example/latest-open-update-failure' } },
+    base: { ref: 'main' },
+  }
+  client.pulls.push(latestUpdateFailurePull)
+}
+let latestUpdateFailureError = null
+try {
+  await runBot({
+    repo: 'example/latest-open-update-failure',
+    targetDir: latestUpdateFailureRepo.work,
+    token: 'test-token',
+    transport: latestUpdateFailureGithub.transport.bind(latestUpdateFailureGithub),
+    vendoredFeeds: path.join(root, 'feeds'),
+    now: new Date('2026-08-01T00:00:00Z'),
+  })
+} catch (error) {
+  latestUpdateFailureError = error
+}
+assert(latestUpdateFailureError?.message.includes('GitHub PATCH') && latestUpdateFailurePull?.state === 'open', 'concurrent latestOpen update failure surfaces instead of claiming publication')
+assert(!gitTry(latestUpdateFailureRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'concurrent latestOpen without a trusted prior head exact-deletes the pushed branch and becomes unmergeable')
+
+const postPushLatestRepo = makeRepo({ name: 'post-push-latest-open-base-drift', files: baseFiles, config: { issues: { enabled: false } } })
+const postPushLatestGithub = new FakeGitHub()
+const postPushLatestBase = headOf(postPushLatestRepo, 'main')
+let postPushLatestPull = null
+postPushLatestGithub.beforeListPulls = (client, count) => {
+  if (count !== 2) return
+  write(path.join(postPushLatestRepo.work, 'README.md'), 'default branch moved as a concurrent trusted PR appeared\n')
+  git(postPushLatestRepo.work, ['add', 'README.md'])
+  git(postPushLatestRepo.work, ['commit', '-m', 'move base during latest-open race'])
+  git(postPushLatestRepo.work, ['push', 'origin', 'main'])
+  const pushedHead = headOf(postPushLatestRepo, branch)
+  postPushLatestPull = {
+    number: 92,
+    state: 'open',
+    labels: [{ name: 'model-eol' }],
+    body: metadataLine({
+      schema: 'model-eol.bot/0.1',
+      id: 'o3-deep-research-2025-06-26',
+      publisher: 'openai',
+      shutdown: '2026-07-23',
+      via: null,
+      replacement: 'gpt-5.6-sol',
+      base_sha: postPushLatestBase,
+      head_sha: pushedHead,
+      feed_digest: 'concurrent-latest-open',
+    }),
+    head: { ref: branch, sha: pushedHead, repo: { full_name: 'example/post-push-latest-open-base-drift' } },
+    base: { ref: 'main' },
+  }
+  client.pulls.push(postPushLatestPull)
+}
+const postPushLatestResult = await runBot({
+  repo: 'example/post-push-latest-open-base-drift',
+  targetDir: postPushLatestRepo.work,
+  token: 'test-token',
+  transport: postPushLatestGithub.transport.bind(postPushLatestGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+})
+const postPushLatestDecision = postPushLatestResult.decisions.find(item => item.group.kind === 'model')
+const postPushLatestMetadata = parseMetadata(postPushLatestPull?.body)
+assert(postPushLatestDecision?.action === 'stand-down' && postPushLatestDecision.reason === 'default-branch-moved', 'post-push drift stands down when a trusted latestOpen race appears')
+assert(postPushLatestPull?.state === 'closed' && postPushLatestMetadata?.stale_closed === true && postPushLatestMetadata?.head_sha === headOf(postPushLatestRepo, branch), 'detected latestOpen race is closed with stale metadata bound to the pushed head')
+assert(postPushLatestGithub.callsFor('POST', '/pulls').length === 0 && !postPushLatestGithub.pulls.some(item => item.state === 'open'), 'latestOpen drift leaves no open unsafe pull request')
+assert(postPushLatestGithub.callsFor('POST', '/comments').some(call => call.body.body?.includes('default branch moved') && call.body.body?.includes('reevaluated')), 'latestOpen drift records a clear reevaluation comment')
+
 const staleRepo = makeRepo({ name: 'stale-reconciliation', files: baseFiles, config: { issues: { enabled: false } } })
 const staleGithub = new FakeGitHub()
 const staleRun = () => runBot({
@@ -464,6 +690,230 @@ const changedOpenai = JSON.parse(fs.readFileSync(path.join(changedFeeds, 'openai
 changedOpenai.note = 'Changed feed note for lifecycle update'
 fs.writeFileSync(path.join(changedFeeds, 'openai.json'), JSON.stringify(changedOpenai, null, 2))
 
+const updateFailureRepo = makeRepo({ name: 'existing-update-failure-rollback', files: baseFiles, config: { issues: { enabled: false } } })
+const updateFailureGithub = new FakeGitHub()
+const updateFailureRun = options => runBot({
+  repo: 'example/existing-update-failure-rollback',
+  targetDir: updateFailureRepo.work,
+  token: 'test-token',
+  transport: updateFailureGithub.transport.bind(updateFailureGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+  ...options,
+})
+const updateFailureFirst = await updateFailureRun()
+const updateFailureFirstDecision = updateFailureFirst.decisions.find(item => item.group.kind === 'model')
+const updateFailurePull = updateFailureGithub.pulls.find(item => item.number === updateFailureFirstDecision?.number)
+const updateFailureOldBody = updateFailurePull?.body
+const updateFailureOldHead = headOf(updateFailureRepo, branch)
+const updateFailureBase = headOf(updateFailureRepo, 'main')
+if (updateFailurePull) updateFailurePull.head.sha = updateFailureOldHead
+updateFailureGithub.pullUpdateStatus = 503
+let updateFailureError = null
+try {
+  await updateFailureRun({ vendoredFeeds: changedFeeds })
+} catch (error) {
+  updateFailureError = error
+}
+assert(updateFailureError?.message.includes('GitHub PATCH'), 'existing pull-request body update failure is surfaced')
+assert(headOf(updateFailureRepo, branch) === updateFailureOldHead && headOf(updateFailureRepo, 'main') === updateFailureBase, 'failed existing update exact-leases the branch back to its prior trusted head without moving the base')
+assert(updateFailurePull?.state === 'open' && updateFailurePull.body === updateFailureOldBody && parseMetadata(updateFailurePull.body)?.head_sha === updateFailureOldHead, 'failed existing update leaves the old pull-request body and restored head aligned')
+updateFailureGithub.pullUpdateStatus = 200
+const updateFailureRecovery = await updateFailureRun({ vendoredFeeds: changedFeeds })
+const updateFailureRecoveryDecision = updateFailureRecovery.decisions.find(item => item.group.kind === 'model')
+assert(updateFailureRecoveryDecision?.action === 'update' && headOf(updateFailureRepo, branch) !== updateFailureOldHead, 'a retry after exact-head rollback successfully publishes the regenerated update')
+assert(parseMetadata(updateFailurePull?.body)?.head_sha === headOf(updateFailureRepo, branch), 'successful retry realigns the pull-request receipt with the regenerated remote head')
+
+const ambiguousUpdateFeeds = path.join(tempRoot, 'ambiguous-update-feeds')
+fs.cpSync(changedFeeds, ambiguousUpdateFeeds, { recursive: true })
+const ambiguousUpdateOpenai = JSON.parse(fs.readFileSync(path.join(ambiguousUpdateFeeds, 'openai.json'), 'utf8'))
+ambiguousUpdateOpenai.note = 'Ambiguous update response after the body was applied'
+fs.writeFileSync(path.join(ambiguousUpdateFeeds, 'openai.json'), JSON.stringify(ambiguousUpdateOpenai, null, 2))
+const ambiguousUpdatePriorBody = updateFailurePull?.body
+const ambiguousUpdatePriorHead = headOf(updateFailureRepo, branch)
+updateFailureGithub.pullUpdateStatus = 503
+updateFailureGithub.pullUpdateApplyBeforeFailure = true
+let ambiguousUpdateError = null
+try {
+  await updateFailureRun({ vendoredFeeds: ambiguousUpdateFeeds })
+} catch (error) {
+  ambiguousUpdateError = error
+}
+const ambiguousUpdateMetadata = parseMetadata(updateFailurePull?.body)
+assert(ambiguousUpdateError?.message.includes('GitHub PATCH') && updateFailurePull?.body !== ambiguousUpdatePriorBody, 'existing update detects a 503 response after GitHub applied the candidate body')
+assert(ambiguousUpdateMetadata?.head_sha !== ambiguousUpdatePriorHead && !gitTry(updateFailureRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'ambiguous existing update never rolls a candidate receipt onto the old head; it deletes the candidate branch so the PR is unmergeable')
+assert(updateFailureGithub.callsFor('GET', `/pulls/${updateFailurePull?.number}`).length >= 2, 'failed existing updates re-read pull-request state before choosing rollback or deletion')
+
+const latestRollbackRepo = makeRepo({ name: 'latest-open-update-rollback', files: baseFiles, config: { issues: { enabled: false } } })
+const latestRollbackGithub = new FakeGitHub()
+const latestRollbackRun = options => runBot({
+  repo: 'example/latest-open-update-rollback',
+  targetDir: latestRollbackRepo.work,
+  token: 'test-token',
+  transport: latestRollbackGithub.transport.bind(latestRollbackGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+  ...options,
+})
+const latestRollbackFirst = await latestRollbackRun()
+const latestRollbackFirstDecision = latestRollbackFirst.decisions.find(item => item.group.kind === 'model')
+const latestRollbackClosed = latestRollbackGithub.pulls.find(item => item.number === latestRollbackFirstDecision?.number)
+const latestRollbackOldBody = latestRollbackClosed?.body
+const latestRollbackOldHead = headOf(latestRollbackRepo, branch)
+if (latestRollbackClosed) {
+  const priorMetadata = parseMetadata(latestRollbackClosed.body)
+  const lines = latestRollbackClosed.body.split(/\r?\n/)
+  lines[0] = metadataLine({ ...priorMetadata, stale_closed: true })
+  latestRollbackClosed.body = lines.join('\n')
+  latestRollbackClosed.state = 'closed'
+  latestRollbackClosed.head.sha = latestRollbackOldHead
+}
+let latestRollbackOpen = null
+latestRollbackGithub.pullUpdateStatus = 503
+latestRollbackGithub.beforeListPulls = (client, count) => {
+  if (count !== 4) return
+  const pushedHead = headOf(latestRollbackRepo, branch)
+  latestRollbackOpen = {
+    number: 94,
+    state: 'open',
+    labels: [{ name: 'model-eol' }],
+    body: latestRollbackOldBody,
+    head: { ref: branch, sha: pushedHead, repo: { full_name: 'example/latest-open-update-rollback' } },
+    base: { ref: 'main' },
+  }
+  client.pulls.push(latestRollbackOpen)
+}
+let latestRollbackError = null
+try {
+  await latestRollbackRun({ vendoredFeeds: changedFeeds })
+} catch (error) {
+  latestRollbackError = error
+}
+assert(latestRollbackError?.message.includes('GitHub PATCH') && latestRollbackOpen?.body === latestRollbackOldBody, 'concurrent latestOpen update failure preserves its prior trusted body')
+assert(headOf(latestRollbackRepo, branch) === latestRollbackOldHead, 'concurrent latestOpen with exact trusted prior metadata rolls the pushed branch back to the available prior head')
+
+const latestAmbiguousRepo = makeRepo({ name: 'latest-open-ambiguous-update', files: baseFiles, config: { issues: { enabled: false } } })
+const latestAmbiguousGithub = new FakeGitHub()
+const latestAmbiguousRun = options => runBot({
+  repo: 'example/latest-open-ambiguous-update',
+  targetDir: latestAmbiguousRepo.work,
+  token: 'test-token',
+  transport: latestAmbiguousGithub.transport.bind(latestAmbiguousGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+  ...options,
+})
+const latestAmbiguousFirst = await latestAmbiguousRun()
+const latestAmbiguousFirstDecision = latestAmbiguousFirst.decisions.find(item => item.group.kind === 'model')
+const latestAmbiguousClosed = latestAmbiguousGithub.pulls.find(item => item.number === latestAmbiguousFirstDecision?.number)
+const latestAmbiguousOldBody = latestAmbiguousClosed?.body
+const latestAmbiguousOldHead = headOf(latestAmbiguousRepo, branch)
+if (latestAmbiguousClosed) {
+  const priorMetadata = parseMetadata(latestAmbiguousClosed.body)
+  const lines = latestAmbiguousClosed.body.split(/\r?\n/)
+  lines[0] = metadataLine({ ...priorMetadata, stale_closed: true })
+  latestAmbiguousClosed.body = lines.join('\n')
+  latestAmbiguousClosed.state = 'closed'
+  latestAmbiguousClosed.head.sha = latestAmbiguousOldHead
+}
+let latestAmbiguousOpen = null
+latestAmbiguousGithub.pullUpdateStatus = 503
+latestAmbiguousGithub.pullUpdateApplyBeforeFailure = true
+latestAmbiguousGithub.pullUpdateThrowAfterApply = true
+latestAmbiguousGithub.beforeListPulls = (client, count) => {
+  if (count !== 4) return
+  const pushedHead = headOf(latestAmbiguousRepo, branch)
+  latestAmbiguousOpen = {
+    number: 95,
+    state: 'open',
+    labels: [{ name: 'model-eol' }],
+    body: latestAmbiguousOldBody,
+    head: { ref: branch, sha: pushedHead, repo: { full_name: 'example/latest-open-ambiguous-update' } },
+    base: { ref: 'main' },
+  }
+  client.pulls.push(latestAmbiguousOpen)
+}
+let latestAmbiguousError = null
+try {
+  await latestAmbiguousRun({ vendoredFeeds: changedFeeds })
+} catch (error) {
+  latestAmbiguousError = error
+}
+const latestAmbiguousMetadata = parseMetadata(latestAmbiguousOpen?.body)
+assert(latestAmbiguousError?.message.includes('transport failure after apply') && latestAmbiguousOpen?.body !== latestAmbiguousOldBody, 'trusted latestOpen path detects a transport failure after the candidate body was applied')
+assert(latestAmbiguousMetadata?.head_sha !== latestAmbiguousOldHead && !gitTry(latestAmbiguousRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'ambiguous trusted latestOpen never rolls a candidate receipt onto its old head; exact deletion makes it unmergeable')
+assert(latestAmbiguousGithub.callsFor('GET', `/pulls/${latestAmbiguousOpen?.number}`).length === 1, 'failed trusted latestOpen update re-reads pull-request state before rejecting rollback')
+
+const closeFailureRepo = makeRepo({ name: 'post-push-close-failure', files: baseFiles, config: { issues: { enabled: false } } })
+const closeFailureGithub = new FakeGitHub()
+const closeFailureRun = options => runBot({
+  repo: 'example/post-push-close-failure',
+  targetDir: closeFailureRepo.work,
+  token: 'test-token',
+  transport: closeFailureGithub.transport.bind(closeFailureGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+  ...options,
+})
+const closeFailureFirst = await closeFailureRun()
+const closeFailureFirstDecision = closeFailureFirst.decisions.find(item => item.group.kind === 'model')
+const closeFailurePull = closeFailureGithub.pulls.find(item => item.number === closeFailureFirstDecision?.number)
+if (closeFailurePull) closeFailurePull.head.sha = parseMetadata(closeFailureFirstDecision?.body)?.head_sha
+const closeFailureAdvance = stageDefaultBranchAdvance(closeFailureRepo, 'close-failure')
+advanceDefaultBranchAfterMigrationPush(closeFailureRepo, branch, closeFailureAdvance.ref)
+closeFailureGithub.pullUpdateStatus = 503
+let closeFailureError = null
+try {
+  await closeFailureRun({ vendoredFeeds: changedFeeds })
+} catch (error) {
+  closeFailureError = error
+}
+assert(closeFailureError?.message.includes('GitHub PATCH') && closeFailurePull?.state === 'open', 'failed drift closure surfaces the GitHub update error and may leave the pull request open')
+assert(!gitTry(closeFailureRepo.bare, ['show-ref', '--verify', `refs/heads/${branch}`]), 'failed drift closure exact-leases away the newly pushed head so the open pull request is unmergeable')
+
+const postPushUpdateRepo = makeRepo({ name: 'post-push-update-base-drift', files: baseFiles, config: { issues: { enabled: false } } })
+const postPushUpdateGithub = new FakeGitHub()
+const postPushUpdateRun = options => runBot({
+  repo: 'example/post-push-update-base-drift',
+  targetDir: postPushUpdateRepo.work,
+  token: 'test-token',
+  transport: postPushUpdateGithub.transport.bind(postPushUpdateGithub),
+  vendoredFeeds: path.join(root, 'feeds'),
+  now: new Date('2026-08-01T00:00:00Z'),
+  ...options,
+})
+const postPushUpdateFirst = await postPushUpdateRun()
+const postPushUpdateFirstDecision = postPushUpdateFirst.decisions.find(item => item.group.kind === 'model')
+const postPushUpdatePull = postPushUpdateGithub.pulls.find(item => item.number === postPushUpdateFirstDecision?.number)
+if (postPushUpdatePull) postPushUpdatePull.head.sha = parseMetadata(postPushUpdateFirstDecision?.body)?.head_sha
+const postPushUpdateBase = headOf(postPushUpdateRepo, 'main')
+const postPushUpdateBranchBefore = headOf(postPushUpdateRepo, branch)
+const postPushUpdateAdvance = stageDefaultBranchAdvance(postPushUpdateRepo, 'update-base-drift')
+advanceDefaultBranchAfterMigrationPush(postPushUpdateRepo, branch, postPushUpdateAdvance.ref)
+const postPushUpdateResult = await postPushUpdateRun({ vendoredFeeds: changedFeeds })
+const postPushUpdateDecision = postPushUpdateResult.decisions.find(item => item.group.kind === 'model')
+const postPushUpdateHead = headOf(postPushUpdateRepo, branch)
+const postPushUpdateMetadata = parseMetadata(postPushUpdatePull?.body)
+assert(
+  postPushUpdateDecision?.action === 'stand-down'
+    && postPushUpdateDecision.reason === 'default-branch-moved'
+    && postPushUpdateDecision.expectedBaseHead === postPushUpdateBase
+    && postPushUpdateDecision.currentBaseHead === postPushUpdateAdvance.head,
+  'existing-PR publication stands down when the default branch moves after its migration branch push',
+)
+assert(postPushUpdateGithub.calls.filter(call => call.method === 'PATCH' && call.path.includes('/pulls/')).length === 1 && postPushUpdateGithub.callsFor('POST', '/pulls').length === 1, 'post-push default-branch drift closes the existing pull request without creating another')
+assert(postPushUpdatePull?.state === 'closed' && !postPushUpdateGithub.pulls.some(item => item.state === 'open'), 'detected post-push drift leaves no open unsafe pull request')
+assert(postPushUpdateMetadata?.stale_closed === true && postPushUpdateMetadata?.head_sha === postPushUpdateHead && postPushUpdateMetadata.head_sha === postPushUpdateDecision?.headSha, 'drift closure records automated stale metadata bound to the newly pushed head')
+assert(postPushUpdateHead !== postPushUpdateBranchBefore, 'existing-PR refusal leaves the newly evaluated migration branch available for the next reconciliation')
+assert(postPushUpdateGithub.callsFor('POST', '/comments').some(call => call.body.body?.includes('default branch moved') && call.body.body?.includes('reevaluated')), 'existing-PR drift records a clear reevaluation comment')
+
+const postPushUpdateRecovery = await postPushUpdateRun({ vendoredFeeds: changedFeeds })
+const postPushUpdateRecoveryDecision = postPushUpdateRecovery.decisions.find(item => item.group.kind === 'model')
+const postPushUpdateRecoveredPull = postPushUpdateGithub.pulls.find(item => item.number === postPushUpdateRecoveryDecision?.number)
+const postPushUpdateRecoveredMetadata = parseMetadata(postPushUpdateRecoveryDecision?.body)
+assert(postPushUpdateRecoveryDecision?.action === 'create' && postPushUpdateRecoveredPull?.state === 'open' && postPushUpdatePull?.state === 'closed', 'a fresh run after base refresh creates safe work without reopening the stale PR')
+assert(postPushUpdateRecoveredMetadata?.base_sha === headOf(postPushUpdateRepo, 'main') && postPushUpdateRecoveredMetadata?.head_sha === headOf(postPushUpdateRepo, branch), 'recovery PR binds its receipt to the refreshed base and regenerated branch head')
+
 const replacementRepo = makeRepo({ name: 'replacement-suppression', files: baseFiles, config })
 const replacementGithub = new FakeGitHub()
 const replacementRun = options => runBot({
@@ -503,6 +953,7 @@ const updateDecision = updated.decisions.find(item => item.group.kind === 'model
 assert(updateDecision?.action === 'update', 'changed digest with intact bot head updates the PR')
 assert(headOf(repo, branch) !== beforeUpdateHead, 'changed digest force-pushes a regenerated branch')
 assert(github.calls.filter(call => call.method === 'PATCH' && call.path.includes('/pulls/')).length === 1, 'changed digest updates the PR body')
+assert(createdPull?.state === 'open' && parseMetadata(createdPull?.body)?.stale_closed !== true, 'successful existing-PR regeneration remains open without stale closure churn')
 
 const foreignHead = '1111111111111111111111111111111111111111'
 const pull = github.pulls.find(item => item.number === firstDecision.number)
diff --git a/lib/config.mjs b/lib/config.mjs
index b7fa75a..4d533e8 100644
--- a/lib/config.mjs
+++ b/lib/config.mjs
@@ -1,8 +1,11 @@
 import fs from 'node:fs'
+import { TextDecoder } from 'node:util'
 
 import { matchesAnyGlob, normalizeRepoPath } from './glob.mjs'
 import { MAX_REPORT_BYTES, MAX_TIMEOUT_MS, MIN_TIMEOUT_MS } from './validate-feed.mjs'
 
+const utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
+
 const frozenArray = () => Object.freeze([])
 
 export const DEFAULT_CONFIG = Object.freeze({
@@ -248,9 +251,21 @@ export const loadConfig = (file, { defaults = DEFAULT_CONFIG, allowMissing = tru
     if (allowMissing) return normalizeConfig({}, { defaults })
     throw new Error(`could not read config ${file}: file not found`)
   }
+  let bytes
+  try {
+    bytes = fs.readFileSync(file)
+  } catch (error) {
+    throw new Error(`could not read config ${file}: ${error.message}`)
+  }
+  let source
+  try {
+    source = utf8Decoder.decode(bytes)
+  } catch {
+    throw new Error(`could not read config ${file}: invalid UTF-8`)
+  }
   let value
   try {
-    value = JSON.parse(fs.readFileSync(file, 'utf8'))
+    value = JSON.parse(source)
   } catch (error) {
     throw new Error(`could not read config ${file}: ${error.message}`)
   }
diff --git a/lib/feeds.mjs b/lib/feeds.mjs
index 0c94ada..8424e0a 100644
--- a/lib/feeds.mjs
+++ b/lib/feeds.mjs
@@ -1,8 +1,11 @@
 import fs from 'node:fs'
 import path from 'node:path'
+import { TextDecoder } from 'node:util'
 
 import { assertIsoDate, assertValidFeed } from './validate-feed.mjs'
 
+const utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
+
 export { assertIsoDate }
 
 export const BUILTIN_CHANNELS = Object.freeze([
@@ -25,7 +28,24 @@ export const loadFeeds = feedsDir => {
 
   for (const f of fs.readdirSync(feedsDir).filter(f => f.endsWith('.json'))) {
     const file = path.join(feedsDir, f)
-    const feed = JSON.parse(fs.readFileSync(file, 'utf8'))
+    let bytes
+    try {
+      bytes = fs.readFileSync(file)
+    } catch (error) {
+      throw new Error(`${file}: could not read feed: ${error.message}`)
+    }
+    let source
+    try {
+      source = utf8Decoder.decode(bytes)
+    } catch {
+      throw new Error(`${file}: invalid UTF-8`)
+    }
+    let feed
+    try {
+      feed = JSON.parse(source)
+    } catch (error) {
+      throw new Error(`${file}: invalid JSON: ${error.message}`)
+    }
     if (feed.spec !== 'model-eol/0.1') {
       throw new Error(`${file}: unsupported feed spec ${JSON.stringify(feed.spec)}; expected "model-eol/0.1"`)
     }
diff --git a/lib/scanner.mjs b/lib/scanner.mjs
index 55e76f9..be3a488 100644
--- a/lib/scanner.mjs
+++ b/lib/scanner.mjs
@@ -29,6 +29,7 @@ export const INCOMPLETE_SCAN_REASONS = new Set([
   'invalid-utf8',
   'symlink-skipped',
   'submodule-skipped',
+  'nested-repository-skipped',
   'file-count-cap',
   'git-listing-failure',
 ])
@@ -263,10 +264,25 @@ export const addedLinesForTargets = (targets, baseRef) => {
     }
     return relative || '.'
   })
+  if (baseRef.startsWith('-')) {
+    throw new Error(`--changed base ref must not begin with "-": ${JSON.stringify(baseRef)}`)
+  }
+  const resolvedBase = spawnSync('git', [
+    'rev-parse', '--verify', '--quiet', '--end-of-options', `${baseRef}^{commit}`,
+  ], {
+    cwd: gitRoot,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  })
+  const baseCommit = (resolvedBase.stdout ?? '').trim()
+  if (resolvedBase.error || resolvedBase.status !== 0 || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(baseCommit)) {
+    const detail = (resolvedBase.stderr ?? '').trim()
+    throw new Error(`--changed could not resolve base ref "${baseRef}" to a commit${detail ? `: ${detail}` : ''}`)
+  }
   const result = spawnSync('git', [
     '-c', 'core.quotePath=true',
     'diff', '--no-color', '--src-prefix=a/', '--dst-prefix=b/',
-    '--text', '--no-ext-diff', '--no-textconv', '--unified=0', baseRef, '--', ...relativeTargets,
+    '--text', '--no-ext-diff', '--no-textconv', '--unified=0', baseCommit, '--', ...relativeTargets,
   ], {
     cwd: gitRoot,
     encoding: 'utf8',
@@ -511,6 +527,10 @@ const collectFilesDetailed = (targets, {
         if (!isIgnoredFile(absoluteFile)) note({ reason: 'submodule-skipped', file: displayPath(absoluteFile) })
         continue
       }
+      if (relativeFile.endsWith('/')) {
+        if (!isIgnoredFile(absoluteFile)) note({ reason: 'nested-repository-skipped', file: displayPath(absoluteFile) })
+        continue
+      }
       if (path.relative(gitRoot, absoluteFile).split(path.sep).some(part => SKIP_DIRS.has(part))) continue
       addFile(absoluteFile)
     }
diff --git a/refresh/test/run.mjs b/refresh/test/run.mjs
index 4759dfd..8e7b724 100644
--- a/refresh/test/run.mjs
+++ b/refresh/test/run.mjs
@@ -1098,7 +1098,10 @@ assert(publishedUatWorkflow.includes("needs.resolve.outputs.moving_current == 't
 assert(publishedUatWorkflow.includes('v0 or immutable v$VERSION moved while the moving Action UAT was running') && publishedUatWorkflow.includes('exit 1'), 'moving Action monitoring fails if either bound ref changes during its two-step round-trip')
 assert(publishedUatWorkflow.includes('const r=Array.isArray(v)?v.at(-1):v') && publishedUatWorkflow.includes('if(typeof r!=="string")process.exit(1)'), 'moving npm-line recheck normalizes npm view arrays to the resolved latest version')
 assert(releaseWorkflow.includes('group: model-eol-release-and-moving-uat') && releaseWorkflow.includes('queue: max') && publishedUatWorkflow.includes('group: model-eol-release-and-moving-uat') && publishedUatWorkflow.includes('queue: max'), 'release and moving-alias UAT serialize through one non-cancelling queued concurrency group')
-assert(publishedUatWorkflow.includes("ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || 'main' }}"), 'workflow-run UAT resolves its receipt with the triggering release commit\'s own verifier')
+assert(publishedUatWorkflow.includes('workflows: [npm-release]') && publishedUatWorkflow.includes('branches: [main]') && publishedUatWorkflow.includes('types: [completed]'), 'published UAT is triggered by completed npm-release runs on main')
+assert(!publishedUatWorkflow.includes('workflow_dispatch') && !publishedUatWorkflow.includes('inputs.version') && !publishedUatWorkflow.includes('REQUESTED_VERSION') && !publishedUatWorkflow.includes('manually requested immutable'), 'published UAT has no workflow-dispatch or manual-version execution fallback')
+assert(publishedUatWorkflow.includes("if: github.event.workflow_run.conclusion == 'success'") && publishedUatWorkflow.includes('ref: ${{ github.event.workflow_run.head_sha }}'), 'workflow-run UAT executes only after success and resolves its receipt with the triggering release commit\'s own verifier')
+assert(publishedUatWorkflow.includes('--expected-source-sha "$EXPECTED_SOURCE_SHA"'), 'workflow-run UAT binds the downloaded receipt to the triggering release source')
 assert(publishedUatWorkflow.includes('ref: ${{ needs.resolve.outputs.release_commit }}'), 'package UAT runs the exact release commit\'s own consumer harness')
 const freshnessScript = fs.readFileSync(path.join(root, 'scripts/update-readme-freshness.mjs'), 'utf8')
 assert(freshnessScript.includes('AWS Bedrock and Google Vertex AI lifecycle pages'), 'README freshness metadata names every automated distributor source')
diff --git a/test/run.mjs b/test/run.mjs
index 5d2d759..afcf3f4 100644
--- a/test/run.mjs
+++ b/test/run.mjs
@@ -336,6 +336,19 @@ const invalidSharedConfig = path.join(tempRoot, 'invalid-shared-config.json')
 fs.writeFileSync(invalidSharedConfig, JSON.stringify({ ignore: { path: ['src/**'] } }))
 const invalidSharedConfigRun = run(['inventory', repositoryConfigDir, '--config', invalidSharedConfig, '--json'])
 assert(invalidSharedConfigRun.code === 2 && invalidSharedConfigRun.err.includes('ignore.path'), 'CLI preserves strict unknown-key validation for shared config')
+const invalidUtf8Config = path.join(tempRoot, 'invalid-utf8-config.json')
+fs.writeFileSync(invalidUtf8Config, Buffer.concat([
+  Buffer.from('{"ignore":{"paths":["noise-'),
+  Buffer.from([0xff]),
+  Buffer.from('"]}}\n'),
+]))
+const invalidUtf8ConfigRun = run(['inventory', repositoryConfigDir, '--config', invalidUtf8Config, '--json'])
+assert(
+  invalidUtf8ConfigRun.code === 2 &&
+    invalidUtf8ConfigRun.err.includes('invalid UTF-8') &&
+    invalidUtf8ConfigRun.err.includes(path.basename(invalidUtf8Config)),
+  'operational config loading rejects invalid UTF-8 with the selected filename',
+)
 
 const partialGlobDir = path.join(tempRoot, 'partial-glob')
 fs.mkdirSync(path.join(partialGlobDir, 'partial', 'nested'), { recursive: true })
@@ -606,6 +619,34 @@ assert(trackedSymlinkGit(['add', '.model-eol.json']).status === 0 && trackedSyml
 const ignoredTrackedSymlink = run([trackedSymlinkRepo, '--json'])
 assert(ignoredTrackedSymlink.code === 0 && JSON.parse(ignoredTrackedSymlink.out).scan_notes.every(note => note.reason !== 'symlink-skipped'), 'repository path policy can explicitly accept an intentional tracked symlink')
 
+const untrackedNestedRepo = path.join(tempRoot, 'untracked-nested-repo')
+const untrackedNestedPath = path.join(untrackedNestedRepo, 'nested')
+fs.mkdirSync(untrackedNestedPath, { recursive: true })
+const untrackedNestedGit = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8' })
+assert(untrackedNestedGit(untrackedNestedRepo, ['init', '-q']).status === 0, 'untracked-nested fixture initializes the parent repository')
+assert(untrackedNestedGit(untrackedNestedPath, ['init', '-q']).status === 0, 'untracked-nested fixture initializes the embedded repository')
+fs.writeFileSync(path.join(untrackedNestedPath, 'app.py'), 'MODEL = "o3-deep-research"\n')
+const untrackedNestedCheck = run([untrackedNestedRepo, '--json'])
+assert(untrackedNestedCheck.code === 2 && untrackedNestedCheck.err.includes('nested-repository-skipped'), 'an untracked nested Git repository fails check as incomplete coverage')
+const untrackedNestedPlan = run(['plan', untrackedNestedRepo])
+assert(untrackedNestedPlan.code === 2 && untrackedNestedPlan.err.includes('nested-repository-skipped'), 'an untracked nested Git repository fails plan as incomplete coverage')
+const allowedUntrackedNested = run([untrackedNestedRepo, '--allow-incomplete', '--json'])
+const allowedUntrackedNestedJson = JSON.parse(allowedUntrackedNested.out)
+assert(
+  allowedUntrackedNested.code === 0 &&
+    allowedUntrackedNestedJson.findings.length === 0 &&
+    allowedUntrackedNestedJson.scan_notes.some(note => note.reason === 'nested-repository-skipped' && note.file.endsWith('/nested')),
+  'allow-incomplete records an untracked nested repository without recursively scanning its retired reference',
+)
+const untrackedNestedConfig = path.join(untrackedNestedRepo, '.model-eol.json')
+fs.writeFileSync(untrackedNestedConfig, '{"ignore":{"paths":["nested"]}}\n')
+const ignoredUntrackedNested = run([untrackedNestedRepo, '--json'])
+assert(
+  ignoredUntrackedNested.code === 0 &&
+    JSON.parse(ignoredUntrackedNested.out).scan_notes.every(note => note.reason !== 'nested-repository-skipped'),
+  'repository path policy can explicitly accept an intentional untracked nested repository',
+)
+
 const trackedSubmoduleRepo = path.join(tempRoot, 'tracked-submodule-repo')
 const trackedSubmodulePath = path.join(trackedSubmoduleRepo, 'sub')
 fs.mkdirSync(trackedSubmodulePath, { recursive: true })
@@ -731,6 +772,23 @@ try {
   invalidControlMessage = error.message
 }
 assert(invalidControlMessage.includes('invalid-control.json') && invalidControlMessage.includes('replacement') && invalidControlMessage.includes('control characters'), 'loadFeeds rejects control characters in replacement fields')
+const invalidUtf8Feeds = path.join(tempRoot, 'invalid-utf8-feeds')
+const invalidUtf8Feed = path.join(invalidUtf8Feeds, 'invalid-utf8.json')
+fs.mkdirSync(invalidUtf8Feeds)
+fs.writeFileSync(invalidUtf8Feed, Buffer.concat([
+  Buffer.from('{"spec":"model-eol/0.1","publisher":"test","generated":"2026-08-01T00:00:00Z","source":"https://example.invalid/utf8","note":"bad-'),
+  Buffer.from([0xff]),
+  Buffer.from('","models":[{"id":"utf8-test-model"}]}\n'),
+]))
+let invalidUtf8FeedMessage = ''
+try {
+  loadFeeds(invalidUtf8Feeds)
+} catch (error) {
+  invalidUtf8FeedMessage = error.message
+}
+assert(invalidUtf8FeedMessage.includes('invalid-utf8.json') && invalidUtf8FeedMessage.includes('invalid UTF-8'), 'operational feed loading rejects invalid UTF-8 with the feed filename')
+const invalidUtf8FeedRun = run(['check', path.join(root, 'test/fixture'), '--feeds', invalidUtf8Feeds, '--json'])
+assert(invalidUtf8FeedRun.code === 2 && invalidUtf8FeedRun.err.includes('invalid-utf8.json') && invalidUtf8FeedRun.err.includes('invalid UTF-8'), 'check exits 2 instead of loading a replacement-decoded feed')
 
 const federationDir = path.join(tempRoot, 'federation')
 const federationFeeds = path.join(federationDir, 'feeds')
@@ -808,6 +866,18 @@ assert(changedRun.out.trim().startsWith('{'), `--changed emits JSON (stderr: ${c
 const changedJson = JSON.parse(changedRun.out)
 assert(changedRun.code === 1, '--changed fails for an added bad model')
 assert(changedJson.findings.length === 1 && changedJson.findings[0].id === 'claude-opus-4-1-20250805', '--changed pins parseable prefixes and color despite hostile Git config')
+const changedExpressionRun = run(['check', changedRepo, '--days', '30', '--changed', 'HEAD~0', '--json'])
+assert(changedExpressionRun.code === 1 && JSON.parse(changedExpressionRun.out).findings.length === 1, '--changed resolves ordinary revision expressions to a commit')
+const injectedDiffOutput = path.join(changedRepo, 'injected.diff')
+const injectedChanged = run(['check', changedRepo, '--days', '30', `--changed=--output=${injectedDiffOutput}`, '--json'])
+assert(
+  injectedChanged.code === 2 &&
+    injectedChanged.err.includes('base ref must not begin') &&
+    !fs.existsSync(injectedDiffOutput),
+  '--changed rejects leading-option input without allowing Git to create an output file',
+)
+const missingChangedBase = run(['check', changedRepo, '--days', '30', '--changed', 'definitely-not-a-ref', '--json'])
+assert(missingChangedBase.code === 2 && missingChangedBase.err.includes('could not resolve base ref'), '--changed fails closed when its revision does not resolve to a commit')
 assert(parseDiffPath('"b/path with spaces.py"') === 'path with spaces.py', 'Git diff path parsing preserves ordinary quoted paths with spaces')
 assert(parseDiffPath('"b/\\303\\251.py"') === 'é.py', 'Git diff path parsing decodes octal UTF-8 bytes')
 let malformedDiffPathFailed = false