From 7aa1d214d8248b70d0d0103240d59071ac3dbacb Mon Sep 17 00:00:00 2001 From: localhost41 Date: Thu, 9 Jul 2026 16:54:13 -0700 Subject: [PATCH] Add qvac-bench release hardening --- .github/workflows/ci.yml | 7 +- CHANGELOG.md | 11 ++- LICENSE | 21 ++++++ README.md | 13 +++- docs/validation-checklist.md | 24 ++++++- package.json | 6 +- scripts/verify-package.mjs | 130 ++++++++++++++++++++++++++++++++--- 7 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 LICENSE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 742bc9c..c1463a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,10 @@ on: jobs: build: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [22, 24, 26] steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -15,7 +19,8 @@ jobs: version: 11.10.0 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: ${{ matrix.node-version }} + cache: pnpm - name: Install dependencies run: | if [ -f pnpm-lock.yaml ]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index d18a916..b0f7f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,13 @@ ## Unreleased -- Initial repository scaffold. +- No unreleased changes. + +## 0.1.0-alpha.0 - 2026-07-09 + +- Initial alpha CLI for OpenAI-compatible streaming QVAC benchmarks. +- Added prompt fixtures, repeated-run summaries, JSON and CSV output, and + sanitized endpoint error handling. +- Added package verification for packed assets and packed-tarball installability. +- Added validation checklist and benchmark report template with real-QVAC + validation explicitly unresolved until live runs are recorded. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..feb2cef --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LocalHost Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 0fb0866..c067507 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,13 @@ pnpm verify:package This runs lint, tests, build, and an `npm pack --dry-run` integrity check. The package check verifies the packed files include the CLI entrypoint, README, -CHANGELOG, and type declarations, and that the package bin points to an existing -built file. +CHANGELOG, LICENSE, type declarations, and shipped docs under `docs/`. It also +packs the real tarball, installs it into a temporary consumer project, imports +the public API, runs the installed `qvac-bench --help` bin, and confirms the +report and validation templates are present after install. + +Supported runtime majors are Node.js 22, 24, and 26. CI runs the full package +verification flow across that matrix. ## CLI @@ -101,7 +106,9 @@ CI runs a smoke test against a lightweight local mock OpenAI-compatible streamin endpoint. This verifies CLI behavior, streaming parsing, time to first token, total duration, and approximate tokens/sec without requiring network access or a live QVAC server. Real QVAC server validation should be run separately against the -target QVAC build and model. +target QVAC build and model, and must be tracked in +[docs/validation-checklist.md](docs/validation-checklist.md). Until a live run is +recorded there, real-QVAC validation is unresolved. ## Methodology diff --git a/docs/validation-checklist.md b/docs/validation-checklist.md index d6d176e..1bdc0f4 100644 --- a/docs/validation-checklist.md +++ b/docs/validation-checklist.md @@ -12,6 +12,23 @@ all automated tests use in‑memory mock servers. > QVAC deployment. Every item that still needs verification is marked as > **[UNRESOLVED]**. +## Release Readiness Status + +Current package version: `0.1.0-alpha.0` + +As of 2026-07-09: + +- Automated mock validation: covered by `pnpm verify:package` and CI on Node.js + 22, 24, and 26. Record the exact CI run before release. +- Packed package installability: covered by `pnpm verify:package`, which packs + the tarball, installs it into a temporary consumer project, imports the public + API, runs the installed CLI help command, and checks shipped docs. +- Real-QVAC validation: **[UNRESOLVED]**. No passing live-QVAC run is claimed by + this repository until the matrix in [Real QVAC Validation](#2-real-qvac-validation) + includes dated pass entries with endpoint, model, command, and notes. +- Public benchmark claims: **blocked** until real-QVAC validation is completed + and a benchmark report is filled from `docs/reports/template.md`. + --- ## 1. Mock CI Validation @@ -32,8 +49,9 @@ run on any developer machine or CI agent – no QVAC access is needed. | Environment | Steps | Expected Outcome | Pass? | Date | Notes | | ------------------------------- | --------------------------------------------------------------------- | ----------------------------------- | :---: | ---------- | ----- | -| macOS 14 (Apple Silicon) – local | `pnpm install && pnpm test && pnpm build` | All tests green; build succeeds | ☐ | | Run on a clean checkout. **[UNRESOLVED]** | -| Linux (fresh install) | `pnpm install && pnpm test && pnpm build` | All tests green; build succeeds | ☐ | | Node.js 20 LTS from a clean image. **[UNRESOLVED]** | +| macOS 14 (Apple Silicon) – local | `pnpm install && pnpm verify:package` | All checks green; tarball installs | ☐ | | Run on a clean checkout. **[UNRESOLVED]** | +| Linux (fresh install) | `pnpm install && pnpm verify:package` | All checks green; tarball installs | ☐ | | Node.js 22, 24, or 26 from a clean image. **[UNRESOLVED]** | +| GitHub Actions Linux | CI workflow matrix | Node.js 22, 24, and 26 all pass | ☐ | | Link the workflow run before release. **[UNRESOLVED]** | ### 1.3 Additional Mock Checks @@ -68,7 +86,7 @@ pipeline that does not have access to the real endpoint. | Environment | QVAC URL | Model | Command (example) | Expected Output | Pass? | Date | Notes | | ------------------------------- | ----------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | :---: | ---------- | ----- | | macOS 14 (Apple Silicon) – local | `` | `gpt-4o-mini` | `node dist/cli.js --url $QVAC_URL --model gpt-4o-mini --prompt "Hello" --max-tokens 50 --iterations 3` | Time‑to‑first‑token and total‑time stats printed; exit code 0. | ☐ | | Run after a clean build. **[UNRESOLVED]** | -| Linux (fresh install) | `` | `gpt-4o-mini` | `node dist/cli.js --url $QVAC_URL --model gpt-4o-mini --prompt "Hello" --max-tokens 50 --iterations 3` | Same as above; no errors. | ☐ | | Node.js 20 LTS from a clean image. **[UNRESOLVED]** | +| Linux (fresh install) | `` | `gpt-4o-mini` | `node dist/cli.js --url $QVAC_URL --model gpt-4o-mini --prompt "Hello" --max-tokens 50 --iterations 3` | Same as above; no errors. | ☐ | | Node.js 22, 24, or 26 from a clean image. **[UNRESOLVED]** | ### 2.3 Steps for Each Row diff --git a/package.json b/package.json index 66d1be4..3102bbd 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,17 @@ "name": "@localhostlabs/qvac-bench", "version": "0.1.0-alpha.0", "description": "QVAC developer tooling package: qvac-bench", + "license": "MIT", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", "bin": { "qvac-bench": "dist/cli.js" }, - "files": ["dist", "README.md", "CHANGELOG.md"], + "files": ["dist", "docs", "README.md", "CHANGELOG.md", "LICENSE"], + "engines": { + "node": ">=22 <27" + }, "scripts": { "build": "node scripts/clean-dist.mjs && tsc -p tsconfig.build.json", "test": "vitest run test", diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 1327a26..bea88c2 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -19,6 +19,17 @@ const binEntries = typeof packageJson.bin === "string" ? [[packageJson.name, packageJson.bin]] : Object.entries(packageJson.bin ?? {}); +const requiredPublishedFiles = new Set([ + "package.json", + "README.md", + "CHANGELOG.md", + "LICENSE", + "docs/reports/template.md", + "docs/validation-checklist.md", + packageJson.main, + packageJson.types, + ...binEntries.map(([, binPath]) => binPath) +]); for (const [name, binPath] of binEntries) { if (!existsSync(join(rootDir, binPath))) { @@ -59,15 +70,8 @@ try { const packedPackage = packOutput[0]; const packedFiles = new Set((packedPackage?.files ?? []).map((file) => file.path)); -const requiredFiles = new Set([ - "README.md", - "CHANGELOG.md", - packageJson.main, - packageJson.types, - ...binEntries.map(([, binPath]) => binPath) -]); -for (const requiredFile of requiredFiles) { +for (const requiredFile of requiredPublishedFiles) { if (!packedFiles.has(requiredFile)) { fail(`packed package is missing ${requiredFile}`); } @@ -77,4 +81,110 @@ if (![...packedFiles].some((file) => file.endsWith(".d.ts"))) { fail("packed package does not include type declarations"); } -console.log(`Package verification passed: ${packedPackage.filename}`); +const installRootDir = mkdtempSync(join(tmpdir(), "qvac-bench-install-")); +const installCacheDir = mkdtempSync(join(tmpdir(), "qvac-bench-npm-install-cache-")); + +try { + const packResult = spawnSync("npm", ["pack", rootDir, "--json"], { + cwd: installRootDir, + encoding: "utf8", + env: { + ...process.env, + npm_config_cache: installCacheDir + } + }); + + if (packResult.error) { + fail(`npm pack could not start: ${packResult.error.message}`); + } + + if (packResult.status !== 0) { + fail(`npm pack exited with ${packResult.status}\n${packResult.stderr.trim()}`); + } + + let packInstallOutput; + try { + packInstallOutput = JSON.parse(packResult.stdout); + } catch { + fail(`npm pack returned invalid JSON\n${packResult.stdout.trim()}`); + } + + const tarballName = packInstallOutput[0]?.filename; + if (typeof tarballName !== "string") { + fail("npm pack did not report a tarball filename"); + } + + const tarballPath = join(installRootDir, tarballName); + const installProjectDir = join(installRootDir, "consumer"); + mkdirSync(installProjectDir); + writeFileSync( + join(installProjectDir, "package.json"), + JSON.stringify({ private: true, type: "module" }, null, 2) + ); + + const installResult = spawnSync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", tarballPath], { + cwd: installProjectDir, + encoding: "utf8", + env: { + ...process.env, + npm_config_cache: installCacheDir + } + }); + + if (installResult.error) { + fail(`npm install packed tarball could not start: ${installResult.error.message}`); + } + + if (installResult.status !== 0) { + fail(`npm install packed tarball exited with ${installResult.status}\n${installResult.stderr.trim()}`); + } + + const importResult = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + [ + `import { name, promptNames } from ${JSON.stringify(packageJson.name)};`, + "if (name() !== 'qvac-bench') throw new Error('unexpected package name export');", + "if (!promptNames().includes('hello')) throw new Error('missing prompt fixture export');" + ].join("\n") + ], + { + cwd: installProjectDir, + encoding: "utf8" + } + ); + + if (importResult.status !== 0) { + fail(`installed package import smoke test failed\n${importResult.stderr.trim()}`); + } + + const binResult = spawnSync("npx", ["--no-install", "qvac-bench", "--help"], { + cwd: installProjectDir, + encoding: "utf8", + env: { + ...process.env, + npm_config_cache: installCacheDir + } + }); + + if (binResult.error) { + fail(`installed package bin smoke test could not start: ${binResult.error.message}`); + } + + if (binResult.status !== 0 || !binResult.stdout.includes("Usage: qvac-bench [options]")) { + fail(`installed package bin smoke test failed\n${binResult.stderr.trim()}`); + } + + for (const docsFile of ["docs/reports/template.md", "docs/validation-checklist.md"]) { + if (!existsSync(join(installProjectDir, "node_modules", packageJson.name, docsFile))) { + fail(`installed package is missing ${docsFile}`); + } + } + + console.log(`Package verification passed: ${packedPackage.filename}`); +} finally { + rmSync(installRootDir, { recursive: true, force: true }); + rmSync(installCacheDir, { recursive: true, force: true }); +}