From 0114e20ec52f1dcf6c33edc25545275c23b65c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B8ran=20A=2E=20Slettemark?= Date: Tue, 21 Jul 2026 14:36:40 +0200 Subject: [PATCH 1/5] Added ops console --- .env.example | 22 +- .github/workflows/test.yml | 23 +- Dockerfile.router | 2 +- README.md | 130 +++- app/JobQueue.d.ts | 35 +- app/JobQueue.js | 47 +- app/JobQueue.js.map | 2 +- app/builder-service.js | 390 ++++++++++-- app/download.js | 13 +- app/downloader-service.js | 342 ++++++++--- app/ops/cache.js | 285 +++++++++ app/ops/config.js | 138 +++++ app/ops/console-router.js | 484 +++++++++++++++ app/ops/http.js | 275 +++++++++ app/ops/internal-router.js | 96 +++ app/ops/schemas.js | 383 ++++++++++++ app/ops/sessions.js | 229 +++++++ app/ops/telemetry.js | 417 +++++++++++++ app/router.js | 119 +++- app/server.js | 33 +- app/service-client.js | 101 ++- app/utils.js | 4 +- app/utils.js.map | 2 +- builder-server.js | 40 +- compose.ops-test.yaml | 65 ++ compose.yaml | 10 +- docs/operations-console.md | 218 ++++++- downloader-server.js | 54 +- package-lock.json | 255 ++++++++ package.json | 5 + playwright.ops.config.js | 24 + scripts/test-ops-integration.js | 194 ++++++ src/JobQueue.ts | 88 ++- src/tsconfig.json | 3 +- src/utils.ts | 11 +- static/ops/console.css | 430 +++++++++++++ static/ops/console.js | 484 +++++++++++++++ static/ops/index.html | 127 ++++ static/ops/login.html | 32 + static/ops/login.js | 37 ++ test/JobQueue.test.js | 99 +-- test/browser/ops.spec.js | 167 +++++ test/builder-service.js | 249 +++++++- test/download.js | 74 ++- test/downloader-service.js | 224 ++++++- test/fixtures/mtls/ca.crt | 20 + test/fixtures/mtls/client.crt | 21 + test/fixtures/mtls/client.key | 28 + test/fixtures/mtls/server.crt | 21 + test/fixtures/mtls/server.key | 28 + test/fixtures/mtls/untrusted-ca.crt | 20 + test/fixtures/mtls/untrusted-client.crt | 21 + test/fixtures/mtls/untrusted-client.key | 28 + test/hurl/ops-auth.hurl | 205 +++++++ test/hurl/ops-cache.hurl | 121 ++++ test/hurl/ops-security.hurl | 33 + test/hurl/ops-snapshot.hurl | 99 +++ test/ops-cache.js | 218 +++++++ test/ops-config.js | 106 ++++ test/ops-console.js | 779 ++++++++++++++++++++++++ test/ops-http.js | 445 ++++++++++++++ test/ops-internal.js | 156 +++++ test/ops-telemetry.js | 166 +++++ test/ops/fixture-service.js | 141 +++++ test/ops/stack.js | 106 ++++ test/router.js | 62 +- test/server.js | 178 +++++- test/test.js | 7 + 68 files changed, 9098 insertions(+), 373 deletions(-) create mode 100644 app/ops/cache.js create mode 100644 app/ops/config.js create mode 100644 app/ops/console-router.js create mode 100644 app/ops/http.js create mode 100644 app/ops/internal-router.js create mode 100644 app/ops/schemas.js create mode 100644 app/ops/sessions.js create mode 100644 app/ops/telemetry.js create mode 100644 compose.ops-test.yaml create mode 100644 playwright.ops.config.js create mode 100644 scripts/test-ops-integration.js create mode 100644 static/ops/console.css create mode 100644 static/ops/console.js create mode 100644 static/ops/index.html create mode 100644 static/ops/login.html create mode 100644 static/ops/login.js create mode 100644 test/browser/ops.spec.js create mode 100644 test/fixtures/mtls/ca.crt create mode 100644 test/fixtures/mtls/client.crt create mode 100644 test/fixtures/mtls/client.key create mode 100644 test/fixtures/mtls/server.crt create mode 100644 test/fixtures/mtls/server.key create mode 100644 test/fixtures/mtls/untrusted-ca.crt create mode 100644 test/fixtures/mtls/untrusted-client.crt create mode 100644 test/fixtures/mtls/untrusted-client.key create mode 100644 test/hurl/ops-auth.hurl create mode 100644 test/hurl/ops-cache.hurl create mode 100644 test/hurl/ops-security.hurl create mode 100644 test/hurl/ops-snapshot.hurl create mode 100644 test/ops-cache.js create mode 100644 test/ops-config.js create mode 100644 test/ops-console.js create mode 100644 test/ops-http.js create mode 100644 test/ops-internal.js create mode 100644 test/ops-telemetry.js create mode 100644 test/ops/fixture-service.js create mode 100644 test/ops/stack.js diff --git a/.env.example b/.env.example index c988caf..1bad03a 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ # Copy this file to .env, replace the secret placeholders, then run: # docker compose up --build --wait -# The router is available at http://localhost:${ROUTER_PORT:-8080}. +# The router is available at http://localhost:${ROUTER_PORT:-8080} by default. # Downloader and builder have no host ports and are reachable only by service name. # Shared by all services. Generate a long random value; do not commit it. @@ -10,11 +10,29 @@ WEBHOOK_SECRET= # Downloader only. Optional for local use, but avoids GitHub API rate limits. GITHUB_TOKEN= -# Host-facing router port and internal service URLs. +# Host-facing router bind/port and internal service URLs. +# Keep ROUTER_BIND_ADDRESS on loopback for local development; set explicitly if needed. +ROUTER_BIND_ADDRESS=127.0.0.1 ROUTER_PORT=8080 DOWNLOADER_URL=http://downloader:8080 BUILDER_URL=http://builder:8080 +# Operations console: disabled by default and intended for development only here. +# Never store the raw operator token in Compose/environment. Store only the +# canonical verifier form: v1.. +OPS_CONSOLE_ENABLED=false +OPS_CONSOLE_TOKEN_VERIFIER= +# Local HTTP requires OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true and an exact loopback origin. +OPS_CONSOLE_ORIGIN= +OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=false +# Local default remains HTTP on loopback and does not require mTLS. +OPS_CONSOLE_MTLS_PORT=8443 +# Production HTTPS requires absolute in-container paths to mounted secrets. +# Add an explicit read-only mount override in Compose matching these paths. +OPS_CONSOLE_MTLS_KEY_PATH= +OPS_CONSOLE_MTLS_CERT_PATH= +OPS_CONSOLE_MTLS_CA_PATH= + # Ephemeral cache roots are fixed by compose.yaml and owned by UID/GID 1000. # They are separate tmpfs mounts and are discarded when containers stop. DOWNLOADER_CACHE_ROOT=/app/downloader-cache diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eff4ea1..9ae6fe5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,22 +6,26 @@ on: - master - develop +permissions: + contents: read + jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: '24' - cache: 'npm' - name: Install dependencies - run: npm install + run: npm ci - name: Check TypeScript compilation is up to date run: | @@ -38,3 +42,12 @@ jobs: - name: Run tests run: npm test + + - name: Run operations console integration tests + run: npm run test:integration:ops + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium firefox + + - name: Run operations console browser tests + run: npm run test:browser:ops diff --git a/Dockerfile.router b/Dockerfile.router index 226cff2..af7b549 100644 --- a/Dockerfile.router +++ b/Dockerfile.router @@ -20,6 +20,6 @@ COPY --chown=node:node assets ./assets COPY --chown=node:node static ./static USER node -EXPOSE 8080 +EXPOSE 8080 8443 ENTRYPOINT ["/sbin/tini", "--"] CMD ["npm", "run", "start:router"] diff --git a/README.md b/README.md index 72580b9..e234db7 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,134 @@ Node.js server that serves Highcharts distribution files built on demand from an ## Operations console -A **development-only** secured operations console is specified for the router. -See [docs/operations-console.md](docs/operations-console.md) for the full specification. +> **Development-only.** The operations console must never be enabled in staging or production. + +A secured operations console is built into the router and is disabled by default. When enabled, it lets trusted Highsoft engineers inspect system health, browse recent request activity, trace individual requests, and perform cache-maintenance operations — all from a same-origin browser UI served at `/_ops/`. + +The full normative specification (1 479 lines) is at [docs/operations-console.md](docs/operations-console.md). This section covers everything needed to configure, start, and operate it locally. + +### Topology + +The console does not change the three-process topology. Only the router exposes a port; the downloader and builder remain on the private Docker network. The browser never contacts internal services directly and never receives the internal bearer token. Console telemetry and cache state are available only under bearer-protected `/v1/ops/*` paths on each internal service. + +### Environment variables + +| Variable | Required when | Description | +|---|---|---| +| `OPS_CONSOLE_ENABLED` | Always evaluated | Must be exactly the string `true` to enable the console. Any other value or absence keeps the console off and every `/_ops/*` path returns 404. | +| `OPS_CONSOLE_TOKEN_VERIFIER` | Console enabled | Canonical domain-separated SHA-256 verifier derived from the shared admin token (see below). Never store the raw token. Validated at startup; missing or malformed values abort startup. | +| `OPS_CONSOLE_ORIGIN` | Console enabled | The single exact external Origin the console accepts for all state-changing requests and for login (e.g. `http://localhost:8080`). Missing or ambiguous values abort startup. | +| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | Set to exactly `true` to allow HTTP only when the Origin is a loopback address and no mTLS settings are present. All other HTTP/mTLS combinations abort startup. | +| `OPS_CONSOLE_MTLS_PORT` | HTTPS console | Port for the mTLS listener. Defaults to `8443`. Must not be combined with HTTP loopback mode. | +| `OPS_CONSOLE_MTLS_KEY_PATH` | HTTPS console | Absolute in-container path to the server TLS private key PEM file. | +| `OPS_CONSOLE_MTLS_CERT_PATH` | HTTPS console | Absolute in-container path to the server TLS certificate PEM file. | +| `OPS_CONSOLE_MTLS_CA_PATH` | HTTPS console | Absolute in-container path to the Envoy client CA certificate PEM file. The router uses this CA to verify the client certificate that Envoy presents during the mTLS handshake. The file must be a CA certificate; startup fails if it is not. This CA is the Envoy-client-issuing CA and is distinct from the server CA used in Contour's `validation.caSecret`. | +| `ROUTER_BIND_ADDRESS` | Optional | Host address the router container binds to. Defaults to `127.0.0.1` in Docker Compose; the console is not reachable from outside loopback unless this is changed deliberately. | + +`OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup. + +**Verifier, not token.** `OPS_CONSOLE_TOKEN_VERIFIER` stores a one-way verifier, never the raw token. Derive it from a canonical 32-byte CSPRNG token (base64url-encoded) with: + +```bash +node -e " +const { createHash } = require('crypto'); +const sep = 'github.highcharts.com:ops-console:v1\x00'; +const raw = Buffer.from('', 'base64url'); +const digest = createHash('sha256').update(sep).update(raw).digest('base64url'); +console.log('v1.' + digest); +" +``` + +Set the printed `v1.` string as `OPS_CONSOLE_TOKEN_VERIFIER` in your `.env`. Keep the raw token only in your password manager or 1Password — never in `.env`, source control, logs, or browser storage. + +### Login and session behaviour + +Navigate to `/_ops/login` and submit the raw base64url token. On success the router creates an opaque server-side session (30-minute idle expiry, 8-hour absolute maximum) and sets a `HttpOnly; SameSite=Strict` cookie. Sessions are router-local and in-memory; restarting the router revokes all active sessions. The UI warns five minutes before expiry and returns to the login page on expiry or logout. + +### Snapshot and cache operations + +`GET /_ops/api/v1/snapshot` returns a concurrent aggregate of router, downloader, and builder state: health, queue depth, cache summary, and recent request activity. Each internal call is bounded to 1.5 seconds. + +`POST /_ops/api/v1/cache-operations` drives cache maintenance: evict a single commit, purge expired entries, or clear a full service cache. Each operation is bounded to 10 seconds. All mutations are recorded as a single sanitized audit event in the router log. + +### Local Compose setup + +```bash +# 1. Copy the example env file (only needed once) +cp .env.example .env + +# 2. Generate and store the raw token in your password manager, then derive the verifier: +node -e " +const { createHash, randomBytes } = require('crypto'); +const sep = 'github.highcharts.com:ops-console:v1\x00'; +const raw = randomBytes(32); +console.log('Raw token (save to password manager):', raw.toString('base64url')); +const digest = createHash('sha256').update(sep).update(raw).digest('base64url'); +console.log('Verifier (put in .env):', 'v1.' + digest); +" + +# 3. Edit .env — set the required console variables: +# OPS_CONSOLE_ENABLED=true +# OPS_CONSOLE_TOKEN_VERIFIER=v1. +# OPS_CONSOLE_ORIGIN=http://localhost:8080 +# OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true +# (leave OPS_CONSOLE_MTLS_* blank for local HTTP loopback mode) + +# 4. Start the stack +docker compose up --build --wait + +# 5. Open http://localhost:8080/_ops/login and log in with the raw token + +# 6. To disable the console again, set OPS_CONSOLE_ENABLED=false (or remove it) and restart: +docker compose up --build --wait +``` + +### Running the console tests + +```bash +# Full unit and lint suite (includes ops-console and ops-config tests) +npm test + +# Integration smoke tests against a running stack (requires docker compose up) +npm run test:integration:ops + +# Browser/accessibility tests (requires a running stack and Playwright browsers installed) +npm run test:browser:ops +``` + +### Operational checklist + +This checklist covers staged local rollout, abort criteria, and rollback. + +**Enable:** + +1. Derive the verifier and set `OPS_CONSOLE_ENABLED=true` plus the three required variables in `.env`. +2. For local HTTP loopback mode, set `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` and leave all `OPS_CONSOLE_MTLS_*` variables blank. For HTTPS mTLS mode, supply absolute in-container paths for `OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and `OPS_CONSOLE_MTLS_CA_PATH`. +3. Restart the router (`docker compose up --build --wait`). Confirm startup log shows `ops-console enabled`. +4. Open `/_ops/login`; confirm login succeeds and the snapshot page loads with data from all three services. +5. Run `npm run test:integration:ops` — all checks must pass. + +**Abort criteria — disable immediately if any of the following occur:** + +- `/_ops/*` paths are reachable from outside the intended network boundary. +- Login returns a session without a valid token being submitted. +- Any audit event is absent from the router log after a cache mutation. +- Internal bearer token appears in a browser response or cookie. +- Router startup fails with a missing-verifier, missing TLS path, or invalid TLS material error that cannot be resolved within 10 minutes. + +**Disable and roll back:** + +```bash +# Remove or unset OPS_CONSOLE_ENABLED in .env (or set it to any value other than "true"), +# then restart the router: +docker compose up --build --wait + +# Confirm every /_ops/* path returns 404: +curl -o /dev/null -w "%{http_code}" http://localhost:8080/_ops/ +# Expected: 404 +``` + +If the router itself must be rolled back, re-tag the prior immutable image to the environment tag and restart the container. The router can be rolled back independently of the internal services provided internal API compatibility is maintained. ## Architecture diff --git a/app/JobQueue.d.ts b/app/JobQueue.d.ts index 5bc68c3..271d0c2 100644 --- a/app/JobQueue.d.ts +++ b/app/JobQueue.d.ts @@ -1,10 +1,33 @@ -export type JobArgs = { +type JobArgs = { func: (...args: any[]) => Promise; args: any[]; }; -export type JobType = JobArgs & { - setDone: (isDone: true) => void; - done: Promise; +type JobType = JobArgs & { + setDone: (result: any) => void; + setFailed: (error: unknown) => void; + done: Promise; + queuedAt: number; }; -export type Queues = 'download' | 'compile'; -export type QueueType = Map; +type Queues = 'download' | 'compile'; +type QueueType = Map; +type QueueMetrics = { + active: number; + queued: number; + limit: number; + available: number; + oldestQueuedAgeMs: number | null; +}; +declare class JobQueue { + static _instance: JobQueue; + private static queues; + private static churns; + doJob(queue: QueueType, jobID: string): Promise; + churn(queue: QueueType): Promise; + private makeJob; + addJob(type: Queues, jobID: string, job: JobArgs): Promise; + getJobs(type: Queues): [string, JobType][]; + getJobPromises(type: Queues): JobType[]; + getMetrics(type: Queues, now?: number): QueueMetrics; + constructor(); +} +export type { JobArgs, JobType, Queues, QueueType, QueueMetrics, JobQueue }; diff --git a/app/JobQueue.js b/app/JobQueue.js index a9d8818..64dd74f 100644 --- a/app/JobQueue.js +++ b/app/JobQueue.js @@ -1,10 +1,6 @@ 'use strict' -const __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { default: mod } -} Object.defineProperty(exports, '__esModule', { value: true }) /* eslint-disable camelcase */ -const node_crypto_1 = __importDefault(require('node:crypto')) const node_process_1 = require('node:process') // Max queue size per queue const MAX_QUEUE_SIZE = Number(node_process_1.env.MAX_QUEUE_SIZE) || 2 @@ -20,19 +16,18 @@ class JobQueue { compile: false } - id async doJob (queue, jobID) { const job = queue.get(jobID) if (job) { try { - await Promise.resolve().then(() => job.func(...job.args)) + const result = await Promise.resolve().then(() => job.func(...job.args)) + job.setDone(result) } catch (error) { - console.log(error) + job.setFailed(error) + throw error } finally { queue.delete(jobID) - job.setDone(true) } - await job.done } } @@ -54,15 +49,16 @@ class JobQueue { } makeJob (job) { - const myJob = { + const { promise: done, resolve: setDone, reject: setFailed } = Promise.withResolvers() + // Prevent an ignored caller promise from becoming an unhandled rejection. + done.catch(() => { }) + return { ...job, - setDone () { }, - done: Promise.resolve(true) + setDone, + setFailed, + done, + queuedAt: Date.now() } - myJob.done = new Promise((resolve) => { - myJob.setDone = resolve - }) - return myJob } addJob (type, jobID, job) { @@ -73,13 +69,14 @@ class JobQueue { return Promise.reject(error) } if (queue.has(jobID)) { - return queue.get(jobID)?.done + return queue.get(jobID).done } const transformedJob = this.makeJob(job) queue.set(jobID, transformedJob) if (!JobQueue.churns[type]) { JobQueue.churns[type] = true - return this.churn(queue) + this.churn(queue) + .catch(console.error) .finally(() => { JobQueue.churns[type] = false }) @@ -97,8 +94,20 @@ class JobQueue { return Array.from(queue.values()) } + getMetrics (type, now = Date.now()) { + const queue = JobQueue.queues[type] + const active = JobQueue.churns[type] && queue.size > 0 ? 1 : 0 + const waiting = Array.from(queue.values()).slice(active) + return { + active, + queued: waiting.length, + limit: MAX_QUEUE_SIZE, + available: Math.max(0, MAX_QUEUE_SIZE - queue.size), + oldestQueuedAgeMs: waiting.length ? Math.max(0, now - waiting[0].queuedAt) : null + } + } + constructor () { - this.id = node_crypto_1.default.randomUUID() if (JobQueue._instance) { return JobQueue._instance } diff --git a/app/JobQueue.js.map b/app/JobQueue.js.map index 9ed4085..08db14d 100644 --- a/app/JobQueue.js.map +++ b/app/JobQueue.js.map @@ -1 +1 @@ -{"version":3,"file":"JobQueue.js","sourceRoot":"","sources":["../src/JobQueue.ts"],"names":[],"mappings":";;;;;AAWA,8BAA8B;AAC9B,8DAAiC;AACjC,+CAAmC;AAEnC,2BAA2B;AAC3B,MAAM,cAAc,GAAG,MAAM,CAAC,kBAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAEvD,MAAM,QAAQ;IACV,MAAM,CAAC,SAAS,CAAU;IAElB,MAAM,CAAC,MAAM,GAA8B;QAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;QACnB,OAAO,EAAE,IAAI,GAAG,EAAE;KACrB,CAAA;IAEO,MAAM,CAAC,MAAM,GAA4B;QAC7C,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,KAAK;KACjB,CAAC;IAEK,EAAE,CAAS;IAElB,KAAK,CAAC,KAAK,CAAC,KAAgB,EAAE,KAAa;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAE7B,IAAI,GAAG,EAAE,CAAC;YACN,IAAI,CAAC;gBACD,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9D,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;oBACO,CAAC;gBACL,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,CAAC;QACnB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAgB;QACxB,YAAY;QACZ,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAExC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAClC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;gBACO,CAAC;YACL,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnC,CAAC;QAED,kCAAkC;QAClC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAEO,OAAO,CAAC,GAAY;QACxB,MAAM,KAAK,GAAY;YACnB,GAAG,GAAG;YACN,OAAO,KAAK,CAAC;YACb,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;SAC9B,CAAC;QAEF,KAAK,CAAC,IAAI,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACjC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QAC5B,CAAC,CAAC,CAAA;QAEF,OAAO,KAAK,CAAC;IACjB,CAAC;IAEM,MAAM,CAAC,IAAY,EAAE,KAAa,EAAE,GAAY;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;YAC3F,KAAK,CAAC,IAAI,GAAG,gBAAgB,CAAC;YAE9B,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAExC,KAAK,CAAC,GAAG,CACL,KAAK,EACL,cAAc,CACjB,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YAE7B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;iBACnB,OAAO,CAAC,GAAG,EAAE;gBACV,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAClC,CAAC,CAAC,CAAA;QACV,CAAC;QAED,OAAO,cAAc,CAAC,IAAI,CAAC;IAC/B,CAAC;IAEM,OAAO,CAAC,IAAY;QACvB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACvC,CAAC;IAEM,cAAc,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;IAED;QACI,IAAI,CAAC,EAAE,GAAG,qBAAM,CAAC,UAAU,EAAE,CAAA;QAC7B,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,SAAS,CAAA;QAC7B,CAAC;QAED,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IAC7B,CAAC;;AAGL,MAAM,CAAC,OAAO,GAAG;IACb,QAAQ;CACX,CAAA"} \ No newline at end of file +{"version":3,"file":"JobQueue.js","sourceRoot":"","sources":["../src/JobQueue.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;AAsBb,8BAA8B;AAC9B,+CAAmC;AAEnC,2BAA2B;AAC3B,MAAM,cAAc,GAAG,MAAM,CAAC,kBAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAEvD,MAAM,QAAQ;IACV,MAAM,CAAC,SAAS,CAAU;IAElB,MAAM,CAAC,MAAM,GAA8B;QAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;QACnB,OAAO,EAAE,IAAI,GAAG,EAAE;KACrB,CAAA;IAEO,MAAM,CAAC,MAAM,GAA4B;QAC7C,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,KAAK;KACjB,CAAC;IAEF,KAAK,CAAC,KAAK,CAAC,KAAgB,EAAE,KAAa;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAE7B,IAAI,GAAG,EAAE,CAAC;YACN,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;gBACzE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACX,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACrB,MAAM,KAAK,CAAC;YAChB,CAAC;oBACO,CAAC;gBACL,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAgB;QACxB,YAAY;QACZ,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAExC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAClC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;gBACO,CAAC;YACL,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnC,CAAC;QAED,kCAAkC;QAClC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAEO,OAAO,CAAC,GAAY;QACxB,MAAM,EACF,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,SAAS,EACpB,GAAG,OAAO,CAAC,aAAa,EAAO,CAAC;QAEjC,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAErB,OAAO;YACH,GAAG,GAAG;YACN,OAAO;YACP,SAAS;YACT,IAAI;YACJ,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;SACvB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,IAAY,EAAE,KAAa,EAAE,GAAY;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;YAC3F,KAAK,CAAC,IAAI,GAAG,gBAAgB,CAAC;YAE9B,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC;QAClC,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAExC,KAAK,CAAC,GAAG,CACL,KAAK,EACL,cAAc,CACjB,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YAE7B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;iBACZ,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;iBACpB,OAAO,CAAC,GAAG,EAAE;gBACV,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAClC,CAAC,CAAC,CAAA;QACV,CAAC;QAED,OAAO,cAAc,CAAC,IAAI,CAAC;IAC/B,CAAC;IAEM,OAAO,CAAC,IAAY;QACvB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACvC,CAAC;IAEM,cAAc,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;IAEM,UAAU,CAAC,IAAY,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEzD,OAAO;YACH,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,cAAc;YACrB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC;YACnD,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;SACpF,CAAC;IACN,CAAC;IAED;QACI,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,SAAS,CAAA;QAC7B,CAAC;QACD,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAA;IAC7B,CAAC;;AAGL,MAAM,CAAC,OAAO,GAAG;IACb,QAAQ;CACX,CAAA"} \ No newline at end of file diff --git a/app/builder-service.js b/app/builder-service.js index 4471446..5593ef3 100644 --- a/app/builder-service.js +++ b/app/builder-service.js @@ -1,18 +1,42 @@ 'use strict' -const { createReadStream, existsSync, promises: fs } = require('node:fs') +const { createReadStream, createWriteStream, existsSync, promises: fs } = require('node:fs') const { spawn } = require('node:child_process') const { join, posix, resolve } = require('node:path') -const { Readable } = require('node:stream') +const { Readable, Transform } = require('node:stream') const { pipeline } = require('node:stream/promises') +const { createGunzip } = require('node:zlib') const config = require('../config.json') const { createServiceClient, ServiceError } = require('./service-client') const { createBuilder } = require('./build') -const { removeDirectory } = require('./filesystem') +const { JobQueue } = require('./JobQueue') +const { CacheManager } = require('./ops/cache') +const { Telemetry } = require('./ops/telemetry') const SHA_PATTERN = /^[a-f0-9]{40}$/ +const WEBPACK_CONFIG_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/ const MODES = new Set(['legacy', 'webpack', 'dashboards', 'esbuild']) const SOURCE_PATHS = ['ts', 'js', 'css', 'tools/webpacks', 'tools/libs'] +const ARCHIVE_BODY_TIMEOUT = 120000 +const MAX_COMPRESSED_BYTES = 32 * 1024 * 1024 +const MAX_INFLATED_BYTES = 64 * 1024 * 1024 +const MAX_ENTRIES = 5000 +const MAX_FILE_BYTES = 4 * 1024 * 1024 +const MAX_PATH_BYTES = 512 +const OPERATIONS = ['source_download', 'build'] +const ROUTES = ['/v1/build'] +const FAILURE_SUMMARIES = { + ARCHIVE_ERROR: 'Source archive extraction failed', + DOWNLOADER_ERROR: 'Downloader request failed', + DOWNLOADER_TIMEOUT: 'Downloader request timed out', + INVALID_BUILD: 'Build did not produce the requested file', + INVALID_COMMIT: 'Commit is invalid', + INVALID_MODE: 'Build mode is invalid', + INVALID_OPTIONS: 'Build options are invalid', + INVALID_PATH: 'Build output path is invalid', + QUEUE_FULL: 'Build capacity is unavailable', + SOURCE_INCOMPLETE: 'Source archive is incomplete' +} class BuilderError extends Error { constructor (code, message, status = 500, cause) { @@ -32,29 +56,41 @@ function validateRequest (body = {}) { if (body.options !== undefined && (!body.options || typeof body.options !== 'object' || Array.isArray(body.options))) { throw new BuilderError('INVALID_OPTIONS', 'Options must be an object', 400) } + if (body.options?.config !== undefined && (typeof body.options.config !== 'string' || !WEBPACK_CONFIG_PATTERN.test(body.options.config))) { + throw new BuilderError('INVALID_OPTIONS', FAILURE_SUMMARIES.INVALID_OPTIONS, 400) + } return { commit: body.commit, path: body.path, mode: body.mode, options: body.options || {} } } function createBuilderService (options = {}) { const cacheRoot = resolve(options.cacheRoot || process.env.BUILDER_CACHE_ROOT || config.builderCacheRoot || join(__dirname, '../tmp')) + const cacheLifetime = Number(options.cacheLifetime || process.env.BUILDER_CACHE_LIFETIME || config.builderCacheLifetime || 7 * 24 * 60 * 60 * 1000) + const now = options.now || Date.now + const queue = options.queue || new JobQueue() + const cacheManager = options.cacheManager || new CacheManager({ root: cacheRoot, service: 'builder', idleExpiryMs: cacheLifetime, now }) + const telemetry = options.telemetry || new Telemetry({ service: 'builder', operations: OPERATIONS, routes: ROUTES, failureSummaries: FAILURE_SUMMARIES, now }) const client = options.client || createServiceClient({ baseURL: options.downloaderURL || process.env.DOWNLOADER_URL || config.downloaderURL, token: options.token === undefined ? process.env.INTERNAL_SERVICE_TOKEN : options.token, timeout: options.timeout || process.env.BUILDER_DOWNLOADER_TIMEOUT || config.builderDownloaderTimeout }) - const builder = options.builder || createBuilder({ ...options, cacheRoot }) - const cacheLifetime = Number(options.cacheLifetime || process.env.BUILDER_CACHE_LIFETIME || config.builderCacheLifetime || 7 * 24 * 60 * 60 * 1000) + const builder = options.builder || createBuilder({ ...options, cacheRoot, queue }) const pending = new Map() function complete (root) { return SOURCE_PATHS.every(path => existsSync(join(root, path))) } - async function ensureSource (commit) { + function ensureSource (commit, context = {}) { + return cacheManager.use(commit, () => ensureSourceInUse(commit, context)) + } + + async function ensureSourceInUse (commit, context) { const root = join(cacheRoot, commit) if (complete(root)) return root if (pending.has(commit)) return pending.get(commit) - const promise = extract(commit, root).finally(() => pending.delete(commit)) + const promise = observe(context.correlationId, 'source_download', () => extract(commit, root), true, commit) + .finally(() => pending.delete(commit)) pending.set(commit, promise) return promise } @@ -62,18 +98,17 @@ function createBuilderService (options = {}) { async function extract (commit, root) { await fs.mkdir(cacheRoot, { recursive: true }) await fs.rm(root, { recursive: true, force: true }) - const temporary = await fs.mkdtemp(join(cacheRoot, `.extract-${commit}-`)) + const archiveDirectory = await fs.mkdtemp(join(cacheRoot, `.archive-${commit}-`)) + const archivePath = join(archiveDirectory, 'source.tar.gz') + let temporary try { - const archive = await client.stream(`/v1/sources/${commit}.tar.gz`) - const tar = (options.spawn || spawn)('tar', ['-xzf', '-', '-C', temporary], { shell: false, stdio: ['pipe', 'ignore', 'pipe'] }) - let stderr = '' - tar.stderr.on('data', chunk => { stderr += chunk }) - const closed = new Promise((resolve, reject) => { - tar.on('error', reject) - tar.on('close', code => code === 0 ? resolve() : reject(new BuilderError('ARCHIVE_ERROR', stderr || `tar exited with ${code}`, 502))) - }) + temporary = await fs.mkdtemp(join(cacheRoot, `.extract-${commit}-`)) + const archive = await client.stream(`/v1/sources/${commit}.tar.gz`, { timeoutThroughBody: true }) const input = typeof archive.getReader === 'function' ? Readable.fromWeb(archive) : archive - await Promise.all([pipeline(input, tar.stdin), closed]) + await saveArchive(input, archivePath) + await preflightArchive(archivePath) + await extractArchive(archivePath, temporary, options.spawn || spawn) + await inspectExtracted(temporary) if (!complete(temporary)) throw new BuilderError('SOURCE_INCOMPLETE', 'Source archive is incomplete', 502) try { await fs.rename(temporary, root) @@ -83,42 +118,323 @@ function createBuilderService (options = {}) { } return root } catch (error) { - if (error instanceof BuilderError || error instanceof ServiceError) throw error - throw new BuilderError('SOURCE_ERROR', error.message, 502, error) + throw sanitize(error) } finally { - await fs.rm(temporary, { recursive: true, force: true }) + await Promise.all([ + fs.rm(archiveDirectory, { recursive: true, force: true }), + temporary && fs.rm(temporary, { recursive: true, force: true }) + ]) } } - async function build (body) { - const request = validateRequest(body) - await ensureSource(request.commit) + async function build (body, context = {}) { + telemetry.startSpan(context.correlationId, 'build') + let release + let request try { + request = validateRequest(body) + release = await cacheManager.acquire(request.commit) + await ensureSourceInUse(request.commit, context) const result = await builder.build(request) if (!result || !result.file || !existsSync(result.file)) throw new BuilderError('INVALID_BUILD', 'Build did not produce the requested file', 400) - return { stream: createReadStream(result.file), builtWith: result.builtWith || request.mode, path: request.path } + await fs.writeFile(join(cacheRoot, request.commit, '.complete'), '') + return { + stream: observedStream(createReadStream(result.file), release, context.correlationId, telemetry, request.commit), + builtWith: result.builtWith || request.mode, + path: request.path + } } catch (error) { - if (error.name === 'QueueFullError') throw new BuilderError('QUEUE_FULL', error.message, 503, error) - throw error + release?.() + failed(context.correlationId, 'build', error, request?.commit) + throw sanitize(error) } } async function cleanup (force = false) { - const entries = await fs.readdir(cacheRoot, { withFileTypes: true }).catch(() => []) - const removed = [] - for (const entry of entries) { - if (!entry.isDirectory() || pending.has(entry.name)) continue - const path = join(cacheRoot, entry.name) - const stat = await fs.stat(path) - if (force || Date.now() - stat.mtimeMs > cacheLifetime) { - await removeDirectory(path) - removed.push(entry.name) + const before = await cacheManager.inspect() + await cacheManager.execute(force ? 'cache.clear' : 'cache.purge_expired') + const remaining = new Set((await cacheManager.inspect()).map(entry => entry.commit)) + return before.map(entry => entry.commit).filter(commit => !remaining.has(commit)) + } + + async function snapshot () { + const queueMetrics = typeof queue.getMetrics === 'function' ? queue.getMetrics('compile', now()) : { active: 0, queued: 0, limit: 0, oldestQueuedAgeMs: null } + let cache + let cacheError = false + try { + cache = await cacheManager.snapshot() + } catch (error) { + cache = null + cacheError = true + } + const downloader = telemetry.dependencies().find(dependency => dependency.name === 'downloader') + const queueSaturated = queueMetrics.limit > 0 && queueMetrics.active + queueMetrics.queued >= queueMetrics.limit + const dependencyFailed = downloader && !['available', 'unknown'].includes(downloader.status) + return telemetry.snapshot({ + capabilities: [ + capability('build_delivery', queueSaturated || dependencyFailed, queueSaturated ? 'QUEUE_SATURATED' : downloader?.status === 'unavailable' ? 'DOWNLOADER_UNAVAILABLE' : 'DOWNLOADER_DEGRADED'), + capability('cache_control', cacheError, 'CACHE_INSPECTION_FAILED') + ], + queues: [{ name: 'build', ...queueMetrics }], + cache + }) + } + + function startRequest (details) { + return telemetry.startTrace(details) + } + + function completeRequest (correlationId, outcome) { + return telemetry.completeTrace(correlationId, outcome) + } + + async function observe (correlationId, operation, work, dependency = false, commit = null) { + const started = now() + telemetry.startSpan(correlationId, operation) + try { + const result = await work() + telemetry.completeSpan(correlationId, operation, { status: 'succeeded' }) + if (dependency) telemetry.recordDependency('downloader', { succeeded: true, latencyMs: now() - started }) + return result + } catch (error) { + const safe = safeError(error) + if (dependency) telemetry.recordDependency('downloader', { succeeded: false, latencyMs: now() - started, errorCode: safe.code }) + failed(correlationId, operation, safe, commit) + throw safe + } + } + + function failed (correlationId, operation, error, commit) { + const safe = safeError(error) + telemetry.completeSpan(correlationId, operation, { status: safe.code === 'QUEUE_FULL' ? 'rejected' : 'failed', httpStatus: safe.status, code: safe.code }) + telemetry.recordFailure({ correlationId, operation, code: safe.code, httpStatus: safe.status, commit }) + } + + return { build, cacheManager, cacheRoot, cleanup, completeRequest, ensureSource, snapshot, startRequest, telemetry } +} + +async function saveArchive (input, archivePath) { + let timedOut = false + let size = 0 + const controller = new AbortController() + const timer = setTimeout(() => { + timedOut = true + controller.abort() + }, ARCHIVE_BODY_TIMEOUT) + const limit = new Transform({ + transform (chunk, encoding, callback) { + size += chunk.length + callback(size > MAX_COMPRESSED_BYTES ? archiveError() : null, chunk) + } + }) + try { + await pipeline(input, limit, createWriteStream(archivePath, { flags: 'wx' }), { signal: controller.signal }) + } catch (error) { + if (timedOut) throw new ServiceError('SERVICE_TIMEOUT', 'Service request timed out', 504, { cause: error }) + throw error + } finally { + clearTimeout(timer) + } +} + +async function preflightArchive (archivePath) { + const input = createReadStream(archivePath) + const gunzip = createGunzip() + const parser = createTarParser() + let inflated = 0 + input.pipe(gunzip) + try { + for await (const chunk of gunzip) { + inflated += chunk.length + if (inflated > MAX_INFLATED_BYTES) throw archiveError() + parser.write(chunk) + } + parser.end() + } catch (error) { + input.destroy() + gunzip.destroy() + if (error instanceof BuilderError) throw error + throw archiveError(error) + } +} + +function createTarParser () { + let buffer = Buffer.alloc(0) + let remaining = 0 + let entries = 0 + let zeroBlocks = 0 + let ended = false + const names = new Set() + + return { + write (chunk) { + buffer = Buffer.concat([buffer, chunk]) + while (buffer.length) { + if (remaining) { + const consumed = Math.min(remaining, buffer.length) + remaining -= consumed + buffer = buffer.subarray(consumed) + continue + } + if (buffer.length < 512) return + const header = buffer.subarray(0, 512) + buffer = buffer.subarray(512) + if (header.every(byte => byte === 0)) { + zeroBlocks++ + if (zeroBlocks === 2) ended = true + continue + } + if (ended || zeroBlocks) throw archiveError() + const entry = parseTarHeader(header) + entries++ + if (entries > MAX_ENTRIES || names.has(entry.path)) throw archiveError() + names.add(entry.path) + remaining = Math.ceil(entry.size / 512) * 512 } + }, + end () { + if (!ended || remaining || buffer.length) throw archiveError() } - return removed } +} + +function parseTarHeader (header) { + const expectedChecksum = parseTarNumber(header.subarray(148, 156)) + let checksum = 0 + for (let i = 0; i < header.length; i++) checksum += i >= 148 && i < 156 ? 32 : header[i] + if (checksum !== expectedChecksum) throw archiveError() + + const name = tarString(header.subarray(0, 100)) + const prefix = tarString(header.subarray(345, 500)) + const rawPath = prefix ? `${prefix}/${name}` : name + const type = header[156] + const directory = type === 53 + if (type !== 0 && type !== 48 && !directory) throw archiveError() + const size = parseTarNumber(header.subarray(124, 136)) + if ((directory && size !== 0) || (!directory && size > MAX_FILE_BYTES)) throw archiveError() + return { path: archivePath(rawPath, directory), size } +} + +function parseTarNumber (field) { + if (field[0] & 0x80) throw archiveError() + const value = field.toString('ascii').replace(/\0.*$/, '').trim() + if (!/^[0-7]+$/.test(value)) throw archiveError() + const number = Number.parseInt(value, 8) + if (!Number.isSafeInteger(number)) throw archiveError() + return number +} + +function tarString (field) { + const nul = field.indexOf(0) + const bytes = nul === -1 ? field : field.subarray(0, nul) + if (nul !== -1 && field.subarray(nul).some(byte => byte !== 0)) throw archiveError() + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes) + } catch (error) { + throw archiveError(error) + } +} + +function archivePath (rawPath, directory) { + if (!rawPath || Buffer.byteLength(rawPath) > MAX_PATH_BYTES || rawPath.includes('\0') || rawPath.includes('\\') || posix.isAbsolute(rawPath)) throw archiveError() + const path = directory && rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath + const segments = path.split('/') + if (!path || segments.some(segment => !segment || segment === '.' || segment === '..') || posix.normalize(path) !== path) throw archiveError() + if (!allowedPath(path, directory)) throw archiveError() + return path +} + +function allowedPath (path, directory) { + return (directory && path === 'tools') || SOURCE_PATHS.some(root => path === root || path.startsWith(`${root}/`)) +} + +async function extractArchive (archivePath, destination, spawnImpl) { + const child = spawnImpl('tar', ['-xzf', archivePath, '-C', destination, '--no-same-owner', '--no-same-permissions'], { + shell: false, + stdio: ['ignore', 'ignore', 'pipe'] + }) + child.stderr?.resume() + await new Promise((resolve, reject) => { + let settled = false + const finish = error => { + if (settled) return + settled = true + if (error) { + child.kill?.() + reject(archiveError(error)) + } else resolve() + } + child.once('error', finish) + child.once('close', code => finish(code === 0 ? null : archiveError())) + }) +} + +async function inspectExtracted (root) { + let nodes = 0 + let totalSize = 0 + const pending = [''] + while (pending.length) { + const parent = pending.pop() + for (const name of await fs.readdir(join(root, parent))) { + const path = parent ? `${parent}/${name}` : name + const stat = await fs.lstat(join(root, path)) + nodes++ + if (nodes > MAX_ENTRIES || (!stat.isDirectory() && !stat.isFile()) || (SOURCE_PATHS.includes(path) && !stat.isDirectory())) throw archiveError() + archivePath(path, stat.isDirectory()) + if (stat.isDirectory()) { + pending.push(path) + } else { + if (stat.size > MAX_FILE_BYTES) throw archiveError() + totalSize += stat.size + if (totalSize > MAX_INFLATED_BYTES) throw archiveError() + } + } + } +} + +function archiveError (cause) { + return new BuilderError('ARCHIVE_ERROR', FAILURE_SUMMARIES.ARCHIVE_ERROR, 502, cause) +} + +function observedStream (stream, release, correlationId, telemetry, commit) { + let completed = false + const finish = error => { + if (completed) return + completed = true + release() + if (error) { + const safe = safeError(error) + telemetry.completeSpan(correlationId, 'build', { status: 'failed', httpStatus: safe.status, code: safe.code }) + telemetry.recordFailure({ correlationId, operation: 'build', code: safe.code, httpStatus: safe.status, commit }) + } else { + telemetry.completeSpan(correlationId, 'build', { status: 'succeeded', httpStatus: 200 }) + } + } + stream.once('error', finish) + stream.once('end', () => finish()) + stream.once('close', () => finish()) + return stream +} + +function safeError (error) { + if (error instanceof BuilderError && Object.hasOwn(FAILURE_SUMMARIES, error.code)) return error + if (error?.name === 'QueueFullError' || error?.code === 'QUEUE_FULL') return new BuilderError('QUEUE_FULL', FAILURE_SUMMARIES.QUEUE_FULL, 503, error) + if (error instanceof ServiceError) { + const code = error.code === 'SERVICE_TIMEOUT' ? 'DOWNLOADER_TIMEOUT' : error.code === 'SOURCE_INCOMPLETE' ? 'SOURCE_INCOMPLETE' : 'DOWNLOADER_ERROR' + const result = new BuilderError(code, FAILURE_SUMMARIES[code], code === 'DOWNLOADER_TIMEOUT' ? 504 : 502, error) + for (const field of ['retryAfter', 'rateLimitRemaining', 'rateLimitReset']) result[field] = error[field] + return result + } + if (Object.hasOwn(FAILURE_SUMMARIES, error?.code)) return new BuilderError(error.code, FAILURE_SUMMARIES[error.code], error.status || 500, error) + return new BuilderError('INTERNAL_ERROR', 'An internal error occurred', 500, error) +} + +function sanitize (error) { + return safeError(error) +} - return { build, cacheRoot, cleanup, ensureSource } +function capability (name, degraded, reasonCode) { + return { name, status: degraded ? 'degraded' : 'available', reasonCode: degraded ? reasonCode : null } } module.exports = { BuilderError, MODES, SOURCE_PATHS, createBuilderService, validateRequest } diff --git a/app/download.js b/app/download.js index 18542d2..68052c7 100644 --- a/app/download.js +++ b/app/download.js @@ -335,9 +335,9 @@ async function downloadSourceFolderGit (outputDir, branch, mode = 'tar') { } /** - * An asynchronous version of https.get, with encoding set to utf8. + * An asynchronous version of https.get. * The Promise resolves with an object containing the status code and the - * response body. + * raw response body Buffer. * * @param {object|string} options Can either be an https request options object, * or an url string. @@ -346,13 +346,12 @@ function get(options, _retried) { return new Promise((resolve, reject) => { const request = httpsGet(options, response => { const body = [] - response.setEncoding('utf8') response.on('error', reject) response.on('data', (data) => { body.push(data) }) response.on('end', () => resolve({ statusCode: response.statusCode, - body: body.join(''), + body: Buffer.concat(body), headers: response.headers }) ) @@ -429,7 +428,7 @@ async function getFilesInFolder(path, branch) { throw error } - const promises = JSON.parse(body).map(obj => { + const promises = JSON.parse(body.toString('utf8')).map(obj => { const name = path + '/' + obj.name return ( (obj.type === 'dir') @@ -483,7 +482,7 @@ async function getBranchInfo (branch) { const rate = logRateLimitIfDepleted(headers, `fetching branch info for ${branch}`) throwIfRateLimited(rate, statusCode) if (statusCode === 200) { - return JSON.parse(body) + return JSON.parse(body.toString('utf8')) } return false }) @@ -511,7 +510,7 @@ async function getCommitInfo (commit) { const rate = logRateLimitIfDepleted(headers, `fetching commit info for ${commit}`) throwIfRateLimited(rate, statusCode) if (statusCode === 200) { - return JSON.parse(body) + return JSON.parse(body.toString('utf8')) } return false }) diff --git a/app/downloader-service.js b/app/downloader-service.js index 558950f..dd5e15c 100644 --- a/app/downloader-service.js +++ b/app/downloader-service.js @@ -6,14 +6,29 @@ const { join, relative, resolve, sep } = require('path') const config = require('../config.json') const { repo } = config const download = require('./download') -const { removeDirectory } = require('./filesystem') const { JobQueue } = require('./JobQueue') +const { CacheManager } = require('./ops/cache') +const { Telemetry } = require('./ops/telemetry') const SHA_PATTERN = /^[a-f0-9]{40}$/i const SOURCE_PATHS = ['ts', 'js', 'css', 'tools/webpacks', 'tools/libs'] const OPTIONAL_SOURCE_PATHS = ['js', 'tools/webpacks', 'tools/libs'] const ESBUILD_DETECTION_FILE = 'ts/masters/highcharts-autoload.src.ts' const ESBUILD_DETECTION_TIMEOUT = 5000 +const SOURCE_COMPLETE = '.source-complete' +const OPERATIONS = ['github_branch_lookup', 'github_commit_lookup', 'github_esbuild_detection', 'source_download', 'file_download', 'source_archive'] +const ROUTES = ['/v1/resolve', '/v1/files/:commit/*', '/v1/sources/:commit.tar.gz'] +const FAILURE_SUMMARIES = { + FILE_NOT_FOUND: 'File was not found', + INVALID_COMMIT: 'Commit is invalid', + INVALID_PATH: 'File path is invalid', + QUEUE_FULL: 'Download capacity is unavailable', + RATE_LIMITED: 'GitHub rate limit is exhausted', + REF_NOT_FOUND: 'Ref was not found', + SOURCE_INCOMPLETE: 'Source tree is incomplete', + UPSTREAM_ERROR: 'GitHub request failed', + UPSTREAM_TIMEOUT: 'GitHub request timed out' +} class DownloaderError extends Error { constructor (code, message, status = 500, cause) { @@ -33,6 +48,11 @@ function validateCommit (commit) { function safeFilePath (root, commit, filepath) { const commitRoot = resolve(root, validateCommit(commit)) + return safeChildPath(commitRoot, filepath) +} + +function safeChildPath (root, filepath) { + const commitRoot = resolve(root) const target = resolve(commitRoot, filepath || '') if (!filepath || target === commitRoot || relative(commitRoot, target).startsWith(`..${sep}`) || relative(commitRoot, target) === '..') { throw new DownloaderError('INVALID_PATH', 'Unsafe file path', 400) @@ -47,38 +67,65 @@ function createDownloaderService (options = {}) { const sourceURL = options.sourceURL || process.env.DOWNLOADER_SOURCE_URL || config.downloaderSourceURL || `https://raw.githubusercontent.com/${repo}/` const fetchImpl = options.fetch || global.fetch const queue = options.queue || new JobQueue() + const now = options.now || Date.now + const cacheManager = options.cacheManager || new CacheManager({ root: cacheRoot, service: 'downloader', idleExpiryMs: cacheLifetime, now }) + const telemetry = options.telemetry || new Telemetry({ service: 'downloader', operations: OPERATIONS, routes: ROUTES, failureSummaries: FAILURE_SUMMARIES, now }) const esbuildCache = new Map() + const esbuildPending = new Map() + const sourcePending = new Map() - async function needsEsbuild (commit) { - if (esbuildCache.has(commit)) return esbuildCache.get(commit) + async function needsEsbuild (commit, context = {}) { + commit = validateCommit(commit) + return cacheManager.use(commit, () => { + const complete = join(cacheRoot, commit, '.complete') + if (esbuildCache.has(commit) && existsSync(complete)) return esbuildCache.get(commit) + esbuildCache.delete(commit) - const controller = new AbortController() - const timer = setTimeout(() => controller.abort(), ESBUILD_DETECTION_TIMEOUT) - try { - const response = await fetchImpl(`${sourceURL.replace(/\/?$/, '/')}${commit}/${ESBUILD_DETECTION_FILE}`, { - method: 'HEAD', - signal: controller.signal, - headers: githubToken ? { Authorization: `token ${githubToken}` } : {} - }) - if (!response.ok && response.status !== 404) { - throw new DownloaderError('UPSTREAM_ERROR', `GitHub returned ${response.status}`, 502) + let pending = esbuildPending.get(commit) + if (!pending) { + pending = detectEsbuild(commit, complete, context) + esbuildPending.set(commit, pending) } - esbuildCache.set(commit, response.ok) - return response.ok - } catch (error) { - if (error instanceof DownloaderError) throw error - const timedOut = controller.signal.aborted - throw new DownloaderError(timedOut ? 'UPSTREAM_TIMEOUT' : 'UPSTREAM_ERROR', timedOut ? 'GitHub request timed out' : error.message, timedOut ? 504 : 502, error) + return pending + }) + } + + async function detectEsbuild (commit, complete, context) { + try { + return await observe(context.correlationId, 'github_esbuild_detection', async () => { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), ESBUILD_DETECTION_TIMEOUT) + try { + const response = await fetchImpl(`${sourceURL.replace(/\/?$/, '/')}${commit}/${ESBUILD_DETECTION_FILE}`, { + method: 'HEAD', + signal: controller.signal, + headers: githubToken ? { Authorization: `token ${githubToken}` } : {} + }) + if (!response.ok && response.status !== 404) { + throw new DownloaderError('UPSTREAM_ERROR', 'GitHub request failed', 502) + } + esbuildCache.set(commit, response.ok) + await fs.mkdir(join(cacheRoot, commit), { recursive: true }) + await fs.writeFile(complete, '') + return response.ok + } catch (error) { + if (error instanceof DownloaderError) throw error + const timedOut = controller.signal.aborted + throw new DownloaderError(timedOut ? 'UPSTREAM_TIMEOUT' : 'UPSTREAM_ERROR', timedOut ? 'GitHub request timed out' : 'GitHub request failed', timedOut ? 504 : 502, error) + } finally { + clearTimeout(timer) + } + }, true, commit) } finally { - clearTimeout(timer) + esbuildPending.delete(commit) } } - async function resolveRef (ref) { - let info = await download.getBranchInfo(ref) + async function resolveRef (ref, context = {}) { + let info = await observe(context.correlationId, 'github_branch_lookup', () => download.getBranchInfo(ref), true) let commit = info && info.commit && info.commit.sha if (!commit) { - info = await download.getCommitInfo(ref) + info = await observe(context.correlationId, 'github_commit_lookup', () => download.getCommitInfo(ref), true) commit = info && info.sha } if (!SHA_PATTERN.test(commit || '')) { @@ -87,23 +134,32 @@ function createDownloaderService (options = {}) { commit = commit.toLowerCase() return { commit, - needsEsbuild: await needsEsbuild(commit), + needsEsbuild: await needsEsbuild(commit, context), rate: download.getRateLimitState() } } - async function ensureSource (commit) { + async function ensureSource (commit, context = {}) { commit = validateCommit(commit) - const root = join(cacheRoot, commit) - const complete = join(root, '.complete') - if (existsSync(complete)) return root - let failure + return cacheManager.use(commit, async () => { + const root = join(cacheRoot, commit) + if (sourceIsComplete(root)) return root + let pending = sourcePending.get(commit) + if (!pending) { + pending = populateSource(commit, root, context) + sourcePending.set(commit, pending) + } + return pending + }) + } + + async function populateSource (commit, root, context) { try { - await queue.addJob('download', `downloader:${commit}`, { - func: async () => { - try { + try { + await queue.addJob('download', `downloader:${commit}`, { + func: async () => { await fs.mkdir(root, { recursive: true }) - const responses = await download.downloadSourceFolder(root, sourceURL.replace(/\/?$/, '/'), commit) + const responses = await observe(context.correlationId, 'source_download', () => download.downloadSourceFolder(root, sourceURL.replace(/\/?$/, '/'), commit), true, commit) if (responses.some(response => !response.success)) { throw new DownloaderError('SOURCE_INCOMPLETE', 'Source tree is incomplete', 502) } @@ -111,71 +167,193 @@ function createDownloaderService (options = {}) { if (!SOURCE_PATHS.every(path => existsSync(join(root, path)))) { throw new DownloaderError('SOURCE_INCOMPLETE', 'Source tree is incomplete', 502) } - await fs.writeFile(complete, '') - } catch (error) { - failure = error - throw error - } - }, - args: [] - }) - if (failure) throw failure - if (!existsSync(complete)) throw new DownloaderError('SOURCE_INCOMPLETE', 'Source tree is incomplete', 502) + await fs.writeFile(join(root, SOURCE_COMPLETE), '') + await fs.writeFile(join(root, '.complete'), '') + }, + args: [] + }) + if (!sourceIsComplete(root)) throw new DownloaderError('SOURCE_INCOMPLETE', 'Source tree is incomplete', 502) + } catch (error) { + await Promise.all([...SOURCE_PATHS, SOURCE_COMPLETE].map(path => fs.rm(join(root, path), { recursive: true, force: true }))) + if (!existsSync(join(root, '.complete'))) await fs.rm(root, { recursive: true, force: true }) + if (error.name === 'QueueFullError') throw new DownloaderError('QUEUE_FULL', 'Download capacity is unavailable', 503, error) + throw error + } + return root + } finally { + sourcePending.delete(commit) + } + } + + async function openFile (commit, filepath, context = {}) { + commit = validateCommit(commit) + const release = await cacheManager.acquire(commit) + telemetry.startSpan(context.correlationId, 'file_download') + try { + const sourceRoot = join(cacheRoot, commit) + const sourcePath = safeFilePath(cacheRoot, commit, filepath) + if (sourceIsComplete(sourceRoot)) { + const stat = await fs.stat(sourcePath).catch(() => false) + if (stat && stat.isFile()) return observedStream(createReadStream(sourcePath), release, context.correlationId, 'file_download', telemetry, failed, commit) + } + + const fileCacheRoot = join(cacheRoot, commit, '.files') + const path = safeChildPath(fileCacheRoot, filepath) + const stat = await fs.stat(path).catch(() => false) + if (!(stat && stat.isFile())) { + const started = now() + let result + try { + result = await download.downloadFile(`${sourceURL.replace(/\/?$/, '/')}${commit}/${filepath}`, path) + telemetry.recordDependency('github', { succeeded: result.success, latencyMs: now() - started, errorCode: result.success ? null : result.statusCode === 404 ? 'FILE_NOT_FOUND' : 'UPSTREAM_ERROR' }) + } catch (error) { + telemetry.recordDependency('github', { succeeded: false, latencyMs: now() - started, errorCode: safeError(error).code }) + throw error + } + if (!result.success) { + const notFound = result.statusCode === 404 + throw new DownloaderError(notFound ? 'FILE_NOT_FOUND' : 'UPSTREAM_ERROR', notFound ? 'File was not found' : 'GitHub request failed', notFound ? 404 : 502) + } + const downloaded = await fs.stat(path).catch(() => false) + if (!(downloaded && downloaded.isFile())) throw new DownloaderError('SOURCE_INCOMPLETE', 'Source tree is incomplete', 502) + await fs.writeFile(join(cacheRoot, commit, '.complete'), '') + } + return observedStream(createReadStream(path), release, context.correlationId, 'file_download', telemetry, failed, commit) } catch (error) { - await fs.rm(root, { recursive: true, force: true }) - if (error.name === 'QueueFullError') throw new DownloaderError('QUEUE_FULL', error.message, 503, error) - throw error + release() + failed(context.correlationId, 'file_download', error, commit) + throw sanitize(error) } - return root } - async function openFile (commit, filepath) { + async function archive (commit, context = {}) { commit = validateCommit(commit) - const sourcePath = safeFilePath(cacheRoot, commit, filepath) - const sourceRoot = join(cacheRoot, commit) - if (existsSync(join(sourceRoot, '.complete'))) { - const stat = await fs.stat(sourcePath).catch(() => false) - if (stat && stat.isFile()) return createReadStream(sourcePath) + const release = await cacheManager.acquire(commit) + telemetry.startSpan(context.correlationId, 'source_archive') + try { + const root = await ensureSource(commit, context) + const child = spawn('tar', ['-czf', '-', '--', ...SOURCE_PATHS], { + cwd: root, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'] + }) + let completed = false + const finish = (error) => { + if (completed) return + completed = true + release() + if (error) failed(context.correlationId, 'source_archive', error, commit) + else telemetry.completeSpan(context.correlationId, 'source_archive', { status: 'succeeded', httpStatus: 200 }) + } + child.once('error', finish) + child.once('close', code => finish(code ? new DownloaderError('INTERNAL_ERROR', 'Archive generation failed', 500) : null)) + return child + } catch (error) { + release() + failed(context.correlationId, 'source_archive', error, commit) + throw sanitize(error) } + } - const fileCacheRoot = join(cacheRoot, '.files') - const path = safeFilePath(fileCacheRoot, commit, filepath) - const stat = await fs.stat(path).catch(() => false) - if (stat && stat.isFile()) return createReadStream(path) + async function cleanup (force = false) { + const before = await cacheManager.inspect() + await cacheManager.execute(force ? 'cache.clear' : 'cache.purge_expired') + const remaining = new Set((await cacheManager.inspect()).map(entry => entry.commit)) + return before.map(entry => entry.commit).filter(commit => !remaining.has(commit)) + } - const result = await download.downloadFile(`${sourceURL.replace(/\/?$/, '/')}${commit}/${filepath}`, path) - if (!result.success) { - const notFound = result.statusCode === 404 - throw new DownloaderError(notFound ? 'FILE_NOT_FOUND' : 'UPSTREAM_ERROR', notFound ? 'File was not found' : `GitHub returned ${result.statusCode}`, notFound ? 404 : 502) + async function snapshot () { + const queueMetrics = typeof queue.getMetrics === 'function' ? queue.getMetrics('download', now()) : { active: 0, queued: 0, limit: 0, oldestQueuedAgeMs: null } + let cache + let cacheError = false + try { + cache = await cacheManager.snapshot() + } catch (error) { + cache = null + cacheError = true } - return createReadStream(path) + const github = telemetry.dependencies().find(dependency => dependency.name === 'github') + const queueSaturated = queueMetrics.active + queueMetrics.queued >= queueMetrics.limit && queueMetrics.limit > 0 + const githubFailure = github && github.status !== 'available' && github.status !== 'unknown' + return telemetry.snapshot({ + capabilities: [ + capability('ref_resolution', githubFailure, github?.status === 'unavailable' ? 'GITHUB_UNAVAILABLE' : 'GITHUB_DEGRADED'), + capability('source_file_delivery', githubFailure, 'GITHUB_DEGRADED'), + capability('source_archive_delivery', githubFailure || queueSaturated, queueSaturated ? 'QUEUE_SATURATED' : 'GITHUB_DEGRADED'), + capability('cache_control', cacheError, 'CACHE_INSPECTION_FAILED') + ], + queues: [{ name: 'download', ...queueMetrics }], + cache + }) } - async function archive (commit) { - const root = await ensureSource(commit) - return spawn('tar', ['-czf', '-', '--', ...SOURCE_PATHS], { - cwd: root, - shell: false, - stdio: ['ignore', 'pipe', 'pipe'] - }) + function startRequest (details) { + return telemetry.startTrace(details) } - async function cleanup (force = false) { - const entries = await fs.readdir(cacheRoot, { withFileTypes: true }).catch(() => []) - const removed = [] - for (const entry of entries) { - if (!entry.isDirectory()) continue - const path = join(cacheRoot, entry.name) - const stat = await fs.stat(path) - if (force || Date.now() - stat.mtimeMs > cacheLifetime) { - await removeDirectory(path) - removed.push(entry.name) - } + function completeRequest (correlationId, outcome) { + return telemetry.completeTrace(correlationId, outcome) + } + + function sourceIsComplete (root) { + return existsSync(join(root, SOURCE_COMPLETE)) || (existsSync(join(root, '.complete')) && SOURCE_PATHS.every(path => existsSync(join(root, path)))) + } + + async function observe (correlationId, operation, work, dependency = false, commit = null) { + const started = now() + telemetry.startSpan(correlationId, operation) + try { + const result = await work() + telemetry.completeSpan(correlationId, operation, { status: 'succeeded' }) + if (dependency) telemetry.recordDependency('github', { succeeded: true, latencyMs: now() - started }) + return result + } catch (error) { + if (dependency) telemetry.recordDependency('github', { succeeded: false, latencyMs: now() - started, errorCode: safeError(error).code }) + failed(correlationId, operation, error, commit) + throw sanitize(error) } - return removed } - return { archive, cacheRoot, cleanup, ensureSource, needsEsbuild, openFile, resolveRef } + function failed (correlationId, operation, error, commit) { + const safe = safeError(error) + telemetry.completeSpan(correlationId, operation, { status: safe.code === 'QUEUE_FULL' ? 'rejected' : 'failed', httpStatus: safe.status, code: safe.code }) + telemetry.recordFailure({ correlationId, operation, code: safe.code, httpStatus: safe.status, commit }) + } + + return { archive, cacheManager, cacheRoot, cleanup, completeRequest, ensureSource, needsEsbuild, openFile, resolveRef, snapshot, startRequest, telemetry } +} + +function observedStream (stream, release, correlationId, operation, telemetry, failed, commit) { + let completed = false + const finish = (error) => { + if (completed) return + completed = true + release() + if (error) failed(correlationId, operation, error, commit) + else telemetry.completeSpan(correlationId, operation, { status: 'succeeded', httpStatus: 200 }) + } + stream.once('error', finish) + stream.once('end', () => finish()) + stream.once('close', () => finish()) + return stream +} + +function safeError (error) { + if (error instanceof DownloaderError && Object.hasOwn(FAILURE_SUMMARIES, error.code)) return error + if (error?.code === 'RATE_LIMITED') { + const result = new DownloaderError('RATE_LIMITED', 'GitHub rate limit is exhausted', 429, error) + for (const field of ['rateLimitLimit', 'rateLimitRemaining', 'rateLimitReset']) result[field] = error[field] + return result + } + return new DownloaderError('INTERNAL_ERROR', 'An internal error occurred', 500, error) +} + +function sanitize (error) { + return safeError(error) +} + +function capability (name, degraded, reasonCode) { + return { name, status: degraded ? 'degraded' : 'available', reasonCode: degraded ? reasonCode : null } } module.exports = { diff --git a/app/ops/cache.js b/app/ops/cache.js new file mode 100644 index 0000000..471fb87 --- /dev/null +++ b/app/ops/cache.js @@ -0,0 +1,285 @@ +'use strict' + +const { randomUUID } = require('node:crypto') +const { promises: fs } = require('node:fs') +const { join, resolve } = require('node:path') + +const SHA = /^[0-9a-f]{40}$/ +const OPERATIONS = new Set(['cache.evict_commit', 'cache.purge_expired', 'cache.clear']) + +class CacheError extends Error { + constructor (code, message) { + super(message) + this.name = 'CacheError' + this.code = code + } +} + +class CacheManager { + constructor ({ root, service, idleExpiryMs, now = Date.now, concurrency = 4, entryLimit = 200, scanLimit = 100000, filesystem = fs } = {}) { + if (typeof root !== 'string' || !root) throw new TypeError('Cache root is required') + if (!['downloader', 'builder'].includes(service)) throw new TypeError('Cache service is invalid') + if (!Number.isSafeInteger(idleExpiryMs) || idleExpiryMs < 0) throw new TypeError('Cache idle expiry is invalid') + if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new TypeError('Cache concurrency is invalid') + this.root = resolve(root) + this.service = service + this.idleExpiryMs = idleExpiryMs + this.now = now + this.concurrency = concurrency + this.entryLimit = entryLimit + this.scanLimit = scanLimit + this.fs = filesystem + this.states = new Map() + this.sizes = new Map() + } + + async acquire (commit) { + commit = parseCommit(commit) + const state = this.state(commit) + while (state.deleting) await state.deleting + state.inUse++ + state.lastUsedAt = this.now() + let released = false + return () => { + if (released) return + released = true + state.inUse-- + state.lastUsedAt = this.now() + } + } + + async use (commit, work) { + if (typeof work !== 'function') throw new TypeError('Cache work must be a function') + const release = await this.acquire(commit) + try { + return await work() + } finally { + release() + } + } + + touch (commit) { + const state = this.state(parseCommit(commit)) + state.lastUsedAt = this.now() + } + + async inspect () { + const names = await this.commitNames() + const entries = (await mapLimit(names, this.concurrency, name => this.inspectEntry(name))).filter(Boolean) + const current = new Set(names) + for (const commit of this.sizes.keys()) { + if (!current.has(commit)) this.sizes.delete(commit) + } + return entries.sort((a, b) => b.lastUsedAt - a.lastUsedAt || a.commit.localeCompare(b.commit)) + } + + async snapshot () { + const entries = await this.inspect() + const totalBytes = entries.reduce((total, entry) => safeAdd(total, entry.sizeBytes), 0) + return { + entryCount: entries.length, + totalBytes, + idleExpiryMs: this.idleExpiryMs, + entriesTruncated: entries.length > this.entryLimit, + entries: entries.slice(0, this.entryLimit).map(entry => ({ + commit: entry.commit, + sizeBytes: entry.sizeBytes, + lastAccessedAt: new Date(entry.lastUsedAt).toISOString(), + expiresAt: new Date(entry.lastUsedAt + this.idleExpiryMs).toISOString(), + inUse: entry.inUse + })) + } + } + + async execute (operation, commit) { + if (!OPERATIONS.has(operation)) throw new CacheError('INVALID_OPERATION', 'Unknown cache operation') + if (operation === 'cache.evict_commit') commit = parseCommit(commit) + else if (commit !== undefined) throw new CacheError('INVALID_OPERATION', 'Commit is not accepted for this operation') + + let candidates + try { + candidates = operation === 'cache.evict_commit' + ? [await this.inspectEntry(commit)].filter(Boolean) + : await this.inspect() + } catch (error) { + return this.result(operation, [], error) + } + + const threshold = this.now() - this.idleExpiryMs + if (operation === 'cache.purge_expired') candidates = candidates.filter(entry => entry.lastUsedAt <= threshold) + const dispositions = await mapLimit(candidates, this.concurrency, entry => this.deleteEntry(entry, operation, threshold)) + return this.result(operation, dispositions, null, operation === 'cache.evict_commit' && candidates.length === 0) + } + + result (operation, dispositions, initialError, absent = false) { + const errors = dispositions.filter(item => item.error) + const removed = dispositions.filter(item => item.removed) + const skippedInUse = dispositions.filter(item => item.inUse).length + const skippedChanged = dispositions.filter(item => item.changed).length + const error = initialError || (errors[0] && errors[0].error) + const target = { + service: this.service, + outcome: error ? 'failed' : removed.length && (skippedInUse || skippedChanged) ? 'partial' : removed.length ? 'completed' : 'no_op', + removedEntries: removed.length, + freedBytes: removed.reduce((total, item) => safeAdd(total, item.sizeBytes), 0), + absent: absent || dispositions.some(item => item.absent), + skippedInUse, + error: error ? publicError(error) : null + } + if (operation !== 'cache.clear') target.skippedChanged = skippedChanged + return target + } + + async deleteEntry (entry, operation, threshold) { + const state = this.state(entry.commit, entry.lastUsedAt) + if (state.deleting) { + await state.deleting + return { absent: true } + } + if (state.inUse) return { inUse: true } + + let finish + state.deleting = new Promise(resolve => { finish = resolve }) + try { + if (state.inUse) return { inUse: true } + const current = await this.entryIdentity(entry.commit) + if (!current || current.dev !== entry.dev || current.ino !== entry.ino) { + return operation === 'cache.purge_expired' && current ? { changed: true } : { absent: true } + } + if (operation === 'cache.purge_expired' && state.lastUsedAt > threshold) return { changed: true } + + const quarantine = join(this.root, `.ops-delete-${randomUUID()}`) + try { + await this.fs.rename(join(this.root, entry.commit), quarantine) + } catch (error) { + if (error.code === 'ENOENT') return { absent: true } + throw error + } + await this.fs.rm(quarantine, { recursive: true, force: true }) + this.states.delete(entry.commit) + this.sizes.delete(entry.commit) + return { removed: true, sizeBytes: entry.sizeBytes } + } catch (error) { + return { error } + } finally { + state.deleting = null + finish() + } + } + + async commitNames () { + let root + try { + root = await this.fs.lstat(this.root) + } catch (error) { + if (error.code === 'ENOENT') return [] + throw error + } + if (!root.isDirectory() || root.isSymbolicLink()) throw new CacheError('UNSAFE_CACHE_ROOT', 'Cache root is not a real directory') + const names = (await this.fs.readdir(this.root, { withFileTypes: true })) + .filter(entry => entry.isDirectory() && SHA.test(entry.name)) + .map(entry => entry.name) + .sort() + if (names.length > this.scanLimit) throw new CacheError('CACHE_SCAN_LIMIT', 'Cache entry limit exceeded') + return names + } + + async inspectEntry (commit) { + if (!SHA.test(commit)) return null + const identity = await this.entryIdentity(commit) + if (!identity) { + this.sizes.delete(commit) + return null + } + const marker = await this.fs.lstat(join(this.root, commit, '.complete')).catch(error => error.code === 'ENOENT' ? null : Promise.reject(error)) + if (!marker || !marker.isFile() || marker.isSymbolicLink()) { + this.sizes.delete(commit) + return null + } + let measured = this.sizes.get(commit) + if (!measured || measured.dev !== identity.dev || measured.ino !== identity.ino || + measured.mtimeMs !== identity.mtimeMs || measured.ctimeMs !== identity.ctimeMs || + measured.markerDev !== marker.dev || measured.markerIno !== marker.ino || + measured.markerMtimeMs !== marker.mtimeMs || measured.markerCtimeMs !== marker.ctimeMs) { + const sizeBytes = await this.directorySize(join(this.root, commit), { nodes: 0 }) + measured = { + dev: identity.dev, + ino: identity.ino, + mtimeMs: identity.mtimeMs, + ctimeMs: identity.ctimeMs, + markerDev: marker.dev, + markerIno: marker.ino, + markerMtimeMs: marker.mtimeMs, + markerCtimeMs: marker.ctimeMs, + sizeBytes + } + this.sizes.set(commit, measured) + } + const fallback = Math.max(identity.mtimeMs, marker.mtimeMs) + const state = this.state(commit, fallback) + return { commit, sizeBytes: measured.sizeBytes, lastUsedAt: state.lastUsedAt, inUse: state.inUse, dev: identity.dev, ino: identity.ino } + } + + async entryIdentity (commit) { + try { + const stat = await this.fs.lstat(join(this.root, commit)) + return stat.isDirectory() && !stat.isSymbolicLink() ? stat : null + } catch (error) { + if (error.code === 'ENOENT') return null + throw error + } + } + + async directorySize (directory, measured) { + let total = 0 + const entries = await this.fs.readdir(directory, { withFileTypes: true }) + for (const entry of entries) { + if (++measured.nodes > this.scanLimit) throw new CacheError('CACHE_SCAN_LIMIT', 'Cache content limit exceeded') + const path = join(directory, entry.name) + if (entry.isDirectory()) total = safeAdd(total, await this.directorySize(path, measured)) + else if (entry.isFile() || entry.isSymbolicLink()) total = safeAdd(total, (await this.fs.lstat(path)).size) + } + return total + } + + state (commit, fallback = this.now()) { + let state = this.states.get(commit) + if (!state) { + state = { inUse: 0, lastUsedAt: fallback, deleting: null } + this.states.set(commit, state) + } + return state + } +} + +function parseCommit (commit) { + if (typeof commit !== 'string' || !SHA.test(commit)) throw new CacheError('INVALID_COMMIT', 'Commit must be a canonical lowercase 40-character SHA') + return commit +} + +function safeAdd (left, right) { + const total = left + right + if (!Number.isSafeInteger(total) || total < 0) throw new CacheError('CACHE_SIZE_OVERFLOW', 'Cache byte total exceeds the supported range') + return total +} + +function publicError (error) { + return { + code: typeof error.code === 'string' && /^[\x21-\x7e]{1,64}$/.test(error.code) ? error.code : 'CACHE_OPERATION_FAILED', + message: error instanceof CacheError ? error.message : 'Cache operation failed' + } +} + +async function mapLimit (values, limit, mapper) { + const results = new Array(values.length) + let next = 0 + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, async () => { + while (next < values.length) { + const index = next++ + results[index] = await mapper(values[index], index) + } + })) + return results +} + +module.exports = { CacheError, CacheManager, parseCommit } diff --git a/app/ops/config.js b/app/ops/config.js new file mode 100644 index 0000000..f804545 --- /dev/null +++ b/app/ops/config.js @@ -0,0 +1,138 @@ +'use strict' + +const { createHash, timingSafeEqual } = require('node:crypto') +const { isIP } = require('node:net') +const { isAbsolute } = require('node:path') + +const DOMAIN_SEPARATOR = Buffer.from('github.highcharts.com:ops-console:v1\0') +const BASE64URL_32 = /^[A-Za-z0-9_-]{43}$/ +const DEFAULT_MTLS_PORT = 8443 + +function decodeCanonical32 (value, name) { + if (typeof value !== 'string' || !BASE64URL_32.test(value)) { + throw new Error(`Invalid ${name}`) + } + const decoded = Buffer.from(value, 'base64url') + if (decoded.length !== 32 || decoded.toString('base64url') !== value) { + throw new Error(`Invalid ${name}`) + } + return decoded +} + +function tokenDigest (token) { + return createHash('sha256') + .update(DOMAIN_SEPARATOR) + .update(decodeCanonical32(token, 'operations console token')) + .digest() +} + +function createTokenVerifier (token) { + return `v1.${tokenDigest(token).toString('base64url')}` +} + +function parseTokenVerifier (value) { + if (typeof value !== 'string' || !value.startsWith('v1.')) { + throw new Error('Invalid OPS_CONSOLE_TOKEN_VERIFIER') + } + return decodeCanonical32(value.slice(3), 'OPS_CONSOLE_TOKEN_VERIFIER') +} + +function verifyToken (token, verifierDigest) { + if (!Buffer.isBuffer(verifierDigest) || verifierDigest.length !== 32) { + throw new Error('Invalid operations console token verifier') + } + let candidate = Buffer.alloc(32) + try { + candidate = tokenDigest(token) + } catch (error) {} + return timingSafeEqual(candidate, verifierDigest) +} + +function parseOrigin (value) { + if (typeof value !== 'string') throw new Error('Invalid OPS_CONSOLE_ORIGIN') + let url + try { + url = new URL(value) + } catch (error) { + throw new Error('Invalid OPS_CONSOLE_ORIGIN') + } + if ((url.protocol !== 'https:' && url.protocol !== 'http:') || url.origin !== value || url.username || url.password) { + throw new Error('Invalid OPS_CONSOLE_ORIGIN') + } + return url +} + +function nonblank (value) { + return typeof value === 'string' && value.trim() !== '' +} + +function parseAbsolutePath (value, name) { + if (typeof value !== 'string' || value.trim() !== value || !isAbsolute(value)) throw new Error(`Invalid ${name}`) + return value +} + +function parsePort (value) { + if (value === undefined || value === '') return DEFAULT_MTLS_PORT + if (typeof value !== 'string' || !/^[1-9]\d{0,4}$/.test(value)) throw new Error('Invalid OPS_CONSOLE_MTLS_PORT') + const port = Number(value) + if (port > 65535) throw new Error('Invalid OPS_CONSOLE_MTLS_PORT') + return port +} + +function isLoopbackHost (hostname) { + if (hostname === 'localhost' || hostname === '::1' || hostname === '[::1]') return true + if (isIP(hostname) !== 4) return false + return Number(hostname.split('.')[0]) === 127 +} + +function parseOpsConfig (env = process.env) { + if (env.OPS_CONSOLE_ENABLED !== 'true') return Object.freeze({ enabled: false }) + + const verifierDigest = parseTokenVerifier(env.OPS_CONSOLE_TOKEN_VERIFIER) + const originURL = parseOrigin(env.OPS_CONSOLE_ORIGIN) + const allowHttpLoopback = env.OPS_CONSOLE_ALLOW_HTTP_LOOPBACK === 'true' + const mtlsNames = ['OPS_CONSOLE_MTLS_KEY_PATH', 'OPS_CONSOLE_MTLS_CERT_PATH', 'OPS_CONSOLE_MTLS_CA_PATH'] + const hasMTLSSetting = mtlsNames.some(name => nonblank(env[name])) || nonblank(env.OPS_CONSOLE_MTLS_PORT) + + if (nonblank(env.OPS_CONSOLE_TRUSTED_PROXY)) { + throw new Error('OPS_CONSOLE_TRUSTED_PROXY is not supported; use operations console mTLS') + } + + if (originURL.protocol === 'http:') { + if (!allowHttpLoopback || !isLoopbackHost(originURL.hostname) || hasMTLSSetting) { + throw new Error('HTTP operations console requires direct loopback mode without mTLS settings') + } + return Object.freeze({ + enabled: true, + origin: originURL.origin, + protocol: 'http', + allowHttpLoopback, + verifierDigest + }) + } + + if (allowHttpLoopback) throw new Error('HTTPS operations console cannot allow HTTP loopback mode') + const mtls = Object.freeze({ + port: parsePort(env.OPS_CONSOLE_MTLS_PORT), + keyPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_KEY_PATH, 'OPS_CONSOLE_MTLS_KEY_PATH'), + certPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_CERT_PATH, 'OPS_CONSOLE_MTLS_CERT_PATH'), + caPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_CA_PATH, 'OPS_CONSOLE_MTLS_CA_PATH') + }) + + return Object.freeze({ + enabled: true, + origin: originURL.origin, + protocol: 'https', + allowHttpLoopback: false, + verifierDigest, + mtls + }) +} + +module.exports = { + createTokenVerifier, + DEFAULT_MTLS_PORT, + parseOpsConfig, + parseTokenVerifier, + verifyToken +} diff --git a/app/ops/console-router.js b/app/ops/console-router.js new file mode 100644 index 0000000..c8e52ad --- /dev/null +++ b/app/ops/console-router.js @@ -0,0 +1,484 @@ +'use strict' + +const { randomUUID } = require('node:crypto') +const path = require('node:path') +const express = require('express') +const appConfig = require('../../config.json') +const { createServiceClient } = require('../service-client') +const { parseOpsConfig, verifyToken } = require('./config') +const { + OpsHttpError, + asyncRoute, + errorBody, + getTrustedRequestContext, + readStrictJSON, + requireCSRF, + requireJSONContentType, + requireOrigin, + requireSameOriginFetch, + setSecurityHeaders +} = require('./http') +const { + deriveCacheOutcome, + validateCacheOperationRequest, + validateCacheOperationResponse, + validateLoginRequest, + validateServiceSnapshot +} = require('./schemas') +const { OpsRateLimiter, SessionStore, sessionResponse } = require('./sessions') +const { mergeActivity, serviceSlot } = require('./telemetry') + +const COOKIE_NAMES = Object.freeze({ http: 'ghhc-console-dev', https: '__Host-hc-ops' }) +const SESSION_ID = /^[A-Za-z0-9_-]{43}$/ +const SNAPSHOT_TIMEOUT_MS = 1500 +const CACHE_OPERATION_TIMEOUT_MS = 10000 +const MAX_SNAPSHOT_BYTES = 1024 * 1024 +const MAX_CACHE_OPERATION_BYTES = 64 * 1024 +const REFRESH_AFTER_MS = 30000 +const SPAN_OPERATIONS = [ + 'public_file_delivery', + 'github_branch_lookup', + 'github_commit_lookup', + 'github_esbuild_detection', + 'source_download', + 'file_download', + 'source_archive', + 'build' +] +const SLOT_ERROR_CODES = new Set(['INCOMPATIBLE_SCHEMA', 'SERVICE_RESPONSE_TOO_LARGE', 'SERVICE_TIMEOUT', 'SERVICE_UNAVAILABLE']) +const STATIC_DIR = path.join(__dirname, '../../static/ops') + +function createOpsConsoleRouter ({ env = process.env, log = logStatus, now = Date.now, sessions, rateLimiter, downloader, builder, routerService } = {}) { + const config = parseOpsConfig(env) + log({ time: new Date(now()).toISOString(), action: 'operations_console_status', enabled: config.enabled }) + + const router = express.Router() + router.use((request, response, next) => { + setSecurityHeaders(response) + next() + }) + + if (!config.enabled) { + router.use((request, response) => response.sendStatus(404)) + return router + } + + const store = sessions || new SessionStore({ now }) + const limiter = rateLimiter || new OpsRateLimiter({ now }) + const token = env.INTERNAL_SERVICE_TOKEN + downloader = downloader || createServiceClient({ baseURL: env.DOWNLOADER_URL || appConfig.downloaderURL, token, timeout: SNAPSHOT_TIMEOUT_MS }) + builder = builder || createServiceClient({ baseURL: env.BUILDER_URL || appConfig.builderURL || 'http://127.0.0.1:8082', token, timeout: SNAPSHOT_TIMEOUT_MS }) + routerService = routerService || defaultRouterService() + const snapshotState = Object.fromEntries(['router', 'downloader', 'builder'].map(service => [service, { + attempt: 0, + lastAttemptAt: null, + lastSuccess: null, + lastSuccessAttempt: 0, + error: null + }])) + const cookieName = COOKIE_NAMES[config.protocol] + const cleanup = setInterval(() => store.pruneExpired(), 60 * 1000) + cleanup.unref() + + router.use((request, response, next) => { + request.correlationId = randomUUID() + if (request.path.startsWith('/api/')) response.set('X-Correlation-ID', request.correlationId) + if (request.method === 'POST' && request.path === '/api/v1/cache-operations') { + request.cacheAudit = cacheAudit(request, now) + const emit = () => { + if (!request.cacheAudit.dispatched) emitCacheAudit(request, log, now) + } + response.once('finish', emit) + response.once('close', emit) + } + try { + request.opsContext = getTrustedRequestContext(request, config) + requireSameOriginFetch(request, { + allowTopLevelNavigation: request.path === '/' || request.path === '/login' + }) + next() + } catch (error) { + next(error) + } + }) + + router.get('/login', exactFile('/login', 'login.html')) + for (const asset of ['console.css', 'console.js', 'login.js']) { + router.get('/' + asset, exactFile('/' + asset, asset)) + } + router.get('/', (request, response, next) => { + if (request.originalUrl.split('?', 1)[0] !== '/_ops/') return next() + const sessionId = readSessionId(request, cookieName) + if (!store.authenticate(sessionId, { renew: false })) return response.redirect(302, '/_ops/login') + response.sendFile(path.join(STATIC_DIR, 'index.html')) + }) + + router.post('/api/v1/session', asyncRoute(async (request, response) => { + requireOrigin(request, config.origin) + enforceRateLimit(limiter.attemptLogin(), response) + requireJSONContentType(request) + const credentials = validateLoginRequest(await readStrictJSON(request)) + if (!verifyToken(credentials.token, config.verifierDigest)) throw authenticationFailed() + + const previousSessionId = readSessionId(request, cookieName) + const created = store.create(previousSessionId) + response.set('Set-Cookie', sessionCookie(cookieName, created.sessionId, config.protocol)) + response.status(201).json(sessionResponse(created.session)) + })) + + router.head('/api/v1/session', (request, response) => sendNotFound(response, request.correlationId)) + + router.get('/api/v1/session', (request, response, next) => { + try { + const { session } = authenticate(request, store, cookieName) + response.json(sessionResponse(session)) + } catch (error) { + next(error) + } + }) + + router.head('/api/v1/snapshot', (request, response) => sendNotFound(response, request.correlationId)) + + router.get('/api/v1/snapshot', asyncRoute(async (request, response) => { + const { sessionId } = authenticate(request, store, cookieName) + enforceRateLimit(limiter.attemptSnapshot(sessionId), response) + const attempts = await Promise.all([ + attemptSnapshot('router', () => routerService.snapshot(), snapshotState, now), + attemptSnapshot('downloader', () => downloader.json('/v1/ops/snapshot', snapshotRequest(request.correlationId)), snapshotState, now), + attemptSnapshot('builder', () => builder.json('/v1/ops/snapshot', snapshotRequest(request.correlationId)), snapshotState, now) + ]) + const byService = Object.fromEntries(attempts.map(attempt => [attempt.service, attempt])) + commitAttempt(byService.downloader, snapshotState.downloader) + commitAttempt(byService.builder, snapshotState.builder) + + const observedAt = new Date(now()).toISOString() + const remoteSlots = { + downloader: serviceSlot(snapshotState.downloader, observedAt), + builder: serviceSlot(snapshotState.builder, observedAt) + } + if (byService.router.snapshot) { + byService.router.snapshot = validateServiceSnapshot(deriveRouterSnapshot(byService.router.snapshot, remoteSlots), 'router') + } + commitAttempt(byService.router, snapshotState.router) + const slots = { + router: serviceSlot(snapshotState.router, observedAt), + ...remoteSlots + } + const fragments = [slots.downloader.snapshot, slots.builder.snapshot].filter(Boolean).flatMap(snapshot => snapshot.activity) + const merged = mergeActivity(slots.router.snapshot?.activity || [], fragments, SPAN_OPERATIONS) + if (slots.router.snapshot) { + slots.router.snapshot.telemetry.spansDropped = Math.min(Number.MAX_SAFE_INTEGER, slots.router.snapshot.telemetry.spansDropped + merged.spansDropped) + } + const failures = Object.values(slots).flatMap(slot => slot.snapshot?.failures || []) + .sort((a, b) => Date.parse(b.occurredAt) - Date.parse(a.occurredAt)) + + response.json({ + schemaVersion: 1, + correlationId: request.correlationId, + observedAt, + refreshAfterMs: REFRESH_AFTER_MS, + services: slots, + activity: merged.activity, + failures + }) + })) + + router.post('/api/v1/cache-operations', asyncRoute(async (request, response) => { + requireOrigin(request, config.origin) + const { sessionId, session } = authenticate(request, store, cookieName) + request.cacheAudit.sessionAuditId = session.auditId + requireCSRF(request, session.csrfToken) + enforceRateLimit(limiter.attemptCacheOperation(sessionId), response) + requireJSONContentType(request) + const command = validateCacheOperationRequest(await readStrictJSON(request)) + Object.assign(request.cacheAudit, { operation: command.operation, targets: command.targets, commit: command.commit || null }) + + const startedAt = new Date(now()).toISOString() + request.cacheAudit.dispatched = true + try { + const targets = await Promise.all(command.targets.map(service => attemptCacheTarget( + service, + service === 'downloader' ? downloader : builder, + command, + request.correlationId + ))) + const outcome = deriveCacheOutcome(targets) + Object.assign(request.cacheAudit, { outcome, targetOutcomes: targets }) + const result = validateCacheOperationResponse({ + correlationId: request.correlationId, + operation: command.operation, + startedAt, + completedAt: new Date(now()).toISOString(), + outcome, + targets + }) + response.status(200).json(result) + } finally { + emitCacheAudit(request, log, now) + } + })) + + router.delete('/api/v1/session', (request, response, next) => { + try { + requireOrigin(request, config.origin) + const { sessionId, session } = authenticate(request, store, cookieName) + requireCSRF(request, session.csrfToken) + store.revoke(sessionId) + limiter.removeSession(sessionId) + response.set('Set-Cookie', clearSessionCookie(cookieName, config.protocol)) + response.status(204).end() + } catch (error) { + next(error) + } + }) + + router.use('/api', (request, response) => sendNotFound(response, request.correlationId)) + router.use((request, response) => response.sendStatus(404)) + router.use((error, request, response, next) => { + if (response.headersSent) return next(error) + if (request.cacheAudit) { + request.cacheAudit.outcome = 'failed' + request.cacheAudit.errorCodes.push(safeMutationErrorCode(error)) + } + const result = errorBody(error, request.correlationId) + response.status(result.status).json(result.body) + }) + return router +} + +function authenticate (request, store, cookieName) { + const sessionId = readSessionId(request, cookieName) + const session = store.authenticate(sessionId) + if (!session) throw new OpsHttpError(401, 'UNAUTHORIZED', 'Authentication required') + return { sessionId, session } +} + +function exactFile (route, file) { + return (request, response, next) => { + if (request.path !== route) return next() + response.sendFile(path.join(STATIC_DIR, file)) + } +} + +function snapshotRequest (correlationId) { + return { correlationId, maxResponseBytes: MAX_SNAPSHOT_BYTES, timeout: SNAPSHOT_TIMEOUT_MS, timeoutThroughBody: true } +} + +async function attemptCacheTarget (service, client, command, correlationId) { + try { + const result = validateCacheOperationResponse(await deadline(client.json('/v1/ops/cache-operations', { + method: 'POST', + body: { ...command, targets: [service] }, + correlationId, + maxRequestBytes: 4 * 1024, + maxResponseBytes: MAX_CACHE_OPERATION_BYTES, + timeout: CACHE_OPERATION_TIMEOUT_MS, + timeoutThroughBody: true + }), CACHE_OPERATION_TIMEOUT_MS)) + if (result.correlationId !== correlationId || result.operation !== command.operation || + result.targets.length !== 1 || result.targets[0].service !== service) { + throw Object.assign(new Error('Incompatible cache operation response'), { code: 'INCOMPATIBLE_SCHEMA' }) + } + return result.targets[0] + } catch (error) { + return failedCacheTarget(service, command.operation, safeMutationErrorCode(error), isAmbiguousMutationError(error)) + } +} + +function failedCacheTarget (service, operation, code, ambiguous) { + const target = { + service, + outcome: ambiguous ? 'unknown' : 'failed', + removedEntries: 0, + freedBytes: 0, + absent: false, + skippedInUse: 0, + error: { code, message: ambiguous ? 'Cache operation outcome is unknown' : 'Cache operation failed' } + } + if (operation !== 'cache.clear') target.skippedChanged = 0 + return target +} + +function isAmbiguousMutationError (error) { + return ['SERVICE_TIMEOUT', 'SERVICE_RESPONSE_TOO_LARGE', 'INVALID_SERVICE_RESPONSE', 'INCOMPATIBLE_SCHEMA'].includes(error?.code) +} + +function safeMutationErrorCode (error) { + if (error?.code === 'SERVICE_TIMEOUT') return 'SERVICE_TIMEOUT' + if (['SERVICE_RESPONSE_TOO_LARGE', 'INVALID_SERVICE_RESPONSE', 'INCOMPATIBLE_SCHEMA'].includes(error?.code)) return 'INCOMPATIBLE_SCHEMA' + if (error?.code === 'SERVICE_UNAVAILABLE') return 'SERVICE_UNAVAILABLE' + if (error instanceof OpsHttpError) return error.code + return 'SERVICE_REJECTED' +} + +function cacheAudit (request, now) { + return { + startedAt: now(), + sessionAuditId: null, + source: null, + userAgent: safeUserAgent(request.headers['user-agent']), + operation: null, + targets: [], + commit: null, + dispatched: false, + outcome: 'failed', + targetOutcomes: [], + errorCodes: [], + emitted: false + } +} + +function emitCacheAudit (request, log, now) { + const audit = request.cacheAudit + if (!audit || audit.emitted) return + audit.emitted = true + audit.source = request.opsContext?.source || null + const errorCodes = [...new Set([ + ...audit.errorCodes, + ...audit.targetOutcomes.map(target => target.error?.code).filter(Boolean) + ])].slice(0, 8) + log({ + time: new Date(now()).toISOString(), + action: 'cache_operation', + correlationId: request.correlationId, + sessionAuditId: audit.sessionAuditId, + source: audit.source, + userAgent: audit.userAgent, + operation: audit.operation, + targets: audit.targets, + commit: audit.commit, + dispatchStatus: audit.dispatched ? 'dispatched' : 'not_dispatched', + outcome: audit.outcome, + targetOutcomes: audit.targetOutcomes.map(target => ({ + service: target.service, + outcome: target.outcome, + removedEntries: target.removedEntries, + freedBytes: target.freedBytes, + absent: target.absent, + skippedInUse: target.skippedInUse, + ...(target.skippedChanged === undefined ? {} : { skippedChanged: target.skippedChanged }) + })), + errorCodes, + durationMs: Math.max(0, now() - audit.startedAt) + }) +} + +function safeUserAgent (value) { + if (typeof value !== 'string') return null + const userAgent = value.toLowerCase() + if (userAgent.includes('firefox/') || userAgent.includes('fxios/')) return 'firefox' + if (['chromium/', 'chrome/', 'crios/', 'edg/', 'edga/', 'edgios/', 'opr/'].some(token => userAgent.includes(token))) return 'chromium' + if (userAgent.includes('safari/')) return 'safari' + return 'other' +} + +async function attemptSnapshot (service, provider, states, now) { + const state = states[service] + const attempt = ++state.attempt + const lastAttemptAt = new Date(now()).toISOString() + state.lastAttemptAt = lastAttemptAt + try { + const snapshot = validateServiceSnapshot(await deadline(provider(), SNAPSHOT_TIMEOUT_MS), service) + return { service, attempt, lastAttemptAt, snapshot, error: null } + } catch (error) { + return { service, attempt, lastAttemptAt, snapshot: null, error: slotError(error) } + } +} + +function commitAttempt (attempt, state) { + if (attempt.snapshot && attempt.attempt >= state.lastSuccessAttempt) { + state.lastSuccess = attempt.snapshot + state.lastSuccessAttempt = attempt.attempt + } + if (attempt.attempt === state.attempt) state.error = attempt.error +} + +function deadline (promise, milliseconds) { + let timer + const timeout = new Promise((resolve, reject) => { + timer = setTimeout(() => reject(Object.assign(new Error('Service deadline exceeded'), { code: 'SERVICE_TIMEOUT' })), milliseconds) + }) + return Promise.race([Promise.resolve(promise), timeout]).finally(() => clearTimeout(timer)) +} + +function slotError (error) { + return { code: SLOT_ERROR_CODES.has(error?.code) ? error.code : 'SERVICE_UNAVAILABLE' } +} + +function deriveRouterSnapshot (snapshot, slots) { + const dependencies = ['downloader', 'builder'].map(name => { + const slot = slots[name] + return { + name, + status: slot.freshness === 'fresh' ? 'available' : slot.freshness === 'stale' ? 'degraded' : 'unavailable', + lastAttemptAt: slot.lastAttemptAt, + lastSuccessAt: slot.lastSuccessAt, + lastFailureAt: slot.error ? slot.lastAttemptAt : null, + lastLatencyMs: null, + errorCode: slot.error?.code || null + } + }) + const downstreamStatus = dependencies.some(dependency => dependency.status === 'unavailable') + ? 'unavailable' + : dependencies.some(dependency => dependency.status === 'degraded') ? 'degraded' : 'available' + const reasonCode = downstreamStatus === 'available' ? null : 'DOWNSTREAM_' + downstreamStatus.toUpperCase() + const capabilities = [ + { name: 'public_file_delivery', status: downstreamStatus, reasonCode }, + { name: 'console_read', status: downstreamStatus, reasonCode }, + { name: 'console_cache_control', status: downstreamStatus, reasonCode } + ] + const reasons = [...new Set(capabilities.map(capability => capability.reasonCode).filter(Boolean))] + .map(code => ({ code, message: `A downstream service is ${downstreamStatus}` })) + return { + ...snapshot, + health: { status: capabilities.every(capability => capability.status === 'unavailable') ? 'unhealthy' : reasons.length ? 'degraded' : 'healthy', reasons }, + capabilities, + dependencies + } +} + +function defaultRouterService () { + const publicRouter = require('../router') + return { snapshot: publicRouter.opsSnapshot, telemetry: publicRouter.opsTelemetry } +} + +function readSessionId (request, cookieName) { + const header = request.headers.cookie + if (typeof header !== 'string' || Buffer.byteLength(header) > 4096) return undefined + const values = header.split(';') + .map(cookie => cookie.trim()) + .filter(cookie => cookie.startsWith(`${cookieName}=`)) + .map(cookie => cookie.slice(cookieName.length + 1)) + return values.length === 1 && SESSION_ID.test(values[0]) ? values[0] : undefined +} + +function sessionCookie (name, value, protocol) { + const secure = protocol === 'https' ? '; Secure' : '' + return `${name}=${value}${secure}; HttpOnly; SameSite=Strict; Path=/` +} + +function clearSessionCookie (name, protocol) { + const secure = protocol === 'https' ? '; Secure' : '' + return `${name}=; Max-Age=0${secure}; HttpOnly; SameSite=Strict; Path=/` +} + +function enforceRateLimit (result, response) { + if (result.allowed) return + response.set('Retry-After', String(result.retryAfter)) + throw new OpsHttpError(429, 'RATE_LIMITED', 'Too many requests') +} + +function authenticationFailed () { + return new OpsHttpError(401, 'UNAUTHORIZED', 'Authentication failed') +} + +function sendNotFound (response, correlationId) { + const result = errorBody(new OpsHttpError(404, 'NOT_FOUND', 'Console route not found'), correlationId) + response.status(result.status).json(result.body) +} + +function logStatus (event) { + console.log(JSON.stringify(event)) // eslint-disable-line no-console +} + +module.exports = { createOpsConsoleRouter } diff --git a/app/ops/http.js b/app/ops/http.js new file mode 100644 index 0000000..29a8d52 --- /dev/null +++ b/app/ops/http.js @@ -0,0 +1,275 @@ +'use strict' + +const { createHash, randomUUID, timingSafeEqual } = require('node:crypto') +const { isIP } = require('node:net') + +const MAX_BODY_BYTES = 4 * 1024 +const CSP = "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self'; connect-src 'self'; font-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'; object-src 'none'" +const SECURITY_HEADERS = Object.freeze({ + 'Cache-Control': 'no-store', + 'Content-Security-Policy': CSP, + 'X-Content-Type-Options': 'nosniff', + 'Referrer-Policy': 'no-referrer', + 'Permissions-Policy': 'camera=(), microphone=(), geolocation=()', + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Resource-Policy': 'same-origin', + 'X-Frame-Options': 'DENY' +}) +const CORRELATION_ID = /^[\x21-\x7e]{1,64}$/ + +class OpsHttpError extends Error { + constructor (status, code, message, fields) { + super(message) + this.name = 'OpsHttpError' + this.status = status + this.code = code + this.fields = fields + } +} + +function strictJSON (input, maximum = MAX_BODY_BYTES) { + const body = Buffer.isBuffer(input) ? input : Buffer.from(input || '') + if (body.length > maximum) throw new OpsHttpError(413, 'PAYLOAD_TOO_LARGE', 'Request body is too large') + + let source + try { + source = new TextDecoder('utf-8', { fatal: true }).decode(body) + } catch (error) { + throw invalidJSON() + } + + let position = 0 + const whitespace = () => { while (' \t\n\r'.includes(source[position]) && position < source.length) position++ } + const fail = () => { throw invalidJSON() } + const string = () => { + const start = position + if (source[position++] !== '"') fail() + while (position < source.length) { + const character = source[position++] + if (character === '"') { + try { return JSON.parse(source.slice(start, position)) } catch (error) { fail() } + } + if (character < ' ' || (character === '\\' && !/^(?:["\\/bfnrt]|u[0-9a-fA-F]{4})/.test(source.slice(position)))) fail() + if (character === '\\') position += source[position] === 'u' ? 5 : 1 + } + fail() + } + const value = () => { + whitespace() + if (source[position] === '"') return string() + if (source[position] === '{') return object() + if (source[position] === '[') return array() + for (const [literal, result] of [['true', true], ['false', false], ['null', null]]) { + if (source.startsWith(literal, position)) { + position += literal.length + return result + } + } + const match = source.slice(position).match(/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/) + if (!match) fail() + position += match[0].length + const number = Number(match[0]) + if (!Number.isFinite(number)) fail() + return number + } + const object = () => { + position++ + whitespace() + const result = Object.create(null) + const keys = new Set() + if (source[position] === '}') { position++; return result } + while (position < source.length) { + if (source[position] !== '"') fail() + const key = string() + if (keys.has(key)) throw new OpsHttpError(400, 'INVALID_JSON', 'Request body is invalid') + keys.add(key) + whitespace() + if (source[position++] !== ':') fail() + result[key] = value() + whitespace() + if (source[position] === '}') { position++; return result } + if (source[position++] !== ',') fail() + whitespace() + } + fail() + } + const array = () => { + position++ + whitespace() + const result = [] + if (source[position] === ']') { position++; return result } + while (position < source.length) { + result.push(value()) + whitespace() + if (source[position] === ']') { position++; return result } + if (source[position++] !== ',') fail() + } + fail() + } + + let result + try { + result = value() + whitespace() + } catch (error) { + if (error instanceof OpsHttpError) throw error + throw invalidJSON() + } + if (position !== source.length || !result || Array.isArray(result) || typeof result !== 'object') throw invalidJSON() + return result +} + +async function readStrictJSON (request, maximum = MAX_BODY_BYTES) { + const chunks = [] + let size = 0 + for await (const chunk of request) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + size += bytes.length + if (size > maximum) throw new OpsHttpError(413, 'PAYLOAD_TOO_LARGE', 'Request body is too large') + chunks.push(bytes) + } + return strictJSON(Buffer.concat(chunks, size), maximum) +} + +function invalidJSON () { + return new OpsHttpError(400, 'INVALID_JSON', 'Request body is invalid') +} + +function setSecurityHeaders (response) { + for (const [name, value] of Object.entries(SECURITY_HEADERS)) response.setHeader(name, value) +} + +function header (request, name) { + const value = request.headers?.[name.toLowerCase()] + return typeof value === 'string' ? value : undefined +} + +function requireOrigin (request, origin) { + if (header(request, 'origin') !== origin) throw new OpsHttpError(403, 'ORIGIN_REJECTED', 'Request origin was rejected') +} + +function requireSameOriginFetch (request, { allowTopLevelNavigation = false } = {}) { + const site = header(request, 'sec-fetch-site') + const topLevelNavigation = allowTopLevelNavigation && + request.method === 'GET' && + header(request, 'sec-fetch-mode') === 'navigate' && + header(request, 'sec-fetch-dest') === 'document' + if (site !== undefined && site !== 'same-origin' && !(site === 'none' && topLevelNavigation)) { + throw new OpsHttpError(403, 'FETCH_METADATA_REJECTED', 'Request context was rejected') + } +} + +function requireJSONContentType (request) { + const contentType = header(request, 'content-type') + if (!contentType || !/^application\/json(?:\s*;\s*charset=utf-8)?$/i.test(contentType)) { + throw new OpsHttpError(415, 'UNSUPPORTED_MEDIA_TYPE', 'Content-Type must be application/json') + } +} + +function timingSafeStringEqual (actual, expected) { + if (typeof expected !== 'string' || expected.length === 0) { + throw new Error('Invalid expected credential') + } + const validActual = typeof actual === 'string' && actual.length > 0 + const digest = value => createHash('sha256').update(value).digest() + return timingSafeEqual(digest(validActual ? actual : '\0'), digest(expected)) && validActual +} + +function requireCSRF (request, expected) { + if (!timingSafeStringEqual(header(request, 'x-ops-csrf'), expected)) { + throw new OpsHttpError(403, 'CSRF_REJECTED', 'CSRF validation failed') + } +} + +function requireBearer (request, expected) { + const authorization = header(request, 'authorization') || '' + const match = authorization.match(/^Bearer ([^\s]+)$/) + if (!timingSafeStringEqual(match?.[1], expected)) { + throw new OpsHttpError(401, 'UNAUTHORIZED', 'Authentication required') + } +} + +function normalizeIP (value) { + if (typeof value !== 'string') return null + const unwrapped = value.startsWith('[') && value.endsWith(']') ? value.slice(1, -1) : value + const mapped = unwrapped.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/i) + const normalized = mapped ? mapped[1] : unwrapped + return isIP(normalized) ? normalized : null +} + +function getTrustedRequestContext (request, config) { + if (config.protocol === 'https') { + if (request.socket?.encrypted !== true || request.socket?.authorized !== true) { + throw new OpsHttpError(400, 'INVALID_REQUEST_CONTEXT', 'Request context is invalid') + } + return Object.freeze({ protocol: 'https', source: null }) + } + + const peerIp = normalizeIP(request.socket?.remoteAddress) + if (request.socket?.encrypted || !peerIp || !isLoopbackIP(peerIp)) { + throw new OpsHttpError(400, 'INVALID_REQUEST_CONTEXT', 'Request context is invalid') + } + return Object.freeze({ protocol: 'http', source: peerIp }) +} + +function isLoopbackIP (address) { + if (address === '::1') return true + return isIP(address) === 4 && Number(address.split('.')[0]) === 127 +} + +function errorBody (error, correlationId = randomUUID()) { + const safe = error instanceof OpsHttpError + ? error + : new OpsHttpError(500, 'INTERNAL_ERROR', 'An internal error occurred') + const body = { error: { code: safe.code, message: safe.message, correlationId } } + if (Array.isArray(safe.fields)) { + const fields = [...new Set(safe.fields.filter(field => typeof field === 'string' && /^[\x20-\x7e]{1,64}$/.test(field)))].slice(0, 16) + if (fields.length) body.error.details = { fields } + } + return { status: safe.status, body } +} + +function asyncRoute (handler) { + return (request, response, next) => Promise.resolve(handler(request, response, next)).catch(next) +} + +function observedRoute (service, route, details, handler) { + return asyncRoute(async (request, response, next) => { + request.correlationId = CORRELATION_ID.test(request.get('x-correlation-id') || '') ? request.get('x-correlation-id') : randomUUID() + response.set('X-Correlation-ID', request.correlationId) + service.startRequest?.({ correlationId: request.correlationId, method: request.method, route, ...details(request) }) + let completed = false + const complete = outcome => { + if (completed) return + completed = true + service.completeRequest?.(request.correlationId, outcome) + } + response.once('finish', () => complete({ status: response.statusCode < 400 ? 'succeeded' : 'failed', httpStatus: response.statusCode })) + response.once('close', () => complete({ status: response.writableEnded ? 'succeeded' : 'aborted', httpStatus: response.statusCode })) + try { + await handler(request, response, next) + } catch (error) { + complete({ status: error.code === 'QUEUE_FULL' ? 'rejected' : 'failed', httpStatus: error.status || 500, code: error.code || 'INTERNAL_ERROR' }) + throw error + } + }) +} + +module.exports = { + MAX_BODY_BYTES, + OpsHttpError, + SECURITY_HEADERS, + asyncRoute, + errorBody, + getTrustedRequestContext, + observedRoute, + readStrictJSON, + requireBearer, + requireCSRF, + requireJSONContentType, + requireOrigin, + requireSameOriginFetch, + setSecurityHeaders, + strictJSON, + timingSafeStringEqual +} diff --git a/app/ops/internal-router.js b/app/ops/internal-router.js new file mode 100644 index 0000000..f912600 --- /dev/null +++ b/app/ops/internal-router.js @@ -0,0 +1,96 @@ +'use strict' + +const { randomUUID } = require('node:crypto') +const express = require('express') +const { OpsHttpError, asyncRoute, errorBody, readStrictJSON, requireBearer, requireJSONContentType } = require('./http') +const { deriveCacheOutcome, validateCacheOperationRequest, validateCacheOperationResponse, validateServiceSnapshot } = require('./schemas') + +const MAX_REQUEST_BYTES = 64 * 1024 +const MAX_RESPONSE_BYTES = 1024 * 1024 +const MAX_MUTATION_RESPONSE_BYTES = 64 * 1024 +const CORRELATION_ID = /^[\x21-\x7e]{1,64}$/ + +function createInternalOpsRouter ({ token, service, snapshot, cache, now = Date.now } = {}) { + if (typeof token !== 'string' || !token) throw new TypeError('Internal service token is required') + if (!['downloader', 'builder'].includes(service)) throw new TypeError('Internal operations service is invalid') + if (typeof snapshot !== 'function') throw new TypeError('Internal snapshot provider is required') + if (!cache || typeof cache.execute !== 'function') throw new TypeError('Internal cache manager is required') + + const router = express.Router() + + router.use((request, response, next) => { + response.set('Cache-Control', 'no-store') + response.set('X-Content-Type-Options', 'nosniff') + request.correlationId = correlationId(request) + response.set('X-Correlation-ID', request.correlationId) + try { + requireBearer(request, token) + next() + } catch (error) { + sendError(response, error, request.correlationId) + } + }) + + router.get('/v1/ops/snapshot', asyncRoute(async (request, response) => { + const value = await snapshot(request.correlationId) + const result = validateGenerated(() => validateServiceSnapshot(value, service)) + sendJSON(response, result, MAX_RESPONSE_BYTES) + })) + + router.post('/v1/ops/cache-operations', asyncRoute(async (request, response) => { + requireJSONContentType(request) + const command = validateCacheOperationRequest(await readStrictJSON(request, MAX_REQUEST_BYTES)) + if (command.targets.length !== 1 || command.targets[0] !== service) { + throw new OpsHttpError(400, 'INVALID_REQUEST', 'Request body is invalid', ['targets']) + } + + const startedAt = new Date(now()).toISOString() + const target = await cache.execute(command.operation, command.commit) + const result = validateGenerated(() => validateCacheOperationResponse({ + correlationId: request.correlationId, + operation: command.operation, + startedAt, + completedAt: new Date(now()).toISOString(), + outcome: deriveCacheOutcome([target]), + targets: [target] + })) + sendJSON(response, result, MAX_MUTATION_RESPONSE_BYTES) + })) + + router.use((error, request, response, next) => { + if (response.headersSent) return next(error) + sendError(response, error, request.correlationId) + }) + return router +} + +function correlationId (request) { + const value = request.get('x-correlation-id') + return CORRELATION_ID.test(value || '') ? value : randomUUID() +} + +function sendJSON (response, value, maximum) { + const body = Buffer.from(JSON.stringify(value)) + if (body.length > maximum) throw new Error('Internal operations response exceeded its limit') + response.status(200).type('application/json').send(body) +} + +function validateGenerated (validator) { + try { + return validator() + } catch (error) { + throw new Error('Internal operations provider returned invalid data') + } +} + +function sendError (response, error, correlationId) { + const result = errorBody(error, correlationId) + response.status(result.status).json(result.body) +} + +module.exports = { + MAX_MUTATION_RESPONSE_BYTES, + MAX_REQUEST_BYTES, + MAX_RESPONSE_BYTES, + createInternalOpsRouter +} diff --git a/app/ops/schemas.js b/app/ops/schemas.js new file mode 100644 index 0000000..16015dc --- /dev/null +++ b/app/ops/schemas.js @@ -0,0 +1,383 @@ +'use strict' + +const { OpsHttpError } = require('./http') + +const OPERATIONS = Object.freeze(['cache.evict_commit', 'cache.purge_expired', 'cache.clear']) +const TARGETS = Object.freeze(['downloader', 'builder']) +const OUTCOMES = Object.freeze(['completed', 'no_op', 'partial', 'failed', 'unknown']) +const SERVICES = Object.freeze(['router', 'downloader', 'builder']) +const SHA = /^[0-9a-f]{40}$/ +const CODE = /^[\x21-\x7e]{1,64}$/ +const RFC3339 = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/ + +class SchemaError extends OpsHttpError { + constructor (fields, code = 'INVALID_REQUEST', message = 'Request body is invalid') { + super(400, code, message, fields) + this.name = 'SchemaError' + } +} + +function record (value, field) { + if (!value || typeof value !== 'object' || Array.isArray(value)) invalid(field) + return value +} + +function exactKeys (value, required, optional = []) { + const allowed = new Set([...required, ...optional]) + const fields = Object.keys(value).filter(key => !allowed.has(key)).concat(required.filter(key => !(key in value))) + if (fields.length) throw new SchemaError(fields) +} + +function text (value, field, maximum = 256) { + if (typeof value !== 'string' || Buffer.byteLength(value) > maximum) invalid(field) + return value +} + +function code (value, field) { + if (typeof value !== 'string' || !CODE.test(value)) invalid(field) + return value +} + +function enumeration (value, values, field) { + if (!values.includes(value)) invalid(field) + return value +} + +function integer (value, field) { + if (!Number.isSafeInteger(value) || value < 0) invalid(field) + return value +} + +function timestamp (value, field) { + if (typeof value !== 'string' || !RFC3339.test(value) || Number.isNaN(Date.parse(value))) invalid(field) + return value +} + +function commit (value, field) { + if (typeof value !== 'string' || !SHA.test(value)) invalid(field) + return value +} + +function nullable (value, parser, field) { + return value === null ? null : parser(value, field) +} + +function list (value, field, maximum, parser) { + if (!Array.isArray(value) || value.length > maximum) invalid(field) + return value.map((entry, index) => parser(entry, `${field}.${index}`)) +} + +function invalid (field) { + throw new SchemaError([field]) +} + +function validateLoginRequest (value) { + const invalidLogin = () => { throw new OpsHttpError(401, 'UNAUTHORIZED', 'Authentication failed') } + if (!value || typeof value !== 'object' || Array.isArray(value) || + Object.keys(value).length !== 1 || !Object.hasOwn(value, 'token') || + typeof value.token !== 'string' || Buffer.byteLength(value.token) > 512 || + !/^[A-Za-z0-9_-]{43}$/.test(value.token)) invalidLogin() + const decoded = Buffer.from(value.token, 'base64url') + if (decoded.length !== 32 || decoded.toString('base64url') !== value.token) invalidLogin() + return { token: value.token } +} + +function validateCacheOperationRequest (value) { + record(value, 'body') + exactKeys(value, ['operation', 'targets'], ['commit']) + const operation = enumeration(value.operation, OPERATIONS, 'operation') + const targets = list(value.targets, 'targets', 2, (target, field) => enumeration(target, TARGETS, field)) + if (!targets.length || new Set(targets).size !== targets.length) invalid('targets') + + if (operation === 'cache.evict_commit') { + if (typeof value.commit !== 'string' || !SHA.test(value.commit)) invalid('commit') + } else if (targets.length !== 1 || 'commit' in value) { + invalid('targets') + } + return operation === 'cache.evict_commit' + ? { operation, targets, commit: value.commit } + : { operation, targets } +} + +function validateError (value, field) { + record(value, field) + if (!('code' in value) || !('message' in value)) invalid(field) + return { code: code(value.code, `${field}.code`), message: text(value.message, `${field}.message`) } +} + +function validateTargetResult (value, field, operation) { + record(value, field) + const required = ['service', 'outcome', 'removedEntries', 'freedBytes', 'absent', 'skippedInUse', 'error'] + if (operation !== 'cache.clear') required.push('skippedChanged') + for (const key of required) if (!(key in value)) invalid(`${field}.${key}`) + if (operation === 'cache.clear' && 'skippedChanged' in value) invalid(`${field}.skippedChanged`) + const target = { + service: enumeration(value.service, TARGETS, `${field}.service`), + outcome: enumeration(value.outcome, OUTCOMES, `${field}.outcome`), + removedEntries: integer(value.removedEntries, `${field}.removedEntries`), + freedBytes: integer(value.freedBytes, `${field}.freedBytes`), + absent: typeof value.absent === 'boolean' ? value.absent : invalid(`${field}.absent`), + skippedInUse: integer(value.skippedInUse, `${field}.skippedInUse`), + error: value.error === null ? null : validateError(value.error, `${field}.error`) + } + if (operation !== 'cache.clear') target.skippedChanged = integer(value.skippedChanged, `${field}.skippedChanged`) + if (target.outcome !== deriveTargetOutcome(target)) invalid(`${field}.outcome`) + return target +} + +function deriveTargetOutcome (target) { + if (target.outcome === 'unknown') return 'unknown' + if (target.error !== null) return 'failed' + const skipped = target.skippedInUse > 0 || (target.skippedChanged || 0) > 0 + if (target.removedEntries > 0 && skipped) return 'partial' + if (target.removedEntries > 0) return 'completed' + return 'no_op' +} + +function deriveCacheOutcome (targets) { + if (targets.some(target => target.outcome === 'unknown')) return 'unknown' + const failed = targets.filter(target => target.outcome === 'failed').length + if (failed === targets.length) return 'failed' + if (failed) return 'partial' + if (targets.some(target => target.outcome === 'partial')) return 'partial' + if (targets.some(target => target.removedEntries > 0)) return 'completed' + return 'no_op' +} + +function validateCacheOperationResponse (value) { + record(value, 'response') + for (const key of ['correlationId', 'operation', 'startedAt', 'completedAt', 'outcome', 'targets']) { + if (!(key in value)) incompatible(key) + } + const operation = enumeration(value.operation, OPERATIONS, 'operation') + const targets = list(value.targets, 'targets', 2, (target, field) => validateTargetResult(target, field, operation)) + if (!targets.length || new Set(targets.map(target => target.service)).size !== targets.length) incompatible('targets') + const result = { + correlationId: code(value.correlationId, 'correlationId'), + operation, + startedAt: timestamp(value.startedAt, 'startedAt'), + completedAt: timestamp(value.completedAt, 'completedAt'), + outcome: enumeration(value.outcome, OUTCOMES, 'outcome'), + targets + } + if (result.outcome !== deriveCacheOutcome(targets)) incompatible('outcome') + return result +} + +function validateServiceSnapshot (value, expectedService) { + try { + return parseServiceSnapshot(value, expectedService) + } catch (error) { + if (error instanceof SchemaError && error.code !== 'INCOMPATIBLE_SCHEMA') { + throw new SchemaError(error.fields, 'INCOMPATIBLE_SCHEMA', 'Internal service response is incompatible') + } + throw error + } +} + +function parseServiceSnapshot (value, expectedService) { + record(value, 'snapshot') + const required = ['schemaVersion', 'service', 'instanceId', 'startedAt', 'observedAt', 'health', 'capabilities', 'queues', 'cache', 'dependencies', 'telemetry'] + for (const key of required) if (!(key in value)) incompatible(key) + if (value.schemaVersion !== 1) incompatible('schemaVersion') + const service = enumeration(value.service, SERVICES, 'service') + if (expectedService && service !== expectedService) incompatible('service') + return { + schemaVersion: 1, + service, + instanceId: code(value.instanceId, 'instanceId'), + startedAt: timestamp(value.startedAt, 'startedAt'), + observedAt: timestamp(value.observedAt, 'observedAt'), + health: validateHealth(value.health), + capabilities: list(value.capabilities, 'capabilities', 16, validateCapability), + queues: list(value.queues, 'queues', 2, validateQueue), + cache: value.cache === null ? null : validateCache(value.cache), + dependencies: list(value.dependencies, 'dependencies', 3, validateDependency), + telemetry: validateTelemetry(value.telemetry), + activity: value.activity === undefined ? [] : list(value.activity, 'activity', 300, validateActivity), + failures: value.failures === undefined ? [] : list(value.failures, 'failures', 100, validateFailure) + } +} + +function validateHealth (value) { + record(value, 'health') + if (!('status' in value) || !('reasons' in value)) incompatible('health') + const health = { + status: enumeration(value.status, ['healthy', 'degraded', 'unhealthy'], 'health.status'), + reasons: list(value.reasons, 'health.reasons', 16, (reason, field) => { + record(reason, field) + return { code: code(reason.code, `${field}.code`), message: text(reason.message, `${field}.message`) } + }) + } + if (health.status !== 'healthy' && !health.reasons.length) incompatible('health.reasons') + return health +} + +function validateCapability (value, field) { + record(value, field) + const capability = { + name: code(value.name, `${field}.name`), + status: enumeration(value.status, ['available', 'degraded', 'unavailable'], `${field}.status`), + reasonCode: nullable(value.reasonCode, code, `${field}.reasonCode`) + } + if (capability.status !== 'available' && capability.reasonCode === null) incompatible(`${field}.reasonCode`) + return capability +} + +function validateQueue (value, field) { + record(value, field) + const queue = {} + for (const key of ['name', 'active', 'queued', 'limit', 'available', 'oldestQueuedAgeMs']) if (!(key in value)) incompatible(field) + queue.name = enumeration(value.name, ['download', 'build'], `${field}.name`) + for (const key of ['active', 'queued', 'limit', 'available']) queue[key] = integer(value[key], `${field}.${key}`) + queue.oldestQueuedAgeMs = nullable(value.oldestQueuedAgeMs, integer, `${field}.oldestQueuedAgeMs`) + if (queue.available !== Math.max(0, queue.limit - queue.active - queue.queued)) incompatible(`${field}.available`) + return queue +} + +function validateCache (value) { + record(value, 'cache') + const result = {} + for (const key of ['entryCount', 'totalBytes', 'idleExpiryMs', 'entriesTruncated', 'entries']) if (!(key in value)) incompatible(`cache.${key}`) + for (const key of ['entryCount', 'totalBytes', 'idleExpiryMs']) result[key] = integer(value[key], `cache.${key}`) + if (typeof value.entriesTruncated !== 'boolean') incompatible('cache.entriesTruncated') + result.entriesTruncated = value.entriesTruncated + result.entries = list(value.entries, 'cache.entries', 200, (entry, field) => { + record(entry, field) + if (typeof entry.commit !== 'string' || !SHA.test(entry.commit)) incompatible(`${field}.commit`) + return { + commit: entry.commit, + sizeBytes: integer(entry.sizeBytes, `${field}.sizeBytes`), + lastAccessedAt: timestamp(entry.lastAccessedAt, `${field}.lastAccessedAt`), + expiresAt: timestamp(entry.expiresAt, `${field}.expiresAt`), + inUse: integer(entry.inUse, `${field}.inUse`) + } + }) + return result +} + +function validateDependency (value, field) { + record(value, field) + return { + name: enumeration(value.name, ['github', 'downloader', 'builder'], `${field}.name`), + status: enumeration(value.status, ['available', 'degraded', 'unavailable', 'unknown'], `${field}.status`), + lastAttemptAt: nullable(value.lastAttemptAt, timestamp, `${field}.lastAttemptAt`), + lastSuccessAt: nullable(value.lastSuccessAt, timestamp, `${field}.lastSuccessAt`), + lastFailureAt: nullable(value.lastFailureAt, timestamp, `${field}.lastFailureAt`), + lastLatencyMs: nullable(value.lastLatencyMs, integer, `${field}.lastLatencyMs`), + errorCode: nullable(value.errorCode, code, `${field}.errorCode`) + } +} + +function validateTelemetry (value) { + record(value, 'telemetry') + const result = {} + for (const key of ['activityDropped', 'completedEvicted', 'failuresEvicted', 'spansDropped']) result[key] = integer(value[key], `telemetry.${key}`) + return result +} + +function validateActivity (value, field) { + record(value, field) + const state = enumeration(value.state, ['active', 'completed'], `${field}.state`) + const activity = { + correlationId: code(value.correlationId, `${field}.correlationId`), + state, + startedAt: timestamp(value.startedAt, `${field}.startedAt`), + completedAt: nullable(value.completedAt, timestamp, `${field}.completedAt`), + durationMs: nullable(value.durationMs, integer, `${field}.durationMs`), + request: validateActivityRequest(value.request, `${field}.request`), + outcome: validateActivityOutcome(value.outcome, `${field}.outcome`), + spans: list(value.spans, `${field}.spans`, 8, validateSpan) + } + const isComplete = activity.completedAt !== null && activity.durationMs !== null + if ((state === 'completed') !== isComplete) incompatible(field) + return activity +} + +function validateActivityRequest (value, field) { + record(value, field) + return { + method: code(value.method, `${field}.method`), + route: text(value.route, `${field}.route`), + commit: nullable(value.commit, commit, `${field}.commit`), + resource: nullable(value.resource, controlFreeText, `${field}.resource`), + buildMode: nullable(value.buildMode, (mode, modeField) => enumeration(mode, ['legacy', 'webpack', 'dashboards', 'esbuild', 'static'], modeField), `${field}.buildMode`) + } +} + +function controlFreeText (value, field) { + text(value, field) + if ([...value].some(character => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127)) invalid(field) + return value +} + +function validateActivityOutcome (value, field) { + record(value, field) + return { + status: enumeration(value.status, ['succeeded', 'failed', 'rejected', 'aborted'], `${field}.status`), + httpStatus: nullable(value.httpStatus, httpStatus, `${field}.httpStatus`), + code: nullable(value.code, code, `${field}.code`) + } +} + +function validateSpan (value, field) { + record(value, field) + const state = enumeration(value.state, ['active', 'completed'], `${field}.state`) + const span = { + service: enumeration(value.service, SERVICES, `${field}.service`), + operation: code(value.operation, `${field}.operation`), + state, + startedAt: timestamp(value.startedAt, `${field}.startedAt`), + completedAt: nullable(value.completedAt, timestamp, `${field}.completedAt`), + durationMs: nullable(value.durationMs, integer, `${field}.durationMs`), + outcome: validateSpanOutcome(value.outcome, `${field}.outcome`) + } + const isComplete = span.completedAt !== null && span.durationMs !== null + if ((state === 'completed') !== isComplete) incompatible(field) + return span +} + +function validateSpanOutcome (value, field) { + record(value, field) + return { + status: code(value.status, `${field}.status`), + httpStatus: nullable(value.httpStatus, httpStatus, `${field}.httpStatus`), + code: nullable(value.code, code, `${field}.code`) + } +} + +function httpStatus (value, field) { + integer(value, field) + if (value < 100 || value > 599) invalid(field) + return value +} + +function validateFailure (value, field) { + record(value, field) + return { + occurredAt: timestamp(value.occurredAt, `${field}.occurredAt`), + correlationId: code(value.correlationId, `${field}.correlationId`), + service: enumeration(value.service, SERVICES, `${field}.service`), + operation: code(value.operation, `${field}.operation`), + code: code(value.code, `${field}.code`), + summary: text(value.summary, `${field}.summary`), + httpStatus: nullable(value.httpStatus, httpStatus, `${field}.httpStatus`), + commit: nullable(value.commit, commit, `${field}.commit`) + } +} + +function incompatible (field) { + throw new SchemaError([field], 'INCOMPATIBLE_SCHEMA', 'Internal service response is incompatible') +} + +module.exports = { + OPERATIONS, + OUTCOMES, + SchemaError, + TARGETS, + deriveCacheOutcome, + validateCacheOperationRequest, + validateCacheOperationResponse, + validateLoginRequest, + validateServiceSnapshot +} diff --git a/app/ops/sessions.js b/app/ops/sessions.js new file mode 100644 index 0000000..5b8f001 --- /dev/null +++ b/app/ops/sessions.js @@ -0,0 +1,229 @@ +'use strict' + +const { randomBytes } = require('node:crypto') +const { OpsHttpError } = require('./http') + +const SESSION_CAPACITY = 64 +const IDLE_TIMEOUT_MS = 30 * 60 * 1000 +const ABSOLUTE_TIMEOUT_MS = 8 * 60 * 60 * 1000 +const LOGIN_WINDOW_MS = 15 * 60 * 1000 +const API_WINDOW_MS = 60 * 1000 + +function randomOpaqueValue (bytes = 32) { + return randomBytes(bytes).toString('base64url') +} + +function createAuditId () { + return randomOpaqueValue(16) +} + +function sessionResponse (session) { + return Object.freeze({ + authenticated: true, + idleExpiresAt: new Date(Math.min(session.lastUsedAt + IDLE_TIMEOUT_MS, session.createdAt + ABSOLUTE_TIMEOUT_MS)).toISOString(), + absoluteExpiresAt: new Date(session.createdAt + ABSOLUTE_TIMEOUT_MS).toISOString(), + csrfToken: session.csrfToken + }) +} + +class SessionStore { + constructor ({ now = Date.now, generation = 0 } = {}) { + this.now = now + this.generation = generation + this.sessions = new Map() + } + + create (replaceSessionId) { + const now = this.currentTime() + this.pruneExpired(now) + if (typeof replaceSessionId === 'string') this.sessions.delete(replaceSessionId) + if (this.sessions.size >= SESSION_CAPACITY) { + throw new OpsHttpError(503, 'SESSION_CAPACITY', 'Session capacity is unavailable') + } + + let sessionId + do sessionId = randomOpaqueValue() + while (this.sessions.has(sessionId)) + + const session = Object.freeze({ + createdAt: now, + lastUsedAt: now, + csrfToken: randomOpaqueValue(), + generation: this.generation, + auditId: createAuditId() + }) + this.sessions.set(sessionId, session) + return Object.freeze({ sessionId, session }) + } + + authenticate (sessionId, { renew = true } = {}) { + if (typeof sessionId !== 'string') return null + const now = this.currentTime() + const session = this.sessions.get(sessionId) + if (!session || this.isExpired(session, now) || session.generation !== this.generation) { + this.sessions.delete(sessionId) + return null + } + if (!renew) return session + + const renewed = Object.freeze({ ...session, lastUsedAt: now }) + this.sessions.set(sessionId, renewed) + return renewed + } + + revoke (sessionId) { + return typeof sessionId === 'string' && this.sessions.delete(sessionId) + } + + rotate (generation = this.generation + 1) { + if (generation === this.generation) return 0 + const revoked = this.sessions.size + this.sessions.clear() + this.generation = generation + return revoked + } + + pruneExpired (now = this.currentTime()) { + let removed = 0 + for (const [sessionId, session] of this.sessions) { + if (!this.isExpired(session, now) && session.generation === this.generation) continue + this.sessions.delete(sessionId) + removed++ + } + return removed + } + + get size () { + return this.sessions.size + } + + currentTime () { + const now = this.now() + if (!Number.isFinite(now)) throw new Error('Session clock returned an invalid time') + return now + } + + isExpired (session, now) { + return now >= session.lastUsedAt + IDLE_TIMEOUT_MS || now >= session.createdAt + ABSOLUTE_TIMEOUT_MS + } +} + +class OpsRateLimiter { + constructor ({ now = Date.now } = {}) { + this.now = now + this.snapshotSessions = new Map() + this.cacheSessions = new Map() + this.loginGlobal = null + this.cacheGlobal = null + } + + attemptLogin () { + const now = this.currentTime() + this.prune(now) + return this.consumeGlobal('loginGlobal', 30, LOGIN_WINDOW_MS, now) + } + + attemptSnapshot (sessionId) { + const now = this.currentTime() + this.prune(now) + return this.consume(this.snapshotSessions, sessionId, 12, API_WINDOW_MS, SESSION_CAPACITY, now) + } + + attemptCacheOperation (sessionId) { + const now = this.currentTime() + this.prune(now) + const perSession = this.consume(this.cacheSessions, sessionId, 5, API_WINDOW_MS, SESSION_CAPACITY, now) + const global = this.consumeGlobal('cacheGlobal', 20, API_WINDOW_MS, now) + return combineLimits(perSession, global) + } + + removeSession (sessionId) { + this.snapshotSessions.delete(sessionId) + this.cacheSessions.delete(sessionId) + } + + prune (now = this.currentTime()) { + for (const buckets of [this.snapshotSessions, this.cacheSessions]) { + for (const [key, bucket] of buckets) { + if (now >= bucket.resetAt) buckets.delete(key) + } + } + if (this.loginGlobal && now >= this.loginGlobal.resetAt) this.loginGlobal = null + if (this.cacheGlobal && now >= this.cacheGlobal.resetAt) this.cacheGlobal = null + } + + get bucketCounts () { + return Object.freeze({ + snapshotSessions: this.snapshotSessions.size, + cacheSessions: this.cacheSessions.size + }) + } + + currentTime () { + const now = this.now() + if (!Number.isFinite(now)) throw new Error('Rate-limit clock returned an invalid time') + return now + } + + consume (buckets, key, limit, windowMs, capacity, now) { + if (typeof key !== 'string' || !key) throw new Error('Invalid rate-limit key') + const bucket = buckets.get(key) + if (!bucket && buckets.size >= capacity) { + const resetAt = Math.min(...Array.from(buckets.values(), value => value.resetAt)) + return limitResult(false, 0, resetAt, now) + } + if (bucket) { + buckets.delete(key) + buckets.set(key, bucket) + } + const current = bucket || { count: 0, resetAt: now + windowMs } + if (current.count >= limit) return limitResult(false, 0, current.resetAt, now) + + current.count++ + buckets.set(key, current) + return limitResult(true, limit - current.count, current.resetAt, now) + } + + consumeGlobal (name, limit, windowMs, now) { + let bucket = this[name] + if (!bucket) bucket = { count: 0, resetAt: now + windowMs } + if (bucket.count >= limit) return limitResult(false, 0, bucket.resetAt, now) + bucket.count++ + this[name] = bucket + return limitResult(true, limit - bucket.count, bucket.resetAt, now) + } +} + +function limitResult (allowed, remaining, resetAt, now) { + return Object.freeze({ + allowed, + remaining, + retryAfter: allowed ? 0 : Math.max(1, Math.ceil((resetAt - now) / 1000)), + resetAt + }) +} + +function combineLimits (...results) { + const denied = results.filter(result => !result.allowed) + if (!denied.length) { + return limitResult(true, Math.min(...results.map(result => result.remaining)), Math.max(...results.map(result => result.resetAt)), 0) + } + return Object.freeze({ + allowed: false, + remaining: 0, + retryAfter: Math.max(...denied.map(result => result.retryAfter)), + resetAt: Math.max(...denied.map(result => result.resetAt)) + }) +} + +module.exports = { + ABSOLUTE_TIMEOUT_MS, + API_WINDOW_MS, + IDLE_TIMEOUT_MS, + LOGIN_WINDOW_MS, + OpsRateLimiter, + SESSION_CAPACITY, + SessionStore, + createAuditId, + sessionResponse +} diff --git a/app/ops/telemetry.js b/app/ops/telemetry.js new file mode 100644 index 0000000..469fc59 --- /dev/null +++ b/app/ops/telemetry.js @@ -0,0 +1,417 @@ +'use strict' + +const { randomUUID } = require('node:crypto') + +const LIMITS = Object.freeze({ + active: 100, + completed: 200, + completedAgeMs: 15 * 60 * 1000, + failures: 100, + failureAgeMs: 60 * 60 * 1000, + spans: 8, + cacheEntries: 200, + windowSamples: 200, + healthWindowMs: 5 * 60 * 1000, + staleMs: 2 * 60 * 1000 +}) +const SERVICES = new Set(['router', 'downloader', 'builder']) +const DEPENDENCIES = new Set(['github', 'downloader', 'builder']) +const CAPABILITIES = Object.freeze({ + router: new Set(['public_file_delivery', 'console_read', 'console_cache_control']), + downloader: new Set(['ref_resolution', 'source_file_delivery', 'source_archive_delivery', 'cache_control']), + builder: new Set(['build_delivery', 'cache_control']) +}) +const OUTCOMES = new Set(['succeeded', 'failed', 'rejected', 'aborted']) +const BUILD_MODES = new Set(['legacy', 'webpack', 'dashboards', 'esbuild', 'static']) +const CODE = /^[\x21-\x7e]{1,64}$/ +const SHA = /^[0-9a-f]{40}$/ +const INTERNAL_SUMMARY = 'An internal error occurred' + +function iso (time) { + return new Date(time).toISOString() +} + +function integer (value) { + return Number.isSafeInteger(value) && value >= 0 ? value : null +} + +function safeCode (value) { + return typeof value === 'string' && CODE.test(value) ? value : null +} + +function boundedText (value, maximum = 256) { + if (typeof value !== 'string') return null + const characters = [...value].filter(character => character.charCodeAt(0) >= 32 && character.charCodeAt(0) !== 127) + while (Buffer.byteLength(characters.join('')) > maximum) characters.pop() + return characters.join('') +} + +function normalizeResource (value) { + if (typeof value !== 'string' || value.includes('://') || value.includes('?') || value.includes('#') || value.includes('\\')) return null + const result = boundedText(value) + if (!result || result.split('/').includes('..') || !/^[\p{L}\p{N}._@/+-]+$/u.test(result)) return null + return result +} + +function safeOutcome (value, active = false) { + if (active) return { status: 'succeeded', httpStatus: null, code: null } + const status = OUTCOMES.has(value?.status) ? value.status : 'failed' + const httpStatus = integer(value?.httpStatus) + return { + status, + httpStatus: httpStatus >= 100 && httpStatus <= 599 ? httpStatus : null, + code: safeCode(value?.code) + } +} + +class Telemetry { + constructor ({ service, operations = [], routes = [], failureSummaries = {}, now = Date.now, instanceId = randomUUID() }) { + if (!SERVICES.has(service)) throw new Error('Invalid telemetry service') + this.service = service + this.operations = new Set(operations.filter(safeCode)) + this.routes = new Set(routes.filter(route => typeof route === 'string' && boundedText(route) === route)) + this.failureSummaries = new Map([['INTERNAL_ERROR', INTERNAL_SUMMARY]]) + for (const [code, summary] of Object.entries(failureSummaries)) { + if (safeCode(code) && boundedText(summary) === summary) this.failureSummaries.set(code, summary) + } + this.now = now + this.instanceId = safeCode(instanceId) || randomUUID() + this.startedAt = iso(now()) + this.active = new Map() + this.completed = [] + this.failures = [] + this.fragments = new Map() + this.dependencyHistory = new Map() + this.window = [] + this.counters = { activityDropped: 0, completedEvicted: 0, failuresEvicted: 0, spansDropped: 0 } + } + + startTrace ({ correlationId, method, route, commit = null, resource = null, buildMode = null }) { + this.prune() + if (!safeCode(correlationId) || !safeCode(method) || !this.routes.has(route)) return null + if (this.active.has(correlationId)) return structuredClone(this.active.get(correlationId)) + if (this.active.size >= LIMITS.active) { + const oldest = this.active.keys().next().value + this.active.delete(oldest) + this.counters.activityDropped++ + } + const started = this.now() + const trace = { + correlationId, + state: 'active', + startedAt: iso(started), + completedAt: null, + durationMs: null, + request: { + method, + route, + commit: SHA.test(commit || '') ? commit : null, + resource: normalizeResource(resource), + buildMode: BUILD_MODES.has(buildMode) ? buildMode : null + }, + outcome: safeOutcome(null, true), + spans: [] + } + Object.defineProperty(trace, '_started', { value: started, enumerable: false }) + this.active.set(correlationId, trace) + return structuredClone(trace) + } + + completeTrace (correlationId, outcome) { + const trace = this.active.get(correlationId) + if (!trace) return null + const completed = this.now() + this.active.delete(correlationId) + trace.state = 'completed' + trace.completedAt = iso(completed) + trace.durationMs = Math.max(0, completed - trace._started) + trace.outcome = safeOutcome(outcome) + trace.spans = this.takeFragments(correlationId) + Object.defineProperty(trace, '_completed', { value: completed, enumerable: false }) + this.completed.push(trace) + this.recordWindow(trace.outcome.status, trace.durationMs, outcome?.timeoutMs) + this.prune() + return structuredClone(trace) + } + + startSpan (correlationId, operation) { + if (!safeCode(correlationId) || !this.operations.has(operation)) return null + if (!this.fragments.has(correlationId) && this.fragments.size >= LIMITS.active) { + const oldest = this.fragments.keys().next().value + this.counters.spansDropped += this.fragments.get(oldest).length + this.fragments.delete(oldest) + } + const spans = this.fragments.get(correlationId) || [] + if (spans.length >= LIMITS.spans) { + this.counters.spansDropped++ + return null + } + const started = this.now() + const span = { + service: this.service, + operation, + state: 'active', + startedAt: iso(started), + completedAt: null, + durationMs: null, + outcome: { status: 'active', httpStatus: null, code: null } + } + Object.defineProperty(span, '_started', { value: started, enumerable: false }) + spans.push(span) + this.fragments.set(correlationId, spans) + return structuredClone(span) + } + + completeSpan (correlationId, operation, outcome) { + const span = this.fragments.get(correlationId)?.find(entry => entry.operation === operation && entry.state === 'active') + if (!span) return null + const completed = this.now() + span.state = 'completed' + span.completedAt = iso(completed) + span.durationMs = Math.max(0, completed - span._started) + span.outcome = safeOutcome(outcome) + return structuredClone(span) + } + + takeFragments (correlationId) { + const spans = this.fragments.get(correlationId) || [] + this.fragments.delete(correlationId) + return structuredClone(spans) + } + + recordFailure ({ correlationId, operation, code, httpStatus = null, commit = null }) { + this.prune() + if (!safeCode(correlationId) || !this.operations.has(operation)) return null + const safeFailureCode = safeCode(code) && this.failureSummaries.has(code) ? code : 'INTERNAL_ERROR' + const occurred = this.now() + const status = integer(httpStatus) + const failure = { + occurredAt: iso(occurred), + correlationId, + service: this.service, + operation, + code: safeFailureCode, + summary: this.failureSummaries.get(safeFailureCode), + httpStatus: status >= 100 && status <= 599 ? status : null, + commit: SHA.test(commit || '') ? commit : null + } + Object.defineProperty(failure, '_occurred', { value: occurred, enumerable: false }) + this.failures.push(failure) + this.prune() + return structuredClone(failure) + } + + recordDependency (name, { succeeded, latencyMs = null, errorCode = null }) { + if (!DEPENDENCIES.has(name)) return null + const history = this.dependencyHistory.get(name) || [] + const safeDependencyCode = safeCode(errorCode) && this.failureSummaries.has(errorCode) ? errorCode : 'INTERNAL_ERROR' + history.push({ at: this.now(), succeeded: succeeded === true, latencyMs: integer(latencyMs), errorCode: safeDependencyCode }) + this.dependencyHistory.set(name, history) + this.prune() + return this.dependencies().find(dependency => dependency.name === name) + } + + recordWindow (status, durationMs, timeoutMs = null) { + this.window.push({ at: this.now(), status: OUTCOMES.has(status) ? status : 'failed', durationMs: integer(durationMs) || 0, timeoutMs: integer(timeoutMs) }) + this.prune() + } + + healthSignals (queues = []) { + this.prune() + const completed = this.window.length + const failures = this.window.filter(sample => sample.status === 'failed' || sample.status === 'rejected').length + const latencies = this.window.filter(sample => sample.timeoutMs !== null).sort((a, b) => a.durationMs - b.durationMs) + const p95 = latencies.length ? latencies[Math.ceil(latencies.length * 0.95) - 1] : null + return { + queueSaturated: queues.some(queue => queue.active + queue.queued >= queue.limit), + capacityRejected: this.window.some(sample => sample.status === 'rejected'), + reliabilityDegraded: completed >= 10 && failures >= 3 && failures / completed >= 0.1, + latencyDegraded: latencies.length >= 10 && p95.durationMs >= p95.timeoutMs * 0.8, + dependencyDegraded: this.dependencies().some(dependency => dependency.status === 'degraded' || dependency.status === 'unavailable') + } + } + + dependencies () { + this.prune() + return [...this.dependencyHistory].map(([name, history]) => { + const latest = history.at(-1) + const successes = history.filter(entry => entry.succeeded) + const failures = history.filter(entry => !entry.succeeded) + const status = !latest + ? 'unknown' + : latest.succeeded + ? 'available' + : history.at(-2)?.succeeded + ? 'degraded' + : 'unavailable' + return { + name, + status, + lastAttemptAt: latest ? iso(latest.at) : null, + lastSuccessAt: successes.length ? iso(successes.at(-1).at) : null, + lastFailureAt: failures.length ? iso(failures.at(-1).at) : null, + lastLatencyMs: latest?.latencyMs ?? null, + errorCode: latest?.succeeded ? null : latest?.errorCode ?? 'INTERNAL_ERROR' + } + }).sort((a, b) => a.name.localeCompare(b.name)) + } + + activity () { + this.prune() + return [...this.active.values()].sort((a, b) => b._started - a._started).concat([...this.completed].sort((a, b) => b._completed - a._completed)).map(value => structuredClone(value)) + } + + getFailures () { + this.prune() + return [...this.failures].sort((a, b) => b._occurred - a._occurred).map(value => structuredClone(value)) + } + + snapshot ({ capabilities, queues = [], cache = null }) { + const observedAt = iso(this.now()) + const normalizedCapabilities = capabilities.map(capability => ({ + name: CAPABILITIES[this.service].has(capability.name) ? capability.name : null, + status: capability.status === 'unavailable' ? 'unavailable' : capability.status === 'degraded' ? 'degraded' : 'available', + reasonCode: capability.status === 'available' ? null : safeCode(capability.reasonCode) || 'INTERNAL_ERROR' + })).filter(capability => capability.name) + const affected = normalizedCapabilities.filter(capability => capability.status !== 'available') + const unavailable = normalizedCapabilities.length > 0 && affected.length === normalizedCapabilities.length && affected.every(capability => capability.status === 'unavailable') + return { + schemaVersion: 1, + service: this.service, + instanceId: this.instanceId, + startedAt: this.startedAt, + observedAt, + health: { + status: unavailable ? 'unhealthy' : affected.length ? 'degraded' : 'healthy', + reasons: [...new Set(normalizedCapabilities.map(capability => capability.reasonCode).filter(Boolean))].map(code => ({ code, message: code.replaceAll('_', ' ').toLowerCase() })) + }, + capabilities: normalizedCapabilities, + queues: queues.map(queue => queueSnapshot(queue.name, queue)), + cache, + dependencies: this.dependencies(), + telemetry: { ...this.counters }, + activity: this.activity(), + failures: this.getFailures() + } + } + + prune () { + const now = this.now() + const trim = (items, age, maximum, counter) => { + let removed = 0 + while (items.length && (now - (items[0]._completed ?? items[0]._occurred ?? items[0].at) > age || items.length > maximum)) { + items.shift() + removed++ + } + if (counter) this.counters[counter] += removed + } + trim(this.completed, LIMITS.completedAgeMs, LIMITS.completed, 'completedEvicted') + trim(this.failures, LIMITS.failureAgeMs, LIMITS.failures, 'failuresEvicted') + trim(this.window, LIMITS.healthWindowMs, LIMITS.windowSamples) + for (const history of this.dependencyHistory.values()) trim(history, LIMITS.healthWindowMs, LIMITS.windowSamples) + } +} + +function queueSnapshot (name, metrics) { + if (name !== 'download' && name !== 'build') throw new Error('Invalid queue name') + const active = integer(metrics.active) || 0 + const queued = integer(metrics.queued) || 0 + const limit = integer(metrics.limit) || 0 + return { + name, + active, + queued, + limit, + available: Math.max(0, limit - active - queued), + oldestQueuedAgeMs: queued ? integer(metrics.oldestQueuedAgeMs) : null + } +} + +function cacheSnapshot (entries, idleExpiryMs) { + const safeEntries = entries.filter(entry => SHA.test(entry.commit || '') && + integer(entry.sizeBytes) !== null && + integer(entry.inUse) !== null && + Number.isFinite(Date.parse(entry.lastAccessedAt)) && + Number.isFinite(Date.parse(entry.expiresAt))) + const visible = [...safeEntries].sort((a, b) => Date.parse(b.lastAccessedAt) - Date.parse(a.lastAccessedAt)).slice(0, LIMITS.cacheEntries) + return { + entryCount: safeEntries.length, + totalBytes: safeEntries.reduce((total, entry) => total + entry.sizeBytes, 0), + idleExpiryMs: integer(idleExpiryMs) || 0, + entriesTruncated: safeEntries.length > LIMITS.cacheEntries, + entries: visible.map(entry => ({ + commit: entry.commit, + sizeBytes: entry.sizeBytes, + lastAccessedAt: iso(Date.parse(entry.lastAccessedAt)), + expiresAt: iso(Date.parse(entry.expiresAt)), + inUse: entry.inUse + })) + } +} + +function mergeActivity (canonicalTraces, fragments, operations = []) { + const traces = canonicalTraces.map(value => structuredClone(value)) + const byCorrelation = new Map(traces.map(trace => [trace.correlationId, trace])) + const allowedOperations = new Set(operations.filter(safeCode)) + let spansDropped = 0 + for (const fragment of fragments) { + const trace = byCorrelation.get(fragment.correlationId) + if (!trace || !Array.isArray(fragment.spans)) continue + const keys = new Set(trace.spans.map(span => `${trace.correlationId}:${span.service}:${span.operation}`)) + for (const span of fragment.spans) { + const key = `${trace.correlationId}:${span.service}:${span.operation}` + if (!SERVICES.has(span.service) || !allowedOperations.has(span.operation) || keys.has(key)) continue + if (trace.spans.length >= LIMITS.spans) { + spansDropped++ + continue + } + trace.spans.push({ + service: span.service, + operation: span.operation, + state: span.state === 'active' ? 'active' : 'completed', + startedAt: span.startedAt, + completedAt: span.state === 'active' ? null : span.completedAt, + durationMs: span.state === 'active' ? null : integer(span.durationMs), + outcome: { + status: safeCode(span.outcome?.status) || 'failed', + httpStatus: integer(span.outcome?.httpStatus), + code: safeCode(span.outcome?.code) + } + }) + keys.add(key) + } + } + return { activity: traces, spansDropped } +} + +function serviceSlot ({ lastAttemptAt = null, lastSuccess = null, error = null }, observedAt) { + const observed = typeof observedAt === 'number' ? observedAt : Date.parse(observedAt) + const successAt = lastSuccess && Date.parse(lastSuccess.observedAt) + if (!lastSuccess || !Number.isFinite(successAt) || observed - successAt > LIMITS.staleMs) { + return { freshness: 'unknown', lastAttemptAt, lastSuccessAt: lastSuccess?.observedAt || null, ageMs: null, snapshot: null, error: safeSlotError(error) } + } + const ageMs = Math.max(0, observed - successAt) + return { + freshness: error ? 'stale' : 'fresh', + lastAttemptAt, + lastSuccessAt: lastSuccess.observedAt, + ageMs, + snapshot: structuredClone(lastSuccess), + error: safeSlotError(error) + } +} + +function safeSlotError (error) { + if (!error) return null + return { code: safeCode(error.code) || 'INTERNAL_ERROR', message: 'Service snapshot unavailable' } +} + +module.exports = { + LIMITS, + Telemetry, + cacheSnapshot, + mergeActivity, + normalizeResource, + queueSnapshot, + serviceSlot +} diff --git a/app/router.js b/app/router.js index 56815f0..6d64233 100644 --- a/app/router.js +++ b/app/router.js @@ -1,8 +1,9 @@ 'use strict' const { Readable } = require('node:stream') +const { randomUUID } = require('node:crypto') const { pipeline } = require('node:stream/promises') -const { join } = require('node:path') +const { join, posix } = require('node:path') const { readFile } = require('node:fs/promises') const express = require('express') const rateLimit = require('express-rate-limit') @@ -10,6 +11,7 @@ const slowDown = require('express-slow-down') const config = require('../config.json') const { version } = require('../package.json') const { createServiceClient, ServiceError } = require('./service-client') +const { Telemetry } = require('./ops/telemetry') const { getBranch, getFile, getFileForEsbuild, getType } = require('./interpreter.js') const { catchAsyncErrors, @@ -22,9 +24,22 @@ const { } = require('./handlers.js') const indexPath = join(__dirname, '..', 'static', 'index.html') +const PUBLIC_ROUTE = '/:ref/*' +const PUBLIC_OPERATION = 'public_file_delivery' +const FAILURE_SUMMARIES = { + FILE_NOT_FOUND: 'File was not found', + INVALID_BUILD: 'Build request was invalid', + QUEUE_FULL: 'Service capacity is unavailable', + RATE_LIMITED: 'GitHub rate limit is exhausted', + REF_NOT_FOUND: 'Ref was not found', + SERVICE_TIMEOUT: 'Internal service timed out', + SERVICE_UNAVAILABLE: 'Internal service is unavailable', + UPSTREAM_TIMEOUT: 'GitHub request timed out' +} function createRouter (options = {}) { const router = express.Router() + const opsConsoleEnabled = options.opsConsoleEnabled ?? process.env.OPS_CONSOLE_ENABLED === 'true' const token = options.token ?? process.env.INTERNAL_SERVICE_TOKEN const downloader = options.downloader || createServiceClient({ baseURL: process.env.DOWNLOADER_URL || config.downloaderURL, @@ -36,16 +51,31 @@ function createRouter (options = {}) { token, timeout: process.env.PUBLIC_BUILDER_TIMEOUT || config.publicBuilderTimeout || 30000 }) + const now = options.now || Date.now + const telemetry = options.telemetry || new Telemetry({ + service: 'router', + operations: [PUBLIC_OPERATION], + routes: [PUBLIC_ROUTE], + failureSummaries: FAILURE_SUMMARIES, + now + }) + + router.use((req, res, next) => { + req.correlationId = randomUUID() + res.set('X-Correlation-ID', req.correlationId) + next() + }) router.get('/health', catchAsyncErrors(handlerHealth)) router.get('/favicon.ico', catchAsyncErrors(handlerIcon)) router.get('/robots.txt', catchAsyncErrors(handlerRobots)) router.get('/cleanup', catchAsyncErrors(async (req, res) => { + if (opsConsoleEnabled) return res.sendStatus(404) if (!req.url.includes('?true')) return send(res, req, 400, 'Something went wrong. Please note that this service is for debugging only. Use our official CDN in production.') const body = { force: false } const [, result] = await Promise.all([ - downloader.json('/v1/cleanup', { method: 'POST', body }), - builder.json('/v1/cleanup', { method: 'POST', body }) + downloader.json('/v1/cleanup', { method: 'POST', body, correlationId: req.correlationId }), + builder.json('/v1/cleanup', { method: 'POST', body, correlationId: req.correlationId }) ]) return send(res, req, 200, result.removed) })) @@ -54,6 +84,11 @@ function createRouter (options = {}) { router.get('/', catchAsyncErrors(async (_, res) => { res.type('html').send((await readFile(indexPath, 'utf8')).replace('{{APP_VERSION}}', version)) })) + router.use((req, res, next) => { + let path + try { path = posix.normalize(decodeURIComponent(req.path)) } catch { return next() } + return path === '/ops' || path.startsWith('/ops/') ? res.sendStatus(404) : next() + }) router.use(express.static('static')) if (!options.disableRateLimit) { @@ -78,13 +113,28 @@ function createRouter (options = {}) { res.once('close', () => { if (!res.writableEnded) abort() }) let rate = {} + let traceStarted = false + let failure + const startTrace = (commit = null, buildMode = null, resource = req.path) => { + if (traceStarted) { + const request = telemetry.active.get(req.correlationId)?.request + if (request && /^[0-9a-f]{40}$/.test(commit || '')) request.commit = commit + if (request && ['legacy', 'dashboards', 'esbuild', 'static'].includes(buildMode)) request.buildMode = buildMode + return + } + traceStarted = true + telemetry.startTrace({ correlationId: req.correlationId, method: req.method, route: PUBLIC_ROUTE, commit, resource, buildMode }) + telemetry.startSpan(req.correlationId, PUBLIC_OPERATION) + } + startTrace() try { const originalBranch = await getBranch(req.path) - const resolved = await downloader.json('/v1/resolve', { + const resolved = await observeDependency(telemetry, 'downloader', now, () => downloader.json('/v1/resolve', { method: 'POST', body: { ref: originalBranch }, - signal: controller.signal - }) + signal: controller.signal, + correlationId: req.correlationId + })) const commit = resolved.commit rate = resolved.rate || {} const url = rewriteRef(req.url, originalBranch, commit) @@ -95,12 +145,16 @@ function createRouter (options = {}) { const parsed = esbuild ? getFileForEsbuild(commit, type, url) : { filename: getFile(commit, type, url), minify: false } const path = dashboards ? req.params.filepath : parsed.filename - if (!path) return send(res, req, 400, 'Could not find the compiled file. Path: ', rate, commit) + if (!path) { + startTrace(commit, dashboards ? 'dashboards' : esbuild ? 'esbuild' : 'legacy') + return send(res, req, 400, 'Could not find the compiled file. Path: ', rate, commit) + } if (!dashboards && !esbuild) { const sourcePath = path.endsWith('.css') ? path : `js/${path}` try { - const response = await downloader.request(`/v1/files/${commit}/${sourcePath}`, { signal: controller.signal }) + const response = await observeDependency(telemetry, 'downloader', now, () => downloader.request(`/v1/files/${commit}/${sourcePath}`, { signal: controller.signal, correlationId: req.correlationId })) + startTrace(commit, 'static') return streamResponse(response, res, req, { commit, rate }) } catch (error) { if (!(error instanceof ServiceError) || error.status !== 404) throw error @@ -108,23 +162,64 @@ function createRouter (options = {}) { } const mode = dashboards ? 'dashboards' : esbuild ? 'esbuild' : 'legacy' - const response = await builder.request('/v1/build', { + startTrace(commit, mode) + const response = await observeDependency(telemetry, 'builder', now, () => builder.request('/v1/build', { method: 'POST', body: { commit, path, mode, options: { minify: parsed.minify, type } }, - signal: controller.signal - }) + signal: controller.signal, + correlationId: req.correlationId + })) return streamResponse(response, res, req, { commit, rate, builtWith: response.headers.get('x-built-with') }) } catch (error) { - if (controller.signal.aborted || res.headersSent) return + startTrace() + if (controller.signal.aborted) return + failure = error + if (res.headersSent) return return sendServiceError(res, req, error, rate) } finally { req.removeListener('aborted', abort) + if (traceStarted) { + const status = controller.signal.aborted ? 'aborted' : (failure || res.statusCode >= 400) ? 'failed' : 'succeeded' + const code = failure ? safeErrorCode(failure) : null + telemetry.completeSpan(req.correlationId, PUBLIC_OPERATION, { status, httpStatus: res.statusCode, code }) + if (failure) telemetry.recordFailure({ correlationId: req.correlationId, operation: PUBLIC_OPERATION, code, httpStatus: res.statusCode }) + telemetry.completeTrace(req.correlationId, { status, httpStatus: res.statusCode, code }) + } } } + router.opsTelemetry = telemetry + router.opsSnapshot = () => telemetry.snapshot({ + capabilities: [ + capability('public_file_delivery'), + capability('console_read'), + capability('console_cache_control', !opsConsoleEnabled, 'NOT_IMPLEMENTED') + ] + }) + return router } +function capability (name, affected = false, reasonCode) { + return { name, status: affected ? 'unavailable' : 'available', reasonCode: affected ? reasonCode : null } +} + +async function observeDependency (telemetry, dependency, now, work) { + const started = now() + try { + const result = await work() + telemetry.recordDependency(dependency, { succeeded: true, latencyMs: now() - started }) + return result + } catch (error) { + telemetry.recordDependency(dependency, { succeeded: false, latencyMs: now() - started, errorCode: safeErrorCode(error) }) + throw error + } +} + +function safeErrorCode (error) { + return typeof error?.code === 'string' && Object.hasOwn(FAILURE_SUMMARIES, error.code) ? error.code : 'INTERNAL_ERROR' +} + function rewriteRef (url, originalBranch, commit) { const prefix = `/${originalBranch}` return url.startsWith(prefix + '/') || url === prefix diff --git a/app/server.js b/app/server.js index ebe50bb..75e20aa 100644 --- a/app/server.js +++ b/app/server.js @@ -1,13 +1,20 @@ 'use strict' const config = require('../config.json') +const { X509Certificate } = require('node:crypto') +const fs = require('node:fs') +const http = require('node:http') +const https = require('node:https') const { bodyJSONParser, clientErrorHandler, logErrors, setConnectionAborted } = require('./middleware.js') +const { parseOpsConfig } = require('./ops/config') +const { createOpsConsoleRouter } = require('./ops/console-router') const router = require('./router.js') const express = require('express') const { join } = require('node:path') function createApp (options = {}) { const app = express() + app.use('/_ops', createOpsConsoleRouter(options.ops)) app.use(setConnectionAborted) app.use(bodyJSONParser) app.use((req, res, next) => { @@ -25,9 +32,31 @@ function createApp (options = {}) { const APP = createApp() function start () { - return APP.listen(process.env.PORT || config.port || 80) + const { publicServer, opsServer, opsConfig } = createServers({ app: APP }) + publicServer.listen(process.env.PORT || config.port || 80) + if (opsServer) opsServer.listen(opsConfig.mtls.port) + return publicServer +} + +function createServers ({ env = process.env, app, readFile = fs.readFileSync } = {}) { + const opsConfig = parseOpsConfig(env) + app = app || createApp({ ops: { env } }) + const publicServer = http.createServer(app) + if (!opsConfig.enabled || opsConfig.protocol === 'http') return { publicServer, opsServer: null, opsConfig } + + const { keyPath, certPath, caPath } = opsConfig.mtls + const ca = readFile(caPath) + if (!new X509Certificate(ca).ca) throw new Error('Invalid OPS_CONSOLE_MTLS_CA_PATH: certificate is not a CA') + const opsServer = https.createServer({ + key: readFile(keyPath), + cert: readFile(certPath), + ca, + requestCert: true, + rejectUnauthorized: true + }, app) + return { publicServer, opsServer, opsConfig } } if (require.main === module || require.main?.filename === join(__dirname, '../server.js')) start() -module.exports = { createApp, default: APP, start } +module.exports = { createApp, createServers, default: APP, start } diff --git a/app/service-client.js b/app/service-client.js index 547ba85..70dcffe 100644 --- a/app/service-client.js +++ b/app/service-client.js @@ -1,5 +1,8 @@ 'use strict' +const { Readable } = require('node:stream') +const { strictJSON } = require('./ops/http') + class ServiceError extends Error { constructor (code, message, status, metadata = {}) { super(message) @@ -16,9 +19,19 @@ function createServiceClient (options = {}) { const timeout = Number(options.timeout || 5000) const fetchImpl = options.fetch || global.fetch - async function request (path, requestOptions = {}) { + async function request (path, requestOptions = {}, retainTimeout = false) { + const { correlationId, maxRequestBytes, maxResponseBytes, timeoutThroughBody, ...fetchOptions } = requestOptions + if (correlationId !== undefined && (typeof correlationId !== 'string' || !/^[\x21-\x7e]{1,64}$/.test(correlationId))) { + throw new TypeError('Invalid correlation ID') + } + const body = fetchOptions.body && typeof fetchOptions.body !== 'string' + ? JSON.stringify(fetchOptions.body) + : fetchOptions.body + if (maxRequestBytes !== undefined && Buffer.byteLength(body || '') > maxRequestBytes) { + throw new ServiceError('REQUEST_TOO_LARGE', 'Service request is too large', 413) + } const controller = new AbortController() - const externalSignal = requestOptions.signal + const externalSignal = fetchOptions.signal const abort = () => controller.abort() externalSignal?.addEventListener('abort', abort, { once: true }) if (externalSignal?.aborted) abort() @@ -26,23 +39,31 @@ function createServiceClient (options = {}) { const timer = setTimeout(() => { timedOut = true controller.abort() - }, requestOptions.timeout || timeout) + }, fetchOptions.timeout || timeout) + let retained = false + const release = () => { + clearTimeout(timer) + externalSignal?.removeEventListener('abort', abort) + } try { const response = await fetchImpl(baseURL + path, { - ...requestOptions, + ...fetchOptions, signal: controller.signal, headers: { - ...(requestOptions.body && typeof requestOptions.body !== 'string' ? { 'Content-Type': 'application/json' } : {}), + ...(fetchOptions.body && typeof fetchOptions.body !== 'string' ? { 'Content-Type': 'application/json' } : {}), + ...fetchOptions.headers, Authorization: `Bearer ${token}`, - ...requestOptions.headers + ...(correlationId ? { 'X-Correlation-ID': correlationId } : {}) }, - body: requestOptions.body && typeof requestOptions.body !== 'string' - ? JSON.stringify(requestOptions.body) - : requestOptions.body + body }) if (!response.ok) { let payload = {} - try { payload = await response.json() } catch (e) {} + try { + payload = maxResponseBytes === undefined ? await response.json() : await boundedJSON(response, maxResponseBytes) + } catch (error) { + if (error instanceof ServiceError) throw error + } const error = payload.error || {} throw new ServiceError(error.code || 'SERVICE_ERROR', error.message || `Service returned ${response.status}`, response.status, { retryAfter: response.headers.get('retry-after'), @@ -51,21 +72,73 @@ function createServiceClient (options = {}) { rateLimitReset: response.headers.get('x-github-ratelimit-reset') }) } + if (retainTimeout && timeoutThroughBody) { + retained = true + return { response, release, timedOut: () => timedOut } + } return response } catch (error) { if (error instanceof ServiceError) throw error throw new ServiceError(timedOut ? 'SERVICE_TIMEOUT' : 'SERVICE_UNAVAILABLE', timedOut ? 'Service request timed out' : error.message, timedOut ? 504 : 502) } finally { - clearTimeout(timer) - externalSignal?.removeEventListener('abort', abort) + if (!retained) release() } } return { - json: async (path, options) => (await request(path, options)).json(), + json: async (path, options = {}) => { + const retained = options.timeoutThroughBody + ? await request(path, options, true) + : { response: await request(path, options), release: () => {}, timedOut: () => false } + try { + return options.maxResponseBytes === undefined + ? await retained.response.json() + : await boundedJSON(retained.response, options.maxResponseBytes) + } catch (error) { + if (error instanceof ServiceError) throw error + throw new ServiceError(retained.timedOut() ? 'SERVICE_TIMEOUT' : 'INVALID_SERVICE_RESPONSE', retained.timedOut() ? 'Service request timed out' : 'Service returned invalid JSON', retained.timedOut() ? 504 : 502) + } finally { + retained.release() + } + }, request, - stream: async (path, options) => (await request(path, options)).body + stream: async (path, options = {}) => { + if (!options.timeoutThroughBody) return (await request(path, options)).body + const { response, release, timedOut } = await request(path, options, true) + const source = typeof response.body.getReader === 'function' ? Readable.fromWeb(response.body) : response.body + const output = Readable.from((async function * () { + try { + yield * source + } catch (error) { + throw new ServiceError(timedOut() ? 'SERVICE_TIMEOUT' : 'SERVICE_UNAVAILABLE', timedOut() ? 'Service request timed out' : error.message, timedOut() ? 504 : 502) + } + })()) + output.once('close', release) + return output + } + } +} + +async function boundedJSON (response, maximum) { + if (!Number.isSafeInteger(maximum) || maximum < 1) throw new TypeError('Invalid response size limit') + const encoding = response.headers?.get?.('content-encoding') + if (encoding && encoding !== 'identity') throw new ServiceError('INVALID_SERVICE_RESPONSE', 'Service returned compressed JSON', 502) + const length = Number(response.headers?.get?.('content-length')) + if (Number.isFinite(length) && length > maximum) throw responseTooLarge() + + const chunks = [] + let size = 0 + for await (const chunk of response.body) { + const bytes = Buffer.from(chunk) + size += bytes.length + if (size > maximum) throw responseTooLarge() + chunks.push(bytes) } + return strictJSON(Buffer.concat(chunks, size), maximum) +} + +function responseTooLarge () { + return new ServiceError('SERVICE_RESPONSE_TOO_LARGE', 'Service response is too large', 502) } module.exports = { ServiceError, createServiceClient } diff --git a/app/utils.js b/app/utils.js index 9d45756..276de11 100644 --- a/app/utils.js +++ b/app/utils.js @@ -14,8 +14,8 @@ function shouldUseWebpack (tsconfig) { function compileWebpack (srcFolder, config = 'highcharts.webpack.mjs') { const configDir = 'tools/webpacks' console.log('Compiling webpack for: ', srcFolder) - const execAsync = (0, node_util_1.promisify)(node_child_process_1.exec) - return execAsync(`npx webpack -c ${(0, node_path_1.join)(configDir, config)} --stats errors-only`, { timeout: 15000, cwd: srcFolder }).then(({ stdout, stderr }) => { + const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile) + return execFileAsync('npx', ['webpack', '-c', (0, node_path_1.join)(configDir, config), '--stats', 'errors-only'], { timeout: 15000, cwd: srcFolder, shell: false }).then(({ stdout, stderr }) => { if (stderr) { console.error(stderr) return diff --git a/app/utils.js.map b/app/utils.js.map index 788e66d..8c779a0 100644 --- a/app/utils.js.map +++ b/app/utils.js.map @@ -1 +1 @@ -{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAAA,8BAA8B;AAC9B,yCAAiC;AACjC,yCAAsC;AACtC,2DAA0C;AAE1C,qCAAkD;AAElD,SAAS,gBAAgB,CAAC,QAAgB;IACxC,IAAI,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,MAAM,GAAE,wBAAwB;IACzE,MAAM,SAAS,GAAG,gBAAgB,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,SAAS,CAAC,CAAA;IAEjD,MAAM,SAAS,GAAG,IAAA,qBAAS,EAAC,yBAAI,CAAC,CAAC;IAElC,OAAO,SAAS,CACd,kBAAkB,IAAA,gBAAI,EAAC,SAAS,EAAE,MAAM,CAAC,sBAAsB,EAC/D,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,CACnC,CAAC,IAAI,CAAC,CAAC,EAAC,MAAM,EAAE,MAAM,EAAC,EAAE,EAAE;QAC1B,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO;QACT,CAAC;QAED,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,IAAA,oBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,IAAA,oBAAU,EAAC,SAAS,CAAC,EAAC,CAAC;YACjD,IAAA,qBAAW,EAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACrB,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;QACb,MAAM,IAAI,KAAK,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC3D,CAAC,CAAC,CAAA;AAEJ,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,gBAAgB;IAChB,cAAc;CACf,CAAA"} \ No newline at end of file +{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAAA,8BAA8B;AAC9B,yCAAiC;AACjC,yCAAsC;AACtC,2DAA8C;AAE9C,qCAAkD;AAElD,SAAS,gBAAgB,CAAC,QAAgB;IACxC,IAAI,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,MAAM,GAAE,wBAAwB;IACzE,MAAM,SAAS,GAAG,gBAAgB,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,SAAS,CAAC,CAAA;IAEjD,MAAM,aAAa,GAAG,IAAA,qBAAS,EAAC,6BAAQ,CAAC,CAAC;IAE1C,OAAO,aAAa,CAClB,KAAK,EACL,CAAC,SAAS,EAAE,IAAI,EAAE,IAAA,gBAAI,EAAC,SAAS,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,EACpE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CACjD,CAAC,IAAI,CAAC,CAAC,EAAC,MAAM,EAAE,MAAM,EAAC,EAAE,EAAE;QAC1B,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO;QACT,CAAC;QAED,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,IAAA,oBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,IAAA,oBAAU,EAAC,SAAS,CAAC,EAAC,CAAC;YACjD,IAAA,qBAAW,EAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACrB,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;QACb,MAAM,IAAI,KAAK,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC3D,CAAC,CAAC,CAAA;AAEJ,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,gBAAgB;IAChB,cAAc;CACf,CAAA"} \ No newline at end of file diff --git a/builder-server.js b/builder-server.js index aa7d93a..5f0a926 100644 --- a/builder-server.js +++ b/builder-server.js @@ -1,9 +1,23 @@ 'use strict' const express = require('express') -const crypto = require('node:crypto') const config = require('./config.json') const { createBuilderService } = require('./app/builder-service') +const { asyncRoute, observedRoute, requireBearer } = require('./app/ops/http') +const { createInternalOpsRouter } = require('./app/ops/internal-router') + +const PUBLIC_ERRORS = { + ARCHIVE_ERROR: 'Source archive extraction failed', + DOWNLOADER_ERROR: 'Downloader request failed', + DOWNLOADER_TIMEOUT: 'Downloader request timed out', + INVALID_BUILD: 'Build did not produce the requested file', + INVALID_COMMIT: 'Commit must be a canonical 40-character SHA', + INVALID_MODE: 'Unknown build mode', + INVALID_OPTIONS: 'Options must be an object', + INVALID_PATH: 'Path must be a normalized relative output path', + QUEUE_FULL: 'Build capacity is unavailable', + SOURCE_INCOMPLETE: 'Source archive is incomplete' +} function createApp (options = {}) { const token = options.token === undefined ? process.env.INTERNAL_SERVICE_TOKEN : options.token @@ -11,17 +25,22 @@ function createApp (options = {}) { const app = express() const service = options.service || createBuilderService(options) - const expectedAuthorization = Buffer.from(`Bearer ${token}`) app.locals.service = service - app.use(express.json()) app.get('/health', (req, res) => res.json({ status: 'ok' })) app.use('/v1', (req, res, next) => { - const authorization = Buffer.from(req.get('authorization') || '') - if (authorization.length !== expectedAuthorization.length || !crypto.timingSafeEqual(authorization, expectedAuthorization)) return res.status(401).json({ error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } }) + try { + requireBearer(req, token) + } catch { + return res.status(401).json({ error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } }) + } next() }) - app.post('/v1/build', asyncRoute(async (req, res) => { - const result = await service.build(req.body) + if (typeof service.snapshot === 'function' && service.cacheManager) { + app.use(createInternalOpsRouter({ token, service: 'builder', snapshot: service.snapshot, cache: service.cacheManager })) + } + app.use(express.json()) + app.post('/v1/build', observedRoute(service, '/v1/build', req => ({ commit: req.body && req.body.commit, buildMode: req.body && req.body.mode }), async (req, res) => { + const result = await service.build(req.body, { correlationId: req.correlationId }) res.set('X-Built-With', result.builtWith) res.set('X-Build-Path', result.path) result.stream.on('error', error => res.destroy(error)) @@ -35,15 +54,12 @@ function createApp (options = {}) { if (error.retryAfter) res.set('Retry-After', error.retryAfter) if (error.rateLimitRemaining) res.set('X-GitHub-RateLimit-Remaining', error.rateLimitRemaining) if (error.rateLimitReset) res.set('X-GitHub-RateLimit-Reset', error.rateLimitReset) - res.status(error.status || 500).json({ error: { code: error.code || 'INTERNAL_ERROR', message: error.message } }) + const code = Object.hasOwn(PUBLIC_ERRORS, error.code) ? error.code : 'INTERNAL_ERROR' + res.status(code === 'INTERNAL_ERROR' ? 500 : error.status || 500).json({ error: { code, message: PUBLIC_ERRORS[code] || 'An internal error occurred' } }) }) return app } -function asyncRoute (handler) { - return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next) -} - function start () { const app = createApp() const interval = Number(process.env.BUILDER_CLEAN_INTERVAL || config.builderCleanInterval || 2 * 60 * 1000) diff --git a/compose.ops-test.yaml b/compose.ops-test.yaml new file mode 100644 index 0000000..ccd6380 --- /dev/null +++ b/compose.ops-test.yaml @@ -0,0 +1,65 @@ +services: + router: + build: + context: . + dockerfile: Dockerfile.router + command: ["node", "test/ops/stack.js"] + environment: + INTERNAL_SERVICE_TOKEN: ops-internal-fixture-token + DOWNLOADER_URL: http://downloader:8080 + BUILDER_URL: http://builder:8080 + OPS_CONSOLE_ENABLED: ${OPS_TEST_CONSOLE_ENABLED:-true} + OPS_CONSOLE_TOKEN_VERIFIER: v1.b1e4_y5sjzm8L9Huzft19Xt3WUAydm4b5LZxUljLqBc + OPS_CONSOLE_ORIGIN: http://localhost:${OPS_TEST_HOST_PORT:-8080} + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: "true" + OPS_TEST_LOGIN_TOKEN: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + ports: + - "127.0.0.1:${OPS_TEST_HOST_PORT:-8080}:8080" + volumes: + - ./test:/app/test:ro + depends_on: + downloader: + condition: service_healthy + builder: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/health"] + interval: 1s + timeout: 1s + retries: 30 + networks: [services] + + downloader: + image: node:24-alpine + command: ["node", "/app/test/ops/fixture-service.js"] + environment: + INTERNAL_SERVICE_TOKEN: ops-internal-fixture-token + OPS_TEST_SERVICE: downloader + PORT: 8080 + volumes: + - ./test:/app/test:ro + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/health"] + interval: 1s + timeout: 1s + retries: 30 + networks: [services] + + builder: + image: node:24-alpine + command: ["node", "/app/test/ops/fixture-service.js"] + environment: + INTERNAL_SERVICE_TOKEN: ops-internal-fixture-token + OPS_TEST_SERVICE: builder + PORT: 8080 + volumes: + - ./test:/app/test:ro + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/health"] + interval: 1s + timeout: 1s + retries: 30 + networks: [services] + +networks: + services: diff --git a/compose.yaml b/compose.yaml index 36e49c6..8bac254 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,8 +10,16 @@ services: BUILDER_URL: http://builder:8080 PUBLIC_DOWNLOADER_TIMEOUT: ${PUBLIC_DOWNLOADER_TIMEOUT:-15000} PUBLIC_BUILDER_TIMEOUT: ${PUBLIC_BUILDER_TIMEOUT:-180000} + OPS_CONSOLE_ENABLED: ${OPS_CONSOLE_ENABLED:-false} + OPS_CONSOLE_TOKEN_VERIFIER: ${OPS_CONSOLE_TOKEN_VERIFIER-} + OPS_CONSOLE_ORIGIN: ${OPS_CONSOLE_ORIGIN-} + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: ${OPS_CONSOLE_ALLOW_HTTP_LOOPBACK:-false} + OPS_CONSOLE_MTLS_PORT: ${OPS_CONSOLE_MTLS_PORT:-8443} + OPS_CONSOLE_MTLS_KEY_PATH: ${OPS_CONSOLE_MTLS_KEY_PATH-} + OPS_CONSOLE_MTLS_CERT_PATH: ${OPS_CONSOLE_MTLS_CERT_PATH-} + OPS_CONSOLE_MTLS_CA_PATH: ${OPS_CONSOLE_MTLS_CA_PATH-} ports: - - "${ROUTER_PORT:-8080}:8080" + - "${ROUTER_BIND_ADDRESS:-127.0.0.1}:${ROUTER_PORT:-8080}:8080" depends_on: downloader: condition: service_healthy diff --git a/docs/operations-console.md b/docs/operations-console.md index c27f99a..2fa08fe 100644 --- a/docs/operations-console.md +++ b/docs/operations-console.md @@ -16,6 +16,9 @@ mandatory behaviour. "Should" and "should not" state strong recommendations. 1. [Purpose, operator journeys, and non-goals](#1-purpose-operator-journeys-and-non-goals) 2. [Topology and trust boundaries](#2-topology-and-trust-boundaries) 3. [Enablement and configuration](#3-enablement-and-configuration) + - [Proxied deployment — Contour/Kubernetes contract](#proxied-deployment--contourkubernetes-contract) + - [Proxied deployment — enablement order](#proxied-deployment--enablement-order) + - [Proxied deployment — certificate lifecycle](#proxied-deployment--certificate-lifecycle) 4. [Route namespace and HTTP routes](#4-route-namespace-and-http-routes) 5. [Shared-token login and session contract](#5-shared-token-login-and-session-contract) 6. [Security headers, CSP, and browser restrictions](#6-security-headers-csp-and-browser-restrictions) @@ -90,7 +93,7 @@ The existing three-process topology is unchanged: | Service | Role | Public port | |---|---|---| -| **Router** | Accepts public requests; hosts the same-origin operations console UI and browser-facing API | 8080 | +| **Router** | Accepts public requests; hosts the same-origin operations console UI and browser-facing API | 8080 (public HTTP); 8443 (mTLS, HTTPS console mode only) | | **Downloader** | Internal only; resolves refs, fetches and caches source trees; exposes authenticated internal `/v1` ops endpoints | None | | **Builder** | Internal only; compiles sources; exposes authenticated internal `/v1` ops endpoints | None | @@ -119,7 +122,14 @@ The existing three-process topology is unchanged: | `OPS_CONSOLE_ENABLED` | Always evaluated | Must be exactly the string `true` to enable the console. Any other value or absence disables it. | | `OPS_CONSOLE_TOKEN_VERIFIER` | Console enabled | Domain-separated SHA-256 verifier derived from the shared admin token. Stored outside source control. Validated at startup. | | `OPS_CONSOLE_ORIGIN` | Console enabled | The single exact external Origin that the console accepts for all state-changing requests and for login. | -| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | When exactly `true`, allows HTTP only if the configured Origin is loopback **and** proxy mode is off. All other HTTP/proxy combinations fail startup. | +| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | When exactly `true`, allows HTTP only if the configured Origin is a loopback address **and** no mTLS settings are present. All other HTTP/mTLS combinations fail startup. | +| `OPS_CONSOLE_MTLS_PORT` | HTTPS console | Port for the separate mTLS listener. Defaults to `8443`. Must not be set when `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true`. | +| `OPS_CONSOLE_MTLS_KEY_PATH` | HTTPS console | Absolute in-container path to the server TLS private key PEM file. | +| `OPS_CONSOLE_MTLS_CERT_PATH` | HTTPS console | Absolute in-container path to the server TLS certificate PEM file. | +| `OPS_CONSOLE_MTLS_CA_PATH` | HTTPS console | Absolute in-container path to the Envoy client CA certificate PEM file. The router uses this CA to verify the client certificate that Envoy presents during the mTLS handshake. The file must be a CA certificate; startup fails otherwise. This is the Envoy-client-issuing CA and is distinct from the server CA used in Contour's `validation.caSecret`. | + +`OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup with: +`OPS_CONSOLE_TRUSTED_PROXY is not supported; use operations console mTLS`. ### Disabled behaviour (default) @@ -134,27 +144,179 @@ When `OPS_CONSOLE_ENABLED` is absent or not exactly `true`: ### Enabled behaviour — fail-closed startup When `OPS_CONSOLE_ENABLED` is exactly `true`, startup **must** fail if any of -the following is missing, malformed, or ambiguous: - -- `OPS_CONSOLE_TOKEN_VERIFIER` — must be a valid verifier value. -- `OPS_CONSOLE_ORIGIN` — must be a valid, unambiguous origin. -- Proxy configuration — must be unambiguous; ambiguity fails startup. +the following conditions apply: + +- `OPS_CONSOLE_TOKEN_VERIFIER` is missing or malformed. +- `OPS_CONSOLE_ORIGIN` is missing or invalid. +- `OPS_CONSOLE_TRUSTED_PROXY` is set to any non-blank value. +- HTTPS mode is chosen but any of `OPS_CONSOLE_MTLS_KEY_PATH`, + `OPS_CONSOLE_MTLS_CERT_PATH`, or `OPS_CONSOLE_MTLS_CA_PATH` is blank, relative + (not an absolute path), missing, unreadable, or malformed TLS material. +- The file at `OPS_CONSOLE_MTLS_CA_PATH` is not a CA certificate. +- `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` is set together with any mTLS setting. +- `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` is set but the Origin is not a loopback address. +- `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` is `true` and the Origin scheme is `https:`. `NODE_ENV` alone must never enable the console. -### HTTPS, proxy, and loopback rules +### Two-listener model -- Deployed development requires HTTPS. -- `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` enables HTTP only when the configured - Origin is loopback and proxy mode is off. Every other HTTP/proxy combination - fails startup. -- Forwarded source address and protocol are trusted only from an explicitly - allowlisted immediate proxy. Otherwise the socket peer is authoritative. -- Ambiguous proxy configuration fails startup. +The router always starts a **public HTTP listener** on port 8080 (configurable via +`PORT`). This listener handles all public file-delivery traffic and the `/health` +endpoint. It is unchanged by console enablement. ---- +When the console is enabled with an `https:` Origin, the router additionally +starts a **separate mTLS listener** on `OPS_CONSOLE_MTLS_PORT` (default `8443`). +The Node.js server is configured with `requestCert: true` and +`rejectUnauthorized: true`; connections without a valid client certificate signed +by the configured CA are rejected at the TLS layer before any application code +runs. See [Node.js `https.createServer`](https://nodejs.org/docs/latest-v24.x/api/https.html#httpscreateserveroptions-requestlistener) +and [`tls.createServer`](https://nodejs.org/docs/latest-v24.x/api/tls.html#tlscreateserveroptions-secureconnectionlistener). + +`/_ops/*` requests that arrive on the **public HTTP listener** are handled by the +disabled router (console disabled) or always fail closed (console enabled in +HTTPS mode). The console is reachable only through the mTLS listener. + +**Local HTTP loopback mode** remains an explicit alternative for development. +It requires `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` and a loopback Origin, and all +`OPS_CONSOLE_MTLS_*` variables must be blank. In this mode the single public HTTP +listener handles `/_ops/*` traffic directly, with no mTLS layer. + +### Client IP and rate limiting + +Node.js does not trust any `Forwarded` or `X-Forwarded-*` header and does not +derive a client IP from proxy headers. Login rate limiting is process-wide only: +a single global counter of 30 attempts per 15 minutes. There is no per-source +or per-IP bucket at the application layer. + +In HTTPS/mTLS mode the audit `source` field is `null`; no client or proxy IP is +attributed. In explicit local HTTP loopback mode the directly observed loopback +peer address is recorded. Operators requiring per-client-IP rate limiting must +enforce it at the proxy layer (e.g. Contour / Envoy) before requests reach the +mTLS port. + +### Proxied deployment — Contour/Kubernetes contract + +> **Prerequisite, not implemented here.** This section describes the platform +> configuration that a proxied deployment requires before the mTLS listener +> can serve `/_ops` traffic in a Kubernetes environment. None of these steps are +> part of this application. -## 4. Route namespace and HTTP routes +The application exposes a second port (default `8443`) for the mTLS-protected +console. The following Kubernetes and Contour configuration is required to route +`/_ops` traffic to it. + +**Kubernetes Service** + +Add a second port to the Service targeting the router Pod: + +```yaml +- name: ops-console + port: 8443 + targetPort: 8443 + protocol: TCP +``` + +**HTTPProxy route** + +Route `/_ops` to the ops-console port using TLS upstream with backend server +verification. The `subjectName` / `subjectNames` field name depends on the +deployed Contour version; check the Contour +[upstream TLS documentation](https://projectcontour.io/docs/1.33/config/upstream-tls/) +and [configuration reference](https://projectcontour.io/docs/1.33/configuration/) +for the correct field for the version in use: + +```yaml +routes: + - conditions: + - prefix: /_ops + services: + - name: router + port: 8443 + protocol: tls + validation: + caSecret: ops-console-server-ca + subjectName: +``` + +`validation.caSecret` is the CA that **Contour uses to verify the router's +server certificate** — it belongs to the server-certificate trust chain, not to +any client-certificate authority. `subjectName` / `subjectNames` must match a +SAN on the router's own server certificate. Use a secret name such as +`ops-console-server-ca` to make the trust direction unambiguous. + +**Envoy client identity** + +These two trust directions are independent and must not be conflated: + +- **Contour → router (server verification).** Contour authenticates the router + by checking its TLS server certificate against `validation.caSecret` + (`ops-console-server-ca` above). +- **Router → Envoy (client verification).** The router authenticates Envoy's + client certificate against the CA mounted at `OPS_CONSOLE_MTLS_CA_PATH`. + This is a *separate* CA that issues only Envoy upstream identities; it has no + relationship to the server CA and must not be reused for one. + +Configure Contour's global `tls.envoy-client-certificate` with a cluster-wide +Envoy client identity. The certificate Envoy presents to the router must be +signed by the dedicated Envoy-client-issuing CA referenced by +`OPS_CONSOLE_MTLS_CA_PATH`. Use a dedicated CA for this role; do not reuse the +server CA or any shared cluster CA merely for convenience. + +**Secret mounts** + +Mount the server key, server certificate, and Envoy client CA as read-only files +inside the container. Use only absolute in-container paths for +`OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and +`OPS_CONSOLE_MTLS_CA_PATH`. Never reference test fixture paths at runtime. + +To exercise mTLS in local Docker Compose, add an explicit read-only volume +override in a `compose.override.yaml` that mounts the secret files at the +configured absolute paths. The default local Compose topology uses HTTP loopback +and does not require this override. + +### Proxied deployment — enablement order + +Follow this order to avoid exposing an unconfigured mTLS port or routing +`/_ops` traffic before the application is ready. + +1. **Provision secrets and platform client identity.** Create the TLS secret + files, mount them read-only into the container, and configure the Contour + global `tls.envoy-client-certificate`. +2. **Deploy the application with the console disabled.** Set + `OPS_CONSOLE_ENABLED=false`. Verify public traffic is unaffected. +3. **Enable the mTLS listener.** Set `OPS_CONSOLE_ENABLED=true` and all four + `OPS_CONSOLE_MTLS_*` variables. Restart the router and confirm the startup + log shows `ops-console enabled`. The mTLS port is now open but `/_ops` is + not yet reachable from outside. +4. **Add and enable the Contour `/_ops` TLS route.** Apply the HTTPProxy change + that routes `/_ops` to port `8443`. +5. **Verify.** Run `npm run test:integration:ops` and confirm snapshot and cache + operations work end-to-end through the proxy. +6. **Expose and use the console.** Public traffic on port 8080 must remain + unaffected at every step. + +To roll back, remove or disable the Contour `/_ops` route **before** disabling +the mTLS listener or rolling back the image (route-first rollback). This ensures +no `/_ops` requests arrive at a listener that is no longer running. + +### Proxied deployment — certificate lifecycle + +**Restart-based reload.** The TLS files are read once at startup by +`fs.readFileSync`. To apply new certificates, rotate the secret files and +restart the router container. There is no live reload. + +**Overlap rotation.** When rotating certificates, provision the new secret files +and restart the router before the old certificate expires. Keep sufficient +overlap to complete the restart and re-verify before the old certificate's +not-after time. + +**Urgent revocation.** There is no OCSP or CRL check in the application. To +contain a compromised key: restart the router immediately with the new +certificate (or with the console disabled), then drain or terminate sessions. +In-memory sessions are invalidated by restart. + +--- The `/_ops/` namespace is reserved from all public branch/file routes. Obscurity is not a security control; the namespace reservation is operational. @@ -370,13 +532,13 @@ All rate-limit state is in-memory and resets on router restart. Rate-limited responses return HTTP `429` with error code `RATE_LIMITED` and a `Retry-After` header. -Source address is determined by the socket peer, unless an immediate proxy is -explicitly allowlisted — in which case the forwarded address from that proxy is -used. Ambiguous proxy configuration fails closed. +Node.js does not trust `Forwarded` or `X-Forwarded-*` headers and does not +derive a client IP from proxy headers. Login rate limiting is process-wide only; +there is no per-source or per-IP application bucket. Per-client-IP limiting in a +proxied deployment must be enforced at the proxy layer. | Area | Limit | |---|---| -| Login — per source IP | 5 attempts / 15 minutes | | Login — global | 30 attempts / 15 minutes | | Snapshot — per session | 12 requests / minute | | Cache operations — per session | 5 requests / minute | @@ -950,7 +1112,7 @@ Required audit event fields: | `action` | Normalized operation name | | `correlationId` | Router request ID | | `sessionAuditId` | Separate random per-session audit ID, independent of cookie, non-authorizing | -| `source` | Trusted source context (IP from allowlisted proxy or socket peer) | +| `source` | `null` in HTTPS/mTLS mode (no client or proxy IP is attributed); directly observed loopback address in explicit local HTTP loopback mode | | `userAgent` | Bounded, truncated | | `operation` | Validated operation | | `targets` | Validated target list | @@ -1209,7 +1371,7 @@ cover at minimum: **Configuration and startup:** - Disabled routes 404; no internal calls; sanitized status event. -- Enabled startup failures for every illegal loopback/proxy combination. +- Enabled startup failures for every illegal loopback/mTLS combination. - `NODE_ENV` alone must not enable the console. **Login and credentials:** @@ -1240,7 +1402,7 @@ cover at minimum: browser storage, external requests, or CORS headers. **Rate limits:** -- Login per-source and global limits enforced. +- Login global limit enforced (30 attempts / 15 minutes); no per-source bucket exists. - Snapshot and cache-operation session/global limits enforced. - Rate-limited responses include `429`, `RATE_LIMITED`, and `Retry-After`. @@ -1294,9 +1456,9 @@ Integration scenarios must cover: - No automatic retry. - Audit event format and 64 KiB response bound. -In deployed-development verification (HTTPS): additionally check HTTPS ingress, -`__Host-hc-ops` cookie, exact configured Origin, and trusted-proxy forwarded-address -handling. +In deployed-development verification (HTTPS mTLS): additionally check HTTPS ingress, +`__Host-hc-ops` cookie, exact configured Origin, and that `/_ops` requests on the +public HTTP listener fail closed. ### Gate 4 — Security and failure injection @@ -1365,7 +1527,7 @@ Automation supports but does not replace manual accessibility acceptance (§16). **Step 5 — Router, console enabled:** Restart the single router with `OPS_CONSOLE_ENABLED=true`, valid `OPS_CONSOLE_TOKEN_VERIFIER`, -`OPS_CONSOLE_ORIGIN`, and proxy configuration. +`OPS_CONSOLE_ORIGIN`, and (for HTTPS mode) the four `OPS_CONSOLE_MTLS_*` variables. Required before declaring success: - Sanitized enabled status event appears in stdout. diff --git a/downloader-server.js b/downloader-server.js index 3bff0dd..040bd12 100644 --- a/downloader-server.js +++ b/downloader-server.js @@ -1,9 +1,22 @@ 'use strict' const express = require('express') -const crypto = require('node:crypto') const config = require('./config.json') const { createDownloaderService } = require('./app/downloader-service') +const { asyncRoute, observedRoute, requireBearer } = require('./app/ops/http') +const { createInternalOpsRouter } = require('./app/ops/internal-router') + +const PUBLIC_ERRORS = { + FILE_NOT_FOUND: 'File was not found', + INVALID_COMMIT: 'Commit must be a 40-character SHA', + INVALID_PATH: 'Unsafe file path', + QUEUE_FULL: 'Download capacity is unavailable', + RATE_LIMITED: 'GitHub rate limit is exhausted', + REF_NOT_FOUND: 'Ref was not found', + SOURCE_INCOMPLETE: 'Source tree is incomplete', + UPSTREAM_ERROR: 'GitHub request failed', + UPSTREAM_TIMEOUT: 'GitHub request timed out' +} function createApp (options = {}) { const token = options.token === undefined ? process.env.INTERNAL_SERVICE_TOKEN : options.token @@ -11,34 +24,36 @@ function createApp (options = {}) { const app = express() const service = options.service || createDownloaderService(options) - const expectedAuthorization = Buffer.from(`Bearer ${token}`) - app.use(express.json()) app.locals.service = service app.get('/health', (req, res) => res.json({ status: 'ok' })) app.use('/v1', (req, res, next) => { - const authorization = Buffer.from(req.get('authorization') || '') - if (authorization.length !== expectedAuthorization.length || !crypto.timingSafeEqual(authorization, expectedAuthorization)) { + try { + requireBearer(req, token) + } catch { return res.status(401).json({ error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } }) } next() }) - app.post('/v1/resolve', asyncRoute(async (req, res) => { - res.json(await service.resolveRef(req.body && req.body.ref)) + if (typeof service.snapshot === 'function' && service.cacheManager) { + app.use(createInternalOpsRouter({ token, service: 'downloader', snapshot: service.snapshot, cache: service.cacheManager })) + } + app.use(express.json()) + app.post('/v1/resolve', observedRoute(service, '/v1/resolve', req => ({ resource: req.body && req.body.ref }), async (req, res) => { + res.json(await service.resolveRef(req.body && req.body.ref, { correlationId: req.correlationId })) })) - app.get('/v1/files/:commit/*', asyncRoute(async (req, res) => { - const stream = await service.openFile(req.params.commit, req.params[0]) + app.get('/v1/files/:commit/*', observedRoute(service, '/v1/files/:commit/*', req => ({ commit: req.params.commit, resource: req.params[0] }), async (req, res) => { + const stream = await service.openFile(req.params.commit, req.params[0], { correlationId: req.correlationId }) res.set('Cache-Control', 'public, max-age=31536000, immutable') stream.on('error', error => res.destroy(error)) stream.pipe(res) })) - app.get('/v1/sources/:commit.tar.gz', asyncRoute(async (req, res, next) => { - const tar = await service.archive(req.params.commit) - let stderr = '' - tar.stderr.on('data', chunk => { stderr += chunk }) + app.get('/v1/sources/:commit.tar.gz', observedRoute(service, '/v1/sources/:commit.tar.gz', req => ({ commit: req.params.commit }), async (req, res, next) => { + const tar = await service.archive(req.params.commit, { correlationId: req.correlationId }) + tar.stderr.resume() tar.on('error', next) tar.on('close', code => { - if (code && !res.headersSent) next(new Error(stderr || `tar exited with ${code}`)) + if (code && !res.headersSent) next(new Error('Archive generation failed')) }) res.type('application/gzip') res.set('Cache-Control', 'public, max-age=31536000, immutable') @@ -52,21 +67,18 @@ function createApp (options = {}) { if (error.rateLimitLimit != null) res.set('X-GitHub-RateLimit-Limit', String(error.rateLimitLimit)) if (error.rateLimitRemaining != null) res.set('X-GitHub-RateLimit-Remaining', String(error.rateLimitRemaining)) if (error.rateLimitReset != null) res.set('X-GitHub-RateLimit-Reset', String(error.rateLimitReset)) - res.status(error.status || 500).json({ - error: { code: error.code || 'INTERNAL_ERROR', message: error.message } + const code = Object.hasOwn(PUBLIC_ERRORS, error.code) ? error.code : 'INTERNAL_ERROR' + res.status(code === 'INTERNAL_ERROR' ? 500 : error.status || 500).json({ + error: { code, message: PUBLIC_ERRORS[code] || 'An internal error occurred' } }) }) return app } -function asyncRoute (handler) { - return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next) -} - function start () { const app = createApp() const interval = Number(process.env.DOWNLOADER_CLEAN_INTERVAL || config.downloaderCleanInterval || 2 * 60 * 1000) - const timer = setInterval(() => app.locals.service?.cleanup().catch(console.error), interval) + const timer = setInterval(() => app.locals.service?.cleanup().catch(() => console.error('Downloader cleanup failed')), interval) timer.unref() return app.listen(Number(process.env.DOWNLOADER_PORT || config.downloaderPort || 8081)) } diff --git a/package-lock.json b/package-lock.json index 69607c5..f74af83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,9 @@ "webpack-cli": "^5.1.4" }, "devDependencies": { + "@axe-core/playwright": "4.12.1", + "@orangeopensource/hurl": "8.0.2", + "@playwright/test": "1.61.1", "@types/node": "^20.4.6", "archiver": "^1.3.0", "chai": "^4.1.1", @@ -44,6 +47,19 @@ "node": ">=24.0.0" } }, + "node_modules/@axe-core/playwright": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.12.1.tgz", + "integrity": "sha512-rMd7xriptqKpP+w5265i4Hdkv2X5kbu6uiBi/B2I7uf3hieRBM3qDCfaKPtxfiYb2mKXfF+yLODJwIx+Jv1GDw==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.12.1" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -3182,6 +3198,19 @@ "node": ">=18" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -3515,6 +3544,72 @@ "@octokit/openapi-types": "^12.11.0" } }, + "node_modules/@orangeopensource/hurl": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@orangeopensource/hurl/-/hurl-8.0.2.tgz", + "integrity": "sha512-Uu3FK4JDh1zMUdkLOcFjWMmmqtTbuafwVWtQOj1YRw4lCKdeB3b4av4DybzJqdm3Sf3THp3euUHH1m4N3N2rGw==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "extract-zip": "2.0.1", + "tar": "7.5.16" + }, + "bin": { + "hurl": "hurl.js", + "hurlfmt": "hurlfmt.js" + } + }, + "node_modules/@orangeopensource/hurl/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@orangeopensource/hurl/node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@orangeopensource/hurl/node_modules/tar": { + "version": "7.5.16", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz", + "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@orangeopensource/hurl/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/@parcel/watcher": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", @@ -3840,6 +3935,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@rollup/plugin-node-resolve": { "version": "15.3.1", "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.1.tgz", @@ -4386,6 +4497,17 @@ "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.2.tgz", @@ -5242,6 +5364,16 @@ "license": "MIT", "optional": true }, + "node_modules/axe-core": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.12.1.tgz", + "integrity": "sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/babel-plugin-add-module-exports": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/babel-plugin-add-module-exports/-/babel-plugin-add-module-exports-0.2.1.tgz", @@ -7944,6 +8076,43 @@ "license": "MIT", "optional": true }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -8006,6 +8175,16 @@ "reusify": "^1.0.4" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -12951,6 +13130,13 @@ "node": "*" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, "node_modules/performance-now": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz", @@ -13157,6 +13343,53 @@ "node": ">=4" } }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -13248,6 +13481,17 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "license": "MIT" }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -16857,6 +17101,17 @@ "node": ">=8" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index ac792f3..df4abae 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,9 @@ "webpack-cli": "^5.1.4" }, "devDependencies": { + "@axe-core/playwright": "4.12.1", + "@orangeopensource/hurl": "8.0.2", + "@playwright/test": "1.61.1", "@types/node": "^20.4.6", "archiver": "^1.3.0", "chai": "^4.1.1", @@ -50,6 +53,8 @@ "test": "npm run test-lint && npm run unit-tests", "test-lint": "standard \"app/*.js\" \"scripts/*.js\" \"test/*.js\"", "unit-tests": "mocha test/test.js", + "test:integration:ops": "node scripts/test-ops-integration.js", + "test:browser:ops": "playwright test --config=playwright.ops.config.js", "artillery-test": "npx artillery run -o /tmp/artillery-report ./test/artillery-load-test.yml", "prepare": "husky" }, diff --git a/playwright.ops.config.js b/playwright.ops.config.js new file mode 100644 index 0000000..37c2b91 --- /dev/null +++ b/playwright.ops.config.js @@ -0,0 +1,24 @@ +'use strict' + +const { defineConfig, devices } = require('@playwright/test') + +module.exports = defineConfig({ + testDir: './test/browser', + testMatch: 'ops.spec.js', + fullyParallel: false, + workers: 1, + timeout: 60 * 1000, + expect: { timeout: 10 * 1000 }, + reporter: process.env.CI ? 'dot' : 'list', + outputDir: 'tmp/playwright-ops-results', + use: { + baseURL: process.env.OPS_TEST_BASE_URL || 'http://localhost:8080', + trace: 'off', + screenshot: 'off', + video: 'off' + }, + projects: [ + { name: 'chromium', use: { ...devices['Desktop Chrome'] } }, + { name: 'firefox', use: { ...devices['Desktop Firefox'] } } + ] +}) diff --git a/scripts/test-ops-integration.js b/scripts/test-ops-integration.js new file mode 100644 index 0000000..62e9f7c --- /dev/null +++ b/scripts/test-ops-integration.js @@ -0,0 +1,194 @@ +'use strict' + +const { spawnSync } = require('node:child_process') +const { randomBytes } = require('node:crypto') +const { basename, resolve } = require('node:path') + +const root = resolve(__dirname, '..') +const composeFile = 'compose.ops-test.yaml' +const project = `ops-console-integration-${process.pid}` +const deploymentComposeFile = 'compose.yaml' +const deploymentProject = `ops-console-deployment-${process.pid}` +const deploymentToken = randomBytes(32).toString('base64url') +const hostPort = Number(process.env.OPS_TEST_HOST_PORT || 8080) +const baseURL = `http://localhost:${hostPort}` +const loginToken = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +const internalToken = 'ops-internal-fixture-token' +const verifier = 'v1.b1e4_y5sjzm8L9Huzft19Xt3WUAydm4b5LZxUljLqBc' +const composeArgs = ['compose', '-p', project, '-f', composeFile] +const deploymentComposeArgs = ['compose', '-p', deploymentProject, '-f', deploymentComposeFile] +const deploymentEnvironment = { ...process.env, INTERNAL_SERVICE_TOKEN: deploymentToken } +let cleaning = false + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.once(signal, () => { + cleanup() + process.exit(128 + (signal === 'SIGINT' ? 2 : 15)) + }) +} + +main().catch(error => { + process.stderr.write(`Operations integration failed: ${error.message}\n`) + process.exitCode = 1 +}).finally(cleanup) + +async function main () { + inspectDeploymentTopology() + try { + deploymentCompose(['up', '--build', '--wait', '--remove-orphans', 'downloader', 'builder']) + checkPrivateAuthentication(deploymentCompose) + } finally { + cleanupDeployment() + } + + inspectTopology() + compose(['up', '--build', '--wait', '--remove-orphans']) + checkPrivateAuthentication() + + hurl('test/hurl/ops-snapshot.hurl') + hurl('test/hurl/ops-cache.hurl') + await checkConcurrentCommands() + checkLogs() + + compose(['down', '--volumes', '--remove-orphans']) + compose(['up', '--build', '--wait', '--remove-orphans']) + hurl('test/hurl/ops-auth.hurl') + checkLogs() + + compose(['down', '--volumes', '--remove-orphans']) + compose(['up', '--build', '--wait', '--remove-orphans'], { OPS_TEST_CONSOLE_ENABLED: 'false' }) + hurl('test/hurl/ops-security.hurl') + hurl('test/hurl/smoke.hurl') + hurl('test/hurl/service-split.hurl') + checkLogs() +} + +function inspectDeploymentTopology () { + const output = deploymentCompose(['config', '--format', 'json'], true) + const config = JSON.parse(output) + const published = Object.entries(config.services).filter(([, service]) => service.ports?.length) + if (published.length !== 1 || published[0][0] !== 'router' || published[0][1].ports.length !== 1 || Number(published[0][1].ports[0].target) !== 8080) { + throw new Error('deployable topology must publish only router:8080') + } + for (const name of ['downloader', 'builder']) { + if (config.services[name].ports?.length) throw new Error(`deployable ${name} publishes a port`) + } +} + +function inspectTopology () { + const output = compose(['config', '--format', 'json'], {}, true) + const config = JSON.parse(output) + const published = Object.entries(config.services).filter(([, service]) => service.ports?.length) + if (published.length !== 1 || published[0][0] !== 'router' || published[0][1].ports.length !== 1 || Number(published[0][1].ports[0].target) !== 8080) { + throw new Error('only router:8080 may be published') + } + for (const name of ['downloader', 'builder']) { + if (config.services[name].ports?.length) throw new Error(`${name} publishes a port`) + } +} + +function checkPrivateAuthentication (runCompose = compose) { + const source = ` + const paths = [ + ['GET', '/v1/ops/snapshot'], + ['POST', '/v1/ops/cache-operations'] + ]; + Promise.all(paths.flatMap(([method, path]) => [undefined, 'Bearer wrong'].map(async authorization => { + const response = await fetch('http://127.0.0.1:8080' + path, { + method, + headers: authorization ? { Authorization: authorization } : {} + }); + if (response.status !== 401) throw new Error(method + ' ' + path + ' returned ' + response.status); + }))).catch(error => { console.error(error.message); process.exit(1) }); + ` + for (const service of ['downloader', 'builder']) runCompose(['exec', '-T', service, 'node', '-e', source]) +} + +async function checkConcurrentCommands () { + const login = await fetch(`${baseURL}/_ops/api/v1/session`, { + method: 'POST', + headers: { Origin: baseURL, 'Content-Type': 'application/json' }, + body: JSON.stringify({ token: loginToken }) + }) + if (login.status !== 201) throw new Error('concurrency login failed') + const cookie = login.headers.get('set-cookie').split(';', 1)[0] + const { csrfToken } = await login.json() + await control('/reset-audits', {}) + await Promise.all(['downloader', 'builder'].map(service => control('/fixture', { service, cache: 'healthy', resetCounts: true }))) + const responses = await Promise.all(['downloader', 'builder'].map(service => fetch(`${baseURL}/_ops/api/v1/cache-operations`, { + method: 'POST', + headers: { Cookie: cookie, Origin: baseURL, 'X-Ops-CSRF': csrfToken, 'Content-Type': 'application/json' }, + body: JSON.stringify({ operation: 'cache.purge_expired', targets: [service] }) + }))) + if (responses.some(response => response.status !== 200)) throw new Error('concurrent cache command failed') + const status = await (await fetch(`${baseURL}/__ops-test/status`)).json() + if (status.audits !== 2 || !status.auditSafe || status.downloader.cache !== 1 || status.builder.cache !== 1) { + throw new Error('concurrent commands were retried or incorrectly audited') + } + checkLogs([cookie, csrfToken]) +} + +async function control (path, body) { + const response = await fetch(`${baseURL}/__ops-test${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + if (!response.ok) throw new Error(`test control ${path} failed`) +} + +function hurl (file) { + const legacyURL = ['test/hurl/smoke.hurl', 'test/hurl/service-split.hurl'].includes(file) + run(resolve(root, 'node_modules/.bin/hurl'), [ + '--test', '--jobs', '1', '--retry', '0', '--no-output', + ...(file === 'test/hurl/ops-auth.hurl' ? ['--no-cookie-store'] : []), + ...(legacyURL && hostPort !== 8080 ? ['--connect-to', `localhost:8080:127.0.0.1:${hostPort}`] : []), + '--secret', `token=${loginToken}`, + '--variable', `base_url=${baseURL}`, + file + ]) +} + +function checkLogs (additionalSecrets = []) { + const logs = compose(['logs', '--no-color', 'router', 'downloader', 'builder'], {}, true) + const forbidden = [loginToken, internalToken, verifier, ...additionalSecrets] + if (forbidden.some(value => value && logs.includes(value)) || /"(?:authorization|cookie|csrfToken|token)"/i.test(logs) || /ghhc-console-dev=[A-Za-z0-9_-]{43}/.test(logs)) { + throw new Error('sensitive value found in service logs') + } + if (/\b(?:Error:|at \/app\/|node:internal)\b/.test(logs)) throw new Error('stack trace found in service logs') +} + +function compose (args, environment = {}, capture = false) { + return run('docker', [...composeArgs, ...args], { ...process.env, ...environment }, capture) +} + +function deploymentCompose (args, capture = false) { + return run('docker', [...deploymentComposeArgs, ...args], deploymentEnvironment, capture) +} + +function run (command, args, environment = process.env, capture = false) { + const result = spawnSync(command, args, { + cwd: root, + env: environment, + encoding: 'utf8', + stdio: capture ? ['ignore', 'pipe', 'pipe'] : 'inherit' + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`${basename(command)} exited with ${result.status}`) + return capture ? result.stdout : '' +} + +function cleanup () { + if (cleaning) return + cleaning = true + cleanupDeployment() + spawnSync('docker', [...composeArgs, 'down', '--volumes', '--remove-orphans'], { cwd: root, stdio: 'ignore' }) +} + +function cleanupDeployment () { + spawnSync('docker', [...deploymentComposeArgs, 'down', '--volumes', '--remove-orphans'], { + cwd: root, + env: deploymentEnvironment, + stdio: 'ignore' + }) +} diff --git a/src/JobQueue.ts b/src/JobQueue.ts index 5212576..61c182e 100644 --- a/src/JobQueue.ts +++ b/src/JobQueue.ts @@ -1,16 +1,26 @@ -export type JobArgs = { +'use strict'; + +type JobArgs = { func: (...args: any[]) => Promise, args: any[]; } -export type JobType = JobArgs & { - setDone: (isDone: true) => void; - done: Promise +type JobType = JobArgs & { + setDone: (result: any) => void; + setFailed: (error: unknown) => void; + done: Promise; + queuedAt: number; +} +type Queues = 'download' | 'compile'; +type QueueType = Map; +type QueueMetrics = { + active: number; + queued: number; + limit: number; + available: number; + oldestQueuedAgeMs: number | null; } -export type Queues = 'download' | 'compile'; -export type QueueType = Map; /* eslint-disable camelcase */ -import crypto from 'node:crypto'; import { env } from 'node:process'; // Max queue size per queue @@ -29,24 +39,21 @@ class JobQueue { compile: false }; - public id: string; - async doJob(queue: QueueType, jobID: string) { const job = queue.get(jobID); if (job) { try { - await Promise.resolve().then(() => job.func(...job.args)); + const result = await Promise.resolve().then(() => job.func(...job.args)); + job.setDone(result); } catch (error) { - console.log(error); + job.setFailed(error); + throw error; } finally { queue.delete(jobID); - job.setDone(true); } - - await job.done; } } @@ -73,17 +80,22 @@ class JobQueue { } private makeJob(job: JobArgs): JobType { - const myJob: JobType = { - ...job, - setDone() { }, - done: Promise.resolve(true) - }; + const { + promise: done, + resolve: setDone, + reject: setFailed + } = Promise.withResolvers(); - myJob.done = new Promise((resolve) => { - myJob.setDone = resolve; - }) + // Prevent an ignored caller promise from becoming an unhandled rejection. + done.catch(() => {}); - return myJob; + return { + ...job, + setDone, + setFailed, + done, + queuedAt: Date.now() + }; } public addJob(type: Queues, jobID: string, job: JobArgs) { @@ -97,7 +109,7 @@ class JobQueue { } if (queue.has(jobID)) { - return queue.get(jobID)?.done; + return queue.get(jobID)!.done; } const transformedJob = this.makeJob(job) @@ -110,7 +122,8 @@ class JobQueue { if (!JobQueue.churns[type]) { JobQueue.churns[type] = true; - return this.churn(queue) + this.churn(queue) + .catch(console.error) .finally(() => { JobQueue.churns[type] = false; }) @@ -129,12 +142,24 @@ class JobQueue { return Array.from(queue.values()); } + public getMetrics(type: Queues, now = Date.now()): QueueMetrics { + const queue = JobQueue.queues[type]; + const active = JobQueue.churns[type] && queue.size > 0 ? 1 : 0; + const waiting = Array.from(queue.values()).slice(active); + + return { + active, + queued: waiting.length, + limit: MAX_QUEUE_SIZE, + available: Math.max(0, MAX_QUEUE_SIZE - queue.size), + oldestQueuedAgeMs: waiting.length ? Math.max(0, now - waiting[0].queuedAt) : null + }; + } + constructor() { - this.id = crypto.randomUUID() if (JobQueue._instance) { return JobQueue._instance } - JobQueue._instance = this } } @@ -142,3 +167,12 @@ class JobQueue { module.exports = { JobQueue } + +export type { + JobArgs, + JobType, + Queues, + QueueType, + QueueMetrics, + JobQueue +}; diff --git a/src/tsconfig.json b/src/tsconfig.json index 475f2af..175c300 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -4,7 +4,8 @@ "_version": "18.1.0", "compilerOptions": { "lib": [ - "es2023" + "es2023", + "esnext.promise" ], "module": "nodenext", "target": "es2022", diff --git a/src/utils.ts b/src/utils.ts index 2079d8d..390ebd7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,7 +1,7 @@ /* eslint-disable camelcase */ import { join } from "node:path"; import { promisify } from 'node:util'; -import { exec } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { symlinkSync, existsSync } from 'node:fs'; @@ -17,11 +17,12 @@ function compileWebpack(srcFolder: string, config ='highcharts.webpack.mjs') { const configDir = 'tools/webpacks'; console.log('Compiling webpack for: ', srcFolder) - const execAsync = promisify(exec); + const execFileAsync = promisify(execFile); - return execAsync( - `npx webpack -c ${join(configDir, config)} --stats errors-only`, - { timeout: 15000, cwd: srcFolder } + return execFileAsync( + 'npx', + ['webpack', '-c', join(configDir, config), '--stats', 'errors-only'], + { timeout: 15000, cwd: srcFolder, shell: false } ).then(({stdout, stderr}) => { if (stderr) { console.error(stderr); diff --git a/static/ops/console.css b/static/ops/console.css new file mode 100644 index 0000000..beb8428 --- /dev/null +++ b/static/ops/console.css @@ -0,0 +1,430 @@ +:root { + color-scheme: light; + --ink: #17211b; + --muted: #59645d; + --paper: #f4f1e8; + --panel: #fffdf7; + --line: #c8c7bd; + --green: #176b45; + --amber: #8b5d00; + --red: #a12a22; + --blue: #175c78; + --focus: #006f98; + --shadow: 0 1px 0 rgba(23, 33, 27, 0.12); + font-family: "Avenir Next", Avenir, "Trebuchet MS", sans-serif; + line-height: 1.45; +} + +* { + box-sizing: border-box; +} + +html { + background: var(--paper); + color: var(--ink); +} + +body { + margin: 0; + min-width: 0; +} + +button, +input { + font: inherit; +} + +button, +input[type="checkbox"] { + min-height: 2.75rem; +} + +button { + border: 1px solid var(--ink); + border-radius: 2px; + background: var(--ink); + color: #fff; + cursor: pointer; + padding: 0.55rem 0.9rem; +} + +button:hover { + background: #304038; +} + +button:disabled { + cursor: wait; + opacity: 0.55; +} + +:focus-visible { + outline: 3px solid var(--focus); + outline-offset: 3px; +} + +[hidden] { + display: none !important; +} + +.skip-link { + background: var(--ink); + color: #fff; + left: 0.75rem; + padding: 0.75rem; + position: fixed; + top: 0.5rem; + transform: translateY(-150%); + z-index: 10; +} + +.skip-link:focus { + transform: none; +} + +.masthead, +main { + margin-inline: auto; + max-width: 88rem; + padding-inline: clamp(1rem, 3vw, 3rem); +} + +.masthead { + align-items: end; + border-bottom: 3px solid var(--ink); + display: flex; + justify-content: space-between; + padding-block: 1.4rem 1rem; +} + +h1, +h2, +h3, +p { + margin-top: 0; +} + +h1 { + font-family: Georgia, "Times New Roman", serif; + font-size: clamp(1.8rem, 4vw, 3rem); + line-height: 1; + margin-bottom: 0; +} + +h2 { + font-size: 1.25rem; + margin-bottom: 0; +} + +h3 { + font-size: 1rem; + margin-block: 1.25rem 0.55rem; +} + +.eyebrow, +.section-number, +.label { + color: var(--muted); + font-size: 0.73rem; + font-weight: 700; + letter-spacing: 0.08em; + margin-bottom: 0.3rem; + text-transform: uppercase; +} + +.toolbar, +.button-row, +.result-heading { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 0.65rem; +} + +main > div > section, +main > noscript > section { + border-bottom: 1px solid var(--line); + padding-block: clamp(1.5rem, 4vw, 3rem); +} + +.section-heading, +.record-head { + align-items: start; + display: flex; + gap: 1rem; + justify-content: space-between; +} + +.quiet { + color: var(--muted); + font-size: 0.88rem; +} + +.service-grid { + display: grid; + gap: 0.75rem; + grid-template-columns: repeat(3, minmax(0, 1fr)); + margin-top: 1rem; +} + +.panel, +.record, +.action-panel, +.operation-result, +.notice { + background: var(--panel); + border: 1px solid var(--line); + box-shadow: var(--shadow); + min-width: 0; + padding: 1rem; +} + +.panel h3, +.record h3, +.result-heading h3 { + margin: 0; +} + +.status { + align-items: center; + display: inline-flex; + font-size: 0.78rem; + font-weight: 800; + gap: 0.4rem; + text-transform: uppercase; +} + +.status::before { + background: currentColor; + border-radius: 50%; + content: ""; + height: 0.55rem; + width: 0.55rem; +} + +.status-healthy, +.status-fresh, +.status-completed, +.status-succeeded, +.status-available { + color: var(--green); +} + +.status-degraded, +.status-stale, +.status-partial, +.status-active, +.status-no_op { + color: var(--amber); +} + +.status-unhealthy, +.status-unknown, +.status-unavailable, +.status-failed, +.status-rejected, +.status-aborted { + color: var(--red); +} + +.facts { + display: grid; + gap: 0.5rem; + grid-template-columns: repeat(2, minmax(0, 1fr)); + margin: 0.85rem 0 0; +} + +.facts div { + min-width: 0; +} + +.facts dt { + color: var(--muted); + font-size: 0.75rem; +} + +.facts dd { + margin: 0; + overflow-wrap: anywhere; +} + +.record-list { + display: grid; + gap: 0.55rem; +} + +.record p:last-child, +.notice p:last-child { + margin-bottom: 0; +} + +.failure-section { + border-top: 3px solid var(--red); +} + +.notice { + border-left: 4px solid var(--blue); + margin-top: 1rem; +} + +.notice-warning { + border-left-color: var(--amber); +} + +.notice-error { + border-left-color: var(--red); +} + +.action-panel, +.cache-actions, +.operation-result { + margin-top: 1rem; +} + +form { + max-width: 45rem; +} + +label, +legend { + font-weight: 700; +} + +input[type="text"], +input[type="password"] { + background: #fff; + border: 1px solid #737b76; + border-radius: 2px; + display: block; + margin-block: 0.35rem 0.8rem; + min-height: 2.75rem; + padding: 0.55rem; + width: 100%; +} + +fieldset { + border: 0; + display: flex; + flex-wrap: wrap; + gap: 1rem; + margin: 0 0 0.75rem; + padding: 0; +} + +.check { + align-items: center; + display: inline-flex; + gap: 0.4rem; + min-height: 2.75rem; +} + +.field-error { + color: var(--red); + font-weight: 700; + min-height: 1.5em; +} + +.danger { + background: #fff; + border-color: var(--red); + color: var(--red); +} + +.danger:hover { + background: #f9e8e4; +} + +.button-quiet { + background: transparent; + color: var(--ink); +} + +.result-heading { + justify-content: space-between; +} + +.mono { + font-family: ui-monospace, "SFMono-Regular", Consolas, monospace; + font-size: 0.84rem; + overflow-wrap: anywhere; +} + +details { + margin-top: 0.7rem; +} + +summary { + cursor: pointer; + font-weight: 700; + min-height: 2.75rem; + padding-block: 0.65rem; +} + +.session-section { + color: var(--muted); +} + +.login-page { + border-top: 0.5rem solid var(--ink); + min-height: 100vh; +} + +.login-shell { + display: grid; + min-height: calc(100vh - 0.5rem); + place-items: center; + padding: 1rem; +} + +.login-panel { + background: var(--panel); + border: 1px solid var(--line); + box-shadow: 0.6rem 0.6rem 0 var(--ink); + max-width: 31rem; + padding: clamp(1.25rem, 6vw, 3rem); + width: 100%; +} + +.login-intro { + margin-block: 1.5rem; +} + +.security-note { + border-top: 1px solid var(--line); + color: var(--muted); + font-size: 0.83rem; + margin-top: 1.5rem; + padding-top: 1rem; +} + +@media (max-width: 44rem) { + .masthead, + .section-heading { + align-items: stretch; + flex-direction: column; + } + + .toolbar { + justify-content: space-between; + } + + .service-grid { + grid-template-columns: 1fr; + } + + .facts { + grid-template-columns: 1fr; + } + + .button-row button { + flex: 1 1 100%; + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + } +} diff --git a/static/ops/console.js b/static/ops/console.js new file mode 100644 index 0000000..25f1f3c --- /dev/null +++ b/static/ops/console.js @@ -0,0 +1,484 @@ +'use strict' + +const API = '/_ops/api/v1' +const SERVICES = ['router', 'downloader', 'builder'] +const REMOTE_SERVICES = ['downloader', 'builder'] +const COMMIT = /^[0-9a-f]{40}$/ +const STATUS = new Set(['healthy', 'degraded', 'unhealthy', 'fresh', 'stale', 'unknown', 'available', 'unavailable', 'active', 'completed', 'no_op', 'partial', 'failed', 'succeeded', 'rejected', 'aborted']) +const ERROR_LABELS = { + INCOMPATIBLE_SCHEMA: 'The service returned an incompatible response.', + SERVICE_RESPONSE_TOO_LARGE: 'The service response exceeded its safety limit.', + SERVICE_TIMEOUT: 'The service did not respond before the deadline.', + SERVICE_UNAVAILABLE: 'The service is unavailable.', + RATE_LIMITED: 'Too many requests. Wait before trying again.' +} + +const ui = Object.fromEntries([ + 'app', 'connection-status', 'refresh', 'snapshot-time', 'aggregate-warning', + 'services', 'queues', 'activity', 'failures', 'cache-summary', 'commit', + 'evict-form', 'evict-error', 'operation-result', 'operation-content', + 'dismiss-result', 'cache-entries', 'expiry-warning', 'stay-signed-in', + 'session-status', 'logout' +].map(id => [id, document.querySelector('#' + id)])) + +let session +let snapshot +let refreshTimer +let expiryTimer +let expiryRedirectTimer +let refreshing = false +const pendingServices = new Set() + +bootstrap() + +async function bootstrap () { + try { + await renewSession() + ui.app.hidden = false + bindEvents() + await refreshSnapshot() + } catch (error) { + if (!error.auth) authLost() + } +} + +function bindEvents () { + ui.refresh.addEventListener('click', refreshSnapshot) + ui['evict-form'].addEventListener('submit', evictCommit) + ui['dismiss-result'].addEventListener('click', () => { ui['operation-result'].hidden = true }) + ui['stay-signed-in'].addEventListener('click', renewSession) + ui.logout.addEventListener('click', logout) + document.querySelectorAll('[data-operation]').forEach(button => { + button.addEventListener('click', () => mutate(button.dataset.operation, [button.dataset.service])) + }) + document.addEventListener('visibilitychange', resumeRefresh) + window.addEventListener('online', resumeRefresh) + window.addEventListener('offline', pauseRefresh) +} + +async function request (path, options) { + const response = await fetch(API + path, { credentials: 'same-origin', ...options }) + if (response.status === 401) { + authLost() + throw Object.assign(new Error('Authentication required'), { auth: true }) + } + if (response.status === 204) return null + const body = await response.json().catch(() => null) + if (!response.ok) { + const code = safeCode(body?.error?.code) + throw Object.assign(new Error(ERROR_LABELS[code] || 'The request could not be completed.'), { code }) + } + return body +} + +async function renewSession () { + session = await request('/session') + renderSession() +} + +async function refreshSnapshot () { + if (refreshing || document.hidden || !navigator.onLine) return + refreshing = true + clearTimeout(refreshTimer) + ui.refresh.disabled = document.activeElement !== ui.refresh + ui.services.setAttribute('aria-busy', 'true') + setConnection(snapshot ? 'Refreshing' : 'Loading snapshot') + + try { + snapshot = await request('/snapshot') + renderSnapshot(snapshot) + setConnection('Current') + } catch (error) { + if (!error.auth) { + setConnection(snapshot ? 'Refresh failed; showing prior snapshot' : error.message) + showAggregateWarning(snapshot ? 'Refresh failed. Existing values may be out of date.' : 'Snapshot unavailable. Try a manual refresh.', true) + } + } finally { + refreshing = false + ui.refresh.disabled = false + ui.services.setAttribute('aria-busy', 'false') + scheduleRefresh(snapshot?.refreshAfterMs) + } +} + +function renderSnapshot (data) { + const observed = date(data.observedAt) + ui['snapshot-time'].textContent = observed ? 'Observed ' + observed : 'Observation time unavailable' + replace(ui.services, SERVICES.map(service => servicePanel(service, data.services?.[service]))) + renderQueues(data.services) + renderActivity(data.activity) + renderFailures(data.failures) + renderCaches(data.services) + const unavailable = SERVICES.filter(service => data.services?.[service]?.freshness === 'unknown') + showAggregateWarning(unavailable.length ? 'Partial snapshot. Unavailable: ' + unavailable.map(label).join(', ') + '.' : '', false) +} + +function servicePanel (service, slot) { + const panel = element('article', 'panel') + const head = element('div', 'record-head') + head.append(element('h3', '', label(service))) + const freshness = slot?.freshness === 'unknown' ? 'unavailable' : slot?.freshness + head.append(status(freshness || 'unknown')) + panel.append(head) + + if (!slot || slot.freshness === 'unknown' || !slot.snapshot) { + panel.append(element('p', 'quiet', 'No current service values.')) + panel.append(facts([ + ['Last attempt', date(slot?.lastAttemptAt) || 'Not yet attempted'], + ['Reason', errorLabel(slot?.error?.code)] + ])) + return panel + } + + panel.append(status(slot.snapshot.health?.status)) + panel.append(facts([ + ['Freshness', slot.freshness === 'stale' ? 'Stale, ' + duration(slot.ageMs) + ' old' : 'Fresh'], + ['Observed', date(slot.snapshot.observedAt) || 'Unavailable'], + ['Started', date(slot.snapshot.startedAt) || 'Unavailable'], + ['Instance', text(slot.snapshot.instanceId, 64) || 'Unavailable'] + ])) + if (slot.error) panel.append(element('p', 'quiet', errorLabel(slot.error.code))) + + const reasons = limited(slot.snapshot.health?.reasons, 16) + if (reasons.length) { + const list = element('ul') + reasons.forEach(reason => list.append(element('li', '', text(reason?.message, 256) || safeCode(reason?.code)))) + panel.append(list) + } + return panel +} + +function renderQueues (services) { + const queues = SERVICES.flatMap(service => limited(services?.[service]?.snapshot?.queues, 8).map(queue => ({ service, queue }))) + const records = queues.map(({ service, queue }) => { + const record = element('article', 'record') + record.append(recordHead(label(service) + ' / ' + text(queue.name, 64), queue.active + queue.queued >= queue.limit && queue.limit > 0 ? 'degraded' : 'available')) + record.append(facts([ + ['Active', integer(queue.active)], ['Queued', integer(queue.queued)], + ['Capacity', integer(queue.limit)], ['Available', integer(queue.available)], + ['Oldest queued', queue.oldestQueuedAgeMs === null ? 'None' : duration(queue.oldestQueuedAgeMs)] + ])) + return record + }) + replace(ui.queues, records.length ? records : [empty('No active or configured queues were reported.')]) +} + +function renderActivity (activity) { + const records = limited(activity, 600) + const children = records.map(trace => { + const record = element('article', 'record') + const outcome = trace.state === 'active' ? 'active' : trace.outcome?.status + record.append(recordHead(text(trace.request?.method, 16) + ' ' + text(trace.request?.route, 256), outcome)) + record.append(facts([ + ['Started', date(trace.startedAt) || 'Unavailable'], + ['Duration', trace.durationMs === null ? 'In progress' : duration(trace.durationMs)], + ['Resource', text(trace.request?.resource, 256) || 'None'], + ['Commit', text(trace.request?.commit, 40) || 'None'], + ['Build mode', text(trace.request?.buildMode, 16) || 'None'], + ['Correlation', text(trace.correlationId, 64)] + ])) + const spans = limited(trace.spans, 8) + if (spans.length) { + const details = element('details') + details.append(element('summary', '', 'Stages (' + spans.length + ')')) + spans.forEach(span => details.append(facts([ + ['Service', label(span.service)], ['Operation', humanize(span.operation)], + ['State', humanize(span.state)], ['Duration', span.durationMs === null ? 'In progress' : duration(span.durationMs)], + ['Outcome', humanize(span.outcome?.status)], ['Code', safeCode(span.outcome?.code) || 'None'] + ]))) + record.append(details) + } + return record + }) + replace(ui.activity, children.length ? children : [empty('No recent public request activity.')]) +} + +function renderFailures (failures) { + const records = limited(failures, 300) + const children = records.map(failure => { + const record = element('article', 'record') + record.append(recordHead(label(failure.service) + ' / ' + humanize(failure.operation), 'failed')) + record.append(element('p', '', text(failure.summary, 256) || 'Operational failure')) + record.append(facts([ + ['Occurred', date(failure.occurredAt) || 'Unavailable'], + ['Code', safeCode(failure.code) || 'INTERNAL_ERROR'], + ['HTTP status', failure.httpStatus === null ? 'None' : integer(failure.httpStatus)], + ['Commit', text(failure.commit, 40) || 'None'], + ['Correlation', text(failure.correlationId, 64)] + ])) + return record + }) + replace(ui.failures, children.length ? children : [empty('No recent failures.')]) +} + +function renderCaches (services) { + const summaries = [] + const entries = [] + REMOTE_SERVICES.forEach(service => { + const slot = services?.[service] + const cache = slot?.snapshot?.cache + const panel = element('article', 'panel') + panel.append(recordHead(label(service), slot?.freshness === 'unknown' ? 'unavailable' : slot?.freshness)) + if (!cache) { + panel.append(element('p', 'quiet', 'Cache inspection unavailable.')) + } else { + panel.append(facts([ + ['Entries', integer(cache.entryCount)], ['Size', bytes(cache.totalBytes)], + ['Idle expiry', duration(cache.idleExpiryMs)], ['List', cache.entriesTruncated ? 'Most recent 200' : 'Complete'] + ])) + limited(cache.entries, 200).forEach(entry => entries.push(cacheEntry(service, entry, slot.freshness))) + } + summaries.push(panel) + }) + replace(ui['cache-summary'], summaries) + replace(ui['cache-entries'], entries.length ? entries : [empty('No cache entries. This is a normal empty state.')]) +} + +function cacheEntry (service, entry, freshness) { + const record = element('article', 'record') + record.append(recordHead(label(service), freshness)) + record.append(element('p', 'mono', text(entry.commit, 40))) + record.append(facts([ + ['Size', bytes(entry.sizeBytes)], ['In use', integer(entry.inUse)], + ['Last accessed', date(entry.lastAccessedAt) || 'Unavailable'], ['Expires', date(entry.expiresAt) || 'Unavailable'] + ])) + const use = element('button', 'button-quiet', 'Prepare eviction') + use.type = 'button' + use.addEventListener('click', () => { + ui.commit.value = text(entry.commit, 40) + document.querySelectorAll('input[name="target"]').forEach(input => { input.checked = input.value === service }) + ui.commit.focus() + }) + record.append(use) + return record +} + +async function evictCommit (event) { + event.preventDefault() + const commit = ui.commit.value + const targets = [...document.querySelectorAll('input[name="target"]:checked')].map(input => input.value) + if (!COMMIT.test(commit) || !targets.length) { + ui['evict-error'].textContent = 'Enter a full lowercase commit SHA and select at least one target.' + ui.commit.setAttribute('aria-invalid', 'true') + return + } + ui['evict-error'].textContent = '' + ui.commit.removeAttribute('aria-invalid') + await mutate('cache.evict_commit', targets, commit) +} + +async function mutate (operation, targets, commit) { + if (targets.some(service => pendingServices.has(service))) return + targets.forEach(service => pendingServices.add(service)) + updateMutationControls() + showPendingResult(operation, targets) + + try { + const body = { operation, targets } + if (commit) body.commit = commit + const result = await request('/cache-operations', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Ops-CSRF': session.csrfToken }, + body: JSON.stringify(body) + }) + renderOperationResult(result) + await refreshSnapshot() + } catch (error) { + if (!error.auth) renderOperationError(error) + } finally { + targets.forEach(service => pendingServices.delete(service)) + updateMutationControls() + } +} + +function showPendingResult (operation, targets) { + ui['operation-result'].hidden = false + replace(ui['operation-content'], [ + status('active'), + element('p', '', humanize(operation) + ' is running for ' + targets.map(label).join(' and ') + '.') + ]) + ui['operation-result'].focus() +} + +function renderOperationResult (result) { + const children = [status(result.outcome), facts([ + ['Operation', humanize(result.operation)], ['Completed', date(result.completedAt) || 'Unavailable'], + ['Correlation', text(result.correlationId, 64)] + ])] + if (result.outcome === 'unknown') children.push(element('p', 'notice notice-warning', 'The outcome is unknown. Inspect a fresh cache snapshot before deciding whether to retry.')) + limited(result.targets, 2).forEach(target => { + const record = element('article', 'record') + record.append(recordHead(label(target.service), target.outcome)) + record.append(facts([ + ['Removed', integer(target.removedEntries)], ['Freed', bytes(target.freedBytes)], + ['Absent', target.absent ? 'Yes' : 'No'], ['Skipped in use', integer(target.skippedInUse)], + ['Skipped changed', target.skippedChanged === undefined ? 'Not applicable' : integer(target.skippedChanged)], + ['Error', target.error ? errorLabel(target.error.code) : 'None'] + ])) + children.push(record) + }) + replace(ui['operation-content'], children) + ui['operation-result'].focus() +} + +function renderOperationError (error) { + replace(ui['operation-content'], [status('failed'), element('p', '', error.message)]) + ui['operation-result'].hidden = false + ui['operation-result'].focus() +} + +function updateMutationControls () { + document.querySelectorAll('[data-service]').forEach(button => { button.disabled = pendingServices.has(button.dataset.service) }) + document.querySelectorAll('input[name="target"]').forEach(input => { input.disabled = pendingServices.has(input.value) }) + const selected = [...document.querySelectorAll('input[name="target"]:checked')] + ui['evict-form'].querySelector('button').disabled = selected.some(input => pendingServices.has(input.value)) +} + +function renderSession () { + clearTimeout(expiryTimer) + clearTimeout(expiryRedirectTimer) + const idle = Date.parse(session.idleExpiresAt) + const absolute = Date.parse(session.absoluteExpiresAt) + const expiry = Math.min(idle, absolute) + ui['session-status'].textContent = 'Idle expiry ' + date(session.idleExpiresAt) + '. Absolute expiry ' + date(session.absoluteExpiresAt) + '.' + const updateWarning = () => { + ui['expiry-warning'].hidden = expiry - Date.now() > 5 * 60 * 1000 + } + updateWarning() + expiryTimer = setTimeout(updateWarning, Math.max(0, expiry - Date.now() - 5 * 60 * 1000)) + expiryRedirectTimer = setTimeout(authLost, Math.max(0, expiry - Date.now())) +} + +async function logout () { + ui.logout.disabled = true + try { + await request('/session', { method: 'DELETE', headers: { 'X-Ops-CSRF': session.csrfToken } }) + } finally { + authLost() + } +} + +function authLost () { + clearTimeout(refreshTimer) + clearTimeout(expiryTimer) + clearTimeout(expiryRedirectTimer) + window.location.replace('/_ops/login') +} + +function pauseRefresh () { + clearTimeout(refreshTimer) + setConnection(navigator.onLine ? 'Refresh paused while hidden' : 'Offline; showing last snapshot') +} + +function resumeRefresh () { + if (!document.hidden && navigator.onLine) refreshSnapshot() + else pauseRefresh() +} + +function scheduleRefresh (milliseconds) { + clearTimeout(refreshTimer) + if (document.hidden || !navigator.onLine) return pauseRefresh() + const delay = Number.isSafeInteger(milliseconds) && milliseconds >= 5000 ? milliseconds : 30000 + const next = new Date(Date.now() + delay) + setConnection('Current; next refresh ' + next.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' })) + refreshTimer = setTimeout(refreshSnapshot, delay) +} + +function showAggregateWarning (message, error) { + ui['aggregate-warning'].hidden = !message + ui['aggregate-warning'].className = 'notice' + (error ? ' notice-error' : ' notice-warning') + ui['aggregate-warning'].textContent = message +} + +function setConnection (message) { + ui['connection-status'].textContent = message +} + +function recordHead (title, state) { + const head = element('div', 'record-head') + head.append(element('h3', '', title || 'Unknown'), status(state)) + return head +} + +function status (value) { + const safe = STATUS.has(value) ? value : 'unknown' + const shown = safe === 'no_op' ? 'No action needed' : safe === 'unknown' ? 'Unavailable' : humanize(safe) + return element('span', 'status status-' + safe, shown) +} + +function facts (items) { + const list = element('dl', 'facts') + items.forEach(([name, value]) => { + const group = element('div') + group.append(element('dt', '', name), element('dd', '', String(value ?? 'Unavailable'))) + list.append(group) + }) + return list +} + +function empty (message) { + return element('p', 'record quiet', message) +} + +function replace (parent, children) { + parent.replaceChildren(...children) +} + +function element (tag, className, content) { + const node = document.createElement(tag) + if (className) node.className = className + if (content !== undefined) node.textContent = content + return node +} + +function limited (value, maximum) { + return Array.isArray(value) ? value.slice(0, maximum) : [] +} + +function text (value, maximum) { + return typeof value === 'string' ? value.slice(0, maximum) : '' +} + +function safeCode (value) { + const code = text(value, 64) + return /^[A-Za-z0-9_.-]+$/.test(code) ? code : '' +} + +function errorLabel (code) { + const safe = safeCode(code) + return ERROR_LABELS[safe] || (safe ? 'Service error: ' + safe : 'Reason unavailable.') +} + +function label (value) { + const labels = { router: 'Router', downloader: 'Downloader', builder: 'Builder' } + return labels[value] || 'Unknown service' +} + +function humanize (value) { + const safe = text(value, 64).replace(/[^A-Za-z0-9_.-]/g, '') + return safe ? safe.replace(/[._-]+/g, ' ').replace(/^./, letter => letter.toUpperCase()) : 'Unknown' +} + +function integer (value) { + return Number.isSafeInteger(value) && value >= 0 ? value.toLocaleString() : 'Unavailable' +} + +function duration (value) { + if (!Number.isSafeInteger(value) || value < 0) return 'Unavailable' + if (value < 1000) return value + ' ms' + if (value < 60000) return (value / 1000).toFixed(1) + ' s' + if (value < 3600000) return Math.round(value / 60000) + ' min' + return (value / 3600000).toFixed(1) + ' h' +} + +function bytes (value) { + if (!Number.isSafeInteger(value) || value < 0) return 'Unavailable' + if (value < 1024) return value + ' B' + if (value < 1024 * 1024) return (value / 1024).toFixed(1) + ' KiB' + return (value / (1024 * 1024)).toFixed(1) + ' MiB' +} + +function date (value) { + const timestamp = typeof value === 'string' ? Date.parse(value) : NaN + return Number.isFinite(timestamp) ? new Date(timestamp).toLocaleString() : '' +} diff --git a/static/ops/index.html b/static/ops/index.html new file mode 100644 index 0000000..c94ea10 --- /dev/null +++ b/static/ops/index.html @@ -0,0 +1,127 @@ + + + + + + Operations console + + + + + +
+
+

Development environment

+

Operations console

+
+
+ Loading + +
+
+ +
+ + + +
+ + diff --git a/static/ops/login.html b/static/ops/login.html new file mode 100644 index 0000000..30b075d --- /dev/null +++ b/static/ops/login.html @@ -0,0 +1,32 @@ + + + + + + Sign in | Operations console + + + + +
+ + +
+ + diff --git a/static/ops/login.js b/static/ops/login.js new file mode 100644 index 0000000..3fd5540 --- /dev/null +++ b/static/ops/login.js @@ -0,0 +1,37 @@ +'use strict' + +const panel = document.querySelector('#login-panel') +const heading = document.querySelector('#login-heading') +const form = document.querySelector('#login-form') +const token = document.querySelector('#token') +const error = document.querySelector('#login-error') +const submit = document.querySelector('#login-submit') + +panel.hidden = false +heading.focus() + +form.addEventListener('submit', async event => { + event.preventDefault() + error.textContent = '' + token.removeAttribute('aria-invalid') + submit.disabled = true + + try { + const response = await fetch('/_ops/api/v1/session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ token: token.value }) + }) + token.value = '' + if (!response.ok) throw new Error('Authentication failed') + window.location.replace('/_ops/') + } catch (failure) { + token.value = '' + error.textContent = 'Sign-in failed. Check the token or try again later.' + token.setAttribute('aria-invalid', 'true') + token.focus() + } finally { + submit.disabled = false + } +}) diff --git a/test/JobQueue.test.js b/test/JobQueue.test.js index aad50fb..ac1c0c6 100644 --- a/test/JobQueue.test.js +++ b/test/JobQueue.test.js @@ -1,62 +1,89 @@ -const { describe, it, before, after } = require('node:test') +const { describe, it } = require('mocha') const assert = require('node:assert') const { JobQueue } = require('../app/JobQueue.js') -function makeJob (time = 0, shouldReject = false) { - return { - func: function () { - return new Promise((resolve, reject) => { - setTimeout(() => { - return shouldReject ? reject(new Error('reason')) : resolve('result') - }, time) - }) - }, - args: [] - } -} +function deferred () { + let resolveJob + let rejectJob + const promise = new Promise((resolve, reject) => { + resolveJob = resolve + rejectJob = reject + }) -describe('JobQueue', async () => { - before(async () => { }) + return { promise, resolve: resolveJob, reject: rejectJob } +} - after(async () => { }) +function makeJob (func) { + return { func, args: [] } +} +describe('JobQueue', () => { it('is a proper singleton', () => { const queue1 = new JobQueue() - const queue2 = new JobQueue() - assert.equal(queue1.id, queue2.id) + assert.strictEqual(queue1, queue2) }) - it('JobQueue', async () => { + it('returns the job result and cleans up the queue', async () => { const queue = new JobQueue() - const commit = '123123123' - - queue.addJob('download', commit, makeJob()) + const result = { built: true } - const jobs = () => queue.getJobs('download') + const promise = queue.addJob('download', 'success', makeJob(async () => result)) - assert.equal(jobs().length, 1, 'Queue should be 1 after adding 1') - queue.addJob('download', commit, makeJob()) + assert.strictEqual(await promise, result) + assert.equal(queue.getJobs('download').length, 0) + }) - assert.equal(jobs().length, 1, 'Queue should be 1 after trying to add another job for same commit') + it('returns the same result promise to duplicate callers', async () => { + const queue = new JobQueue() + const job = deferred() + const first = queue.addJob('download', 'duplicate-success', makeJob(() => job.promise)) + const duplicate = queue.addJob('download', 'duplicate-success', makeJob(async () => 'wrong')) - queue.addJob('download', 'another', makeJob()) - assert.equal(jobs().length, 2, 'Queue length should be 2 after adding job for new commit') + assert.strictEqual(first, duplicate) - console.log(jobs()) + job.resolve('result') + assert.equal(await first, 'result') + assert.equal(await duplicate, 'result') + }) - const firstJob = jobs()[0][1].done - await firstJob + it('rejects duplicate callers with the original error', async () => { + const queue = new JobQueue() + const job = deferred() + const error = new Error('reason') + const first = queue.addJob('download', 'duplicate-failure', makeJob(() => job.promise)) + const duplicate = queue.addJob('download', 'duplicate-failure', makeJob(async () => 'wrong')) - assert.equal(jobs().length, 1, 'Queue length should be 1 after first job has finished') + assert.strictEqual(first, duplicate) - const oneLastJob = queue.addJob('download', 'anotherfailing', makeJob(1000, true)) - console.log({ oneLastJob }) - await oneLastJob + job.reject(error) + await assert.rejects(first, (caught) => caught === error) + await assert.rejects(duplicate, (caught) => caught === error) + assert.equal(queue.getJobs('download').length, 0) + }) - assert.equal(jobs().length, 0, 'All jobs are done') + it('continues with the next queued job after a failure', async () => { + const queue = new JobQueue() + const firstJob = deferred() + const error = new Error('first failed') + const calls = [] + const first = queue.addJob('download', 'failing-first', makeJob(async () => { + calls.push('first') + return firstJob.promise + })) + const second = queue.addJob('download', 'successful-second', makeJob(async () => { + calls.push('second') + return 'second result' + })) + + firstJob.reject(error) + + await assert.rejects(first, (caught) => caught === error) + assert.equal(await second, 'second result') + assert.deepEqual(calls, ['first', 'second']) + assert.equal(queue.getJobs('download').length, 0) }) }) diff --git a/test/browser/ops.spec.js b/test/browser/ops.spec.js new file mode 100644 index 0000000..1388424 --- /dev/null +++ b/test/browser/ops.spec.js @@ -0,0 +1,167 @@ +'use strict' + +const { test, expect } = require('@playwright/test') +const AxeBuilder = require('@axe-core/playwright').default +const { spawnSync } = require('node:child_process') +const { resolve } = require('node:path') + +const root = resolve(__dirname, '../..') +const composeFile = 'compose.ops-test.yaml' +const project = `ops-console-browser-${process.pid}` +const port = Number(process.env.OPS_TEST_HOST_PORT || 8080) +const baseURL = `http://localhost:${port}` +const loginToken = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +const axeTags = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22aa'] + +test.describe.configure({ mode: 'serial' }) + +test.beforeAll(() => { + inspectTopology() + compose(['up', '--build', '--wait', '--remove-orphans']) +}) + +test.afterAll(() => { + compose(['down', '--volumes', '--remove-orphans'], {}, true) +}) + +test.beforeEach(async ({ context, page }) => { + await context.clearCookies() + await control('/fixture', { service: 'downloader', snapshot: 'fresh', cache: 'healthy', resetCounts: true }) + await control('/fixture', { service: 'builder', snapshot: 'fresh', cache: 'healthy', resetCounts: true }) + await page.goto('/_ops/login') +}) + +test('authenticates, renders snapshots, mutates caches, and scans key states', async ({ page }) => { + await expect(page.getByRole('heading', { name: 'Operations console' })).toBeFocused() + await axe(page) + + await login(page) + await waitForOverview(page) + await expect(page.getByRole('heading', { name: 'Service status' })).toBeVisible() + await expect(page.locator('#services')).toContainText('Router') + await expect(page.locator('#services')).toContainText('Downloader') + await expect(page.locator('#services')).toContainText('Builder') + await expect(page.locator('#services')).toContainText('Fresh') + await axe(page) + + await page.getByRole('button', { name: 'Purge expired downloader' }).click() + await expect(page.getByRole('region', { name: 'Cache operation result' })).toBeFocused() + await expect(page.locator('#operation-content')).toContainText('No action needed') + await expect(page.locator('#operation-content')).toContainText('Cache purge expired') + await expect(page.locator('#operation-content')).toContainText('Downloader') + await expect(page.locator('#operation-content')).toContainText('Removed') + await waitForOverview(page) + await expect(page.locator('#session-status')).toContainText('Idle expiry') + await axe(page) +}) + +test('shows stale, unavailable, expiry, and logout auth states', async ({ page }) => { + await login(page) + await waitForOverview(page) + + await control('/fixture', { service: 'downloader', snapshot: 'slow' }) + await page.getByRole('button', { name: 'Refresh' }).click() + await expect(page.locator('#services')).toContainText('Stale', { timeout: 15000 }) + + compose(['down', '--volumes', '--remove-orphans'], {}, true) + compose(['up', '--build', '--wait', '--remove-orphans']) + await control('/fixture', { service: 'downloader', snapshot: 'fresh', cache: 'healthy', resetCounts: true }) + await control('/fixture', { service: 'builder', snapshot: 'disconnect' }) + await page.goto('/_ops/login') + await login(page) + await waitForOverview(page) + await expect(page.locator('#aggregate-warning')).toContainText('Partial snapshot') + await expect(page.locator('#services')).toContainText('No current service values') + + await control('/clock', { advanceMs: 31 * 60 * 1000 }) + await page.getByRole('button', { name: 'Refresh' }).click() + await expect(page).toHaveURL(/\/_ops\/login$/) + + await login(page) + await waitForOverview(page) + await page.getByRole('button', { name: 'Sign out' }).click() + await expect(page).toHaveURL(/\/_ops\/login$/) +}) + +test('keeps keyboard flow and narrow, zoomed, reduced-motion layouts usable', async ({ page }) => { + await page.emulateMedia({ reducedMotion: 'reduce' }) + await page.setViewportSize({ width: 320, height: 720 }) + await login(page) + await waitForOverview(page) + + expect(await page.evaluate(() => document.documentElement.scrollWidth <= document.documentElement.clientWidth)).toBe(true) + await page.keyboard.press('Tab') + await expect(page.locator('.skip-link')).toBeFocused() + await page.keyboard.press('Enter') + await expect(page).toHaveURL(/#main$/) + + await page.setViewportSize({ width: 640, height: 720 }) + await page.evaluate(() => { document.documentElement.style.fontSize = '200%' }) + expect(await page.evaluate(() => document.documentElement.scrollWidth <= document.documentElement.clientWidth)).toBe(true) + await expect(page.getByRole('button', { name: 'Refresh' })).toBeVisible() + await expect(page.getByRole('heading', { name: 'Service status' })).toBeVisible() +}) + +test('keeps focus on manual refresh while snapshot refresh completes', async ({ page }) => { + await login(page) + await waitForOverview(page) + await control('/fixture', { service: 'downloader', snapshot: 'slow' }) + + const refresh = page.getByRole('button', { name: 'Refresh' }) + await refresh.focus() + await refresh.press('Enter') + + await expect(page.locator('#services')).toHaveAttribute('aria-busy', 'true') + await expect(page.locator('#connection-status')).toContainText('Refreshing') + await expect(refresh).toBeFocused() + await waitForOverview(page) + await expect(refresh).toBeFocused() +}) + +async function login (page) { + await page.getByLabel('Operations token').fill(loginToken) + await page.getByRole('button', { name: 'Sign in' }).click() + await expect(page).toHaveURL(/\/_ops\/$/) +} + +async function waitForOverview (page) { + await expect(page.locator('#services')).toHaveAttribute('aria-busy', 'false') + await expect(page.getByText('Waiting for the first snapshot')).toBeHidden() +} + +async function axe (page) { + const results = await new AxeBuilder({ page }).withTags(axeTags).analyze() + expect(results.violations).toEqual([]) +} + +async function control (path, body) { + const response = await fetch(`${baseURL}/__ops-test${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + if (!response.ok) throw new Error(`test control ${path} failed`) +} + +function inspectTopology () { + const config = JSON.parse(compose(['config', '--format', 'json'], {}, true)) + const published = Object.entries(config.services).filter(([, service]) => service.ports?.length) + if (published.length !== 1 || published[0][0] !== 'router' || published[0][1].ports.length !== 1 || Number(published[0][1].ports[0].target) !== 8080) { + throw new Error('only router:8080 may be published') + } + for (const name of ['downloader', 'builder']) { + if (config.services[name].ports?.length) throw new Error(`${name} publishes a port`) + } +} + +function compose (args, environment = {}, quiet = false) { + const result = spawnSync('docker', ['compose', '-p', project, '-f', composeFile, ...args], { + cwd: root, + env: { ...process.env, OPS_TEST_HOST_PORT: String(port), ...environment }, + encoding: 'utf8', + stdio: quiet ? ['ignore', 'pipe', 'pipe'] : 'inherit' + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`docker exited with ${result.status}`) + return quiet ? result.stdout : '' +} diff --git a/test/builder-service.js b/test/builder-service.js index 47aba51..dfc2257 100644 --- a/test/builder-service.js +++ b/test/builder-service.js @@ -5,7 +5,10 @@ const fs = require('node:fs') const http = require('node:http') const os = require('node:os') const { execFileSync } = require('node:child_process') +const { EventEmitter } = require('node:events') const { join } = require('node:path') +const { PassThrough, Readable } = require('node:stream') +const { gzipSync } = require('node:zlib') const { afterEach, beforeEach, describe, it } = require('mocha') const sinon = require('sinon') const { createBuilder } = require('../app/build') @@ -17,6 +20,54 @@ const config = require('../config.json') const SHA = '0123456789abcdef0123456789abcdef01234567' +function tarArchive (entries) { + const blocks = [] + for (const entry of entries) { + const data = Buffer.from(entry.data || '') + const header = Buffer.alloc(512) + let name = entry.name + let prefix = '' + if (Buffer.byteLength(name) > 100) { + const slash = name.lastIndexOf('/', 155) + prefix = name.slice(0, slash) + name = name.slice(slash + 1) + } + header.write(name, 0, 100) + header.write('0000755\0', 100, 8) + header.write('0000000\0', 108, 8) + header.write('0000000\0', 116, 8) + header.write(`${(entry.size === undefined ? data.length : entry.size).toString(8).padStart(11, '0')}\0`, 124, 12) + header.write('00000000000\0', 136, 12) + header.fill(32, 148, 156) + header[156] = (entry.type || (entry.name.endsWith('/') ? '5' : '0')).charCodeAt(0) + header.write('ustar\0', 257, 6) + header.write('00', 263, 2) + header.write(prefix, 345, 155) + let checksum = 0 + for (const byte of header) checksum += byte + header.write(`${checksum.toString(8).padStart(6, '0')}\0 `, 148, 8) + blocks.push(header, data, Buffer.alloc((512 - data.length % 512) % 512)) + } + blocks.push(Buffer.alloc(1024)) + return gzipSync(Buffer.concat(blocks)) +} + +function sourceArchive (extra = []) { + return tarArchive([ + ...SOURCE_PATHS.map(name => ({ name: `${name}/`, type: '5' })), + { name: 'ts/sample', data: 'source' }, + ...extra + ]) +} + +async function expectArchiveFailure (cacheRoot, archive, options = {}) { + const service = createBuilderService({ cacheRoot, client: { stream: () => Readable.from(archive) }, builder: {}, ...options }) + let error + try { await service.ensureSource(SHA) } catch (caught) { error = caught } + expect(error).to.include({ code: options.code || 'ARCHIVE_ERROR', status: options.status || 502 }) + expect(await fs.promises.readdir(cacheRoot)).to.deep.equal([]) +} + function request (app, path, options = {}) { return new Promise((resolve, reject) => { const server = app.listen(0, () => { @@ -49,12 +100,25 @@ describe('builder service', () => { }) it('exposes health without auth and protects every v1 route', async () => { - const app = createApp({ token: 'secret', service: {} }) + const service = { + build: sinon.stub(), + snapshot: sinon.stub(), + cacheManager: { execute: sinon.stub() } + } + const app = createApp({ token: 'secret', service }) + const unauthorized = { error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } } expect((await request(app, '/health')).status).to.equal(200) - expect((await request(app, '/v1/build', { method: 'POST' })).status).to.equal(401) - expect((await request(app, '/v1/build', { method: 'POST', headers: { Authorization: 'Bearer secrets' } })).status).to.equal(401) - expect((await request(app, '/v1/build', { method: 'POST', headers: { Authorization: 'Bearer wrong!' } })).status).to.equal(401) - expect((await request(app, '/v1/cleanup', { method: 'POST', headers: { Authorization: 'Bearer secret' }, body: {} })).status).to.equal(500) + for (const authorization of [undefined, 'Basic secret', 'Bearer ', 'Bearer x', 'Bearer wrong!', 'Bearer secrets']) { + const headers = authorization === undefined ? {} : { Authorization: authorization } + for (const [path, method] of [['/v1/build', 'POST'], ['/v1/ops/snapshot', 'GET']]) { + const response = await request(app, path, { method, headers }) + expect(response.status).to.equal(401) + expect(JSON.parse(response.body)).to.deep.equal(unauthorized) + } + } + expect(service.build.callCount).to.equal(0) + expect(service.snapshot.callCount).to.equal(0) + expect(service.cacheManager.execute.callCount).to.equal(0) }) it('fails startup without an internal service token', () => { @@ -72,6 +136,10 @@ describe('builder service', () => { it('validates canonical commits, normalized paths, modes, and options', () => { expect(validateRequest({ commit: SHA, path: 'modules/exporting.src.js', mode: 'legacy' })).to.include({ commit: SHA, path: 'modules/exporting.src.js', mode: 'legacy' }) + expect(validateRequest({ commit: SHA, path: 'a.js', mode: 'webpack' }).options).to.deep.equal({}) + for (const config of ['highcharts.webpack.mjs', 'custom-1.webpack_mjs']) { + expect(validateRequest({ commit: SHA, path: 'a.js', mode: 'webpack', options: { config } }).options.config).to.equal(config) + } for (const body of [ { commit: SHA.toUpperCase(), path: 'a.js', mode: 'legacy' }, { commit: SHA, path: '../a.js', mode: 'legacy' }, @@ -81,13 +149,44 @@ describe('builder service', () => { ]) expect(() => validateRequest(body)).to.throw() }) + it('rejects unsafe webpack configs before invoking the builder', async () => { + const builder = { build: sinon.stub() } + const service = createBuilderService({ cacheRoot, client: {}, builder }) + const invalidConfigs = [null, 1, {}, [], '../config.mjs', '/config.mjs', 'dir/config.mjs', 'dir\\config.mjs', 'config file.mjs', 'config\nfile.mjs', 'config;touch.mjs', 'config$(id).mjs', 'a'.repeat(129)] + for (const config of invalidConfigs) { + let error + try { await service.build({ commit: SHA, path: 'a.js', mode: 'webpack', options: { config } }) } catch (e) { error = e } + expect(error).to.include({ code: 'INVALID_OPTIONS', status: 400, message: 'Build options are invalid' }) + } + expect(builder.build.callCount).to.equal(0) + }) + + it('executes webpack with the config as one argument and no shell', async () => { + const childProcess = require('node:child_process') + const execFile = sinon.stub(childProcess, 'execFile').callsFake((file, args, options, callback) => callback(null, { stdout: '', stderr: '' })) + const utilsPath = require.resolve('../app/utils') + delete require.cache[utilsPath] + try { + const { compileWebpack } = require('../app/utils') + await compileWebpack('/workspace', 'custom.webpack.mjs') + expect(execFile.firstCall.args.slice(0, 3)).to.deep.equal([ + 'npx', + ['webpack', '-c', join('tools/webpacks', 'custom.webpack.mjs'), '--stats', 'errors-only'], + { timeout: 15000, cwd: '/workspace', shell: false } + ]) + } finally { + execFile.restore() + delete require.cache[utilsPath] + } + }) + it('streams an archive into tar, validates roots, atomically caches, and hits cache', async () => { for (const path of SOURCE_PATHS) { await fs.promises.mkdir(join(sourceRoot, path), { recursive: true }) await fs.promises.writeFile(join(sourceRoot, path, 'sample'), path) } const archive = join(sourceRoot, 'source.tar.gz') - execFileSync('tar', ['-czf', archive, '--', ...SOURCE_PATHS], { cwd: sourceRoot }) + execFileSync('tar', ['--format', 'ustar', '-czf', archive, '--', ...SOURCE_PATHS], { cwd: sourceRoot, env: { ...process.env, COPYFILE_DISABLE: '1' } }) const client = { stream: sinon.stub().callsFake(() => fs.createReadStream(archive)) } const service = createBuilderService({ cacheRoot, client, builder: {} }) expect(await service.ensureSource(SHA)).to.equal(join(cacheRoot, SHA)) @@ -100,7 +199,7 @@ describe('builder service', () => { it('removes partial extraction after required-root failure', async () => { await fs.promises.mkdir(join(sourceRoot, 'ts')) const archive = join(sourceRoot, 'bad.tar.gz') - execFileSync('tar', ['-czf', archive, 'ts'], { cwd: sourceRoot }) + execFileSync('tar', ['--format', 'ustar', '-czf', archive, 'ts'], { cwd: sourceRoot, env: { ...process.env, COPYFILE_DISABLE: '1' } }) const service = createBuilderService({ cacheRoot, client: { stream: () => fs.createReadStream(archive) }, builder: {} }) let error try { await service.ensureSource(SHA) } catch (e) { error = e } @@ -108,6 +207,93 @@ describe('builder service', () => { expect(await fs.promises.readdir(cacheRoot)).to.have.length(0) }) + it('rejects compressed and inflated archive overflows without promotion', async () => { + await expectArchiveFailure(cacheRoot, Buffer.alloc(32 * 1024 * 1024 + 1)) + const files = Array.from({ length: 17 }, (_, index) => ({ name: `ts/large-${index}`, data: Buffer.alloc(4 * 1024 * 1024) })) + await expectArchiveFailure(cacheRoot, sourceArchive(files)) + }) + + it('rejects excessive entries and regular files without promotion', async () => { + await expectArchiveFailure(cacheRoot, sourceArchive([{ name: 'ts/large', size: 4 * 1024 * 1024 + 1 }])) + const entries = Array.from({ length: 5001 }, (_, index) => ({ name: `ts/e-${index}`, data: '' })) + await expectArchiveFailure(cacheRoot, tarArchive(entries)) + }) + + it('rejects non-canonical, duplicate, and disallowed archive paths', async () => { + for (const entries of [ + [{ name: '../escape', data: '' }], + [{ name: '/absolute', data: '' }], + [{ name: 'ts\\backslash', data: '' }], + [{ name: 'ts//empty', data: '' }], + [{ name: 'ts/./dot', data: '' }], + [{ name: 'ts/../escape', data: '' }], + [{ name: 'ts/duplicate', data: '' }, { name: 'ts/duplicate', data: '' }], + [{ name: 'docs/readme', data: '' }] + ]) await expectArchiveFailure(cacheRoot, tarArchive(entries)) + }) + + it('rejects links, special entries, and unsupported tar extensions', async () => { + for (const type of ['1', '2', '3', '4', '6', 'x', 'g', 'L', 'K', 'S']) { + await expectArchiveFailure(cacheRoot, tarArchive([{ name: 'ts/unsupported', type }])) + } + }) + + it('keeps the downloader timeout active after response headers', async () => { + const fetch = (url, options) => { + const body = new PassThrough() + options.signal.addEventListener('abort', () => body.destroy(new Error('aborted'))) + return Promise.resolve({ ok: true, body }) + } + const client = createServiceClient({ baseURL: 'http://downloader', token: 'secret', timeout: 5, fetch }) + const stream = await client.stream('/v1/source', { timeoutThroughBody: true }) + let error + let bytes = 0 + try { + for await (const chunk of stream) bytes += chunk.length + } catch (caught) { + error = caught + } + expect(bytes).to.equal(0) + expect(error).to.include({ code: 'SERVICE_TIMEOUT', status: 504 }) + }) + + it('rejects unsafe extracted nodes and sizes before promotion', async () => { + const cases = [ + async root => fs.promises.symlink('/tmp', join(root, 'ts/link')), + async root => execFileSync('mkfifo', [join(root, 'ts/fifo')]), + async root => { + await fs.promises.rm(join(root, 'ts'), { recursive: true }) + await fs.promises.writeFile(join(root, 'ts'), 'not a source directory') + }, + async root => { + for (let index = 0; index < 17; index++) { + const file = await fs.promises.open(join(root, `ts/large-${index}`), 'w') + await file.truncate(4 * 1024 * 1024) + await file.close() + } + } + ] + for (const mutate of cases) { + const spawn = (command, args) => { + const child = new EventEmitter() + child.stderr = new PassThrough() + child.kill = sinon.stub() + process.nextTick(async () => { + try { + const root = args[3] + for (const path of SOURCE_PATHS) await fs.promises.mkdir(join(root, path), { recursive: true }) + await mutate(root) + child.emit('close', 0) + } catch (error) { + child.emit('error', error) + } + }) + return child + } + await expectArchiveFailure(cacheRoot, sourceArchive(), { spawn }) + } + }) + it('returns build metadata and structured failures', async () => { for (const path of SOURCE_PATHS) await fs.promises.mkdir(join(cacheRoot, SHA, path), { recursive: true }) const output = join(cacheRoot, SHA, 'result.js') @@ -124,6 +310,8 @@ describe('builder service', () => { expect(result.status).to.equal(200) expect(result.body.toString()).to.equal('built') expect(result.headers['x-built-with']).to.equal('webpack') + expect(result.headers['x-correlation-id']).to.be.a('string') + expect(fs.existsSync(join(cacheRoot, SHA, '.complete'))).to.equal(true) service.build = () => Promise.reject(Object.assign(new Error('full'), { name: 'QueueFullError', code: 'QUEUE_FULL', status: 503 })) const failed = await request(app, '/v1/build', { method: 'POST', headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, body: {} }) @@ -282,10 +470,57 @@ describe('builder service', () => { it('cleans only the builder cache', async () => { const path = join(cacheRoot, SHA) await fs.promises.mkdir(path) + await fs.promises.writeFile(join(path, '.complete'), '') const service = createBuilderService({ cacheRoot, client: {}, builder: {}, cacheLifetime: 1 }) const old = new Date(Date.now() - 10000) await fs.promises.utimes(path, old, old) + await fs.promises.utimes(join(path, '.complete'), old, old) expect(await service.cleanup()).to.deep.equal([SHA]) expect(fs.existsSync(sourceRoot)).to.equal(true) }) + + it('exposes bounded operations data and protects active build cache entries', async () => { + const root = join(cacheRoot, SHA) + for (const path of SOURCE_PATHS) await fs.promises.mkdir(join(root, path), { recursive: true }) + const output = join(root, 'result.js') + await fs.promises.writeFile(output, 'built') + await fs.promises.writeFile(join(root, '.complete'), '') + let finishBuild + let markBuildStarted + const buildStarted = new Promise(resolve => { markBuildStarted = resolve }) + const service = createBuilderService({ + cacheRoot, + client: {}, + queue: { getMetrics: () => ({ active: 1, queued: 0, limit: 2, oldestQueuedAgeMs: null }) }, + builder: { + build: () => new Promise(resolve => { + finishBuild = () => resolve({ file: output, builtWith: 'webpack' }) + markBuildStarted() + }) + } + }) + const app = createApp({ token: 'secret', service }) + const building = request(app, '/v1/build', { + method: 'POST', + headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json', 'X-Correlation-ID': 'builder-request' }, + body: { commit: SHA, path: 'result.js', mode: 'webpack' } + }) + await buildStarted + + const operation = await request(app, '/v1/ops/cache-operations', { + method: 'POST', + headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, + body: { operation: 'cache.evict_commit', targets: ['builder'], commit: SHA } + }) + expect(JSON.parse(operation.body).targets[0].skippedInUse).to.equal(1) + finishBuild() + await building + + const snapshot = await request(app, '/v1/ops/snapshot', { headers: { Authorization: 'Bearer secret' } }) + const body = JSON.parse(snapshot.body) + expect(body).to.include({ schemaVersion: 1, service: 'builder' }) + expect(body.queues[0]).to.include({ name: 'build', active: 1, limit: 2 }) + expect(body.activity[0].request).to.deep.include({ commit: SHA, resource: null, buildMode: 'webpack' }) + expect(JSON.stringify(body)).not.to.include(cacheRoot) + }) }) diff --git a/test/download.js b/test/download.js index 85fdd2a..dee46f5 100644 --- a/test/download.js +++ b/test/download.js @@ -1,10 +1,47 @@ const defaults = require('../app/download.js') const { token } = require('../config.json') const { expect } = require('chai') +const { EventEmitter } = require('events') const fs = require('fs') +const https = require('https') const { after, afterEach, before, describe, it } = require('mocha') const sinon = require('sinon') +function loadDownloadWithHttpsGet (httpsGet) { + const downloadPath = require.resolve('../app/download.js') + const cachedDownload = require.cache[downloadPath] + const originalHttpsGet = https.get + + delete require.cache[downloadPath] + https.get = httpsGet + try { + return require('../app/download.js') + } finally { + https.get = originalHttpsGet + delete require.cache[downloadPath] + if (cachedDownload) { + require.cache[downloadPath] = cachedDownload + } + } +} + +function fakeHttpsGet (chunks) { + return (options, callback) => { + const request = new EventEmitter() + request.end = () => { + const response = new EventEmitter() + response.statusCode = 200 + response.headers = {} + callback(response) + process.nextTick(() => { + chunks.forEach(chunk => response.emit('data', chunk)) + response.emit('end') + }) + } + return request + } +} + describe('download.js', () => { describe('exported properties', () => { const functions = [ @@ -51,9 +88,22 @@ describe('download.js', () => { return httpsGetPromise(downloadURL) .then(({ body, statusCode }) => { expect(statusCode).to.equal(404) - expect(body).to.equal('404: Not Found') + expect(body.toString('utf8')).to.equal('404: Not Found') }) }) + it('should preserve arbitrary response bytes', async () => { + const bytes = Buffer.from([0, 0xff, 0xfe, 0xfd, 0x61, 0xc3, 0x28]) + const { httpsGetPromise } = loadDownloadWithHttpsGet(fakeHttpsGet([ + bytes.subarray(0, 3), + bytes.subarray(3) + ])) + + const { body, statusCode } = await httpsGetPromise('https://example.test/binary') + + expect(statusCode).to.equal(200) + expect(Buffer.isBuffer(body)).to.equal(true) + expect(body.equals(bytes)).to.equal(true) + }) }) describe('downloadFile', () => { @@ -63,6 +113,7 @@ describe('download.js', () => { [ 'tmp/test/downloaded-file1.js', 'tmp/test/downloaded-file2.js', + 'tmp/test/downloaded-binary.bin', 'tmp/test' ].forEach(p => { try { @@ -114,6 +165,19 @@ describe('download.js', () => { expect(e.message).to.not.equal('Promise resolved unexpectedly.') }) }) + it('should write arbitrary response bytes unchanged', async () => { + const outputPath = './tmp/test/downloaded-binary.bin' + const bytes = Buffer.from([0xde, 0xad, 0x00, 0xff, 0xfe, 0x62]) + const { downloadFile } = loadDownloadWithHttpsGet(fakeHttpsGet([ + bytes.subarray(0, 2), + bytes.subarray(2) + ])) + + const result = await downloadFile('https://example.test/binary', outputPath) + + expect(result).to.include({ outputPath, statusCode: 200, success: true }) + expect(fs.readFileSync(outputPath).equals(bytes)).to.equal(true) + }) }) describe('downloadFiles', () => { it('is missing tests') @@ -134,7 +198,7 @@ describe('download.js', () => { it('allows root 404s only for optional source folders', async () => { const stub = sinon.stub().callsFake(({ path }) => Promise.resolve({ statusCode: /\/contents\/(js|tools\/webpacks|tools\/libs)\?/.test(path) ? 404 : 200, - body: '[]', + body: Buffer.from('[]'), headers: {} })) __setGitHubRequest(stub) @@ -147,7 +211,7 @@ describe('download.js', () => { for (const [failedPath, statusCode] of [['ts', 404], ['js', 403]]) { __setGitHubRequest(sinon.stub().callsFake(({ path }) => Promise.resolve({ statusCode: path.includes(`/contents/${failedPath}?`) ? statusCode : 200, - body: '[]', + body: Buffer.from('[]'), headers: {} }))) let error @@ -177,7 +241,7 @@ describe('download.js', () => { it('reuses the same request for parallel branch lookups', async () => { const stub = sinon.stub().resolves({ statusCode: 200, - body: JSON.stringify({ commit: { sha: 'abc123' } }) + body: Buffer.from(JSON.stringify({ commit: { sha: 'abc123' } })) }) __clearGitHubCache() @@ -196,7 +260,7 @@ describe('download.js', () => { it('returns cached commit info on subsequent calls', async () => { const stub = sinon.stub().resolves({ statusCode: 200, - body: JSON.stringify({ sha: 'deadbeef' }) + body: Buffer.from(JSON.stringify({ sha: 'deadbeef' })) }) __clearGitHubCache() diff --git a/test/downloader-service.js b/test/downloader-service.js index 1a3d36f..84c40b7 100644 --- a/test/downloader-service.js +++ b/test/downloader-service.js @@ -14,6 +14,14 @@ const { createApp, start } = require('../downloader-server') const SHA = '0123456789abcdef0123456789abcdef01234567' +function deferred () { + const result = {} + result.promise = new Promise((resolve, reject) => { + Object.assign(result, { reject, resolve }) + }) + return result +} + function request (app, path, options = {}) { return new Promise((resolve, reject) => { const server = app.listen(0, () => { @@ -54,21 +62,25 @@ describe('downloader service', () => { }) it('exposes unauthenticated health and protects all v1 routes', async () => { - const app = createApp({ token: 'secret', service: {} }) + const service = { + resolveRef: sinon.stub(), + snapshot: sinon.stub(), + cacheManager: { execute: sinon.stub() } + } + const app = createApp({ token: 'secret', service }) + const unauthorized = { error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } } expect((await request(app, '/health')).status).to.equal(200) - expect((await request(app, '/v1/cleanup', { method: 'POST' })).status).to.equal(401) - expect((await request(app, '/v1/cleanup', { - method: 'POST', - headers: { Authorization: 'Bearer wrong' } - })).status).to.equal(401) - expect((await request(app, '/v1/cleanup', { - method: 'POST', - headers: { Authorization: 'Bearer secrets' } - })).status).to.equal(401) - expect((await request(app, '/v1/cleanup', { - method: 'POST', - headers: { Authorization: 'Bearer wrong!' } - })).status).to.equal(401) + for (const authorization of [undefined, 'Basic secret', 'Bearer ', 'Bearer x', 'Bearer wrong!', 'Bearer secrets']) { + const headers = authorization === undefined ? {} : { Authorization: authorization } + for (const [path, method] of [['/v1/resolve', 'POST'], ['/v1/ops/snapshot', 'GET']]) { + const response = await request(app, path, { method, headers }) + expect(response.status).to.equal(401) + expect(JSON.parse(response.body)).to.deep.equal(unauthorized) + } + } + expect(service.resolveRef.callCount).to.equal(0) + expect(service.snapshot.callCount).to.equal(0) + expect(service.cacheManager.execute.callCount).to.equal(0) }) it('fails startup without an internal service token', () => { @@ -129,7 +141,7 @@ describe('downloader service', () => { }) expect(response.status).to.equal(429) - expect(JSON.parse(response.body)).to.deep.equal({ error: { code: 'RATE_LIMITED', message: 'GitHub returned 403' } }) + expect(JSON.parse(response.body)).to.deep.equal({ error: { code: 'RATE_LIMITED', message: 'GitHub rate limit is exhausted' } }) expect(response.headers).to.include({ 'x-github-ratelimit-limit': '5000', 'x-github-ratelimit-remaining': '0', @@ -152,20 +164,68 @@ describe('downloader service', () => { }) it('returns structured timeout errors', async () => { - const clock = sinon.useFakeTimers() - const fetch = (url, options) => new Promise((resolve, reject) => { - options.signal.addEventListener('abort', () => reject(new Error('aborted'))) + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }) + try { + let fetchStarted + const started = new Promise(resolve => { fetchStarted = resolve }) + const fetch = (url, options) => new Promise((resolve, reject) => { + fetchStarted() + options.signal.addEventListener('abort', () => reject(new Error('aborted'))) + }) + const service = createDownloaderService({ cacheRoot, fetch }) + const pending = service.needsEsbuild(SHA).catch(error => error) + await started + clock.tick(ESBUILD_DETECTION_TIMEOUT - 1) + expect(await Promise.race([pending, Promise.resolve('pending')])).to.equal('pending') + clock.tick(1) + const error = await pending + expect(error).to.include({ code: 'UPSTREAM_TIMEOUT', status: 504 }) + } finally { + clock.restore() + } + }) + + it('shares one concurrent esbuild detection request and success', async () => { + const started = deferred() + const response = deferred() + const fetch = sinon.stub().callsFake(() => { + started.resolve() + return response.promise + }) + const service = createDownloaderService({ cacheRoot, fetch }) + + const first = service.needsEsbuild(SHA) + const second = service.needsEsbuild(SHA) + await started.promise + expect(fetch.callCount).to.equal(1) + response.resolve({ ok: true, status: 200 }) + + expect(await Promise.all([first, second])).to.deep.equal([true, true]) + expect(fetch.callCount).to.equal(1) + }) + + it('shares sanitized esbuild failures and retries after failure', async () => { + const started = deferred() + const response = deferred() + const fetch = sinon.stub() + fetch.onFirstCall().callsFake(() => { + started.resolve() + return response.promise }) + fetch.onSecondCall().resolves({ ok: false, status: 404 }) const service = createDownloaderService({ cacheRoot, fetch }) - const pending = service.needsEsbuild(SHA).catch(error => error) - clock.tick(ESBUILD_DETECTION_TIMEOUT - 1) - await Promise.resolve() - expect(await Promise.race([pending, Promise.resolve('pending')])).to.equal('pending') - clock.tick(1) - await Promise.resolve() - const error = await pending - clock.restore() - expect(error).to.include({ code: 'UPSTREAM_TIMEOUT', status: 504 }) + + const first = service.needsEsbuild(SHA).catch(error => error) + const second = service.needsEsbuild(SHA).catch(error => error) + await started.promise + expect(fetch.callCount).to.equal(1) + response.reject(new Error('sensitive upstream failure')) + const errors = await Promise.all([first, second]) + + expect(errors[0]).to.equal(errors[1]) + expect(errors[0]).to.include({ code: 'UPSTREAM_ERROR', status: 502, message: 'GitHub request failed' }) + expect(await service.needsEsbuild(SHA)).to.equal(false) + expect(fetch.callCount).to.equal(2) }) it('creates absent optional roots and marks complete only after success', async () => { @@ -183,7 +243,63 @@ describe('downloader service', () => { expect(fs.existsSync(join(root, '.complete'))).to.equal(true) }) + it('shares one concurrent source population attempt and success', async () => { + const started = deferred() + const downloadComplete = deferred() + const source = sinon.stub(download, 'downloadSourceFolder').callsFake(async root => { + started.resolve() + await downloadComplete.promise + await fs.promises.mkdir(join(root, 'ts'), { recursive: true }) + await fs.promises.mkdir(join(root, 'css'), { recursive: true }) + return [] + }) + const service = createDownloaderService({ cacheRoot }) + + const first = service.ensureSource(SHA) + const second = service.ensureSource(SHA) + await started.promise + expect(source.callCount).to.equal(1) + downloadComplete.resolve() + + expect(await Promise.all([first, second])).to.deep.equal([join(cacheRoot, SHA), join(cacheRoot, SHA)]) + expect(source.callCount).to.equal(1) + }) + + it('shares sanitized source failures and retries after failure', async () => { + const started = deferred() + const downloadComplete = deferred() + const source = sinon.stub(download, 'downloadSourceFolder') + source.onFirstCall().callsFake(async () => { + started.resolve() + await downloadComplete.promise + throw new Error('sensitive source failure') + }) + source.onSecondCall().callsFake(async root => { + await fs.promises.mkdir(join(root, 'ts'), { recursive: true }) + await fs.promises.mkdir(join(root, 'css'), { recursive: true }) + return [] + }) + const service = createDownloaderService({ cacheRoot }) + + const first = service.ensureSource(SHA).catch(error => error) + const second = service.ensureSource(SHA).catch(error => error) + await started.promise + expect(source.callCount).to.equal(1) + downloadComplete.resolve() + const errors = await Promise.all([first, second]) + + expect(errors[0]).to.equal(errors[1]) + expect(errors[0]).to.include({ code: 'INTERNAL_ERROR', status: 500, message: 'An internal error occurred' }) + expect(await service.ensureSource(SHA)).to.equal(join(cacheRoot, SHA)) + expect(source.callCount).to.equal(2) + }) + it('removes partial source and does not mark complete after failure', async () => { + const root = join(cacheRoot, SHA) + const file = join(root, '.files', 'js/sample.js') + await fs.promises.mkdir(join(file, '..'), { recursive: true }) + await fs.promises.writeFile(file, 'cached file') + await fs.promises.writeFile(join(root, '.complete'), '') sinon.stub(download, 'downloadSourceFolder').callsFake(async root => { await fs.promises.mkdir(join(root, 'ts'), { recursive: true }) throw Object.assign(new Error('forbidden'), { statusCode: 403 }) @@ -193,8 +309,11 @@ describe('downloader service', () => { try { await service.ensureSource(SHA) } catch (e) { error = e } - expect(error).to.include({ statusCode: 403 }) - expect(fs.existsSync(join(cacheRoot, SHA))).to.equal(false) + expect(error).to.include({ code: 'INTERNAL_ERROR', status: 500, message: 'An internal error occurred' }) + expect(fs.existsSync(join(root, 'ts'))).to.equal(false) + expect(fs.existsSync(join(root, '.source-complete'))).to.equal(false) + expect(fs.existsSync(join(root, '.complete'))).to.equal(true) + expect(await fs.promises.readFile(file, 'utf8')).to.equal('cached file') }) it('rejects invalid SHAs, unsafe paths, and retains queue-full identity', async () => { @@ -244,7 +363,7 @@ describe('downloader service', () => { expect(file.callCount).to.equal(1) expect(file.firstCall.args[0]).to.match(new RegExp(`${SHA}/js/sample\\.js$`)) expect(source.callCount).to.equal(0) - expect(fs.existsSync(join(cacheRoot, SHA))).to.equal(false) + expect(fs.existsSync(join(cacheRoot, SHA, '.complete'))).to.equal(true) }) it('preserves static file upstream errors', async () => { @@ -254,7 +373,7 @@ describe('downloader service', () => { try { await service.openFile(SHA, 'js/sample.js') } catch (e) { error = e } - expect(error).to.include({ code: 'UPSTREAM_ERROR', status: 502, message: 'GitHub returned 403' }) + expect(error).to.include({ code: 'UPSTREAM_ERROR', status: 502, message: 'GitHub request failed' }) expect(download.getRateLimitState()).to.deep.equal({ remaining: undefined, reset: undefined, limited: false, retryAfter: undefined }) }) @@ -283,11 +402,52 @@ describe('downloader service', () => { it('cleans only expired downloader cache entries', async () => { const old = join(cacheRoot, SHA) - await fs.promises.mkdir(old) + await fs.promises.mkdir(join(old, '.files'), { recursive: true }) + await fs.promises.writeFile(join(old, '.files', 'cached'), 'data') + await fs.promises.writeFile(join(old, '.complete'), '') const timestamp = new Date(Date.now() - 10000) await fs.promises.utimes(old, timestamp, timestamp) + await fs.promises.utimes(join(old, '.complete'), timestamp, timestamp) const service = createDownloaderService({ cacheRoot, cacheLifetime: 1 }) expect(await service.cleanup()).to.deep.equal([SHA]) expect(fs.existsSync(old)).to.equal(false) }) + + it('exposes a bounded authenticated ops snapshot and named cache operations', async () => { + const root = join(cacheRoot, SHA) + await fs.promises.mkdir(join(root, '.files'), { recursive: true }) + await fs.promises.writeFile(join(root, '.files', 'cached'), 'data') + await fs.promises.writeFile(join(root, '.complete'), '') + const app = createApp({ token: 'secret', cacheRoot }) + + const snapshot = await request(app, '/v1/ops/snapshot', { + headers: { Authorization: 'Bearer secret', 'X-Correlation-ID': 'snapshot-id' } + }) + expect(snapshot.status).to.equal(200) + expect(snapshot.headers['x-correlation-id']).to.equal('snapshot-id') + expect(JSON.parse(snapshot.body)).to.include({ schemaVersion: 1, service: 'downloader' }) + expect(JSON.parse(snapshot.body).cache.entries[0].commit).to.equal(SHA) + + const operation = await request(app, '/v1/ops/cache-operations', { + method: 'POST', + headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, + body: { operation: 'cache.evict_commit', targets: ['downloader'], commit: SHA } + }) + expect(operation.status).to.equal(200) + expect(JSON.parse(operation.body).targets[0]).to.include({ service: 'downloader', outcome: 'completed', removedEntries: 1 }) + }) + + it('keeps a complete logical entry in use for the full file response lifetime', async () => { + const root = join(cacheRoot, SHA) + for (const path of SOURCE_PATHS) await fs.promises.mkdir(join(root, path), { recursive: true }) + await fs.promises.writeFile(join(root, 'js/sample.js'), 'raw source') + await fs.promises.writeFile(join(root, '.complete'), '') + const service = createDownloaderService({ cacheRoot }) + + const stream = await service.openFile(SHA, 'js/sample.js') + expect((await service.cacheManager.snapshot()).entries[0].inUse).to.equal(1) + expect(await service.cacheManager.execute('cache.evict_commit', SHA)).to.include({ outcome: 'no_op', skippedInUse: 1 }) + for await (const chunk of stream) expect(chunk.toString()).to.equal('raw source') + expect(await service.cacheManager.execute('cache.evict_commit', SHA)).to.include({ outcome: 'completed', removedEntries: 1 }) + }) }) diff --git a/test/fixtures/mtls/ca.crt b/test/fixtures/mtls/ca.crt new file mode 100644 index 0000000..146efcc --- /dev/null +++ b/test/fixtures/mtls/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVzCCAj+gAwIBAgIUXIdQnSu6cDMuCpar0PediZHbfO4wDQYJKoZIhvcNAQEL +BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ +RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAyMTAwLgYD +VQQDDCdISUdIQ0hBUlRTIFRFU1QgVFJVU1RFRCBQUk9YWSBDTElFTlQgQ0EwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKNjsRMjjamhr+FqIY6LBjcUkm +drJSUIV7MBZx7cmYShGspWoTYzxJKZcL5jniVq4rCQOjf0ZHc5zOWEtknOc0fQyu +5YlNxLbdcikwfbEny2zk7QDvSSN13XB77OE7lRCLTpRfSM66/O9KzcCBe9qz+ya7 +A3MKtGEhFR3Tk0AS/awLZNFq+YoOoAhwj+38EU0ZH4Mz60kYF9pVIOv3RFgD3Jmd +O4Gv8UB72uv6Wjdwo7tdtRwycHxnS6uODkBJTLB3zkH0Kn+w2u+kqzthl71IAxA2 +cDhoLtf6+dz0Kk94Rbqr4PBOYj4MdnOhEaB7eTNBVw4zVojLNo9p0elkupdNAgMB +AAGjYzBhMB0GA1UdDgQWBBS0NfsJDbelcn2MSv4MH/VTekCM7TAfBgNVHSMEGDAW +gBS0NfsJDbelcn2MSv4MH/VTekCM7TAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB +/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEASBdrQw+ajIXPzRDN6KxP5B+BOvsz +hAmtzK0lmI+OuL6/dEdfWePvYCbV11ygd9nBdTPL3SxA45tEDR8dI6f8wdJqAiPE +HxRK5vcYXWewFBuCl1bFqGFH6BFjshAh3nk5pdbilEG6i6H2eqiCxI91jFoqkr8X +JVB5aEhlODMfm/eZr2MuoqJazKYlFzZGjZEKIayrTUhUFurdliD8j1HZ9TUq8zlc +wmA3DBK4m8EeT4ABTXXdyNmEKlDKOA8YdwLe6mGPw9F1AWdI0Zgi1hc0hr/JCm8o +3cg1nOU+VM5eA3b3dybntOCSfNcMqrkIzwVoiwUVa2Gpdoi5ne/+c3L1pg== +-----END CERTIFICATE----- diff --git a/test/fixtures/mtls/client.crt b/test/fixtures/mtls/client.crt new file mode 100644 index 0000000..e8979ba --- /dev/null +++ b/test/fixtures/mtls/client.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZjCCAk6gAwIBAgIUS1aaNaNolAJyG1LiJHPw7vHw11gwDQYJKoZIhvcNAQEL +BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ +RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAvMS0wKwYD +VQQDDCRISUdIQ0hBUlRTIFRFU1QgVFJVU1RFRCBQUk9YWSBDTElFTlQwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDue8uu1Zlh9BNbHumlLJJMBYj19al +kPAoHD76nzdmWjQt/0SwPtD/gkyxrjspShcnc3RWIB5g6hnnFYMWh9ZMcU7woiYp +2qiLygw0QCGn/F6b4nEuMLaAkca6/iO781gRTFBN/zYhjk5ev1B8bDaPMCvd+BLN +djDNy0PrZy4dxYZSj03H2Ybfzd5xskpePW/a3ifH67r9Mo9mXDBUufhjBSsU8Ray +UWMbFalhHeB2oGfV0B5ZicDtlFQPAjLa8CzA7RdUsBXn0TTPr7uCh33ZJiXTGvcf +mZqWXWQtZT8JLAbHq375kFUaguLckOtWmJlESfnJwpvd0SpVfyWVrm6zAgMBAAGj +dTBzMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsG +AQUFBwMCMB0GA1UdDgQWBBSByyXC8yFoSzuH4//diB1WxVcNADAfBgNVHSMEGDAW +gBS0NfsJDbelcn2MSv4MH/VTekCM7TANBgkqhkiG9w0BAQsFAAOCAQEAIUoSU7vN +MF42QiX3xTpdgwjXGoCA0AKLYgMjPk0JQ1uPByVm0kEqlypXsmo2JOpEc59UqYAJ +4zqyBLu0F0d8oSVNw2GRIKDrWToMyGCQWxawQJd6thkRD+aZxPuwQQ4DfZwxqI/I +4Y74V2QIHNif+/T7OWOrbBGCkvZRB1l+StlbdYYh87vsfXuGa0C2UnnXZnalN6aS +L1B/J7UrAS+QXkZBHA89zceOgaPm4rpEp8wjfzzecHpH430wA818UPcX6NEcMRwp +oshrHeivOiA4tFcNfrKw93om+vguQ+ylnHdlb11FJm6DFsFBLXhFxXrrdHnoOBh9 +Km6EtLJaqdgKJw== +-----END CERTIFICATE----- diff --git a/test/fixtures/mtls/client.key b/test/fixtures/mtls/client.key new file mode 100644 index 0000000..b238a2c --- /dev/null +++ b/test/fixtures/mtls/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDue8uu1Zlh9BN +bHumlLJJMBYj19alkPAoHD76nzdmWjQt/0SwPtD/gkyxrjspShcnc3RWIB5g6hnn +FYMWh9ZMcU7woiYp2qiLygw0QCGn/F6b4nEuMLaAkca6/iO781gRTFBN/zYhjk5e +v1B8bDaPMCvd+BLNdjDNy0PrZy4dxYZSj03H2Ybfzd5xskpePW/a3ifH67r9Mo9m +XDBUufhjBSsU8RayUWMbFalhHeB2oGfV0B5ZicDtlFQPAjLa8CzA7RdUsBXn0TTP +r7uCh33ZJiXTGvcfmZqWXWQtZT8JLAbHq375kFUaguLckOtWmJlESfnJwpvd0SpV +fyWVrm6zAgMBAAECggEANW/9Wo/xTbUf7ROSSu/MIWlMkiqqwvdoajsUAs8XjA1S +s8A/G7N60lfb4qMEKgi9e5rtB1qrkKA5xDq+WJdrreE9wTs0GjdFzyyx2k4sIjYo +Cn1vk0HfggjK7mDWlskgoVBpmHH2cIDu6rVnHyFYYA2x3F+PmqMLPhSiDZJVJ/E8 +uBgEPBO2HlPjMsaz2nLiMw+cpFQTZtEgDFQZCSI9jAMnt0B4mXxmjCsjnSSRrP7m +j35IeOOYWr0CJZkkIt0/t2cFM0eGMW+BtqjRUOLLx10++PLr1LLRuQihS9g895Ae +R4Hr6MbQcIAtA409svbG35USP+XanOdYiJKfU5D1cQKBgQDhz3AU929gLauiYwSF +g9CT9MwmnEDwxB3eE+voZRQzUelieMsPyKp+3alyGAP57uKI3i4BpKArGKICNih5 +i7rxPgmkKcCH+AOYCLM58rGa7ZwZ6PyNbTI5BxJUJtMKyYQhFveXoDF8fiy9c8HX +VAIcON1TPsJAulm2nEMbYmpNiwKBgQDd5Ngq1abh+jpdClhYdYxlsV1zeOM1FeRH +XsT15RlgilLxKk2Lukt7X7FwCdiazt6DH5EsPe6lhaovIQMiBjTsd/YkDvu5IYZ1 +0PNvhk4k0SurxFINDGbNgpym+ZKM15ViyLjGykJB0mUXAYG6vn4jym1h9FDTvO1P +IHHofVhYeQKBgDgYyXZb9e5FykLAKIpmsbVf9iuNW9C0V9soxc1o9vi826bb7U5R +gpGbzZGLh8laYCqyT2mXFTc/mlfETo/Ld7igudJvkOX2ZiYp2ySFNzwO1V3WdI9J +1lU2fYYsUvd2En4J755abJDJ46F5FWnB8/hA5DLe/3EHGmx0K3OtIk17AoGBAMw2 +n8eUR/kzjOEx8yq+TE8PFB2AxUKG+kfA7W4MwfU6eKkhMKsG8g8Ce8/MEAAxoVF2 +DOp1uRu2z3B+Zl667Zwvr2VyMLMqKpBllJUwOtzhcNqtXIJLxpUevsNhb0GV6xM1 +/fBeFupzErxAk79lL7wKwe5jprun5ZNsHclFCripAoGAR+B866iyJmaZtO1hW4aO +d3gHdaquKbguvTYiQHcI85sXiPuJrZhcyWrY3qEpVwBbReAAyUfg4/W80hcr4MA7 +g2zkxiCrLtm1GlgEErpQhpzAW3MpmPq184RU4XjqyiQXxtVTNr8L4QmQaj1W4v/+ +9tHHWQ7ZGbEmFWEng0pcJQc= +-----END PRIVATE KEY----- diff --git a/test/fixtures/mtls/server.crt b/test/fixtures/mtls/server.crt new file mode 100644 index 0000000..4be1571 --- /dev/null +++ b/test/fixtures/mtls/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDejCCAmKgAwIBAgIUS1aaNaNolAJyG1LiJHPw7vHw11cwDQYJKoZIhvcNAQEL +BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ +RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAlMSMwIQYD +VQQDDBpISUdIQ0hBUlRTIFRFU1QgT1BTIFNFUlZFUjCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAJVHcT35zgp7NQF98XUgLXbsjB+RfOkpjbtx50cTvFH5 +Fx0pCm1aPW4mbY5UMx2KYmxE4A+nDyLVbihAZOt+tka7d+qdWT4c3UU/WH49jAN1 +twyJ1F+GDMCx5sM69DvWUlDH48dFHjjsanVa4y9/V5nqimyNsZxvR0S12ggbXSot +ZfsY/GKiwgzl76l9UZbZbMBQ+3dSiTBDw+CxNTb894Up8GY+LhOBBEOk00ssCBCO +WYXD+HFRGKtEQQmluyBMQZU5mgTVYGZfd6dPZC1eLXWrIK+CvyqJBc/D7gyMlNyo +tQHtbJ4gGR32eZv06my1dvPZlcd8sM2ZXkSh6xFPSTcCAwEAAaOBkjCBjzAaBgNV +HREEEzARgglsb2NhbGhvc3SHBH8AAAEwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8E +BAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYEFIHcbY+KiAWVTzKw +qWHq3RjBZmbkMB8GA1UdIwQYMBaAFLQ1+wkNt6VyfYxK/gwf9VN6QIztMA0GCSqG +SIb3DQEBCwUAA4IBAQAHuPi95GdRSa2DB6fvO1w6oUZRbfGqsjF/RkiQXYK16IpT +8qbTXUF4cB1PiGJZng+lS1z5xSgQBtSN6sK3hqWLQe+ZwjKasOrvK+0clZn3cwpy +u7BaGegpaxie9/dQOOuI/2CfmET/vjjueH8VW7nvEneKftdSdWc8uyTWlFcOONiu +5GtmIz6UDlvzwoiIymb+MCRbyogWsBM+v7/yIMU4hG28eG2C+24hwEsMqI30wFNA +UeEgZkUeoZXAoRQqaCx8G/edY3ZzxFLZwrecZLDCmLAx3nh0CoWJ+lUrYggPo0OR +vN7f0eQ2B9LqwHBs55o3DQSNZ5T2Esl1SVmt5S85 +-----END CERTIFICATE----- diff --git a/test/fixtures/mtls/server.key b/test/fixtures/mtls/server.key new file mode 100644 index 0000000..bf63e3c --- /dev/null +++ b/test/fixtures/mtls/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCVR3E9+c4KezUB +ffF1IC127IwfkXzpKY27cedHE7xR+RcdKQptWj1uJm2OVDMdimJsROAPpw8i1W4o +QGTrfrZGu3fqnVk+HN1FP1h+PYwDdbcMidRfhgzAsebDOvQ71lJQx+PHRR447Gp1 +WuMvf1eZ6opsjbGcb0dEtdoIG10qLWX7GPxiosIM5e+pfVGW2WzAUPt3UokwQ8Pg +sTU2/PeFKfBmPi4TgQRDpNNLLAgQjlmFw/hxURirREEJpbsgTEGVOZoE1WBmX3en +T2QtXi11qyCvgr8qiQXPw+4MjJTcqLUB7WyeIBkd9nmb9OpstXbz2ZXHfLDNmV5E +oesRT0k3AgMBAAECggEACqx3DmVkNB+nJJoqv6MzXQOA6Wjvs4RDHBoC3XQtzPaw +jmc21abUKaIZx0mB9iTE8NTj6HqbfHQiUkJ4dNY0lk4lPuNNVEGgGKl03GHuNvkd +w4m/Y60kEEsoIuF+QSEL9ba1NLReetd8rTN4dxb13o2EpEplXrgeMm2GT3oVaUSF +w7t42suo6bFt07+ARVGgw298UAI3M82Ey6v7WiWDae5uBRSiT2SbfrWcvcv1El9n +oFKEPJbdmqkKfWOpQgvnp0zqJ7L0xXkRJuUYWBa/zoPeiRIOQjU4Vk5X69yZuLo1 +iD3xL+x/WMVmDrmGjht8IfU1PbwH/zapF59JP3oJkQKBgQDJ9Egj4YA/ALPeTl0u +i1XFrOVj45gE5rnEhiFtXIzy5rm4LupsybXRhfAEyOZ78tcwfl9hzq1gMh3xoj7m +3BcrlaBOUjKL5Bt7XLQJUSdqz+yFbAeY8Cr2/9W4DzSQLlnHpDC/PEEWJAsx3wPK +nFqRibYrJxi6c/sBxlEgMSx8wwKBgQC9Om3SlvWDFT7gq5Qr9arfgYrazPPsFbWr +YDNmDp4KjKMjX7ffOXawwcw3+ofSWnBb2ba4/ggDH/T4aHxtd5snFi9Xdusit3wc +ooyJc0NUxTrHko3epbDZMM7zHDyB0npKNsJj20cthNvwNnaua50YeGDdhK2CkvHu +A7QQUwJKfQKBgQCeid54DHaY/vw08F/GQiu7WtdZaznT3yzGUmW7bIRZyzbQmEP/ +0vmg2fxqRSxq8WBs+Uf3iEAi3DUVk8C9itnFpViLI4v6tb+9QDE1fzfqaf/LXds4 +/JE+BejI7WbeKQTh7Ms31R1jPDhtlh1r2QJgbjNL/Q00kgfihMT6+J9r8QKBgCls +AuJYXUHmgq8XoAXHbzIh301qE/MYBX6QPnAWvw28H3H83/kjURH8OkH+u4CWf4X7 +sH3qTcKxWiSOar5jsjjqKE7TH0GoPKjgBDeKXbDOw8EwGZIlXwMMJiEdizk348Ef +H4pQU9JpBOQeZ/hiYi8bGski5ABzPjZF5UK1iQjZAoGAMr8VO7+hc3Nd7PGGG7Zg +KIS+Mjy1NzAkVvYfgiuthKcNm4cLDjPFapRz5Y/au4csqIlpJkfczq6u0wlb8tey +jLMQ27eOdSBnNhfHpOBGqqEcE0SWTwZOQw3jEBK08zxB7XdBiUtoXyIldr2oDk55 +jJwS5TYLDAxTaj5JO5qXYC0= +-----END PRIVATE KEY----- diff --git a/test/fixtures/mtls/untrusted-ca.crt b/test/fixtures/mtls/untrusted-ca.crt new file mode 100644 index 0000000..55232eb --- /dev/null +++ b/test/fixtures/mtls/untrusted-ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDTzCCAjegAwIBAgIUBFPgylvNJTthRpraK5pc8sOnxU8wDQYJKoZIhvcNAQEL +BQAwLjEsMCoGA1UEAwwjSElHSENIQVJUUyBURVNUIFVOVFJVU1RFRCBDTElFTlQg +Q0EwIBcNMjYwNzIxMTEwMDIzWhgPMjEyNjA2MjcxMTAwMjNaMC4xLDAqBgNVBAMM +I0hJR0hDSEFSVFMgVEVTVCBVTlRSVVNURUQgQ0xJRU5UIENBMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAm7uVWJ/Y3bToqhX/TplpV+nvjHwihxKpy/pu +oXEnzjyzPk5EeKROn81JCwFfW4NRaM31i1/F/Chboz8J/0VoI9V0PgbFCMIxEI8g +BzyYcmlQmIEom/1bW100keC8CWKdXu1sCz6XDmLqkpbntVP1iLJQoUwyHeJVDZWK +KuW96T/7JkSFanIVkBxOxImXWsIydGqMRYUxcGrkRRjcbggwNtQ/CrinkzGE4433 +Hhlq0qAZk87OVrJEnjT24HfaHmQ70DHeJbE6E859X8Q5qTBVtdsC9JwUNOVUhnKA +C+zlLv921iFdIVZb/ff4yfjkGUiywNDK9wgFMDQKEStJaoVPTwIDAQABo2MwYTAd +BgNVHQ4EFgQUEEvT8qmkVIxZSv7fYkK7XOllClAwHwYDVR0jBBgwFoAUEEvT8qmk +VIxZSv7fYkK7XOllClAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +DQYJKoZIhvcNAQELBQADggEBACwtUa5VGeWsMnWbdejtJ8mX5JVt4GjnUugCxv5A +gGxnr/rfeL4r9cjeBuUydYQoCSYgCwD7n/J2I0yE/0odMRKmuXNT3JCdluvtNwAr +0+tiq0lClanfeO9MTrk9Ri9I7yVgXZAAUypNoxceJ0w4LsJ0jhOLZCT2+UMTuDfh +sn4CORDA16jhgqiLlA2G/+jrOGAmc+4Dmwkl7c221ahaAj+rTMsdWx6XdE/B6Y6w +op9jLP+s8uvhobYyb1QZFXbzToszTiaZyNj7St2M0pFWe3H4Zl3CqqND3IPD3fOq +YB5oBTqYGTOvsLOSqPOp3BKgrgZtoIU4ar9D4oZpwZYBnZM= +-----END CERTIFICATE----- diff --git a/test/fixtures/mtls/untrusted-client.crt b/test/fixtures/mtls/untrusted-client.crt new file mode 100644 index 0000000..30f7ea2 --- /dev/null +++ b/test/fixtures/mtls/untrusted-client.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDXjCCAkagAwIBAgIUDo9v4dFgQ11j9iLP3MmD02+FgwIwDQYJKoZIhvcNAQEL +BQAwLjEsMCoGA1UEAwwjSElHSENIQVJUUyBURVNUIFVOVFJVU1RFRCBDTElFTlQg +Q0EwIBcNMjYwNzIxMTEwMDIzWhgPMjEyNjA2MjcxMTAwMjNaMCsxKTAnBgNVBAMM +IEhJR0hDSEFSVFMgVEVTVCBVTlRSVVNURUQgQ0xJRU5UMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAkR2Yx3codyLyWx+xE9mKrwrdAzQ0zg4jnW0ePUi0 +WuesK/5x3WSzwqK3Opw+MwD7rTQPRc5USsfdViyYHo36VnBP7aiCRHo3F+Le5DQZ +KIrUIfH2USOfG0Guc2wvotvgp2xHThFayySRFrCifnLNbuHeyUwF5e0C7ytrj7mf +YOipS0z/sHoOLwmB3djzQ8f/6XBKaIr15pUZa1PIulWmSCtlFKRVnEljB3sRGatt +EyLDHOqqKC2adZjZYxEvzt6ZJCYj8UuoB3+sF3W2TVttDKiB8sol2mx4fkDLLf8F +8AyIFFDlOk1eWiAI8WEjMUoNugKeLld/ZHuDjVIefMOiMwIDAQABo3UwczAMBgNV +HRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAjAd +BgNVHQ4EFgQUxZb84ATJ7qvD80uffCLDkcMML3AwHwYDVR0jBBgwFoAUEEvT8qmk +VIxZSv7fYkK7XOllClAwDQYJKoZIhvcNAQELBQADggEBAF202eemY9zihOrNCLV1 +RQh+GeKodbZEuv/bv90jj/xJFSdWVeVK97mCF39LVgVf8t3hAxkSh6TYm3NDUSM8 +BRVQ704bRXNAgHbG1Q3eBgUafTCPR7UJpmiqd8zIVQYSM6f7WbEb9n4vJvI5uA05 +rZPHR1tqldqgPRSPBZWpc0Cqm1Slgdrt/GGV0YOJlcJeGOLNGDDcVp6vlowujax5 +meliD+JI0Aq7/m2dVBzEOFs8G5tzgAs8m4dR+w8rtB3VpCNBEpkGJvoeabZX+nPA +fhtzcifa8wDFtT6tIQElN2ZUxZk7SlI0t81YOtXaMMw4oVAoaFORMSnihce0d1QV +HRo= +-----END CERTIFICATE----- diff --git a/test/fixtures/mtls/untrusted-client.key b/test/fixtures/mtls/untrusted-client.key new file mode 100644 index 0000000..24bef50 --- /dev/null +++ b/test/fixtures/mtls/untrusted-client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCRHZjHdyh3IvJb +H7ET2YqvCt0DNDTODiOdbR49SLRa56wr/nHdZLPCorc6nD4zAPutNA9FzlRKx91W +LJgejfpWcE/tqIJEejcX4t7kNBkoitQh8fZRI58bQa5zbC+i2+CnbEdOEVrLJJEW +sKJ+cs1u4d7JTAXl7QLvK2uPuZ9g6KlLTP+weg4vCYHd2PNDx//pcEpoivXmlRlr +U8i6VaZIK2UUpFWcSWMHexEZq20TIsMc6qooLZp1mNljES/O3pkkJiPxS6gHf6wX +dbZNW20MqIHyyiXabHh+QMst/wXwDIgUUOU6TV5aIAjxYSMxSg26Ap4uV39ke4ON +Uh58w6IzAgMBAAECggEAGWlx6igjNW2wvCVaGIxFXW7NEjUPtC/Eq9pCa9/x+WNN +gqy9mtP6KLDe3kwjFkJrUELoE8TUfP3v9Bm+D8e0GXP0gz05hq1SPYQUnSjEaRWa +nVEmXcIbqCXB22OMGfxgJGFxQSoH2MAQCkWnRvZqpCq4nU6LT97H30MexF3wm4MP +LLTe2zvastK0ht/OIc6F4vX3IDk4YS6HScCIDoFT+8ODdIUW4y26BY0KvvHHoHGA ++39TT+88CNqtlier7amgWSF7Uz454NqnyPnm3UN6HGFV9FpFf8dBcBw0apW8cYHd +3oB3tKZ/1e8dLQ5f32zKDQnQaBKU0jMbKvfphlZRlQKBgQDG2IOs+EydFzGvkBum +EnNcjJbCGgNy3UEo+th/fcydTV3aMmHiVo5UsQ3aRJOODMJ16lsEhCElgsy336py +6FjqgF+a8gcQ4EClzYR03FPs8JKrpthua2JofxFIYPMdCDpVkSvIIQkXYDhdJdgj +styf92L59ch7LIribLkSgWhxPwKBgQC604GVjBhi0CQfwtRsCj8KiqMUmVZkCFwP +N02Swfmdsdxh8IjijaTmtxQ9ABxUtXIJbjOjHUh+zk94/eQ1t0ah/HseNg0q17CB +auKGmVcxam0/pVUPxv/GPlfXJXdFJgEqkvmciGduuk915hy2gvlvjHG5khuxyjdV +KTqI1i+eDQKBgEpQbt29KnznRzVy6TJm4OBs1ocSS8mo8QpMvY7gSBFd8b5zZ+u8 +J5N1XFT/nhnPoPmKrZuBehXXzYTGvaAWP+zcL17sJ0HiXabZqTA3i9IYkug+AdM6 +pNui6aWgPhvSkGKcT72KwygOWOJPG3k0BopuXXpHpjaKjhlTQtOss4jnAoGAO8X/ +58Dy7Gp7pE3JeOeuU8kXdDe7FDY4fgXLo/C71GBbilpS8WkwIUDiEMJATjnfSnUf +wEhWMXwAoU1Z/nWs+5a0LW5NuC9aIY34++eDpiPO4FYdaPbaTqsTn5o1FsKaDO0J +P7PEpACn+6ir9xjghXgBysWcZr20BmMyfyVIBuECgYEAmqPBqCesFgX0OYAHUt0D +E4GEW3D86QL7JDzq4F4fe/T5bqPmFYujPczKwK1TkqOyN0U88K6yKVTUF3P0FSXF +pu+wMYVqYFkWlP3rLLEMneZj5zxMWci91Hk0bcgO4gTlH42lyB4B8qcuj4y0MXEa +wapiO8F/nTpJf9MgA0DlT+c= +-----END PRIVATE KEY----- diff --git a/test/hurl/ops-auth.hurl b/test/hurl/ops-auth.hurl new file mode 100644 index 0000000..e156370 --- /dev/null +++ b/test/hurl/ops-auth.hurl @@ -0,0 +1,205 @@ +# Invalid credentials are generic and never reflected. +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" +} +HTTP 401 +[Asserts] +jsonpath "$.error.code" == "UNAUTHORIZED" +body not contains "BBBB" +header "Cache-Control" == "no-store" +header "Content-Security-Policy" == "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self'; connect-src 'self'; font-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'; object-src 'none'" +header "X-Content-Type-Options" == "nosniff" +header "Referrer-Policy" == "no-referrer" +header "Permissions-Policy" == "camera=(), microphone=(), geolocation=()" +header "Cross-Origin-Opener-Policy" == "same-origin" +header "Cross-Origin-Resource-Policy" == "same-origin" +header "X-Frame-Options" == "DENY" +header "Access-Control-Allow-Origin" not exists + +# Loopback mode uses the non-Secure development cookie. +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 201 +[Captures] +old_cookie: header "Set-Cookie" regex "ghhc-console-dev=([^;]+)" +csrf: jsonpath "$.csrfToken" +[Asserts] +header "Set-Cookie" matches /^ghhc-console-dev=[A-Za-z0-9_-]{43}; HttpOnly; SameSite=Strict; Path=\/$/ +header "Set-Cookie" not contains "Secure" +jsonpath "$.authenticated" == true + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{old_cookie}} +HTTP 200 +[Asserts] +jsonpath "$.authenticated" == true + +# Replacement login rotates the session ID and revokes the old session. +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +Cookie: ghhc-console-dev={{old_cookie}} +{ + "token": "{{token}}" +} +HTTP 201 +[Captures] +new_cookie: header "Set-Cookie" regex "ghhc-console-dev=([^;]+)" +csrf: jsonpath "$.csrfToken" +[Asserts] +variable "new_cookie" != "{{old_cookie}}" + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{old_cookie}} +HTTP 401 + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{new_cookie}} +HTTP 200 + +# Server-side token rotation revokes every current session. +POST {{base_url}}/__ops-test/rotate +{} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{new_cookie}} +HTTP 401 + +# Expiry is deterministic through the injected clock. +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 201 +[Captures] +expiring_cookie: header "Set-Cookie" regex "ghhc-console-dev=([^;]+)" + +POST {{base_url}}/__ops-test/clock +Content-Type: application/json +{ + "advanceMs": 1800000 +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{expiring_cookie}} +HTTP 401 + +# Logout clears and revokes the active cookie. +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 201 +[Captures] +logout_cookie: header "Set-Cookie" regex "ghhc-console-dev=([^;]+)" +logout_csrf: jsonpath "$.csrfToken" + +DELETE {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +X-Ops-CSRF: {{logout_csrf}} +Cookie: ghhc-console-dev={{logout_cookie}} +HTTP 204 +[Asserts] +header "Set-Cookie" contains "Max-Age=0" + +GET {{base_url}}/_ops/api/v1/session +Cookie: ghhc-console-dev={{logout_cookie}} +HTTP 401 + +# Origin, Fetch Metadata, content type, CSRF, method, and CORS negatives. +POST {{base_url}}/_ops/api/v1/session +Origin: http://evil.invalid +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 403 +[Asserts] +jsonpath "$.error.code" == "ORIGIN_REJECTED" + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Sec-Fetch-Site: cross-site +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 403 +[Asserts] +jsonpath "$.error.code" == "FETCH_METADATA_REJECTED" + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: text/plain +{ + "token": "{{token}}" +} +HTTP 415 + +HEAD {{base_url}}/_ops/api/v1/session +HTTP 404 + +OPTIONS {{base_url}}/_ops/api/v1/session +HTTP 404 +[Asserts] +header "Access-Control-Allow-Origin" not exists + +# The five-attempt source limit produces a bounded 429 with Retry-After. +POST {{base_url}}/__ops-test/clock +Content-Type: application/json +{ + "advanceMs": 900000 +} +HTTP 200 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 429 +[Asserts] +jsonpath "$.error.code" == "RATE_LIMITED" +header "Retry-After" matches /^\d+$/ diff --git a/test/hurl/ops-cache.hurl b/test/hurl/ops-cache.hurl new file mode 100644 index 0000000..1df457e --- /dev/null +++ b/test/hurl/ops-cache.hurl @@ -0,0 +1,121 @@ +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 201 +[Captures] +csrf: jsonpath "$.csrfToken" + +POST {{base_url}}/__ops-test/reset-audits +{} +HTTP 200 + +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "downloader", + "cache": "healthy", + "resetCounts": true +} +HTTP 200 + +# Named healthy operation completes with one dispatch and one sanitized audit. +POST {{base_url}}/_ops/api/v1/cache-operations +Origin: {{base_url}} +X-Ops-CSRF: {{csrf}} +Content-Type: application/json +{ + "operation": "cache.clear", + "targets": ["downloader"] +} +HTTP 200 +[Asserts] +jsonpath "$.outcome" == "completed" +jsonpath "$.targets[0].outcome" == "completed" +header "Content-Length" toInt < 65536 + +GET {{base_url}}/__ops-test/status +HTTP 200 +[Asserts] +jsonpath "$.audits" == 1 +jsonpath "$.auditSafe" == true +jsonpath "$.downloader.cache" == 1 + +# Wrong Origin and CSRF never dispatch a mutation. +POST {{base_url}}/_ops/api/v1/cache-operations +Origin: http://evil.invalid +X-Ops-CSRF: {{csrf}} +Content-Type: application/json +{"operation":"cache.clear","targets":["downloader"]} +HTTP 403 + +POST {{base_url}}/_ops/api/v1/cache-operations +Origin: {{base_url}} +X-Ops-CSRF: wrong +Content-Type: application/json +{"operation":"cache.clear","targets":["downloader"]} +HTTP 403 + +GET {{base_url}}/__ops-test/status +HTTP 200 +[Asserts] +jsonpath "$.downloader.cache" == 1 +jsonpath "$.audits" == 3 +jsonpath "$.auditSafe" == true + +# An unavailable target fails once without retry. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "builder", + "cache": "disconnect", + "resetCounts": true +} +HTTP 200 + +POST {{base_url}}/_ops/api/v1/cache-operations +Origin: {{base_url}} +X-Ops-CSRF: {{csrf}} +Content-Type: application/json +{"operation":"cache.clear","targets":["builder"]} +HTTP 200 +[Asserts] +jsonpath "$.outcome" == "failed" +jsonpath "$.targets[0].error.code" == "SERVICE_UNAVAILABLE" +header "Content-Length" toInt < 65536 + +GET {{base_url}}/__ops-test/status +HTTP 200 +[Asserts] +jsonpath "$.builder.cache" == 1 +jsonpath "$.audits" == 4 + +# A response beyond ten seconds is ambiguous, bounded, audited, and not retried. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "builder", + "cache": "slow", + "resetCounts": true +} +HTTP 200 + +POST {{base_url}}/_ops/api/v1/cache-operations +Origin: {{base_url}} +X-Ops-CSRF: {{csrf}} +Content-Type: application/json +{"operation":"cache.clear","targets":["builder"]} +HTTP 200 +[Asserts] +jsonpath "$.outcome" == "unknown" +jsonpath "$.targets[0].error.code" == "SERVICE_TIMEOUT" +header "Content-Length" toInt < 65536 + +GET {{base_url}}/__ops-test/status +HTTP 200 +[Asserts] +jsonpath "$.builder.cache" == 1 +jsonpath "$.audits" == 5 +jsonpath "$.auditSafe" == true diff --git a/test/hurl/ops-security.hurl b/test/hurl/ops-security.hurl new file mode 100644 index 0000000..807e69b --- /dev/null +++ b/test/hurl/ops-security.hurl @@ -0,0 +1,33 @@ +# This suite runs after the stack is restarted with the console disabled. +GET {{base_url}}/_ops +HTTP 404 + +GET {{base_url}}/_ops/ +HTTP 404 + +GET {{base_url}}/_ops/login +HTTP 404 + +GET {{base_url}}/_ops/console.js +HTTP 404 + +GET {{base_url}}/_ops/api/v1/session +HTTP 404 + +POST {{base_url}}/_ops/api/v1/session +Content-Type: application/json +{} +HTTP 404 + +GET {{base_url}}/health +HTTP 200 +[Asserts] +body == "OK" + +GET {{base_url}}/ +HTTP 200 +[Asserts] +body contains "github.highcharts.com" + +GET {{base_url}}/cleanup?true +HTTP 200 diff --git a/test/hurl/ops-snapshot.hurl b/test/hurl/ops-snapshot.hurl new file mode 100644 index 0000000..cabccbe --- /dev/null +++ b/test/hurl/ops-snapshot.hurl @@ -0,0 +1,99 @@ +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{ + "token": "{{token}}" +} +HTTP 201 + +# A service down before any success is unknown; the other slot remains fresh. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "builder", + "snapshot": "disconnect", + "resetCounts": true +} +HTTP 200 + +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "downloader", + "snapshot": "fresh", + "resetCounts": true +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/snapshot +HTTP 200 +[Asserts] +jsonpath "$.schemaVersion" == 1 +jsonpath "$.services.downloader.freshness" == "fresh" +jsonpath "$.services.builder.freshness" == "unknown" +jsonpath "$.services.builder.error.code" == "SERVICE_UNAVAILABLE" +header "Content-Length" toInt < 1048576 + +# Recovery is visible on the next refresh. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "builder", + "snapshot": "fresh" +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/snapshot +HTTP 200 +[Asserts] +jsonpath "$.services.downloader.freshness" == "fresh" +jsonpath "$.services.builder.freshness" == "fresh" +jsonpath "$.services.downloader.snapshot.cache.entries" count == 1 + +# The 1.5 second downloader timeout is independent; builder remains fresh and +# the prior downloader success becomes stale. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "downloader", + "snapshot": "slow" +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/snapshot +HTTP 200 +[Asserts] +jsonpath "$.services.downloader.freshness" == "stale" +jsonpath "$.services.downloader.error.code" == "SERVICE_TIMEOUT" +jsonpath "$.services.builder.freshness" == "fresh" + +# Oversized and malformed internal payloads are bounded and never forwarded. +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "downloader", + "snapshot": "oversized" +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/snapshot +HTTP 200 +[Asserts] +jsonpath "$.services.downloader.freshness" == "stale" +jsonpath "$.services.downloader.error.code" == "SERVICE_RESPONSE_TOO_LARGE" +body not contains "padding" +header "Content-Length" toInt < 1048576 + +POST {{base_url}}/__ops-test/fixture +Content-Type: application/json +{ + "service": "downloader", + "snapshot": "malformed" +} +HTTP 200 + +GET {{base_url}}/_ops/api/v1/snapshot +HTTP 200 +[Asserts] +jsonpath "$.services.downloader.freshness" == "stale" +body not contains "not-json" diff --git a/test/ops-cache.js b/test/ops-cache.js new file mode 100644 index 0000000..e19a688 --- /dev/null +++ b/test/ops-cache.js @@ -0,0 +1,218 @@ +'use strict' + +const assert = require('node:assert/strict') +const { promises: fs } = require('node:fs') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { afterEach, describe, it } = require('mocha') +const { CacheError, CacheManager, parseCommit } = require('../app/ops/cache') + +const A = 'a'.repeat(40) +const B = 'b'.repeat(40) +const C = 'c'.repeat(40) +const roots = [] + +async function temporaryRoot () { + const root = await fs.mkdtemp(join(tmpdir(), 'ops-cache-')) + roots.push(root) + return root +} + +async function entry (root, commit, content = 'data', complete = true) { + const directory = join(root, commit) + await fs.mkdir(directory, { recursive: true }) + await fs.writeFile(join(directory, 'content'), content) + if (complete) await fs.writeFile(join(directory, '.complete'), '') + return directory +} + +function manager (root, options = {}) { + return new CacheManager({ root, service: 'builder', idleExpiryMs: 1000, ...options }) +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))) +}) + +describe('operations cache', () => { + it('inspects only complete canonical entries with exact bytes and deterministic idle data', async () => { + const root = await temporaryRoot() + await entry(root, A, '1234') + await entry(root, B, 'ignored', false) + await entry(root, 'not-a-commit', 'ignored') + const time = Date.parse('2026-01-01T00:00:00.000Z') + const cache = manager(root, { now: () => time }) + cache.touch(A) + const snapshot = await cache.snapshot() + assert.deepEqual(snapshot, { + entryCount: 1, + totalBytes: 4, + idleExpiryMs: 1000, + entriesTruncated: false, + entries: [{ + commit: A, + sizeBytes: 4, + lastAccessedAt: '2026-01-01T00:00:00.000Z', + expiresAt: '2026-01-01T00:00:01.000Z', + inUse: 0 + }] + }) + }) + + it('reuses sealed entry sizes and invalidates changed and replaced entries', async () => { + const root = await temporaryRoot() + const directory = await entry(root, A, '1234') + let walks = 0 + const filesystem = Object.create(fs) + filesystem.readdir = async (path, options) => { + if (path.startsWith(directory)) walks++ + return fs.readdir(path, options) + } + const cache = manager(root, { filesystem }) + + assert.equal((await cache.snapshot()).totalBytes, 4) + assert.equal(walks, 1) + assert.equal((await cache.snapshot()).totalBytes, 4) + assert.equal(walks, 1) + + await fs.writeFile(join(directory, 'content'), '123456') + await fs.utimes(join(directory, '.complete'), new Date(1), new Date(2)) + assert.equal((await cache.snapshot()).totalBytes, 6) + assert.equal(walks, 2) + + await fs.rm(directory, { recursive: true }) + await entry(root, A, '12345678') + assert.equal((await cache.snapshot()).totalBytes, 8) + assert.equal(walks, 3) + }) + + it('tracks shared users and excludes active entries from deletion', async () => { + const root = await temporaryRoot() + await entry(root, A) + let time = Date.parse('2026-01-01T00:00:00.000Z') + const cache = manager(root, { now: () => time }) + const releaseOne = await cache.acquire(A) + const releaseTwo = await cache.acquire(A) + assert.equal((await cache.snapshot()).entries[0].inUse, 2) + assert.deepEqual(await cache.execute('cache.evict_commit', A), { + service: 'builder', + outcome: 'no_op', + removedEntries: 0, + freedBytes: 0, + absent: false, + skippedInUse: 1, + error: null, + skippedChanged: 0 + }) + time += 500 + releaseOne() + releaseTwo() + assert.equal((await cache.snapshot()).entries[0].lastAccessedAt, '2026-01-01T00:00:00.500Z') + }) + + it('validates commit paths and ignores symlink entries and markers', async function () { + const root = await temporaryRoot() + const outside = await temporaryRoot() + await entry(outside, A) + try { + await fs.symlink(join(outside, A), join(root, A), 'dir') + await entry(root, B) + await fs.rm(join(root, B, '.complete')) + await fs.symlink(join(outside, A, '.complete'), join(root, B, '.complete')) + } catch (error) { + if (error.code === 'EPERM') return this.skip() + throw error + } + assert.equal((await manager(root).snapshot()).entryCount, 0) + assert.throws(() => parseCommit('../outside'), CacheError) + assert.throws(() => parseCommit('A'.repeat(40)), CacheError) + assert.equal((await fs.readFile(join(outside, A, 'content'), 'utf8')), 'data') + }) + + it('reports absent and deletes complete entries without touching incomplete ones', async () => { + const root = await temporaryRoot() + await entry(root, B, 'keep', false) + const cache = manager(root) + assert.equal((await cache.execute('cache.evict_commit', A)).absent, true) + assert.equal((await cache.execute('cache.clear')).outcome, 'no_op') + assert.equal(await fs.readFile(join(root, B, 'content'), 'utf8'), 'keep') + await entry(root, A, 'gone') + const result = await cache.execute('cache.evict_commit', A) + assert.equal(result.outcome, 'completed') + assert.equal(result.removedEntries, 1) + assert.equal(result.freedBytes, 4) + await assert.rejects(fs.lstat(join(root, A)), { code: 'ENOENT' }) + }) + + it('rechecks idle use under the deletion claim and maps mixed results to partial', async () => { + const root = await temporaryRoot() + await entry(root, A) + await entry(root, B) + let time = Date.parse('2026-01-01T00:00:00.000Z') + const cache = manager(root, { now: () => time, concurrency: 1 }) + cache.touch(A) + cache.touch(B) + time += 1000 + const original = cache.deleteEntry.bind(cache) + cache.deleteEntry = async (candidate, operation, threshold) => { + if (candidate.commit === A) cache.touch(B) + return original(candidate, operation, threshold) + } + const result = await cache.execute('cache.purge_expired') + assert.equal(result.outcome, 'partial') + assert.equal(result.removedEntries, 1) + assert.equal(result.skippedChanged, 1) + assert.equal((await cache.snapshot()).entries[0].commit, B) + }) + + it('serializes competing deleters and lets new users rebuild after deletion', async () => { + const root = await temporaryRoot() + await entry(root, A) + const cache = manager(root) + const [first, second] = await Promise.all([ + cache.execute('cache.evict_commit', A), + cache.execute('cache.evict_commit', A) + ]) + assert.equal(first.removedEntries + second.removedEntries, 1) + assert.equal(first.absent || second.absent, true) + const release = await cache.acquire(A) + release() + }) + + it('bounds visible snapshots, scan work, and deletion concurrency', async () => { + const root = await temporaryRoot() + for (const commit of [A, B, C]) await entry(root, commit) + let active = 0 + let maximum = 0 + const filesystem = Object.create(fs) + filesystem.rm = async (...args) => { + active++ + maximum = Math.max(maximum, active) + await new Promise(resolve => setTimeout(resolve, 5)) + try { return await fs.rm(...args) } finally { active-- } + } + const cache = manager(root, { entryLimit: 2, concurrency: 2, filesystem }) + const snapshot = await cache.snapshot() + assert.equal(snapshot.entryCount, 3) + assert.equal(snapshot.entries.length, 2) + assert.equal(snapshot.entriesTruncated, true) + assert.equal((await cache.execute('cache.clear')).removedEntries, 3) + assert.equal(maximum, 2) + + const limitedRoot = await temporaryRoot() + await entry(limitedRoot, A) + await assert.rejects(manager(limitedRoot, { scanLimit: 1 }).snapshot(), error => error.code === 'CACHE_SCAN_LIMIT') + }) + + it('returns sanitized failed outcomes and rejects unnamed operations', async () => { + const root = await temporaryRoot() + await entry(root, A) + const filesystem = Object.create(fs) + filesystem.rename = async () => { throw new Error(`secret path: ${root}`) } + const result = await manager(root, { filesystem }).execute('cache.evict_commit', A) + assert.equal(result.outcome, 'failed') + assert.deepEqual(result.error, { code: 'CACHE_OPERATION_FAILED', message: 'Cache operation failed' }) + assert.doesNotMatch(JSON.stringify(result), new RegExp(root)) + await assert.rejects(manager(root).execute('cache.delete', A), error => error.code === 'INVALID_OPERATION') + }) +}) diff --git a/test/ops-config.js b/test/ops-config.js new file mode 100644 index 0000000..08b87b3 --- /dev/null +++ b/test/ops-config.js @@ -0,0 +1,106 @@ +'use strict' + +const { expect } = require('chai') +const { randomBytes } = require('node:crypto') +const { join } = require('node:path') +const { describe, it } = require('mocha') +const { + createTokenVerifier, + DEFAULT_MTLS_PORT, + parseOpsConfig, + verifyToken +} = require('../app/ops/config') + +describe('operations console configuration', () => { + const token = randomBytes(32).toString('base64url') + const absolute = name => join(__dirname, 'fixtures/mtls', name) + const enabled = overrides => ({ + OPS_CONSOLE_ENABLED: 'true', + OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), + OPS_CONSOLE_ORIGIN: 'https://ops.example.test', + OPS_CONSOLE_MTLS_KEY_PATH: absolute('server.key'), + OPS_CONSOLE_MTLS_CERT_PATH: absolute('server.crt'), + OPS_CONSOLE_MTLS_CA_PATH: absolute('ca.crt'), + ...overrides + }) + + it('is disabled unless explicitly enabled and ignores disabled secrets', () => { + expect(parseOpsConfig({ NODE_ENV: 'development', OPS_CONSOLE_TOKEN_VERIFIER: 'secret' })).to.deep.equal({ enabled: false }) + expect(parseOpsConfig({ OPS_CONSOLE_ENABLED: 'TRUE' })).to.deep.equal({ enabled: false }) + }) + + it('creates and checks only canonical domain-separated verifiers', () => { + const verifier = createTokenVerifier(token) + const config = parseOpsConfig(enabled()) + expect(verifier).to.match(/^v1\.[A-Za-z0-9_-]{43}$/) + expect(verifyToken(token, config.verifierDigest)).to.equal(true) + expect(verifyToken(randomBytes(32).toString('base64url'), config.verifierDigest)).to.equal(false) + expect(verifyToken('not-a-token', config.verifierDigest)).to.equal(false) + expect(verifyToken('', config.verifierDigest)).to.equal(false) + expect(verifyToken(undefined, config.verifierDigest)).to.equal(false) + for (const invalid of [undefined, '', Buffer.alloc(0), Buffer.alloc(31)]) { + expect(() => verifyToken(token, invalid)).to.throw('Invalid operations console token verifier') + } + }) + + it('fails enabled startup for missing, malformed, or noncanonical settings', () => { + for (const env of [ + enabled({ OPS_CONSOLE_TOKEN_VERIFIER: undefined }), + enabled({ OPS_CONSOLE_TOKEN_VERIFIER: `v2.${'A'.repeat(43)}` }), + enabled({ OPS_CONSOLE_ORIGIN: 'https://ops.example.test/' }), + enabled({ OPS_CONSOLE_ORIGIN: 'HTTPS://ops.example.test' }), + enabled({ OPS_CONSOLE_ORIGIN: 'ftp://ops.example.test' }), + enabled({ OPS_CONSOLE_MTLS_KEY_PATH: undefined }), + enabled({ OPS_CONSOLE_MTLS_CERT_PATH: undefined }), + enabled({ OPS_CONSOLE_MTLS_CA_PATH: undefined }), + enabled({ OPS_CONSOLE_MTLS_KEY_PATH: 'server.key' }), + enabled({ OPS_CONSOLE_MTLS_CERT_PATH: './server.crt' }), + enabled({ OPS_CONSOLE_MTLS_CA_PATH: 'ca.crt' }) + ]) expect(() => parseOpsConfig(env)).to.throw() + }) + + it('requires dedicated HTTPS mTLS paths and a strict bounded port defaulting to 8443', () => { + const config = parseOpsConfig(enabled()) + expect(DEFAULT_MTLS_PORT).to.equal(8443) + expect(config.mtls).to.deep.equal({ + port: 8443, + keyPath: absolute('server.key'), + certPath: absolute('server.crt'), + caPath: absolute('ca.crt') + }) + expect(parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: '1' })).mtls.port).to.equal(1) + expect(parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: '65535' })).mtls.port).to.equal(65535) + for (const port of ['0', '01', '65536', '-1', '8443 ', '1.5']) { + expect(() => parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: port }))).to.throw('Invalid OPS_CONSOLE_MTLS_PORT') + } + }) + + it('rejects the legacy trusted-proxy setting whenever the console is enabled', () => { + expect(() => parseOpsConfig(enabled({ OPS_CONSOLE_TRUSTED_PROXY: '127.0.0.1' }))).to.throw('OPS_CONSOLE_TRUSTED_PROXY is not supported') + expect(parseOpsConfig(enabled({ OPS_CONSOLE_TRUSTED_PROXY: '' })).protocol).to.equal('https') + }) + + it('allows HTTP only for direct loopback mode', () => { + expect(parseOpsConfig(enabled({ + OPS_CONSOLE_ORIGIN: 'http://127.1.2.3:8080', + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', + OPS_CONSOLE_MTLS_KEY_PATH: undefined, + OPS_CONSOLE_MTLS_CERT_PATH: undefined, + OPS_CONSOLE_MTLS_CA_PATH: undefined + }))).to.include({ enabled: true, protocol: 'http', allowHttpLoopback: true }) + expect(parseOpsConfig(enabled({ + OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', + OPS_CONSOLE_MTLS_KEY_PATH: undefined, + OPS_CONSOLE_MTLS_CERT_PATH: undefined, + OPS_CONSOLE_MTLS_CA_PATH: undefined + }))).to.include({ enabled: true, protocol: 'http', allowHttpLoopback: true }) + + for (const overrides of [ + { OPS_CONSOLE_ORIGIN: 'http://localhost:8080' }, + { OPS_CONSOLE_ORIGIN: 'http://example.test', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' }, + { OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', OPS_CONSOLE_MTLS_PORT: '9443' }, + { OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', OPS_CONSOLE_MTLS_CA_PATH: absolute('ca.crt') } + ]) expect(() => parseOpsConfig(enabled(overrides))).to.throw('HTTP operations console') + }) +}) diff --git a/test/ops-console.js b/test/ops-console.js new file mode 100644 index 0000000..b93164a --- /dev/null +++ b/test/ops-console.js @@ -0,0 +1,779 @@ +'use strict' + +const { expect } = require('chai') +const { randomBytes } = require('node:crypto') +const http = require('node:http') +const express = require('express') +const { describe, it } = require('mocha') +const { createTokenVerifier } = require('../app/ops/config') +const { createOpsConsoleRouter } = require('../app/ops/console-router') +const { SECURITY_HEADERS } = require('../app/ops/http') +const { SESSION_CAPACITY, SessionStore } = require('../app/ops/sessions') +const { LIMITS } = require('../app/ops/telemetry') +const { createApp } = require('../app/server') + +describe('operations console router', () => { + const origin = 'http://127.0.0.1:8080' + const enabled = (token, overrides = {}) => ({ + OPS_CONSOLE_ENABLED: 'true', + OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), + OPS_CONSOLE_ORIGIN: origin, + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', + ...overrides + }) + + it('reserves every disabled path before legacy and public middleware and logs one safe status', async () => { + const events = [] + let publicCalls = 0 + const publicRouter = express.Router().use((request, response) => { + publicCalls++ + response.sendStatus(200) + }) + const app = createApp({ ops: { env: {}, log: event => events.push(event) }, router: publicRouter }) + + for (const path of ['/_ops', '/_ops/', '/_ops/login', '/_ops/console.css', '/_ops/console.js', '/_ops/login.js', '/_ops/api/v1/session', '/_ops/anything']) { + const response = await request(app, { path }) + expect(response.status).to.equal(404) + expectSecurityHeaders(response.headers) + } + expect(publicCalls).to.equal(0) + expect(events).to.have.length(1) + expect(events[0]).to.include({ action: 'operations_console_status', enabled: false }) + expect(Object.keys(events[0])).to.have.members(['time', 'action', 'enabled']) + }) + + it('serves only the enabled login page and allow-listed assets without authentication', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + const files = [ + ['/_ops/login', 'text/html', '
'], + ['/_ops/console.css', 'text/css', '.login-panel'], + ['/_ops/console.js', 'application/javascript', "const API = '/_ops/api/v1'"], + ['/_ops/login.js', 'application/javascript', "window.location.replace('/_ops/')"] + ] + + for (const [path, contentType, marker] of files) { + const response = await request(app, { path }) + expect(response.status).to.equal(200) + expect(response.headers['content-type']).to.include(contentType) + expect(response.body).to.include(marker) + expectSecurityHeaders(response.headers) + expect(response.headers).not.to.have.property('access-control-allow-origin') + } + }) + + it('allows direct address-bar console documents and rejects none elsewhere', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + const navigation = { + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Dest': 'document' + } + + expect((await request(app, { path: '/_ops/login?direct=1', headers: navigation })).status).to.equal(200) + const shell = await request(app, { path: '/_ops/?direct=1', headers: navigation }) + expect(shell.status).to.equal(302) + expect(shell.headers.location).to.equal('/_ops/login') + + for (const [method, path] of [ + ['GET', '/_ops/console.css'], + ['GET', '/_ops/api/v1/session'], + ['POST', '/_ops/api/v1/session'], + ['DELETE', '/_ops/api/v1/session'], + ['GET', '/_ops/unknown'], + ['GET', '/_ops/%2e%2e/config.json'] + ]) { + const response = await request(app, { method, path, headers: navigation }) + expect(response).to.nested.include({ status: 403, 'json.error.code': 'FETCH_METADATA_REJECTED' }) + expectSecurityHeaders(response.headers) + } + }) + + it('serves only self-hosted static behavior without inline code or browser storage', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + const authenticated = await login(app, token) + const responses = await Promise.all([ + request(app, { path: '/_ops/login' }), + request(app, { path: '/_ops/', headers: { Cookie: cookiePair(authenticated) } }), + request(app, { path: '/_ops/console.css' }), + request(app, { path: '/_ops/console.js' }), + request(app, { path: '/_ops/login.js' }) + ]) + const assets = responses.map(response => response.body).join('\n') + + expect(responses.every(response => response.status === 200)).to.equal(true) + expect(assets).not.to.match(/https?:\/\/| response.body)) { + expect([...html.matchAll(/]*)>/gi)].every(match => /src="\/_ops\/(?:console|login)\.js"/.test(match[1]))).to.equal(true) + expect([...html.matchAll(/]*)>/gi)].every(match => /href="\/_ops\/console\.css"/.test(match[1]))).to.equal(true) + } + }) + + it('protects the exact UI shell without renewing its session', async () => { + const token = randomBytes(32).toString('base64url') + let now = Date.parse('2026-07-20T12:00:00.000Z') + const app = opsApp({ env: enabled(token), log: () => {}, now: () => now }) + + const anonymous = await request(app, { path: '/_ops/' }) + expect(anonymous.status).to.equal(302) + expect(anonymous.headers.location).to.equal('/_ops/login') + expect(anonymous.body).not.to.include('UNAUTHORIZED') + expectSecurityHeaders(anonymous.headers) + + const invalid = await request(app, { path: '/_ops/', headers: { Cookie: 'ghhc-console-dev=invalid' } }) + expect(invalid.status).to.equal(302) + expect(invalid.headers.location).to.equal('/_ops/login') + + const authenticated = await login(app, token) + const cookie = cookiePair(authenticated) + now += 29 * 60 * 1000 + const shell = await request(app, { path: '/_ops/', headers: { Cookie: cookie } }) + expect(shell.status).to.equal(200) + expect(shell.headers['content-type']).to.include('text/html') + expect(shell.body).to.include('

Operations console

') + expectSecurityHeaders(shell.headers) + + now += 2 * 60 * 1000 + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: cookie } }))).to.nested.include({ + status: 401, + 'json.error.code': 'UNAUTHORIZED' + }) + }) + + it('does not serve, leak, or redirect unknown and traversal-like static paths', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + + for (const path of ['/_ops', '/_ops/unknown', '/_ops/static/', '/_ops/console.css/', '/_ops/.hidden', '/_ops/%2e%2e/config.json']) { + const response = await request(app, { path }) + expect(response.status).to.equal(404) + expect(response.headers).not.to.have.property('location') + expect(response.body).not.to.include('downloaderURL') + expectSecurityHeaders(response.headers) + } + + const api = await request(app, { path: '/_ops/api/v1/unknown' }) + expect(api).to.nested.include({ status: 404, 'json.error.code': 'NOT_FOUND' }) + expect(api.json.error.correlationId).to.equal(api.headers['x-correlation-id']) + }) + + it('creates, bootstraps, and revokes a loopback HTTP session', async () => { + const token = randomBytes(32).toString('base64url') + const events = [] + const app = opsApp({ env: enabled(token), log: event => events.push(event) }) + const login = await request(app, { + method: 'POST', + path: '/_ops/api/v1/session', + headers: { Origin: origin, 'Content-Type': 'application/json', 'Sec-Fetch-Site': 'same-origin' }, + body: JSON.stringify({ token }) + }) + + expect(login.status).to.equal(201) + expect(login.headers['set-cookie']).to.have.length(1) + expect(login.headers['set-cookie'][0]).to.match(/^ghhc-console-dev=[A-Za-z0-9_-]{43}; HttpOnly; SameSite=Strict; Path=\/$/) + expect(login.headers['set-cookie'][0]).not.to.include('Secure') + expect(login.json).to.include({ authenticated: true }) + expect(login.json.csrfToken).to.match(/^[A-Za-z0-9_-]{43}$/) + expect(login.headers['x-correlation-id']).to.match(/^[0-9a-f-]{36}$/) + expect(events).to.have.length(1) + + const cookie = cookiePair(login) + const status = await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: cookie } }) + expect(status.status).to.equal(200) + expect(status.json).to.include({ + authenticated: true, + absoluteExpiresAt: login.json.absoluteExpiresAt, + csrfToken: login.json.csrfToken + }) + expect(Date.parse(status.json.idleExpiresAt)).to.be.at.least(Date.parse(login.json.idleExpiresAt)) + + const rejectedLogout = await request(app, { + method: 'DELETE', + path: '/_ops/api/v1/session', + headers: { Cookie: cookie, Origin: origin, 'X-Ops-CSRF': 'wrong' } + }) + expect(rejectedLogout).to.nested.include({ status: 403, 'json.error.code': 'CSRF_REJECTED' }) + + const logout = await request(app, { + method: 'DELETE', + path: '/_ops/api/v1/session', + headers: { Cookie: cookie, Origin: origin, 'X-Ops-CSRF': login.json.csrfToken } + }) + expect(logout.status).to.equal(204) + expect(logout.body).to.equal('') + expect(logout.headers['set-cookie'][0]).to.equal('ghhc-console-dev=; Max-Age=0; HttpOnly; SameSite=Strict; Path=/') + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: cookie } }))).to.nested.include({ + status: 401, + 'json.error.code': 'UNAUTHORIZED' + }) + }) + + it('returns generic authentication failures without reflecting credentials', async () => { + const token = randomBytes(32).toString('base64url') + const wrongToken = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + + for (const body of [JSON.stringify({ token: wrongToken }), JSON.stringify({ token, extra: true })]) { + const response = await request(app, { + method: 'POST', + path: '/_ops/api/v1/session', + headers: { Origin: origin, 'Content-Type': 'application/json' }, + body + }) + expect(response.status).to.equal(401) + expect(response.json.error).to.include({ code: 'UNAUTHORIZED', message: 'Authentication failed' }) + expect(response.body).not.to.include(token) + expect(response.body).not.to.include(wrongToken) + } + }) + + it('enforces context, Origin, Fetch Metadata, content type, and body bounds', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + const attempt = headers => request(app, { + method: 'POST', + path: '/_ops/api/v1/session', + headers, + body: JSON.stringify({ token }) + }) + + expect(await attempt({ 'Content-Type': 'application/json' })).to.nested.include({ status: 403, 'json.error.code': 'ORIGIN_REJECTED' }) + expect(await attempt({ Origin: origin, 'Content-Type': 'application/json', 'Sec-Fetch-Site': 'cross-site' })).to.nested.include({ status: 403, 'json.error.code': 'FETCH_METADATA_REJECTED' }) + expect(await attempt({ Origin: origin, 'Content-Type': 'text/plain' })).to.nested.include({ status: 415, 'json.error.code': 'UNSUPPORTED_MEDIA_TYPE' }) + + const oversized = await request(app, { + method: 'POST', + path: '/_ops/api/v1/session', + headers: { Origin: origin, 'Content-Type': 'application/json' }, + body: JSON.stringify({ token, padding: 'x'.repeat(4096) }) + }) + expect(oversized).to.nested.include({ status: 413, 'json.error.code': 'PAYLOAD_TOO_LARGE' }) + }) + + it('rotates an existing session only after successful authentication', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + const first = await login(app, token) + const firstCookie = cookiePair(first) + const failed = await login(app, randomBytes(32).toString('base64url'), firstCookie) + expect(failed.status).to.equal(401) + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: firstCookie } })).status).to.equal(200) + + const replacement = await login(app, token, firstCookie) + expect(replacement.status).to.equal(201) + expect(cookiePair(replacement)).not.to.equal(firstCookie) + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: firstCookie } })).status).to.equal(401) + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: cookiePair(replacement) } })).status).to.equal(200) + }) + + it('enforces login rate and session capacity without evicting sessions', async () => { + const token = randomBytes(32).toString('base64url') + const rateApp = opsApp({ env: enabled(token), log: () => {} }) + for (let index = 0; index < 30; index++) { + const headers = { 'X-Forwarded-For': `192.0.2.${index + 1}`, 'X-Forwarded-Proto': 'https' } + expect((await login(rateApp, randomBytes(32).toString('base64url'), undefined, headers)).status).to.equal(401) + } + const limited = await login(rateApp, token) + expect(limited).to.nested.include({ status: 429, 'json.error.code': 'RATE_LIMITED' }) + expect(limited.headers['retry-after']).to.equal('900') + + const store = new SessionStore() + for (let index = 0; index < SESSION_CAPACITY; index++) store.create() + const capacityApp = opsApp({ env: enabled(token), log: () => {}, sessions: store }) + const capacity = await login(capacityApp, token) + expect(capacity).to.nested.include({ status: 503, 'json.error.code': 'SESSION_CAPACITY' }) + expect(store.size).to.equal(SESSION_CAPACITY) + }) + + it('expires sessions, rejects ambiguous cookies, and does not renew on HEAD', async () => { + const token = randomBytes(32).toString('base64url') + let now = Date.parse('2026-07-20T12:00:00.000Z') + const app = opsApp({ env: enabled(token), log: () => {}, now: () => now }) + const created = await login(app, token) + const cookie = cookiePair(created) + expect((await request(app, { method: 'HEAD', path: '/_ops/api/v1/session', headers: { Cookie: cookie } })).status).to.equal(404) + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: `${cookie}; ${cookie}` } })).status).to.equal(401) + now += 30 * 60 * 1000 + expect((await request(app, { path: '/_ops/api/v1/session', headers: { Cookie: cookie } })).status).to.equal(401) + }) + + it('authenticates, concurrently aggregates bounded snapshots, and propagates one correlation ID', async () => { + const token = randomBytes(32).toString('base64url') + const started = new Set() + const calls = [] + const observedAt = new Date().toISOString() + const makeProvider = (service, value) => async options => { + started.add(service) + calls.push({ service, options }) + await new Promise(resolve => setImmediate(resolve)) + expect([...started]).to.have.members(['router', 'downloader', 'builder']) + return value + } + const routerSnapshot = serviceSnapshot('router', observedAt, { + activity: [activity('public-request', 'router', 'public_file_delivery')] + }) + const downloaderSnapshot = serviceSnapshot('downloader', observedAt, { + activity: [activity('public-request', 'downloader', 'file_download')], + failures: [failure('public-request', 'downloader', 'file_download')] + }) + const app = opsApp({ + env: enabled(token), + log: () => {}, + routerService: { snapshot: makeProvider('router', routerSnapshot) }, + downloader: { json: (path, options) => makeProvider('downloader', downloaderSnapshot)(options) }, + builder: { json: (path, options) => makeProvider('builder', serviceSnapshot('builder', observedAt))(options) } + }) + const authenticated = await login(app, token) + const cookie = cookiePair(authenticated) + + expect(await request(app, { path: '/_ops/api/v1/snapshot' })).to.nested.include({ status: 401, 'json.error.code': 'UNAUTHORIZED' }) + expect(await request(app, { path: '/_ops/api/v1/snapshot', headers: { Cookie: cookie, 'Sec-Fetch-Site': 'cross-site' } })).to.nested.include({ status: 403, 'json.error.code': 'FETCH_METADATA_REJECTED' }) + expect((await request(app, { method: 'HEAD', path: '/_ops/api/v1/snapshot', headers: { Cookie: cookie } })).status).to.equal(404) + + const snapshot = await request(app, { path: '/_ops/api/v1/snapshot', headers: { Cookie: cookie, 'X-Correlation-ID': 'browser-value' } }) + expect(snapshot.status).to.equal(200) + expect(snapshot.headers['x-correlation-id']).to.match(/^[0-9a-f-]{36}$/).and.not.equal('browser-value') + expect(snapshot.json).to.include({ schemaVersion: 1, correlationId: snapshot.headers['x-correlation-id'], refreshAfterMs: 30000 }) + expect(snapshot.json.services.downloader).to.include({ freshness: 'fresh', error: null }) + expect(snapshot.json.services.builder).to.include({ freshness: 'fresh', error: null }) + expect(snapshot.json.activity).to.have.length(1) + expect(snapshot.json.activity[0].spans.map(span => span.service)).to.have.members(['router', 'downloader']) + expect(snapshot.json.failures).to.have.length(1) + expect(snapshot.json.services.router.snapshot.capabilities.find(capability => capability.name === 'console_cache_control')).to.include({ status: 'available', reasonCode: null }) + expect(calls.filter(call => call.service !== 'router')).to.have.length(2) + for (const call of calls.filter(call => call.service !== 'router')) { + expect(call.options).to.include({ + correlationId: snapshot.json.correlationId, + maxResponseBytes: 1024 * 1024, + timeout: 1500, + timeoutThroughBody: true + }) + } + expect(JSON.stringify(snapshot.json)).not.to.include('browser-value') + }) + + it('retains last-success snapshots across failures, expires stale data, and replaces restarted instances', async () => { + const token = randomBytes(32).toString('base64url') + let now = Date.parse('2026-07-20T12:00:00.000Z') + let downloaderFails = false + let builderInstance = 'builder-1' + const currentSnapshot = (service, instanceId = `${service}-1`) => serviceSnapshot(service, new Date(now).toISOString(), { instanceId }) + const app = opsApp({ + env: enabled(token), + log: () => {}, + now: () => now, + routerService: { snapshot: () => currentSnapshot('router') }, + downloader: { json: () => downloaderFails ? Promise.reject(new Error('secret /path')) : Promise.resolve(currentSnapshot('downloader')) }, + builder: { json: () => Promise.resolve(currentSnapshot('builder', builderInstance)) } + }) + const authenticated = await login(app, token) + const headers = { Cookie: cookiePair(authenticated) } + const fresh = await request(app, { path: '/_ops/api/v1/snapshot', headers }) + expect(fresh.json.services.downloader).to.include({ freshness: 'fresh', ageMs: 0 }) + + now += 1000 + downloaderFails = true + builderInstance = 'builder-2' + const stale = await request(app, { path: '/_ops/api/v1/snapshot', headers }) + expect(stale.json.services.downloader).to.nested.include({ + freshness: 'stale', + ageMs: 1000, + 'snapshot.instanceId': 'downloader-1', + 'error.code': 'SERVICE_UNAVAILABLE', + 'error.message': 'Service snapshot unavailable' + }) + expect(stale.json.services.builder).to.nested.include({ freshness: 'fresh', 'snapshot.instanceId': 'builder-2' }) + expect(JSON.stringify(stale.json)).not.to.include('secret').and.not.to.include('/path') + + now += LIMITS.staleMs + const unknown = await request(app, { path: '/_ops/api/v1/snapshot', headers }) + expect(unknown.json.services.downloader).to.include({ freshness: 'unknown', snapshot: null, ageMs: null }) + expect(unknown.json.services.downloader.lastSuccessAt).to.equal(fresh.json.services.downloader.lastSuccessAt) + }) + + it('applies concurrent independent 1.5-second deadlines without retries', async () => { + const token = randomBytes(32).toString('base64url') + const observedAt = new Date().toISOString() + const calls = { router: 0, downloader: 0, builder: 0 } + const never = () => new Promise(() => {}) + const app = opsApp({ + env: enabled(token), + log: () => {}, + routerService: { snapshot: () => { calls.router++; return serviceSnapshot('router', observedAt) } }, + downloader: { json: () => { calls.downloader++; return never() } }, + builder: { json: () => { calls.builder++; return never() } } + }) + const authenticated = await login(app, token) + const startedAt = Date.now() + const snapshot = await request(app, { path: '/_ops/api/v1/snapshot', headers: { Cookie: cookiePair(authenticated) } }) + const duration = Date.now() - startedAt + + expect(duration).to.be.at.least(1400).and.below(2500) + expect(calls).to.deep.equal({ router: 1, downloader: 1, builder: 1 }) + expect(snapshot.json.services.router.freshness).to.equal('fresh') + for (const service of ['downloader', 'builder']) { + expect(snapshot.json.services[service]).to.nested.include({ freshness: 'unknown', snapshot: null, 'error.code': 'SERVICE_TIMEOUT' }) + } + }) + + it('enforces the per-session snapshot rate limit without dispatching excess calls', async () => { + const token = randomBytes(32).toString('base64url') + const observedAt = new Date().toISOString() + let calls = 0 + const app = opsApp({ + env: enabled(token), + log: () => {}, + routerService: { snapshot: () => serviceSnapshot('router', observedAt) }, + downloader: { json: () => { calls++; return serviceSnapshot('downloader', observedAt) } }, + builder: { json: () => serviceSnapshot('builder', observedAt) } + }) + const authenticated = await login(app, token) + const headers = { Cookie: cookiePair(authenticated) } + for (let index = 0; index < 12; index++) expect((await request(app, { path: '/_ops/api/v1/snapshot', headers })).status).to.equal(200) + const limited = await request(app, { path: '/_ops/api/v1/snapshot', headers }) + expect(limited).to.nested.include({ status: 429, 'json.error.code': 'RATE_LIMITED' }) + expect(limited.headers['retry-after']).to.equal('60') + expect(calls).to.equal(12) + }) + + it('authenticates and concurrently dispatches exact cache commands with one correlation ID and audit event', async () => { + const token = randomBytes(32).toString('base64url') + const secret = 'must-not-appear' + const events = [] + const started = new Set() + const calls = [] + const service = (name, target) => ({ + json: async (path, options) => { + started.add(name) + calls.push({ name, path, options }) + await new Promise(resolve => setImmediate(resolve)) + expect([...started]).to.have.members(['downloader', 'builder']) + return cacheOperationResponse(name, options, target) + } + }) + const app = opsApp({ + env: enabled(token), + log: event => events.push(event), + downloader: service('downloader', { outcome: 'completed', removedEntries: 1, freedBytes: 42 }), + builder: service('builder', { absent: true }) + }) + const authenticated = await login(app, token) + const response = await request(app, { + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers: { + Cookie: cookiePair(authenticated), + Origin: origin, + 'Content-Type': 'application/json', + 'X-Ops-CSRF': authenticated.json.csrfToken, + Forwarded: 'for=203.0.113.9;proto=https', + 'X-Forwarded-For': '198.51.100.8', + 'X-Forwarded-Proto': 'https', + 'User-Agent': `Mozilla/5.0 ${secret} Chrome/126.0 Safari/537.36` + }, + body: JSON.stringify({ operation: 'cache.evict_commit', targets: ['downloader', 'builder'], commit: 'a'.repeat(40) }) + }) + + expect(response.status).to.equal(200) + expect(response.json).to.include({ correlationId: response.headers['x-correlation-id'], operation: 'cache.evict_commit', outcome: 'completed' }) + expect(response.json.targets.map(target => target.outcome)).to.deep.equal(['completed', 'no_op']) + expect(calls).to.have.length(2) + for (const call of calls) { + expect(call.path).to.equal('/v1/ops/cache-operations') + expect(call.options).to.include({ + method: 'POST', + correlationId: response.json.correlationId, + maxRequestBytes: 4096, + maxResponseBytes: 64 * 1024, + timeout: 10000, + timeoutThroughBody: true + }) + expect(call.options.body).to.deep.equal({ operation: 'cache.evict_commit', targets: [call.name], commit: 'a'.repeat(40) }) + } + + const audits = events.filter(event => event.action === 'cache_operation') + expect(audits).to.have.length(1) + expect(audits[0]).to.include({ + correlationId: response.json.correlationId, + source: '127.0.0.1', + userAgent: 'chromium', + operation: 'cache.evict_commit', + commit: 'a'.repeat(40), + dispatchStatus: 'dispatched', + outcome: 'completed' + }) + expect(audits[0].sessionAuditId).to.match(/^[A-Za-z0-9_-]{22}$/) + expect(audits[0].targets).to.deep.equal(['downloader', 'builder']) + expect(audits[0].targetOutcomes).to.have.length(2) + expect(JSON.stringify(audits[0])).not.to.include(token).and.not.to.include(authenticated.json.csrfToken).and.not.to.include(secret) + }) + + it('rejects unsafe cache attempts before dispatch and emits one bounded sanitized audit', async () => { + const token = randomBytes(32).toString('base64url') + const events = [] + let calls = 0 + const client = { json: () => { calls++; throw new Error('must not dispatch') } } + const app = opsApp({ env: enabled(token), log: event => events.push(event), downloader: client, builder: client }) + const authenticated = await login(app, token) + const headers = { + Cookie: cookiePair(authenticated), + Origin: origin, + 'Content-Type': 'application/json', + 'X-Ops-CSRF': authenticated.json.csrfToken + } + const rejected = await request(app, { + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers, + body: JSON.stringify({ operation: 'cache.evict_commit', targets: ['downloader'], commit: 'A'.repeat(40) }) + }) + + expect(rejected).to.nested.include({ status: 400, 'json.error.code': 'INVALID_REQUEST' }) + expect(calls).to.equal(0) + const audits = events.filter(event => event.action === 'cache_operation') + expect(audits).to.have.length(1) + expect(audits[0]).to.include({ userAgent: null, operation: null, dispatchStatus: 'not_dispatched', outcome: 'failed' }) + expect(audits[0].errorCodes).to.deep.equal(['INVALID_REQUEST']) + + for (const missing of [ + { Origin: origin, 'Content-Type': 'application/json', 'X-Ops-CSRF': authenticated.json.csrfToken }, + { Cookie: cookiePair(authenticated), Origin: origin, 'Content-Type': 'application/json', 'X-Ops-CSRF': 'wrong' }, + { Cookie: cookiePair(authenticated), Origin: origin, 'X-Ops-CSRF': authenticated.json.csrfToken } + ]) { + const response = await request(app, { method: 'POST', path: '/_ops/api/v1/cache-operations', headers: missing, body: '{}' }) + expect(response.status).to.be.oneOf([401, 403, 415]) + } + expect(events.filter(event => event.action === 'cache_operation')).to.have.length(4) + }) + + it('finalizes one cache audit with settled target outcomes after the client closes', async () => { + const token = randomBytes(32).toString('base64url') + const events = [] + const releases = {} + let resolveDispatched + let resolveResponseClosed + let resolveAudited + const dispatched = new Promise(resolve => { resolveDispatched = resolve }) + const responseClosed = new Promise(resolve => { resolveResponseClosed = resolve }) + const audited = new Promise(resolve => { resolveAudited = resolve }) + const service = (name, target) => ({ + json: (path, options) => new Promise(resolve => { + releases[name] = () => resolve(cacheOperationResponse(name, options, target)) + if (Object.keys(releases).length === 2) resolveDispatched() + }) + }) + const ops = { + env: enabled(token), + log: event => { + events.push(event) + if (event.action === 'cache_operation') resolveAudited() + }, + downloader: service('downloader', { outcome: 'completed', removedEntries: 1, freedBytes: 42 }), + builder: service('builder', { absent: true }) + } + const app = express() + app.use((request, response, next) => { + if (request.method === 'POST' && request.path === '/_ops/api/v1/cache-operations') response.once('close', resolveResponseClosed) + next() + }) + app.use('/_ops', createOpsConsoleRouter(ops)) + const authenticated = await login(app, token) + const server = app.listen(0, '127.0.0.1') + await new Promise(resolve => server.once('listening', resolve)) + const outgoing = http.request({ + host: '127.0.0.1', + port: server.address().port, + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers: { + Cookie: cookiePair(authenticated), + Origin: origin, + 'Content-Type': 'application/json', + 'X-Ops-CSRF': authenticated.json.csrfToken + } + }) + outgoing.on('error', () => {}) + outgoing.end(JSON.stringify({ operation: 'cache.evict_commit', targets: ['downloader', 'builder'], commit: 'a'.repeat(40) })) + + await dispatched + outgoing.destroy() + await responseClosed + expect(events.filter(event => event.action === 'cache_operation')).to.have.length(0) + + releases.downloader() + releases.builder() + await audited + await new Promise(resolve => server.close(resolve)) + + const audits = events.filter(event => event.action === 'cache_operation') + expect(audits).to.have.length(1) + expect(audits[0]).to.include({ dispatchStatus: 'dispatched', outcome: 'completed' }) + expect(audits[0].targetOutcomes).to.deep.equal([ + { service: 'downloader', outcome: 'completed', removedEntries: 1, freedBytes: 42, absent: false, skippedInUse: 0, skippedChanged: 0 }, + { service: 'builder', outcome: 'no_op', removedEntries: 0, freedBytes: 0, absent: true, skippedInUse: 0, skippedChanged: 0 } + ]) + }) + + it('enforces the cache-operation session rate before dispatch', async () => { + const token = randomBytes(32).toString('base64url') + let calls = 0 + const client = { json: (path, options) => { calls++; return cacheOperationResponse('builder', options) } } + const app = opsApp({ env: enabled(token), log: () => {}, downloader: client, builder: client }) + const authenticated = await login(app, token) + const options = { + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers: { Cookie: cookiePair(authenticated), Origin: origin, 'Content-Type': 'application/json', 'X-Ops-CSRF': authenticated.json.csrfToken }, + body: JSON.stringify({ operation: 'cache.clear', targets: ['builder'] }) + } + for (let index = 0; index < 5; index++) expect((await request(app, options)).status).to.equal(200) + const limited = await request(app, options) + expect(limited).to.nested.include({ status: 429, 'json.error.code': 'RATE_LIMITED' }) + expect(limited.headers['retry-after']).to.equal('60') + expect(calls).to.equal(5) + }) + + it('returns unknown without retrying when a dispatched target is ambiguous', async () => { + const token = randomBytes(32).toString('base64url') + const events = [] + let calls = 0 + const app = opsApp({ + env: enabled(token), + log: event => events.push(event), + downloader: { json: () => { calls++; return Promise.reject(Object.assign(new Error('secret /path'), { code: 'SERVICE_TIMEOUT' })) } }, + builder: { json: () => { throw new Error('not targeted') } } + }) + const authenticated = await login(app, token) + const response = await request(app, { + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers: { Cookie: cookiePair(authenticated), Origin: origin, 'Content-Type': 'application/json', 'X-Ops-CSRF': authenticated.json.csrfToken }, + body: JSON.stringify({ operation: 'cache.evict_commit', targets: ['downloader'], commit: 'a'.repeat(40) }) + }) + + expect(response).to.nested.include({ status: 200, 'json.outcome': 'unknown', 'json.targets.0.outcome': 'unknown', 'json.targets.0.error.code': 'SERVICE_TIMEOUT' }) + expect(calls).to.equal(1) + const audit = events.find(event => event.action === 'cache_operation') + expect(audit).to.include({ dispatchStatus: 'dispatched', outcome: 'unknown' }) + expect(audit.errorCodes).to.deep.equal(['SERVICE_TIMEOUT']) + expect(JSON.stringify(response.json) + JSON.stringify(audit)).not.to.include('secret').and.not.to.include('/path') + }) + + it('keeps future APIs reserved and returns uniform bounded errors with security headers', async () => { + const token = randomBytes(32).toString('base64url') + const app = opsApp({ env: enabled(token), log: () => {} }) + for (const path of ['/_ops/api/v1/cache-operations', '/_ops/api/v1/unknown']) { + const response = await request(app, { path }) + expect(response).to.nested.include({ status: 404, 'json.error.code': 'NOT_FOUND' }) + expect(response.json.error.correlationId).to.equal(response.headers['x-correlation-id']) + expectSecurityHeaders(response.headers) + expect(response.headers).not.to.have.property('access-control-allow-origin') + } + }) +}) + +function serviceSnapshot (service, observedAt, overrides = {}) { + const capabilityNames = { + router: ['public_file_delivery', 'console_read', 'console_cache_control'], + downloader: ['ref_resolution', 'source_file_delivery', 'source_archive_delivery', 'cache_control'], + builder: ['build_delivery', 'cache_control'] + } + return { + schemaVersion: 1, + service, + instanceId: overrides.instanceId || `${service}-1`, + startedAt: observedAt, + observedAt, + health: { status: 'healthy', reasons: [] }, + capabilities: capabilityNames[service].map(name => ({ name, status: 'available', reasonCode: null })), + queues: [], + cache: null, + dependencies: [], + telemetry: { activityDropped: 0, completedEvicted: 0, failuresEvicted: 0, spansDropped: 0 }, + activity: overrides.activity || [], + failures: overrides.failures || [] + } +} + +function activity (correlationId, service, operation) { + const time = '2026-07-20T12:00:00.000Z' + return { + correlationId, + state: 'completed', + startedAt: time, + completedAt: time, + durationMs: 0, + request: { method: 'GET', route: '/:ref/*', commit: null, resource: null, buildMode: null }, + outcome: { status: 'succeeded', httpStatus: 200, code: null }, + spans: [{ service, operation, state: 'completed', startedAt: time, completedAt: time, durationMs: 0, outcome: { status: 'succeeded', httpStatus: 200, code: null } }] + } +} + +function failure (correlationId, service, operation) { + return { occurredAt: '2026-07-20T12:00:00.000Z', correlationId, service, operation, code: 'INTERNAL_ERROR', summary: 'An internal error occurred', httpStatus: 500, commit: null } +} + +function cacheOperationResponse (service, options, overrides = {}) { + const operation = options.body.operation + const target = { + service, + outcome: 'no_op', + removedEntries: 0, + freedBytes: 0, + absent: false, + skippedInUse: 0, + error: null, + ...overrides + } + if (operation !== 'cache.clear') target.skippedChanged = 0 + return { + correlationId: options.correlationId, + operation, + startedAt: '2026-07-20T12:00:00.000Z', + completedAt: '2026-07-20T12:00:00.000Z', + outcome: target.outcome, + targets: [target] + } +} + +function opsApp (ops) { + return createApp({ ops, router: express.Router().use((request, response) => response.sendStatus(418)) }) +} + +function login (app, token, cookie, extraHeaders = {}) { + const headers = { Origin: 'http://127.0.0.1:8080', 'Content-Type': 'application/json', ...extraHeaders } + if (cookie) headers.Cookie = cookie + return request(app, { method: 'POST', path: '/_ops/api/v1/session', headers, body: JSON.stringify({ token }) }) +} + +function cookiePair (response) { + return response.headers['set-cookie'][0].split(';', 1)[0] +} + +function expectSecurityHeaders (headers) { + for (const [name, value] of Object.entries(SECURITY_HEADERS)) expect(headers[name.toLowerCase()]).to.equal(value) +} + +function request (app, { method = 'GET', path, headers = {}, body } = {}) { + return new Promise((resolve, reject) => { + const server = app.listen(0, '127.0.0.1', () => { + const outgoing = http.request({ host: '127.0.0.1', port: server.address().port, method, path, headers }, response => { + const chunks = [] + response.on('data', chunk => chunks.push(chunk)) + response.on('end', () => { + server.close() + const responseBody = Buffer.concat(chunks).toString() + let json + try { json = JSON.parse(responseBody) } catch (error) {} + resolve({ status: response.statusCode, headers: response.headers, body: responseBody, json }) + }) + }) + outgoing.on('error', error => server.close(() => reject(error))) + if (body) outgoing.write(body) + outgoing.end() + }) + }) +} diff --git a/test/ops-http.js b/test/ops-http.js new file mode 100644 index 0000000..b907865 --- /dev/null +++ b/test/ops-http.js @@ -0,0 +1,445 @@ +'use strict' + +const { expect } = require('chai') +const { randomBytes } = require('node:crypto') +const { Readable } = require('node:stream') +const { describe, it } = require('mocha') +const { + MAX_BODY_BYTES, + OpsHttpError, + SECURITY_HEADERS, + errorBody, + getTrustedRequestContext, + readStrictJSON, + requireBearer, + requireCSRF, + requireJSONContentType, + requireOrigin, + requireSameOriginFetch, + setSecurityHeaders, + strictJSON +} = require('../app/ops/http') +const { + deriveCacheOutcome, + validateCacheOperationRequest, + validateCacheOperationResponse, + validateLoginRequest, + validateServiceSnapshot +} = require('../app/ops/schemas') +const { + ABSOLUTE_TIMEOUT_MS, + IDLE_TIMEOUT_MS, + OpsRateLimiter, + SESSION_CAPACITY, + SessionStore, + createAuditId, + sessionResponse +} = require('../app/ops/sessions') + +describe('operations console HTTP boundaries', () => { + it('parses one bounded UTF-8 object and rejects duplicate or ambiguous JSON', async () => { + expect(strictJSON('{"name":"ok","nested":{"x":1}}')).to.deep.equal({ name: 'ok', nested: { x: 1 } }) + expect(await readStrictJSON(Readable.from(['{"ok":', 'true}']))).to.deep.equal({ ok: true }) + const bounded = `{"value":"${'x'.repeat(MAX_BODY_BYTES - 12)}"}` + expect(Buffer.byteLength(bounded)).to.equal(MAX_BODY_BYTES) + expect(strictJSON(bounded).value).to.have.length(MAX_BODY_BYTES - 12) + for (const input of [ + '{"x":1,"x":2}', + '{"x":1,"\\u0078":2}', + '[]', + 'null', + '{"x":NaN}', + '{"x":1} trailing', + '{\u00a0"x":1}', + Buffer.from([0xc3, 0x28]) + ]) expect(() => strictJSON(input)).to.throw(OpsHttpError).with.property('code', 'INVALID_JSON') + expect(() => strictJSON(Buffer.alloc(MAX_BODY_BYTES + 1))).to.throw(OpsHttpError).with.property('status', 413) + }) + + it('sets every required security header exactly and no CORS or HSTS header', () => { + const headers = {} + setSecurityHeaders({ setHeader: (name, value) => { headers[name] = value } }) + expect(headers).to.deep.equal(SECURITY_HEADERS) + expect(headers).not.to.have.keys('Access-Control-Allow-Origin', 'Strict-Transport-Security') + }) + + it('enforces exact Origin, Fetch Metadata, JSON media type, CSRF, and bearer credentials', () => { + const request = { + headers: { + origin: 'https://ops.example.test', + 'sec-fetch-site': 'same-origin', + 'content-type': 'application/json; charset=utf-8', + 'x-ops-csrf': 'csrf', + authorization: 'Bearer internal-token' + } + } + expect(() => requireOrigin(request, 'https://ops.example.test')).not.to.throw() + expect(() => requireSameOriginFetch(request)).not.to.throw() + expect(() => requireJSONContentType(request)).not.to.throw() + expect(() => requireCSRF(request, 'csrf')).not.to.throw() + expect(() => requireBearer(request, 'internal-token')).not.to.throw() + + for (const guard of [ + () => requireOrigin({ headers: {} }, 'https://ops.example.test'), + () => requireOrigin({ headers: { origin: 'https://OPS.example.test' } }, 'https://ops.example.test'), + () => requireSameOriginFetch({ headers: { 'sec-fetch-site': 'cross-site' } }), + () => requireJSONContentType({ headers: { 'content-type': 'application/json; charset=latin1' } }), + () => requireCSRF({ headers: { 'x-ops-csrf': 'wrong' } }, 'csrf'), + () => requireBearer({ headers: { authorization: 'Bearer wrong' } }, 'internal-token') + ]) expect(guard).to.throw(OpsHttpError) + }) + + it('allows Fetch Metadata none only for an explicitly permitted passive document navigation', () => { + const navigation = { + method: 'GET', + headers: { + 'sec-fetch-site': 'none', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-dest': 'document' + } + } + expect(() => requireSameOriginFetch({ headers: {} })).not.to.throw() + expect(() => requireSameOriginFetch({ headers: { 'sec-fetch-site': 'same-origin' } })).not.to.throw() + expect(() => requireSameOriginFetch(navigation, { allowTopLevelNavigation: true })).not.to.throw() + + for (const request of [ + navigation, + { ...navigation, method: 'POST' }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-mode': 'cors' } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-mode': '' } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-mode': ['navigate'] } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-mode': undefined } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-dest': 'script' } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-dest': '' } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-dest': ['document'] } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-dest': undefined } }, + { ...navigation, headers: { ...navigation.headers, 'sec-fetch-site': 'cross-site' } } + ]) { + expect(() => requireSameOriginFetch(request)).to.throw(OpsHttpError).that.includes({ + status: 403, + code: 'FETCH_METADATA_REJECTED' + }) + } + }) + + it('fails closed for absent, empty, or non-string CSRF and bearer credentials', () => { + for (const headers of [ + {}, + { 'x-ops-csrf': '' }, + { 'x-ops-csrf': ['csrf'] } + ]) { + expect(() => requireCSRF({ headers }, 'csrf')).to.throw(OpsHttpError).that.includes({ status: 403, code: 'CSRF_REJECTED' }) + } + for (const headers of [ + {}, + { authorization: '' }, + { authorization: ['Bearer internal-token'] }, + { authorization: 'Bearer ' } + ]) { + expect(() => requireBearer({ headers }, 'internal-token')).to.throw(OpsHttpError).that.includes({ status: 401, code: 'UNAUTHORIZED' }) + } + for (const expected of [undefined, '', null, 1]) { + expect(() => requireCSRF({ headers: { 'x-ops-csrf': '' } }, expected)).to.throw('Invalid expected credential') + expect(() => requireBearer({ headers: {} }, expected)).to.throw('Invalid expected credential') + } + }) + + it('requires an authorized encrypted socket for HTTPS and ignores forwarded claims', () => { + const context = getTrustedRequestContext({ + socket: { remoteAddress: '10.2.3.4', encrypted: true, authorized: true }, + headers: { + forwarded: 'for=192.0.2.8;proto=http', + 'x-forwarded-for': '192.0.2.8', + 'x-forwarded-proto': 'http' + } + }, { protocol: 'https' }) + expect(context).to.deep.equal({ protocol: 'https', source: null }) + + for (const socket of [ + { encrypted: false, authorized: true }, + { encrypted: true, authorized: false }, + { encrypted: true }, + {} + ]) expect(() => getTrustedRequestContext({ socket, headers: {} }, { protocol: 'https' })).to.throw(OpsHttpError) + }) + + it('requires direct HTTP request sources to be loopback', () => { + const config = { protocol: 'http' } + expect(getTrustedRequestContext({ socket: { remoteAddress: '::1' }, headers: { 'x-forwarded-for': '192.0.2.1' } }, config).source).to.equal('::1') + expect(() => getTrustedRequestContext({ socket: { remoteAddress: '192.0.2.1' }, headers: {} }, config)).to.throw(OpsHttpError) + expect(() => getTrustedRequestContext({ socket: { remoteAddress: '127.0.0.1', encrypted: true }, headers: {} }, config)).to.throw(OpsHttpError) + }) + + it('produces bounded generic errors without reflecting sensitive values', () => { + const secret = randomBytes(32).toString('base64url') + const internal = errorBody(new Error(`failed for ${secret}`), 'correlation-id') + expect(internal).to.deep.equal({ status: 500, body: { error: { code: 'INTERNAL_ERROR', message: 'An internal error occurred', correlationId: 'correlation-id' } } }) + expect(JSON.stringify(internal)).not.to.include(secret) + + const safe = errorBody(new OpsHttpError(400, 'INVALID_REQUEST', 'Request body is invalid', ['token', 'token', secret.repeat(2)]), 'id') + expect(safe.body.error.details.fields).to.deep.equal(['token']) + }) +}) + +describe('operations console schemas', () => { + const time = '2026-07-17T12:00:00.000Z' + + it('keeps malformed login attempts generic and accepts only an exact canonical token object', () => { + const token = randomBytes(32).toString('base64url') + expect(validateLoginRequest({ token })).to.deep.equal({ token }) + for (const body of [ + undefined, + null, + [], + 'token', + {}, + { token, extra: true }, + { token: undefined }, + { token: null }, + { token: '' }, + { token: 'not-valid' }, + { token: 'x'.repeat(513) } + ]) { + expect(() => validateLoginRequest(body)).to.throw(OpsHttpError).that.includes({ + status: 401, + code: 'UNAUTHORIZED', + message: 'Authentication failed', + fields: undefined + }) + } + let loginError + try { validateLoginRequest({ token, extra: true }) } catch (error) { loginError = error } + const failure = errorBody(loginError, 'login-failure') + expect(failure).to.deep.equal({ + status: 401, + body: { error: { code: 'UNAUTHORIZED', message: 'Authentication failed', correlationId: 'login-failure' } } + }) + expect(JSON.stringify(failure)).not.to.include(token) + }) + + it('strictly validates named cache requests', () => { + const commit = 'a'.repeat(40) + expect(validateCacheOperationRequest({ operation: 'cache.evict_commit', targets: ['downloader', 'builder'], commit })).to.deep.equal({ operation: 'cache.evict_commit', targets: ['downloader', 'builder'], commit }) + expect(validateCacheOperationRequest({ operation: 'cache.clear', targets: ['builder'] })).to.deep.equal({ operation: 'cache.clear', targets: ['builder'] }) + for (const body of [ + { operation: 'cache.clear', targets: ['builder'], commit }, + { operation: 'cache.purge_expired', targets: ['builder', 'downloader'] }, + { operation: 'cache.evict_commit', targets: ['builder'], commit: commit.toUpperCase() }, + { operation: 'cache.clear', targets: ['builder', 'builder'] }, + { operation: 'cleanup', targets: ['builder'] } + ]) expect(() => validateCacheOperationRequest(body)).to.throw(OpsHttpError) + }) + + it('derives and validates cache outcomes without treating absent as partial', () => { + const target = (service, outcome, values = {}) => ({ + service, + outcome, + removedEntries: 0, + freedBytes: 0, + absent: false, + skippedInUse: 0, + skippedChanged: 0, + error: null, + ...values + }) + expect(deriveCacheOutcome([target('builder', 'completed', { removedEntries: 1 }), target('downloader', 'no_op', { absent: true })])).to.equal('completed') + expect(deriveCacheOutcome([target('builder', 'partial', { removedEntries: 1, skippedInUse: 1 })])).to.equal('partial') + expect(deriveCacheOutcome([target('builder', 'unknown')])).to.equal('unknown') + + const response = { + correlationId: 'correlation-id', + operation: 'cache.evict_commit', + startedAt: time, + completedAt: time, + outcome: 'completed', + targets: [target('builder', 'completed', { removedEntries: 1 })], + ignoredAdditiveField: true + } + expect(validateCacheOperationResponse(response).outcome).to.equal('completed') + expect(() => validateCacheOperationResponse({ ...response, outcome: 'partial' })).to.throw() + }) + + it('accepts additive internal fields but rejects missing, incompatible, and unbounded snapshots', () => { + const snapshot = { + schemaVersion: 1, + service: 'builder', + instanceId: 'instance-id', + startedAt: time, + observedAt: time, + health: { status: 'healthy', reasons: [] }, + capabilities: [{ name: 'build_delivery', status: 'available', reasonCode: null }], + queues: [{ name: 'build', active: 1, queued: 1, limit: 4, available: 2, oldestQueuedAgeMs: 10 }], + cache: null, + dependencies: [], + telemetry: { activityDropped: 0, completedEvicted: 0, failuresEvicted: 0, spansDropped: 0 }, + futureField: true + } + expect(validateServiceSnapshot(snapshot, 'builder')).to.include({ schemaVersion: 1, service: 'builder' }) + for (const invalid of [ + { ...snapshot, schemaVersion: 2 }, + { ...snapshot, service: 'downloader' }, + { ...snapshot, queues: [{ ...snapshot.queues[0], available: 3 }] }, + { ...snapshot, capabilities: Array(17).fill(snapshot.capabilities[0]) }, + { ...snapshot, health: { status: 'degraded', reasons: [] } }, + { ...snapshot, activity: [{ correlationId: 'id', state: 'active' }] }, + { ...snapshot, instanceId: 'x'.repeat(65) }, + { ...snapshot, observedAt: 'not-a-time' } + ]) expect(() => validateServiceSnapshot(invalid, 'builder')).to.throw().with.property('code', 'INCOMPATIBLE_SCHEMA') + }) +}) + +describe('operations console sessions', () => { + const start = Date.parse('2026-07-17T12:00:00.000Z') + + it('creates opaque server-side sessions with bounded public state and audit IDs', () => { + const store = new SessionStore({ now: () => start }) + const created = store.create() + const response = sessionResponse(created.session) + + expect(created.sessionId).to.match(/^[A-Za-z0-9_-]{43}$/) + expect(created.session.csrfToken).to.match(/^[A-Za-z0-9_-]{43}$/) + expect(created.session.auditId).to.match(/^[A-Za-z0-9_-]{22}$/) + expect(new Set([created.sessionId, created.session.csrfToken, created.session.auditId]).size).to.equal(3) + expect(response).to.deep.equal({ + authenticated: true, + idleExpiresAt: new Date(start + IDLE_TIMEOUT_MS).toISOString(), + absoluteExpiresAt: new Date(start + ABSOLUTE_TIMEOUT_MS).toISOString(), + csrfToken: created.session.csrfToken + }) + expect(response).not.to.have.keys('sessionId', 'auditId', 'generation') + expect(createAuditId()).to.match(/^[A-Za-z0-9_-]{22}$/) + }) + + it('renews idle lifetime exactly but never extends absolute lifetime', () => { + let now = start + const store = new SessionStore({ now: () => now }) + const { sessionId } = store.create() + + for (now = start + IDLE_TIMEOUT_MS - 1; now < start + ABSOLUTE_TIMEOUT_MS; now += IDLE_TIMEOUT_MS - 1) { + expect(store.authenticate(sessionId)).not.to.equal(null) + } + now = start + ABSOLUTE_TIMEOUT_MS - 1 + expect(store.authenticate(sessionId, { renew: false })).not.to.equal(null) + now++ + expect(store.authenticate(sessionId)).to.equal(null) + expect(store.size).to.equal(0) + }) + + it('rejects at the exact idle boundary and pruning cannot revive expiry', () => { + let now = start + const store = new SessionStore({ now: () => now }) + const { sessionId } = store.create() + now += IDLE_TIMEOUT_MS + + expect(store.pruneExpired()).to.equal(1) + now = start + expect(store.authenticate(sessionId)).to.equal(null) + }) + + it('rotates replacement logins, logout revocation, and verifier generations', () => { + const store = new SessionStore({ now: () => start, generation: 4 }) + const first = store.create() + const replacement = store.create(first.sessionId) + expect(replacement.sessionId).not.to.equal(first.sessionId) + expect(store.authenticate(first.sessionId)).to.equal(null) + expect(store.authenticate(replacement.sessionId)).not.to.equal(null) + expect(store.revoke(replacement.sessionId)).to.equal(true) + expect(store.revoke(replacement.sessionId)).to.equal(false) + + const rotated = store.create() + expect(store.rotate(5)).to.equal(1) + expect(store.authenticate(rotated.sessionId)).to.equal(null) + expect(store.rotate(5)).to.equal(0) + }) + + it('has hard fail-closed capacity without evicting live sessions', () => { + const store = new SessionStore({ now: () => start }) + const ids = Array.from({ length: SESSION_CAPACITY }, () => store.create().sessionId) + expect(() => store.create()).to.throw(OpsHttpError).that.includes({ status: 503, code: 'SESSION_CAPACITY' }) + expect(store.size).to.equal(SESSION_CAPACITY) + expect(ids.every(id => store.authenticate(id, { renew: false }))).to.equal(true) + + const replacement = store.create(ids[0]) + expect(store.size).to.equal(SESSION_CAPACITY) + expect(store.authenticate(ids[0])).to.equal(null) + expect(store.authenticate(replacement.sessionId)).not.to.equal(null) + }) + + it('uses Phase 1.1 CSRF guards and never leaks session secrets in failures', () => { + const store = new SessionStore({ now: () => start }) + const { sessionId, session } = store.create() + const request = { headers: { 'x-ops-csrf': session.csrfToken } } + expect(() => requireCSRF(request, session.csrfToken)).not.to.throw() + expect(() => requireCSRF({ headers: {} }, session.csrfToken)).to.throw(OpsHttpError) + + const failure = errorBody(new OpsHttpError(503, 'SESSION_CAPACITY', 'Session capacity is unavailable'), 'audit-safe') + expect(JSON.stringify(failure)).not.to.include(sessionId) + expect(JSON.stringify(failure)).not.to.include(session.csrfToken) + expect(JSON.stringify(failure)).not.to.include(session.auditId) + expect(store.authenticate(`missing-${sessionId}`)).to.equal(null) + }) +}) + +describe('operations console rate limits', () => { + const start = Date.parse('2026-07-17T12:00:00.000Z') + + it('enforces only the process-wide 30-attempt login limit for 15 minutes', () => { + let now = start + const limiter = new OpsRateLimiter({ now: () => now }) + for (let index = 0; index < 30; index++) expect(limiter.attemptLogin()).to.include({ allowed: true, remaining: 29 - index }) + expect(limiter.attemptLogin()).to.include({ allowed: false, retryAfter: 900 }) + now += 15 * 60 * 1000 - 1 + expect(limiter.attemptLogin()).to.include({ allowed: false, retryAfter: 1 }) + now++ + expect(limiter.attemptLogin()).to.include({ allowed: true, remaining: 29 }) + }) + + it('enforces snapshot, per-session cache, and global cache limits', () => { + let now = start + const limiter = new OpsRateLimiter({ now: () => now }) + for (let index = 0; index < 12; index++) expect(limiter.attemptSnapshot('session-a').allowed).to.equal(true) + expect(limiter.attemptSnapshot('session-a')).to.include({ allowed: false, retryAfter: 60 }) + + for (let index = 0; index < 5; index++) expect(limiter.attemptCacheOperation('session-a').allowed).to.equal(true) + expect(limiter.attemptCacheOperation('session-a')).to.include({ allowed: false, retryAfter: 60 }) + for (let index = 0; index < 14; index++) expect(limiter.attemptCacheOperation(`session-${index + 1}`).allowed).to.equal(true) + expect(limiter.attemptCacheOperation('session-global')).to.include({ allowed: false, retryAfter: 60 }) + + now += 60 * 1000 + expect(limiter.attemptSnapshot('session-a')).to.include({ allowed: true, remaining: 11 }) + expect(limiter.attemptCacheOperation('session-a')).to.include({ allowed: true, remaining: 4 }) + }) + + it('cleans session buckets and bounds their storage without eviction', () => { + let now = start + const limiter = new OpsRateLimiter({ now: () => now }) + limiter.attemptSnapshot('session') + limiter.attemptCacheOperation('session') + limiter.removeSession('session') + expect(limiter.bucketCounts).to.deep.equal({ snapshotSessions: 0, cacheSessions: 0 }) + + for (let index = 0; index < SESSION_CAPACITY; index++) { + limiter.attemptSnapshot(`snapshot-${index}`) + limiter.attemptCacheOperation(`cache-${index}`) + } + expect(limiter.attemptSnapshot('snapshot-overflow').allowed).to.equal(false) + expect(limiter.attemptCacheOperation('cache-overflow').allowed).to.equal(false) + expect(limiter.bucketCounts).to.deep.equal({ + snapshotSessions: SESSION_CAPACITY, + cacheSessions: SESSION_CAPACITY + }) + + now += 15 * 60 * 1000 + limiter.prune() + expect(limiter.bucketCounts).to.deep.equal({ snapshotSessions: 0, cacheSessions: 0 }) + }) + + it('returns generic rate results without retaining or exposing keys', () => { + const limiter = new OpsRateLimiter({ now: () => start }) + const secret = randomBytes(32).toString('base64url') + const result = limiter.attemptSnapshot(secret) + expect(result).to.deep.equal({ allowed: true, remaining: 11, retryAfter: 0, resetAt: start + 60000 }) + expect(JSON.stringify(result)).not.to.include(secret) + expect(JSON.stringify(limiter.bucketCounts)).not.to.include(secret) + }) +}) diff --git a/test/ops-internal.js b/test/ops-internal.js new file mode 100644 index 0000000..bfe535f --- /dev/null +++ b/test/ops-internal.js @@ -0,0 +1,156 @@ +'use strict' + +const { expect } = require('chai') +const http = require('node:http') +const express = require('express') +const { describe, it } = require('mocha') +const sinon = require('sinon') +const { createInternalOpsRouter, MAX_REQUEST_BYTES } = require('../app/ops/internal-router') +const { createServiceClient } = require('../app/service-client') + +const TIME = '2026-07-17T12:00:00.000Z' + +function snapshot () { + return { + schemaVersion: 1, + service: 'builder', + instanceId: 'builder-instance', + startedAt: TIME, + observedAt: TIME, + health: { status: 'healthy', reasons: [] }, + capabilities: [{ name: 'build_delivery', status: 'available', reasonCode: null }], + queues: [], + cache: null, + dependencies: [], + telemetry: { activityDropped: 0, completedEvicted: 0, failuresEvicted: 0, spansDropped: 0 }, + activity: [], + failures: [] + } +} + +function target (values = {}) { + return { + service: 'builder', + outcome: 'no_op', + removedEntries: 0, + freedBytes: 0, + absent: true, + skippedInUse: 0, + skippedChanged: 0, + error: null, + ...values + } +} + +function request (app, path, options = {}) { + return new Promise((resolve, reject) => { + const server = app.listen(0, () => { + const request = http.request({ port: server.address().port, path, method: options.method || 'GET', headers: options.headers }, response => { + const chunks = [] + response.on('data', chunk => chunks.push(chunk)) + response.on('end', () => { + server.close() + resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks) }) + }) + }) + request.on('error', reject) + request.end(options.body) + }) + }) +} + +function appFor (options = {}) { + const app = express() + app.use(createInternalOpsRouter({ + token: 'secret', + service: 'builder', + snapshot, + cache: { execute: sinon.stub().resolves(target()) }, + now: () => Date.parse(TIME), + ...options + })) + return app +} + +describe('internal operations router', () => { + it('protects both routes before processing input and returns safe correlated errors', async () => { + const app = appFor() + const unauthorized = await request(app, '/v1/ops/snapshot', { headers: { 'X-Correlation-ID': 'request-id' } }) + expect(unauthorized.status).to.equal(401) + expect(unauthorized.headers['x-correlation-id']).to.equal('request-id') + expect(JSON.parse(unauthorized.body)).to.deep.equal({ error: { code: 'UNAUTHORIZED', message: 'Authentication required', correlationId: 'request-id' } }) + + const failure = await request(appFor({ snapshot: () => { throw new Error('secret path') } }), '/v1/ops/snapshot', { headers: { Authorization: 'Bearer secret' } }) + expect(failure.status).to.equal(500) + expect(JSON.parse(failure.body).error).to.include({ code: 'INTERNAL_ERROR', message: 'An internal error occurred' }) + expect(failure.body.toString()).not.to.include('secret path') + }) + + it('returns only a validated schema-v1 snapshot and propagates correlation', async () => { + const provider = sinon.stub().resolves(snapshot()) + const response = await request(appFor({ snapshot: provider }), '/v1/ops/snapshot', { + headers: { Authorization: 'Bearer secret', 'X-Correlation-ID': 'correlation-id' } + }) + expect(response.status).to.equal(200) + expect(response.headers['x-correlation-id']).to.equal('correlation-id') + expect(response.headers['cache-control']).to.equal('no-store') + expect(JSON.parse(response.body)).to.deep.equal(snapshot()) + expect(provider.calledOnceWithExactly('correlation-id')).to.equal(true) + + const invalid = await request(appFor({ snapshot: () => ({ ...snapshot(), schemaVersion: 2 }) }), '/v1/ops/snapshot', { headers: { Authorization: 'Bearer secret' } }) + expect(invalid.status).to.equal(500) + expect(JSON.parse(invalid.body).error.code).to.equal('INTERNAL_ERROR') + }) + + it('strictly bounds and validates service-specific cache commands', async () => { + const execute = sinon.stub().resolves(target()) + const app = appFor({ cache: { execute } }) + const response = await request(app, '/v1/ops/cache-operations', { + method: 'POST', + headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json', 'X-Correlation-ID': 'operation-id' }, + body: JSON.stringify({ operation: 'cache.evict_commit', targets: ['builder'], commit: 'a'.repeat(40) }) + }) + expect(response.status).to.equal(200) + expect(JSON.parse(response.body)).to.deep.equal({ + correlationId: 'operation-id', + operation: 'cache.evict_commit', + startedAt: TIME, + completedAt: TIME, + outcome: 'no_op', + targets: [target()] + }) + expect(execute.calledOnceWithExactly('cache.evict_commit', 'a'.repeat(40))).to.equal(true) + + for (const options of [ + { headers: { Authorization: 'Bearer secret' }, body: '{}' }, + { headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, body: '{"operation":"cache.clear","operation":"cache.clear","targets":["builder"]}' }, + { headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, body: JSON.stringify({ operation: 'cache.clear', targets: ['downloader'] }) }, + { headers: { Authorization: 'Bearer secret', 'Content-Type': 'application/json' }, body: Buffer.alloc(MAX_REQUEST_BYTES + 1, 32) } + ]) { + const rejected = await request(app, '/v1/ops/cache-operations', { method: 'POST', ...options }) + expect(rejected.status).to.be.oneOf([400, 413, 415]) + } + expect(execute.callCount).to.equal(1) + }) +}) + +describe('bounded internal service JSON transport', () => { + it('bounds request and response bodies and sends correlation without retries', async () => { + const fetch = sinon.stub().resolves(new Response('{"ok":true}', { headers: { 'Content-Type': 'application/json' } })) + const client = createServiceClient({ baseURL: 'http://builder', token: 'secret', fetch }) + expect(await client.json('/v1/ops/snapshot', { correlationId: 'request-id', maxResponseBytes: 64 })).to.deep.equal({ ok: true }) + expect(fetch.calledOnce).to.equal(true) + expect(fetch.firstCall.args[1].headers['X-Correlation-ID']).to.equal('request-id') + + let error + try { await client.json('/v1/ops/cache-operations', { method: 'POST', body: { value: 'x'.repeat(20) }, maxRequestBytes: 8, maxResponseBytes: 64 }) } catch (caught) { error = caught } + expect(error).to.include({ code: 'REQUEST_TOO_LARGE', status: 413 }) + expect(fetch.calledOnce).to.equal(true) + + fetch.resetBehavior() + fetch.resolves(new Response(JSON.stringify({ value: 'x'.repeat(100) }))) + try { await client.json('/v1/ops/snapshot', { maxResponseBytes: 32 }) } catch (caught) { error = caught } + expect(error).to.include({ code: 'SERVICE_RESPONSE_TOO_LARGE', status: 502 }) + expect(fetch.callCount).to.equal(2) + }) +}) diff --git a/test/ops-telemetry.js b/test/ops-telemetry.js new file mode 100644 index 0000000..902b730 --- /dev/null +++ b/test/ops-telemetry.js @@ -0,0 +1,166 @@ +'use strict' + +const assert = require('node:assert/strict') +const { describe, it } = require('mocha') + +const { JobQueue } = require('../app/JobQueue') +const { LIMITS, Telemetry, cacheSnapshot, mergeActivity, normalizeResource, serviceSlot } = require('../app/ops/telemetry') + +function clock () { + let time = Date.parse('2026-01-01T00:00:00.000Z') + return { now: () => time, advance: amount => { time += amount } } +} + +function telemetry (time, options = {}) { + return new Telemetry({ + service: 'router', + operations: ['deliver', ...Array.from({ length: LIMITS.spans + 1 }, (_, index) => `operation-${index}`)], + routes: ['/files/:commit/*'], + failureSummaries: { UPSTREAM_FAILED: 'Upstream request failed' }, + now: time.now, + instanceId: 'instance-1', + ...options + }) +} + +describe('operations telemetry', () => { + it('bounds and expires completed traces and failures deterministically', () => { + const time = clock() + const store = telemetry(time) + const activeStore = telemetry(time) + for (let index = 0; index <= LIMITS.active; index++) { + activeStore.startTrace({ correlationId: `active-${index}`, method: 'GET', route: '/files/:commit/*' }) + } + assert.equal(activeStore.activity().length, LIMITS.active) + assert.equal(activeStore.activity().some(trace => trace.correlationId === 'active-0'), false) + assert.equal(activeStore.counters.activityDropped, 1) + + for (let index = 0; index <= LIMITS.completed; index++) { + store.startTrace({ correlationId: `request-${index}`, method: 'GET', route: '/files/:commit/*' }) + store.completeTrace(`request-${index}`, { status: 'succeeded' }) + } + assert.equal(store.activity().length, LIMITS.completed) + assert.equal(store.counters.completedEvicted, 1) + + for (let index = 0; index <= LIMITS.failures; index++) { + store.recordFailure({ correlationId: `failure-${index}`, operation: 'deliver', code: 'UPSTREAM_FAILED' }) + } + assert.equal(store.getFailures().length, LIMITS.failures) + assert.equal(store.counters.failuresEvicted, 1) + + time.advance(LIMITS.failureAgeMs + 1) + assert.equal(store.getFailures().length, 0) + assert.equal(store.counters.failuresEvicted, LIMITS.failures + 1) + }) + + it('caps local spans and merges only correlated, unique fragments', () => { + const time = clock() + const store = telemetry(time) + store.startTrace({ correlationId: 'known', method: 'GET', route: '/files/:commit/*' }) + for (let index = 0; index <= LIMITS.spans; index++) { + store.startSpan('known', `operation-${index}`) + } + const trace = store.completeTrace('known', { status: 'succeeded' }) + assert.equal(trace.spans.length, LIMITS.spans) + assert.equal(store.counters.spansDropped, 1) + + const merged = mergeActivity([trace], [ + { correlationId: 'orphan', spans: [{ service: 'builder', operation: 'build' }] }, + { correlationId: 'known', spans: [{ service: 'builder', operation: 'build' }, { service: 'builder', operation: 'build' }] } + ], ['build']) + assert.equal(merged.activity.length, 1) + assert.equal(merged.activity[0].spans.length, LIMITS.spans) + assert.equal(merged.spansDropped, 2) + }) + + it('derives passive dependency and rolling health semantics', () => { + const time = clock() + const store = telemetry(time) + assert.deepEqual(store.dependencies(), []) + assert.equal(store.recordDependency('github', { succeeded: false, errorCode: 'RATE_LIMITED', latencyMs: 10 }).status, 'unavailable') + assert.equal(store.recordDependency('github', { succeeded: true, latencyMs: 5 }).status, 'available') + assert.equal(store.recordDependency('github', { succeeded: false, errorCode: 'UPSTREAM_FAILED', latencyMs: 7 }).status, 'degraded') + + for (let index = 0; index < 10; index++) store.recordWindow(index < 3 ? 'failed' : 'succeeded', 80, 100) + assert.deepEqual(store.healthSignals([]), { + queueSaturated: false, + capacityRejected: false, + reliabilityDegraded: true, + latencyDegraded: true, + dependencyDegraded: true + }) + time.advance(LIMITS.healthWindowMs + 1) + assert.equal(store.healthSignals([]).reliabilityDegraded, false) + }) + + it('reports queue state without exposing jobs or arguments', async () => { + const queue = new JobQueue() + let release + const blocked = new Promise(resolve => { release = resolve }) + const first = queue.addJob('compile', 'metrics-active', { func: () => blocked, args: ['secret'] }) + const second = queue.addJob('compile', 'metrics-waiting', { func: async () => {}, args: ['also-secret'] }) + const metrics = queue.getMetrics('compile', Date.now() + 25) + assert.deepEqual(Object.keys(metrics), ['active', 'queued', 'limit', 'available', 'oldestQueuedAgeMs']) + assert.equal(metrics.active, 1) + assert.equal(metrics.queued, 1) + assert.equal(metrics.available, 0) + assert.ok(metrics.oldestQueuedAgeMs >= 25) + assert.doesNotMatch(JSON.stringify(metrics), /secret/) + release() + await Promise.all([first, second]) + }) + + it('assembles bounded cache summaries and deterministic freshness slots', () => { + const entries = Array.from({ length: LIMITS.cacheEntries + 1 }, (_, index) => ({ + commit: index.toString(16).padStart(40, '0'), + sizeBytes: 2, + lastAccessedAt: new Date(index * 1000).toISOString(), + expiresAt: new Date(index * 1000 + 10000).toISOString(), + inUse: 0, + path: '/secret/cache' + })) + const cache = cacheSnapshot(entries, 1000) + assert.equal(cache.entryCount, LIMITS.cacheEntries + 1) + assert.equal(cache.totalBytes, (LIMITS.cacheEntries + 1) * 2) + assert.equal(cache.entries.length, LIMITS.cacheEntries) + assert.equal(cache.entriesTruncated, true) + assert.doesNotMatch(JSON.stringify(cache), /secret|path/) + + const snapshot = { observedAt: '2026-01-01T00:00:00.000Z', token: undefined } + assert.equal(serviceSlot({ lastSuccess: snapshot }, snapshot.observedAt).freshness, 'fresh') + assert.equal(serviceSlot({ lastSuccess: snapshot, error: { code: 'TIMEOUT' } }, Date.parse(snapshot.observedAt) + 1).freshness, 'stale') + assert.equal(serviceSlot({ lastSuccess: snapshot }, Date.parse(snapshot.observedAt) + LIMITS.staleMs + 1).freshness, 'unknown') + }) + + it('sanitizes request resources and failures by construction', () => { + const time = clock() + const store = telemetry(time) + const trace = store.startTrace({ + correlationId: 'safe-id', + method: 'GET', + route: '/files/:commit/*', + resource: 'https://example.test/private?token=secret', + headers: { authorization: 'secret' }, + args: ['secret'] + }) + const failure = store.recordFailure({ + correlationId: 'safe-id', + operation: 'deliver', + code: 'UNKNOWN', + summary: 'secret', + stack: '/Users/private/file.js' + }) + assert.equal(trace.request.resource, null) + assert.equal(failure.code, 'INTERNAL_ERROR') + assert.equal(failure.summary, 'An internal error occurred') + assert.doesNotMatch(JSON.stringify({ trace, failure }), /secret|authorization|Users|stack/) + }) + + it('truncates bounded text without splitting astral characters', () => { + const astralLetter = '\u{10400}' + const exact = `${'a'.repeat(252)}${astralLetter}` + assert.equal(Buffer.byteLength(exact), 256) + assert.equal(normalizeResource(exact), exact) + assert.equal(normalizeResource(`${'a'.repeat(253)}${astralLetter}`), 'a'.repeat(253)) + }) +}) diff --git a/test/ops/fixture-service.js b/test/ops/fixture-service.js new file mode 100644 index 0000000..f970ae4 --- /dev/null +++ b/test/ops/fixture-service.js @@ -0,0 +1,141 @@ +'use strict' + +const http = require('node:http') + +const service = process.env.OPS_TEST_SERVICE +const token = process.env.INTERNAL_SERVICE_TOKEN +const port = Number(process.env.PORT || 8080) +const commit = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +const modes = { snapshot: 'fresh', cache: 'healthy' } +const calls = { snapshot: 0, cache: 0 } + +if (!['downloader', 'builder'].includes(service) || !token) throw new Error('Invalid fixture configuration') + +http.createServer(async (request, response) => { + try { + const url = new URL(request.url, 'http://fixture') + if (url.pathname === '/health') return json(response, 200, { status: 'ok' }) + if (url.pathname.startsWith('/v1/') || url.pathname.startsWith('/__ops-test/')) { + if (request.headers.authorization !== `Bearer ${token}`) return json(response, 401, { error: { code: 'UNAUTHORIZED', message: 'Unauthorized' } }) + } + + if (request.method === 'POST' && url.pathname === '/__ops-test/control') { + const body = await readJSON(request) + if (body.snapshot && !['fresh', 'slow', 'disconnect', 'malformed', 'oversized'].includes(body.snapshot)) return json(response, 400, { ok: false }) + if (body.cache && !['healthy', 'slow', 'disconnect'].includes(body.cache)) return json(response, 400, { ok: false }) + if (body.snapshot) modes.snapshot = body.snapshot + if (body.cache) modes.cache = body.cache + if (body.resetCounts) Object.assign(calls, { snapshot: 0, cache: 0 }) + return json(response, 200, { ok: true }) + } + if (request.method === 'GET' && url.pathname === '/__ops-test/status') return json(response, 200, { service, modes, calls }) + + if (request.method === 'GET' && url.pathname === '/v1/ops/snapshot') { + calls.snapshot++ + if (modes.snapshot === 'slow') await delay(1700) + if (modes.snapshot === 'disconnect') return request.socket.destroy() + if (modes.snapshot === 'malformed') return raw(response, 200, 'application/json', '{not-json') + if (modes.snapshot === 'oversized') return raw(response, 200, 'application/json', JSON.stringify({ padding: 'x'.repeat(1024 * 1024) })) + return json(response, 200, snapshot(request.headers['x-correlation-id'])) + } + + if (request.method === 'POST' && url.pathname === '/v1/ops/cache-operations') { + calls.cache++ + const body = await readJSON(request) + if (modes.cache === 'slow') await delay(10500) + if (modes.cache === 'disconnect') return request.socket.destroy() + const removedEntries = body.operation === 'cache.clear' ? 1 : 0 + const target = { + service, + outcome: removedEntries ? 'completed' : 'no_op', + removedEntries, + freedBytes: removedEntries ? 128 : 0, + absent: body.operation === 'cache.evict_commit', + skippedInUse: 0, + error: null, + ...(body.operation === 'cache.clear' ? {} : { skippedChanged: 0 }) + } + const now = new Date().toISOString() + return json(response, 200, { + correlationId: request.headers['x-correlation-id'], + operation: body.operation, + startedAt: now, + completedAt: now, + outcome: target.outcome, + targets: [target] + }) + } + + if (service === 'downloader' && request.method === 'POST' && url.pathname === '/v1/resolve') { + const body = await readJSON(request) + return json(response, 200, { commit, needsEsbuild: body.ref === 'master', rate: {} }) + } + if (service === 'downloader' && request.method === 'GET' && url.pathname.startsWith('/v1/files/')) { + return json(response, 404, { error: { code: 'FILE_NOT_FOUND', message: 'File was not found' } }) + } + if (service === 'builder' && request.method === 'POST' && url.pathname === '/v1/build') { + const body = await readJSON(request) + response.setHeader('X-Built-With', body.mode === 'esbuild' ? 'esbuild' : 'assembler') + return raw(response, 200, 'application/javascript', body.mode === 'dashboards' ? '/* Dashboards fixture */' : '/* Highcharts fixture */') + } + if (request.method === 'POST' && url.pathname === '/v1/cleanup') return json(response, 200, { removed: '0' }) + json(response, 404, { error: { code: 'NOT_FOUND', message: 'Not found' } }) + } catch (error) { + json(response, 400, { error: { code: 'INVALID_REQUEST', message: 'Invalid request' } }) + } +}).listen(port, '0.0.0.0') + +function snapshot (correlationId) { + const now = new Date().toISOString() + return { + schemaVersion: 1, + service, + instanceId: `${service}-fixture`, + startedAt: now, + observedAt: now, + health: { status: 'healthy', reasons: [] }, + capabilities: [{ name: service === 'builder' ? 'build_delivery' : 'source_delivery', status: 'available', reasonCode: null }], + queues: [{ name: service === 'builder' ? 'build' : 'download', active: 0, queued: 0, limit: 2, available: 2, oldestQueuedAgeMs: null }], + cache: { + entryCount: 1, + totalBytes: 128, + idleExpiryMs: 60000, + entriesTruncated: false, + entries: [{ commit, sizeBytes: 128, lastAccessedAt: now, expiresAt: new Date(Date.now() + 60000).toISOString(), inUse: 0 }] + }, + dependencies: [], + telemetry: { activityDropped: 0, completedEvicted: 0, failuresEvicted: 0, spansDropped: 0 }, + activity: correlationId ? [] : [], + failures: [] + } +} + +function readJSON (request) { + return new Promise((resolve, reject) => { + const chunks = [] + let bytes = 0 + request.on('data', chunk => { + bytes += chunk.length + if (bytes > 65536) request.destroy() + else chunks.push(chunk) + }) + request.on('end', () => { + try { resolve(JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}')) } catch (error) { reject(error) } + }) + request.on('error', reject) + }) +} + +function json (response, status, body) { + raw(response, status, 'application/json', JSON.stringify(body)) +} + +function raw (response, status, type, body) { + if (response.destroyed) return + response.writeHead(status, { 'Content-Type': type, 'Content-Length': Buffer.byteLength(body) }) + response.end(body) +} + +function delay (milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)) +} diff --git a/test/ops/stack.js b/test/ops/stack.js new file mode 100644 index 0000000..705ec58 --- /dev/null +++ b/test/ops/stack.js @@ -0,0 +1,106 @@ +'use strict' + +const express = require('express') +const { createOpsConsoleRouter } = require('../../app/ops/console-router') +const { OpsRateLimiter, SessionStore } = require('../../app/ops/sessions') +const { createRouter } = require('../../app/router') + +const port = Number(process.env.PORT || 8080) +const internalToken = process.env.INTERNAL_SERVICE_TOKEN +let clockOffset = 0 +const now = () => Date.now() + clockOffset +const sessions = new SessionStore({ now }) +const rateLimiter = new OpsRateLimiter({ now }) +const events = [] +const publicRouter = createRouter({ disableRateLimit: true, now }) +const app = express() + +if (!internalToken) throw new Error('INTERNAL_SERVICE_TOKEN is required') + +// Published Docker traffic arrives from a bridge address; this harness models the +// production direct-loopback listener before invoking the real trust-boundary code. +app.use((request, response, next) => { + Object.defineProperty(request.socket, 'remoteAddress', { configurable: true, value: '127.0.0.1' }) + next() +}) +app.use('/__ops-test', express.json({ limit: '4kb' }), testControls) +app.use('/_ops', createOpsConsoleRouter({ + env: process.env, + log: event => { + events.push(event) + console.log(JSON.stringify(event)) // eslint-disable-line no-console + }, + now, + sessions, + rateLimiter, + routerService: { snapshot: publicRouter.opsSnapshot } +})) +app.use(publicRouter) +app.listen(port, '0.0.0.0') + +async function testControls (request, response) { + try { + if (request.method === 'POST' && request.path === '/fixture') { + const { service, snapshot, cache, resetCounts } = request.body || {} + if (!['downloader', 'builder'].includes(service)) return response.sendStatus(400) + const result = await fixture(service, '/__ops-test/control', { + method: 'POST', + body: { snapshot, cache, resetCounts } + }) + return response.json(result) + } + if (request.method === 'POST' && request.path === '/clock') { + const milliseconds = request.body?.advanceMs + if (!Number.isSafeInteger(milliseconds) || milliseconds < 0 || milliseconds > 9 * 60 * 60 * 1000) return response.sendStatus(400) + clockOffset += milliseconds + return response.json({ ok: true }) + } + if (request.method === 'POST' && request.path === '/rotate') { + sessions.rotate() + return response.json({ ok: true }) + } + if (request.method === 'POST' && request.path === '/reset-audits') { + events.splice(0) + return response.json({ ok: true }) + } + if (request.method === 'GET' && request.path === '/status') { + const [downloader, builder] = await Promise.all([ + fixture('downloader', '/__ops-test/status'), + fixture('builder', '/__ops-test/status') + ]) + const audits = events.filter(event => event.action === 'cache_operation') + return response.json({ + audits: audits.length, + auditSafe: audits.every(auditIsSafe), + downloader: downloader.calls, + builder: builder.calls + }) + } + response.sendStatus(404) + } catch (error) { + response.status(502).json({ error: 'fixture unavailable' }) + } +} + +async function fixture (service, path, options = {}) { + const result = await fetch(`http://${service}:8080${path}`, { + method: options.method, + headers: { + Authorization: `Bearer ${internalToken}`, + ...(options.body ? { 'Content-Type': 'application/json' } : {}) + }, + body: options.body ? JSON.stringify(options.body) : undefined + }) + if (!result.ok) throw new Error('Fixture request failed') + return result.json() +} + +function auditIsSafe (event) { + const encoded = JSON.stringify(event) + return event.action === 'cache_operation' && + !encoded.includes(process.env.OPS_TEST_LOGIN_TOKEN || '\u0000') && + !encoded.includes(process.env.OPS_CONSOLE_TOKEN_VERIFIER || '\u0000') && + !encoded.includes(internalToken) && + !/(authorization|cookie|csrfToken|token)/i.test(encoded) && + Buffer.byteLength(encoded) < 65536 +} diff --git a/test/router.js b/test/router.js index f2e9f3a..82a5ab3 100644 --- a/test/router.js +++ b/test/router.js @@ -67,10 +67,25 @@ describe('public router delegation', () => { it('streams ordinary files from downloader with public cache, ETag, and rate headers', async () => { const clients = services() - const result = await request(createRouter({ ...clients, disableRateLimit: true }), '/trettan/highcharts.js') + const router = createRouter({ ...clients, disableRateLimit: true }) + const result = await request(router, '/trettan/highcharts.js', { headers: { 'X-Correlation-ID': 'browser-value' } }) expect(result).to.include({ status: 200, body: 'downloaded' }) expect(result.headers).to.include({ etag: SHA, 'cache-control': 'max-age=3600', 'cdn-cache-control': 'max-age=3600', 'x-github-ratelimit-remaining': '12' }) + expect(result.headers['x-correlation-id']).to.match(/^[0-9a-f-]{36}$/).and.not.equal('browser-value') expect(clients.downloader.request.firstCall.args[0]).to.equal(`/v1/files/${SHA}/js/highcharts.src.js`) + expect(clients.downloader.json.firstCall.args[1].correlationId).to.equal(result.headers['x-correlation-id']) + expect(clients.downloader.request.firstCall.args[1].correlationId).to.equal(result.headers['x-correlation-id']) + const activity = router.opsSnapshot().activity + expect(activity).to.have.length(1) + expect(activity[0]).to.nested.include({ + correlationId: result.headers['x-correlation-id'], + state: 'completed', + 'request.commit': SHA, + 'request.resource': '/trettan/highcharts.js', + 'request.buildMode': 'static', + 'outcome.status': 'succeeded' + }) + expect(JSON.stringify(activity)).not.to.include('browser-value') }) it('falls back to explicit legacy builder mode while retaining downloader rate metadata', async () => { @@ -139,8 +154,49 @@ describe('public router delegation', () => { clients.downloader.json.withArgs('/v1/cleanup').resolves({ removed: ['source'] }) const result = await request(createRouter({ ...clients, disableRateLimit: true }), '/cleanup?true') expect(result).to.include({ status: 200, body: '[]' }) - expect(clients.downloader.json.calledWith('/v1/cleanup', { method: 'POST', body: { force: false } })).to.equal(true) - expect(clients.builder.json.calledWith('/v1/cleanup', { method: 'POST', body: { force: false } })).to.equal(true) + expect(clients.downloader.json.calledWithMatch('/v1/cleanup', { method: 'POST', body: { force: false }, correlationId: sinon.match.string })).to.equal(true) + expect(clients.builder.json.calledWithMatch('/v1/cleanup', { method: 'POST', body: { force: false }, correlationId: sinon.match.string })).to.equal(true) + }) + + it('disables legacy public cleanup only while the operations console is enabled', async () => { + const clients = services() + const result = await request(createRouter({ ...clients, disableRateLimit: true, opsConsoleEnabled: true }), '/cleanup?true') + expect(result.status).to.equal(404) + expect(clients.downloader.json.called).to.equal(false) + expect(clients.builder.json.called).to.equal(false) + }) + + it('never exposes operations-console assets through the public static root', async () => { + const paths = [ + '/ops', + '/ops/', + '/ops/index.html', + '/ops/login.html', + '/ops/console.js', + '/ops/login.js', + '/ops/console.css', + '/%6f%70%73/index.html', + '/public/../ops/login.html', + '/public/%2e%2e/ops/console.js', + '/ops%2fconsole.css' + ] + for (const opsConsoleEnabled of [false, true]) { + const clients = services() + const router = createRouter({ ...clients, disableRateLimit: true, opsConsoleEnabled }) + for (const path of paths) { + const result = await request(router, path) + expect(result, path).to.include({ status: 404, body: 'Not Found' }) + expect(result.headers['content-type'], path).to.match(/^text\/plain/) + expect(result.headers, path).not.to.have.property('location') + } + expect(clients.downloader.json.called).to.equal(false) + expect(clients.builder.request.called).to.equal(false) + } + + const clients = services() + const router = createRouter({ ...clients, disableRateLimit: true }) + expect(await request(router, '/test.html')).to.include({ status: 200 }) + expect(await request(router, '/master/highcharts.js')).to.include({ status: 200, body: 'downloaded' }) }) it('only inspects cache files for canonical commit SHAs', async () => { diff --git a/test/server.js b/test/server.js index cac5d22..e59ecc4 100644 --- a/test/server.js +++ b/test/server.js @@ -1,17 +1,23 @@ 'use strict' const { expect } = require('chai') +const { randomBytes } = require('node:crypto') const http = require('node:http') +const https = require('node:https') const fs = require('node:fs') const { join } = require('node:path') const { describe, it } = require('mocha') -const { createApp } = require('../app/server') +const { createTokenVerifier } = require('../app/ops/config') +const { createApp, createServers } = require('../app/server') const { createRouter } = require('../app/router') describe('public server', () => { it('serves health without starting a listener or calling services', async () => { const unavailable = () => { throw new Error('service called') } - const app = createApp({ router: createRouter({ downloader: { json: unavailable }, builder: {}, disableRateLimit: true }) }) + const app = createApp({ + ops: { env: {}, log: () => {} }, + router: createRouter({ downloader: { json: unavailable }, builder: {}, disableRateLimit: true }) + }) const response = await request(app, '/health') expect(response).to.include({ status: 200, body: 'OK' }) }) @@ -23,8 +29,176 @@ describe('public server', () => { expect(server).not.to.include('setInterval') expect(handlers).not.to.include("require('./server.js')") }) + + it('creates only the public listener when disabled or in explicit HTTP loopback mode', () => { + expect(createServers({ env: {}, app: () => {} }).opsServer).to.equal(null) + const token = randomBytes(32).toString('base64url') + const env = { + OPS_CONSOLE_ENABLED: 'true', + OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), + OPS_CONSOLE_ORIGIN: 'http://127.0.0.1:8080', + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' + } + expect(createServers({ env, app: () => {} }).opsServer).to.equal(null) + }) + + it('fails before listening for unreadable or invalid mTLS material', () => { + const env = mtlsEnv() + const handler = (request, response) => response.end('OK') + const invalid = join(__dirname, '../app/server.js') + const missing = join(__dirname, 'fixtures/mtls/missing.pem') + + expect(() => createServers({ env: { ...env, OPS_CONSOLE_MTLS_CA_PATH: missing }, app: handler })).to.throw() + expect(() => createServers({ + env, + app: handler, + readFile: path => { + if (path === env.OPS_CONSOLE_MTLS_CA_PATH) throw Object.assign(new Error('unreadable test CA'), { code: 'EACCES' }) + return fs.readFileSync(path) + } + })).to.throw('unreadable test CA') + for (const name of ['OPS_CONSOLE_MTLS_KEY_PATH', 'OPS_CONSOLE_MTLS_CERT_PATH', 'OPS_CONSOLE_MTLS_CA_PATH']) { + expect(() => createServers({ env: { ...env, [name]: invalid }, app: handler })).to.throw() + } + expect(() => createServers({ + env: { ...env, OPS_CONSOLE_MTLS_KEY_PATH: fixture('client.key') }, + app: handler + })).to.throw() + }) + + it('serves public HTTP separately and admits console HTTPS only after trusted client mTLS', async () => { + const token = randomBytes(32).toString('base64url') + const env = mtlsEnv(token) + const events = [] + const app = createApp({ ops: { env, log: event => events.push(event) } }) + const { publicServer, opsServer } = createServers({ env, app }) + expect(opsServer.requestCert).to.equal(true) + expect(opsServer.rejectUnauthorized).to.equal(true) + await Promise.all([listen(publicServer), listen(opsServer)]) + + try { + const publicHealth = await networkRequest(http, { port: publicServer.address().port, path: '/health' }) + expect(publicHealth).to.include({ status: 200, body: 'OK' }) + + const publicConsole = await networkRequest(http, { port: publicServer.address().port, path: '/_ops/login' }) + expect(publicConsole.status).to.equal(400) + expect(JSON.parse(publicConsole.body).error.code).to.equal('INVALID_REQUEST_CONTEXT') + + const tls = { + port: opsServer.address().port, + path: '/_ops/login', + ca: fs.readFileSync(fixture('ca.crt')), + cert: fs.readFileSync(fixture('client.crt')), + key: fs.readFileSync(fixture('client.key')), + servername: 'localhost' + } + const consoleResponse = await networkRequest(https, { + ...tls, + headers: { + Forwarded: 'for=203.0.113.9;proto=http', + 'X-Forwarded-For': '198.51.100.8', + 'X-Forwarded-Proto': 'http' + } + }) + expect(consoleResponse.status).to.equal(200) + expect(consoleResponse.body).to.include('') + + const login = await networkRequest(https, { + ...tls, + method: 'POST', + path: '/_ops/api/v1/session', + headers: { + Origin: env.OPS_CONSOLE_ORIGIN, + 'Content-Type': 'application/json', + 'X-Forwarded-For': '198.51.100.8', + 'X-Forwarded-Proto': 'http' + }, + body: JSON.stringify({ token }) + }) + expect(login.status).to.equal(201) + expect(login.headers['set-cookie'][0]).to.match(/^__Host-hc-ops=.*; Secure;/) + const session = JSON.parse(login.body) + const rejectedCache = await networkRequest(https, { + ...tls, + method: 'POST', + path: '/_ops/api/v1/cache-operations', + headers: { + Cookie: login.headers['set-cookie'][0].split(';', 1)[0], + Origin: env.OPS_CONSOLE_ORIGIN, + 'Content-Type': 'application/json', + 'X-Ops-CSRF': session.csrfToken, + Forwarded: 'for=203.0.113.9;proto=http', + 'X-Forwarded-For': '198.51.100.8', + 'X-Forwarded-Proto': 'http' + }, + body: JSON.stringify({ operation: 'not-an-operation', targets: ['builder'] }) + }) + expect(rejectedCache.status).to.equal(400) + expect(events.find(event => event.action === 'cache_operation')).to.include({ source: null, dispatchStatus: 'not_dispatched' }) + + await expectNetworkFailure(networkRequest(https, { + port: opsServer.address().port, + path: '/_ops/login', + ca: tls.ca, + servername: 'localhost' + })) + await expectNetworkFailure(networkRequest(https, { + ...tls, + cert: fs.readFileSync(fixture('untrusted-client.crt')), + key: fs.readFileSync(fixture('untrusted-client.key')) + })) + await expectNetworkFailure(networkRequest(http, { port: opsServer.address().port, path: '/_ops/login' })) + } finally { + await Promise.all([close(publicServer), close(opsServer)]) + } + }) }) +function fixture (name) { + return join(__dirname, 'fixtures/mtls', name) +} + +function mtlsEnv (token = randomBytes(32).toString('base64url')) { + return { + OPS_CONSOLE_ENABLED: 'true', + OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), + OPS_CONSOLE_ORIGIN: 'https://localhost:8443', + OPS_CONSOLE_MTLS_KEY_PATH: fixture('server.key'), + OPS_CONSOLE_MTLS_CERT_PATH: fixture('server.crt'), + OPS_CONSOLE_MTLS_CA_PATH: fixture('ca.crt') + } +} + +function listen (server) { + return new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) +} + +function close (server) { + return new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) +} + +async function expectNetworkFailure (promise) { + let error + try { await promise } catch (failure) { error = failure } + expect(error).to.be.instanceOf(Error) +} + +function networkRequest (protocol, options) { + return new Promise((resolve, reject) => { + const { body, ...requestOptions } = options + const outgoing = protocol.request({ host: '127.0.0.1', method: 'GET', ...requestOptions }, response => { + const chunks = [] + response.on('data', chunk => chunks.push(chunk)) + response.on('end', () => resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString() })) + }) + outgoing.on('error', reject) + outgoing.end(body) + }) +} + function request (app, path) { return new Promise((resolve, reject) => { const server = app.listen(0, () => { diff --git a/test/test.js b/test/test.js index 409bf24..5b8f803 100644 --- a/test/test.js +++ b/test/test.js @@ -1,10 +1,17 @@ require('./download.js') +require('./JobQueue.test.js') require('./downloader-service.js') require('./builder-service.js') require('./esbuild.js') require('./filesystem.js') require('./interpreter.js') require('./middleware.js') +require('./ops-cache.js') +require('./ops-config.js') +require('./ops-console.js') +require('./ops-http.js') +require('./ops-internal.js') +require('./ops-telemetry.js') require('./router.js') require('./server.js') require('./utilities.js') From 1d9ec7ee4914e1a90565b5f7cb3768b4b2f02480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B8ran=20A=2E=20Slettemark?= Date: Wed, 22 Jul 2026 09:57:58 +0200 Subject: [PATCH 2/5] Fixed hurl test --- test/hurl/ops-auth.hurl | 152 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) diff --git a/test/hurl/ops-auth.hurl b/test/hurl/ops-auth.hurl index e156370..dfb5fe2 100644 --- a/test/hurl/ops-auth.hurl +++ b/test/hurl/ops-auth.hurl @@ -157,7 +157,7 @@ HTTP 404 [Asserts] header "Access-Control-Allow-Origin" not exists -# The five-attempt source limit produces a bounded 429 with Retry-After. +# The global 30-attempt login limit produces a bounded 429 with Retry-After. POST {{base_url}}/__ops-test/clock Content-Type: application/json { @@ -195,6 +195,156 @@ Content-Type: application/json {"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} HTTP 401 +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + +POST {{base_url}}/_ops/api/v1/session +Origin: {{base_url}} +Content-Type: application/json +{"token":"BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"} +HTTP 401 + POST {{base_url}}/_ops/api/v1/session Origin: {{base_url}} Content-Type: application/json From 2728c1eb6c3176656e56d4ed31ef2affe5883bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B8ran=20A=2E=20Slettemark?= Date: Wed, 22 Jul 2026 12:13:53 +0200 Subject: [PATCH 3/5] Removed overengineered auth setup --- .env.example | 9 +- Dockerfile.router | 2 +- README.md | 13 +- app/ops/config.js | 42 ++---- app/ops/console-router.js | 3 - app/ops/http.js | 30 ---- app/server.js | 30 +--- compose.yaml | 2 +- docs/operations-console.md | 185 ++---------------------- test/fixtures/mtls/ca.crt | 20 --- test/fixtures/mtls/client.crt | 21 --- test/fixtures/mtls/client.key | 28 ---- test/fixtures/mtls/server.crt | 21 --- test/fixtures/mtls/server.key | 28 ---- test/fixtures/mtls/untrusted-ca.crt | 20 --- test/fixtures/mtls/untrusted-client.crt | 21 --- test/fixtures/mtls/untrusted-client.key | 28 ---- test/ops-config.js | 56 +++---- test/ops-console.js | 2 +- test/ops-http.js | 27 ---- test/server.js | 131 +++++------------ 21 files changed, 96 insertions(+), 623 deletions(-) delete mode 100644 test/fixtures/mtls/ca.crt delete mode 100644 test/fixtures/mtls/client.crt delete mode 100644 test/fixtures/mtls/client.key delete mode 100644 test/fixtures/mtls/server.crt delete mode 100644 test/fixtures/mtls/server.key delete mode 100644 test/fixtures/mtls/untrusted-ca.crt delete mode 100644 test/fixtures/mtls/untrusted-client.crt delete mode 100644 test/fixtures/mtls/untrusted-client.key diff --git a/.env.example b/.env.example index 1bad03a..d1604fd 100644 --- a/.env.example +++ b/.env.example @@ -25,13 +25,8 @@ OPS_CONSOLE_TOKEN_VERIFIER= # Local HTTP requires OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true and an exact loopback origin. OPS_CONSOLE_ORIGIN= OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=false -# Local default remains HTTP on loopback and does not require mTLS. -OPS_CONSOLE_MTLS_PORT=8443 -# Production HTTPS requires absolute in-container paths to mounted secrets. -# Add an explicit read-only mount override in Compose matching these paths. -OPS_CONSOLE_MTLS_KEY_PATH= -OPS_CONSOLE_MTLS_CERT_PATH= -OPS_CONSOLE_MTLS_CA_PATH= +# OPS_CONSOLE_MTLS_PORT, OPS_CONSOLE_MTLS_KEY_PATH, OPS_CONSOLE_MTLS_CERT_PATH, +# and OPS_CONSOLE_MTLS_CA_PATH are retired and unsupported; do not set them. # Ephemeral cache roots are fixed by compose.yaml and owned by UID/GID 1000. # They are separate tmpfs mounts and are discarded when containers stop. diff --git a/Dockerfile.router b/Dockerfile.router index af7b549..226cff2 100644 --- a/Dockerfile.router +++ b/Dockerfile.router @@ -20,6 +20,6 @@ COPY --chown=node:node assets ./assets COPY --chown=node:node static ./static USER node -EXPOSE 8080 8443 +EXPOSE 8080 ENTRYPOINT ["/sbin/tini", "--"] CMD ["npm", "run", "start:router"] diff --git a/README.md b/README.md index e234db7..c521a3a 100644 --- a/README.md +++ b/README.md @@ -22,15 +22,13 @@ The console does not change the three-process topology. Only the router exposes | `OPS_CONSOLE_ENABLED` | Always evaluated | Must be exactly the string `true` to enable the console. Any other value or absence keeps the console off and every `/_ops/*` path returns 404. | | `OPS_CONSOLE_TOKEN_VERIFIER` | Console enabled | Canonical domain-separated SHA-256 verifier derived from the shared admin token (see below). Never store the raw token. Validated at startup; missing or malformed values abort startup. | | `OPS_CONSOLE_ORIGIN` | Console enabled | The single exact external Origin the console accepts for all state-changing requests and for login (e.g. `http://localhost:8080`). Missing or ambiguous values abort startup. | -| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | Set to exactly `true` to allow HTTP only when the Origin is a loopback address and no mTLS settings are present. All other HTTP/mTLS combinations abort startup. | -| `OPS_CONSOLE_MTLS_PORT` | HTTPS console | Port for the mTLS listener. Defaults to `8443`. Must not be combined with HTTP loopback mode. | -| `OPS_CONSOLE_MTLS_KEY_PATH` | HTTPS console | Absolute in-container path to the server TLS private key PEM file. | -| `OPS_CONSOLE_MTLS_CERT_PATH` | HTTPS console | Absolute in-container path to the server TLS certificate PEM file. | -| `OPS_CONSOLE_MTLS_CA_PATH` | HTTPS console | Absolute in-container path to the Envoy client CA certificate PEM file. The router uses this CA to verify the client certificate that Envoy presents during the mTLS handshake. The file must be a CA certificate; startup fails if it is not. This CA is the Envoy-client-issuing CA and is distinct from the server CA used in Contour's `validation.caSecret`. | +| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | Set to exactly `true` to allow HTTP when the Origin is a loopback address. All other HTTP combinations abort startup. | | `ROUTER_BIND_ADDRESS` | Optional | Host address the router container binds to. Defaults to `127.0.0.1` in Docker Compose; the console is not reachable from outside loopback unless this is changed deliberately. | `OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup. +`OPS_CONSOLE_MTLS_PORT`, `OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and `OPS_CONSOLE_MTLS_CA_PATH` are retired and unsupported; setting any to a non-blank value aborts startup. + **Verifier, not token.** `OPS_CONSOLE_TOKEN_VERIFIER` stores a one-way verifier, never the raw token. Derive it from a canonical 32-byte CSPRNG token (base64url-encoded) with: ```bash @@ -76,7 +74,6 @@ console.log('Verifier (put in .env):', 'v1.' + digest); # OPS_CONSOLE_TOKEN_VERIFIER=v1. # OPS_CONSOLE_ORIGIN=http://localhost:8080 # OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true -# (leave OPS_CONSOLE_MTLS_* blank for local HTTP loopback mode) # 4. Start the stack docker compose up --build --wait @@ -107,7 +104,7 @@ This checklist covers staged local rollout, abort criteria, and rollback. **Enable:** 1. Derive the verifier and set `OPS_CONSOLE_ENABLED=true` plus the three required variables in `.env`. -2. For local HTTP loopback mode, set `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` and leave all `OPS_CONSOLE_MTLS_*` variables blank. For HTTPS mTLS mode, supply absolute in-container paths for `OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and `OPS_CONSOLE_MTLS_CA_PATH`. +2. For local HTTP loopback mode, set `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true`. If `OPS_CONSOLE_ORIGIN` uses an `https:` scheme, TLS termination is handled at the proxy layer; the Node backend remains ordinary HTTP. 3. Restart the router (`docker compose up --build --wait`). Confirm startup log shows `ops-console enabled`. 4. Open `/_ops/login`; confirm login succeeds and the snapshot page loads with data from all three services. 5. Run `npm run test:integration:ops` — all checks must pass. @@ -118,7 +115,7 @@ This checklist covers staged local rollout, abort criteria, and rollback. - Login returns a session without a valid token being submitted. - Any audit event is absent from the router log after a cache mutation. - Internal bearer token appears in a browser response or cookie. -- Router startup fails with a missing-verifier, missing TLS path, or invalid TLS material error that cannot be resolved within 10 minutes. +- Router startup fails with a missing-verifier or invalid-origin error that cannot be resolved within 10 minutes. **Disable and roll back:** diff --git a/app/ops/config.js b/app/ops/config.js index f804545..391188a 100644 --- a/app/ops/config.js +++ b/app/ops/config.js @@ -2,11 +2,15 @@ const { createHash, timingSafeEqual } = require('node:crypto') const { isIP } = require('node:net') -const { isAbsolute } = require('node:path') const DOMAIN_SEPARATOR = Buffer.from('github.highcharts.com:ops-console:v1\0') const BASE64URL_32 = /^[A-Za-z0-9_-]{43}$/ -const DEFAULT_MTLS_PORT = 8443 +const RETIRED_MTLS_SETTINGS = [ + 'OPS_CONSOLE_MTLS_PORT', + 'OPS_CONSOLE_MTLS_KEY_PATH', + 'OPS_CONSOLE_MTLS_CERT_PATH', + 'OPS_CONSOLE_MTLS_CA_PATH' +] function decodeCanonical32 (value, name) { if (typeof value !== 'string' || !BASE64URL_32.test(value)) { @@ -66,19 +70,6 @@ function nonblank (value) { return typeof value === 'string' && value.trim() !== '' } -function parseAbsolutePath (value, name) { - if (typeof value !== 'string' || value.trim() !== value || !isAbsolute(value)) throw new Error(`Invalid ${name}`) - return value -} - -function parsePort (value) { - if (value === undefined || value === '') return DEFAULT_MTLS_PORT - if (typeof value !== 'string' || !/^[1-9]\d{0,4}$/.test(value)) throw new Error('Invalid OPS_CONSOLE_MTLS_PORT') - const port = Number(value) - if (port > 65535) throw new Error('Invalid OPS_CONSOLE_MTLS_PORT') - return port -} - function isLoopbackHost (hostname) { if (hostname === 'localhost' || hostname === '::1' || hostname === '[::1]') return true if (isIP(hostname) !== 4) return false @@ -88,19 +79,20 @@ function isLoopbackHost (hostname) { function parseOpsConfig (env = process.env) { if (env.OPS_CONSOLE_ENABLED !== 'true') return Object.freeze({ enabled: false }) + if (RETIRED_MTLS_SETTINGS.some(name => nonblank(env[name]))) { + throw new Error('OPS_CONSOLE_MTLS_* settings are retired and unsupported') + } const verifierDigest = parseTokenVerifier(env.OPS_CONSOLE_TOKEN_VERIFIER) const originURL = parseOrigin(env.OPS_CONSOLE_ORIGIN) const allowHttpLoopback = env.OPS_CONSOLE_ALLOW_HTTP_LOOPBACK === 'true' - const mtlsNames = ['OPS_CONSOLE_MTLS_KEY_PATH', 'OPS_CONSOLE_MTLS_CERT_PATH', 'OPS_CONSOLE_MTLS_CA_PATH'] - const hasMTLSSetting = mtlsNames.some(name => nonblank(env[name])) || nonblank(env.OPS_CONSOLE_MTLS_PORT) if (nonblank(env.OPS_CONSOLE_TRUSTED_PROXY)) { - throw new Error('OPS_CONSOLE_TRUSTED_PROXY is not supported; use operations console mTLS') + throw new Error('OPS_CONSOLE_TRUSTED_PROXY is not supported') } if (originURL.protocol === 'http:') { - if (!allowHttpLoopback || !isLoopbackHost(originURL.hostname) || hasMTLSSetting) { - throw new Error('HTTP operations console requires direct loopback mode without mTLS settings') + if (!allowHttpLoopback || !isLoopbackHost(originURL.hostname)) { + throw new Error('HTTP operations console requires loopback origin opt-in') } return Object.freeze({ enabled: true, @@ -112,26 +104,18 @@ function parseOpsConfig (env = process.env) { } if (allowHttpLoopback) throw new Error('HTTPS operations console cannot allow HTTP loopback mode') - const mtls = Object.freeze({ - port: parsePort(env.OPS_CONSOLE_MTLS_PORT), - keyPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_KEY_PATH, 'OPS_CONSOLE_MTLS_KEY_PATH'), - certPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_CERT_PATH, 'OPS_CONSOLE_MTLS_CERT_PATH'), - caPath: parseAbsolutePath(env.OPS_CONSOLE_MTLS_CA_PATH, 'OPS_CONSOLE_MTLS_CA_PATH') - }) return Object.freeze({ enabled: true, origin: originURL.origin, protocol: 'https', allowHttpLoopback: false, - verifierDigest, - mtls + verifierDigest }) } module.exports = { createTokenVerifier, - DEFAULT_MTLS_PORT, parseOpsConfig, parseTokenVerifier, verifyToken diff --git a/app/ops/console-router.js b/app/ops/console-router.js index c8e52ad..14e4217 100644 --- a/app/ops/console-router.js +++ b/app/ops/console-router.js @@ -10,7 +10,6 @@ const { OpsHttpError, asyncRoute, errorBody, - getTrustedRequestContext, readStrictJSON, requireCSRF, requireJSONContentType, @@ -92,7 +91,6 @@ function createOpsConsoleRouter ({ env = process.env, log = logStatus, now = Dat response.once('close', emit) } try { - request.opsContext = getTrustedRequestContext(request, config) requireSameOriginFetch(request, { allowTopLevelNavigation: request.path === '/' || request.path === '/login' }) @@ -332,7 +330,6 @@ function emitCacheAudit (request, log, now) { const audit = request.cacheAudit if (!audit || audit.emitted) return audit.emitted = true - audit.source = request.opsContext?.source || null const errorCodes = [...new Set([ ...audit.errorCodes, ...audit.targetOutcomes.map(target => target.error?.code).filter(Boolean) diff --git a/app/ops/http.js b/app/ops/http.js index 29a8d52..54fd291 100644 --- a/app/ops/http.js +++ b/app/ops/http.js @@ -1,7 +1,6 @@ 'use strict' const { createHash, randomUUID, timingSafeEqual } = require('node:crypto') -const { isIP } = require('node:net') const MAX_BODY_BYTES = 4 * 1024 const CSP = "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self'; connect-src 'self'; font-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'; object-src 'none'" @@ -189,34 +188,6 @@ function requireBearer (request, expected) { } } -function normalizeIP (value) { - if (typeof value !== 'string') return null - const unwrapped = value.startsWith('[') && value.endsWith(']') ? value.slice(1, -1) : value - const mapped = unwrapped.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/i) - const normalized = mapped ? mapped[1] : unwrapped - return isIP(normalized) ? normalized : null -} - -function getTrustedRequestContext (request, config) { - if (config.protocol === 'https') { - if (request.socket?.encrypted !== true || request.socket?.authorized !== true) { - throw new OpsHttpError(400, 'INVALID_REQUEST_CONTEXT', 'Request context is invalid') - } - return Object.freeze({ protocol: 'https', source: null }) - } - - const peerIp = normalizeIP(request.socket?.remoteAddress) - if (request.socket?.encrypted || !peerIp || !isLoopbackIP(peerIp)) { - throw new OpsHttpError(400, 'INVALID_REQUEST_CONTEXT', 'Request context is invalid') - } - return Object.freeze({ protocol: 'http', source: peerIp }) -} - -function isLoopbackIP (address) { - if (address === '::1') return true - return isIP(address) === 4 && Number(address.split('.')[0]) === 127 -} - function errorBody (error, correlationId = randomUUID()) { const safe = error instanceof OpsHttpError ? error @@ -261,7 +232,6 @@ module.exports = { SECURITY_HEADERS, asyncRoute, errorBody, - getTrustedRequestContext, observedRoute, readStrictJSON, requireBearer, diff --git a/app/server.js b/app/server.js index 75e20aa..34f1a21 100644 --- a/app/server.js +++ b/app/server.js @@ -1,12 +1,8 @@ 'use strict' const config = require('../config.json') -const { X509Certificate } = require('node:crypto') -const fs = require('node:fs') const http = require('node:http') -const https = require('node:https') const { bodyJSONParser, clientErrorHandler, logErrors, setConnectionAborted } = require('./middleware.js') -const { parseOpsConfig } = require('./ops/config') const { createOpsConsoleRouter } = require('./ops/console-router') const router = require('./router.js') const express = require('express') @@ -32,31 +28,9 @@ function createApp (options = {}) { const APP = createApp() function start () { - const { publicServer, opsServer, opsConfig } = createServers({ app: APP }) - publicServer.listen(process.env.PORT || config.port || 80) - if (opsServer) opsServer.listen(opsConfig.mtls.port) - return publicServer -} - -function createServers ({ env = process.env, app, readFile = fs.readFileSync } = {}) { - const opsConfig = parseOpsConfig(env) - app = app || createApp({ ops: { env } }) - const publicServer = http.createServer(app) - if (!opsConfig.enabled || opsConfig.protocol === 'http') return { publicServer, opsServer: null, opsConfig } - - const { keyPath, certPath, caPath } = opsConfig.mtls - const ca = readFile(caPath) - if (!new X509Certificate(ca).ca) throw new Error('Invalid OPS_CONSOLE_MTLS_CA_PATH: certificate is not a CA') - const opsServer = https.createServer({ - key: readFile(keyPath), - cert: readFile(certPath), - ca, - requestCert: true, - rejectUnauthorized: true - }, app) - return { publicServer, opsServer, opsConfig } + return http.createServer(APP).listen(process.env.PORT || config.port || 80) } if (require.main === module || require.main?.filename === join(__dirname, '../server.js')) start() -module.exports = { createApp, createServers, default: APP, start } +module.exports = { createApp, default: APP, start } diff --git a/compose.yaml b/compose.yaml index 8bac254..11c6056 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,7 +14,7 @@ services: OPS_CONSOLE_TOKEN_VERIFIER: ${OPS_CONSOLE_TOKEN_VERIFIER-} OPS_CONSOLE_ORIGIN: ${OPS_CONSOLE_ORIGIN-} OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: ${OPS_CONSOLE_ALLOW_HTTP_LOOPBACK:-false} - OPS_CONSOLE_MTLS_PORT: ${OPS_CONSOLE_MTLS_PORT:-8443} + OPS_CONSOLE_MTLS_PORT: ${OPS_CONSOLE_MTLS_PORT-} OPS_CONSOLE_MTLS_KEY_PATH: ${OPS_CONSOLE_MTLS_KEY_PATH-} OPS_CONSOLE_MTLS_CERT_PATH: ${OPS_CONSOLE_MTLS_CERT_PATH-} OPS_CONSOLE_MTLS_CA_PATH: ${OPS_CONSOLE_MTLS_CA_PATH-} diff --git a/docs/operations-console.md b/docs/operations-console.md index 2fa08fe..81574c9 100644 --- a/docs/operations-console.md +++ b/docs/operations-console.md @@ -16,9 +16,6 @@ mandatory behaviour. "Should" and "should not" state strong recommendations. 1. [Purpose, operator journeys, and non-goals](#1-purpose-operator-journeys-and-non-goals) 2. [Topology and trust boundaries](#2-topology-and-trust-boundaries) 3. [Enablement and configuration](#3-enablement-and-configuration) - - [Proxied deployment — Contour/Kubernetes contract](#proxied-deployment--contourkubernetes-contract) - - [Proxied deployment — enablement order](#proxied-deployment--enablement-order) - - [Proxied deployment — certificate lifecycle](#proxied-deployment--certificate-lifecycle) 4. [Route namespace and HTTP routes](#4-route-namespace-and-http-routes) 5. [Shared-token login and session contract](#5-shared-token-login-and-session-contract) 6. [Security headers, CSP, and browser restrictions](#6-security-headers-csp-and-browser-restrictions) @@ -93,7 +90,7 @@ The existing three-process topology is unchanged: | Service | Role | Public port | |---|---|---| -| **Router** | Accepts public requests; hosts the same-origin operations console UI and browser-facing API | 8080 (public HTTP); 8443 (mTLS, HTTPS console mode only) | +| **Router** | Accepts public requests; hosts the same-origin operations console UI and browser-facing API | 8080 | | **Downloader** | Internal only; resolves refs, fetches and caches source trees; exposes authenticated internal `/v1` ops endpoints | None | | **Builder** | Internal only; compiles sources; exposes authenticated internal `/v1` ops endpoints | None | @@ -122,14 +119,11 @@ The existing three-process topology is unchanged: | `OPS_CONSOLE_ENABLED` | Always evaluated | Must be exactly the string `true` to enable the console. Any other value or absence disables it. | | `OPS_CONSOLE_TOKEN_VERIFIER` | Console enabled | Domain-separated SHA-256 verifier derived from the shared admin token. Stored outside source control. Validated at startup. | | `OPS_CONSOLE_ORIGIN` | Console enabled | The single exact external Origin that the console accepts for all state-changing requests and for login. | -| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | When exactly `true`, allows HTTP only if the configured Origin is a loopback address **and** no mTLS settings are present. All other HTTP/mTLS combinations fail startup. | -| `OPS_CONSOLE_MTLS_PORT` | HTTPS console | Port for the separate mTLS listener. Defaults to `8443`. Must not be set when `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true`. | -| `OPS_CONSOLE_MTLS_KEY_PATH` | HTTPS console | Absolute in-container path to the server TLS private key PEM file. | -| `OPS_CONSOLE_MTLS_CERT_PATH` | HTTPS console | Absolute in-container path to the server TLS certificate PEM file. | -| `OPS_CONSOLE_MTLS_CA_PATH` | HTTPS console | Absolute in-container path to the Envoy client CA certificate PEM file. The router uses this CA to verify the client certificate that Envoy presents during the mTLS handshake. The file must be a CA certificate; startup fails otherwise. This is the Envoy-client-issuing CA and is distinct from the server CA used in Contour's `validation.caSecret`. | +| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | When exactly `true`, allows HTTP when the configured Origin is a loopback address. All other HTTP combinations fail startup. | -`OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup with: -`OPS_CONSOLE_TRUSTED_PROXY is not supported; use operations console mTLS`. +`OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup. + +`OPS_CONSOLE_MTLS_PORT`, `OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and `OPS_CONSOLE_MTLS_CA_PATH` are retired and unsupported. Setting any to a non-blank value aborts startup. ### Disabled behaviour (default) @@ -149,39 +143,11 @@ the following conditions apply: - `OPS_CONSOLE_TOKEN_VERIFIER` is missing or malformed. - `OPS_CONSOLE_ORIGIN` is missing or invalid. - `OPS_CONSOLE_TRUSTED_PROXY` is set to any non-blank value. -- HTTPS mode is chosen but any of `OPS_CONSOLE_MTLS_KEY_PATH`, - `OPS_CONSOLE_MTLS_CERT_PATH`, or `OPS_CONSOLE_MTLS_CA_PATH` is blank, relative - (not an absolute path), missing, unreadable, or malformed TLS material. -- The file at `OPS_CONSOLE_MTLS_CA_PATH` is not a CA certificate. -- `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` is set together with any mTLS setting. - `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` is set but the Origin is not a loopback address. - `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` is `true` and the Origin scheme is `https:`. `NODE_ENV` alone must never enable the console. -### Two-listener model - -The router always starts a **public HTTP listener** on port 8080 (configurable via -`PORT`). This listener handles all public file-delivery traffic and the `/health` -endpoint. It is unchanged by console enablement. - -When the console is enabled with an `https:` Origin, the router additionally -starts a **separate mTLS listener** on `OPS_CONSOLE_MTLS_PORT` (default `8443`). -The Node.js server is configured with `requestCert: true` and -`rejectUnauthorized: true`; connections without a valid client certificate signed -by the configured CA are rejected at the TLS layer before any application code -runs. See [Node.js `https.createServer`](https://nodejs.org/docs/latest-v24.x/api/https.html#httpscreateserveroptions-requestlistener) -and [`tls.createServer`](https://nodejs.org/docs/latest-v24.x/api/tls.html#tlscreateserveroptions-secureconnectionlistener). - -`/_ops/*` requests that arrive on the **public HTTP listener** are handled by the -disabled router (console disabled) or always fail closed (console enabled in -HTTPS mode). The console is reachable only through the mTLS listener. - -**Local HTTP loopback mode** remains an explicit alternative for development. -It requires `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true` and a loopback Origin, and all -`OPS_CONSOLE_MTLS_*` variables must be blank. In this mode the single public HTTP -listener handles `/_ops/*` traffic directly, with no mTLS layer. - ### Client IP and rate limiting Node.js does not trust any `Forwarded` or `X-Forwarded-*` header and does not @@ -189,132 +155,9 @@ derive a client IP from proxy headers. Login rate limiting is process-wide only: a single global counter of 30 attempts per 15 minutes. There is no per-source or per-IP bucket at the application layer. -In HTTPS/mTLS mode the audit `source` field is `null`; no client or proxy IP is -attributed. In explicit local HTTP loopback mode the directly observed loopback -peer address is recorded. Operators requiring per-client-IP rate limiting must -enforce it at the proxy layer (e.g. Contour / Envoy) before requests reach the -mTLS port. - -### Proxied deployment — Contour/Kubernetes contract - -> **Prerequisite, not implemented here.** This section describes the platform -> configuration that a proxied deployment requires before the mTLS listener -> can serve `/_ops` traffic in a Kubernetes environment. None of these steps are -> part of this application. - -The application exposes a second port (default `8443`) for the mTLS-protected -console. The following Kubernetes and Contour configuration is required to route -`/_ops` traffic to it. - -**Kubernetes Service** - -Add a second port to the Service targeting the router Pod: - -```yaml -- name: ops-console - port: 8443 - targetPort: 8443 - protocol: TCP -``` - -**HTTPProxy route** - -Route `/_ops` to the ops-console port using TLS upstream with backend server -verification. The `subjectName` / `subjectNames` field name depends on the -deployed Contour version; check the Contour -[upstream TLS documentation](https://projectcontour.io/docs/1.33/config/upstream-tls/) -and [configuration reference](https://projectcontour.io/docs/1.33/configuration/) -for the correct field for the version in use: - -```yaml -routes: - - conditions: - - prefix: /_ops - services: - - name: router - port: 8443 - protocol: tls - validation: - caSecret: ops-console-server-ca - subjectName: -``` - -`validation.caSecret` is the CA that **Contour uses to verify the router's -server certificate** — it belongs to the server-certificate trust chain, not to -any client-certificate authority. `subjectName` / `subjectNames` must match a -SAN on the router's own server certificate. Use a secret name such as -`ops-console-server-ca` to make the trust direction unambiguous. - -**Envoy client identity** - -These two trust directions are independent and must not be conflated: - -- **Contour → router (server verification).** Contour authenticates the router - by checking its TLS server certificate against `validation.caSecret` - (`ops-console-server-ca` above). -- **Router → Envoy (client verification).** The router authenticates Envoy's - client certificate against the CA mounted at `OPS_CONSOLE_MTLS_CA_PATH`. - This is a *separate* CA that issues only Envoy upstream identities; it has no - relationship to the server CA and must not be reused for one. - -Configure Contour's global `tls.envoy-client-certificate` with a cluster-wide -Envoy client identity. The certificate Envoy presents to the router must be -signed by the dedicated Envoy-client-issuing CA referenced by -`OPS_CONSOLE_MTLS_CA_PATH`. Use a dedicated CA for this role; do not reuse the -server CA or any shared cluster CA merely for convenience. - -**Secret mounts** - -Mount the server key, server certificate, and Envoy client CA as read-only files -inside the container. Use only absolute in-container paths for -`OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and -`OPS_CONSOLE_MTLS_CA_PATH`. Never reference test fixture paths at runtime. - -To exercise mTLS in local Docker Compose, add an explicit read-only volume -override in a `compose.override.yaml` that mounts the secret files at the -configured absolute paths. The default local Compose topology uses HTTP loopback -and does not require this override. - -### Proxied deployment — enablement order - -Follow this order to avoid exposing an unconfigured mTLS port or routing -`/_ops` traffic before the application is ready. - -1. **Provision secrets and platform client identity.** Create the TLS secret - files, mount them read-only into the container, and configure the Contour - global `tls.envoy-client-certificate`. -2. **Deploy the application with the console disabled.** Set - `OPS_CONSOLE_ENABLED=false`. Verify public traffic is unaffected. -3. **Enable the mTLS listener.** Set `OPS_CONSOLE_ENABLED=true` and all four - `OPS_CONSOLE_MTLS_*` variables. Restart the router and confirm the startup - log shows `ops-console enabled`. The mTLS port is now open but `/_ops` is - not yet reachable from outside. -4. **Add and enable the Contour `/_ops` TLS route.** Apply the HTTPProxy change - that routes `/_ops` to port `8443`. -5. **Verify.** Run `npm run test:integration:ops` and confirm snapshot and cache - operations work end-to-end through the proxy. -6. **Expose and use the console.** Public traffic on port 8080 must remain - unaffected at every step. - -To roll back, remove or disable the Contour `/_ops` route **before** disabling -the mTLS listener or rolling back the image (route-first rollback). This ensures -no `/_ops` requests arrive at a listener that is no longer running. - -### Proxied deployment — certificate lifecycle - -**Restart-based reload.** The TLS files are read once at startup by -`fs.readFileSync`. To apply new certificates, rotate the secret files and -restart the router container. There is no live reload. - -**Overlap rotation.** When rotating certificates, provision the new secret files -and restart the router before the old certificate expires. Keep sufficient -overlap to complete the restart and re-verify before the old certificate's -not-after time. - -**Urgent revocation.** There is no OCSP or CRL check in the application. To -contain a compromised key: restart the router immediately with the new -certificate (or with the console disabled), then drain or terminate sessions. -In-memory sessions are invalidated by restart. +The audit `source` field is always `null`; no network identity is trusted and no +client or proxy IP is attributed. Operators requiring per-client-IP rate limiting must +enforce it at the proxy layer before requests reach the router. --- @@ -1112,7 +955,7 @@ Required audit event fields: | `action` | Normalized operation name | | `correlationId` | Router request ID | | `sessionAuditId` | Separate random per-session audit ID, independent of cookie, non-authorizing | -| `source` | `null` in HTTPS/mTLS mode (no client or proxy IP is attributed); directly observed loopback address in explicit local HTTP loopback mode | +| `source` | Always `null`; no network identity is trusted and no client or proxy IP is attributed | | `userAgent` | Bounded, truncated | | `operation` | Validated operation | | `targets` | Validated target list | @@ -1371,7 +1214,7 @@ cover at minimum: **Configuration and startup:** - Disabled routes 404; no internal calls; sanitized status event. -- Enabled startup failures for every illegal loopback/mTLS combination. +- Enabled startup failures for every illegal loopback combination. - `NODE_ENV` alone must not enable the console. **Login and credentials:** @@ -1456,10 +1299,6 @@ Integration scenarios must cover: - No automatic retry. - Audit event format and 64 KiB response bound. -In deployed-development verification (HTTPS mTLS): additionally check HTTPS ingress, -`__Host-hc-ops` cookie, exact configured Origin, and that `/_ops` requests on the -public HTTP listener fail closed. - ### Gate 4 — Security and failure injection Failure injection cases map to OWASP ASVS 5.0 V3/V4/V7/V8/V16 and relevant WSTG @@ -1526,8 +1365,8 @@ Automation supports but does not replace manual accessibility acceptance (§16). - All `/_ops/*` paths must return `404`. **Step 5 — Router, console enabled:** Restart the single router with -`OPS_CONSOLE_ENABLED=true`, valid `OPS_CONSOLE_TOKEN_VERIFIER`, -`OPS_CONSOLE_ORIGIN`, and (for HTTPS mode) the four `OPS_CONSOLE_MTLS_*` variables. +`OPS_CONSOLE_ENABLED=true`, valid `OPS_CONSOLE_TOKEN_VERIFIER`, and +`OPS_CONSOLE_ORIGIN`. Required before declaring success: - Sanitized enabled status event appears in stdout. diff --git a/test/fixtures/mtls/ca.crt b/test/fixtures/mtls/ca.crt deleted file mode 100644 index 146efcc..0000000 --- a/test/fixtures/mtls/ca.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDVzCCAj+gAwIBAgIUXIdQnSu6cDMuCpar0PediZHbfO4wDQYJKoZIhvcNAQEL -BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ -RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAyMTAwLgYD -VQQDDCdISUdIQ0hBUlRTIFRFU1QgVFJVU1RFRCBQUk9YWSBDTElFTlQgQ0EwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKNjsRMjjamhr+FqIY6LBjcUkm -drJSUIV7MBZx7cmYShGspWoTYzxJKZcL5jniVq4rCQOjf0ZHc5zOWEtknOc0fQyu -5YlNxLbdcikwfbEny2zk7QDvSSN13XB77OE7lRCLTpRfSM66/O9KzcCBe9qz+ya7 -A3MKtGEhFR3Tk0AS/awLZNFq+YoOoAhwj+38EU0ZH4Mz60kYF9pVIOv3RFgD3Jmd -O4Gv8UB72uv6Wjdwo7tdtRwycHxnS6uODkBJTLB3zkH0Kn+w2u+kqzthl71IAxA2 -cDhoLtf6+dz0Kk94Rbqr4PBOYj4MdnOhEaB7eTNBVw4zVojLNo9p0elkupdNAgMB -AAGjYzBhMB0GA1UdDgQWBBS0NfsJDbelcn2MSv4MH/VTekCM7TAfBgNVHSMEGDAW -gBS0NfsJDbelcn2MSv4MH/VTekCM7TAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB -/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEASBdrQw+ajIXPzRDN6KxP5B+BOvsz -hAmtzK0lmI+OuL6/dEdfWePvYCbV11ygd9nBdTPL3SxA45tEDR8dI6f8wdJqAiPE -HxRK5vcYXWewFBuCl1bFqGFH6BFjshAh3nk5pdbilEG6i6H2eqiCxI91jFoqkr8X -JVB5aEhlODMfm/eZr2MuoqJazKYlFzZGjZEKIayrTUhUFurdliD8j1HZ9TUq8zlc -wmA3DBK4m8EeT4ABTXXdyNmEKlDKOA8YdwLe6mGPw9F1AWdI0Zgi1hc0hr/JCm8o -3cg1nOU+VM5eA3b3dybntOCSfNcMqrkIzwVoiwUVa2Gpdoi5ne/+c3L1pg== ------END CERTIFICATE----- diff --git a/test/fixtures/mtls/client.crt b/test/fixtures/mtls/client.crt deleted file mode 100644 index e8979ba..0000000 --- a/test/fixtures/mtls/client.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDZjCCAk6gAwIBAgIUS1aaNaNolAJyG1LiJHPw7vHw11gwDQYJKoZIhvcNAQEL -BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ -RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAvMS0wKwYD -VQQDDCRISUdIQ0hBUlRTIFRFU1QgVFJVU1RFRCBQUk9YWSBDTElFTlQwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDue8uu1Zlh9BNbHumlLJJMBYj19al -kPAoHD76nzdmWjQt/0SwPtD/gkyxrjspShcnc3RWIB5g6hnnFYMWh9ZMcU7woiYp -2qiLygw0QCGn/F6b4nEuMLaAkca6/iO781gRTFBN/zYhjk5ev1B8bDaPMCvd+BLN -djDNy0PrZy4dxYZSj03H2Ybfzd5xskpePW/a3ifH67r9Mo9mXDBUufhjBSsU8Ray -UWMbFalhHeB2oGfV0B5ZicDtlFQPAjLa8CzA7RdUsBXn0TTPr7uCh33ZJiXTGvcf -mZqWXWQtZT8JLAbHq375kFUaguLckOtWmJlESfnJwpvd0SpVfyWVrm6zAgMBAAGj -dTBzMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsG -AQUFBwMCMB0GA1UdDgQWBBSByyXC8yFoSzuH4//diB1WxVcNADAfBgNVHSMEGDAW -gBS0NfsJDbelcn2MSv4MH/VTekCM7TANBgkqhkiG9w0BAQsFAAOCAQEAIUoSU7vN -MF42QiX3xTpdgwjXGoCA0AKLYgMjPk0JQ1uPByVm0kEqlypXsmo2JOpEc59UqYAJ -4zqyBLu0F0d8oSVNw2GRIKDrWToMyGCQWxawQJd6thkRD+aZxPuwQQ4DfZwxqI/I -4Y74V2QIHNif+/T7OWOrbBGCkvZRB1l+StlbdYYh87vsfXuGa0C2UnnXZnalN6aS -L1B/J7UrAS+QXkZBHA89zceOgaPm4rpEp8wjfzzecHpH430wA818UPcX6NEcMRwp -oshrHeivOiA4tFcNfrKw93om+vguQ+ylnHdlb11FJm6DFsFBLXhFxXrrdHnoOBh9 -Km6EtLJaqdgKJw== ------END CERTIFICATE----- diff --git a/test/fixtures/mtls/client.key b/test/fixtures/mtls/client.key deleted file mode 100644 index b238a2c..0000000 --- a/test/fixtures/mtls/client.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDue8uu1Zlh9BN -bHumlLJJMBYj19alkPAoHD76nzdmWjQt/0SwPtD/gkyxrjspShcnc3RWIB5g6hnn -FYMWh9ZMcU7woiYp2qiLygw0QCGn/F6b4nEuMLaAkca6/iO781gRTFBN/zYhjk5e -v1B8bDaPMCvd+BLNdjDNy0PrZy4dxYZSj03H2Ybfzd5xskpePW/a3ifH67r9Mo9m -XDBUufhjBSsU8RayUWMbFalhHeB2oGfV0B5ZicDtlFQPAjLa8CzA7RdUsBXn0TTP -r7uCh33ZJiXTGvcfmZqWXWQtZT8JLAbHq375kFUaguLckOtWmJlESfnJwpvd0SpV -fyWVrm6zAgMBAAECggEANW/9Wo/xTbUf7ROSSu/MIWlMkiqqwvdoajsUAs8XjA1S -s8A/G7N60lfb4qMEKgi9e5rtB1qrkKA5xDq+WJdrreE9wTs0GjdFzyyx2k4sIjYo -Cn1vk0HfggjK7mDWlskgoVBpmHH2cIDu6rVnHyFYYA2x3F+PmqMLPhSiDZJVJ/E8 -uBgEPBO2HlPjMsaz2nLiMw+cpFQTZtEgDFQZCSI9jAMnt0B4mXxmjCsjnSSRrP7m -j35IeOOYWr0CJZkkIt0/t2cFM0eGMW+BtqjRUOLLx10++PLr1LLRuQihS9g895Ae -R4Hr6MbQcIAtA409svbG35USP+XanOdYiJKfU5D1cQKBgQDhz3AU929gLauiYwSF -g9CT9MwmnEDwxB3eE+voZRQzUelieMsPyKp+3alyGAP57uKI3i4BpKArGKICNih5 -i7rxPgmkKcCH+AOYCLM58rGa7ZwZ6PyNbTI5BxJUJtMKyYQhFveXoDF8fiy9c8HX -VAIcON1TPsJAulm2nEMbYmpNiwKBgQDd5Ngq1abh+jpdClhYdYxlsV1zeOM1FeRH -XsT15RlgilLxKk2Lukt7X7FwCdiazt6DH5EsPe6lhaovIQMiBjTsd/YkDvu5IYZ1 -0PNvhk4k0SurxFINDGbNgpym+ZKM15ViyLjGykJB0mUXAYG6vn4jym1h9FDTvO1P -IHHofVhYeQKBgDgYyXZb9e5FykLAKIpmsbVf9iuNW9C0V9soxc1o9vi826bb7U5R -gpGbzZGLh8laYCqyT2mXFTc/mlfETo/Ld7igudJvkOX2ZiYp2ySFNzwO1V3WdI9J -1lU2fYYsUvd2En4J755abJDJ46F5FWnB8/hA5DLe/3EHGmx0K3OtIk17AoGBAMw2 -n8eUR/kzjOEx8yq+TE8PFB2AxUKG+kfA7W4MwfU6eKkhMKsG8g8Ce8/MEAAxoVF2 -DOp1uRu2z3B+Zl667Zwvr2VyMLMqKpBllJUwOtzhcNqtXIJLxpUevsNhb0GV6xM1 -/fBeFupzErxAk79lL7wKwe5jprun5ZNsHclFCripAoGAR+B866iyJmaZtO1hW4aO -d3gHdaquKbguvTYiQHcI85sXiPuJrZhcyWrY3qEpVwBbReAAyUfg4/W80hcr4MA7 -g2zkxiCrLtm1GlgEErpQhpzAW3MpmPq184RU4XjqyiQXxtVTNr8L4QmQaj1W4v/+ -9tHHWQ7ZGbEmFWEng0pcJQc= ------END PRIVATE KEY----- diff --git a/test/fixtures/mtls/server.crt b/test/fixtures/mtls/server.crt deleted file mode 100644 index 4be1571..0000000 --- a/test/fixtures/mtls/server.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDejCCAmKgAwIBAgIUS1aaNaNolAJyG1LiJHPw7vHw11cwDQYJKoZIhvcNAQEL -BQAwMjEwMC4GA1UEAwwnSElHSENIQVJUUyBURVNUIFRSVVNURUQgUFJPWFkgQ0xJ -RU5UIENBMCAXDTI2MDcyMTExMDAyM1oYDzIxMjYwNjI3MTEwMDIzWjAlMSMwIQYD -VQQDDBpISUdIQ0hBUlRTIFRFU1QgT1BTIFNFUlZFUjCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAJVHcT35zgp7NQF98XUgLXbsjB+RfOkpjbtx50cTvFH5 -Fx0pCm1aPW4mbY5UMx2KYmxE4A+nDyLVbihAZOt+tka7d+qdWT4c3UU/WH49jAN1 -twyJ1F+GDMCx5sM69DvWUlDH48dFHjjsanVa4y9/V5nqimyNsZxvR0S12ggbXSot -ZfsY/GKiwgzl76l9UZbZbMBQ+3dSiTBDw+CxNTb894Up8GY+LhOBBEOk00ssCBCO -WYXD+HFRGKtEQQmluyBMQZU5mgTVYGZfd6dPZC1eLXWrIK+CvyqJBc/D7gyMlNyo -tQHtbJ4gGR32eZv06my1dvPZlcd8sM2ZXkSh6xFPSTcCAwEAAaOBkjCBjzAaBgNV -HREEEzARgglsb2NhbGhvc3SHBH8AAAEwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8E -BAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYEFIHcbY+KiAWVTzKw -qWHq3RjBZmbkMB8GA1UdIwQYMBaAFLQ1+wkNt6VyfYxK/gwf9VN6QIztMA0GCSqG -SIb3DQEBCwUAA4IBAQAHuPi95GdRSa2DB6fvO1w6oUZRbfGqsjF/RkiQXYK16IpT -8qbTXUF4cB1PiGJZng+lS1z5xSgQBtSN6sK3hqWLQe+ZwjKasOrvK+0clZn3cwpy -u7BaGegpaxie9/dQOOuI/2CfmET/vjjueH8VW7nvEneKftdSdWc8uyTWlFcOONiu -5GtmIz6UDlvzwoiIymb+MCRbyogWsBM+v7/yIMU4hG28eG2C+24hwEsMqI30wFNA -UeEgZkUeoZXAoRQqaCx8G/edY3ZzxFLZwrecZLDCmLAx3nh0CoWJ+lUrYggPo0OR -vN7f0eQ2B9LqwHBs55o3DQSNZ5T2Esl1SVmt5S85 ------END CERTIFICATE----- diff --git a/test/fixtures/mtls/server.key b/test/fixtures/mtls/server.key deleted file mode 100644 index bf63e3c..0000000 --- a/test/fixtures/mtls/server.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCVR3E9+c4KezUB -ffF1IC127IwfkXzpKY27cedHE7xR+RcdKQptWj1uJm2OVDMdimJsROAPpw8i1W4o -QGTrfrZGu3fqnVk+HN1FP1h+PYwDdbcMidRfhgzAsebDOvQ71lJQx+PHRR447Gp1 -WuMvf1eZ6opsjbGcb0dEtdoIG10qLWX7GPxiosIM5e+pfVGW2WzAUPt3UokwQ8Pg -sTU2/PeFKfBmPi4TgQRDpNNLLAgQjlmFw/hxURirREEJpbsgTEGVOZoE1WBmX3en -T2QtXi11qyCvgr8qiQXPw+4MjJTcqLUB7WyeIBkd9nmb9OpstXbz2ZXHfLDNmV5E -oesRT0k3AgMBAAECggEACqx3DmVkNB+nJJoqv6MzXQOA6Wjvs4RDHBoC3XQtzPaw -jmc21abUKaIZx0mB9iTE8NTj6HqbfHQiUkJ4dNY0lk4lPuNNVEGgGKl03GHuNvkd -w4m/Y60kEEsoIuF+QSEL9ba1NLReetd8rTN4dxb13o2EpEplXrgeMm2GT3oVaUSF -w7t42suo6bFt07+ARVGgw298UAI3M82Ey6v7WiWDae5uBRSiT2SbfrWcvcv1El9n -oFKEPJbdmqkKfWOpQgvnp0zqJ7L0xXkRJuUYWBa/zoPeiRIOQjU4Vk5X69yZuLo1 -iD3xL+x/WMVmDrmGjht8IfU1PbwH/zapF59JP3oJkQKBgQDJ9Egj4YA/ALPeTl0u -i1XFrOVj45gE5rnEhiFtXIzy5rm4LupsybXRhfAEyOZ78tcwfl9hzq1gMh3xoj7m -3BcrlaBOUjKL5Bt7XLQJUSdqz+yFbAeY8Cr2/9W4DzSQLlnHpDC/PEEWJAsx3wPK -nFqRibYrJxi6c/sBxlEgMSx8wwKBgQC9Om3SlvWDFT7gq5Qr9arfgYrazPPsFbWr -YDNmDp4KjKMjX7ffOXawwcw3+ofSWnBb2ba4/ggDH/T4aHxtd5snFi9Xdusit3wc -ooyJc0NUxTrHko3epbDZMM7zHDyB0npKNsJj20cthNvwNnaua50YeGDdhK2CkvHu -A7QQUwJKfQKBgQCeid54DHaY/vw08F/GQiu7WtdZaznT3yzGUmW7bIRZyzbQmEP/ -0vmg2fxqRSxq8WBs+Uf3iEAi3DUVk8C9itnFpViLI4v6tb+9QDE1fzfqaf/LXds4 -/JE+BejI7WbeKQTh7Ms31R1jPDhtlh1r2QJgbjNL/Q00kgfihMT6+J9r8QKBgCls -AuJYXUHmgq8XoAXHbzIh301qE/MYBX6QPnAWvw28H3H83/kjURH8OkH+u4CWf4X7 -sH3qTcKxWiSOar5jsjjqKE7TH0GoPKjgBDeKXbDOw8EwGZIlXwMMJiEdizk348Ef -H4pQU9JpBOQeZ/hiYi8bGski5ABzPjZF5UK1iQjZAoGAMr8VO7+hc3Nd7PGGG7Zg -KIS+Mjy1NzAkVvYfgiuthKcNm4cLDjPFapRz5Y/au4csqIlpJkfczq6u0wlb8tey -jLMQ27eOdSBnNhfHpOBGqqEcE0SWTwZOQw3jEBK08zxB7XdBiUtoXyIldr2oDk55 -jJwS5TYLDAxTaj5JO5qXYC0= ------END PRIVATE KEY----- diff --git a/test/fixtures/mtls/untrusted-ca.crt b/test/fixtures/mtls/untrusted-ca.crt deleted file mode 100644 index 55232eb..0000000 --- a/test/fixtures/mtls/untrusted-ca.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIUBFPgylvNJTthRpraK5pc8sOnxU8wDQYJKoZIhvcNAQEL -BQAwLjEsMCoGA1UEAwwjSElHSENIQVJUUyBURVNUIFVOVFJVU1RFRCBDTElFTlQg -Q0EwIBcNMjYwNzIxMTEwMDIzWhgPMjEyNjA2MjcxMTAwMjNaMC4xLDAqBgNVBAMM -I0hJR0hDSEFSVFMgVEVTVCBVTlRSVVNURUQgQ0xJRU5UIENBMIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAm7uVWJ/Y3bToqhX/TplpV+nvjHwihxKpy/pu -oXEnzjyzPk5EeKROn81JCwFfW4NRaM31i1/F/Chboz8J/0VoI9V0PgbFCMIxEI8g -BzyYcmlQmIEom/1bW100keC8CWKdXu1sCz6XDmLqkpbntVP1iLJQoUwyHeJVDZWK -KuW96T/7JkSFanIVkBxOxImXWsIydGqMRYUxcGrkRRjcbggwNtQ/CrinkzGE4433 -Hhlq0qAZk87OVrJEnjT24HfaHmQ70DHeJbE6E859X8Q5qTBVtdsC9JwUNOVUhnKA -C+zlLv921iFdIVZb/ff4yfjkGUiywNDK9wgFMDQKEStJaoVPTwIDAQABo2MwYTAd -BgNVHQ4EFgQUEEvT8qmkVIxZSv7fYkK7XOllClAwHwYDVR0jBBgwFoAUEEvT8qmk -VIxZSv7fYkK7XOllClAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw -DQYJKoZIhvcNAQELBQADggEBACwtUa5VGeWsMnWbdejtJ8mX5JVt4GjnUugCxv5A -gGxnr/rfeL4r9cjeBuUydYQoCSYgCwD7n/J2I0yE/0odMRKmuXNT3JCdluvtNwAr -0+tiq0lClanfeO9MTrk9Ri9I7yVgXZAAUypNoxceJ0w4LsJ0jhOLZCT2+UMTuDfh -sn4CORDA16jhgqiLlA2G/+jrOGAmc+4Dmwkl7c221ahaAj+rTMsdWx6XdE/B6Y6w -op9jLP+s8uvhobYyb1QZFXbzToszTiaZyNj7St2M0pFWe3H4Zl3CqqND3IPD3fOq -YB5oBTqYGTOvsLOSqPOp3BKgrgZtoIU4ar9D4oZpwZYBnZM= ------END CERTIFICATE----- diff --git a/test/fixtures/mtls/untrusted-client.crt b/test/fixtures/mtls/untrusted-client.crt deleted file mode 100644 index 30f7ea2..0000000 --- a/test/fixtures/mtls/untrusted-client.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDXjCCAkagAwIBAgIUDo9v4dFgQ11j9iLP3MmD02+FgwIwDQYJKoZIhvcNAQEL -BQAwLjEsMCoGA1UEAwwjSElHSENIQVJUUyBURVNUIFVOVFJVU1RFRCBDTElFTlQg -Q0EwIBcNMjYwNzIxMTEwMDIzWhgPMjEyNjA2MjcxMTAwMjNaMCsxKTAnBgNVBAMM -IEhJR0hDSEFSVFMgVEVTVCBVTlRSVVNURUQgQ0xJRU5UMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAkR2Yx3codyLyWx+xE9mKrwrdAzQ0zg4jnW0ePUi0 -WuesK/5x3WSzwqK3Opw+MwD7rTQPRc5USsfdViyYHo36VnBP7aiCRHo3F+Le5DQZ -KIrUIfH2USOfG0Guc2wvotvgp2xHThFayySRFrCifnLNbuHeyUwF5e0C7ytrj7mf -YOipS0z/sHoOLwmB3djzQ8f/6XBKaIr15pUZa1PIulWmSCtlFKRVnEljB3sRGatt -EyLDHOqqKC2adZjZYxEvzt6ZJCYj8UuoB3+sF3W2TVttDKiB8sol2mx4fkDLLf8F -8AyIFFDlOk1eWiAI8WEjMUoNugKeLld/ZHuDjVIefMOiMwIDAQABo3UwczAMBgNV -HRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAjAd -BgNVHQ4EFgQUxZb84ATJ7qvD80uffCLDkcMML3AwHwYDVR0jBBgwFoAUEEvT8qmk -VIxZSv7fYkK7XOllClAwDQYJKoZIhvcNAQELBQADggEBAF202eemY9zihOrNCLV1 -RQh+GeKodbZEuv/bv90jj/xJFSdWVeVK97mCF39LVgVf8t3hAxkSh6TYm3NDUSM8 -BRVQ704bRXNAgHbG1Q3eBgUafTCPR7UJpmiqd8zIVQYSM6f7WbEb9n4vJvI5uA05 -rZPHR1tqldqgPRSPBZWpc0Cqm1Slgdrt/GGV0YOJlcJeGOLNGDDcVp6vlowujax5 -meliD+JI0Aq7/m2dVBzEOFs8G5tzgAs8m4dR+w8rtB3VpCNBEpkGJvoeabZX+nPA -fhtzcifa8wDFtT6tIQElN2ZUxZk7SlI0t81YOtXaMMw4oVAoaFORMSnihce0d1QV -HRo= ------END CERTIFICATE----- diff --git a/test/fixtures/mtls/untrusted-client.key b/test/fixtures/mtls/untrusted-client.key deleted file mode 100644 index 24bef50..0000000 --- a/test/fixtures/mtls/untrusted-client.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCRHZjHdyh3IvJb -H7ET2YqvCt0DNDTODiOdbR49SLRa56wr/nHdZLPCorc6nD4zAPutNA9FzlRKx91W -LJgejfpWcE/tqIJEejcX4t7kNBkoitQh8fZRI58bQa5zbC+i2+CnbEdOEVrLJJEW -sKJ+cs1u4d7JTAXl7QLvK2uPuZ9g6KlLTP+weg4vCYHd2PNDx//pcEpoivXmlRlr -U8i6VaZIK2UUpFWcSWMHexEZq20TIsMc6qooLZp1mNljES/O3pkkJiPxS6gHf6wX -dbZNW20MqIHyyiXabHh+QMst/wXwDIgUUOU6TV5aIAjxYSMxSg26Ap4uV39ke4ON -Uh58w6IzAgMBAAECggEAGWlx6igjNW2wvCVaGIxFXW7NEjUPtC/Eq9pCa9/x+WNN -gqy9mtP6KLDe3kwjFkJrUELoE8TUfP3v9Bm+D8e0GXP0gz05hq1SPYQUnSjEaRWa -nVEmXcIbqCXB22OMGfxgJGFxQSoH2MAQCkWnRvZqpCq4nU6LT97H30MexF3wm4MP -LLTe2zvastK0ht/OIc6F4vX3IDk4YS6HScCIDoFT+8ODdIUW4y26BY0KvvHHoHGA -+39TT+88CNqtlier7amgWSF7Uz454NqnyPnm3UN6HGFV9FpFf8dBcBw0apW8cYHd -3oB3tKZ/1e8dLQ5f32zKDQnQaBKU0jMbKvfphlZRlQKBgQDG2IOs+EydFzGvkBum -EnNcjJbCGgNy3UEo+th/fcydTV3aMmHiVo5UsQ3aRJOODMJ16lsEhCElgsy336py -6FjqgF+a8gcQ4EClzYR03FPs8JKrpthua2JofxFIYPMdCDpVkSvIIQkXYDhdJdgj -styf92L59ch7LIribLkSgWhxPwKBgQC604GVjBhi0CQfwtRsCj8KiqMUmVZkCFwP -N02Swfmdsdxh8IjijaTmtxQ9ABxUtXIJbjOjHUh+zk94/eQ1t0ah/HseNg0q17CB -auKGmVcxam0/pVUPxv/GPlfXJXdFJgEqkvmciGduuk915hy2gvlvjHG5khuxyjdV -KTqI1i+eDQKBgEpQbt29KnznRzVy6TJm4OBs1ocSS8mo8QpMvY7gSBFd8b5zZ+u8 -J5N1XFT/nhnPoPmKrZuBehXXzYTGvaAWP+zcL17sJ0HiXabZqTA3i9IYkug+AdM6 -pNui6aWgPhvSkGKcT72KwygOWOJPG3k0BopuXXpHpjaKjhlTQtOss4jnAoGAO8X/ -58Dy7Gp7pE3JeOeuU8kXdDe7FDY4fgXLo/C71GBbilpS8WkwIUDiEMJATjnfSnUf -wEhWMXwAoU1Z/nWs+5a0LW5NuC9aIY34++eDpiPO4FYdaPbaTqsTn5o1FsKaDO0J -P7PEpACn+6ir9xjghXgBysWcZr20BmMyfyVIBuECgYEAmqPBqCesFgX0OYAHUt0D -E4GEW3D86QL7JDzq4F4fe/T5bqPmFYujPczKwK1TkqOyN0U88K6yKVTUF3P0FSXF -pu+wMYVqYFkWlP3rLLEMneZj5zxMWci91Hk0bcgO4gTlH42lyB4B8qcuj4y0MXEa -wapiO8F/nTpJf9MgA0DlT+c= ------END PRIVATE KEY----- diff --git a/test/ops-config.js b/test/ops-config.js index 08b87b3..e57c4cc 100644 --- a/test/ops-config.js +++ b/test/ops-config.js @@ -2,25 +2,19 @@ const { expect } = require('chai') const { randomBytes } = require('node:crypto') -const { join } = require('node:path') const { describe, it } = require('mocha') const { createTokenVerifier, - DEFAULT_MTLS_PORT, parseOpsConfig, verifyToken } = require('../app/ops/config') describe('operations console configuration', () => { const token = randomBytes(32).toString('base64url') - const absolute = name => join(__dirname, 'fixtures/mtls', name) const enabled = overrides => ({ OPS_CONSOLE_ENABLED: 'true', OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), OPS_CONSOLE_ORIGIN: 'https://ops.example.test', - OPS_CONSOLE_MTLS_KEY_PATH: absolute('server.key'), - OPS_CONSOLE_MTLS_CERT_PATH: absolute('server.crt'), - OPS_CONSOLE_MTLS_CA_PATH: absolute('ca.crt'), ...overrides }) @@ -49,29 +43,27 @@ describe('operations console configuration', () => { enabled({ OPS_CONSOLE_TOKEN_VERIFIER: `v2.${'A'.repeat(43)}` }), enabled({ OPS_CONSOLE_ORIGIN: 'https://ops.example.test/' }), enabled({ OPS_CONSOLE_ORIGIN: 'HTTPS://ops.example.test' }), - enabled({ OPS_CONSOLE_ORIGIN: 'ftp://ops.example.test' }), - enabled({ OPS_CONSOLE_MTLS_KEY_PATH: undefined }), - enabled({ OPS_CONSOLE_MTLS_CERT_PATH: undefined }), - enabled({ OPS_CONSOLE_MTLS_CA_PATH: undefined }), - enabled({ OPS_CONSOLE_MTLS_KEY_PATH: 'server.key' }), - enabled({ OPS_CONSOLE_MTLS_CERT_PATH: './server.crt' }), - enabled({ OPS_CONSOLE_MTLS_CA_PATH: 'ca.crt' }) + enabled({ OPS_CONSOLE_ORIGIN: 'ftp://ops.example.test' }) ]) expect(() => parseOpsConfig(env)).to.throw() }) - it('requires dedicated HTTPS mTLS paths and a strict bounded port defaulting to 8443', () => { + it('uses HTTPS as the external origin without backend transport configuration', () => { const config = parseOpsConfig(enabled()) - expect(DEFAULT_MTLS_PORT).to.equal(8443) - expect(config.mtls).to.deep.equal({ - port: 8443, - keyPath: absolute('server.key'), - certPath: absolute('server.crt'), - caPath: absolute('ca.crt') - }) - expect(parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: '1' })).mtls.port).to.equal(1) - expect(parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: '65535' })).mtls.port).to.equal(65535) - for (const port of ['0', '01', '65536', '-1', '8443 ', '1.5']) { - expect(() => parseOpsConfig(enabled({ OPS_CONSOLE_MTLS_PORT: port }))).to.throw('Invalid OPS_CONSOLE_MTLS_PORT') + expect(config).to.include({ enabled: true, origin: 'https://ops.example.test', protocol: 'https', allowHttpLoopback: false }) + expect(config).not.to.have.property('mtls') + }) + + it('rejects nonblank retired mTLS settings and ignores blank or disabled settings', () => { + for (const name of [ + 'OPS_CONSOLE_MTLS_PORT', + 'OPS_CONSOLE_MTLS_KEY_PATH', + 'OPS_CONSOLE_MTLS_CERT_PATH', + 'OPS_CONSOLE_MTLS_CA_PATH' + ]) { + expect(() => parseOpsConfig(enabled({ [name]: name.endsWith('PORT') ? '9443' : '/retired.pem' }))) + .to.throw('OPS_CONSOLE_MTLS_* settings are retired and unsupported') + expect(parseOpsConfig(enabled({ [name]: ' ' })).protocol).to.equal('https') + expect(parseOpsConfig({ OPS_CONSOLE_ENABLED: 'false', [name]: '/retired.pem' })).to.deep.equal({ enabled: false }) } }) @@ -83,24 +75,16 @@ describe('operations console configuration', () => { it('allows HTTP only for direct loopback mode', () => { expect(parseOpsConfig(enabled({ OPS_CONSOLE_ORIGIN: 'http://127.1.2.3:8080', - OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', - OPS_CONSOLE_MTLS_KEY_PATH: undefined, - OPS_CONSOLE_MTLS_CERT_PATH: undefined, - OPS_CONSOLE_MTLS_CA_PATH: undefined + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' }))).to.include({ enabled: true, protocol: 'http', allowHttpLoopback: true }) expect(parseOpsConfig(enabled({ OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', - OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', - OPS_CONSOLE_MTLS_KEY_PATH: undefined, - OPS_CONSOLE_MTLS_CERT_PATH: undefined, - OPS_CONSOLE_MTLS_CA_PATH: undefined + OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' }))).to.include({ enabled: true, protocol: 'http', allowHttpLoopback: true }) for (const overrides of [ { OPS_CONSOLE_ORIGIN: 'http://localhost:8080' }, - { OPS_CONSOLE_ORIGIN: 'http://example.test', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' }, - { OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', OPS_CONSOLE_MTLS_PORT: '9443' }, - { OPS_CONSOLE_ORIGIN: 'http://[::1]:8080', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true', OPS_CONSOLE_MTLS_CA_PATH: absolute('ca.crt') } + { OPS_CONSOLE_ORIGIN: 'http://example.test', OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' } ]) expect(() => parseOpsConfig(enabled(overrides))).to.throw('HTTP operations console') }) }) diff --git a/test/ops-console.js b/test/ops-console.js index b93164a..31029e2 100644 --- a/test/ops-console.js +++ b/test/ops-console.js @@ -498,7 +498,7 @@ describe('operations console router', () => { expect(audits).to.have.length(1) expect(audits[0]).to.include({ correlationId: response.json.correlationId, - source: '127.0.0.1', + source: null, userAgent: 'chromium', operation: 'cache.evict_commit', commit: 'a'.repeat(40), diff --git a/test/ops-http.js b/test/ops-http.js index b907865..afa44ad 100644 --- a/test/ops-http.js +++ b/test/ops-http.js @@ -9,7 +9,6 @@ const { OpsHttpError, SECURITY_HEADERS, errorBody, - getTrustedRequestContext, readStrictJSON, requireBearer, requireCSRF, @@ -144,32 +143,6 @@ describe('operations console HTTP boundaries', () => { } }) - it('requires an authorized encrypted socket for HTTPS and ignores forwarded claims', () => { - const context = getTrustedRequestContext({ - socket: { remoteAddress: '10.2.3.4', encrypted: true, authorized: true }, - headers: { - forwarded: 'for=192.0.2.8;proto=http', - 'x-forwarded-for': '192.0.2.8', - 'x-forwarded-proto': 'http' - } - }, { protocol: 'https' }) - expect(context).to.deep.equal({ protocol: 'https', source: null }) - - for (const socket of [ - { encrypted: false, authorized: true }, - { encrypted: true, authorized: false }, - { encrypted: true }, - {} - ]) expect(() => getTrustedRequestContext({ socket, headers: {} }, { protocol: 'https' })).to.throw(OpsHttpError) - }) - - it('requires direct HTTP request sources to be loopback', () => { - const config = { protocol: 'http' } - expect(getTrustedRequestContext({ socket: { remoteAddress: '::1' }, headers: { 'x-forwarded-for': '192.0.2.1' } }, config).source).to.equal('::1') - expect(() => getTrustedRequestContext({ socket: { remoteAddress: '192.0.2.1' }, headers: {} }, config)).to.throw(OpsHttpError) - expect(() => getTrustedRequestContext({ socket: { remoteAddress: '127.0.0.1', encrypted: true }, headers: {} }, config)).to.throw(OpsHttpError) - }) - it('produces bounded generic errors without reflecting sensitive values', () => { const secret = randomBytes(32).toString('base64url') const internal = errorBody(new Error(`failed for ${secret}`), 'correlation-id') diff --git a/test/server.js b/test/server.js index e59ecc4..8434235 100644 --- a/test/server.js +++ b/test/server.js @@ -3,12 +3,11 @@ const { expect } = require('chai') const { randomBytes } = require('node:crypto') const http = require('node:http') -const https = require('node:https') const fs = require('node:fs') const { join } = require('node:path') const { describe, it } = require('mocha') const { createTokenVerifier } = require('../app/ops/config') -const { createApp, createServers } = require('../app/server') +const { createApp } = require('../app/server') const { createRouter } = require('../app/router') describe('public server', () => { @@ -30,70 +29,22 @@ describe('public server', () => { expect(handlers).not.to.include("require('./server.js')") }) - it('creates only the public listener when disabled or in explicit HTTP loopback mode', () => { - expect(createServers({ env: {}, app: () => {} }).opsServer).to.equal(null) + it('serves an external HTTPS console origin on the single plain HTTP listener', async () => { const token = randomBytes(32).toString('base64url') - const env = { - OPS_CONSOLE_ENABLED: 'true', - OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), - OPS_CONSOLE_ORIGIN: 'http://127.0.0.1:8080', - OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' - } - expect(createServers({ env, app: () => {} }).opsServer).to.equal(null) - }) - - it('fails before listening for unreadable or invalid mTLS material', () => { - const env = mtlsEnv() - const handler = (request, response) => response.end('OK') - const invalid = join(__dirname, '../app/server.js') - const missing = join(__dirname, 'fixtures/mtls/missing.pem') - - expect(() => createServers({ env: { ...env, OPS_CONSOLE_MTLS_CA_PATH: missing }, app: handler })).to.throw() - expect(() => createServers({ - env, - app: handler, - readFile: path => { - if (path === env.OPS_CONSOLE_MTLS_CA_PATH) throw Object.assign(new Error('unreadable test CA'), { code: 'EACCES' }) - return fs.readFileSync(path) - } - })).to.throw('unreadable test CA') - for (const name of ['OPS_CONSOLE_MTLS_KEY_PATH', 'OPS_CONSOLE_MTLS_CERT_PATH', 'OPS_CONSOLE_MTLS_CA_PATH']) { - expect(() => createServers({ env: { ...env, [name]: invalid }, app: handler })).to.throw() - } - expect(() => createServers({ - env: { ...env, OPS_CONSOLE_MTLS_KEY_PATH: fixture('client.key') }, - app: handler - })).to.throw() - }) - - it('serves public HTTP separately and admits console HTTPS only after trusted client mTLS', async () => { - const token = randomBytes(32).toString('base64url') - const env = mtlsEnv(token) + const env = enabledEnv(token, 'https://ops.example.test') const events = [] const app = createApp({ ops: { env, log: event => events.push(event) } }) - const { publicServer, opsServer } = createServers({ env, app }) - expect(opsServer.requestCert).to.equal(true) - expect(opsServer.rejectUnauthorized).to.equal(true) - await Promise.all([listen(publicServer), listen(opsServer)]) + const server = http.createServer(app) + await listen(server) try { - const publicHealth = await networkRequest(http, { port: publicServer.address().port, path: '/health' }) + const port = server.address().port + const publicHealth = await networkRequest({ port, path: '/health' }) expect(publicHealth).to.include({ status: 200, body: 'OK' }) - const publicConsole = await networkRequest(http, { port: publicServer.address().port, path: '/_ops/login' }) - expect(publicConsole.status).to.equal(400) - expect(JSON.parse(publicConsole.body).error.code).to.equal('INVALID_REQUEST_CONTEXT') - - const tls = { - port: opsServer.address().port, + const consoleResponse = await networkRequest({ + port, path: '/_ops/login', - ca: fs.readFileSync(fixture('ca.crt')), - cert: fs.readFileSync(fixture('client.crt')), - key: fs.readFileSync(fixture('client.key')), - servername: 'localhost' - } - const consoleResponse = await networkRequest(https, { - ...tls, headers: { Forwarded: 'for=203.0.113.9;proto=http', 'X-Forwarded-For': '198.51.100.8', @@ -103,8 +54,8 @@ describe('public server', () => { expect(consoleResponse.status).to.equal(200) expect(consoleResponse.body).to.include('') - const login = await networkRequest(https, { - ...tls, + const login = await networkRequest({ + port, method: 'POST', path: '/_ops/api/v1/session', headers: { @@ -118,8 +69,8 @@ describe('public server', () => { expect(login.status).to.equal(201) expect(login.headers['set-cookie'][0]).to.match(/^__Host-hc-ops=.*; Secure;/) const session = JSON.parse(login.body) - const rejectedCache = await networkRequest(https, { - ...tls, + const rejectedCache = await networkRequest({ + port, method: 'POST', path: '/_ops/api/v1/cache-operations', headers: { @@ -135,37 +86,39 @@ describe('public server', () => { }) expect(rejectedCache.status).to.equal(400) expect(events.find(event => event.action === 'cache_operation')).to.include({ source: null, dispatchStatus: 'not_dispatched' }) + } finally { + await close(server) + } + }) - await expectNetworkFailure(networkRequest(https, { - port: opsServer.address().port, - path: '/_ops/login', - ca: tls.ca, - servername: 'localhost' - })) - await expectNetworkFailure(networkRequest(https, { - ...tls, - cert: fs.readFileSync(fixture('untrusted-client.crt')), - key: fs.readFileSync(fixture('untrusted-client.key')) - })) - await expectNetworkFailure(networkRequest(http, { port: opsServer.address().port, path: '/_ops/login' })) + it('retains explicit HTTP loopback mode without a Secure cookie', async () => { + const token = randomBytes(32).toString('base64url') + const env = enabledEnv(token, 'http://127.0.0.1:8080', { OPS_CONSOLE_ALLOW_HTTP_LOOPBACK: 'true' }) + const server = http.createServer(createApp({ ops: { env, log: () => {} } })) + await listen(server) + try { + const login = await networkRequest({ + port: server.address().port, + method: 'POST', + path: '/_ops/api/v1/session', + headers: { Origin: env.OPS_CONSOLE_ORIGIN, 'Content-Type': 'application/json' }, + body: JSON.stringify({ token }) + }) + expect(login.status).to.equal(201) + expect(login.headers['set-cookie'][0]).to.match(/^ghhc-console-dev=/) + expect(login.headers['set-cookie'][0]).not.to.include('; Secure') } finally { - await Promise.all([close(publicServer), close(opsServer)]) + await close(server) } }) }) -function fixture (name) { - return join(__dirname, 'fixtures/mtls', name) -} - -function mtlsEnv (token = randomBytes(32).toString('base64url')) { +function enabledEnv (token, origin, overrides = {}) { return { OPS_CONSOLE_ENABLED: 'true', OPS_CONSOLE_TOKEN_VERIFIER: createTokenVerifier(token), - OPS_CONSOLE_ORIGIN: 'https://localhost:8443', - OPS_CONSOLE_MTLS_KEY_PATH: fixture('server.key'), - OPS_CONSOLE_MTLS_CERT_PATH: fixture('server.crt'), - OPS_CONSOLE_MTLS_CA_PATH: fixture('ca.crt') + OPS_CONSOLE_ORIGIN: origin, + ...overrides } } @@ -180,16 +133,10 @@ function close (server) { return new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) } -async function expectNetworkFailure (promise) { - let error - try { await promise } catch (failure) { error = failure } - expect(error).to.be.instanceOf(Error) -} - -function networkRequest (protocol, options) { +function networkRequest (options) { return new Promise((resolve, reject) => { const { body, ...requestOptions } = options - const outgoing = protocol.request({ host: '127.0.0.1', method: 'GET', ...requestOptions }, response => { + const outgoing = http.request({ host: '127.0.0.1', method: 'GET', ...requestOptions }, response => { const chunks = [] response.on('data', chunk => chunks.push(chunk)) response.on('end', () => resolve({ status: response.statusCode, headers: response.headers, body: Buffer.concat(chunks).toString() })) From e6988ff4d7361a7dff50227daf9777c30bafe3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B8ran=20A=2E=20Slettemark?= Date: Wed, 22 Jul 2026 12:59:17 +0200 Subject: [PATCH 4/5] Fixed review finding --- app/service-client.js | 27 ++++++++++++++++++--------- test/builder-service.js | 29 ++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/app/service-client.js b/app/service-client.js index 70dcffe..384467d 100644 --- a/app/service-client.js +++ b/app/service-client.js @@ -105,16 +105,25 @@ function createServiceClient (options = {}) { stream: async (path, options = {}) => { if (!options.timeoutThroughBody) return (await request(path, options)).body const { response, release, timedOut } = await request(path, options, true) - const source = typeof response.body.getReader === 'function' ? Readable.fromWeb(response.body) : response.body - const output = Readable.from((async function * () { - try { - yield * source - } catch (error) { - throw new ServiceError(timedOut() ? 'SERVICE_TIMEOUT' : 'SERVICE_UNAVAILABLE', timedOut() ? 'Service request timed out' : error.message, timedOut() ? 504 : 502) + try { + const source = typeof response.body.getReader === 'function' ? Readable.fromWeb(response.body) : response.body + const output = Readable.from((async function * () { + try { + yield * source + } catch (error) { + throw new ServiceError(timedOut() ? 'SERVICE_TIMEOUT' : 'SERVICE_UNAVAILABLE', timedOut() ? 'Service request timed out' : error.message, timedOut() ? 504 : 502) + } + })()) + const safeRelease = () => { + output.off('end', safeRelease).off('error', safeRelease).off('close', safeRelease) + release() } - })()) - output.once('close', release) - return output + output.once('end', safeRelease).once('error', safeRelease).once('close', safeRelease) + return output + } catch (error) { + release() + throw error + } } } } diff --git a/test/builder-service.js b/test/builder-service.js index dfc2257..703d1c0 100644 --- a/test/builder-service.js +++ b/test/builder-service.js @@ -5,7 +5,7 @@ const fs = require('node:fs') const http = require('node:http') const os = require('node:os') const { execFileSync } = require('node:child_process') -const { EventEmitter } = require('node:events') +const { EventEmitter, once } = require('node:events') const { join } = require('node:path') const { PassThrough, Readable } = require('node:stream') const { gzipSync } = require('node:zlib') @@ -257,6 +257,33 @@ describe('builder service', () => { expect(error).to.include({ code: 'SERVICE_TIMEOUT', status: 504 }) }) + it('releases retained stream cleanup at end only once', async () => { + const controller = new AbortController() + const removeEventListener = sinon.spy(controller.signal, 'removeEventListener') + const fetch = async () => ({ ok: true, body: Readable.from(['archive']) }) + const client = createServiceClient({ baseURL: 'http://downloader', token: 'secret', timeout: 1000, fetch }) + const stream = await client.stream('/v1/source', { signal: controller.signal, timeoutThroughBody: true }) + const closed = once(stream, 'close') + stream.once('end', () => expect(removeEventListener.callCount).to.equal(1)) + + stream.resume() + await closed + expect(removeEventListener.callCount).to.equal(1) + }) + + it('releases retained stream cleanup when stream setup rejects', async () => { + const controller = new AbortController() + const removeEventListener = sinon.spy(controller.signal, 'removeEventListener') + const fetch = async () => ({ ok: true, body: null }) + const client = createServiceClient({ baseURL: 'http://downloader', token: 'secret', timeout: 1000, fetch }) + let error + + await client.stream('/v1/source', { signal: controller.signal, timeoutThroughBody: true }).catch(caught => { error = caught }) + + expect(error).to.be.instanceOf(TypeError) + expect(removeEventListener.callCount).to.equal(1) + }) + it('rejects unsafe extracted nodes and sizes before promotion', async () => { const cases = [ async root => fs.promises.symlink('/tmp', join(root, 'ts/link')), From 2add26fd2165aeb14cf22fb2d319a0f0f353ffda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B8ran=20A=2E=20Slettemark?= Date: Wed, 22 Jul 2026 13:11:04 +0200 Subject: [PATCH 5/5] Mocked response for failing tests --- test/download.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/download.js b/test/download.js index dee46f5..332b3be 100644 --- a/test/download.js +++ b/test/download.js @@ -25,13 +25,13 @@ function loadDownloadWithHttpsGet (httpsGet) { } } -function fakeHttpsGet (chunks) { +function fakeHttpsGet (chunks, statusCode = 200, headers = {}) { return (options, callback) => { const request = new EventEmitter() request.end = () => { const response = new EventEmitter() - response.statusCode = 200 - response.headers = {} + response.statusCode = statusCode + response.headers = headers callback(response) process.nextTick(() => { chunks.forEach(chunk => response.emit('data', chunk)) @@ -85,6 +85,11 @@ describe('download.js', () => { }) }) it('should return a response when a url is provided', () => { + const body = '404: Not Found' + const { httpsGetPromise } = loadDownloadWithHttpsGet(fakeHttpsGet([ + Buffer.from(body) + ], 404)) + return httpsGetPromise(downloadURL) .then(({ body, statusCode }) => { expect(statusCode).to.equal(404) @@ -128,9 +133,15 @@ describe('download.js', () => { } after(cleanFiles) before(() => { + cleanFiles() fs.mkdirSync('tmp/test', { recursive: true }) }) it('should resolve with an informational object, and a newly created file.', () => { + const body = 'downloaded file content' + const { downloadFile } = loadDownloadWithHttpsGet(fakeHttpsGet([ + Buffer.from(body) + ])) + return downloadFile( downloadURL + 'master/ts/masters/highcharts.src.ts', './tmp/test/downloaded-file1.js' @@ -140,9 +151,14 @@ describe('download.js', () => { expect(success).to.equal(true) expect(url).to.equal(downloadURL + 'master/ts/masters/highcharts.src.ts') expect(fs.lstatSync('./tmp/test/downloaded-file1.js').size).to.be.greaterThan(0) + expect(fs.readFileSync('./tmp/test/downloaded-file1.js', 'utf8')).to.equal(body) }) }) it('should only create a file if response status is 200', () => { + const { downloadFile } = loadDownloadWithHttpsGet(fakeHttpsGet([ + Buffer.from('404: Not Found') + ], 404)) + return downloadFile( downloadURL + 'master/i-do-not-exist.js', './tmp/test/downloaded-file2.js'