Added ops console

.env.example

@@ -1,6 +1,6 @@
 # Copy this file to .env, replace the secret placeholders, then run:
 #   docker compose up --build --wait
-# The router is available at http://localhost:${ROUTER_PORT:-8080}.
+# The router is available at http://localhost:${ROUTER_PORT:-8080} by default.
 # Downloader and builder have no host ports and are reachable only by service name.
 
 # Shared by all services. Generate a long random value; do not commit it.
@@ -10,11 +10,24 @@ WEBHOOK_SECRET=
 # Downloader only. Optional for local use, but avoids GitHub API rate limits.
 GITHUB_TOKEN=
 
-# Host-facing router port and internal service URLs.
+# Host-facing router bind/port and internal service URLs.
+# Keep ROUTER_BIND_ADDRESS on loopback for local development; set explicitly if needed.
+ROUTER_BIND_ADDRESS=127.0.0.1
 ROUTER_PORT=8080
 DOWNLOADER_URL=http://downloader:8080
 BUILDER_URL=http://builder:8080
 
+# Operations console: disabled by default and intended for development only here.
+# Never store the raw operator token in Compose/environment. Store only the
+# canonical verifier form: v1.<base64url-sha256>.
+OPS_CONSOLE_ENABLED=false
+OPS_CONSOLE_TOKEN_VERIFIER=
+# Local HTTP requires OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true and an exact loopback origin.
+OPS_CONSOLE_ORIGIN=
+OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=false
+# OPS_CONSOLE_MTLS_PORT, OPS_CONSOLE_MTLS_KEY_PATH, OPS_CONSOLE_MTLS_CERT_PATH,
+# and OPS_CONSOLE_MTLS_CA_PATH are retired and unsupported; do not set them.
+
 # Ephemeral cache roots are fixed by compose.yaml and owned by UID/GID 1000.
 # They are separate tmpfs mounts and are discarded when containers stop.
 DOWNLOADER_CACHE_ROOT=/app/downloader-cache

.github/workflows/test.yml

@@ -6,22 +6,26 @@ on:
       - master
       - develop
 
+permissions:
+  contents: read
+
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '24'
-          cache: 'npm'
 
       - name: Install dependencies
-        run: npm install
+        run: npm ci
 
       - name: Check TypeScript compilation is up to date
         run: |
@@ -38,3 +42,12 @@ jobs:
 
       - name: Run tests
         run: npm test
+
+      - name: Run operations console integration tests
+        run: npm run test:integration:ops
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium firefox
+
+      - name: Run operations console browser tests
+        run: npm run test:browser:ops

README.md

@@ -5,8 +5,131 @@ Node.js server that serves Highcharts distribution files built on demand from an
 
 ## Operations console
 
-A **development-only** secured operations console is specified for the router.
-See [docs/operations-console.md](docs/operations-console.md) for the full specification.
+> **Development-only.** The operations console must never be enabled in staging or production.
+
+A secured operations console is built into the router and is disabled by default. When enabled, it lets trusted Highsoft engineers inspect system health, browse recent request activity, trace individual requests, and perform cache-maintenance operations — all from a same-origin browser UI served at `/_ops/`.
+
+The full normative specification (1 479 lines) is at [docs/operations-console.md](docs/operations-console.md). This section covers everything needed to configure, start, and operate it locally.
+
+### Topology
+
+The console does not change the three-process topology. Only the router exposes a port; the downloader and builder remain on the private Docker network. The browser never contacts internal services directly and never receives the internal bearer token. Console telemetry and cache state are available only under bearer-protected `/v1/ops/*` paths on each internal service.
+
+### Environment variables
+
+| Variable | Required when | Description |
+|---|---|---|
+| `OPS_CONSOLE_ENABLED` | Always evaluated | Must be exactly the string `true` to enable the console. Any other value or absence keeps the console off and every `/_ops/*` path returns 404. |
+| `OPS_CONSOLE_TOKEN_VERIFIER` | Console enabled | Canonical domain-separated SHA-256 verifier derived from the shared admin token (see below). Never store the raw token. Validated at startup; missing or malformed values abort startup. |
+| `OPS_CONSOLE_ORIGIN` | Console enabled | The single exact external Origin the console accepts for all state-changing requests and for login (e.g. `http://localhost:8080`). Missing or ambiguous values abort startup. |
+| `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK` | Optional | Set to exactly `true` to allow HTTP when the Origin is a loopback address. All other HTTP combinations abort startup. |
+| `ROUTER_BIND_ADDRESS` | Optional | Host address the router container binds to. Defaults to `127.0.0.1` in Docker Compose; the console is not reachable from outside loopback unless this is changed deliberately. |
+
+`OPS_CONSOLE_TRUSTED_PROXY` is not supported. Setting it to any non-blank value aborts startup.
+
+`OPS_CONSOLE_MTLS_PORT`, `OPS_CONSOLE_MTLS_KEY_PATH`, `OPS_CONSOLE_MTLS_CERT_PATH`, and `OPS_CONSOLE_MTLS_CA_PATH` are retired and unsupported; setting any to a non-blank value aborts startup.
+
+**Verifier, not token.** `OPS_CONSOLE_TOKEN_VERIFIER` stores a one-way verifier, never the raw token. Derive it from a canonical 32-byte CSPRNG token (base64url-encoded) with:
+
+```bash
+node -e "
+const { createHash } = require('crypto');
+const sep = 'github.highcharts.com:ops-console:v1\x00';
+const raw = Buffer.from('<your-base64url-token>', 'base64url');
+const digest = createHash('sha256').update(sep).update(raw).digest('base64url');
+console.log('v1.' + digest);
+"
+```
+
+Set the printed `v1.<base64url-sha256>` string as `OPS_CONSOLE_TOKEN_VERIFIER` in your `.env`. Keep the raw token only in your password manager or 1Password — never in `.env`, source control, logs, or browser storage.
+
+### Login and session behaviour
+
+Navigate to `/_ops/login` and submit the raw base64url token. On success the router creates an opaque server-side session (30-minute idle expiry, 8-hour absolute maximum) and sets a `HttpOnly; SameSite=Strict` cookie. Sessions are router-local and in-memory; restarting the router revokes all active sessions. The UI warns five minutes before expiry and returns to the login page on expiry or logout.
+
+### Snapshot and cache operations
+
+`GET /_ops/api/v1/snapshot` returns a concurrent aggregate of router, downloader, and builder state: health, queue depth, cache summary, and recent request activity. Each internal call is bounded to 1.5 seconds.
+
+`POST /_ops/api/v1/cache-operations` drives cache maintenance: evict a single commit, purge expired entries, or clear a full service cache. Each operation is bounded to 10 seconds. All mutations are recorded as a single sanitized audit event in the router log.
+
+### Local Compose setup
+
+```bash
+# 1. Copy the example env file (only needed once)
+cp .env.example .env
+
+# 2. Generate and store the raw token in your password manager, then derive the verifier:
+node -e "
+const { createHash, randomBytes } = require('crypto');
+const sep = 'github.highcharts.com:ops-console:v1\x00';
+const raw = randomBytes(32);
+console.log('Raw token (save to password manager):', raw.toString('base64url'));
+const digest = createHash('sha256').update(sep).update(raw).digest('base64url');
+console.log('Verifier (put in .env):', 'v1.' + digest);
+"
+
+# 3. Edit .env — set the required console variables:
+#   OPS_CONSOLE_ENABLED=true
+#   OPS_CONSOLE_TOKEN_VERIFIER=v1.<digest from above>
+#   OPS_CONSOLE_ORIGIN=http://localhost:8080
+#   OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true
+
+# 4. Start the stack
+docker compose up --build --wait
+
+# 5. Open http://localhost:8080/_ops/login and log in with the raw token
+
+# 6. To disable the console again, set OPS_CONSOLE_ENABLED=false (or remove it) and restart:
+docker compose up --build --wait
+```
+
+### Running the console tests
+
+```bash
+# Full unit and lint suite (includes ops-console and ops-config tests)
+npm test
+
+# Integration smoke tests against a running stack (requires docker compose up)
+npm run test:integration:ops
+
+# Browser/accessibility tests (requires a running stack and Playwright browsers installed)
+npm run test:browser:ops
+```
+
+### Operational checklist
+
+This checklist covers staged local rollout, abort criteria, and rollback.
+
+**Enable:**
+
+1. Derive the verifier and set `OPS_CONSOLE_ENABLED=true` plus the three required variables in `.env`.
+2. For local HTTP loopback mode, set `OPS_CONSOLE_ALLOW_HTTP_LOOPBACK=true`. If `OPS_CONSOLE_ORIGIN` uses an `https:` scheme, TLS termination is handled at the proxy layer; the Node backend remains ordinary HTTP.
+3. Restart the router (`docker compose up --build --wait`). Confirm startup log shows `ops-console enabled`.
+4. Open `/_ops/login`; confirm login succeeds and the snapshot page loads with data from all three services.
+5. Run `npm run test:integration:ops` — all checks must pass.
+
+**Abort criteria — disable immediately if any of the following occur:**
+
+- `/_ops/*` paths are reachable from outside the intended network boundary.
+- Login returns a session without a valid token being submitted.
+- Any audit event is absent from the router log after a cache mutation.
+- Internal bearer token appears in a browser response or cookie.
+- Router startup fails with a missing-verifier or invalid-origin error that cannot be resolved within 10 minutes.
+
+**Disable and roll back:**
+
+```bash
+# Remove or unset OPS_CONSOLE_ENABLED in .env (or set it to any value other than "true"),
+# then restart the router:
+docker compose up --build --wait
+
+# Confirm every /_ops/* path returns 404:
+curl -o /dev/null -w "%{http_code}" http://localhost:8080/_ops/
+# Expected: 404
+```
+
+If the router itself must be rolled back, re-tag the prior immutable image to the environment tag and restart the container. The router can be rolled back independently of the internal services provided internal API compatibility is maintained.
 
 ## Architecture
 

app/JobQueue.d.ts

@@ -1,10 +1,33 @@
-export type JobArgs = {
+type JobArgs = {
     func: (...args: any[]) => Promise<any>;
     args: any[];
 };
-export type JobType = JobArgs & {
-    setDone: (isDone: true) => void;
-    done: Promise<true>;
+type JobType = JobArgs & {
+    setDone: (result: any) => void;
+    setFailed: (error: unknown) => void;
+    done: Promise<any>;
+    queuedAt: number;
 };
-export type Queues = 'download' | 'compile';
-export type QueueType = Map<string, JobType>;
+type Queues = 'download' | 'compile';
+type QueueType = Map<string, JobType>;
+type QueueMetrics = {
+    active: number;
+    queued: number;
+    limit: number;
+    available: number;
+    oldestQueuedAgeMs: number | null;
+};
+declare class JobQueue {
+    static _instance: JobQueue;
+    private static queues;
+    private static churns;
+    doJob(queue: QueueType, jobID: string): Promise<void>;
+    churn(queue: QueueType): Promise<QueueType>;
+    private makeJob;
+    addJob(type: Queues, jobID: string, job: JobArgs): Promise<any>;
+    getJobs(type: Queues): [string, JobType][];
+    getJobPromises(type: Queues): JobType[];
+    getMetrics(type: Queues, now?: number): QueueMetrics;
+    constructor();
+}
+export type { JobArgs, JobType, Queues, QueueType, QueueMetrics, JobQueue };

app/JobQueue.js

@@ -1,10 +1,6 @@
 'use strict'
-const __importDefault = (this && this.__importDefault) || function (mod) {
-  return (mod && mod.__esModule) ? mod : { default: mod }
-}
 Object.defineProperty(exports, '__esModule', { value: true })
 /* eslint-disable camelcase */
-const node_crypto_1 = __importDefault(require('node:crypto'))
 const node_process_1 = require('node:process')
 // Max queue size per queue
 const MAX_QUEUE_SIZE = Number(node_process_1.env.MAX_QUEUE_SIZE) || 2
@@ -20,19 +16,18 @@ class JobQueue {
     compile: false
   }
 
-  id
   async doJob (queue, jobID) {
     const job = queue.get(jobID)
     if (job) {
       try {
-        await Promise.resolve().then(() => job.func(...job.args))
+        const result = await Promise.resolve().then(() => job.func(...job.args))
+        job.setDone(result)
       } catch (error) {
-        console.log(error)
+        job.setFailed(error)
+        throw error
       } finally {
         queue.delete(jobID)
-        job.setDone(true)
       }
-      await job.done
     }
   }
 
@@ -54,15 +49,16 @@ class JobQueue {
   }
 
   makeJob (job) {
-    const myJob = {
+    const { promise: done, resolve: setDone, reject: setFailed } = Promise.withResolvers()
+    // Prevent an ignored caller promise from becoming an unhandled rejection.
+    done.catch(() => { })
+    return {
       ...job,
-      setDone () { },
-      done: Promise.resolve(true)
+      setDone,
+      setFailed,
+      done,
+      queuedAt: Date.now()
     }
-    myJob.done = new Promise((resolve) => {
-      myJob.setDone = resolve
-    })
-    return myJob
   }
 
   addJob (type, jobID, job) {
@@ -73,13 +69,14 @@ class JobQueue {
       return Promise.reject(error)
     }
     if (queue.has(jobID)) {
-      return queue.get(jobID)?.done
+      return queue.get(jobID).done
     }
     const transformedJob = this.makeJob(job)
     queue.set(jobID, transformedJob)
     if (!JobQueue.churns[type]) {
       JobQueue.churns[type] = true
-      return this.churn(queue)
+      this.churn(queue)
+        .catch(console.error)
         .finally(() => {
           JobQueue.churns[type] = false
         })
@@ -97,8 +94,20 @@ class JobQueue {
     return Array.from(queue.values())
   }
 
+  getMetrics (type, now = Date.now()) {
+    const queue = JobQueue.queues[type]
+    const active = JobQueue.churns[type] && queue.size > 0 ? 1 : 0
+    const waiting = Array.from(queue.values()).slice(active)
+    return {
+      active,
+      queued: waiting.length,
+      limit: MAX_QUEUE_SIZE,
+      available: Math.max(0, MAX_QUEUE_SIZE - queue.size),
+      oldestQueuedAgeMs: waiting.length ? Math.max(0, now - waiting[0].queuedAt) : null
+    }
+  }
+
   constructor () {
-    this.id = node_crypto_1.default.randomUUID()
     if (JobQueue._instance) {
       return JobQueue._instance
     }