feat(routing): add LiteLLM backend to run open-weight models on the Claude Code harness

evalboard/app/runs/[id]/[...task]/_sections.tsx

@@ -1098,7 +1098,9 @@ function ReconciliationRow({ m, unit }: { m: MessageEvent; unit: Unit }) {
                 <span className="tabular-nums text-right text-amber-700">
                     {fmtTok(m.outputTokens, "output")}
                 </span>
-                <span className="text-right text-gray-400">—</span>
+                <span className="tabular-nums text-right text-amber-700">
+                    {m.costUsd != null ? fmtUsd(m.costUsd) : "—"}
+                </span>
             </div>
         </li>
     );

evalboard/lib/__tests__/parseMessages.test.ts

@@ -769,4 +769,65 @@ describe("parseMessages — reconciliation entry", () => {
         expect(recon.parentToolUseId).toBeNull();
         expect(recon.toolUses).toEqual([]);
     });
+
+    test("prices the reconciliation row from the turn model so the Cost column sums to the turn total", () => {
+        // Mirrors the OpenRouter/LiteLLM sparse-stream case that motivated this:
+        // the per-message stream carries only output, and the bulk of the input +
+        // cache-read tokens land in the single reconciliation row. Before the fix
+        // that row read "—" for cost, so the Cost column summed to far less than
+        // the real total.
+        const turns: TurnEntry[] = [
+            {
+                model_used: "deepseek/deepseek-v4-pro",
+                messages: [
+                    {
+                        role: "assistant",
+                        model: "deepseek/deepseek-v4-pro",
+                        started_at: "2026-01-01T00:00:00.000Z",
+                        completed_at: "2026-01-01T00:00:01.000Z",
+                        generation_duration_ms: 1000,
+                        content_blocks: [{ block_type: "text", text: "hi" }],
+                        input_tokens: 0,
+                        output_tokens: 1000,
+                        cache_read_tokens: 0,
+                    },
+                    {
+                        role: "reconciliation",
+                        input_tokens: 100000,
+                        output_tokens: 0,
+                        cache_creation_tokens: 0,
+                        cache_read_tokens: 300000,
+                        note: "billed but not streamed",
+                    },
+                ],
+            },
+        ];
+        const events = parseMessages(turns);
+        const [asst, recon] = events;
+        // deepseek-v4-pro: in 0.435 / out 0.87 / cacheRead 0.003625 per MTok.
+        // reconciliation = 100000*0.435 + 300000*0.003625 = 44587.5 tok-$ / 1e6.
+        expect(recon.model).toBe("deepseek/deepseek-v4-pro");
+        expect(recon.costUsd).toBeCloseTo(0.0445875, 9);
+        // The whole point: assistant + reconciliation sum to the turn total
+        // (1000*0.87/1e6 + 0.0445875), instead of the reconciliation row reading "—".
+        expect((asst.costUsd ?? 0) + (recon.costUsd ?? 0)).toBeCloseTo(0.0454575, 9);
+    });
+
+    test("reconciliation cost stays null when the turn model is absent (legacy runs)", () => {
+        const turns: TurnEntry[] = [
+            {
+                messages: [
+                    {
+                        role: "reconciliation",
+                        input_tokens: 512,
+                        cache_creation_tokens: 1000,
+                        note: "no model",
+                    },
+                ],
+            },
+        ];
+        const recon = parseMessages(turns)[0];
+        expect(recon.model).toBeNull();
+        expect(recon.costUsd).toBeNull();
+    });
 });

evalboard/lib/__tests__/pricing-parity.test.ts

@@ -5,12 +5,17 @@ import { describe, expect, test } from "vitest";
 import { PRICING } from "../pricing";
 
 // Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative
-// Python table in src/coder_eval/pricing.py. If the backend reprices a
-// model (or adds one) and this mirror isn't updated, the frontend's "estimated"
-// USD figures silently disagree with the backend's authoritative Cost on the
-// same tokens. This test parses the Python table and asserts both tables have
-// the same model ids and the same four per-MTok rates — turning silent drift
-// into a failing build.
+// Python table in src/coder_eval/pricing.py. It exists so the frontend's
+// "estimated" USD figures agree with the backend's authoritative Cost on the
+// same tokens.
+//
+// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts
+// must exist in pricing.py with identical rates (a frontend rate that disagrees
+// with the backend, or prices a model the backend doesn't, fails the build).
+// The frontend is NOT required to mirror every backend model — it only needs to
+// price the ones it displays, and the backend legitimately prices models the
+// evalboard never renders. (Exact-match was too strict: it forced unrelated
+// backend model additions into this file to keep the build green.)
 
 const here = dirname(fileURLToPath(import.meta.url));
 const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py");
@@ -44,20 +49,24 @@ describe("pricing.ts ↔ pricing.py parity", () => {
         expect(Object.keys(py).length).toBeGreaterThan(10);
     });
 
-    test("both tables list the same model ids", () => {
-        expect(Object.keys(PRICING).sort()).toEqual(Object.keys(py).sort());
+    test("every model in lib/pricing.ts exists in pricing.py", () => {
+        const orphans = Object.keys(PRICING).filter((m) => !(m in py));
+        expect(
+            orphans,
+            `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`,
+        ).toEqual([]);
     });
 
-    test("every model has identical input/output/cacheWrite/cacheRead rates", () => {
-        for (const [model, [input, output, cw, cr]] of Object.entries(py)) {
-            const ts = PRICING[model];
-            expect(ts, `missing in lib/pricing.ts: ${model}`).toBeDefined();
+    test("shared models have identical input/output/cacheWrite/cacheRead rates", () => {
+        for (const [model, ts] of Object.entries(PRICING)) {
+            const rates = py[model];
+            expect(rates, `not priced in pricing.py: ${model}`).toBeDefined();
             expect([
                 ts.inputPerMTok,
                 ts.outputPerMTok,
                 ts.cacheWritePerMTok,
                 ts.cacheReadPerMTok,
-            ]).toEqual([input, output, cw, cr]);
+            ]).toEqual(rates);
         }
     });
 });

evalboard/lib/pricing.ts

@@ -50,6 +50,14 @@ export const PRICING: Record<string, Pricing> = {
     "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2),
     "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15),
     "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15),
+    // OpenRouter open-weight models (litellm backend). Mirror of pricing.py;
+    // these providers cache implicitly (cache_write == input, unused) so only
+    // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request,
+    // so these rates are only accurate when the litellm config pins the provider
+    // (sort: price) — otherwise the billed rate can differ from the headline.
+    "moonshotai/kimi-k3": p(3, 15, 3, 0.3),
+    "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534),
+    "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625),
 };
 
 function p(

evalboard/lib/runs.ts

@@ -1198,6 +1198,9 @@ interface MessageEntry {
 export interface TurnEntry {
     commands?: CommandEntry[];
     messages?: MessageEntry[];
+    // Model the turn ran on (iteration `model_used`). Used to price the synthetic
+    // reconciliation row, which carries no model of its own.
+    model_used?: string | null;
     token_usage?: TokenUsageEntry | null;
     result_summary?: {
         result?: string | null;
@@ -1716,8 +1719,23 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
                 reasoningTokens: null,
                 thinkingOutputTokens: null,
                 textOutputTokens: null,
-                model: null,
-                costUsd: null,
+                // Price the reconciliation row so summing the Cost column across
+                // the stream reproduces the turn total. Critical for providers
+                // (e.g. OpenRouter/LiteLLM) whose per-message stream is sparse and
+                // dumps most tokens into this one row — leaving it uncosted made
+                // the visible per-row costs sum to far less than the real total.
+                // Excluded from the cost simulator (by role), and the authoritative
+                // task total_cost_usd is unaffected (it reads the backend aggregate,
+                // not a sum of per-message costUsd), so there is no double-count.
+                model: turn.model_used ?? null,
+                costUsd: messageCostUsd({
+                    model: turn.model_used ?? null,
+                    inputTokens: typeof msg.input_tokens === "number" ? msg.input_tokens : null,
+                    outputTokens: typeof msg.output_tokens === "number" ? msg.output_tokens : null,
+                    cacheWriteTokens:
+                        typeof msg.cache_creation_tokens === "number" ? msg.cache_creation_tokens : null,
+                    cacheReadTokens: typeof msg.cache_read_tokens === "number" ? msg.cache_read_tokens : null,
+                }),
                 note: typeof msg.note === "string" ? msg.note : null,
             });
         }

litellm/README.md

@@ -0,0 +1,193 @@
+# LiteLLM proxy — coder_eval `litellm` (open-weight) backend
+
+coder_eval evaluates UiPath skills on **non-Claude open-weight models** by driving
+them through the **Claude Code SDK** harness (so the native `Skill` tool +
+progressive disclosure fire unchanged). The SDK only speaks the **Anthropic
+Messages** format (`POST /v1/messages`), but the target models don't:
+
+| Target | Wire format | Needs translation? |
+|---|---|---|
+| Bedrock open-weight (GLM, DeepSeek, Kimi) | Converse | yes |
+| OpenRouter (any model) | OpenAI Chat Completions | yes |
+
+**LiteLLM is the translation shim.** The SDK is pointed at LiteLLM via
+`ANTHROPIC_BASE_URL`; LiteLLM translates Anthropic ↔ Converse / OpenAI per model
+and forwards to the real backend.
+
+```
+Claude Code SDK ──Anthropic /v1/messages──▶ LiteLLM (localhost:4000) ──▶ Bedrock Converse (eu-north-1)
+  ANTHROPIC_BASE_URL=localhost:4000                                   └──▶ OpenRouter (OpenAI format)
+```
+
+Files:
+- `litellm/litellm-config.yaml` — the model list + settings (this is the file you edit to add models).
+- `litellm/start-litellm.sh` — launches the proxy, reading credentials out of `.env`.
+
+---
+
+## When to run `start-litellm.sh`
+
+Run the proxy **whenever you run coder_eval against the `litellm` backend** — i.e.
+`API_BACKEND=litellm` in `.env`, or `coder-eval run ... --backend litellm`. coder_eval
+does **not** own the proxy lifecycle: it expects an already-running proxy at
+`LITELLM_BASE_URL` and **fails fast at startup** if it isn't reachable
+(`LiteLLM proxy not reachable at ... — Start it (e.g. litellm/start-litellm.sh)`).
+
+You do **not** need it for the `direct` (Anthropic) or `bedrock` (Claude-on-Bedrock)
+backends — those talk to their APIs directly.
+
+### Prerequisites
+
+`start-litellm.sh` reads these from `.env` and exports them into the proxy's own
+environment before launching:
+
+| Var | For | Notes |
+|---|---|---|
+| `AWS_BEARER_TOKEN_BEDROCK` | Bedrock models | required if you use any `bedrock/*` model |
+| `AWS_REGION` | Bedrock models | defaults to `eu-north-1` |
+| `OPENROUTER_API_KEY` | OpenRouter models | required if you use any `openrouter/*` model |
+| `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local`. It can be customly designed.|
+
+### Start it
+
+```bash
+bash litellm/start-litellm.sh          # foreground on :4000, Ctrl-C to stop
+```
+
+Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`,
+`LITELLM_MASTER_KEY`.
+
+### Point coder_eval at it
+
+In `.env`:
+```
+API_BACKEND=litellm
+LITELLM_BASE_URL=http://localhost:4000
+LITELLM_AUTH_TOKEN=<same value start-litellm.sh printed as "master key">
+LITELLM_MODEL=deepseek/deepseek-v4-pro   # default model when a task doesn't pin one
+```
+Then run — `--model` overrides the default:
+```bash
+coder-eval run tasks/<task>.yaml --backend litellm --model deepseek/deepseek-v4-pro
+```
+
+### Verify the proxy
+
+```bash
+KEY=$(grep -E '^LITELLM_AUTH_TOKEN=' .env | head -1 | sed -E 's/^LITELLM_AUTH_TOKEN=//; s/^"//; s/"$//')
+# health
+curl -s http://localhost:4000/health/liveliness
+# which models are loaded
+curl -s http://localhost:4000/v1/models -H "x-api-key: $KEY" \
+  | python3 -c "import sys,json; print([m['id'] for m in json.load(sys.stdin)['data']])"
+```
+
+---
+
+## Config file: `litellm-config.yaml`
+
+```yaml
+model_list:
+  - model_name: <what clients pass as --model>   # also the pricing-table key
+    litellm_params:
+      model: <litellm provider-routed target>    # e.g. bedrock/converse/... or openrouter/...
+      # ...provider-specific params
+
+litellm_settings:
+  drop_params: true          # silently drop provider-unsupported params instead of 400
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY   # never hardcode a secret here
+```
+
+- **`model_name`** is the id clients (and `--model`) use, **and** the key coder_eval
+  prices on. Keep it identical to the `pricing.py` key.
+- **`model`** is LiteLLM's routing target — the `provider/...` prefix selects the
+  translator + backend.
+
+---
+
+## Adding a new model
+
+### 1. Add an entry to `model_list`
+
+**Bedrock open-weight** (Converse, eu-north-1):
+```yaml
+  - model_name: some.bedrock-model
+    litellm_params:
+      model: bedrock/converse/some.bedrock-model
+      aws_region_name: eu-north-1
+```
+
+**OpenRouter** (OpenAI format — pin the provider, see caveat below):
+```yaml
+  - model_name: vendor/some-model
+    litellm_params:
+      model: openrouter/vendor/some-model
+      api_key: os.environ/OPENROUTER_API_KEY
+      extra_body:
+        provider:
+          sort: price            # route to the cheapest upstream
+          allow_fallbacks: false # never silently upgrade to a pricier provider
+```
+Find the exact OpenRouter slug + rates:
+```bash
+KEY=$(grep -E '^OPENROUTER_API_KEY=' .env | head -1 | sed -E 's/^OPENROUTER_API_KEY=//; s/^"//; s/"$//')
+curl -s https://openrouter.ai/api/v1/models -H "Authorization: Bearer $KEY" \
+  | python3 -c "import sys,json;[print(m['id'],m['pricing']) for m in json.load(sys.stdin)['data'] if 'SEARCH' in m['id'].lower()]"
+```
+
+### 2. Register pricing in **both** tables (rates must match)
+
+- `src/coder_eval/pricing.py` — add to the `_PRICING` dict, keyed on the
+  `model_name` (bare id as passed in `agent.model`):
+  ```python
+  "vendor/some-model": ModelPricing(input, output, cache_write, cache_read),
+  ```
+  These implicit-caching providers charge no separate cache-write fee, so
+  `cache_write == input` (unused) and `cache_read` is the discounted rate.
+- `evalboard/lib/pricing.ts` — add the same entry so the evalboard cost columns
+  populate. A test (`lib/__tests__/pricing-parity.test.ts`) fails the build if a
+  rate here disagrees with `pricing.py`.
+
+### 3. Restart the proxy
+
+LiteLLM reads `model_list` **once at startup**. After editing the yaml, `Ctrl-C`
+and re-run `start-litellm.sh` — otherwise you get
+`Invalid model name passed in model=...`.
+
+### 4. Verify
+
+`curl .../v1/models` should list the new `model_name`, then run a task with
+`--model vendor/some-model`.
+
+---
+
+## ⚠️ OpenRouter cost accuracy (provider routing)
+
+OpenRouter is a **multi-provider router**: the rate on a model's page is the
+*cheapest* provider's headline, but a request can be routed to a pricier upstream
+(observed: DeepSeek V4 Pro → Novita at ~$1.47/M input vs the $0.435/M headline,
+3.4×). Because coder_eval prices `litellm`-backend runs from the **static**
+`pricing.py` table, an un-pinned model can bill very differently from what
+coder_eval reports.
+
+**Mitigation:** always pin providers (`extra_body.provider: {sort: price,
+allow_fallbacks: false}`), which makes the billed rate deterministic and equal to
+the headline the pricing table uses. To confirm the actual provider/cost for a
+run, read the OpenRouter **Activity** page (`provider_name`, `usage`) or query
+`GET /api/v1/models/<id>/endpoints` for per-provider rates.
+
+Token *counts* are exact and provider-independent; only the per-token *rate* is
+subject to this. Bedrock models are single-provider and not affected.
+
+---
+
+## Troubleshooting
+
+| Symptom | Cause / fix |
+|---|---|
+| `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. |
+| `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. |
+| HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. |
+| evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. |