diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx
index 62a5bae0..13ef0fd9 100644
--- a/evalboard/app/runs/[id]/[...task]/_sections.tsx
+++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx
@@ -1098,7 +1098,9 @@ function ReconciliationRow({ m, unit }: { m: MessageEvent; unit: Unit }) {
{fmtTok(m.outputTokens, "output")}
- —
+
+ {m.costUsd != null ? fmtUsd(m.costUsd) : "—"}
+
);
diff --git a/evalboard/lib/__tests__/parseMessages.test.ts b/evalboard/lib/__tests__/parseMessages.test.ts
index 96c3e807..e3bd183a 100644
--- a/evalboard/lib/__tests__/parseMessages.test.ts
+++ b/evalboard/lib/__tests__/parseMessages.test.ts
@@ -769,4 +769,65 @@ describe("parseMessages — reconciliation entry", () => {
expect(recon.parentToolUseId).toBeNull();
expect(recon.toolUses).toEqual([]);
});
+
+ test("prices the reconciliation row from the turn model so the Cost column sums to the turn total", () => {
+ // Mirrors the OpenRouter/LiteLLM sparse-stream case that motivated this:
+ // the per-message stream carries only output, and the bulk of the input +
+ // cache-read tokens land in the single reconciliation row. Before the fix
+ // that row read "—" for cost, so the Cost column summed to far less than
+ // the real total.
+ const turns: TurnEntry[] = [
+ {
+ model_used: "deepseek/deepseek-v4-pro",
+ messages: [
+ {
+ role: "assistant",
+ model: "deepseek/deepseek-v4-pro",
+ started_at: "2026-01-01T00:00:00.000Z",
+ completed_at: "2026-01-01T00:00:01.000Z",
+ generation_duration_ms: 1000,
+ content_blocks: [{ block_type: "text", text: "hi" }],
+ input_tokens: 0,
+ output_tokens: 1000,
+ cache_read_tokens: 0,
+ },
+ {
+ role: "reconciliation",
+ input_tokens: 100000,
+ output_tokens: 0,
+ cache_creation_tokens: 0,
+ cache_read_tokens: 300000,
+ note: "billed but not streamed",
+ },
+ ],
+ },
+ ];
+ const events = parseMessages(turns);
+ const [asst, recon] = events;
+ // deepseek-v4-pro: in 0.435 / out 0.87 / cacheRead 0.003625 per MTok.
+ // reconciliation = 100000*0.435 + 300000*0.003625 = 44587.5 tok-$ / 1e6.
+ expect(recon.model).toBe("deepseek/deepseek-v4-pro");
+ expect(recon.costUsd).toBeCloseTo(0.0445875, 9);
+ // The whole point: assistant + reconciliation sum to the turn total
+ // (1000*0.87/1e6 + 0.0445875), instead of the reconciliation row reading "—".
+ expect((asst.costUsd ?? 0) + (recon.costUsd ?? 0)).toBeCloseTo(0.0454575, 9);
+ });
+
+ test("reconciliation cost stays null when the turn model is absent (legacy runs)", () => {
+ const turns: TurnEntry[] = [
+ {
+ messages: [
+ {
+ role: "reconciliation",
+ input_tokens: 512,
+ cache_creation_tokens: 1000,
+ note: "no model",
+ },
+ ],
+ },
+ ];
+ const recon = parseMessages(turns)[0];
+ expect(recon.model).toBeNull();
+ expect(recon.costUsd).toBeNull();
+ });
});
diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts
index 5fe056d6..648f49f6 100644
--- a/evalboard/lib/__tests__/pricing-parity.test.ts
+++ b/evalboard/lib/__tests__/pricing-parity.test.ts
@@ -5,12 +5,17 @@ import { describe, expect, test } from "vitest";
import { PRICING } from "../pricing";
// Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative
-// Python table in src/coder_eval/pricing.py. If the backend reprices a
-// model (or adds one) and this mirror isn't updated, the frontend's "estimated"
-// USD figures silently disagree with the backend's authoritative Cost on the
-// same tokens. This test parses the Python table and asserts both tables have
-// the same model ids and the same four per-MTok rates — turning silent drift
-// into a failing build.
+// Python table in src/coder_eval/pricing.py. It exists so the frontend's
+// "estimated" USD figures agree with the backend's authoritative Cost on the
+// same tokens.
+//
+// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts
+// must exist in pricing.py with identical rates (a frontend rate that disagrees
+// with the backend, or prices a model the backend doesn't, fails the build).
+// The frontend is NOT required to mirror every backend model — it only needs to
+// price the ones it displays, and the backend legitimately prices models the
+// evalboard never renders. (Exact-match was too strict: it forced unrelated
+// backend model additions into this file to keep the build green.)
const here = dirname(fileURLToPath(import.meta.url));
const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py");
@@ -44,20 +49,49 @@ describe("pricing.ts ↔ pricing.py parity", () => {
expect(Object.keys(py).length).toBeGreaterThan(10);
});
- test("both tables list the same model ids", () => {
- expect(Object.keys(PRICING).sort()).toEqual(Object.keys(py).sort());
+ test("every model in lib/pricing.ts exists in pricing.py", () => {
+ const orphans = Object.keys(PRICING).filter((m) => !(m in py));
+ expect(
+ orphans,
+ `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`,
+ ).toEqual([]);
});
- test("every model has identical input/output/cacheWrite/cacheRead rates", () => {
- for (const [model, [input, output, cw, cr]] of Object.entries(py)) {
- const ts = PRICING[model];
- expect(ts, `missing in lib/pricing.ts: ${model}`).toBeDefined();
+ test("shared models have identical input/output/cacheWrite/cacheRead rates", () => {
+ for (const [model, ts] of Object.entries(PRICING)) {
+ const rates = py[model];
+ expect(rates, `not priced in pricing.py: ${model}`).toBeDefined();
expect([
ts.inputPerMTok,
ts.outputPerMTok,
ts.cacheWritePerMTok,
ts.cacheReadPerMTok,
- ]).toEqual([input, output, cw, cr]);
+ ]).toEqual(rates);
}
});
+
+ // Python-priced models we deliberately do NOT mirror to the frontend: heavy
+ // frontier Claude/GPT variants the evalboard never runs, so pricing them here
+ // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model
+ // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build —
+ // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids
+ // that previously rendered "—" for cost).
+ const DELIBERATELY_UNMIRRORED = new Set([
+ "claude-sonnet-5",
+ "gpt-5.4-mini",
+ "gpt-5.4-nano",
+ "gpt-5.4-pro",
+ "gpt-5.5-pro",
+ "gpt-5.6-sol",
+ "gpt-5.6-terra",
+ "gpt-5.6-luna",
+ ]);
+
+ test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => {
+ const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m));
+ expect(
+ missing,
+ `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`,
+ ).toEqual([]);
+ });
});
diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts
index 0238810e..66302cf1 100644
--- a/evalboard/lib/__tests__/pricing.test.ts
+++ b/evalboard/lib/__tests__/pricing.test.ts
@@ -38,6 +38,14 @@ describe("resolvePricing", () => {
test("knows the current default opus id", () => {
expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75);
});
+
+ test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => {
+ // The recorded model arrives prefixed on litellm/Bedrock runs; without the
+ // strip these rendered "—" for the whole cost column.
+ expect(resolvePricing("converse/zai.glm-5")?.outputPerMTok).toBe(3.84);
+ expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74);
+ expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15);
+ });
});
describe("tokenBucketUsd", () => {
diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts
index 29f27360..93ac23c8 100644
--- a/evalboard/lib/pricing.ts
+++ b/evalboard/lib/pricing.ts
@@ -50,6 +50,20 @@ export const PRICING: Record = {
"gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2),
"gemini-3.5-flash": p(1.5, 9, 1.5, 0.15),
"gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15),
+ // OpenRouter open-weight models (litellm backend). Mirror of pricing.py;
+ // these providers cache implicitly (cache_write == input, unused) so only
+ // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request,
+ // so these rates are only accurate when the litellm config pins the provider
+ // (sort: price) — otherwise the billed rate can differ from the headline.
+ "moonshotai/kimi-k3": p(3, 15, 3, 0.3),
+ "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534),
+ "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625),
+ // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py.
+ // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so
+ // resolvePricing strips the routing/region prefixes before lookup.
+ "deepseek.v3.2": p(0.74, 2.22, 0.74, 0),
+ "zai.glm-5": p(1.2, 3.84, 1.2, 0),
+ "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0),
};
function p(
@@ -66,21 +80,45 @@ function p(
};
}
-// Resolve pricing for a model id, tolerating undated aliases (the recorded
-// model is usually the canonical id like "claude-sonnet-4-6", but be lenient
-// about a trailing date suffix). Matches the Python source's exact-match
-// semantics, plus a date-suffix strip — deliberately NO loose prefix match: a
-// substring fallback would silently price `gpt-5-mini` at full `gpt-5` rates,
-// presenting a multi-x overcharge as an authoritative-looking figure. Unknown
-// ids return null (render "—") rather than a wrong number.
+// Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare
+// pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the
+// recorded model_used arrives qualified (e.g. "converse/zai.glm-5",
+// "eu.anthropic.claude-sonnet-4-6"). Idempotent on already-bare ids.
+const _ROUTING_PREFIXES = ["bedrock/converse/", "bedrock/", "converse/"];
+const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."];
+function normalizeModel(model: string): string {
+ let m = model.trim();
+ for (const pre of _ROUTING_PREFIXES) {
+ if (m.startsWith(pre)) {
+ m = m.slice(pre.length);
+ break;
+ }
+ }
+ for (const pre of _REGION_PREFIXES) {
+ if (m.startsWith(pre)) {
+ m = m.slice(pre.length);
+ break;
+ }
+ }
+ if (m.startsWith("anthropic.")) m = m.slice("anthropic.".length);
+ return m;
+}
+
+// Resolve pricing for a model id, tolerating routing/region prefixes and undated
+// aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6",
+// but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date).
+// Deliberately NO loose *substring* match: that would silently price `gpt-5-mini`
+// at full `gpt-5` rates, presenting a multi-x overcharge as an authoritative-looking
+// figure. Unknown ids return null (render "—") rather than a wrong number.
//
// Object.hasOwn (not `PRICING[model]` truthiness) guards against a degenerate
// id like "constructor"/"toString" resolving to an inherited prototype member.
export function resolvePricing(model: string | null): Pricing | null {
if (!model) return null;
- if (Object.hasOwn(PRICING, model)) return PRICING[model];
+ const norm = normalizeModel(model);
+ if (Object.hasOwn(PRICING, norm)) return PRICING[norm];
// Try stripping a trailing -YYYYMMDD date.
- const undated = model.replace(/-\d{8}$/, "");
+ const undated = norm.replace(/-\d{8}$/, "");
if (Object.hasOwn(PRICING, undated)) return PRICING[undated];
return null;
}
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index c1d3f824..45de6f98 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -1198,6 +1198,9 @@ interface MessageEntry {
export interface TurnEntry {
commands?: CommandEntry[];
messages?: MessageEntry[];
+ // Model the turn ran on (iteration `model_used`). Used to price the synthetic
+ // reconciliation row, which carries no model of its own.
+ model_used?: string | null;
token_usage?: TokenUsageEntry | null;
result_summary?: {
result?: string | null;
@@ -1716,8 +1719,23 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] {
reasoningTokens: null,
thinkingOutputTokens: null,
textOutputTokens: null,
- model: null,
- costUsd: null,
+ // Price the reconciliation row so summing the Cost column across
+ // the stream reproduces the turn total. Critical for providers
+ // (e.g. OpenRouter/LiteLLM) whose per-message stream is sparse and
+ // dumps most tokens into this one row — leaving it uncosted made
+ // the visible per-row costs sum to far less than the real total.
+ // Excluded from the cost simulator (by role), and the authoritative
+ // task total_cost_usd is unaffected (it reads the backend aggregate,
+ // not a sum of per-message costUsd), so there is no double-count.
+ model: turn.model_used ?? null,
+ costUsd: messageCostUsd({
+ model: turn.model_used ?? null,
+ inputTokens: typeof msg.input_tokens === "number" ? msg.input_tokens : null,
+ outputTokens: typeof msg.output_tokens === "number" ? msg.output_tokens : null,
+ cacheWriteTokens:
+ typeof msg.cache_creation_tokens === "number" ? msg.cache_creation_tokens : null,
+ cacheReadTokens: typeof msg.cache_read_tokens === "number" ? msg.cache_read_tokens : null,
+ }),
note: typeof msg.note === "string" ? msg.note : null,
});
}
diff --git a/litellm/README.md b/litellm/README.md
new file mode 100644
index 00000000..6c3afdc1
--- /dev/null
+++ b/litellm/README.md
@@ -0,0 +1,193 @@
+# LiteLLM proxy — coder_eval `litellm` (open-weight) backend
+
+coder_eval evaluates UiPath skills on **non-Claude open-weight models** by driving
+them through the **Claude Code SDK** harness (so the native `Skill` tool +
+progressive disclosure fire unchanged). The SDK only speaks the **Anthropic
+Messages** format (`POST /v1/messages`), but the target models don't:
+
+| Target | Wire format | Needs translation? |
+|---|---|---|
+| Bedrock open-weight (GLM, DeepSeek, Kimi) | Converse | yes |
+| OpenRouter (any model) | OpenAI Chat Completions | yes |
+
+**LiteLLM is the translation shim.** The SDK is pointed at LiteLLM via
+`ANTHROPIC_BASE_URL`; LiteLLM translates Anthropic ↔ Converse / OpenAI per model
+and forwards to the real backend.
+
+```
+Claude Code SDK ──Anthropic /v1/messages──▶ LiteLLM (localhost:4000) ──▶ Bedrock Converse (eu-north-1)
+ ANTHROPIC_BASE_URL=localhost:4000 └──▶ OpenRouter (OpenAI format)
+```
+
+Files:
+- `litellm/litellm-config.yaml` — the model list + settings (this is the file you edit to add models).
+- `litellm/start-litellm.sh` — launches the proxy, reading credentials out of `.env`.
+
+---
+
+## When to run `start-litellm.sh`
+
+Run the proxy **whenever you run coder_eval against the `litellm` backend** — i.e.
+`API_BACKEND=litellm` in `.env`, or `coder-eval run ... --backend litellm`. coder_eval
+does **not** own the proxy lifecycle: it expects an already-running proxy at
+`LITELLM_BASE_URL` and **fails fast at startup** if it isn't reachable
+(`LiteLLM proxy not reachable at ... — Start it (e.g. litellm/start-litellm.sh)`).
+
+You do **not** need it for the `direct` (Anthropic) or `bedrock` (Claude-on-Bedrock)
+backends — those talk to their APIs directly.
+
+### Prerequisites
+
+`start-litellm.sh` reads these from `.env` and exports them into the proxy's own
+environment before launching:
+
+| Var | For | Notes |
+|---|---|---|
+| `AWS_BEARER_TOKEN_BEDROCK` | Bedrock models | required if you use any `bedrock/*` model |
+| `AWS_REGION` | Bedrock models | defaults to `eu-north-1` |
+| `OPENROUTER_API_KEY` | OpenRouter models | required if you use any `openrouter/*` model |
+| `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local`. It can be customly designed.|
+
+### Start it
+
+```bash
+bash litellm/start-litellm.sh # foreground on :4000, Ctrl-C to stop
+```
+
+Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`,
+`LITELLM_MASTER_KEY`.
+
+### Point coder_eval at it
+
+In `.env`:
+```
+API_BACKEND=litellm
+LITELLM_BASE_URL=http://localhost:4000
+LITELLM_AUTH_TOKEN=
+LITELLM_MODEL=deepseek/deepseek-v4-pro # default model when a task doesn't pin one
+```
+Then run — `--model` overrides the default:
+```bash
+coder-eval run tasks/.yaml --backend litellm --model deepseek/deepseek-v4-pro
+```
+
+### Verify the proxy
+
+```bash
+KEY=$(grep -E '^LITELLM_AUTH_TOKEN=' .env | head -1 | sed -E 's/^LITELLM_AUTH_TOKEN=//; s/^"//; s/"$//')
+# health
+curl -s http://localhost:4000/health/liveliness
+# which models are loaded
+curl -s http://localhost:4000/v1/models -H "x-api-key: $KEY" \
+ | python3 -c "import sys,json; print([m['id'] for m in json.load(sys.stdin)['data']])"
+```
+
+---
+
+## Config file: `litellm-config.yaml`
+
+```yaml
+model_list:
+ - model_name: # also the pricing-table key
+ litellm_params:
+ model: # e.g. bedrock/converse/... or openrouter/...
+ # ...provider-specific params
+
+litellm_settings:
+ drop_params: true # silently drop provider-unsupported params instead of 400
+
+general_settings:
+ master_key: os.environ/LITELLM_MASTER_KEY # never hardcode a secret here
+```
+
+- **`model_name`** is the id clients (and `--model`) use, **and** the key coder_eval
+ prices on. Keep it identical to the `pricing.py` key.
+- **`model`** is LiteLLM's routing target — the `provider/...` prefix selects the
+ translator + backend.
+
+---
+
+## Adding a new model
+
+### 1. Add an entry to `model_list`
+
+**Bedrock open-weight** (Converse, eu-north-1):
+```yaml
+ - model_name: some.bedrock-model
+ litellm_params:
+ model: bedrock/converse/some.bedrock-model
+ aws_region_name: eu-north-1
+```
+
+**OpenRouter** (OpenAI format — pin the provider, see caveat below):
+```yaml
+ - model_name: vendor/some-model
+ litellm_params:
+ model: openrouter/vendor/some-model
+ api_key: os.environ/OPENROUTER_API_KEY
+ extra_body:
+ provider:
+ sort: price # route to the cheapest upstream
+ allow_fallbacks: false # never silently upgrade to a pricier provider
+```
+Find the exact OpenRouter slug + rates:
+```bash
+KEY=$(grep -E '^OPENROUTER_API_KEY=' .env | head -1 | sed -E 's/^OPENROUTER_API_KEY=//; s/^"//; s/"$//')
+curl -s https://openrouter.ai/api/v1/models -H "Authorization: Bearer $KEY" \
+ | python3 -c "import sys,json;[print(m['id'],m['pricing']) for m in json.load(sys.stdin)['data'] if 'SEARCH' in m['id'].lower()]"
+```
+
+### 2. Register pricing in **both** tables (rates must match)
+
+- `src/coder_eval/pricing.py` — add to the `_PRICING` dict, keyed on the
+ `model_name` (bare id as passed in `agent.model`):
+ ```python
+ "vendor/some-model": ModelPricing(input, output, cache_write, cache_read),
+ ```
+ These implicit-caching providers charge no separate cache-write fee, so
+ `cache_write == input` (unused) and `cache_read` is the discounted rate.
+- `evalboard/lib/pricing.ts` — add the same entry so the evalboard cost columns
+ populate. A test (`lib/__tests__/pricing-parity.test.ts`) fails the build if a
+ rate here disagrees with `pricing.py`.
+
+### 3. Restart the proxy
+
+LiteLLM reads `model_list` **once at startup**. After editing the yaml, `Ctrl-C`
+and re-run `start-litellm.sh` — otherwise you get
+`Invalid model name passed in model=...`.
+
+### 4. Verify
+
+`curl .../v1/models` should list the new `model_name`, then run a task with
+`--model vendor/some-model`.
+
+---
+
+## ⚠️ OpenRouter cost accuracy (provider routing)
+
+OpenRouter is a **multi-provider router**: the rate on a model's page is the
+*cheapest* provider's headline, but a request can be routed to a pricier upstream
+(observed: DeepSeek V4 Pro → Novita at ~$1.47/M input vs the $0.435/M headline,
+3.4×). Because coder_eval prices `litellm`-backend runs from the **static**
+`pricing.py` table, an un-pinned model can bill very differently from what
+coder_eval reports.
+
+**Mitigation:** always pin providers (`extra_body.provider: {sort: price,
+allow_fallbacks: false}`), which makes the billed rate deterministic and equal to
+the headline the pricing table uses. To confirm the actual provider/cost for a
+run, read the OpenRouter **Activity** page (`provider_name`, `usage`) or query
+`GET /api/v1/models//endpoints` for per-provider rates.
+
+Token *counts* are exact and provider-independent; only the per-token *rate* is
+subject to this. Bedrock models are single-provider and not affected.
+
+---
+
+## Troubleshooting
+
+| Symptom | Cause / fix |
+|---|---|
+| `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. |
+| `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. |
+| HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. |
+| evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. |
diff --git a/litellm/litellm-config.yaml b/litellm/litellm-config.yaml
new file mode 100644
index 00000000..a9189740
--- /dev/null
+++ b/litellm/litellm-config.yaml
@@ -0,0 +1,81 @@
+# LiteLLM proxy config for the coder_eval `litellm` (open-weight) backend.
+#
+# Full guide (when to run the proxy, how to add models): litellm/README.md
+#
+# Translates Anthropic Messages (what the Claude Code SDK speaks) <-> Bedrock
+# Converse (what the non-Claude open-weight models speak), so the native `Skill`
+# tool and UiPath skills fire unchanged. Verified 2026-07-21: HTTP 200 on both
+# models via POST /v1/messages, and tool_use blocks survive the translation.
+#
+# Two ways to run the proxy:
+# - Manual: the operator starts the proxy and points coder_eval at it via
+# LITELLM_BASE_URL (coder_eval owns no proxy lifecycle).
+# - Autostart: coder_eval spawns the proxy against this same file, or an
+# always-on docker sidecar mounts it.
+#
+# Required env in the proxy's environment (NOT committed):
+# AWS_BEARER_TOKEN_BEDROCK — Bedrock bearer token (LiteLLM forwards it).
+# AWS_REGION=eu-north-1 — EU residency (Stockholm).
+# LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN.
+#
+# Run manually:
+# uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000
+
+model_list:
+ # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok).
+ - model_name: deepseek.v3.2
+ litellm_params:
+ model: bedrock/converse/deepseek.v3.2
+ aws_region_name: eu-north-1
+ # GLM 5 — quality lead ($1.20 / $3.84 per Mtok, eu-north-1).
+ - model_name: zai.glm-5
+ litellm_params:
+ model: bedrock/converse/zai.glm-5
+ aws_region_name: eu-north-1
+ # Kimi K2.5 (Moonshot AI) — Bedrock eu-north-1 (bare id, Converse).
+ - model_name: moonshotai.kimi-k2.5
+ litellm_params:
+ model: bedrock/converse/moonshotai.kimi-k2.5
+ aws_region_name: eu-north-1
+ # OpenRouter models — cost-optimization path. Not on Bedrock; OpenRouter is
+ # OpenAI-format, so LiteLLM does the Anthropic<->OpenAI translation.
+ #
+ # provider routing (extra_body.provider): OpenRouter is a multi-provider router,
+ # and its per-request routing is nondeterministic — a request can land on a
+ # provider far pricier than the model page's headline rate (observed: DeepSeek
+ # V4 Pro routed to Novita at ~$1.47/M input vs the $0.435/M headline, 3.4x).
+ # `sort: price` pins the cheapest upstream and `allow_fallbacks: false` forbids
+ # silently upgrading to a pricier one, so the billed rate is deterministic and
+ # matches the pricing.py table (and is the cost floor we benchmark for).
+ - model_name: moonshotai/kimi-k3
+ litellm_params:
+ model: openrouter/moonshotai/kimi-k3
+ api_key: os.environ/OPENROUTER_API_KEY
+ extra_body:
+ provider:
+ sort: price
+ allow_fallbacks: false
+ - model_name: z-ai/glm-5.2
+ litellm_params:
+ model: openrouter/z-ai/glm-5.2
+ api_key: os.environ/OPENROUTER_API_KEY
+ extra_body:
+ provider:
+ sort: price
+ allow_fallbacks: false
+ - model_name: deepseek/deepseek-v4-pro
+ litellm_params:
+ model: openrouter/deepseek/deepseek-v4-pro
+ api_key: os.environ/OPENROUTER_API_KEY
+ extra_body:
+ provider:
+ sort: price
+ allow_fallbacks: false
+
+litellm_settings:
+ # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400.
+ drop_params: true
+
+general_settings:
+ # Read the virtual key from the environment — never hardcode a secret here.
+ master_key: os.environ/LITELLM_MASTER_KEY
diff --git a/litellm/start-litellm.sh b/litellm/start-litellm.sh
new file mode 100755
index 00000000..da3192db
--- /dev/null
+++ b/litellm/start-litellm.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Start the LiteLLM proxy for coder_eval's `litellm` (open-weight) backend.
+#
+# Why this script exists: the proxy is a separate process that reads its Bedrock
+# credential + master key from ITS OWN environment — it does NOT read coder_eval's
+# .env. And coder_eval's .env uses bare `KEY=value` (no `export`), so `source .env`
+# alone leaves those vars unexported → the proxy can't see them → HTTP 401
+# "Unable to locate credentials". This script reads the needed values out of .env
+# and exports them explicitly before launching, so that can't happen.
+#
+# Usage:
+# litellm/start-litellm.sh # foreground; Ctrl-C to stop
+# Overridable via env:
+# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$REPO_ROOT/.env}"
+CONFIG="${LITELLM_CONFIG:-$REPO_ROOT/litellm/litellm-config.yaml}"
+PORT="${LITELLM_PORT:-4000}"
+
+# Read a key from .env, returning ONLY the value: content between surrounding
+# quotes if quoted (dropping any trailing `# comment`), else the bare value with
+# an inline comment stripped. Pure bash — avoids BSD-sed backreference quirks and
+# handles base64 values containing / + = safely.
+read_env() {
+ local line val
+ line=$(grep -E "^$1=" "$ENV_FILE" 2>/dev/null | head -1) || line=""
+ [ -z "$line" ] && return 0
+ val="${line#*=}" # strip leading `KEY=`
+ val="${val#"${val%%[![:space:]]*}"}" # trim leading whitespace
+ if [[ "$val" == '"'* ]]; then # double-quoted → content between quotes
+ val="${val#\"}"; val="${val%%\"*}"
+ elif [[ "$val" == "'"* ]]; then # single-quoted
+ val="${val#\'}"; val="${val%%\'*}"
+ else # bare → drop inline comment + trailing ws
+ val="${val%%#*}"
+ val="${val%"${val##*[![:space:]]}"}"
+ fi
+ printf '%s' "$val"
+}
+
+# --- credentials the proxy needs (already-exported values win over .env) ---
+export AWS_BEARER_TOKEN_BEDROCK="${AWS_BEARER_TOKEN_BEDROCK:-$(read_env AWS_BEARER_TOKEN_BEDROCK)}"
+export AWS_REGION="${AWS_REGION:-$(read_env AWS_REGION)}"
+export AWS_REGION="${AWS_REGION:-eu-north-1}"
+# OpenRouter key for the openrouter/* models in the config (cost-optimization path).
+export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-$(read_env OPENROUTER_API_KEY)}"
+# Master key = the key clients present. Default to .env's LITELLM_AUTH_TOKEN so the
+# client and proxy match; fall back to the local dev key.
+export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-$(read_env LITELLM_AUTH_TOKEN)}"
+export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-spike-local}"
+
+# --- preflight: fail loud instead of a runtime 401 ---
+[ -f "$CONFIG" ] || { echo "ERROR: config not found: $CONFIG" >&2; exit 1; }
+if [ -z "$AWS_BEARER_TOKEN_BEDROCK" ]; then
+ echo "ERROR: AWS_BEARER_TOKEN_BEDROCK is empty — not in $ENV_FILE and not exported." >&2
+ echo " Set it in .env or 'export AWS_BEARER_TOKEN_BEDROCK=...' before running." >&2
+ exit 1
+fi
+echo "config : $CONFIG"
+echo "region : $AWS_REGION"
+echo "bedrock tok: set (${#AWS_BEARER_TOKEN_BEDROCK} chars)"
+echo "master key : $LITELLM_MASTER_KEY"
+
+# --- stop any stale proxy on the port (the classic 'creds-less running proxy') ---
+existing=$(lsof -tiTCP:"$PORT" -sTCP:LISTEN 2>/dev/null || true)
+if [ -n "$existing" ]; then
+ echo "stopping existing listener on :$PORT (pids: $existing)"
+ # shellcheck disable=SC2086
+ kill $existing 2>/dev/null || true
+ sleep 1
+fi
+
+cat < --model zai.glm-5 (or deepseek.v3.2 / moonshotai.kimi-k2.5)
+
+EOF
+
+exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT"
diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py
index cd24451a..9c2e2887 100644
--- a/src/coder_eval/agents/claude_code_agent.py
+++ b/src/coder_eval/agents/claude_code_agent.py
@@ -45,6 +45,7 @@
CommandTelemetry,
ContentBlock,
DirectRoute,
+ LiteLLMRoute,
ResultSummary,
TokenUsage,
TranscriptMessage,
@@ -551,6 +552,30 @@ def on_user_message(self, message: Message) -> None:
)
)
+ def _finalize_token_usage(self) -> TokenUsage:
+ """Build the turn's cumulative TokenUsage, repricing for LiteLLM.
+
+ Extracted from ``finalize`` so the LiteLLM repricing *wiring* (not just the
+ static ``_reprice_for_litellm`` helper) is directly testable. The SDK's
+ cost estimate assumes Claude pricing and is wrong for an open-weight model
+ behind LiteLLM, so reprice the top-line from the token buckets at the
+ model's real rate — buckets untouched, so the reconciliation invariant
+ holds.
+ """
+ usage = (
+ self._agent._build_token_usage(
+ self.sdk_messages,
+ self.sdk_result_usage,
+ self.sdk_result_cost,
+ self.sdk_result_model_usage,
+ self.effective_model,
+ )
+ or TokenUsage()
+ )
+ if isinstance(self._agent.route, LiteLLMRoute):
+ self._agent._reprice_for_litellm(usage, self.effective_model)
+ return usage
+
def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None:
"""Close orphaned tools + the open turn, emit the terminal AgentEndEvent,
and on a crash build the partial TurnRecord. Idempotent."""
@@ -591,16 +616,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso
)
self.current_turn_id = None
- usage = (
- self._agent._build_token_usage(
- self.sdk_messages,
- self.sdk_result_usage,
- self.sdk_result_cost,
- self.sdk_result_model_usage,
- self.effective_model,
- )
- or TokenUsage()
- )
+ usage = self._finalize_token_usage()
try:
agent_output = self._agent._format_messages(self.messages)
@@ -763,6 +779,38 @@ def _build_sdk_env(
case DirectRoute():
return base_env, None
+ case LiteLLMRoute() as cr:
+ # Point the SDK at the custom Anthropic-compatible endpoint (e.g.
+ # a LiteLLM gateway). These override any inherited value: the SDK
+ # merges {**os.environ, ..., **options.env} at spawn, so setting
+ # them here wins over the parent environment.
+ env = {
+ "ANTHROPIC_BASE_URL": cr.base_url,
+ "ANTHROPIC_AUTH_TOKEN": cr.auth_token,
+ # Neutralize any inherited ANTHROPIC_API_KEY: auth on this
+ # route is the bearer ANTHROPIC_AUTH_TOKEN, and a stray
+ # x-api-key (e.g. a real Anthropic key exported from .env)
+ # would conflict with the gateway's key auth.
+ "ANTHROPIC_API_KEY": "",
+ # Claude Code attaches usage-attribution metadata (metadata.user_id)
+ # that Bedrock's requestMetadata regex rejects (HTTP 400) once LiteLLM
+ # forwards it to Bedrock. Disable it, mirroring the BedrockRoute case above.
+ "CLAUDE_CODE_ATTRIBUTION_HEADER": "0",
+ # Neutralize inherited Bedrock creds. The CLI auto-selects Bedrock DIRECT
+ # when AWS_BEARER_TOKEN_BEDROCK is present (`if(process.env.AWS_BEARER_TOKEN_BEDROCK)`),
+ # and that token is forwarded into docker task containers via the default
+ # env-passthrough allowlist — so without blanking it the CLI bypasses
+ # ANTHROPIC_BASE_URL (the LiteLLM proxy) and calls Bedrock directly. Empty string
+ # is falsy in the CLI's check, so this forces it back onto the gateway.
+ "AWS_BEARER_TOKEN_BEDROCK": "",
+ "CLAUDE_CODE_USE_BEDROCK": "",
+ }
+ if cr.model:
+ env["ANTHROPIC_MODEL"] = cr.model
+ if cr.small_model:
+ env["ANTHROPIC_SMALL_FAST_MODEL"] = cr.small_model
+ return {**base_env, **env}, cr.model
+
raise AssertionError(f"Unhandled route type: {type(route).__name__}")
def _resolve_effective_model(
@@ -784,6 +832,13 @@ def _resolve_effective_model(
if effective:
env["ANTHROPIC_MODEL"] = effective
return effective
+ if isinstance(self.route, LiteLLMRoute):
+ # Same env-sync as Bedrock, but pass the id verbatim (no
+ # inference-profile qualification — the gateway maps it).
+ effective = config_model or route_model
+ if effective:
+ env["ANTHROPIC_MODEL"] = effective
+ return effective
return config_model or route_model
async def communicate(
@@ -1371,6 +1426,24 @@ def _build_token_usage(
model,
)
+ @staticmethod
+ def _price_from_buckets(usage: TokenUsage, model: str | None) -> float | None:
+ """Price the four token buckets at ``model``'s list rate.
+
+ Shared by ``_backfill_cost`` (price-if-absent) and ``_reprice_for_litellm``
+ (always-reprice). Returns ``None`` when ``model`` is unset or absent from
+ the rate card.
+ """
+ if not model:
+ return None
+ return calculate_cost(
+ model,
+ uncached_input_tokens=usage.uncached_input_tokens,
+ output_tokens=usage.output_tokens,
+ cache_creation_tokens=usage.cache_creation_input_tokens,
+ cache_read_tokens=usage.cache_read_input_tokens,
+ )
+
@staticmethod
def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage:
"""Price the token buckets when the SDK gave no cost (timeout / kill).
@@ -1384,13 +1457,7 @@ def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage:
"""
if usage.total_cost_usd is not None or not model:
return usage
- cost = calculate_cost(
- model,
- uncached_input_tokens=usage.uncached_input_tokens,
- output_tokens=usage.output_tokens,
- cache_creation_tokens=usage.cache_creation_input_tokens,
- cache_read_tokens=usage.cache_read_input_tokens,
- )
+ cost = ClaudeCodeAgent._price_from_buckets(usage, model)
if cost is not None:
usage.total_cost_usd = cost
else:
@@ -1400,6 +1467,30 @@ def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage:
logger.warning("No pricing for model %r; timeout/kill turn cost left unset", model)
return usage
+ @staticmethod
+ def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> None:
+ """Recompute the top-line cost for the LiteLLM backend, in place.
+
+ The Claude Agent SDK's ``costUSD``/``total_cost_usd`` is a client-side
+ estimate that assumes Claude/Anthropic pricing, so it is wrong for an
+ open-weight model driven through LiteLLM. Reprice from the (already
+ authoritative) token buckets at the model's real rate. The token buckets
+ are left untouched, so the per-message stream / reconciliation invariant
+ is unaffected — only the cost scalar changes.
+
+ An unknown/unpriced model sets the cost to ``None`` (an honest "N/A")
+ **and logs a warning** — mirroring ``_backfill_cost`` — because a proxy
+ model missing from the rate card otherwise silently yields
+ ``total_cost_usd = None``, which makes the orchestrator skip the
+ ``max_usd`` gate with no diagnostic.
+ """
+ cost = ClaudeCodeAgent._price_from_buckets(usage, model)
+ usage.total_cost_usd = cost
+ if cost is None:
+ logger.warning(
+ "No pricing for litellm model %r; turn cost left unset (max_usd gate will be skipped)", model
+ )
+
def get_sdk_options(self) -> dict[str, Any] | None:
"""Get the raw SDK options used for the last agent query.
diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py
index dff081da..9b22929c 100644
--- a/src/coder_eval/cli/run_command.py
+++ b/src/coder_eval/cli/run_command.py
@@ -4,6 +4,9 @@
import logging
import os
import sys
+import urllib.error
+import urllib.parse
+import urllib.request
from collections.abc import Callable
from pathlib import Path
from typing import Any
@@ -12,7 +15,7 @@
import typer
from tqdm import tqdm
-from ..config import settings
+from ..config import Settings, settings
from ..logging_config import setup_logging
from ..models import PreservationMode, ResolvedTask, RunSummary, TaskResult
from ..orchestration.config import BatchRunConfig
@@ -66,6 +69,46 @@ def _resolve_experiment_path(experiment: Path | None) -> Path | None:
raise typer.BadParameter(f"Experiment not found: {experiment}.{hint}")
+def _litellm_preflight_error(current_settings: Settings) -> str | None:
+ """Return an error message if the ``litellm`` backend's external proxy is
+ unreachable, else ``None``.
+
+ Only applies when ``api_backend=litellm`` with an explicit ``LITELLM_BASE_URL``
+ (the manual proxy / always-on-sidecar path). Without this check a dead proxy
+ makes the Claude SDK hang on the endpoint instead of failing fast. Any HTTP
+ response (even non-200) counts as reachable — only a connection/timeout error
+ is treated as "proxy down".
+ """
+ from ..models import ApiBackend
+
+ if current_settings.api_backend != ApiBackend.LITELLM or not current_settings.litellm_base_url:
+ return None
+ base_url = current_settings.litellm_base_url
+ # Reject a scheme-less/non-http(s) URL with a clear message instead of letting
+ # urlopen raise a bare ValueError ("unknown url type") that escapes as a
+ # traceback. Also makes the `# nosec B310` below honest — the scheme is now
+ # constrained to http(s), which is exactly what B310 audits.
+ if urllib.parse.urlsplit(base_url).scheme not in ("http", "https"):
+ return (
+ f"LITELLM_BASE_URL must be an http(s) URL, got {base_url!r}. "
+ "Set it to e.g. http://localhost:4000 (or unset LITELLM_BASE_URL and switch backends)."
+ )
+ url = f"{base_url.rstrip('/')}/health/liveliness"
+ try:
+ # B310: url is built from the operator-configured LITELLM_BASE_URL, whose
+ # scheme is validated to http(s) just above — not untrusted input; this
+ # only probes reachability of that proxy endpoint.
+ urllib.request.urlopen(url, timeout=5).close() # nosec B310
+ except urllib.error.HTTPError:
+ return None # server responded (up), just not 200 on this path
+ except (urllib.error.URLError, OSError) as exc:
+ return (
+ f"LiteLLM proxy not reachable at {current_settings.litellm_base_url} (tried {url}): {exc}. "
+ "Start it (e.g. litellm/start-litellm.sh) or unset LITELLM_BASE_URL."
+ )
+ return None
+
+
def _build_overrides(
*,
model: str | None,
@@ -225,7 +268,7 @@ def run_command(
None,
"--backend",
"-b",
- click_type=click.Choice(["direct", "bedrock"], case_sensitive=False),
+ click_type=click.Choice(["direct", "bedrock", "litellm"], case_sensitive=False),
help="API backend (default: from API_BACKEND env var)",
),
experiment: Path | None = typer.Option( # noqa: B008
@@ -460,6 +503,13 @@ async def _run_all_tasks(
},
)
+ # Fail fast if the litellm backend points at an unreachable external proxy —
+ # otherwise the agent hangs on the dead endpoint instead of erroring.
+ preflight_error = await asyncio.to_thread(_litellm_preflight_error, settings)
+ if preflight_error:
+ console.print(f"[red]{preflight_error}[/red]")
+ raise typer.Exit(1)
+
try:
# Always run through experiment layer (defaults to experiments/default.yaml)
summary, failed_suite_gates = await _run_with_experiment(
diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py
index 9111634a..860b7ea1 100644
--- a/src/coder_eval/config.py
+++ b/src/coder_eval/config.py
@@ -7,6 +7,7 @@
import os
from pathlib import Path
from typing import Any
+from urllib.parse import urlsplit
from dotenv import dotenv_values, load_dotenv
from pydantic import AliasChoices, Field
@@ -103,6 +104,17 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
bedrock_model: str | None = None # Cross-region model ID
bedrock_small_model: str | None = None # Cross-region small model ID
+ # LiteLLM (Anthropic-compatible) endpoint settings (used when api_backend == "litellm").
+ # These map to ANTHROPIC_BASE_URL / ANTHROPIC_AUTH_TOKEN / ANTHROPIC_MODEL /
+ # ANTHROPIC_SMALL_FAST_MODEL, but ONLY inside the SDK subprocess env (see
+ # ClaudeCodeAgent._build_sdk_env). They are deliberately NOT named anthropic_*
+ # so the os.environ export loop below can't leak ANTHROPIC_BASE_URL process-wide
+ # (which would silently redirect the judge's in-process Anthropic() client).
+ litellm_base_url: str | None = None
+ litellm_auth_token: str | None = None
+ litellm_model: str | None = None
+ litellm_small_model: str | None = None
+
# Codex settings (CodexAgent). CODEX_MODEL is the fallback model/deployment
# used when a task doesn't pin agent.model; CODEX_BASE_URL routes to a custom
# OpenAI-/responses-compatible endpoint (incl. Azure OpenAI). For Azure also
@@ -169,6 +181,36 @@ def _validate_bedrock_settings(self) -> None:
+ " Please set them in your .env file."
)
+ def _validate_litellm_settings(self) -> None:
+ """Validate that required custom Anthropic-endpoint settings are present.
+
+ Raises:
+ ValueError: If required custom settings are missing
+ """
+ missing = []
+ if not self.litellm_base_url:
+ missing.append("LITELLM_BASE_URL")
+ if not self.litellm_auth_token:
+ missing.append("LITELLM_AUTH_TOKEN")
+ # LITELLM_MODEL is required for the same reason BEDROCK_MODEL is: a None
+ # model sent to the SDK/gateway yields an opaque 400. Fail fast instead.
+ if not self.litellm_model:
+ missing.append("LITELLM_MODEL")
+ if missing:
+ raise ValueError(
+ f"LiteLLM-endpoint routing is enabled but missing required settings: {', '.join(missing)}."
+ + " Please set them in your .env file."
+ )
+ # base_url is present (not in `missing`); reject a malformed one so the
+ # downstream preflight (urlopen) and environment_info (urlparse hostname)
+ # get a well-formed absolute URL instead of a raw ValueError / empty host.
+ parts = urlsplit(self.litellm_base_url or "")
+ if parts.scheme not in ("http", "https") or not parts.hostname:
+ raise ValueError(
+ f"LITELLM_BASE_URL must be an http(s) URL with a host, got {self.litellm_base_url!r}. "
+ + "Set it to e.g. http://localhost:4000 in your .env file."
+ )
+
def validate_api_keys(self, agent_type: str) -> None:
"""Validate that required API keys are present.
@@ -186,6 +228,9 @@ def validate_api_keys(self, agent_type: str) -> None:
if self.api_backend == ApiBackend.BEDROCK:
self._validate_bedrock_settings()
+ if self.api_backend == ApiBackend.LITELLM:
+ self._validate_litellm_settings()
+
# Claude Code agent can use either:
# 1. ANTHROPIC_API_KEY environment variable
# 2. Cached CLI authentication from 'claude-code login' (subscription account)
diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py
index 73db9f4a..64d0abfa 100644
--- a/src/coder_eval/criteria/llm_judge.py
+++ b/src/coder_eval/criteria/llm_judge.py
@@ -28,6 +28,7 @@
JudgeCriterionResult,
JudgeTranscript,
JudgeVerdict,
+ LiteLLMRoute,
LLMJudgeCriterion,
TokenUsage,
)
@@ -203,9 +204,15 @@ def _invoke_tool_channel(
)
verdict, err = extract_verdict_from_anthropic_response(anthropic_response)
response_usage = token_usage_from_anthropic_dict(anthropic_response)
- case _:
- # route is None or an unexpected type — the unconfigured-arm guard in
- # _check_impl handles None before dispatch, so this is defensive only.
+ case LiteLLMRoute():
+ # Defensive: the evaluation route is pinned to Bedrock/Direct by
+ # resolve_evaluation_route, so a LiteLLM route should never reach the
+ # judge. Fail loudly rather than silently scoring 0.0. (Explicit arm
+ # keeps the match exhaustive so pyright flags any future route member.)
+ return None, "llm_judge: evaluation route must be Bedrock/Direct, got LiteLLM", "(litellm route)", None
+ case None:
+ # Handled by the unconfigured-arm guard in _check_impl before dispatch;
+ # defensive only.
return None, "llm_judge: no usable API route", "(no route)", None
if verdict is not None:
diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py
index 0c50ecf5..53ff28be 100644
--- a/src/coder_eval/isolation/docker_runner.py
+++ b/src/coder_eval/isolation/docker_runner.py
@@ -59,6 +59,29 @@
# `COPY` destination in docker/Dockerfile -- a drift guard test enforces that.
CONTAINER_ENTRYPOINT = "/usr/local/bin/coder_eval_entrypoint.sh"
+# Docker Desktop's stable alias for the host, from inside a bridge-network
+# container. Auto-resolves on macOS/Windows; on Linux it must be published
+# explicitly via `--add-host host.docker.internal:host-gateway`.
+_DOCKER_HOST_ALIAS = "host.docker.internal"
+_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+
+
+def _rewrite_loopback_for_container(url: str) -> str | None:
+ """Rewrite a loopback URL to the docker host alias, preserving scheme/port/path.
+
+ Returns the rewritten URL, or None if the host is not loopback (forward as-is).
+ A LiteLLM proxy on the HOST is unreachable at localhost from inside a bridge
+ container, so ``http://localhost:4000`` -> ``http://host.docker.internal:4000``.
+ """
+ from urllib.parse import urlsplit, urlunsplit
+
+ parts = urlsplit(url)
+ if parts.hostname not in _LOOPBACK_HOSTS:
+ return None
+ netloc = _DOCKER_HOST_ALIAS if parts.port is None else f"{_DOCKER_HOST_ALIAS}:{parts.port}"
+ return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+
# Top-level entries under ~/.claude that the per-task RW copy SKIPS. We copy
# the host's ~/.claude into a throwaway tmp dir and mount that copy read-WRITE
# so the in-container CLI can write anywhere it needs without ever touching the
@@ -1088,9 +1111,27 @@ def _build_argv(
# would silently default to DIRECT — downgrading the judge (and agent) route.
merged_allowlist = set(cfg.env_passthrough) | set(cfg.env_passthrough_extra)
for env_var in merged_allowlist:
+ # LITELLM_BASE_URL is forwarded below with a value rewrite, not name-only.
+ if env_var == "LITELLM_BASE_URL":
+ continue
if env_var in os.environ:
argv += ["--env", env_var]
+ # LITELLM_BASE_URL points at a proxy on the HOST. A bridge-network container
+ # can't reach the host's loopback, so rewrite localhost/127.0.0.1 to the
+ # docker host alias and publish that alias (`--add-host`) for Linux parity
+ # (it's automatic on macOS/Windows Docker Desktop). It's only a URL, so an
+ # explicit `--env VAR=value` is safe to render in the logged argv — unlike
+ # the auth token, which stays name-only above. Skipped when the container
+ # has no network (the proxy is unreachable anyway → validation errors).
+ litellm_base_url = os.environ.get("LITELLM_BASE_URL")
+ if litellm_base_url and "LITELLM_BASE_URL" in merged_allowlist and cfg.network != "none":
+ rewritten = _rewrite_loopback_for_container(litellm_base_url)
+ if rewritten is not None:
+ argv += ["--env", f"LITELLM_BASE_URL={rewritten}", "--add-host", f"{_DOCKER_HOST_ALIAS}:host-gateway"]
+ else:
+ argv += ["--env", "LITELLM_BASE_URL"]
+
# Signal to in-container agents that the harness already provides OS-level
# isolation. The Codex agent reads this to fall back to its full-access
# sandbox: Codex's Landlock-backed read-only / workspace-write sandboxes
diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py
index 82354063..3a9b3796 100644
--- a/src/coder_eval/models/__init__.py
+++ b/src/coder_eval/models/__init__.py
@@ -136,6 +136,8 @@
BedrockRoute,
DirectRoute,
JudgeTransport,
+ LiteLLMRoute,
+ resolve_evaluation_route,
resolve_route,
to_bedrock_inference_profile,
)
@@ -229,8 +231,10 @@
"ApiRoute",
"DirectRoute",
"BedrockRoute",
+ "LiteLLMRoute",
"JudgeTransport",
"resolve_route",
+ "resolve_evaluation_route",
"to_bedrock_inference_profile",
# Templates
"BaseTemplateSource",
diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py
index 36173746..03afe885 100644
--- a/src/coder_eval/models/enums.py
+++ b/src/coder_eval/models/enums.py
@@ -67,6 +67,7 @@ class ApiBackend(StrEnum):
DIRECT = "direct" # Anthropic API directly (ANTHROPIC_API_KEY)
BEDROCK = "bedrock" # AWS Bedrock (bearer token auth)
+ LITELLM = "litellm" # LiteLLM (Anthropic-compatible) endpoint, e.g. LiteLLM -> Bedrock
class PermissionMode(StrEnum):
diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py
index f1a90687..1b338e62 100644
--- a/src/coder_eval/models/routing.py
+++ b/src/coder_eval/models/routing.py
@@ -6,6 +6,7 @@
from typing import TYPE_CHECKING, Literal
from coder_eval.models.enums import ApiBackend
+from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL
if TYPE_CHECKING:
@@ -101,24 +102,42 @@ class BedrockRoute:
disable_attribution_header: bool = True
-ApiRoute = DirectRoute | BedrockRoute
+@dataclass(frozen=True)
+class LiteLLMRoute:
+ """Route through a custom Anthropic-compatible endpoint (e.g. a LiteLLM
+ gateway fronting Bedrock open-weight models).
+
+ The Claude Code SDK is pointed at ``base_url`` via ``ANTHROPIC_BASE_URL`` and
+ authenticates with ``auth_token`` via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The
+ ``model``/``small_model`` ids are passed **verbatim** (no Bedrock
+ inference-profile qualification) — the gateway maps them to its backend.
+ """
+
+ base_url: str
+ auth_token: str
+ model: str | None = None
+ small_model: str | None = None
+
+
+ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute
# Stable string names for environment_info recording (decoupled from class names)
ROUTE_NAMES: dict[type, str] = {
DirectRoute: "anthropic_direct",
BedrockRoute: "aws_bedrock",
+ LiteLLMRoute: "litellm",
}
def resolve_route(settings: Settings) -> ApiRoute:
"""Resolve an ``ApiRoute`` from static settings.
- Handles the two supported backends (``DIRECT`` and ``BEDROCK``), whose
- route is fully determined by ``Settings``.
+ Handles the three supported backends (``DIRECT``, ``BEDROCK``, ``LITELLM``),
+ whose route is fully determined by ``Settings``.
Called after ``validate_api_keys()`` has verified credentials. Uses
- ``assert`` for type narrowing (not ``ValueError``) since the Bedrock
+ ``assert`` for type narrowing (not ``ValueError``) since the Bedrock/custom
credential checks are an internal contract.
"""
match settings.api_backend:
@@ -145,6 +164,48 @@ def resolve_route(settings: Settings) -> ApiRoute:
)
case ApiBackend.DIRECT:
return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings))
+ case ApiBackend.LITELLM:
+ # base_url/auth_token are guaranteed by validate_api_keys (run first);
+ # asserts narrow the Optional types, mirroring the Bedrock arm.
+ assert settings.litellm_base_url is not None, "LiteLLM backend requires litellm_base_url"
+ assert settings.litellm_auth_token is not None, "LiteLLM backend requires litellm_auth_token"
+ # No inference-profile qualification: the id is passed verbatim to the gateway.
+ small_model = settings.litellm_small_model or settings.litellm_model
+ return LiteLLMRoute(
+ base_url=settings.litellm_base_url,
+ auth_token=settings.litellm_auth_token,
+ model=settings.litellm_model,
+ small_model=small_model,
+ )
+
+
+def resolve_evaluation_route(settings: Settings, agent_route: ApiRoute) -> ApiRoute:
+ """Resolve the route used by the *evaluation* side — the ``llm_judge`` /
+ ``agent_judge`` criteria and the simulated user — which must stay on a
+ constant Claude backend regardless of the agent under test, so grading and
+ simulation stay comparable across models.
+
+ - Agent on Bedrock/Direct: the judge already runs on Claude via that route,
+ so reuse it unchanged (no behavior change for existing runs).
+ - Agent on LiteLLM (open-weight): the agent route cannot serve a Claude
+ judge, so pin evaluation to Bedrock (preferred, from the AWS bearer token)
+ or Direct (``ANTHROPIC_API_KEY``). If neither is configured, fall back to a
+ ``DirectRoute`` with no judge transport so ``llm_judge`` fails with its
+ clean "unconfigured" error rather than silently scoring 0.0.
+ """
+ if isinstance(agent_route, BedrockRoute | DirectRoute):
+ return agent_route
+ # agent_route is LiteLLMRoute → pin evaluation to a constant Claude backend.
+ if settings.aws_bearer_token_bedrock and settings.aws_region:
+ judge_model = settings.bedrock_model or DEFAULT_JUDGE_MODEL
+ qualified = to_bedrock_inference_profile(judge_model, settings.aws_region)
+ return BedrockRoute(
+ bearer_token=settings.aws_bearer_token_bedrock,
+ region=settings.aws_region,
+ model=qualified,
+ small_model=qualified,
+ )
+ return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings))
def _resolve_direct_judge_transport(settings: Settings) -> JudgeTransport | None:
diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py
index 282cdbdc..906add84 100644
--- a/src/coder_eval/models/sandbox.py
+++ b/src/coder_eval/models/sandbox.py
@@ -227,6 +227,15 @@ class DockerDriverConfig(BaseModel):
# through Bedrock instead of falling back to ~/.claude OAuth.
"CLAUDE_CODE_USE_BEDROCK",
"ANTHROPIC_MODEL",
+ # LiteLLM (Anthropic-compatible) open-weight backend. The proxy runs on
+ # the HOST, so LITELLM_BASE_URL is rewritten to host.docker.internal at
+ # the container boundary (see docker_runner); the rest forward verbatim.
+ # Without these the in-container Settings sees API_BACKEND=litellm with no
+ # creds and _validate_litellm_settings raises a hard ValueError.
+ "LITELLM_BASE_URL",
+ "LITELLM_AUTH_TOKEN",
+ "LITELLM_MODEL",
+ "LITELLM_SMALL_MODEL",
# Codex agent auth/routing — without these the in-container codex
# binary falls back to a ChatGPT login that doesn't exist in the
# container and auth fails. CODEX_API_KEY drives login_api_key;
diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index b1581470..f5a5206e 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -11,6 +11,7 @@
from inspect import isawaitable
from pathlib import Path
from typing import Any
+from urllib.parse import urlparse
from .agent import Agent
from .agents.watchdog import ThreadedWatchdog
@@ -37,6 +38,7 @@
EvaluationResult,
FinalStatus,
JudgeCriterionResult,
+ LiteLLMRoute,
PostRunCommand,
PostRunResult,
PreRunCommand,
@@ -48,6 +50,7 @@
TokenUsage,
TurnRecord,
UserMessage,
+ resolve_evaluation_route,
resolve_route,
)
from .orchestration.early_stop import EarlyStopWatcher, validate_early_stop
@@ -115,16 +118,21 @@ async def _pump_stream(
_UTTERANCE_TAG_RE = re.compile(r"^\[(ASSISTANT|RESULT - SUCCESS|RESULT - ERROR|TOOL USE)\](?: (.*))?$")
-def _format_routing(route: ApiRoute) -> str:
+def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str:
"""Format the route name for the ``API routing:`` log line.
For ``DirectRoute`` the resolved judge transport is appended so the choice
(anthropic / none) is visible on every run, not only in the persisted
- ``environment_info`` record.
+ ``environment_info`` record. For ``LiteLLMRoute`` the model is shown — the
+ ``effective_model`` (the resolved ``agent.model``, e.g. from ``--model``)
+ when supplied, else the route's own default — so the line reflects what the
+ agent will actually send rather than the route-level fallback.
"""
name = ROUTE_NAMES[type(route)]
if isinstance(route, DirectRoute):
return f"{name} (judge transport: {route.judge_transport or 'none'})"
+ if isinstance(route, LiteLLMRoute):
+ return f"{name} (model: {effective_model or route.model or 'default'})"
return name
@@ -361,6 +369,11 @@ def __init__(
# API routing (initialized in _setup)
self.route: ApiRoute | None = None
+ # Route for the evaluation side (llm_judge / agent_judge / simulated user):
+ # pinned to a constant Claude backend so grading stays comparable when the
+ # agent runs on an open-weight (LiteLLM) model. Equals self.route for the
+ # Direct/Bedrock backends.
+ self.eval_route: ApiRoute | None = None
# Result tracking
self.result: EvaluationResult | None = None
@@ -890,8 +903,11 @@ async def _setup(self) -> None:
self.result.sandbox_path = str(self.sandbox.sandbox_dir)
self.route = resolve_route(settings)
- logger.info("API routing: %s", _format_routing(self.route))
- self.success_checker = SuccessChecker(self.sandbox, route=self.route)
+ self.eval_route = resolve_evaluation_route(settings, self.route)
+ logger.info(
+ "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)
+ )
+ self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route)
self._record_route_environment_info()
return
@@ -957,8 +973,9 @@ async def _setup_sandbox() -> Any:
# Determine API routing from settings.api_backend enum
self.route = resolve_route(settings)
- logger.info("API routing: %s", _format_routing(self.route))
- self.success_checker = SuccessChecker(self.sandbox, route=self.route)
+ self.eval_route = resolve_evaluation_route(settings, self.route)
+ logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None))
+ self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route)
# Create and start the agent. For a no-op (type: none) task this dispatches
# to NoOpAgent, whose start/communicate/stop are no-ops — the orchestrator
@@ -1074,6 +1091,12 @@ def _record_route_environment_info(self) -> None:
assert self.result is not None
assert self.route is not None
self.result.environment_info["api_routing"] = ROUTE_NAMES[type(self.route)]
+ # The evaluation side (llm_judge / agent_judge / simulated user) may run on
+ # a different, constant backend — pinned to Claude when the agent is on
+ # LiteLLM — so record it: a run then shows what actually graded/simulated
+ # it, distinct from the agent's api_routing.
+ if self.eval_route is not None:
+ self.result.environment_info["eval_routing"] = ROUTE_NAMES[type(self.eval_route)]
if isinstance(self.route, BedrockRoute):
self.result.environment_info["aws_region"] = self.route.region
if self.route.model:
@@ -1082,6 +1105,12 @@ def _record_route_environment_info(self) -> None:
# Record which transport llm_judge will use under DirectRoute so the
# choice is visible in run artifacts (and not just the startup log).
self.result.environment_info["judge_transport"] = self.route.judge_transport or "none"
+ elif isinstance(self.route, LiteLLMRoute):
+ # Host only (never the base_url or auth token) — mirrors the Codex
+ # agent's host-only recording so secrets stay out of run artifacts.
+ self.result.environment_info["litellm_base_url_host"] = urlparse(self.route.base_url).hostname or ""
+ if self.route.model:
+ self.result.environment_info["litellm_model"] = self.route.model
# Agent-specific routing (e.g. Codex custom-endpoint / Azure). No-op for
# the evaluate-only path (no agent) and for agents that add nothing.
if self.agent is not None:
@@ -1660,7 +1689,10 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir:
config=sim_config,
task_description=self.task.description,
initial_prompt=initial_prompt,
- route=self.route,
+ # Pin the simulated user to the constant Claude eval route, not the
+ # agent's (possibly open-weight) route, so the simulator behaves
+ # identically across the models under test.
+ route=self.eval_route,
)
await simulator.start()
diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index ebe6797b..28784360 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -88,6 +88,21 @@ class ModelPricing:
"gemini-3.1-pro-preview-customtools": ModelPricing(2.0, 12.0, 2.0, 0.20),
"gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15),
"gemini-3-flash-preview": ModelPricing(1.5, 9.0, 1.5, 0.15),
+ # Open-weight models on Bedrock (eu-north-1), driven via the LiteLLM backend.
+ # Bedrock lists no prompt-cache read/write rate for these, so cache-creation
+ # is priced at the input rate and cache-read at 0 (conservative — see the
+ # per-provider cost-accounting caveat; revisit against the AWS model cards).
+ "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0),
+ "zai.glm-5": ModelPricing(1.2, 3.84, 1.2, 0.0),
+ "moonshotai.kimi-k2.5": ModelPricing(0.72, 3.6, 0.72, 0.0),
+ # OpenRouter models (cost-optimization path). These providers cache prompt
+ # prefixes IMPLICITLY (no cache_control, no cache-write fee), so cache-creation
+ # is priced at input (unused — cache_creation_tokens is always 0) and cache-read
+ # at OpenRouter's published input_cache_read rate. Rates per OpenRouter's
+ # /models endpoint (per-token x 1e6).
+ "moonshotai/kimi-k3": ModelPricing(3.0, 15.0, 3.0, 0.30),
+ "z-ai/glm-5.2": ModelPricing(0.826, 2.596, 0.826, 0.1534),
+ "deepseek/deepseek-v4-pro": ModelPricing(0.435, 0.87, 0.435, 0.003625),
}
@@ -144,6 +159,12 @@ def _normalize_model(model: str) -> str:
so it is safe to apply unconditionally for every route.
"""
model = model.strip()
+ # LiteLLM/Bedrock routing prefixes (e.g. "converse/zai.glm-5",
+ # "bedrock/converse/deepseek.v3.2") → bare model id.
+ for routing_prefix in ("bedrock/converse/", "bedrock/", "converse/"):
+ if model.startswith(routing_prefix):
+ model = model[len(routing_prefix) :]
+ break
for prefix in _BEDROCK_REGION_PREFIXES:
if model.startswith(prefix):
model = model[len(prefix) :]
diff --git a/tests/test_cli_backend_flag.py b/tests/test_cli_backend_flag.py
index 18177367..bd0c1ba6 100644
--- a/tests/test_cli_backend_flag.py
+++ b/tests/test_cli_backend_flag.py
@@ -29,7 +29,7 @@ async def _noop_run_all_tasks(*args: object, **kwargs: object) -> None:
return None
-@pytest.mark.parametrize("backend", ["bedrock", "direct"])
+@pytest.mark.parametrize("backend", ["bedrock", "direct", "litellm"])
def test_backend_flag_syncs_api_backend_env(monkeypatch, backend) -> None:
# delenv snapshots the key so monkeypatch's teardown restores it even though
# the command mutates os.environ directly (no test pollution across params).
diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py
index 60be6b86..b726786c 100644
--- a/tests/test_config_precedence.py
+++ b/tests/test_config_precedence.py
@@ -279,12 +279,13 @@ def test_cli_override_applies_with_lineage_detail():
def test_api_backend_enum_values():
- """Verify ApiBackend has exactly 2 values with expected string representations."""
+ """Verify ApiBackend has exactly 3 values with expected string representations."""
from coder_eval.models.enums import ApiBackend
- assert set(ApiBackend) == {ApiBackend.DIRECT, ApiBackend.BEDROCK}
+ assert set(ApiBackend) == {ApiBackend.DIRECT, ApiBackend.BEDROCK, ApiBackend.LITELLM}
assert str(ApiBackend.DIRECT) == "direct"
assert str(ApiBackend.BEDROCK) == "bedrock"
+ assert str(ApiBackend.LITELLM) == "litellm"
def test_settings_api_backend_default():
diff --git a/tests/test_docker_litellm_env.py b/tests/test_docker_litellm_env.py
new file mode 100644
index 00000000..8d8d99e8
--- /dev/null
+++ b/tests/test_docker_litellm_env.py
@@ -0,0 +1,119 @@
+"""Tests for forwarding the LiteLLM backend into a docker-driver container.
+
+The LiteLLM proxy runs on the HOST, so the container must (a) receive the
+LITELLM_* credentials via the env allowlist and (b) reach the host via
+host.docker.internal rather than an unreachable loopback address.
+"""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from coder_eval.isolation.docker_runner import (
+ DockerRunner,
+ _rewrite_loopback_for_container,
+)
+from coder_eval.models import DockerDriverConfig, FileExistsCriterion, SandboxConfig, TaskDefinition
+
+
+pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="docker driver is POSIX-only")
+
+
+class TestRewriteLoopbackForContainer:
+ """localhost/127.0.0.1 -> host.docker.internal, preserving scheme/port/path."""
+
+ def test_localhost_with_port(self):
+ assert _rewrite_loopback_for_container("http://localhost:4000") == "http://host.docker.internal:4000"
+
+ def test_127_0_0_1_with_port(self):
+ assert _rewrite_loopback_for_container("http://127.0.0.1:4000") == "http://host.docker.internal:4000"
+
+ def test_preserves_scheme_and_path(self):
+ assert _rewrite_loopback_for_container("https://localhost:8443/v1") == "https://host.docker.internal:8443/v1"
+
+ def test_no_port(self):
+ assert _rewrite_loopback_for_container("http://localhost") == "http://host.docker.internal"
+
+ def test_non_loopback_returns_none(self):
+ assert _rewrite_loopback_for_container("http://litellm.internal:4000") is None
+
+
+class TestLitellmEnvForwarding:
+ """_build_argv forwards LITELLM_* and rewrites the base URL for the container."""
+
+ def _make_runner(self) -> DockerRunner:
+ task = TaskDefinition(
+ task_id="t",
+ description="d",
+ initial_prompt="p",
+ sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig()),
+ success_criteria=[FileExistsCriterion(description="c", path="x.txt")],
+ )
+ rt = MagicMock()
+ rt.task = task
+ rt.run_dir = Path(tempfile.gettempdir()) / "test_run_litellm"
+ rt.task_file = None
+ return DockerRunner(rt)
+
+ def _build(self, runner: DockerRunner) -> list[str]:
+ with tempfile.TemporaryDirectory() as tmp:
+ input_dir = Path(tmp) / "in"
+ output_dir = Path(tmp) / "out"
+ input_dir.mkdir()
+ output_dir.mkdir()
+ return runner._build_argv(input_dir, output_dir, container_name="c")
+
+ def test_loopback_url_rewritten_and_host_published(self, monkeypatch):
+ monkeypatch.setenv("API_BACKEND", "litellm")
+ monkeypatch.setenv("LITELLM_BASE_URL", "http://localhost:4000")
+ monkeypatch.setenv("LITELLM_AUTH_TOKEN", "sk-master")
+ monkeypatch.setenv("LITELLM_MODEL", "zai.glm-5")
+ argv = self._build(self._make_runner())
+
+ # Base URL forwarded as an explicit rewritten value (safe: URL, not a secret).
+ assert "LITELLM_BASE_URL=http://host.docker.internal:4000" in argv
+ # The bare name-only form must NOT also be forwarded (it would carry localhost).
+ assert "LITELLM_BASE_URL" not in argv
+ # host alias published for Linux parity.
+ assert "--add-host" in argv
+ assert "host.docker.internal:host-gateway" in argv
+ # Credentials + model forwarded name-only (value copied by docker, stays out of argv).
+ assert "LITELLM_AUTH_TOKEN" in argv
+ assert "LITELLM_MODEL" in argv
+ assert "sk-master" not in " ".join(argv)
+
+ def test_non_loopback_url_forwarded_name_only(self, monkeypatch):
+ monkeypatch.setenv("API_BACKEND", "litellm")
+ monkeypatch.setenv("LITELLM_BASE_URL", "http://litellm.internal:4000")
+ monkeypatch.setenv("LITELLM_AUTH_TOKEN", "sk-master")
+ monkeypatch.setenv("LITELLM_MODEL", "zai.glm-5")
+ argv = self._build(self._make_runner())
+
+ # Forwarded name-only; value never rendered, no host alias needed.
+ assert argv.count("LITELLM_BASE_URL") == 1
+ assert "litellm.internal" not in " ".join(argv)
+ assert "host.docker.internal:host-gateway" not in argv
+
+ def test_no_network_skips_base_url(self, monkeypatch):
+ monkeypatch.setenv("API_BACKEND", "litellm")
+ monkeypatch.setenv("LITELLM_BASE_URL", "http://localhost:4000")
+ task = TaskDefinition(
+ task_id="t",
+ description="d",
+ initial_prompt="p",
+ sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(network="none")),
+ success_criteria=[FileExistsCriterion(description="c", path="x.txt")],
+ )
+ rt = MagicMock()
+ rt.task = task
+ rt.run_dir = Path(tempfile.gettempdir()) / "test_run_litellm_nonet"
+ rt.task_file = None
+ argv = self._build(DockerRunner(rt))
+
+ assert "LITELLM_BASE_URL" not in " ".join(argv)
+ assert "host.docker.internal:host-gateway" not in argv
diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py
new file mode 100644
index 00000000..4f9b3835
--- /dev/null
+++ b/tests/test_litellm_route.py
@@ -0,0 +1,441 @@
+"""Tests for the LiteLLM (Anthropic-compatible) open-weight backend (LiteLLMRoute).
+
+Covers route resolution, config validation, SDK env building, effective-model
+sync, pricing, and cost repricing. Mirrors the Bedrock equivalents in
+``tests/test_routing.py``.
+"""
+
+from __future__ import annotations
+
+import urllib.error
+import urllib.request
+from unittest.mock import MagicMock
+
+import pytest
+
+from coder_eval.agents.claude_code_agent import ClaudeCodeAgent
+from coder_eval.cli.run_command import _litellm_preflight_error
+from coder_eval.config import Settings
+from coder_eval.models import (
+ AgentKind,
+ BedrockRoute,
+ DirectRoute,
+ LiteLLMRoute,
+ TokenUsage,
+ parse_agent_config,
+)
+from coder_eval.models.enums import ApiBackend
+from coder_eval.models.routing import ROUTE_NAMES, resolve_evaluation_route, resolve_route
+from coder_eval.pricing import _normalize_model, calculate_cost
+
+
+def _make_agent(route, *, config_model: str | None = None) -> ClaudeCodeAgent:
+ return ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, model=config_model), route=route)
+
+
+class TestResolveEvaluationRoute:
+ """resolve_evaluation_route() pins the judge + simulated user to a constant
+ Claude backend regardless of the agent's backend, so grading/simulation stay
+ comparable across the models under test."""
+
+ @staticmethod
+ def _isolated_settings(monkeypatch, **kwargs):
+ # Skip .env and clear the credential env vars so presence/absence is
+ # driven purely by kwargs (config republishes .env into os.environ).
+ for var in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION", "ANTHROPIC_API_KEY"):
+ monkeypatch.delenv(var, raising=False)
+ return Settings(_env_file=None, **kwargs)
+
+ def test_bedrock_agent_route_is_reused_unchanged(self, monkeypatch):
+ route = BedrockRoute(bearer_token="tok", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6")
+ settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK)
+ assert resolve_evaluation_route(settings, route) is route
+
+ def test_direct_agent_route_is_reused_unchanged(self, monkeypatch):
+ route = DirectRoute(judge_transport="anthropic")
+ settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT)
+ assert resolve_evaluation_route(settings, route) is route
+
+ def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch):
+ agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5")
+ settings = self._isolated_settings(
+ monkeypatch,
+ api_backend=ApiBackend.LITELLM,
+ aws_bearer_token_bedrock="aws-tok",
+ aws_region="eu-north-1",
+ )
+ ev = resolve_evaluation_route(settings, agent)
+ assert isinstance(ev, BedrockRoute)
+ assert ev.bearer_token == "aws-tok"
+ assert ev.region == "eu-north-1"
+ # Judge + simulator run on a Claude model, region-qualified.
+ assert ev.model == "eu.anthropic.claude-sonnet-4-6"
+
+ def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch):
+ agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5")
+ settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM, anthropic_api_key="sk-ant")
+ ev = resolve_evaluation_route(settings, agent)
+ assert isinstance(ev, DirectRoute)
+ assert ev.judge_transport == "anthropic"
+
+ def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkeypatch):
+ # No Bedrock creds and no ANTHROPIC_API_KEY → DirectRoute(None), which makes
+ # llm_judge fail with its clean "unconfigured" error rather than scoring 0.0.
+ agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5")
+ settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM)
+ ev = resolve_evaluation_route(settings, agent)
+ assert isinstance(ev, DirectRoute)
+ assert ev.judge_transport is None
+
+
+class TestEvalRouteWiring:
+ """The orchestrator must hand the simulated user the eval_route (constant
+ Claude), never the agent's (possibly open-weight) route — guards the
+ simulation path the senior review flagged as untested."""
+
+ async def test_simulator_receives_eval_route_not_agent_route(self, monkeypatch):
+ from pathlib import Path
+ from types import SimpleNamespace
+
+ from coder_eval import orchestrator as orch_mod
+ from coder_eval.orchestrator import Orchestrator
+
+ eval_route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6")
+ agent_route = LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5")
+ captured: dict = {}
+
+ class _SpySimulator:
+ def __init__(self, **kwargs):
+ captured.update(kwargs)
+
+ async def start(self):
+ # Abort before the dialog loop; we only care which route was passed.
+ raise RuntimeError("__stop_dialog__")
+
+ monkeypatch.setattr(orch_mod, "UserSimulator", _SpySimulator)
+ fake = SimpleNamespace(
+ result=SimpleNamespace(simulation=None),
+ task=SimpleNamespace(simulation=object(), agent=object(), description="d"),
+ agent=object(),
+ success_checker=object(),
+ eval_route=eval_route,
+ route=agent_route,
+ )
+ with pytest.raises(RuntimeError, match="__stop_dialog__"):
+ await Orchestrator._simulation_dialog_loop(fake, initial_prompt="hi", sandbox_dir=Path("/tmp"))
+ # If someone reverts to route=self.route this flips to the litellm route.
+ assert captured["route"] is eval_route
+ assert captured["route"] is not agent_route
+
+
+class TestResolveRouteCustom:
+ """resolve_route() builds a LiteLLMRoute for the CUSTOM backend."""
+
+ def test_resolves_custom_route_with_all_fields(self):
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="http://localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="deepseek.v3.2",
+ )
+ route = resolve_route(settings)
+ assert isinstance(route, LiteLLMRoute)
+ assert route.base_url == "http://localhost:4000"
+ assert route.auth_token == "sk-master"
+ assert route.model == "deepseek.v3.2"
+
+ def test_small_model_falls_back_to_model(self):
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="http://localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="deepseek.v3.2",
+ litellm_small_model=None,
+ )
+ route = resolve_route(settings)
+ assert isinstance(route, LiteLLMRoute)
+ assert route.small_model == "deepseek.v3.2"
+ assert route.small_model == route.model
+
+ def test_explicit_small_model_wins(self):
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="http://localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="zai.glm-5",
+ litellm_small_model="deepseek.v3.2",
+ )
+ route = resolve_route(settings)
+ assert isinstance(route, LiteLLMRoute)
+ assert route.small_model == "deepseek.v3.2"
+
+ def test_model_passed_verbatim_no_inference_profile(self):
+ """The dotted Bedrock id must NOT get an eu./anthropic. prefix (unlike Bedrock)."""
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="http://localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="zai.glm-5",
+ )
+ route = resolve_route(settings)
+ assert isinstance(route, LiteLLMRoute)
+ assert route.model == "zai.glm-5"
+
+
+class TestRouteRegistration:
+ def test_route_names_covers_custom(self):
+ assert ROUTE_NAMES[LiteLLMRoute] == "litellm"
+
+ def test_importable_from_models(self):
+ from coder_eval.models import LiteLLMRoute as Imported
+
+ assert Imported is LiteLLMRoute
+
+
+class TestValidateApiKeysCustom:
+ """validate_api_keys() fails fast on missing custom settings."""
+
+ def test_missing_base_url_raises_naming_it(self):
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url=None,
+ litellm_auth_token="sk-master",
+ litellm_model="zai.glm-5",
+ )
+ with pytest.raises(ValueError, match="LITELLM_BASE_URL"):
+ settings.validate_api_keys("claude-code")
+
+ def test_scheme_less_base_url_raises_naming_it(self):
+ # "localhost:4000" has no http(s) scheme — must fail at validation with a
+ # field-named message, not later as a raw urlopen ValueError.
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="zai.glm-5",
+ )
+ with pytest.raises(ValueError, match="LITELLM_BASE_URL must be an http"):
+ settings.validate_api_keys("claude-code")
+
+ def test_all_present_does_not_raise(self):
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url="http://localhost:4000",
+ litellm_auth_token="sk-master",
+ litellm_model="zai.glm-5",
+ )
+ settings.validate_api_keys("claude-code") # no raise
+
+ def test_none_agent_skips_custom_validation(self):
+ """The no-op agent needs no backend creds — validation must be skipped."""
+ settings = Settings(
+ api_backend=ApiBackend.LITELLM,
+ litellm_base_url=None,
+ litellm_auth_token=None,
+ litellm_model=None,
+ )
+ settings.validate_api_keys("none") # no raise
+
+
+class TestBuildSdkEnvCustom:
+ """_build_sdk_env() for the LiteLLM route."""
+
+ def test_custom_route_env_has_anthropic_vars_only(self):
+ route = LiteLLMRoute(
+ base_url="http://x:4000",
+ auth_token="sk-1",
+ model="deepseek.v3.2",
+ small_model="deepseek.v3.2",
+ )
+ env, model = ClaudeCodeAgent._build_sdk_env(route)
+ assert env["ANTHROPIC_BASE_URL"] == "http://x:4000"
+ assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-1"
+ assert env["ANTHROPIC_MODEL"] == "deepseek.v3.2"
+ assert env["ANTHROPIC_SMALL_FAST_MODEL"] == "deepseek.v3.2"
+ assert model == "deepseek.v3.2"
+ # Inherited Bedrock creds are neutralized (blanked to ""), not merely
+ # absent, so the CLI can't auto-select Bedrock-direct and bypass the proxy.
+ assert env["CLAUDE_CODE_USE_BEDROCK"] == ""
+ assert env["AWS_BEARER_TOKEN_BEDROCK"] == ""
+ assert "AWS_REGION" not in env
+
+ def test_custom_route_no_model_omits_model_vars(self):
+ route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")
+ env, model = ClaudeCodeAgent._build_sdk_env(route)
+ assert model is None
+ assert "ANTHROPIC_MODEL" not in env
+ assert "ANTHROPIC_SMALL_FAST_MODEL" not in env
+
+ def test_custom_route_forwards_path(self, monkeypatch):
+ import os
+
+ custom_path = f"/custom/bin{os.pathsep}/usr/bin"
+ monkeypatch.setenv("PATH", custom_path)
+ env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1"))
+ assert env["PATH"] == custom_path
+
+ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch):
+ """The SDK merges os.environ, so a stray x-api-key must be overridden to
+ empty in options.env (not merely omitted) — else it would fight the
+ bearer auth_token against the gateway."""
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key")
+ env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1"))
+ assert env["ANTHROPIC_API_KEY"] == ""
+
+
+class TestResolveEffectiveModelCustom:
+ """_resolve_effective_model() on the LiteLLM route — no prefixing."""
+
+ def test_config_model_synced_verbatim(self):
+ route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2")
+ env, route_model = ClaudeCodeAgent._build_sdk_env(route)
+ agent = _make_agent(route, config_model="zai.glm-5")
+ effective = agent._resolve_effective_model("zai.glm-5", env, route_model)
+ assert effective == "zai.glm-5"
+ assert env["ANTHROPIC_MODEL"] == "zai.glm-5" # no eu./anthropic. prefix
+
+ def test_route_model_used_when_config_none(self):
+ route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2")
+ env, route_model = ClaudeCodeAgent._build_sdk_env(route)
+ agent = _make_agent(route)
+ effective = agent._resolve_effective_model(None, env, route_model)
+ assert effective == "deepseek.v3.2"
+
+ def test_both_none_returns_none(self):
+ route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")
+ env, route_model = ClaudeCodeAgent._build_sdk_env(route)
+ agent = _make_agent(route)
+ effective = agent._resolve_effective_model(None, env, route_model)
+ assert effective is None
+ assert "ANTHROPIC_MODEL" not in env
+
+
+class TestOpenWeightPricing:
+ """Pricing for the open-weight models + LiteLLM/Bedrock prefix normalization."""
+
+ def test_glm5_rate(self):
+ # 1M input + 1M output → input_per_mtok + output_per_mtok.
+ assert calculate_cost("zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.2 + 3.84)
+
+ def test_deepseek_rate(self):
+ assert calculate_cost("deepseek.v3.2", 1_000_000, 1_000_000) == pytest.approx(0.74 + 2.22)
+
+ def test_kimi_rate(self):
+ assert calculate_cost("moonshotai.kimi-k2.5", 1_000_000, 1_000_000) == pytest.approx(0.72 + 3.6)
+
+ def test_kimi_converse_prefixed_prices_same(self):
+ assert calculate_cost("converse/moonshotai.kimi-k2.5", 1_000_000, 1_000_000) == pytest.approx(0.72 + 3.6)
+
+ def test_normalize_strips_converse_prefix(self):
+ assert _normalize_model("converse/zai.glm-5") == "zai.glm-5"
+ assert _normalize_model("bedrock/converse/deepseek.v3.2") == "deepseek.v3.2"
+
+ def test_normalize_identity_on_bare_ids(self):
+ assert _normalize_model("zai.glm-5") == "zai.glm-5"
+ assert _normalize_model("deepseek.v3.2") == "deepseek.v3.2"
+
+ def test_converse_prefixed_id_prices_same(self):
+ """The SDK reports model_used as e.g. 'converse/zai.glm-5' — must still price."""
+ assert calculate_cost("converse/zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.2 + 3.84)
+
+
+class TestRepriceForLitellm:
+ """The litellm backend recomputes cost from tokens, overriding the SDK estimate."""
+
+ def test_overrides_wrong_sdk_cost_with_real_rate(self):
+ u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=3.68)
+ ClaudeCodeAgent._reprice_for_litellm(u, "zai.glm-5")
+ assert u.total_cost_usd == pytest.approx(1.2 + 3.84)
+
+ def test_converse_prefixed_model_reprices(self):
+ u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=99.0)
+ ClaudeCodeAgent._reprice_for_litellm(u, "converse/zai.glm-5")
+ assert u.total_cost_usd == pytest.approx(1.2 + 3.84)
+
+ def test_unpriced_model_yields_none_not_sdk_figure(self):
+ """An unknown model must show N/A (None), never the misleading SDK cost."""
+ u = TokenUsage(uncached_input_tokens=100, output_tokens=100, total_cost_usd=9.99)
+ ClaudeCodeAgent._reprice_for_litellm(u, "some-unknown-model")
+ assert u.total_cost_usd is None
+
+ def test_none_model_yields_none(self):
+ u = TokenUsage(uncached_input_tokens=100, output_tokens=100, total_cost_usd=9.99)
+ ClaudeCodeAgent._reprice_for_litellm(u, None)
+ assert u.total_cost_usd is None
+
+
+class TestRepriceWiring:
+ """The finalize path (not just the static helper) reprices a litellm turn. A
+ regression that skips the reprice would silently persist the SDK's Claude cost
+ and disable the max_usd gate — the static-only tests above wouldn't catch it."""
+
+ def _usage_after_finalize(self, effective_model: str | None) -> TokenUsage:
+ from types import SimpleNamespace
+
+ from coder_eval.agents.claude_code_agent import _ClaudeTurnState
+
+ agent = _make_agent(
+ LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5"),
+ config_model="zai.glm-5",
+ )
+ stub = SimpleNamespace(
+ _agent=agent,
+ sdk_messages=[],
+ sdk_result_usage=None,
+ sdk_result_cost=None,
+ # model_usage carries the SDK's Claude-priced estimate (3.68); the
+ # reprice must override it from the litellm rate table.
+ sdk_result_model_usage={"m": {"inputTokens": 1_000_000, "outputTokens": 1_000_000, "costUSD": 3.68}},
+ effective_model=effective_model,
+ )
+ return _ClaudeTurnState._finalize_token_usage(stub) # type: ignore[arg-type]
+
+ def test_finalize_reprices_priced_litellm_model(self):
+ usage = self._usage_after_finalize("zai.glm-5")
+ # litellm rate (1.2 + 3.84), NOT the SDK's Claude estimate of 3.68.
+ assert usage.total_cost_usd == pytest.approx(1.2 + 3.84)
+ assert usage.uncached_input_tokens == 1_000_000 # token buckets untouched
+
+ def test_finalize_unpriced_litellm_model_yields_none(self):
+ # An unpriced model → None (not the misleading 3.68), so the orchestrator
+ # skips the max_usd gate rather than gating on a wrong figure.
+ usage = self._usage_after_finalize("some-unpriced-model")
+ assert usage.total_cost_usd is None
+
+
+class TestLitellmPreflight:
+ """External-proxy reachability preflight — fail fast instead of hanging on a dead proxy."""
+
+ def test_none_for_non_litellm_backend(self):
+ s = Settings(api_backend=ApiBackend.BEDROCK, litellm_base_url="http://x:4000")
+ assert _litellm_preflight_error(s) is None
+
+ def test_none_when_no_base_url(self):
+ s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url=None, litellm_model="m")
+ assert _litellm_preflight_error(s) is None
+
+ def test_scheme_less_base_url_returns_clean_error(self):
+ # Regression: a scheme-less URL used to make urlopen raise a bare
+ # ValueError that escaped as a traceback. Now it returns a clean message.
+ s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="localhost:4000", litellm_model="m")
+ err = _litellm_preflight_error(s)
+ assert err is not None and "http(s)" in err
+
+ def test_error_when_proxy_down(self, monkeypatch):
+ monkeypatch.setattr(urllib.request, "urlopen", MagicMock(side_effect=urllib.error.URLError("refused")))
+ s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:9", litellm_model="m")
+ err = _litellm_preflight_error(s)
+ assert err is not None
+ assert "not reachable" in err and "http://127.0.0.1:9" in err
+
+ def test_none_when_reachable(self, monkeypatch):
+ monkeypatch.setattr(urllib.request, "urlopen", MagicMock(return_value=MagicMock()))
+ s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:4000", litellm_model="m")
+ assert _litellm_preflight_error(s) is None
+
+ def test_none_on_http_error_means_server_up(self, monkeypatch):
+ monkeypatch.setattr(
+ urllib.request, "urlopen", MagicMock(side_effect=urllib.error.HTTPError("u", 404, "nf", {}, None))
+ )
+ s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:4000", litellm_model="m")
+ assert _litellm_preflight_error(s) is None
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index 6b3f25e8..be8de5e9 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -12,6 +12,7 @@
ClaudeCodeAgentConfig,
DirectRoute,
FileExistsCriterion,
+ LiteLLMRoute,
PreservationMode,
SandboxConfig,
TaskDefinition,
@@ -47,6 +48,22 @@ def test_format_routing_non_direct_routes_unchanged():
assert _format_routing(BedrockRoute(bearer_token="t", region="us-east-1")) == "aws_bedrock"
+def test_format_routing_litellm_shows_model():
+ out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5"))
+ assert out.startswith("litellm")
+ assert "zai.glm-5" in out
+
+
+def test_format_routing_litellm_effective_model_wins_over_route_default():
+ """The --model override (effective_model) must be logged, not the route's LITELLM_MODEL default."""
+ out = _format_routing(
+ LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5"),
+ effective_model="deepseek.v3.2",
+ )
+ assert "deepseek.v3.2" in out
+ assert "zai.glm-5" not in out
+
+
def _make_orchestrator_with_route(tmp_path: Path, route) -> Orchestrator:
"""Build a minimal Orchestrator pre-populated with a route + EvaluationResult.
@@ -98,6 +115,24 @@ def test_record_route_environment_info_bedrock(tmp_path):
assert info["api_routing"] == "aws_bedrock"
assert info["aws_region"] == "eu-north-1"
assert info["bedrock_model"] == "eu.anthropic.claude-sonnet-4-6"
+
+
+def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path):
+ """LiteLLM route records host + model, but NEVER the auth token or full base_url."""
+ orchestrator = _make_orchestrator_with_route(
+ tmp_path,
+ LiteLLMRoute(base_url="http://localhost:4000", auth_token="sk-super-secret", model="zai.glm-5"),
+ )
+ orchestrator._record_route_environment_info()
+ assert orchestrator.result is not None
+ info = orchestrator.result.environment_info
+ assert info["api_routing"] == "litellm"
+ assert info["litellm_base_url_host"] == "localhost"
+ assert info["litellm_model"] == "zai.glm-5"
+ # No secret and no full URL anywhere in the recorded audit dict.
+ blob = str(info)
+ assert "sk-super-secret" not in blob
+ assert "http://localhost:4000" not in blob
assert "judge_transport" not in info # Direct-only field
diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py
new file mode 100644
index 00000000..e1b41f19
--- /dev/null
+++ b/tests/test_route_seam_exhaustiveness.py
@@ -0,0 +1,87 @@
+"""Guardrail: every ``ApiRoute`` member must be handled at every route-matching seam.
+
+The route union is matched via ``match``/``isinstance`` at several seams:
+``_build_sdk_env``, ``_format_routing``, ``ROUTE_NAMES``, the ``llm_judge``
+dispatch (``_invoke_tool_channel``), and ``Orchestrator._record_route_environment_info``.
+A new route added without extending each seam would silently no-op
+(``_record_route_environment_info``), silently mis-score (the judge dispatch), or
+raise (``_build_sdk_env``). This test iterates the union and forces a fixture +
+per-seam check for every member, so adding a 4th route fails here until each seam
+is extended. (``_record_route_environment_info`` uses ``if/elif isinstance`` — not
+a ``match`` — so pyright does NOT flag a missing route there; this runtime test is
+its only guard.)
+"""
+
+from __future__ import annotations
+
+import typing
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+from coder_eval.agents.claude_code_agent import ClaudeCodeAgent
+from coder_eval.criteria import llm_judge
+from coder_eval.criteria.llm_judge import _invoke_tool_channel
+from coder_eval.models import ROUTE_NAMES, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute
+from coder_eval.orchestrator import Orchestrator, _format_routing
+
+
+# One minimal instance per ApiRoute member. The set-equality assertion below
+# forces this dict to grow whenever the union does.
+_INSTANCES: list[object] = [
+ DirectRoute(),
+ BedrockRoute(bearer_token="t", region="eu-north-1", model="x"),
+ LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="m"),
+]
+
+
+def test_fixtures_cover_every_union_member():
+ """Fails when a new ApiRoute member is added without a fixture here."""
+ assert {type(r) for r in _INSTANCES} == set(typing.get_args(ApiRoute))
+
+
+def test_every_route_has_a_route_name():
+ for r in _INSTANCES:
+ assert type(r) in ROUTE_NAMES
+
+
+def test_build_sdk_env_handles_every_route():
+ """_build_sdk_env raises AssertionError on an unhandled route — must not for any member."""
+ for r in _INSTANCES:
+ env, _model = ClaudeCodeAgent._build_sdk_env(r) # type: ignore[arg-type]
+ assert isinstance(env, dict)
+
+
+def test_format_routing_handles_every_route():
+ """No silent no-op: each route formats to a non-empty string led by its ROUTE_NAMES value."""
+ for r in _INSTANCES:
+ out = _format_routing(r) # type: ignore[arg-type]
+ assert out and out.startswith(ROUTE_NAMES[type(r)])
+
+
+def test_invoke_tool_channel_handles_every_route(monkeypatch):
+ """The llm_judge dispatch must handle every route type — an unhandled member
+ would fall past all cases and raise UnboundLocalError. Network-touching arms
+ (Bedrock/Direct) are stubbed so this only checks dispatch coverage."""
+ monkeypatch.setattr(llm_judge, "invoke_bedrock_judge", lambda **_: {})
+ monkeypatch.setattr(llm_judge, "invoke_anthropic_judge", lambda **_: {})
+ monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub"))
+ monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp: None)
+ criterion = MagicMock()
+ for r in _INSTANCES:
+ result = _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u") # type: ignore[arg-type]
+ # (verdict, parse_error, raw_text, response_usage) — a 4-tuple means the
+ # route matched an explicit arm rather than falling through.
+ assert isinstance(result, tuple) and len(result) == 4
+
+
+def test_record_route_environment_info_handles_every_route():
+ """Orchestrator._record_route_environment_info records a route-specific
+ dimension for every route (not just the generic api_routing key). A new route
+ missing from its if/elif would record only api_routing — caught here since
+ pyright can't check the isinstance chain."""
+ for r in _INSTANCES:
+ fake = SimpleNamespace(route=r, eval_route=r, result=SimpleNamespace(environment_info={}), agent=None)
+ Orchestrator._record_route_environment_info(fake) # type: ignore[arg-type]
+ env = fake.result.environment_info
+ assert env.get("api_routing") == ROUTE_NAMES[type(r)]
+ assert len(env) > 1, f"{type(r).__name__} recorded no route-specific env info"