From 96d03c26a9cb1f217562c76ebc09e74f7e27300a Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 21 Jul 2026 14:34:42 +0100 Subject: [PATCH 01/23] =?UTF-8?q?feat(routing):=20add=20LiteLLM=20backend?= =?UTF-8?q?=20=E2=80=94=20skill=20activation=20works=20on=20open-weight=20?= =?UTF-8?q?models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a LiteLLMRoute (ApiBackend.LITELLM) so the Claude Code SDK can drive EU-resident Bedrock open-weight models (GLM 5, DeepSeek V3.2) through a self-hosted LiteLLM proxy that translates Anthropic Messages <-> Bedrock Converse. Because the model runs inside the native Claude Code harness, the Skill tool and progressive disclosure fire unchanged — so UiPath skill activation works on a non-Claude model via this path. Verified end-to-end: the LiteLLM integration works AND drives skill activations. Tested on a real uipath maestro_flow skill task (cli_dice_roller) using GLM 5 — 4/4 criteria passed (api_routing=litellm, model=zai.glm-5). The agent engaged the uipath-maestro-flow skill and used `uip maestro flow node add` / `edge add` / `validate` correctly, producing a schema-valid flow. - ApiBackend.LITELLM + LiteLLMRoute frozen dataclass in the ApiRoute union - litellm_* settings + fail-fast _validate_litellm_settings (config.py) - SDK env wiring in ClaudeCodeAgent (_build_sdk_env / _resolve_effective_model): sets ANTHROPIC_BASE_URL/AUTH_TOKEN/MODEL, neutralizes inherited ANTHROPIC_API_KEY - docker/litellm-config.yaml: GLM 5 / DeepSeek V3.2 -> Bedrock Converse @ eu-north-1 - tests/test_litellm_route.py Co-Authored-By: Claude Opus 4.8 --- docker/litellm-config.yaml | 40 +++++ src/coder_eval/agents/claude_code_agent.py | 28 +++ src/coder_eval/config.py | 35 ++++ src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/enums.py | 1 + src/coder_eval/models/routing.py | 39 +++- tests/test_litellm_route.py | 197 +++++++++++++++++++++ 7 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 docker/litellm-config.yaml create mode 100644 tests/test_litellm_route.py diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml new file mode 100644 index 00000000..d4fd6546 --- /dev/null +++ b/docker/litellm-config.yaml @@ -0,0 +1,40 @@ +# LiteLLM proxy config for the coder_eval `custom` (open-weight) backend. +# +# Translates Anthropic Messages (what the Claude Code SDK speaks) <-> Bedrock +# Converse (what the non-Claude open-weight models speak), so the native `Skill` +# tool and UiPath skills fire unchanged. Verified 2026-07-21: HTTP 200 on both +# models via POST /v1/messages, and tool_use blocks survive the translation. +# +# Shared by both proxy modes: +# - Phase 1 — operator starts the proxy manually and points coder_eval at it +# via LITELLM_BASE_URL (no coder_eval-owned lifecycle). +# - Phase 1.5 — coder_eval autostarts the proxy against this same file, or an +# always-on docker sidecar mounts it. +# +# Required env in the proxy's environment (NOT committed): +# AWS_BEARER_TOKEN_BEDROCK — Bedrock bearer token (LiteLLM forwards it). +# AWS_REGION=eu-north-1 — EU residency (Stockholm). +# LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. +# +# Run manually (Phase 1): +# uvx --from 'litellm[proxy]' litellm --config docker/litellm-config.yaml --port 4000 + +model_list: + # DeepSeek V3.2 — Phase-1 cost lead ($0.74 / $2.22 per Mtok). + - model_name: deepseek.v3.2 + litellm_params: + model: bedrock/converse/deepseek.v3.2 + aws_region_name: eu-north-1 + # GLM 5 — Phase-1 quality lead ($1.55 / $4.96 per Mtok). + - model_name: zai.glm-5 + litellm_params: + model: bedrock/converse/zai.glm-5 + aws_region_name: eu-north-1 + +litellm_settings: + # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400. + drop_params: true + +general_settings: + # Read the virtual key from the environment — never hardcode a secret here. + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index cd24451a..3a1a1664 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -45,6 +45,7 @@ CommandTelemetry, ContentBlock, DirectRoute, + LiteLLMRoute, ResultSummary, TokenUsage, TranscriptMessage, @@ -763,6 +764,26 @@ def _build_sdk_env( case DirectRoute(): return base_env, None + case LiteLLMRoute() as cr: + # Point the SDK at the custom Anthropic-compatible endpoint (e.g. + # a LiteLLM gateway). These override any inherited value: the SDK + # merges {**os.environ, ..., **options.env} at spawn, so setting + # them here wins over the parent environment. + env = { + "ANTHROPIC_BASE_URL": cr.base_url, + "ANTHROPIC_AUTH_TOKEN": cr.auth_token, + # Neutralize any inherited ANTHROPIC_API_KEY: auth on this + # route is the bearer ANTHROPIC_AUTH_TOKEN, and a stray + # x-api-key (e.g. a real Anthropic key exported from .env) + # would conflict with the gateway's key auth. + "ANTHROPIC_API_KEY": "", + } + if cr.model: + env["ANTHROPIC_MODEL"] = cr.model + if cr.small_model: + env["ANTHROPIC_SMALL_FAST_MODEL"] = cr.small_model + return {**base_env, **env}, cr.model + raise AssertionError(f"Unhandled route type: {type(route).__name__}") def _resolve_effective_model( @@ -784,6 +805,13 @@ def _resolve_effective_model( if effective: env["ANTHROPIC_MODEL"] = effective return effective + if isinstance(self.route, LiteLLMRoute): + # Same env-sync as Bedrock, but pass the id verbatim (no + # inference-profile qualification — the gateway maps it). + effective = config_model or route_model + if effective: + env["ANTHROPIC_MODEL"] = effective + return effective return config_model or route_model async def communicate( diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py index 9111634a..cb51193b 100644 --- a/src/coder_eval/config.py +++ b/src/coder_eval/config.py @@ -103,6 +103,17 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: bedrock_model: str | None = None # Cross-region model ID bedrock_small_model: str | None = None # Cross-region small model ID + # LiteLLM (Anthropic-compatible) endpoint settings (used when api_backend == "litellm"). + # These map to ANTHROPIC_BASE_URL / ANTHROPIC_AUTH_TOKEN / ANTHROPIC_MODEL / + # ANTHROPIC_SMALL_FAST_MODEL, but ONLY inside the SDK subprocess env (see + # ClaudeCodeAgent._build_sdk_env). They are deliberately NOT named anthropic_* + # so the os.environ export loop below can't leak ANTHROPIC_BASE_URL process-wide + # (which would silently redirect the judge's in-process Anthropic() client). + litellm_base_url: str | None = None + litellm_auth_token: str | None = None + litellm_model: str | None = None + litellm_small_model: str | None = None + # Codex settings (CodexAgent). CODEX_MODEL is the fallback model/deployment # used when a task doesn't pin agent.model; CODEX_BASE_URL routes to a custom # OpenAI-/responses-compatible endpoint (incl. Azure OpenAI). For Azure also @@ -169,6 +180,27 @@ def _validate_bedrock_settings(self) -> None: + " Please set them in your .env file." ) + def _validate_litellm_settings(self) -> None: + """Validate that required custom Anthropic-endpoint settings are present. + + Raises: + ValueError: If required custom settings are missing + """ + missing = [] + if not self.litellm_base_url: + missing.append("LITELLM_BASE_URL") + if not self.litellm_auth_token: + missing.append("LITELLM_AUTH_TOKEN") + # LITELLM_MODEL is required for the same reason BEDROCK_MODEL is: a None + # model sent to the SDK/gateway yields an opaque 400. Fail fast instead. + if not self.litellm_model: + missing.append("LITELLM_MODEL") + if missing: + raise ValueError( + f"LiteLLM-endpoint routing is enabled but missing required settings: {', '.join(missing)}." + + " Please set them in your .env file." + ) + def validate_api_keys(self, agent_type: str) -> None: """Validate that required API keys are present. @@ -186,6 +218,9 @@ def validate_api_keys(self, agent_type: str) -> None: if self.api_backend == ApiBackend.BEDROCK: self._validate_bedrock_settings() + if self.api_backend == ApiBackend.LITELLM: + self._validate_litellm_settings() + # Claude Code agent can use either: # 1. ANTHROPIC_API_KEY environment variable # 2. Cached CLI authentication from 'claude-code login' (subscription account) diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 82354063..7a6b2a07 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -136,6 +136,7 @@ BedrockRoute, DirectRoute, JudgeTransport, + LiteLLMRoute, resolve_route, to_bedrock_inference_profile, ) @@ -229,6 +230,7 @@ "ApiRoute", "DirectRoute", "BedrockRoute", + "LiteLLMRoute", "JudgeTransport", "resolve_route", "to_bedrock_inference_profile", diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 36173746..03afe885 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -67,6 +67,7 @@ class ApiBackend(StrEnum): DIRECT = "direct" # Anthropic API directly (ANTHROPIC_API_KEY) BEDROCK = "bedrock" # AWS Bedrock (bearer token auth) + LITELLM = "litellm" # LiteLLM (Anthropic-compatible) endpoint, e.g. LiteLLM -> Bedrock class PermissionMode(StrEnum): diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index f1a90687..6e7512e7 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -101,24 +101,42 @@ class BedrockRoute: disable_attribution_header: bool = True -ApiRoute = DirectRoute | BedrockRoute +@dataclass(frozen=True) +class LiteLLMRoute: + """Route through a custom Anthropic-compatible endpoint (e.g. a LiteLLM + gateway fronting Bedrock open-weight models). + + The Claude Code SDK is pointed at ``base_url`` via ``ANTHROPIC_BASE_URL`` and + authenticates with ``auth_token`` via ``ANTHROPIC_AUTH_TOKEN`` (bearer). The + ``model``/``small_model`` ids are passed **verbatim** (no Bedrock + inference-profile qualification) — the gateway maps them to its backend. + """ + + base_url: str + auth_token: str + model: str | None = None + small_model: str | None = None + + +ApiRoute = DirectRoute | BedrockRoute | LiteLLMRoute # Stable string names for environment_info recording (decoupled from class names) ROUTE_NAMES: dict[type, str] = { DirectRoute: "anthropic_direct", BedrockRoute: "aws_bedrock", + LiteLLMRoute: "litellm", } def resolve_route(settings: Settings) -> ApiRoute: """Resolve an ``ApiRoute`` from static settings. - Handles the two supported backends (``DIRECT`` and ``BEDROCK``), whose - route is fully determined by ``Settings``. + Handles the three supported backends (``DIRECT``, ``BEDROCK``, ``LITELLM``), + whose route is fully determined by ``Settings``. Called after ``validate_api_keys()`` has verified credentials. Uses - ``assert`` for type narrowing (not ``ValueError``) since the Bedrock + ``assert`` for type narrowing (not ``ValueError``) since the Bedrock/custom credential checks are an internal contract. """ match settings.api_backend: @@ -145,6 +163,19 @@ def resolve_route(settings: Settings) -> ApiRoute: ) case ApiBackend.DIRECT: return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) + case ApiBackend.LITELLM: + # base_url/auth_token are guaranteed by validate_api_keys (run first); + # asserts narrow the Optional types, mirroring the Bedrock arm. + assert settings.litellm_base_url is not None, "LiteLLM backend requires litellm_base_url" + assert settings.litellm_auth_token is not None, "LiteLLM backend requires litellm_auth_token" + # No inference-profile qualification: the id is passed verbatim to the gateway. + small_model = settings.litellm_small_model or settings.litellm_model + return LiteLLMRoute( + base_url=settings.litellm_base_url, + auth_token=settings.litellm_auth_token, + model=settings.litellm_model, + small_model=small_model, + ) def _resolve_direct_judge_transport(settings: Settings) -> JudgeTransport | None: diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py new file mode 100644 index 00000000..9d170c7c --- /dev/null +++ b/tests/test_litellm_route.py @@ -0,0 +1,197 @@ +"""Tests for the custom Anthropic-compatible backend (LiteLLMRoute). + +Covers Phase 1 (route resolution + config validation) and Phase 2 (SDK env +building + effective-model sync) of the open-weight support plan. Mirrors the +Bedrock equivalents in ``tests/test_routing.py``. +""" + +from __future__ import annotations + +import pytest + +from coder_eval.agents.claude_code_agent import ClaudeCodeAgent +from coder_eval.config import Settings +from coder_eval.models import ( + AgentKind, + LiteLLMRoute, + parse_agent_config, +) +from coder_eval.models.enums import ApiBackend +from coder_eval.models.routing import ROUTE_NAMES, resolve_route + + +def _make_agent(route, *, config_model: str | None = None) -> ClaudeCodeAgent: + return ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, model=config_model), route=route) + + +class TestResolveRouteCustom: + """resolve_route() builds a LiteLLMRoute for the CUSTOM backend.""" + + def test_resolves_custom_route_with_all_fields(self): + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="http://localhost:4000", + litellm_auth_token="sk-master", + litellm_model="deepseek.v3.2", + ) + route = resolve_route(settings) + assert isinstance(route, LiteLLMRoute) + assert route.base_url == "http://localhost:4000" + assert route.auth_token == "sk-master" + assert route.model == "deepseek.v3.2" + + def test_small_model_falls_back_to_model(self): + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="http://localhost:4000", + litellm_auth_token="sk-master", + litellm_model="deepseek.v3.2", + litellm_small_model=None, + ) + route = resolve_route(settings) + assert isinstance(route, LiteLLMRoute) + assert route.small_model == "deepseek.v3.2" + assert route.small_model == route.model + + def test_explicit_small_model_wins(self): + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="http://localhost:4000", + litellm_auth_token="sk-master", + litellm_model="zai.glm-5", + litellm_small_model="deepseek.v3.2", + ) + route = resolve_route(settings) + assert isinstance(route, LiteLLMRoute) + assert route.small_model == "deepseek.v3.2" + + def test_model_passed_verbatim_no_inference_profile(self): + """The dotted Bedrock id must NOT get an eu./anthropic. prefix (unlike Bedrock).""" + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="http://localhost:4000", + litellm_auth_token="sk-master", + litellm_model="zai.glm-5", + ) + route = resolve_route(settings) + assert isinstance(route, LiteLLMRoute) + assert route.model == "zai.glm-5" + + +class TestRouteRegistration: + def test_route_names_covers_custom(self): + assert ROUTE_NAMES[LiteLLMRoute] == "litellm" + + def test_importable_from_models(self): + from coder_eval.models import LiteLLMRoute as Imported + + assert Imported is LiteLLMRoute + + +class TestValidateApiKeysCustom: + """validate_api_keys() fails fast on missing custom settings.""" + + def test_missing_base_url_raises_naming_it(self): + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url=None, + litellm_auth_token="sk-master", + litellm_model="zai.glm-5", + ) + with pytest.raises(ValueError, match="LITELLM_BASE_URL"): + settings.validate_api_keys("claude-code") + + def test_all_present_does_not_raise(self): + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="http://localhost:4000", + litellm_auth_token="sk-master", + litellm_model="zai.glm-5", + ) + settings.validate_api_keys("claude-code") # no raise + + def test_none_agent_skips_custom_validation(self): + """The no-op agent needs no backend creds — validation must be skipped.""" + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url=None, + litellm_auth_token=None, + litellm_model=None, + ) + settings.validate_api_keys("none") # no raise + + +class TestBuildSdkEnvCustom: + """Phase 2: _build_sdk_env() for the custom route.""" + + def test_custom_route_env_has_anthropic_vars_only(self): + route = LiteLLMRoute( + base_url="http://x:4000", + auth_token="sk-1", + model="deepseek.v3.2", + small_model="deepseek.v3.2", + ) + env, model = ClaudeCodeAgent._build_sdk_env(route) + assert env["ANTHROPIC_BASE_URL"] == "http://x:4000" + assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-1" + assert env["ANTHROPIC_MODEL"] == "deepseek.v3.2" + assert env["ANTHROPIC_SMALL_FAST_MODEL"] == "deepseek.v3.2" + assert model == "deepseek.v3.2" + # No Bedrock/Direct-specific vars. + assert "CLAUDE_CODE_USE_BEDROCK" not in env + assert "AWS_BEARER_TOKEN_BEDROCK" not in env + assert "AWS_REGION" not in env + + def test_custom_route_no_model_omits_model_vars(self): + route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + env, model = ClaudeCodeAgent._build_sdk_env(route) + assert model is None + assert "ANTHROPIC_MODEL" not in env + assert "ANTHROPIC_SMALL_FAST_MODEL" not in env + + def test_custom_route_forwards_path(self, monkeypatch): + import os + + custom_path = f"/custom/bin{os.pathsep}/usr/bin" + monkeypatch.setenv("PATH", custom_path) + env, _ = ClaudeCodeAgent._build_sdk_env( + LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + ) + assert env["PATH"] == custom_path + + def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch): + """The SDK merges os.environ, so a stray x-api-key must be overridden to + empty in options.env (not merely omitted) — else it would fight the + bearer auth_token against the gateway.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key") + env, _ = ClaudeCodeAgent._build_sdk_env( + LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + ) + assert env["ANTHROPIC_API_KEY"] == "" + + +class TestResolveEffectiveModelCustom: + """Phase 2: _resolve_effective_model() on the custom route — no prefixing.""" + + def test_config_model_synced_verbatim(self): + route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + env, route_model = ClaudeCodeAgent._build_sdk_env(route) + agent = _make_agent(route, config_model="zai.glm-5") + effective = agent._resolve_effective_model("zai.glm-5", env, route_model) + assert effective == "zai.glm-5" + assert env["ANTHROPIC_MODEL"] == "zai.glm-5" # no eu./anthropic. prefix + + def test_route_model_used_when_config_none(self): + route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") + env, route_model = ClaudeCodeAgent._build_sdk_env(route) + agent = _make_agent(route) + effective = agent._resolve_effective_model(None, env, route_model) + assert effective == "deepseek.v3.2" + + def test_both_none_returns_none(self): + route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") + env, route_model = ClaudeCodeAgent._build_sdk_env(route) + agent = _make_agent(route) + effective = agent._resolve_effective_model(None, env, route_model) + assert effective is None + assert "ANTHROPIC_MODEL" not in env From 49cb7dc3a8ab287ea91945d1b0cf9f776c60f0df Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 21 Jul 2026 15:10:04 +0100 Subject: [PATCH 02/23] feat(routing): --backend litellm, open-weight pricing, and correct cost accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observability + CLI: - `--backend litellm` flag (alias for API_BACKEND=litellm) - Record the LiteLLM route in run artifacts (host + model only — never the auth token or full base_url) and in the routing log line - Guardrail test: every ApiRoute member must be handled at each route-matching seam (ROUTE_NAMES / _build_sdk_env / _format_routing) Pricing + cost: - Add rates for zai.glm-5 ($1.55/$4.96) and deepseek.v3.2 ($0.74/$2.22) - _normalize_model strips LiteLLM/Bedrock routing prefixes (converse/, bedrock/converse/) so SDK-reported model ids resolve to a rate - On the litellm backend, recompute total_cost_usd from the token buckets at the model's real rate: the Claude SDK's costUSD assumes Claude pricing and is wrong for open-weight models. Token buckets are untouched (the per-message reconciliation invariant is preserved); an unpriced model yields None (an honest N/A) instead of the misleading SDK figure. Also drops internal plan-phase references from the shared LiteLLM config comments. Co-Authored-By: Claude Opus 4.8 --- docker/litellm-config.yaml | 18 +++---- src/coder_eval/agents/claude_code_agent.py | 31 +++++++++++ src/coder_eval/cli/run_command.py | 2 +- src/coder_eval/orchestrator.py | 10 ++++ src/coder_eval/pricing.py | 12 +++++ tests/test_cli_backend_flag.py | 2 +- tests/test_litellm_route.py | 62 +++++++++++++++++++--- tests/test_orchestrator.py | 25 +++++++++ tests/test_route_seam_exhaustiveness.py | 50 +++++++++++++++++ 9 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 tests/test_route_seam_exhaustiveness.py diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml index d4fd6546..0892e367 100644 --- a/docker/litellm-config.yaml +++ b/docker/litellm-config.yaml @@ -1,31 +1,31 @@ -# LiteLLM proxy config for the coder_eval `custom` (open-weight) backend. +# LiteLLM proxy config for the coder_eval `litellm` (open-weight) backend. # # Translates Anthropic Messages (what the Claude Code SDK speaks) <-> Bedrock # Converse (what the non-Claude open-weight models speak), so the native `Skill` # tool and UiPath skills fire unchanged. Verified 2026-07-21: HTTP 200 on both # models via POST /v1/messages, and tool_use blocks survive the translation. # -# Shared by both proxy modes: -# - Phase 1 — operator starts the proxy manually and points coder_eval at it -# via LITELLM_BASE_URL (no coder_eval-owned lifecycle). -# - Phase 1.5 — coder_eval autostarts the proxy against this same file, or an -# always-on docker sidecar mounts it. +# Two ways to run the proxy: +# - Manual: the operator starts the proxy and points coder_eval at it via +# LITELLM_BASE_URL (coder_eval owns no proxy lifecycle). +# - Autostart: coder_eval spawns the proxy against this same file, or an +# always-on docker sidecar mounts it. # # Required env in the proxy's environment (NOT committed): # AWS_BEARER_TOKEN_BEDROCK — Bedrock bearer token (LiteLLM forwards it). # AWS_REGION=eu-north-1 — EU residency (Stockholm). # LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. # -# Run manually (Phase 1): +# Run manually: # uvx --from 'litellm[proxy]' litellm --config docker/litellm-config.yaml --port 4000 model_list: - # DeepSeek V3.2 — Phase-1 cost lead ($0.74 / $2.22 per Mtok). + # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok). - model_name: deepseek.v3.2 litellm_params: model: bedrock/converse/deepseek.v3.2 aws_region_name: eu-north-1 - # GLM 5 — Phase-1 quality lead ($1.55 / $4.96 per Mtok). + # GLM 5 — quality lead ($1.55 / $4.96 per Mtok). - model_name: zai.glm-5 litellm_params: model: bedrock/converse/zai.glm-5 diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 3a1a1664..d1197cf0 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -602,6 +602,12 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso ) or TokenUsage() ) + # The SDK's cost estimate assumes Claude pricing; it is wrong for an + # open-weight model behind LiteLLM. Reprice the top-line from the token + # buckets at the model's real rate (buckets untouched → reconciliation + # invariant holds). + if isinstance(self._agent.route, LiteLLMRoute): + self._agent._reprice_for_litellm(usage, self.effective_model) try: agent_output = self._agent._format_messages(self.messages) @@ -1428,6 +1434,31 @@ def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage: logger.warning("No pricing for model %r; timeout/kill turn cost left unset", model) return usage + @staticmethod + def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> TokenUsage: + """Recompute the top-line cost for the LiteLLM backend. + + The Claude Agent SDK's ``costUSD``/``total_cost_usd`` is a client-side + estimate that assumes Claude/Anthropic pricing, so it is wrong for an + open-weight model driven through LiteLLM. Reprice from the (already + authoritative) token buckets at the model's real rate. The token buckets + are left untouched, so the per-message stream / reconciliation invariant + is unaffected — only the cost scalar changes. An unknown/unpriced model + yields ``None`` (an honest "N/A") rather than the misleading SDK figure. + """ + usage.total_cost_usd = ( + calculate_cost( + model, + uncached_input_tokens=usage.uncached_input_tokens, + output_tokens=usage.output_tokens, + cache_creation_tokens=usage.cache_creation_input_tokens, + cache_read_tokens=usage.cache_read_input_tokens, + ) + if model + else None + ) + return usage + def get_sdk_options(self) -> dict[str, Any] | None: """Get the raw SDK options used for the last agent query. diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index b35366ca..54928c5b 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -220,7 +220,7 @@ def run_command( None, "--backend", "-b", - click_type=click.Choice(["direct", "bedrock"], case_sensitive=False), + click_type=click.Choice(["direct", "bedrock", "litellm"], case_sensitive=False), help="API backend (default: from API_BACKEND env var)", ), experiment: Path | None = typer.Option( # noqa: B008 diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index b1581470..945a55e4 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -11,6 +11,7 @@ from inspect import isawaitable from pathlib import Path from typing import Any +from urllib.parse import urlparse from .agent import Agent from .agents.watchdog import ThreadedWatchdog @@ -37,6 +38,7 @@ EvaluationResult, FinalStatus, JudgeCriterionResult, + LiteLLMRoute, PostRunCommand, PostRunResult, PreRunCommand, @@ -125,6 +127,8 @@ def _format_routing(route: ApiRoute) -> str: name = ROUTE_NAMES[type(route)] if isinstance(route, DirectRoute): return f"{name} (judge transport: {route.judge_transport or 'none'})" + if isinstance(route, LiteLLMRoute): + return f"{name} (model: {route.model or 'default'})" return name @@ -1082,6 +1086,12 @@ def _record_route_environment_info(self) -> None: # Record which transport llm_judge will use under DirectRoute so the # choice is visible in run artifacts (and not just the startup log). self.result.environment_info["judge_transport"] = self.route.judge_transport or "none" + elif isinstance(self.route, LiteLLMRoute): + # Host only (never the base_url or auth token) — mirrors the Codex + # agent's host-only recording so secrets stay out of run artifacts. + self.result.environment_info["litellm_base_url_host"] = urlparse(self.route.base_url).hostname or "" + if self.route.model: + self.result.environment_info["litellm_model"] = self.route.model # Agent-specific routing (e.g. Codex custom-endpoint / Azure). No-op for # the evaluate-only path (no agent) and for agents that add nothing. if self.agent is not None: diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index ebe6797b..1dbec677 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -88,6 +88,12 @@ class ModelPricing: "gemini-3.1-pro-preview-customtools": ModelPricing(2.0, 12.0, 2.0, 0.20), "gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15), "gemini-3-flash-preview": ModelPricing(1.5, 9.0, 1.5, 0.15), + # Open-weight models on Bedrock (eu-north-1), driven via the LiteLLM backend. + # Bedrock lists no prompt-cache read/write rate for these, so cache-creation + # is priced at the input rate and cache-read at 0 (conservative — see the + # per-provider cost-accounting caveat; revisit against the AWS model cards). + "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0), + "zai.glm-5": ModelPricing(1.55, 4.96, 1.55, 0.0), } @@ -144,6 +150,12 @@ def _normalize_model(model: str) -> str: so it is safe to apply unconditionally for every route. """ model = model.strip() + # LiteLLM/Bedrock routing prefixes (e.g. "converse/zai.glm-5", + # "bedrock/converse/deepseek.v3.2") → bare model id. + for routing_prefix in ("bedrock/converse/", "bedrock/", "converse/"): + if model.startswith(routing_prefix): + model = model[len(routing_prefix) :] + break for prefix in _BEDROCK_REGION_PREFIXES: if model.startswith(prefix): model = model[len(prefix) :] diff --git a/tests/test_cli_backend_flag.py b/tests/test_cli_backend_flag.py index 18177367..bd0c1ba6 100644 --- a/tests/test_cli_backend_flag.py +++ b/tests/test_cli_backend_flag.py @@ -29,7 +29,7 @@ async def _noop_run_all_tasks(*args: object, **kwargs: object) -> None: return None -@pytest.mark.parametrize("backend", ["bedrock", "direct"]) +@pytest.mark.parametrize("backend", ["bedrock", "direct", "litellm"]) def test_backend_flag_syncs_api_backend_env(monkeypatch, backend) -> None: # delenv snapshots the key so monkeypatch's teardown restores it even though # the command mutates os.environ directly (no test pollution across params). diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 9d170c7c..71a93b1c 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -1,8 +1,8 @@ -"""Tests for the custom Anthropic-compatible backend (LiteLLMRoute). +"""Tests for the LiteLLM (Anthropic-compatible) open-weight backend (LiteLLMRoute). -Covers Phase 1 (route resolution + config validation) and Phase 2 (SDK env -building + effective-model sync) of the open-weight support plan. Mirrors the -Bedrock equivalents in ``tests/test_routing.py``. +Covers route resolution, config validation, SDK env building, effective-model +sync, pricing, and cost repricing. Mirrors the Bedrock equivalents in +``tests/test_routing.py``. """ from __future__ import annotations @@ -14,10 +14,12 @@ from coder_eval.models import ( AgentKind, LiteLLMRoute, + TokenUsage, parse_agent_config, ) from coder_eval.models.enums import ApiBackend from coder_eval.models.routing import ROUTE_NAMES, resolve_route +from coder_eval.pricing import _normalize_model, calculate_cost def _make_agent(route, *, config_model: str | None = None) -> ClaudeCodeAgent: @@ -122,7 +124,7 @@ def test_none_agent_skips_custom_validation(self): class TestBuildSdkEnvCustom: - """Phase 2: _build_sdk_env() for the custom route.""" + """_build_sdk_env() for the LiteLLM route.""" def test_custom_route_env_has_anthropic_vars_only(self): route = LiteLLMRoute( @@ -171,7 +173,7 @@ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch) class TestResolveEffectiveModelCustom: - """Phase 2: _resolve_effective_model() on the custom route — no prefixing.""" + """_resolve_effective_model() on the LiteLLM route — no prefixing.""" def test_config_model_synced_verbatim(self): route = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="deepseek.v3.2") @@ -195,3 +197,51 @@ def test_both_none_returns_none(self): effective = agent._resolve_effective_model(None, env, route_model) assert effective is None assert "ANTHROPIC_MODEL" not in env + + +class TestOpenWeightPricing: + """Pricing for the open-weight models + LiteLLM/Bedrock prefix normalization.""" + + def test_glm5_rate(self): + # 1M input + 1M output → input_per_mtok + output_per_mtok. + assert calculate_cost("zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.55 + 4.96) + + def test_deepseek_rate(self): + assert calculate_cost("deepseek.v3.2", 1_000_000, 1_000_000) == pytest.approx(0.74 + 2.22) + + def test_normalize_strips_converse_prefix(self): + assert _normalize_model("converse/zai.glm-5") == "zai.glm-5" + assert _normalize_model("bedrock/converse/deepseek.v3.2") == "deepseek.v3.2" + + def test_normalize_identity_on_bare_ids(self): + assert _normalize_model("zai.glm-5") == "zai.glm-5" + assert _normalize_model("deepseek.v3.2") == "deepseek.v3.2" + + def test_converse_prefixed_id_prices_same(self): + """The SDK reports model_used as e.g. 'converse/zai.glm-5' — must still price.""" + assert calculate_cost("converse/zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.55 + 4.96) + + +class TestRepriceForLitellm: + """The litellm backend recomputes cost from tokens, overriding the SDK estimate.""" + + def test_overrides_wrong_sdk_cost_with_real_rate(self): + u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=3.68) + ClaudeCodeAgent._reprice_for_litellm(u, "zai.glm-5") + assert u.total_cost_usd == pytest.approx(1.55 + 4.96) + + def test_converse_prefixed_model_reprices(self): + u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=99.0) + ClaudeCodeAgent._reprice_for_litellm(u, "converse/zai.glm-5") + assert u.total_cost_usd == pytest.approx(1.55 + 4.96) + + def test_unpriced_model_yields_none_not_sdk_figure(self): + """An unknown model must show N/A (None), never the misleading SDK cost.""" + u = TokenUsage(uncached_input_tokens=100, output_tokens=100, total_cost_usd=9.99) + ClaudeCodeAgent._reprice_for_litellm(u, "some-unknown-model") + assert u.total_cost_usd is None + + def test_none_model_yields_none(self): + u = TokenUsage(uncached_input_tokens=100, output_tokens=100, total_cost_usd=9.99) + ClaudeCodeAgent._reprice_for_litellm(u, None) + assert u.total_cost_usd is None diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 6b3f25e8..e0c75d6f 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -12,6 +12,7 @@ ClaudeCodeAgentConfig, DirectRoute, FileExistsCriterion, + LiteLLMRoute, PreservationMode, SandboxConfig, TaskDefinition, @@ -47,6 +48,12 @@ def test_format_routing_non_direct_routes_unchanged(): assert _format_routing(BedrockRoute(bearer_token="t", region="us-east-1")) == "aws_bedrock" +def test_format_routing_litellm_shows_model(): + out = _format_routing(LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5")) + assert out.startswith("litellm") + assert "zai.glm-5" in out + + def _make_orchestrator_with_route(tmp_path: Path, route) -> Orchestrator: """Build a minimal Orchestrator pre-populated with a route + EvaluationResult. @@ -98,6 +105,24 @@ def test_record_route_environment_info_bedrock(tmp_path): assert info["api_routing"] == "aws_bedrock" assert info["aws_region"] == "eu-north-1" assert info["bedrock_model"] == "eu.anthropic.claude-sonnet-4-6" + + +def test_record_route_environment_info_litellm_records_host_only_no_secret(tmp_path): + """LiteLLM route records host + model, but NEVER the auth token or full base_url.""" + orchestrator = _make_orchestrator_with_route( + tmp_path, + LiteLLMRoute(base_url="http://localhost:4000", auth_token="sk-super-secret", model="zai.glm-5"), + ) + orchestrator._record_route_environment_info() + assert orchestrator.result is not None + info = orchestrator.result.environment_info + assert info["api_routing"] == "litellm" + assert info["litellm_base_url_host"] == "localhost" + assert info["litellm_model"] == "zai.glm-5" + # No secret and no full URL anywhere in the recorded audit dict. + blob = str(info) + assert "sk-super-secret" not in blob + assert "http://localhost:4000" not in blob assert "judge_transport" not in info # Direct-only field diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py new file mode 100644 index 00000000..dda703cb --- /dev/null +++ b/tests/test_route_seam_exhaustiveness.py @@ -0,0 +1,50 @@ +"""Guardrail: every ``ApiRoute`` member must be handled at every route-matching seam. + +The route union is matched via ``match``/``isinstance`` at several seams +(``_build_sdk_env``, ``_format_routing``, ``ROUTE_NAMES``). A new route added +without extending each seam would silently no-op (``_format_routing`` / +recording) or raise (``_build_sdk_env``). This test iterates the union and +forces a fixture + per-seam check for every member, so adding a 4th route +fails here until each seam is extended. +""" + +from __future__ import annotations + +import typing + +from coder_eval.agents.claude_code_agent import ClaudeCodeAgent +from coder_eval.models import ROUTE_NAMES, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute +from coder_eval.orchestrator import _format_routing + + +# One minimal instance per ApiRoute member. The set-equality assertion below +# forces this dict to grow whenever the union does. +_INSTANCES: list[object] = [ + DirectRoute(), + BedrockRoute(bearer_token="t", region="eu-north-1", model="x"), + LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="m"), +] + + +def test_fixtures_cover_every_union_member(): + """Fails when a new ApiRoute member is added without a fixture here.""" + assert {type(r) for r in _INSTANCES} == set(typing.get_args(ApiRoute)) + + +def test_every_route_has_a_route_name(): + for r in _INSTANCES: + assert type(r) in ROUTE_NAMES + + +def test_build_sdk_env_handles_every_route(): + """_build_sdk_env raises AssertionError on an unhandled route — must not for any member.""" + for r in _INSTANCES: + env, _model = ClaudeCodeAgent._build_sdk_env(r) # type: ignore[arg-type] + assert isinstance(env, dict) + + +def test_format_routing_handles_every_route(): + """No silent no-op: each route formats to a non-empty string led by its ROUTE_NAMES value.""" + for r in _INSTANCES: + out = _format_routing(r) # type: ignore[arg-type] + assert out and out.startswith(ROUTE_NAMES[type(r)]) From ea6ba9099999bc8cd7747d88304ef8fde53e5b2b Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 21 Jul 2026 17:08:41 +0100 Subject: [PATCH 03/23] fix(routing): log the effective litellm model + cover LITELLM in the enum test - _format_routing now shows the effective agent model (e.g. from --model), not the route's LITELLM_MODEL default, so the 'API routing:' line reflects what the agent actually sends. - Update test_api_backend_enum_values to include ApiBackend.LITELLM (ripple from adding the enum member). Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/orchestrator.py | 17 ++++++++++++----- tests/test_config_precedence.py | 5 +++-- tests/test_orchestrator.py | 10 ++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 945a55e4..da66a10f 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -117,18 +117,21 @@ async def _pump_stream( _UTTERANCE_TAG_RE = re.compile(r"^\[(ASSISTANT|RESULT - SUCCESS|RESULT - ERROR|TOOL USE)\](?: (.*))?$") -def _format_routing(route: ApiRoute) -> str: +def _format_routing(route: ApiRoute, effective_model: str | None = None) -> str: """Format the route name for the ``API routing:`` log line. For ``DirectRoute`` the resolved judge transport is appended so the choice (anthropic / none) is visible on every run, not only in the persisted - ``environment_info`` record. + ``environment_info`` record. For ``LiteLLMRoute`` the model is shown — the + ``effective_model`` (the resolved ``agent.model``, e.g. from ``--model``) + when supplied, else the route's own default — so the line reflects what the + agent will actually send rather than the route-level fallback. """ name = ROUTE_NAMES[type(route)] if isinstance(route, DirectRoute): return f"{name} (judge transport: {route.judge_transport or 'none'})" if isinstance(route, LiteLLMRoute): - return f"{name} (model: {route.model or 'default'})" + return f"{name} (model: {effective_model or route.model or 'default'})" return name @@ -894,7 +897,9 @@ async def _setup(self) -> None: self.result.sandbox_path = str(self.sandbox.sandbox_dir) self.route = resolve_route(settings) - logger.info("API routing: %s", _format_routing(self.route)) + logger.info( + "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) + ) self.success_checker = SuccessChecker(self.sandbox, route=self.route) self._record_route_environment_info() return @@ -961,7 +966,9 @@ async def _setup_sandbox() -> Any: # Determine API routing from settings.api_backend enum self.route = resolve_route(settings) - logger.info("API routing: %s", _format_routing(self.route)) + logger.info( + "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) + ) self.success_checker = SuccessChecker(self.sandbox, route=self.route) # Create and start the agent. For a no-op (type: none) task this dispatches diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py index 60be6b86..b726786c 100644 --- a/tests/test_config_precedence.py +++ b/tests/test_config_precedence.py @@ -279,12 +279,13 @@ def test_cli_override_applies_with_lineage_detail(): def test_api_backend_enum_values(): - """Verify ApiBackend has exactly 2 values with expected string representations.""" + """Verify ApiBackend has exactly 3 values with expected string representations.""" from coder_eval.models.enums import ApiBackend - assert set(ApiBackend) == {ApiBackend.DIRECT, ApiBackend.BEDROCK} + assert set(ApiBackend) == {ApiBackend.DIRECT, ApiBackend.BEDROCK, ApiBackend.LITELLM} assert str(ApiBackend.DIRECT) == "direct" assert str(ApiBackend.BEDROCK) == "bedrock" + assert str(ApiBackend.LITELLM) == "litellm" def test_settings_api_backend_default(): diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index e0c75d6f..be8de5e9 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -54,6 +54,16 @@ def test_format_routing_litellm_shows_model(): assert "zai.glm-5" in out +def test_format_routing_litellm_effective_model_wins_over_route_default(): + """The --model override (effective_model) must be logged, not the route's LITELLM_MODEL default.""" + out = _format_routing( + LiteLLMRoute(base_url="http://localhost:4000", auth_token="k", model="zai.glm-5"), + effective_model="deepseek.v3.2", + ) + assert "deepseek.v3.2" in out + assert "zai.glm-5" not in out + + def _make_orchestrator_with_route(tmp_path: Path, route) -> Orchestrator: """Build a minimal Orchestrator pre-populated with a route + EvaluationResult. From 4b5f1837b4267dd7e37d57c739efcb6dd15f283a Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Wed, 22 Jul 2026 11:29:02 +0100 Subject: [PATCH 04/23] feat(litellm): add Kimi K2.5, correct GLM eu-north-1 rate, drop Qwen; add proxy start script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - config: add moonshotai.kimi-k2.5 to the shared litellm model_list; remove qwen.qwen3-coder-next (needs a Bedrock model-access grant — deferred) - pricing: Kimi K2.5 ($0.72/$3.60); correct GLM 5 (zai.glm-5) to the eu-north-1 rate ($1.20/$3.84, was the eu-west $1.55/$4.96) - docker/start-litellm.sh: launch the proxy with the required creds exported. Reads AWS_BEARER_TOKEN_BEDROCK / AWS_REGION out of .env and exports them into the proxy's process (the bare-KEY=value .env is otherwise unexported → "Unable to locate credentials" 401), sets LITELLM_MASTER_KEY, fails loud on a missing token, and kills any stale listener before relaunching. Pure-bash value extraction (handles quoted values + inline comments; no BSD-sed quirks). Co-Authored-By: Claude Opus 4.8 --- docker/litellm-config.yaml | 7 ++- docker/start-litellm.sh | 86 +++++++++++++++++++++++++++++++++++++ src/coder_eval/pricing.py | 3 +- tests/test_litellm_route.py | 14 ++++-- 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100755 docker/start-litellm.sh diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml index 0892e367..28a97a22 100644 --- a/docker/litellm-config.yaml +++ b/docker/litellm-config.yaml @@ -25,11 +25,16 @@ model_list: litellm_params: model: bedrock/converse/deepseek.v3.2 aws_region_name: eu-north-1 - # GLM 5 — quality lead ($1.55 / $4.96 per Mtok). + # GLM 5 — quality lead ($1.20 / $3.84 per Mtok, eu-north-1). - model_name: zai.glm-5 litellm_params: model: bedrock/converse/zai.glm-5 aws_region_name: eu-north-1 + # Kimi K2.5 (Moonshot AI) — Bedrock eu-north-1 (bare id, Converse). + - model_name: moonshotai.kimi-k2.5 + litellm_params: + model: bedrock/converse/moonshotai.kimi-k2.5 + aws_region_name: eu-north-1 litellm_settings: # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400. diff --git a/docker/start-litellm.sh b/docker/start-litellm.sh new file mode 100755 index 00000000..2c306132 --- /dev/null +++ b/docker/start-litellm.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Start the LiteLLM proxy for coder_eval's `litellm` (open-weight) backend. +# +# Why this script exists: the proxy is a separate process that reads its Bedrock +# credential + master key from ITS OWN environment — it does NOT read coder_eval's +# .env. And coder_eval's .env uses bare `KEY=value` (no `export`), so `source .env` +# alone leaves those vars unexported → the proxy can't see them → HTTP 401 +# "Unable to locate credentials". This script reads the needed values out of .env +# and exports them explicitly before launching, so that can't happen. +# +# Usage: +# docker/start-litellm.sh # foreground; Ctrl-C to stop +# Overridable via env: +# LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ENV_FILE="${ENV_FILE:-$REPO_ROOT/.env}" +CONFIG="${LITELLM_CONFIG:-$REPO_ROOT/docker/litellm-config.yaml}" +PORT="${LITELLM_PORT:-4000}" + +# Read a key from .env, returning ONLY the value: content between surrounding +# quotes if quoted (dropping any trailing `# comment`), else the bare value with +# an inline comment stripped. Pure bash — avoids BSD-sed backreference quirks and +# handles base64 values containing / + = safely. +read_env() { + local line val + line=$(grep -E "^$1=" "$ENV_FILE" 2>/dev/null | head -1) || line="" + [ -z "$line" ] && return 0 + val="${line#*=}" # strip leading `KEY=` + val="${val#"${val%%[![:space:]]*}"}" # trim leading whitespace + if [[ "$val" == '"'* ]]; then # double-quoted → content between quotes + val="${val#\"}"; val="${val%%\"*}" + elif [[ "$val" == "'"* ]]; then # single-quoted + val="${val#\'}"; val="${val%%\'*}" + else # bare → drop inline comment + trailing ws + val="${val%%#*}" + val="${val%"${val##*[![:space:]]}"}" + fi + printf '%s' "$val" +} + +# --- credentials the proxy needs (already-exported values win over .env) --- +export AWS_BEARER_TOKEN_BEDROCK="${AWS_BEARER_TOKEN_BEDROCK:-$(read_env AWS_BEARER_TOKEN_BEDROCK)}" +export AWS_REGION="${AWS_REGION:-$(read_env AWS_REGION)}" +export AWS_REGION="${AWS_REGION:-eu-north-1}" +# Master key = the key clients present. Default to .env's LITELLM_AUTH_TOKEN so the +# client and proxy match; fall back to the local dev key. +export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-$(read_env LITELLM_AUTH_TOKEN)}" +export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-spike-local}" + +# --- preflight: fail loud instead of a runtime 401 --- +[ -f "$CONFIG" ] || { echo "ERROR: config not found: $CONFIG" >&2; exit 1; } +if [ -z "$AWS_BEARER_TOKEN_BEDROCK" ]; then + echo "ERROR: AWS_BEARER_TOKEN_BEDROCK is empty — not in $ENV_FILE and not exported." >&2 + echo " Set it in .env or 'export AWS_BEARER_TOKEN_BEDROCK=...' before running." >&2 + exit 1 +fi +echo "config : $CONFIG" +echo "region : $AWS_REGION" +echo "bedrock tok: set (${#AWS_BEARER_TOKEN_BEDROCK} chars)" +echo "master key : $LITELLM_MASTER_KEY" + +# --- stop any stale proxy on the port (the classic 'creds-less running proxy') --- +existing=$(lsof -tiTCP:"$PORT" -sTCP:LISTEN 2>/dev/null || true) +if [ -n "$existing" ]; then + echo "stopping existing listener on :$PORT (pids: $existing)" + # shellcheck disable=SC2086 + kill $existing 2>/dev/null || true + sleep 1 +fi + +cat < --model zai.glm-5 (or deepseek.v3.2 / moonshotai.kimi-k2.5) + +EOF + +exec uvx --from 'litellm[proxy]' litellm --config "$CONFIG" --host 127.0.0.1 --port "$PORT" diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 1dbec677..89c418fe 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -93,7 +93,8 @@ class ModelPricing: # is priced at the input rate and cache-read at 0 (conservative — see the # per-provider cost-accounting caveat; revisit against the AWS model cards). "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0), - "zai.glm-5": ModelPricing(1.55, 4.96, 1.55, 0.0), + "zai.glm-5": ModelPricing(1.2, 3.84, 1.2, 0.0), + "moonshotai.kimi-k2.5": ModelPricing(0.72, 3.6, 0.72, 0.0), } diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 71a93b1c..9ae9c9a4 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -204,11 +204,17 @@ class TestOpenWeightPricing: def test_glm5_rate(self): # 1M input + 1M output → input_per_mtok + output_per_mtok. - assert calculate_cost("zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.55 + 4.96) + assert calculate_cost("zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.2 + 3.84) def test_deepseek_rate(self): assert calculate_cost("deepseek.v3.2", 1_000_000, 1_000_000) == pytest.approx(0.74 + 2.22) + def test_kimi_rate(self): + assert calculate_cost("moonshotai.kimi-k2.5", 1_000_000, 1_000_000) == pytest.approx(0.72 + 3.6) + + def test_kimi_converse_prefixed_prices_same(self): + assert calculate_cost("converse/moonshotai.kimi-k2.5", 1_000_000, 1_000_000) == pytest.approx(0.72 + 3.6) + def test_normalize_strips_converse_prefix(self): assert _normalize_model("converse/zai.glm-5") == "zai.glm-5" assert _normalize_model("bedrock/converse/deepseek.v3.2") == "deepseek.v3.2" @@ -219,7 +225,7 @@ def test_normalize_identity_on_bare_ids(self): def test_converse_prefixed_id_prices_same(self): """The SDK reports model_used as e.g. 'converse/zai.glm-5' — must still price.""" - assert calculate_cost("converse/zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.55 + 4.96) + assert calculate_cost("converse/zai.glm-5", 1_000_000, 1_000_000) == pytest.approx(1.2 + 3.84) class TestRepriceForLitellm: @@ -228,12 +234,12 @@ class TestRepriceForLitellm: def test_overrides_wrong_sdk_cost_with_real_rate(self): u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=3.68) ClaudeCodeAgent._reprice_for_litellm(u, "zai.glm-5") - assert u.total_cost_usd == pytest.approx(1.55 + 4.96) + assert u.total_cost_usd == pytest.approx(1.2 + 3.84) def test_converse_prefixed_model_reprices(self): u = TokenUsage(uncached_input_tokens=1_000_000, output_tokens=1_000_000, total_cost_usd=99.0) ClaudeCodeAgent._reprice_for_litellm(u, "converse/zai.glm-5") - assert u.total_cost_usd == pytest.approx(1.55 + 4.96) + assert u.total_cost_usd == pytest.approx(1.2 + 3.84) def test_unpriced_model_yields_none_not_sdk_figure(self): """An unknown model must show N/A (None), never the misleading SDK cost.""" From 889707b575a40d7f839837ebeb402e1338b5825d Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Wed, 22 Jul 2026 14:51:33 +0100 Subject: [PATCH 05/23] feat(litellm): preflight external proxy reachability (fail fast, no hang) Ping /health/liveliness once before the batch when the litellm backend targets an external proxy. If unreachable, exit with a clear error instead of letting every task hang on the dead endpoint. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/cli/run_command.py | 38 ++++++++++++++++++++++++++++++- tests/test_litellm_route.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 54928c5b..1ffe9a2d 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -4,6 +4,8 @@ import logging import os import sys +import urllib.error +import urllib.request from collections.abc import Callable from pathlib import Path from typing import Any @@ -12,7 +14,7 @@ import typer from tqdm import tqdm -from ..config import settings +from ..config import Settings, settings from ..logging_config import setup_logging from ..models import PreservationMode, ResolvedTask, RunSummary, TaskResult from ..orchestration.config import BatchRunConfig @@ -66,6 +68,33 @@ def _resolve_experiment_path(experiment: Path | None) -> Path | None: raise typer.BadParameter(f"Experiment not found: {experiment}.{hint}") +def _litellm_preflight_error(current_settings: Settings) -> str | None: + """Return an error message if the ``litellm`` backend's external proxy is + unreachable, else ``None``. + + Only applies when ``api_backend=litellm`` with an explicit ``LITELLM_BASE_URL`` + (the manual proxy / always-on-sidecar path). Without this check a dead proxy + makes the Claude SDK hang on the endpoint instead of failing fast. Any HTTP + response (even non-200) counts as reachable — only a connection/timeout error + is treated as "proxy down". + """ + from ..models import ApiBackend + + if current_settings.api_backend != ApiBackend.LITELLM or not current_settings.litellm_base_url: + return None + url = f"{current_settings.litellm_base_url.rstrip('/')}/health/liveliness" + try: + urllib.request.urlopen(url, timeout=5).close() + except urllib.error.HTTPError: + return None # server responded (up), just not 200 on this path + except (urllib.error.URLError, OSError) as exc: + return ( + f"LiteLLM proxy not reachable at {current_settings.litellm_base_url} (tried {url}): {exc}. " + "Start it (e.g. docker/start-litellm.sh) or unset LITELLM_BASE_URL." + ) + return None + + def _build_overrides( *, model: str | None, @@ -451,6 +480,13 @@ async def _run_all_tasks( }, ) + # Fail fast if the litellm backend points at an unreachable external proxy — + # otherwise the agent hangs on the dead endpoint instead of erroring. + preflight_error = await asyncio.to_thread(_litellm_preflight_error, settings) + if preflight_error: + console.print(f"[red]{preflight_error}[/red]") + raise typer.Exit(1) + try: # Always run through experiment layer (defaults to experiments/default.yaml) summary, failed_suite_gates = await _run_with_experiment( diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 9ae9c9a4..3ce66a42 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -7,9 +7,14 @@ from __future__ import annotations +import urllib.error +import urllib.request +from unittest.mock import MagicMock + import pytest from coder_eval.agents.claude_code_agent import ClaudeCodeAgent +from coder_eval.cli.run_command import _litellm_preflight_error from coder_eval.config import Settings from coder_eval.models import ( AgentKind, @@ -251,3 +256,34 @@ def test_none_model_yields_none(self): u = TokenUsage(uncached_input_tokens=100, output_tokens=100, total_cost_usd=9.99) ClaudeCodeAgent._reprice_for_litellm(u, None) assert u.total_cost_usd is None + + +class TestLitellmPreflight: + """External-proxy reachability preflight — fail fast instead of hanging on a dead proxy.""" + + def test_none_for_non_litellm_backend(self): + s = Settings(api_backend=ApiBackend.BEDROCK, litellm_base_url="http://x:4000") + assert _litellm_preflight_error(s) is None + + def test_none_when_no_base_url(self): + s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url=None, litellm_model="m") + assert _litellm_preflight_error(s) is None + + def test_error_when_proxy_down(self, monkeypatch): + monkeypatch.setattr(urllib.request, "urlopen", MagicMock(side_effect=urllib.error.URLError("refused"))) + s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:9", litellm_model="m") + err = _litellm_preflight_error(s) + assert err is not None + assert "not reachable" in err and "http://127.0.0.1:9" in err + + def test_none_when_reachable(self, monkeypatch): + monkeypatch.setattr(urllib.request, "urlopen", MagicMock(return_value=MagicMock())) + s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:4000", litellm_model="m") + assert _litellm_preflight_error(s) is None + + def test_none_on_http_error_means_server_up(self, monkeypatch): + monkeypatch.setattr( + urllib.request, "urlopen", MagicMock(side_effect=urllib.error.HTTPError("u", 404, "nf", {}, None)) + ) + s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:4000", litellm_model="m") + assert _litellm_preflight_error(s) is None From 16ddae5a1bcea25b85295b03754c2f95e3d01566 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Wed, 22 Jul 2026 17:11:17 +0100 Subject: [PATCH 06/23] feat(docker): forward LiteLLM env into containers, rewrite loopback proxy URL Docker-driver tasks on the litellm backend failed because the LITELLM_* creds weren't in the env allowlist: the in-container Settings saw API_BACKEND=litellm with no creds and validation raised. Add LITELLM_BASE_URL/AUTH_TOKEN/MODEL/SMALL_MODEL to the default env_passthrough allowlist. LITELLM_BASE_URL is special-cased: a loopback host is rewritten to host.docker.internal (the proxy runs on the host, not in the container) and published via --add-host for Linux parity; the token stays name-only so it's never rendered in the logged argv. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/isolation/docker_runner.py | 41 ++++++++ src/coder_eval/models/sandbox.py | 9 ++ tests/test_docker_litellm_env.py | 119 ++++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 tests/test_docker_litellm_env.py diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 0c50ecf5..53ff28be 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -59,6 +59,29 @@ # `COPY` destination in docker/Dockerfile -- a drift guard test enforces that. CONTAINER_ENTRYPOINT = "/usr/local/bin/coder_eval_entrypoint.sh" +# Docker Desktop's stable alias for the host, from inside a bridge-network +# container. Auto-resolves on macOS/Windows; on Linux it must be published +# explicitly via `--add-host host.docker.internal:host-gateway`. +_DOCKER_HOST_ALIAS = "host.docker.internal" +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) + + +def _rewrite_loopback_for_container(url: str) -> str | None: + """Rewrite a loopback URL to the docker host alias, preserving scheme/port/path. + + Returns the rewritten URL, or None if the host is not loopback (forward as-is). + A LiteLLM proxy on the HOST is unreachable at localhost from inside a bridge + container, so ``http://localhost:4000`` -> ``http://host.docker.internal:4000``. + """ + from urllib.parse import urlsplit, urlunsplit + + parts = urlsplit(url) + if parts.hostname not in _LOOPBACK_HOSTS: + return None + netloc = _DOCKER_HOST_ALIAS if parts.port is None else f"{_DOCKER_HOST_ALIAS}:{parts.port}" + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) + + # Top-level entries under ~/.claude that the per-task RW copy SKIPS. We copy # the host's ~/.claude into a throwaway tmp dir and mount that copy read-WRITE # so the in-container CLI can write anywhere it needs without ever touching the @@ -1088,9 +1111,27 @@ def _build_argv( # would silently default to DIRECT — downgrading the judge (and agent) route. merged_allowlist = set(cfg.env_passthrough) | set(cfg.env_passthrough_extra) for env_var in merged_allowlist: + # LITELLM_BASE_URL is forwarded below with a value rewrite, not name-only. + if env_var == "LITELLM_BASE_URL": + continue if env_var in os.environ: argv += ["--env", env_var] + # LITELLM_BASE_URL points at a proxy on the HOST. A bridge-network container + # can't reach the host's loopback, so rewrite localhost/127.0.0.1 to the + # docker host alias and publish that alias (`--add-host`) for Linux parity + # (it's automatic on macOS/Windows Docker Desktop). It's only a URL, so an + # explicit `--env VAR=value` is safe to render in the logged argv — unlike + # the auth token, which stays name-only above. Skipped when the container + # has no network (the proxy is unreachable anyway → validation errors). + litellm_base_url = os.environ.get("LITELLM_BASE_URL") + if litellm_base_url and "LITELLM_BASE_URL" in merged_allowlist and cfg.network != "none": + rewritten = _rewrite_loopback_for_container(litellm_base_url) + if rewritten is not None: + argv += ["--env", f"LITELLM_BASE_URL={rewritten}", "--add-host", f"{_DOCKER_HOST_ALIAS}:host-gateway"] + else: + argv += ["--env", "LITELLM_BASE_URL"] + # Signal to in-container agents that the harness already provides OS-level # isolation. The Codex agent reads this to fall back to its full-access # sandbox: Codex's Landlock-backed read-only / workspace-write sandboxes diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index bf86106e..2daf30df 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -227,6 +227,15 @@ class DockerDriverConfig(BaseModel): # through Bedrock instead of falling back to ~/.claude OAuth. "CLAUDE_CODE_USE_BEDROCK", "ANTHROPIC_MODEL", + # LiteLLM (Anthropic-compatible) open-weight backend. The proxy runs on + # the HOST, so LITELLM_BASE_URL is rewritten to host.docker.internal at + # the container boundary (see docker_runner); the rest forward verbatim. + # Without these the in-container Settings sees API_BACKEND=litellm with no + # creds and _validate_litellm_settings raises a hard ValueError. + "LITELLM_BASE_URL", + "LITELLM_AUTH_TOKEN", + "LITELLM_MODEL", + "LITELLM_SMALL_MODEL", # Codex agent auth/routing — without these the in-container codex # binary falls back to a ChatGPT login that doesn't exist in the # container and auth fails. CODEX_API_KEY drives login_api_key; diff --git a/tests/test_docker_litellm_env.py b/tests/test_docker_litellm_env.py new file mode 100644 index 00000000..8d8d99e8 --- /dev/null +++ b/tests/test_docker_litellm_env.py @@ -0,0 +1,119 @@ +"""Tests for forwarding the LiteLLM backend into a docker-driver container. + +The LiteLLM proxy runs on the HOST, so the container must (a) receive the +LITELLM_* credentials via the env allowlist and (b) reach the host via +host.docker.internal rather than an unreachable loopback address. +""" + +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from coder_eval.isolation.docker_runner import ( + DockerRunner, + _rewrite_loopback_for_container, +) +from coder_eval.models import DockerDriverConfig, FileExistsCriterion, SandboxConfig, TaskDefinition + + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="docker driver is POSIX-only") + + +class TestRewriteLoopbackForContainer: + """localhost/127.0.0.1 -> host.docker.internal, preserving scheme/port/path.""" + + def test_localhost_with_port(self): + assert _rewrite_loopback_for_container("http://localhost:4000") == "http://host.docker.internal:4000" + + def test_127_0_0_1_with_port(self): + assert _rewrite_loopback_for_container("http://127.0.0.1:4000") == "http://host.docker.internal:4000" + + def test_preserves_scheme_and_path(self): + assert _rewrite_loopback_for_container("https://localhost:8443/v1") == "https://host.docker.internal:8443/v1" + + def test_no_port(self): + assert _rewrite_loopback_for_container("http://localhost") == "http://host.docker.internal" + + def test_non_loopback_returns_none(self): + assert _rewrite_loopback_for_container("http://litellm.internal:4000") is None + + +class TestLitellmEnvForwarding: + """_build_argv forwards LITELLM_* and rewrites the base URL for the container.""" + + def _make_runner(self) -> DockerRunner: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig()), + success_criteria=[FileExistsCriterion(description="c", path="x.txt")], + ) + rt = MagicMock() + rt.task = task + rt.run_dir = Path(tempfile.gettempdir()) / "test_run_litellm" + rt.task_file = None + return DockerRunner(rt) + + def _build(self, runner: DockerRunner) -> list[str]: + with tempfile.TemporaryDirectory() as tmp: + input_dir = Path(tmp) / "in" + output_dir = Path(tmp) / "out" + input_dir.mkdir() + output_dir.mkdir() + return runner._build_argv(input_dir, output_dir, container_name="c") + + def test_loopback_url_rewritten_and_host_published(self, monkeypatch): + monkeypatch.setenv("API_BACKEND", "litellm") + monkeypatch.setenv("LITELLM_BASE_URL", "http://localhost:4000") + monkeypatch.setenv("LITELLM_AUTH_TOKEN", "sk-master") + monkeypatch.setenv("LITELLM_MODEL", "zai.glm-5") + argv = self._build(self._make_runner()) + + # Base URL forwarded as an explicit rewritten value (safe: URL, not a secret). + assert "LITELLM_BASE_URL=http://host.docker.internal:4000" in argv + # The bare name-only form must NOT also be forwarded (it would carry localhost). + assert "LITELLM_BASE_URL" not in argv + # host alias published for Linux parity. + assert "--add-host" in argv + assert "host.docker.internal:host-gateway" in argv + # Credentials + model forwarded name-only (value copied by docker, stays out of argv). + assert "LITELLM_AUTH_TOKEN" in argv + assert "LITELLM_MODEL" in argv + assert "sk-master" not in " ".join(argv) + + def test_non_loopback_url_forwarded_name_only(self, monkeypatch): + monkeypatch.setenv("API_BACKEND", "litellm") + monkeypatch.setenv("LITELLM_BASE_URL", "http://litellm.internal:4000") + monkeypatch.setenv("LITELLM_AUTH_TOKEN", "sk-master") + monkeypatch.setenv("LITELLM_MODEL", "zai.glm-5") + argv = self._build(self._make_runner()) + + # Forwarded name-only; value never rendered, no host alias needed. + assert argv.count("LITELLM_BASE_URL") == 1 + assert "litellm.internal" not in " ".join(argv) + assert "host.docker.internal:host-gateway" not in argv + + def test_no_network_skips_base_url(self, monkeypatch): + monkeypatch.setenv("API_BACKEND", "litellm") + monkeypatch.setenv("LITELLM_BASE_URL", "http://localhost:4000") + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(network="none")), + success_criteria=[FileExistsCriterion(description="c", path="x.txt")], + ) + rt = MagicMock() + rt.task = task + rt.run_dir = Path(tempfile.gettempdir()) / "test_run_litellm_nonet" + rt.task_file = None + argv = self._build(DockerRunner(rt)) + + assert "LITELLM_BASE_URL" not in " ".join(argv) + assert "host.docker.internal:host-gateway" not in argv From 731d8ff4a537cd8e1476806999ffe50f4054b298 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 23 Jul 2026 11:06:04 +0100 Subject: [PATCH 07/23] fix(litellm): disable Claude Code attribution metadata (Bedrock requestMetadata 400) --- src/coder_eval/agents/claude_code_agent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index d1197cf0..9275c1d5 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -783,6 +783,10 @@ def _build_sdk_env( # x-api-key (e.g. a real Anthropic key exported from .env) # would conflict with the gateway's key auth. "ANTHROPIC_API_KEY": "", + # Claude Code attaches usage-attribution metadata (metadata.user_id) + # that Bedrock's requestMetadata regex rejects (HTTP 400) once LiteLLM + # forwards it to Bedrock. Disable it, mirroring the BedrockRoute case above. + "CLAUDE_CODE_ATTRIBUTION_HEADER": "0", } if cr.model: env["ANTHROPIC_MODEL"] = cr.model From adfefd35851cb78ec91706fd345da0e8bebc7148 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 23 Jul 2026 11:46:04 +0100 Subject: [PATCH 08/23] fix(litellm): neutralize inherited Bedrock creds so the CLI uses the proxy, not Bedrock-direct --- src/coder_eval/agents/claude_code_agent.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 9275c1d5..68d6687d 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -787,6 +787,14 @@ def _build_sdk_env( # that Bedrock's requestMetadata regex rejects (HTTP 400) once LiteLLM # forwards it to Bedrock. Disable it, mirroring the BedrockRoute case above. "CLAUDE_CODE_ATTRIBUTION_HEADER": "0", + # Neutralize inherited Bedrock creds. The CLI auto-selects Bedrock DIRECT + # when AWS_BEARER_TOKEN_BEDROCK is present (`if(process.env.AWS_BEARER_TOKEN_BEDROCK)`), + # and that token is forwarded into docker task containers via the default + # env-passthrough allowlist — so without blanking it the CLI bypasses + # ANTHROPIC_BASE_URL (the LiteLLM proxy) and calls Bedrock directly. Empty string + # is falsy in the CLI's check, so this forces it back onto the gateway. + "AWS_BEARER_TOKEN_BEDROCK": "", + "CLAUDE_CODE_USE_BEDROCK": "", } if cr.model: env["ANTHROPIC_MODEL"] = cr.model From 8d5031c1c5e9193a851a40764002120867ca93d7 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 23 Jul 2026 17:32:04 +0100 Subject: [PATCH 09/23] feat(pricing): add OpenRouter open-weight models (Kimi K3, GLM 5.2, DeepSeek V4 Pro) Wire three OpenRouter-hosted open-weight models through the existing litellm backend (cost-optimization path) and register their OpenRouter-published rates. These providers cache prompt prefixes implicitly, so cache-creation stays 0 and only cache-read carries a (discounted) rate. Also corrects the Kimi K3 cache-read rate from $0 to the published $0.30/M. Co-Authored-By: Claude Opus 4.8 --- docker/litellm-config.yaml | 14 ++++++++++++++ src/coder_eval/pricing.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml index 28a97a22..671f523c 100644 --- a/docker/litellm-config.yaml +++ b/docker/litellm-config.yaml @@ -35,6 +35,20 @@ model_list: litellm_params: model: bedrock/converse/moonshotai.kimi-k2.5 aws_region_name: eu-north-1 + # OpenRouter models — cost-optimization path. Not on Bedrock; OpenRouter is + # OpenAI-format, so LiteLLM does the Anthropic<->OpenAI translation. + - model_name: moonshotai/kimi-k3 + litellm_params: + model: openrouter/moonshotai/kimi-k3 + api_key: os.environ/OPENROUTER_API_KEY + - model_name: z-ai/glm-5.2 + litellm_params: + model: openrouter/z-ai/glm-5.2 + api_key: os.environ/OPENROUTER_API_KEY + - model_name: deepseek/deepseek-v4-pro + litellm_params: + model: openrouter/deepseek/deepseek-v4-pro + api_key: os.environ/OPENROUTER_API_KEY litellm_settings: # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400. diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 89c418fe..28784360 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -95,6 +95,14 @@ class ModelPricing: "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0), "zai.glm-5": ModelPricing(1.2, 3.84, 1.2, 0.0), "moonshotai.kimi-k2.5": ModelPricing(0.72, 3.6, 0.72, 0.0), + # OpenRouter models (cost-optimization path). These providers cache prompt + # prefixes IMPLICITLY (no cache_control, no cache-write fee), so cache-creation + # is priced at input (unused — cache_creation_tokens is always 0) and cache-read + # at OpenRouter's published input_cache_read rate. Rates per OpenRouter's + # /models endpoint (per-token x 1e6). + "moonshotai/kimi-k3": ModelPricing(3.0, 15.0, 3.0, 0.30), + "z-ai/glm-5.2": ModelPricing(0.826, 2.596, 0.826, 0.1534), + "deepseek/deepseek-v4-pro": ModelPricing(0.435, 0.87, 0.435, 0.003625), } From 6fe833cabf7ba2b6564d728a957b59a5cc700eec Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 09:03:12 +0100 Subject: [PATCH 10/23] feat(litellm): pin OpenRouter provider by price for deterministic cost OpenRouter routes each request to a provider chosen at request time, which can be pricier than the model page's headline. Pin extra_body.provider={sort: price, allow_fallbacks: false} on the OpenRouter models so the cheapest upstream is used deterministically and the billed rate matches the pricing table. Also reindents an evaluate-only logging call. Co-Authored-By: Claude Opus 4.8 --- docker/litellm-config.yaml | 20 ++++++++++++++++++++ src/coder_eval/orchestrator.py | 8 +++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml index 671f523c..9f889d36 100644 --- a/docker/litellm-config.yaml +++ b/docker/litellm-config.yaml @@ -37,18 +37,38 @@ model_list: aws_region_name: eu-north-1 # OpenRouter models — cost-optimization path. Not on Bedrock; OpenRouter is # OpenAI-format, so LiteLLM does the Anthropic<->OpenAI translation. + # + # provider routing (extra_body.provider): OpenRouter is a multi-provider router, + # and its per-request routing is nondeterministic — a request can land on a + # provider far pricier than the model page's headline rate (observed: DeepSeek + # V4 Pro routed to Novita at ~$1.47/M input vs the $0.435/M headline, 3.4x). + # `sort: price` pins the cheapest upstream and `allow_fallbacks: false` forbids + # silently upgrading to a pricier one, so the billed rate is deterministic and + # matches the pricing.py table (and is the cost floor we benchmark for). - model_name: moonshotai/kimi-k3 litellm_params: model: openrouter/moonshotai/kimi-k3 api_key: os.environ/OPENROUTER_API_KEY + extra_body: + provider: + sort: price + allow_fallbacks: false - model_name: z-ai/glm-5.2 litellm_params: model: openrouter/z-ai/glm-5.2 api_key: os.environ/OPENROUTER_API_KEY + extra_body: + provider: + sort: price + allow_fallbacks: false - model_name: deepseek/deepseek-v4-pro litellm_params: model: openrouter/deepseek/deepseek-v4-pro api_key: os.environ/OPENROUTER_API_KEY + extra_body: + provider: + sort: price + allow_fallbacks: false litellm_settings: # Silently drop provider-unsupported OpenAI/Anthropic params rather than 400. diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index da66a10f..d913fa42 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -898,8 +898,8 @@ async def _setup(self) -> None: self.route = resolve_route(settings) logger.info( - "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) - ) + "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) + ) self.success_checker = SuccessChecker(self.sandbox, route=self.route) self._record_route_environment_info() return @@ -966,9 +966,7 @@ async def _setup_sandbox() -> Any: # Determine API routing from settings.api_backend enum self.route = resolve_route(settings) - logger.info( - "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) - ) + logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) self.success_checker = SuccessChecker(self.sandbox, route=self.route) # Create and start the agent. For a no-op (type: none) task this dispatches From 6bb2ac53e2b17e3f681bbc523a23a1f830bbfdc7 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 10:15:49 +0100 Subject: [PATCH 11/23] feat(evalboard): mirror OpenRouter model rates into pricing table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Kimi K3, GLM 5.2, DeepSeek V4 Pro to evalboard's ported pricing table so the per-stage / per-task cost column populates for litellm-backend runs (was blank — the models were absent). Rates mirror src/coder_eval/pricing.py; accurate only with provider pinning (sort: price) in the LiteLLM config. Co-Authored-By: Claude Opus 4.8 --- evalboard/lib/pricing.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 29f27360..26baa144 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -50,6 +50,14 @@ export const PRICING: Record = { "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), + // OpenRouter open-weight models (litellm backend). Mirror of pricing.py; + // these providers cache implicitly (cache_write == input, unused) so only + // cache_read carries a discounted rate. NOTE: OpenRouter routes per-request, + // so these rates are only accurate when the litellm config pins the provider + // (sort: price) — otherwise the billed rate can differ from the headline. + "moonshotai/kimi-k3": p(3, 15, 3, 0.3), + "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534), + "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625), }; function p( From d0b262b159d99da55958f269052596000c9a8ec6 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 12:23:37 +0100 Subject: [PATCH 12/23] fix(evalboard): price the reconciliation row so the Cost column sums to the turn total MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthetic reconciliation row (tokens billed but not streamed per-message) was rendered with a hardcoded '—' cost and built with costUsd=null. For providers whose per-message stream is sparse (OpenRouter/LiteLLM), that row holds most of the turn's input+cache tokens, so summing the visible Cost column fell far short of the real total. Now the reconciliation entry is priced from the turn's model (added model_used to TurnEntry) and its row renders that cost. Safe from double-counting: the cost simulator excludes reconciliation by role, and the authoritative task total reads the backend aggregate, not a sum of per-message costUsd. Co-Authored-By: Claude Opus 4.8 --- .../app/runs/[id]/[...task]/_sections.tsx | 4 +- evalboard/lib/__tests__/parseMessages.test.ts | 61 +++++++++++++++++++ evalboard/lib/runs.ts | 22 ++++++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 6e7f4407..5368c24f 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -1087,7 +1087,9 @@ function ReconciliationRow({ m, unit }: { m: MessageEvent; unit: Unit }) { {fmtTok(m.outputTokens, "output")} - — + + {m.costUsd != null ? fmtUsd(m.costUsd) : "—"} + ); diff --git a/evalboard/lib/__tests__/parseMessages.test.ts b/evalboard/lib/__tests__/parseMessages.test.ts index 96c3e807..e3bd183a 100644 --- a/evalboard/lib/__tests__/parseMessages.test.ts +++ b/evalboard/lib/__tests__/parseMessages.test.ts @@ -769,4 +769,65 @@ describe("parseMessages — reconciliation entry", () => { expect(recon.parentToolUseId).toBeNull(); expect(recon.toolUses).toEqual([]); }); + + test("prices the reconciliation row from the turn model so the Cost column sums to the turn total", () => { + // Mirrors the OpenRouter/LiteLLM sparse-stream case that motivated this: + // the per-message stream carries only output, and the bulk of the input + + // cache-read tokens land in the single reconciliation row. Before the fix + // that row read "—" for cost, so the Cost column summed to far less than + // the real total. + const turns: TurnEntry[] = [ + { + model_used: "deepseek/deepseek-v4-pro", + messages: [ + { + role: "assistant", + model: "deepseek/deepseek-v4-pro", + started_at: "2026-01-01T00:00:00.000Z", + completed_at: "2026-01-01T00:00:01.000Z", + generation_duration_ms: 1000, + content_blocks: [{ block_type: "text", text: "hi" }], + input_tokens: 0, + output_tokens: 1000, + cache_read_tokens: 0, + }, + { + role: "reconciliation", + input_tokens: 100000, + output_tokens: 0, + cache_creation_tokens: 0, + cache_read_tokens: 300000, + note: "billed but not streamed", + }, + ], + }, + ]; + const events = parseMessages(turns); + const [asst, recon] = events; + // deepseek-v4-pro: in 0.435 / out 0.87 / cacheRead 0.003625 per MTok. + // reconciliation = 100000*0.435 + 300000*0.003625 = 44587.5 tok-$ / 1e6. + expect(recon.model).toBe("deepseek/deepseek-v4-pro"); + expect(recon.costUsd).toBeCloseTo(0.0445875, 9); + // The whole point: assistant + reconciliation sum to the turn total + // (1000*0.87/1e6 + 0.0445875), instead of the reconciliation row reading "—". + expect((asst.costUsd ?? 0) + (recon.costUsd ?? 0)).toBeCloseTo(0.0454575, 9); + }); + + test("reconciliation cost stays null when the turn model is absent (legacy runs)", () => { + const turns: TurnEntry[] = [ + { + messages: [ + { + role: "reconciliation", + input_tokens: 512, + cache_creation_tokens: 1000, + note: "no model", + }, + ], + }, + ]; + const recon = parseMessages(turns)[0]; + expect(recon.model).toBeNull(); + expect(recon.costUsd).toBeNull(); + }); }); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index c8538f0d..e2997b25 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -1192,6 +1192,9 @@ interface MessageEntry { export interface TurnEntry { commands?: CommandEntry[]; messages?: MessageEntry[]; + // Model the turn ran on (iteration `model_used`). Used to price the synthetic + // reconciliation row, which carries no model of its own. + model_used?: string | null; token_usage?: TokenUsageEntry | null; result_summary?: { result?: string | null; @@ -1710,8 +1713,23 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] { reasoningTokens: null, thinkingOutputTokens: null, textOutputTokens: null, - model: null, - costUsd: null, + // Price the reconciliation row so summing the Cost column across + // the stream reproduces the turn total. Critical for providers + // (e.g. OpenRouter/LiteLLM) whose per-message stream is sparse and + // dumps most tokens into this one row — leaving it uncosted made + // the visible per-row costs sum to far less than the real total. + // Excluded from the cost simulator (by role), and the authoritative + // task total_cost_usd is unaffected (it reads the backend aggregate, + // not a sum of per-message costUsd), so there is no double-count. + model: turn.model_used ?? null, + costUsd: messageCostUsd({ + model: turn.model_used ?? null, + inputTokens: typeof msg.input_tokens === "number" ? msg.input_tokens : null, + outputTokens: typeof msg.output_tokens === "number" ? msg.output_tokens : null, + cacheWriteTokens: + typeof msg.cache_creation_tokens === "number" ? msg.cache_creation_tokens : null, + cacheReadTokens: typeof msg.cache_read_tokens === "number" ? msg.cache_read_tokens : null, + }), note: typeof msg.note === "string" ? msg.note : null, }); } From f6dc90db52922a4a2f550d8049750f2420dde31a Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 12:55:07 +0100 Subject: [PATCH 13/23] test(evalboard): relax pricing-parity guard to subset-match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exact key-set parity forced lib/pricing.ts to mirror every backend model, even ones the evalboard never renders — so unrelated backend additions had to be copied here just to keep the build green. Relax to subset semantics: every model priced in lib/pricing.ts must exist in pricing.py with identical rates (catches a wrong or orphan frontend rate), without requiring the frontend to carry models it doesn't display. Co-Authored-By: Claude Opus 4.8 --- .../lib/__tests__/pricing-parity.test.ts | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 5fe056d6..0fd3dba7 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -5,12 +5,17 @@ import { describe, expect, test } from "vitest"; import { PRICING } from "../pricing"; // Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative -// Python table in src/coder_eval/pricing.py. If the backend reprices a -// model (or adds one) and this mirror isn't updated, the frontend's "estimated" -// USD figures silently disagree with the backend's authoritative Cost on the -// same tokens. This test parses the Python table and asserts both tables have -// the same model ids and the same four per-MTok rates — turning silent drift -// into a failing build. +// Python table in src/coder_eval/pricing.py. It exists so the frontend's +// "estimated" USD figures agree with the backend's authoritative Cost on the +// same tokens. +// +// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts +// must exist in pricing.py with identical rates (a frontend rate that disagrees +// with the backend, or prices a model the backend doesn't, fails the build). +// The frontend is NOT required to mirror every backend model — it only needs to +// price the ones it displays, and the backend legitimately prices models the +// evalboard never renders. (Exact-match was too strict: it forced unrelated +// backend model additions into this file to keep the build green.) const here = dirname(fileURLToPath(import.meta.url)); const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py"); @@ -44,20 +49,24 @@ describe("pricing.ts ↔ pricing.py parity", () => { expect(Object.keys(py).length).toBeGreaterThan(10); }); - test("both tables list the same model ids", () => { - expect(Object.keys(PRICING).sort()).toEqual(Object.keys(py).sort()); + test("every model in lib/pricing.ts exists in pricing.py", () => { + const orphans = Object.keys(PRICING).filter((m) => !(m in py)); + expect( + orphans, + `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`, + ).toEqual([]); }); - test("every model has identical input/output/cacheWrite/cacheRead rates", () => { - for (const [model, [input, output, cw, cr]] of Object.entries(py)) { - const ts = PRICING[model]; - expect(ts, `missing in lib/pricing.ts: ${model}`).toBeDefined(); + test("shared models have identical input/output/cacheWrite/cacheRead rates", () => { + for (const [model, ts] of Object.entries(PRICING)) { + const rates = py[model]; + expect(rates, `not priced in pricing.py: ${model}`).toBeDefined(); expect([ ts.inputPerMTok, ts.outputPerMTok, ts.cacheWritePerMTok, ts.cacheReadPerMTok, - ]).toEqual([input, output, cw, cr]); + ]).toEqual(rates); } }); }); From dd4b69615caedb7cf39783c25cd2d5549598fa46 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 12:55:07 +0100 Subject: [PATCH 14/23] feat(litellm): export OPENROUTER_API_KEY into the proxy env The proxy's openrouter/* models authenticate with OPENROUTER_API_KEY, which the proxy reads from its own environment. Read it out of .env alongside the Bedrock creds so the OpenRouter models work end-to-end without a manual export. Co-Authored-By: Claude Opus 4.8 --- docker/start-litellm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/start-litellm.sh b/docker/start-litellm.sh index 2c306132..9315aea9 100755 --- a/docker/start-litellm.sh +++ b/docker/start-litellm.sh @@ -46,6 +46,8 @@ read_env() { export AWS_BEARER_TOKEN_BEDROCK="${AWS_BEARER_TOKEN_BEDROCK:-$(read_env AWS_BEARER_TOKEN_BEDROCK)}" export AWS_REGION="${AWS_REGION:-$(read_env AWS_REGION)}" export AWS_REGION="${AWS_REGION:-eu-north-1}" +# OpenRouter key for the openrouter/* models in the config (cost-optimization path). +export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-$(read_env OPENROUTER_API_KEY)}" # Master key = the key clients present. Default to .env's LITELLM_AUTH_TOKEN so the # client and proxy match; fall back to the local dev key. export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-$(read_env LITELLM_AUTH_TOKEN)}" From 7b355c99c0cd0b238e7c6076453af1681e309d4f Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 13:51:22 +0100 Subject: [PATCH 15/23] style(tests): ruff-format LiteLLMRoute test calls (collapse to one line) Pure formatting: ruff at line-length 120 collapses two multi-line _build_sdk_env(LiteLLMRoute(...)) calls onto single lines. No logic change. Co-Authored-By: Claude Opus 4.8 --- tests/test_litellm_route.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 3ce66a42..ae39b91e 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -161,9 +161,7 @@ def test_custom_route_forwards_path(self, monkeypatch): custom_path = f"/custom/bin{os.pathsep}/usr/bin" monkeypatch.setenv("PATH", custom_path) - env, _ = ClaudeCodeAgent._build_sdk_env( - LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") - ) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) assert env["PATH"] == custom_path def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch): @@ -171,9 +169,7 @@ def test_custom_route_neutralizes_inherited_anthropic_api_key(self, monkeypatch) empty in options.env (not merely omitted) — else it would fight the bearer auth_token against the gateway.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-key") - env, _ = ClaudeCodeAgent._build_sdk_env( - LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1") - ) + env, _ = ClaudeCodeAgent._build_sdk_env(LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1")) assert env["ANTHROPIC_API_KEY"] == "" From 6bb2e8cee067f291dc853ea4d821e30311ffb1c0 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 24 Jul 2026 15:41:09 +0100 Subject: [PATCH 16/23] =?UTF-8?q?docs(litellm):=20add=20docker/LITELLM.md?= =?UTF-8?q?=20=E2=80=94=20proxy=20setup=20+=20how=20to=20add=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explains when to run start-litellm.sh, the litellm-config.yaml structure, and the add-a-model recipe (Bedrock + OpenRouter, incl. mandatory pricing.py/evalboard pricing entries, proxy restart, and the OpenRouter provider-routing cost caveat). Adds a pointer to it from the config header. Co-Authored-By: Claude Opus 4.8 --- docker/LITELLM.md | 193 +++++++++++++++++++++++++++++++++++++ docker/litellm-config.yaml | 2 + 2 files changed, 195 insertions(+) create mode 100644 docker/LITELLM.md diff --git a/docker/LITELLM.md b/docker/LITELLM.md new file mode 100644 index 00000000..e132aca7 --- /dev/null +++ b/docker/LITELLM.md @@ -0,0 +1,193 @@ +# LiteLLM proxy — coder_eval `litellm` (open-weight) backend + +coder_eval evaluates UiPath skills on **non-Claude open-weight models** by driving +them through the **Claude Code SDK** harness (so the native `Skill` tool + +progressive disclosure fire unchanged). The SDK only speaks the **Anthropic +Messages** format (`POST /v1/messages`), but the target models don't: + +| Target | Wire format | Needs translation? | +|---|---|---| +| Bedrock open-weight (GLM, DeepSeek, Kimi) | Converse | yes | +| OpenRouter (any model) | OpenAI Chat Completions | yes | + +**LiteLLM is the translation shim.** The SDK is pointed at LiteLLM via +`ANTHROPIC_BASE_URL`; LiteLLM translates Anthropic ↔ Converse / OpenAI per model +and forwards to the real backend. + +``` +Claude Code SDK ──Anthropic /v1/messages──▶ LiteLLM (localhost:4000) ──▶ Bedrock Converse (eu-north-1) + ANTHROPIC_BASE_URL=localhost:4000 └──▶ OpenRouter (OpenAI format) +``` + +Files: +- `docker/litellm-config.yaml` — the model list + settings (this is the file you edit to add models). +- `docker/start-litellm.sh` — launches the proxy, reading credentials out of `.env`. + +--- + +## When to run `start-litellm.sh` + +Run the proxy **whenever you run coder_eval against the `litellm` backend** — i.e. +`API_BACKEND=litellm` in `.env`, or `coder-eval run ... --backend litellm`. coder_eval +does **not** own the proxy lifecycle: it expects an already-running proxy at +`LITELLM_BASE_URL` and **fails fast at startup** if it isn't reachable +(`LiteLLM proxy not reachable at ... — Start it (e.g. docker/start-litellm.sh)`). + +You do **not** need it for the `direct` (Anthropic) or `bedrock` (Claude-on-Bedrock) +backends — those talk to their APIs directly. + +### Prerequisites (in `.env`, never committed) + +`start-litellm.sh` reads these from `.env` and exports them into the proxy's own +environment before launching: + +| Var | For | Notes | +|---|---|---| +| `AWS_BEARER_TOKEN_BEDROCK` | Bedrock models | required if you use any `bedrock/*` model | +| `AWS_REGION` | Bedrock models | defaults to `eu-north-1` | +| `OPENROUTER_API_KEY` | OpenRouter models | required if you use any `openrouter/*` model | +| `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local` | + +### Start it + +```bash +bash docker/start-litellm.sh # foreground on :4000, Ctrl-C to stop +``` + +Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`, +`LITELLM_MASTER_KEY`. + +### Point coder_eval at it + +In `.env`: +``` +API_BACKEND=litellm +LITELLM_BASE_URL=http://localhost:4000 +LITELLM_AUTH_TOKEN= +LITELLM_MODEL=deepseek/deepseek-v4-pro # default model when a task doesn't pin one +``` +Then run — `--model` overrides the default: +```bash +coder-eval run tasks/.yaml --backend litellm --model deepseek/deepseek-v4-pro +``` + +### Verify the proxy + +```bash +KEY=$(grep -E '^LITELLM_AUTH_TOKEN=' .env | head -1 | sed -E 's/^LITELLM_AUTH_TOKEN=//; s/^"//; s/"$//') +# health +curl -s http://localhost:4000/health/liveliness +# which models are loaded +curl -s http://localhost:4000/v1/models -H "x-api-key: $KEY" \ + | python3 -c "import sys,json; print([m['id'] for m in json.load(sys.stdin)['data']])" +``` + +--- + +## Config file: `litellm-config.yaml` + +```yaml +model_list: + - model_name: # also the pricing-table key + litellm_params: + model: # e.g. bedrock/converse/... or openrouter/... + # ...provider-specific params + +litellm_settings: + drop_params: true # silently drop provider-unsupported params instead of 400 + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY # never hardcode a secret here +``` + +- **`model_name`** is the id clients (and `--model`) use, **and** the key coder_eval + prices on. Keep it identical to the `pricing.py` key. +- **`model`** is LiteLLM's routing target — the `provider/...` prefix selects the + translator + backend. + +--- + +## Adding a new model + +### 1. Add an entry to `model_list` + +**Bedrock open-weight** (Converse, eu-north-1): +```yaml + - model_name: some.bedrock-model + litellm_params: + model: bedrock/converse/some.bedrock-model + aws_region_name: eu-north-1 +``` + +**OpenRouter** (OpenAI format — pin the provider, see caveat below): +```yaml + - model_name: vendor/some-model + litellm_params: + model: openrouter/vendor/some-model + api_key: os.environ/OPENROUTER_API_KEY + extra_body: + provider: + sort: price # route to the cheapest upstream + allow_fallbacks: false # never silently upgrade to a pricier provider +``` +Find the exact OpenRouter slug + rates: +```bash +KEY=$(grep -E '^OPENROUTER_API_KEY=' .env | head -1 | sed -E 's/^OPENROUTER_API_KEY=//; s/^"//; s/"$//') +curl -s https://openrouter.ai/api/v1/models -H "Authorization: Bearer $KEY" \ + | python3 -c "import sys,json;[print(m['id'],m['pricing']) for m in json.load(sys.stdin)['data'] if 'SEARCH' in m['id'].lower()]" +``` + +### 2. Register pricing in **both** tables (rates must match) + +- `src/coder_eval/pricing.py` — add to the `_PRICING` dict, keyed on the + `model_name` (bare id as passed in `agent.model`): + ```python + "vendor/some-model": ModelPricing(input, output, cache_write, cache_read), + ``` + These implicit-caching providers charge no separate cache-write fee, so + `cache_write == input` (unused) and `cache_read` is the discounted rate. +- `evalboard/lib/pricing.ts` — add the same entry so the evalboard cost columns + populate. A test (`lib/__tests__/pricing-parity.test.ts`) fails the build if a + rate here disagrees with `pricing.py`. + +### 3. Restart the proxy + +LiteLLM reads `model_list` **once at startup**. After editing the yaml, `Ctrl-C` +and re-run `start-litellm.sh` — otherwise you get +`Invalid model name passed in model=...`. + +### 4. Verify + +`curl .../v1/models` should list the new `model_name`, then run a task with +`--model vendor/some-model`. + +--- + +## ⚠️ OpenRouter cost accuracy (provider routing) + +OpenRouter is a **multi-provider router**: the rate on a model's page is the +*cheapest* provider's headline, but a request can be routed to a pricier upstream +(observed: DeepSeek V4 Pro → Novita at ~$1.47/M input vs the $0.435/M headline, +3.4×). Because coder_eval prices `litellm`-backend runs from the **static** +`pricing.py` table, an un-pinned model can bill very differently from what +coder_eval reports. + +**Mitigation:** always pin providers (`extra_body.provider: {sort: price, +allow_fallbacks: false}`), which makes the billed rate deterministic and equal to +the headline the pricing table uses. To confirm the actual provider/cost for a +run, read the OpenRouter **Activity** page (`provider_name`, `usage`) or query +`GET /api/v1/models//endpoints` for per-provider rates. + +Token *counts* are exact and provider-independent; only the per-token *rate* is +subject to this. Bedrock models are single-provider and not affected. + +--- + +## Troubleshooting + +| Symptom | Cause / fix | +|---|---| +| `LiteLLM proxy not reachable at ...` (coder_eval startup) | Proxy not running — start it, or unset `LITELLM_BASE_URL`. | +| `Invalid model name passed in model=...` | Model added to yaml but proxy not restarted — restart it. | +| HTTP 401 / "Unable to locate credentials" | Missing `AWS_BEARER_TOKEN_BEDROCK` / `OPENROUTER_API_KEY` in `.env`, or key mismatch between `LITELLM_AUTH_TOKEN` (client) and the proxy's master key. | +| evalboard cost column blank for a model | Model missing from `evalboard/lib/pricing.ts`. | diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml index 9f889d36..82a31c95 100644 --- a/docker/litellm-config.yaml +++ b/docker/litellm-config.yaml @@ -1,5 +1,7 @@ # LiteLLM proxy config for the coder_eval `litellm` (open-weight) backend. # +# Full guide (when to run the proxy, how to add models): docker/LITELLM.md +# # Translates Anthropic Messages (what the Claude Code SDK speaks) <-> Bedrock # Converse (what the non-Claude open-weight models speak), so the native `Skill` # tool and UiPath skills fire unchanged. Verified 2026-07-21: HTTP 200 on both From f3e49f96f501f1cbf97699715a2a385d28a54a01 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Mon, 27 Jul 2026 14:50:17 +0100 Subject: [PATCH 17/23] refactor(litellm): move proxy config/script/docs to top-level litellm/ The LiteLLM proxy is a first-class, long-lived component (launched via uvx, not Docker), so it lives in its own top-level litellm/ dir rather than under docker/. Moves litellm-config.yaml + start-litellm.sh + LITELLM.md->README.md and updates all path references (start-script CONFIG default, run_command preflight message, config header, README). Co-Authored-By: Claude Opus 4.8 --- docker/LITELLM.md => litellm/README.md | 12 ++++++------ {docker => litellm}/litellm-config.yaml | 4 ++-- {docker => litellm}/start-litellm.sh | 4 ++-- src/coder_eval/cli/run_command.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) rename docker/LITELLM.md => litellm/README.md (94%) rename {docker => litellm}/litellm-config.yaml (95%) rename {docker => litellm}/start-litellm.sh (96%) diff --git a/docker/LITELLM.md b/litellm/README.md similarity index 94% rename from docker/LITELLM.md rename to litellm/README.md index e132aca7..6c3afdc1 100644 --- a/docker/LITELLM.md +++ b/litellm/README.md @@ -20,8 +20,8 @@ Claude Code SDK ──Anthropic /v1/messages──▶ LiteLLM (localhost:4000) ``` Files: -- `docker/litellm-config.yaml` — the model list + settings (this is the file you edit to add models). -- `docker/start-litellm.sh` — launches the proxy, reading credentials out of `.env`. +- `litellm/litellm-config.yaml` — the model list + settings (this is the file you edit to add models). +- `litellm/start-litellm.sh` — launches the proxy, reading credentials out of `.env`. --- @@ -31,12 +31,12 @@ Run the proxy **whenever you run coder_eval against the `litellm` backend** — `API_BACKEND=litellm` in `.env`, or `coder-eval run ... --backend litellm`. coder_eval does **not** own the proxy lifecycle: it expects an already-running proxy at `LITELLM_BASE_URL` and **fails fast at startup** if it isn't reachable -(`LiteLLM proxy not reachable at ... — Start it (e.g. docker/start-litellm.sh)`). +(`LiteLLM proxy not reachable at ... — Start it (e.g. litellm/start-litellm.sh)`). You do **not** need it for the `direct` (Anthropic) or `bedrock` (Claude-on-Bedrock) backends — those talk to their APIs directly. -### Prerequisites (in `.env`, never committed) +### Prerequisites `start-litellm.sh` reads these from `.env` and exports them into the proxy's own environment before launching: @@ -46,12 +46,12 @@ environment before launching: | `AWS_BEARER_TOKEN_BEDROCK` | Bedrock models | required if you use any `bedrock/*` model | | `AWS_REGION` | Bedrock models | defaults to `eu-north-1` | | `OPENROUTER_API_KEY` | OpenRouter models | required if you use any `openrouter/*` model | -| `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local` | +| `LITELLM_AUTH_TOKEN` | the virtual key clients present | becomes the proxy's `LITELLM_MASTER_KEY`; falls back to `sk-spike-local`. It can be customly designed.| ### Start it ```bash -bash docker/start-litellm.sh # foreground on :4000, Ctrl-C to stop +bash litellm/start-litellm.sh # foreground on :4000, Ctrl-C to stop ``` Overridable via env: `LITELLM_PORT` (default 4000), `LITELLM_CONFIG`, `ENV_FILE`, diff --git a/docker/litellm-config.yaml b/litellm/litellm-config.yaml similarity index 95% rename from docker/litellm-config.yaml rename to litellm/litellm-config.yaml index 82a31c95..a9189740 100644 --- a/docker/litellm-config.yaml +++ b/litellm/litellm-config.yaml @@ -1,6 +1,6 @@ # LiteLLM proxy config for the coder_eval `litellm` (open-weight) backend. # -# Full guide (when to run the proxy, how to add models): docker/LITELLM.md +# Full guide (when to run the proxy, how to add models): litellm/README.md # # Translates Anthropic Messages (what the Claude Code SDK speaks) <-> Bedrock # Converse (what the non-Claude open-weight models speak), so the native `Skill` @@ -19,7 +19,7 @@ # LITELLM_MASTER_KEY — the virtual key clients present as LITELLM_AUTH_TOKEN. # # Run manually: -# uvx --from 'litellm[proxy]' litellm --config docker/litellm-config.yaml --port 4000 +# uvx --from 'litellm[proxy]' litellm --config litellm/litellm-config.yaml --port 4000 model_list: # DeepSeek V3.2 — cost lead ($0.74 / $2.22 per Mtok). diff --git a/docker/start-litellm.sh b/litellm/start-litellm.sh similarity index 96% rename from docker/start-litellm.sh rename to litellm/start-litellm.sh index 9315aea9..da3192db 100755 --- a/docker/start-litellm.sh +++ b/litellm/start-litellm.sh @@ -10,7 +10,7 @@ # and exports them explicitly before launching, so that can't happen. # # Usage: -# docker/start-litellm.sh # foreground; Ctrl-C to stop +# litellm/start-litellm.sh # foreground; Ctrl-C to stop # Overridable via env: # LITELLM_PORT (default 4000), LITELLM_CONFIG, ENV_FILE, LITELLM_MASTER_KEY set -euo pipefail @@ -18,7 +18,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" ENV_FILE="${ENV_FILE:-$REPO_ROOT/.env}" -CONFIG="${LITELLM_CONFIG:-$REPO_ROOT/docker/litellm-config.yaml}" +CONFIG="${LITELLM_CONFIG:-$REPO_ROOT/litellm/litellm-config.yaml}" PORT="${LITELLM_PORT:-4000}" # Read a key from .env, returning ONLY the value: content between surrounding diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 1ffe9a2d..fa817305 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -90,7 +90,7 @@ def _litellm_preflight_error(current_settings: Settings) -> str | None: except (urllib.error.URLError, OSError) as exc: return ( f"LiteLLM proxy not reachable at {current_settings.litellm_base_url} (tried {url}): {exc}. " - "Start it (e.g. docker/start-litellm.sh) or unset LITELLM_BASE_URL." + "Start it (e.g. litellm/start-litellm.sh) or unset LITELLM_BASE_URL." ) return None From 8631122668d6e707c818a2e16a350bfc570967c8 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Mon, 27 Jul 2026 14:50:17 +0100 Subject: [PATCH 18/23] test(litellm): fix stale _build_sdk_env assertion for the LiteLLM route The custom-route env test asserted CLAUDE_CODE_USE_BEDROCK / AWS_BEARER_TOKEN_BEDROCK were absent, but the LiteLLM route deliberately blanks them to "" to neutralize inherited Bedrock creds (adfefd3) so the CLI can't bypass the proxy. Assert they are present-and-empty instead. Co-Authored-By: Claude Opus 4.8 --- tests/test_litellm_route.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index ae39b91e..b4927ce4 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -144,9 +144,10 @@ def test_custom_route_env_has_anthropic_vars_only(self): assert env["ANTHROPIC_MODEL"] == "deepseek.v3.2" assert env["ANTHROPIC_SMALL_FAST_MODEL"] == "deepseek.v3.2" assert model == "deepseek.v3.2" - # No Bedrock/Direct-specific vars. - assert "CLAUDE_CODE_USE_BEDROCK" not in env - assert "AWS_BEARER_TOKEN_BEDROCK" not in env + # Inherited Bedrock creds are neutralized (blanked to ""), not merely + # absent, so the CLI can't auto-select Bedrock-direct and bypass the proxy. + assert env["CLAUDE_CODE_USE_BEDROCK"] == "" + assert env["AWS_BEARER_TOKEN_BEDROCK"] == "" assert "AWS_REGION" not in env def test_custom_route_no_model_omits_model_vars(self): From cdc48f4958f0a5c992cfba21fc5eb08297a13bc0 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Mon, 27 Jul 2026 16:58:55 +0100 Subject: [PATCH 19/23] fix(litellm): silence bandit B310 on the preflight urlopen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The litellm preflight opens LITELLM_BASE_URL/health/liveliness to fail fast when the proxy is down. bandit flags urlopen (B310, permitted-schemes) at medium — but the URL is built from the operator-configured LITELLM_BASE_URL, not untrusted input, so scope a nosec B310 with justification. Unblocks the Quality Gate. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/cli/run_command.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 8e45bb3e..f8253c2b 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -84,7 +84,9 @@ def _litellm_preflight_error(current_settings: Settings) -> str | None: return None url = f"{current_settings.litellm_base_url.rstrip('/')}/health/liveliness" try: - urllib.request.urlopen(url, timeout=5).close() + # B310: url is built from the operator-configured LITELLM_BASE_URL (not + # untrusted input); this only probes reachability of that proxy endpoint. + urllib.request.urlopen(url, timeout=5).close() # nosec B310 except urllib.error.HTTPError: return None # server responded (up), just not 200 on this path except (urllib.error.URLError, OSError) as exc: From 020c4576316604558017c19a9a4ff0a8aa8c42b6 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 28 Jul 2026 15:42:33 +0100 Subject: [PATCH 20/23] fix(routing): pin llm_judge + simulated user to a constant Claude eval route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under --backend litellm the run's LiteLLMRoute reached the judge and the simulated user: the judge dispatch had no arm for it (silent score=0.0 with a misleading 'no usable API route'), and the simulator ran on the open-weight model under test — both break comparability against the Claude baseline. Add resolve_evaluation_route(settings, agent_route): Bedrock/Direct agents reuse their route unchanged; a LiteLLM agent pins evaluation to Bedrock (from the AWS bearer token) or Direct (ANTHROPIC_API_KEY), else DirectRoute(None) so llm_judge returns its clean 'unconfigured' error rather than 0.0. The orchestrator wires eval_route to SuccessChecker (llm_judge/agent_judge) and UserSimulator while the agent keeps the LiteLLM route; eval_routing is recorded in environment_info. Make the llm_judge dispatch exhaustive (explicit LiteLLMRoute + None arms, no wildcard) so pyright flags future route members. Extend test_route_seam_exhaustiveness to the judge-dispatch and _record_route_environment_info seams, and add a wiring test that the simulator receives the eval route, not the agent route. Addresses PR #54 review blocker 1. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/criteria/llm_judge.py | 13 +++- src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/routing.py | 30 ++++++++ src/coder_eval/orchestrator.py | 23 +++++- tests/test_litellm_route.py | 99 ++++++++++++++++++++++++- tests/test_route_seam_exhaustiveness.py | 51 +++++++++++-- 6 files changed, 204 insertions(+), 14 deletions(-) diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 73db9f4a..64d0abfa 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -28,6 +28,7 @@ JudgeCriterionResult, JudgeTranscript, JudgeVerdict, + LiteLLMRoute, LLMJudgeCriterion, TokenUsage, ) @@ -203,9 +204,15 @@ def _invoke_tool_channel( ) verdict, err = extract_verdict_from_anthropic_response(anthropic_response) response_usage = token_usage_from_anthropic_dict(anthropic_response) - case _: - # route is None or an unexpected type — the unconfigured-arm guard in - # _check_impl handles None before dispatch, so this is defensive only. + case LiteLLMRoute(): + # Defensive: the evaluation route is pinned to Bedrock/Direct by + # resolve_evaluation_route, so a LiteLLM route should never reach the + # judge. Fail loudly rather than silently scoring 0.0. (Explicit arm + # keeps the match exhaustive so pyright flags any future route member.) + return None, "llm_judge: evaluation route must be Bedrock/Direct, got LiteLLM", "(litellm route)", None + case None: + # Handled by the unconfigured-arm guard in _check_impl before dispatch; + # defensive only. return None, "llm_judge: no usable API route", "(no route)", None if verdict is not None: diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 7a6b2a07..3a9b3796 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -137,6 +137,7 @@ DirectRoute, JudgeTransport, LiteLLMRoute, + resolve_evaluation_route, resolve_route, to_bedrock_inference_profile, ) @@ -233,6 +234,7 @@ "LiteLLMRoute", "JudgeTransport", "resolve_route", + "resolve_evaluation_route", "to_bedrock_inference_profile", # Templates "BaseTemplateSource", diff --git a/src/coder_eval/models/routing.py b/src/coder_eval/models/routing.py index 6e7512e7..1b338e62 100644 --- a/src/coder_eval/models/routing.py +++ b/src/coder_eval/models/routing.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Literal from coder_eval.models.enums import ApiBackend +from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL if TYPE_CHECKING: @@ -178,6 +179,35 @@ def resolve_route(settings: Settings) -> ApiRoute: ) +def resolve_evaluation_route(settings: Settings, agent_route: ApiRoute) -> ApiRoute: + """Resolve the route used by the *evaluation* side — the ``llm_judge`` / + ``agent_judge`` criteria and the simulated user — which must stay on a + constant Claude backend regardless of the agent under test, so grading and + simulation stay comparable across models. + + - Agent on Bedrock/Direct: the judge already runs on Claude via that route, + so reuse it unchanged (no behavior change for existing runs). + - Agent on LiteLLM (open-weight): the agent route cannot serve a Claude + judge, so pin evaluation to Bedrock (preferred, from the AWS bearer token) + or Direct (``ANTHROPIC_API_KEY``). If neither is configured, fall back to a + ``DirectRoute`` with no judge transport so ``llm_judge`` fails with its + clean "unconfigured" error rather than silently scoring 0.0. + """ + if isinstance(agent_route, BedrockRoute | DirectRoute): + return agent_route + # agent_route is LiteLLMRoute → pin evaluation to a constant Claude backend. + if settings.aws_bearer_token_bedrock and settings.aws_region: + judge_model = settings.bedrock_model or DEFAULT_JUDGE_MODEL + qualified = to_bedrock_inference_profile(judge_model, settings.aws_region) + return BedrockRoute( + bearer_token=settings.aws_bearer_token_bedrock, + region=settings.aws_region, + model=qualified, + small_model=qualified, + ) + return DirectRoute(judge_transport=_resolve_direct_judge_transport(settings)) + + def _resolve_direct_judge_transport(settings: Settings) -> JudgeTransport | None: """Pick the judge transport for ``DirectRoute``. diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index d913fa42..f5a5206e 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -50,6 +50,7 @@ TokenUsage, TurnRecord, UserMessage, + resolve_evaluation_route, resolve_route, ) from .orchestration.early_stop import EarlyStopWatcher, validate_early_stop @@ -368,6 +369,11 @@ def __init__( # API routing (initialized in _setup) self.route: ApiRoute | None = None + # Route for the evaluation side (llm_judge / agent_judge / simulated user): + # pinned to a constant Claude backend so grading stays comparable when the + # agent runs on an open-weight (LiteLLM) model. Equals self.route for the + # Direct/Bedrock backends. + self.eval_route: ApiRoute | None = None # Result tracking self.result: EvaluationResult | None = None @@ -897,10 +903,11 @@ async def _setup(self) -> None: self.result.sandbox_path = str(self.sandbox.sandbox_dir) self.route = resolve_route(settings) + self.eval_route = resolve_evaluation_route(settings, self.route) logger.info( "API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None) ) - self.success_checker = SuccessChecker(self.sandbox, route=self.route) + self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) self._record_route_environment_info() return @@ -966,8 +973,9 @@ async def _setup_sandbox() -> Any: # Determine API routing from settings.api_backend enum self.route = resolve_route(settings) + self.eval_route = resolve_evaluation_route(settings, self.route) logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) - self.success_checker = SuccessChecker(self.sandbox, route=self.route) + self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) # Create and start the agent. For a no-op (type: none) task this dispatches # to NoOpAgent, whose start/communicate/stop are no-ops — the orchestrator @@ -1083,6 +1091,12 @@ def _record_route_environment_info(self) -> None: assert self.result is not None assert self.route is not None self.result.environment_info["api_routing"] = ROUTE_NAMES[type(self.route)] + # The evaluation side (llm_judge / agent_judge / simulated user) may run on + # a different, constant backend — pinned to Claude when the agent is on + # LiteLLM — so record it: a run then shows what actually graded/simulated + # it, distinct from the agent's api_routing. + if self.eval_route is not None: + self.result.environment_info["eval_routing"] = ROUTE_NAMES[type(self.eval_route)] if isinstance(self.route, BedrockRoute): self.result.environment_info["aws_region"] = self.route.region if self.route.model: @@ -1675,7 +1689,10 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: config=sim_config, task_description=self.task.description, initial_prompt=initial_prompt, - route=self.route, + # Pin the simulated user to the constant Claude eval route, not the + # agent's (possibly open-weight) route, so the simulator behaves + # identically across the models under test. + route=self.eval_route, ) await simulator.start() diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index b4927ce4..3e7978ba 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -18,12 +18,14 @@ from coder_eval.config import Settings from coder_eval.models import ( AgentKind, + BedrockRoute, + DirectRoute, LiteLLMRoute, TokenUsage, parse_agent_config, ) from coder_eval.models.enums import ApiBackend -from coder_eval.models.routing import ROUTE_NAMES, resolve_route +from coder_eval.models.routing import ROUTE_NAMES, resolve_evaluation_route, resolve_route from coder_eval.pricing import _normalize_model, calculate_cost @@ -31,6 +33,101 @@ def _make_agent(route, *, config_model: str | None = None) -> ClaudeCodeAgent: return ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, model=config_model), route=route) +class TestResolveEvaluationRoute: + """resolve_evaluation_route() pins the judge + simulated user to a constant + Claude backend regardless of the agent's backend, so grading/simulation stay + comparable across the models under test.""" + + @staticmethod + def _isolated_settings(monkeypatch, **kwargs): + # Skip .env and clear the credential env vars so presence/absence is + # driven purely by kwargs (config republishes .env into os.environ). + for var in ("AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION", "ANTHROPIC_API_KEY"): + monkeypatch.delenv(var, raising=False) + return Settings(_env_file=None, **kwargs) + + def test_bedrock_agent_route_is_reused_unchanged(self, monkeypatch): + route = BedrockRoute(bearer_token="tok", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.BEDROCK) + assert resolve_evaluation_route(settings, route) is route + + def test_direct_agent_route_is_reused_unchanged(self, monkeypatch): + route = DirectRoute(judge_transport="anthropic") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.DIRECT) + assert resolve_evaluation_route(settings, route) is route + + def test_litellm_agent_pins_evaluation_to_bedrock_when_aws_creds_present(self, monkeypatch): + agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + settings = self._isolated_settings( + monkeypatch, + api_backend=ApiBackend.LITELLM, + aws_bearer_token_bedrock="aws-tok", + aws_region="eu-north-1", + ) + ev = resolve_evaluation_route(settings, agent) + assert isinstance(ev, BedrockRoute) + assert ev.bearer_token == "aws-tok" + assert ev.region == "eu-north-1" + # Judge + simulator run on a Claude model, region-qualified. + assert ev.model == "eu.anthropic.claude-sonnet-4-6" + + def test_litellm_agent_falls_back_to_direct_when_only_anthropic_key(self, monkeypatch): + agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM, anthropic_api_key="sk-ant") + ev = resolve_evaluation_route(settings, agent) + assert isinstance(ev, DirectRoute) + assert ev.judge_transport == "anthropic" + + def test_litellm_agent_unconfigured_yields_direct_with_no_transport(self, monkeypatch): + # No Bedrock creds and no ANTHROPIC_API_KEY → DirectRoute(None), which makes + # llm_judge fail with its clean "unconfigured" error rather than scoring 0.0. + agent = LiteLLMRoute(base_url="http://x:4000", auth_token="sk-1", model="zai.glm-5") + settings = self._isolated_settings(monkeypatch, api_backend=ApiBackend.LITELLM) + ev = resolve_evaluation_route(settings, agent) + assert isinstance(ev, DirectRoute) + assert ev.judge_transport is None + + +class TestEvalRouteWiring: + """The orchestrator must hand the simulated user the eval_route (constant + Claude), never the agent's (possibly open-weight) route — guards the + simulation path the senior review flagged as untested.""" + + async def test_simulator_receives_eval_route_not_agent_route(self, monkeypatch): + from pathlib import Path + from types import SimpleNamespace + + from coder_eval import orchestrator as orch_mod + from coder_eval.orchestrator import Orchestrator + + eval_route = BedrockRoute(bearer_token="t", region="eu-north-1", model="eu.anthropic.claude-sonnet-4-6") + agent_route = LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5") + captured: dict = {} + + class _SpySimulator: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def start(self): + # Abort before the dialog loop; we only care which route was passed. + raise RuntimeError("__stop_dialog__") + + monkeypatch.setattr(orch_mod, "UserSimulator", _SpySimulator) + fake = SimpleNamespace( + result=SimpleNamespace(simulation=None), + task=SimpleNamespace(simulation=object(), agent=object(), description="d"), + agent=object(), + success_checker=object(), + eval_route=eval_route, + route=agent_route, + ) + with pytest.raises(RuntimeError, match="__stop_dialog__"): + await Orchestrator._simulation_dialog_loop(fake, initial_prompt="hi", sandbox_dir=Path("/tmp")) + # If someone reverts to route=self.route this flips to the litellm route. + assert captured["route"] is eval_route + assert captured["route"] is not agent_route + + class TestResolveRouteCustom: """resolve_route() builds a LiteLLMRoute for the CUSTOM backend.""" diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index dda703cb..e1b41f19 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -1,20 +1,28 @@ """Guardrail: every ``ApiRoute`` member must be handled at every route-matching seam. -The route union is matched via ``match``/``isinstance`` at several seams -(``_build_sdk_env``, ``_format_routing``, ``ROUTE_NAMES``). A new route added -without extending each seam would silently no-op (``_format_routing`` / -recording) or raise (``_build_sdk_env``). This test iterates the union and -forces a fixture + per-seam check for every member, so adding a 4th route -fails here until each seam is extended. +The route union is matched via ``match``/``isinstance`` at several seams: +``_build_sdk_env``, ``_format_routing``, ``ROUTE_NAMES``, the ``llm_judge`` +dispatch (``_invoke_tool_channel``), and ``Orchestrator._record_route_environment_info``. +A new route added without extending each seam would silently no-op +(``_record_route_environment_info``), silently mis-score (the judge dispatch), or +raise (``_build_sdk_env``). This test iterates the union and forces a fixture + +per-seam check for every member, so adding a 4th route fails here until each seam +is extended. (``_record_route_environment_info`` uses ``if/elif isinstance`` — not +a ``match`` — so pyright does NOT flag a missing route there; this runtime test is +its only guard.) """ from __future__ import annotations import typing +from types import SimpleNamespace +from unittest.mock import MagicMock from coder_eval.agents.claude_code_agent import ClaudeCodeAgent +from coder_eval.criteria import llm_judge +from coder_eval.criteria.llm_judge import _invoke_tool_channel from coder_eval.models import ROUTE_NAMES, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute -from coder_eval.orchestrator import _format_routing +from coder_eval.orchestrator import Orchestrator, _format_routing # One minimal instance per ApiRoute member. The set-equality assertion below @@ -48,3 +56,32 @@ def test_format_routing_handles_every_route(): for r in _INSTANCES: out = _format_routing(r) # type: ignore[arg-type] assert out and out.startswith(ROUTE_NAMES[type(r)]) + + +def test_invoke_tool_channel_handles_every_route(monkeypatch): + """The llm_judge dispatch must handle every route type — an unhandled member + would fall past all cases and raise UnboundLocalError. Network-touching arms + (Bedrock/Direct) are stubbed so this only checks dispatch coverage.""" + monkeypatch.setattr(llm_judge, "invoke_bedrock_judge", lambda **_: {}) + monkeypatch.setattr(llm_judge, "invoke_anthropic_judge", lambda **_: {}) + monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub")) + monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp: None) + criterion = MagicMock() + for r in _INSTANCES: + result = _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u") # type: ignore[arg-type] + # (verdict, parse_error, raw_text, response_usage) — a 4-tuple means the + # route matched an explicit arm rather than falling through. + assert isinstance(result, tuple) and len(result) == 4 + + +def test_record_route_environment_info_handles_every_route(): + """Orchestrator._record_route_environment_info records a route-specific + dimension for every route (not just the generic api_routing key). A new route + missing from its if/elif would record only api_routing — caught here since + pyright can't check the isinstance chain.""" + for r in _INSTANCES: + fake = SimpleNamespace(route=r, eval_route=r, result=SimpleNamespace(environment_info={}), agent=None) + Orchestrator._record_route_environment_info(fake) # type: ignore[arg-type] + env = fake.result.environment_info + assert env.get("api_routing") == ROUTE_NAMES[type(r)] + assert len(env) > 1, f"{type(r).__name__} recorded no route-specific env info" From 90ec8a801d3c23f3ad1dbdbf5209edc15f6e42a2 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 28 Jul 2026 15:53:17 +0100 Subject: [PATCH 21/23] fix(config): validate LITELLM_BASE_URL scheme (clean error, honest nosec) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scheme-less LITELLM_BASE_URL (e.g. 'localhost:4000') made the preflight's urlopen raise a bare ValueError ('unknown url type') that escaped as a traceback instead of a clean exit, and left environment_info's urlparse hostname empty. _validate_litellm_settings now rejects a non-http(s) / host-less URL with a field-named error (surfaced via validate_api_keys), and the CLI preflight guards the scheme before urlopen, returning a clean message. The preflight guard also makes the '# nosec B310' honest — the scheme is now constrained to http(s), which is exactly what B310 audits. Addresses PR #54 review blocker 2 (+ nit: nosec B310 scheme constraint). Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/cli/run_command.py | 18 +++++++++++++++--- src/coder_eval/config.py | 10 ++++++++++ tests/test_litellm_route.py | 19 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index f8253c2b..9b22929c 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -5,6 +5,7 @@ import os import sys import urllib.error +import urllib.parse import urllib.request from collections.abc import Callable from pathlib import Path @@ -82,10 +83,21 @@ def _litellm_preflight_error(current_settings: Settings) -> str | None: if current_settings.api_backend != ApiBackend.LITELLM or not current_settings.litellm_base_url: return None - url = f"{current_settings.litellm_base_url.rstrip('/')}/health/liveliness" + base_url = current_settings.litellm_base_url + # Reject a scheme-less/non-http(s) URL with a clear message instead of letting + # urlopen raise a bare ValueError ("unknown url type") that escapes as a + # traceback. Also makes the `# nosec B310` below honest — the scheme is now + # constrained to http(s), which is exactly what B310 audits. + if urllib.parse.urlsplit(base_url).scheme not in ("http", "https"): + return ( + f"LITELLM_BASE_URL must be an http(s) URL, got {base_url!r}. " + "Set it to e.g. http://localhost:4000 (or unset LITELLM_BASE_URL and switch backends)." + ) + url = f"{base_url.rstrip('/')}/health/liveliness" try: - # B310: url is built from the operator-configured LITELLM_BASE_URL (not - # untrusted input); this only probes reachability of that proxy endpoint. + # B310: url is built from the operator-configured LITELLM_BASE_URL, whose + # scheme is validated to http(s) just above — not untrusted input; this + # only probes reachability of that proxy endpoint. urllib.request.urlopen(url, timeout=5).close() # nosec B310 except urllib.error.HTTPError: return None # server responded (up), just not 200 on this path diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py index cb51193b..860b7ea1 100644 --- a/src/coder_eval/config.py +++ b/src/coder_eval/config.py @@ -7,6 +7,7 @@ import os from pathlib import Path from typing import Any +from urllib.parse import urlsplit from dotenv import dotenv_values, load_dotenv from pydantic import AliasChoices, Field @@ -200,6 +201,15 @@ def _validate_litellm_settings(self) -> None: f"LiteLLM-endpoint routing is enabled but missing required settings: {', '.join(missing)}." + " Please set them in your .env file." ) + # base_url is present (not in `missing`); reject a malformed one so the + # downstream preflight (urlopen) and environment_info (urlparse hostname) + # get a well-formed absolute URL instead of a raw ValueError / empty host. + parts = urlsplit(self.litellm_base_url or "") + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError( + f"LITELLM_BASE_URL must be an http(s) URL with a host, got {self.litellm_base_url!r}. " + + "Set it to e.g. http://localhost:4000 in your .env file." + ) def validate_api_keys(self, agent_type: str) -> None: """Validate that required API keys are present. diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 3e7978ba..d97bbbc3 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -205,6 +205,18 @@ def test_missing_base_url_raises_naming_it(self): with pytest.raises(ValueError, match="LITELLM_BASE_URL"): settings.validate_api_keys("claude-code") + def test_scheme_less_base_url_raises_naming_it(self): + # "localhost:4000" has no http(s) scheme — must fail at validation with a + # field-named message, not later as a raw urlopen ValueError. + settings = Settings( + api_backend=ApiBackend.LITELLM, + litellm_base_url="localhost:4000", + litellm_auth_token="sk-master", + litellm_model="zai.glm-5", + ) + with pytest.raises(ValueError, match="LITELLM_BASE_URL must be an http"): + settings.validate_api_keys("claude-code") + def test_all_present_does_not_raise(self): settings = Settings( api_backend=ApiBackend.LITELLM, @@ -363,6 +375,13 @@ def test_none_when_no_base_url(self): s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url=None, litellm_model="m") assert _litellm_preflight_error(s) is None + def test_scheme_less_base_url_returns_clean_error(self): + # Regression: a scheme-less URL used to make urlopen raise a bare + # ValueError that escaped as a traceback. Now it returns a clean message. + s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="localhost:4000", litellm_model="m") + err = _litellm_preflight_error(s) + assert err is not None and "http(s)" in err + def test_error_when_proxy_down(self, monkeypatch): monkeypatch.setattr(urllib.request, "urlopen", MagicMock(side_effect=urllib.error.URLError("refused"))) s = Settings(api_backend=ApiBackend.LITELLM, litellm_base_url="http://127.0.0.1:9", litellm_model="m") From 15466802e1a8edb745a6929422ed669a26d5b3c8 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 28 Jul 2026 16:09:03 +0100 Subject: [PATCH 22/23] fix(evalboard): mirror Bedrock open-weight rates + strip routing prefixes in resolvePricing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cost column rendered '—' for Bedrock-routed litellm runs: the 3 Bedrock open-weight ids (deepseek.v3.2, zai.glm-5, moonshotai.kimi-k2.5) were priced in pricing.py but absent from pricing.ts, and resolvePricing lacked the bedrock/converse/ + region prefix stripping that _normalize_model has (the recorded model_used arrives as e.g. 'converse/zai.glm-5'). Add the three ids, mirror _normalize_model's prefix stripping in resolvePricing, and add a parity direction (pricing.py -> pricing.ts) with an explicit DELIBERATELY_UNMIRRORED allowlist so a future litellm-relevant omission breaks the build instead of silently rendering '—'. Addresses PR #54 review blocker 4. Co-Authored-By: Claude Opus 4.8 --- .../lib/__tests__/pricing-parity.test.ts | 25 ++++++++++ evalboard/lib/__tests__/pricing.test.ts | 8 ++++ evalboard/lib/pricing.ts | 48 +++++++++++++++---- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 0fd3dba7..648f49f6 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -69,4 +69,29 @@ describe("pricing.ts ↔ pricing.py parity", () => { ]).toEqual(rates); } }); + + // Python-priced models we deliberately do NOT mirror to the frontend: heavy + // frontier Claude/GPT variants the evalboard never runs, so pricing them here + // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model + // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build — + // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids + // that previously rendered "—" for cost). + const DELIBERATELY_UNMIRRORED = new Set([ + "claude-sonnet-5", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5-pro", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + ]); + + test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => { + const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m)); + expect( + missing, + `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`, + ).toEqual([]); + }); }); diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 0238810e..66302cf1 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -38,6 +38,14 @@ describe("resolvePricing", () => { test("knows the current default opus id", () => { expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75); }); + + test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { + // The recorded model arrives prefixed on litellm/Bedrock runs; without the + // strip these rendered "—" for the whole cost column. + expect(resolvePricing("converse/zai.glm-5")?.outputPerMTok).toBe(3.84); + expect(resolvePricing("bedrock/converse/deepseek.v3.2")?.inputPerMTok).toBe(0.74); + expect(resolvePricing("eu.anthropic.claude-sonnet-4-6")?.outputPerMTok).toBe(15); + }); }); describe("tokenBucketUsd", () => { diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 26baa144..93ac23c8 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -58,6 +58,12 @@ export const PRICING: Record = { "moonshotai/kimi-k3": p(3, 15, 3, 0.3), "z-ai/glm-5.2": p(0.826, 2.596, 0.826, 0.1534), "deepseek/deepseek-v4-pro": p(0.435, 0.87, 0.435, 0.003625), + // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py. + // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so + // resolvePricing strips the routing/region prefixes before lookup. + "deepseek.v3.2": p(0.74, 2.22, 0.74, 0), + "zai.glm-5": p(1.2, 3.84, 1.2, 0), + "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0), }; function p( @@ -74,21 +80,45 @@ function p( }; } -// Resolve pricing for a model id, tolerating undated aliases (the recorded -// model is usually the canonical id like "claude-sonnet-4-6", but be lenient -// about a trailing date suffix). Matches the Python source's exact-match -// semantics, plus a date-suffix strip — deliberately NO loose prefix match: a -// substring fallback would silently price `gpt-5-mini` at full `gpt-5` rates, -// presenting a multi-x overcharge as an authoritative-looking figure. Unknown -// ids return null (render "—") rather than a wrong number. +// Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare +// pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the +// recorded model_used arrives qualified (e.g. "converse/zai.glm-5", +// "eu.anthropic.claude-sonnet-4-6"). Idempotent on already-bare ids. +const _ROUTING_PREFIXES = ["bedrock/converse/", "bedrock/", "converse/"]; +const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; +function normalizeModel(model: string): string { + let m = model.trim(); + for (const pre of _ROUTING_PREFIXES) { + if (m.startsWith(pre)) { + m = m.slice(pre.length); + break; + } + } + for (const pre of _REGION_PREFIXES) { + if (m.startsWith(pre)) { + m = m.slice(pre.length); + break; + } + } + if (m.startsWith("anthropic.")) m = m.slice("anthropic.".length); + return m; +} + +// Resolve pricing for a model id, tolerating routing/region prefixes and undated +// aliases (the recorded model is usually the canonical id like "claude-sonnet-4-6", +// but LiteLLM/Bedrock runs record it prefixed, and some carry a trailing date). +// Deliberately NO loose *substring* match: that would silently price `gpt-5-mini` +// at full `gpt-5` rates, presenting a multi-x overcharge as an authoritative-looking +// figure. Unknown ids return null (render "—") rather than a wrong number. // // Object.hasOwn (not `PRICING[model]` truthiness) guards against a degenerate // id like "constructor"/"toString" resolving to an inherited prototype member. export function resolvePricing(model: string | null): Pricing | null { if (!model) return null; - if (Object.hasOwn(PRICING, model)) return PRICING[model]; + const norm = normalizeModel(model); + if (Object.hasOwn(PRICING, norm)) return PRICING[norm]; // Try stripping a trailing -YYYYMMDD date. - const undated = model.replace(/-\d{8}$/, ""); + const undated = norm.replace(/-\d{8}$/, ""); if (Object.hasOwn(PRICING, undated)) return PRICING[undated]; return null; } From 46ff8294f7e8479996a183e6b9d2a4f85ce36d39 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 28 Jul 2026 16:09:04 +0100 Subject: [PATCH 23/23] fix(agents): test litellm reprice wiring; warn on rate-card miss; dedupe pricing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reprice wiring (finalize -> _reprice_for_litellm) was untested — inverting the guard left the suite green while every litellm turn silently kept the SDK's Claude-priced cost, and an unpriced model yielded total_cost_usd=None that made the orchestrator skip the max_usd gate with no diagnostic. Extract _finalize_token_usage on _ClaudeTurnState so the wiring is directly testable, and add tests asserting a priced model reprices to the rate-table figure and an unpriced one yields None. Give _reprice_for_litellm the rate-card-miss warning its sibling _backfill_cost has, make it return None (the result was discarded), and extract the shared _price_from_buckets helper. Addresses PR #54 review blocker 3. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/agents/claude_code_agent.py | 96 +++++++++++++--------- tests/test_litellm_route.py | 39 +++++++++ 2 files changed, 97 insertions(+), 38 deletions(-) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 68d6687d..9c2e2887 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -552,6 +552,30 @@ def on_user_message(self, message: Message) -> None: ) ) + def _finalize_token_usage(self) -> TokenUsage: + """Build the turn's cumulative TokenUsage, repricing for LiteLLM. + + Extracted from ``finalize`` so the LiteLLM repricing *wiring* (not just the + static ``_reprice_for_litellm`` helper) is directly testable. The SDK's + cost estimate assumes Claude pricing and is wrong for an open-weight model + behind LiteLLM, so reprice the top-line from the token buckets at the + model's real rate — buckets untouched, so the reconciliation invariant + holds. + """ + usage = ( + self._agent._build_token_usage( + self.sdk_messages, + self.sdk_result_usage, + self.sdk_result_cost, + self.sdk_result_model_usage, + self.effective_model, + ) + or TokenUsage() + ) + if isinstance(self._agent.route, LiteLLMRoute): + self._agent._reprice_for_litellm(usage, self.effective_model) + return usage + def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -592,22 +616,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso ) self.current_turn_id = None - usage = ( - self._agent._build_token_usage( - self.sdk_messages, - self.sdk_result_usage, - self.sdk_result_cost, - self.sdk_result_model_usage, - self.effective_model, - ) - or TokenUsage() - ) - # The SDK's cost estimate assumes Claude pricing; it is wrong for an - # open-weight model behind LiteLLM. Reprice the top-line from the token - # buckets at the model's real rate (buckets untouched → reconciliation - # invariant holds). - if isinstance(self._agent.route, LiteLLMRoute): - self._agent._reprice_for_litellm(usage, self.effective_model) + usage = self._finalize_token_usage() try: agent_output = self._agent._format_messages(self.messages) @@ -1417,6 +1426,24 @@ def _build_token_usage( model, ) + @staticmethod + def _price_from_buckets(usage: TokenUsage, model: str | None) -> float | None: + """Price the four token buckets at ``model``'s list rate. + + Shared by ``_backfill_cost`` (price-if-absent) and ``_reprice_for_litellm`` + (always-reprice). Returns ``None`` when ``model`` is unset or absent from + the rate card. + """ + if not model: + return None + return calculate_cost( + model, + uncached_input_tokens=usage.uncached_input_tokens, + output_tokens=usage.output_tokens, + cache_creation_tokens=usage.cache_creation_input_tokens, + cache_read_tokens=usage.cache_read_input_tokens, + ) + @staticmethod def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage: """Price the token buckets when the SDK gave no cost (timeout / kill). @@ -1430,13 +1457,7 @@ def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage: """ if usage.total_cost_usd is not None or not model: return usage - cost = calculate_cost( - model, - uncached_input_tokens=usage.uncached_input_tokens, - output_tokens=usage.output_tokens, - cache_creation_tokens=usage.cache_creation_input_tokens, - cache_read_tokens=usage.cache_read_input_tokens, - ) + cost = ClaudeCodeAgent._price_from_buckets(usage, model) if cost is not None: usage.total_cost_usd = cost else: @@ -1447,29 +1468,28 @@ def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage: return usage @staticmethod - def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> TokenUsage: - """Recompute the top-line cost for the LiteLLM backend. + def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> None: + """Recompute the top-line cost for the LiteLLM backend, in place. The Claude Agent SDK's ``costUSD``/``total_cost_usd`` is a client-side estimate that assumes Claude/Anthropic pricing, so it is wrong for an open-weight model driven through LiteLLM. Reprice from the (already authoritative) token buckets at the model's real rate. The token buckets are left untouched, so the per-message stream / reconciliation invariant - is unaffected — only the cost scalar changes. An unknown/unpriced model - yields ``None`` (an honest "N/A") rather than the misleading SDK figure. + is unaffected — only the cost scalar changes. + + An unknown/unpriced model sets the cost to ``None`` (an honest "N/A") + **and logs a warning** — mirroring ``_backfill_cost`` — because a proxy + model missing from the rate card otherwise silently yields + ``total_cost_usd = None``, which makes the orchestrator skip the + ``max_usd`` gate with no diagnostic. """ - usage.total_cost_usd = ( - calculate_cost( - model, - uncached_input_tokens=usage.uncached_input_tokens, - output_tokens=usage.output_tokens, - cache_creation_tokens=usage.cache_creation_input_tokens, - cache_read_tokens=usage.cache_read_input_tokens, + cost = ClaudeCodeAgent._price_from_buckets(usage, model) + usage.total_cost_usd = cost + if cost is None: + logger.warning( + "No pricing for litellm model %r; turn cost left unset (max_usd gate will be skipped)", model ) - if model - else None - ) - return usage def get_sdk_options(self) -> dict[str, Any] | None: """Get the raw SDK options used for the last agent query. diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index d97bbbc3..4f9b3835 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -364,6 +364,45 @@ def test_none_model_yields_none(self): assert u.total_cost_usd is None +class TestRepriceWiring: + """The finalize path (not just the static helper) reprices a litellm turn. A + regression that skips the reprice would silently persist the SDK's Claude cost + and disable the max_usd gate — the static-only tests above wouldn't catch it.""" + + def _usage_after_finalize(self, effective_model: str | None) -> TokenUsage: + from types import SimpleNamespace + + from coder_eval.agents.claude_code_agent import _ClaudeTurnState + + agent = _make_agent( + LiteLLMRoute(base_url="http://x:4000", auth_token="k", model="zai.glm-5"), + config_model="zai.glm-5", + ) + stub = SimpleNamespace( + _agent=agent, + sdk_messages=[], + sdk_result_usage=None, + sdk_result_cost=None, + # model_usage carries the SDK's Claude-priced estimate (3.68); the + # reprice must override it from the litellm rate table. + sdk_result_model_usage={"m": {"inputTokens": 1_000_000, "outputTokens": 1_000_000, "costUSD": 3.68}}, + effective_model=effective_model, + ) + return _ClaudeTurnState._finalize_token_usage(stub) # type: ignore[arg-type] + + def test_finalize_reprices_priced_litellm_model(self): + usage = self._usage_after_finalize("zai.glm-5") + # litellm rate (1.2 + 3.84), NOT the SDK's Claude estimate of 3.68. + assert usage.total_cost_usd == pytest.approx(1.2 + 3.84) + assert usage.uncached_input_tokens == 1_000_000 # token buckets untouched + + def test_finalize_unpriced_litellm_model_yields_none(self): + # An unpriced model → None (not the misleading 3.68), so the orchestrator + # skips the max_usd gate rather than gating on a wrong figure. + usage = self._usage_after_finalize("some-unpriced-model") + assert usage.total_cost_usd is None + + class TestLitellmPreflight: """External-proxy reachability preflight — fail fast instead of hanging on a dead proxy."""