diff --git a/CLAUDE.md b/CLAUDE.md index 4301431..220212a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,4 @@ Docker image → GHCR (`ghcr.io/c4g/va-dat`) → Coolify at `https://va-dat.c4g. - **Silent dry runs.** A request with no resolvable key returns `200 OK` with no LLM findings and no CSV. The only signal is `summary.dry_run` in the response. - HTML test files can be enormous (500K+ tokens) — do not read them fully into context. - Some files may have non-UTF-8 encoding; always use `errors='replace'`. -- Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-4-20250514`, `api_server.py` uses `claude-haiku-4-5-20251001`. +- Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-5`, `api_server.py` uses `claude-haiku-4-5-20251001`. diff --git a/README.md b/README.md index 9e4598c..909446b 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ uv run python entry_points/generate_report.py --output-dir ./output --report-dir |------|---------|-------------| | `--html` | (required) | Path to the HTML file to analyze | | `--output-dir` | `./output` | Directory for results | -| `--model` | `claude-sonnet-4-20250514` | Anthropic model to use | +| `--model` | `claude-sonnet-5` | Anthropic model to use | | `--dry-run` | off | Generate prompts without calling the API | | `--include-summaries` | off | Include the 3 cross-cutting summary prompts | | `--show-cost` | off | Print estimated dollar cost of the run based on model pricing | diff --git a/entry_points/generate_report.py b/entry_points/generate_report.py index 69b0323..fb86401 100644 --- a/entry_points/generate_report.py +++ b/entry_points/generate_report.py @@ -880,13 +880,17 @@ def deduplicate_with_llm( response_text = response.choices[0].message.content or "[]" else: import anthropic + from processing_scripts.llm_client.client import supports_temperature + client = anthropic.Anthropic(api_key=api_key) - message = client.messages.create( - model=model, - max_tokens=512, - temperature=0.0, - messages=[{"role": "user", "content": prompt}], - ) + request_kwargs = { + "model": model, + "max_tokens": 512, + "messages": [{"role": "user", "content": prompt}], + } + if supports_temperature(model): + request_kwargs["temperature"] = 0.0 + message = client.messages.create(**request_kwargs) response_text = "".join( block.text for block in message.content if block.type == "text" ) @@ -1069,8 +1073,8 @@ def main(): ) parser.add_argument( "--model", - default="claude-sonnet-4-20250514", - help="Model to use for LLM deduplication (default: claude-sonnet-4-20250514)", + default="claude-sonnet-5", + help="Model to use for LLM deduplication (default: claude-sonnet-5)", ) args = parser.parse_args() diff --git a/entry_points/run_pipeline.py b/entry_points/run_pipeline.py index c2c758e..1334021 100644 --- a/entry_points/run_pipeline.py +++ b/entry_points/run_pipeline.py @@ -53,6 +53,9 @@ # Pricing per million tokens: (input, output) MODEL_PRICING = { + "claude-fable-5": (10.00, 50.00), + "claude-opus-5": (5.00, 25.00), + "claude-sonnet-5": (3.00, 15.00), "claude-sonnet-4": (3.00, 15.00), "claude-opus-4": (15.00, 75.00), "claude-haiku-4": (0.80, 4.00), @@ -149,12 +152,17 @@ def call(self, prompt: str) -> dict: def _call_anthropic(self, prompt: str, start: float) -> dict: """Call the Anthropic Messages API.""" - message = self._client.messages.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=0.1, - messages=[{"role": "user", "content": prompt}], - ) + from processing_scripts.llm_client.client import supports_temperature + + request_kwargs = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + if supports_temperature(self.model): + request_kwargs["temperature"] = 0.1 + + message = self._client.messages.create(**request_kwargs) response_text = "".join( block.text for block in message.content if block.type == "text" @@ -519,8 +527,11 @@ def main(): parser.add_argument( "--model", type=str, - default="claude-sonnet-4-20250514", - help="Model to use (default: claude-sonnet-4-20250514)", + default="claude-sonnet-5", + help=( + "Model to use (default: claude-sonnet-5). Accepts any " + "Claude, OpenAI, or Gemini model ID, e.g. claude-fable-5." + ), ) parser.add_argument( "--dry-run", diff --git a/index.html b/index.html index 0112713..e5cdb93 100644 --- a/index.html +++ b/index.html @@ -550,8 +550,14 @@

Accessibility Audit Tool

- + + diff --git a/processing_scripts/llm_client/__init__.py b/processing_scripts/llm_client/__init__.py index 73776d6..01e8aa9 100644 --- a/processing_scripts/llm_client/__init__.py +++ b/processing_scripts/llm_client/__init__.py @@ -7,7 +7,7 @@ ----- from llm_client import create_audit_client, load_all_prompts, run_all - client = create_audit_client(model="claude-sonnet-4-6") # or "gpt-4o" + client = create_audit_client(model="claude-sonnet-5") # or "gpt-4o" prompts = load_all_prompts(PROMPTS_DIR) report = run_all(client, prompts, all_slices) """ diff --git a/processing_scripts/llm_client/client.py b/processing_scripts/llm_client/client.py index 7a2c2ad..9f0aae1 100644 --- a/processing_scripts/llm_client/client.py +++ b/processing_scripts/llm_client/client.py @@ -34,6 +34,25 @@ def is_gemini_model(model: str) -> bool: return model.startswith(_GEMINI_PREFIXES) +# Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) that reject +# temperature/top_p/top_k (400 error) in favor of adaptive thinking + effort. +_NO_SAMPLING_PARAMS_MODELS = { + "claude-fable-5", + "claude-mythos-5", + "claude-opus-5", + "claude-opus-4-8", + "claude-opus-4-7", + "claude-opus-4-6", + "claude-sonnet-5", + "claude-sonnet-4-6", +} + + +def supports_temperature(model: str) -> bool: + """Return False for Claude models that reject the `temperature` param.""" + return model not in _NO_SAMPLING_PARAMS_MODELS + + class AuditClient: """ Sends filled accessibility-audit prompts to the Claude API and returns @@ -42,16 +61,21 @@ class AuditClient: Parameters ---------- model : str - Claude model ID to use (default: claude-sonnet-4-6). + Claude model ID to use (default: claude-sonnet-5). Also accepts + ``claude-opus-5`` and ``claude-fable-5``, Anthropic's most capable + model. Newer Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) + reject a ``temperature`` param entirely (see ``supports_temperature``), + which this client already accounts for. temperature : float Sampling temperature (default: 0.1 for consistent structured output). + Silently omitted from the request for models that reject it. max_tokens : int Maximum tokens to generate per response (default: 8192). """ def __init__( self, - model: str = "claude-sonnet-4-6", + model: str = "claude-sonnet-5", temperature: float = 0.1, max_tokens: int = 8192, ): @@ -82,12 +106,15 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: """ filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) - message = self._client.messages.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=self.temperature, - messages=[{"role": "user", "content": filled}], - ) + request_kwargs = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": [{"role": "user", "content": filled}], + } + if supports_temperature(self.model): + request_kwargs["temperature"] = self.temperature + + message = self._client.messages.create(**request_kwargs) usage = { "input_tokens": message.usage.input_tokens, @@ -243,7 +270,7 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: def create_audit_client( - model: str = "claude-sonnet-4-6", + model: str = "claude-sonnet-5", temperature: float = 0.1, max_tokens: int = 8192, ) -> AuditClient | OpenAIAuditClient | GeminiAuditClient: