Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,4 @@ Docker image → GHCR (`ghcr.io/c4g/va-dat`) → Coolify at `https://va-dat.c4g.
- **Silent dry runs.** A request with no resolvable key returns `200 OK` with no LLM findings and no CSV. The only signal is `summary.dry_run` in the response.
- HTML test files can be enormous (500K+ tokens) — do not read them fully into context.
- Some files may have non-UTF-8 encoding; always use `errors='replace'`.
- Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-4-20250514`, `api_server.py` uses `claude-haiku-4-5-20251001`.
- Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-5`, `api_server.py` uses `claude-haiku-4-5-20251001`.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ uv run python entry_points/generate_report.py --output-dir ./output --report-dir
|------|---------|-------------|
| `--html` | (required) | Path to the HTML file to analyze |
| `--output-dir` | `./output` | Directory for results |
| `--model` | `claude-sonnet-4-20250514` | Anthropic model to use |
| `--model` | `claude-sonnet-5` | Anthropic model to use |
| `--dry-run` | off | Generate prompts without calling the API |
| `--include-summaries` | off | Include the 3 cross-cutting summary prompts |
| `--show-cost` | off | Print estimated dollar cost of the run based on model pricing |
Expand Down
20 changes: 12 additions & 8 deletions entry_points/generate_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -880,13 +880,17 @@ def deduplicate_with_llm(
response_text = response.choices[0].message.content or "[]"
else:
import anthropic
from processing_scripts.llm_client.client import supports_temperature

client = anthropic.Anthropic(api_key=api_key)
message = client.messages.create(
model=model,
max_tokens=512,
temperature=0.0,
messages=[{"role": "user", "content": prompt}],
)
request_kwargs = {
"model": model,
"max_tokens": 512,
"messages": [{"role": "user", "content": prompt}],
}
if supports_temperature(model):
request_kwargs["temperature"] = 0.0
message = client.messages.create(**request_kwargs)
response_text = "".join(
block.text for block in message.content if block.type == "text"
)
Expand Down Expand Up @@ -1069,8 +1073,8 @@ def main():
)
parser.add_argument(
"--model",
default="claude-sonnet-4-20250514",
help="Model to use for LLM deduplication (default: claude-sonnet-4-20250514)",
default="claude-sonnet-5",
help="Model to use for LLM deduplication (default: claude-sonnet-5)",
)
args = parser.parse_args()

Expand Down
27 changes: 19 additions & 8 deletions entry_points/run_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@

# Pricing per million tokens: (input, output)
MODEL_PRICING = {
"claude-fable-5": (10.00, 50.00),
"claude-opus-5": (5.00, 25.00),
"claude-sonnet-5": (3.00, 15.00),
"claude-sonnet-4": (3.00, 15.00),
"claude-opus-4": (15.00, 75.00),
"claude-haiku-4": (0.80, 4.00),
Expand Down Expand Up @@ -149,12 +152,17 @@ def call(self, prompt: str) -> dict:

def _call_anthropic(self, prompt: str, start: float) -> dict:
"""Call the Anthropic Messages API."""
message = self._client.messages.create(
model=self.model,
max_tokens=self.max_tokens,
temperature=0.1,
messages=[{"role": "user", "content": prompt}],
)
from processing_scripts.llm_client.client import supports_temperature

request_kwargs = {
"model": self.model,
"max_tokens": self.max_tokens,
"messages": [{"role": "user", "content": prompt}],
}
if supports_temperature(self.model):
request_kwargs["temperature"] = 0.1

message = self._client.messages.create(**request_kwargs)

response_text = "".join(
block.text for block in message.content if block.type == "text"
Expand Down Expand Up @@ -519,8 +527,11 @@ def main():
parser.add_argument(
"--model",
type=str,
default="claude-sonnet-4-20250514",
help="Model to use (default: claude-sonnet-4-20250514)",
default="claude-sonnet-5",
help=(
"Model to use (default: claude-sonnet-5). Accepts any "
"Claude, OpenAI, or Gemini model ID, e.g. claude-fable-5."
),
)
parser.add_argument(
"--dry-run",
Expand Down
10 changes: 8 additions & 2 deletions index.html
Original file line number Diff line number Diff line change
Expand Up @@ -550,8 +550,14 @@ <h2 id="audit-heading">Accessibility Audit Tool</h2>
<option value="claude-haiku-4-5-20251001">
Claude Haiku 4.5 — faster, cheaper
</option>
<option value="claude-sonnet-4-20250514">
Claude Sonnet 4 — higher quality
<option value="claude-sonnet-5">
Claude Sonnet 5 — higher quality
</option>
<option value="claude-opus-5">
Claude Opus 5 — complex agentic/coding tasks
</option>
<option value="claude-fable-5">
Claude Fable 5 — most capable, highest cost
</option>
</optgroup>
<optgroup label="OpenAI">
Expand Down
2 changes: 1 addition & 1 deletion processing_scripts/llm_client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
-----
from llm_client import create_audit_client, load_all_prompts, run_all

client = create_audit_client(model="claude-sonnet-4-6") # or "gpt-4o"
client = create_audit_client(model="claude-sonnet-5") # or "gpt-4o"
prompts = load_all_prompts(PROMPTS_DIR)
report = run_all(client, prompts, all_slices)
"""
Expand Down
45 changes: 36 additions & 9 deletions processing_scripts/llm_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,25 @@ def is_gemini_model(model: str) -> bool:
return model.startswith(_GEMINI_PREFIXES)


# Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) that reject
# temperature/top_p/top_k (400 error) in favor of adaptive thinking + effort.
_NO_SAMPLING_PARAMS_MODELS = {
"claude-fable-5",
"claude-mythos-5",
"claude-opus-5",
"claude-opus-4-8",
"claude-opus-4-7",
"claude-opus-4-6",
"claude-sonnet-5",
"claude-sonnet-4-6",
}


def supports_temperature(model: str) -> bool:
"""Return False for Claude models that reject the `temperature` param."""
return model not in _NO_SAMPLING_PARAMS_MODELS


class AuditClient:
"""
Sends filled accessibility-audit prompts to the Claude API and returns
Expand All @@ -42,16 +61,21 @@ class AuditClient:
Parameters
----------
model : str
Claude model ID to use (default: claude-sonnet-4-6).
Claude model ID to use (default: claude-sonnet-5). Also accepts
``claude-opus-5`` and ``claude-fable-5``, Anthropic's most capable
model. Newer Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5)
reject a ``temperature`` param entirely (see ``supports_temperature``),
which this client already accounts for.
temperature : float
Sampling temperature (default: 0.1 for consistent structured output).
Silently omitted from the request for models that reject it.
max_tokens : int
Maximum tokens to generate per response (default: 8192).
"""

def __init__(
self,
model: str = "claude-sonnet-4-6",
model: str = "claude-sonnet-5",
temperature: float = 0.1,
max_tokens: int = 8192,
):
Expand Down Expand Up @@ -82,12 +106,15 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]:
"""
filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":")))

message = self._client.messages.create(
model=self.model,
max_tokens=self.max_tokens,
temperature=self.temperature,
messages=[{"role": "user", "content": filled}],
)
request_kwargs = {
"model": self.model,
"max_tokens": self.max_tokens,
"messages": [{"role": "user", "content": filled}],
}
if supports_temperature(self.model):
request_kwargs["temperature"] = self.temperature

message = self._client.messages.create(**request_kwargs)

usage = {
"input_tokens": message.usage.input_tokens,
Expand Down Expand Up @@ -243,7 +270,7 @@ def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]:


def create_audit_client(
model: str = "claude-sonnet-4-6",
model: str = "claude-sonnet-5",
temperature: float = 0.1,
max_tokens: int = 8192,
) -> AuditClient | OpenAIAuditClient | GeminiAuditClient:
Expand Down