diff --git a/engine/argument_risk_engine/classification/classifier.py b/engine/argument_risk_engine/classification/classifier.py index e69de29..8e7dc58 100644 --- a/engine/argument_risk_engine/classification/classifier.py +++ b/engine/argument_risk_engine/classification/classifier.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import json +from typing import Any, Literal + +from argument_risk_engine.classification.deterministic import classify_deterministic +from argument_risk_engine.classification.llm_client import LLMClient, LLMClientError +from argument_risk_engine.classification.model_provider import ProviderProfile +from argument_risk_engine.classification.prompts import build_classification_prompt, build_messages +from argument_risk_engine.taxonomy.models import TaxonomyEntry +from pydantic import BaseModel, Field + +ClassificationMode = Literal["deterministic_baseline", "llm"] + + +class RiskAssessment(BaseModel): + risk_id: str + category: str + label: str + severity: str + confidence: float + evidence_span: str + evidence_start_char: int + evidence_end_char: int + explanation: str + false_positive_warning: str = "" + classification_mode: str + model_provider_id: str + model_name: str + llm_used: bool + deterministic_fallback_used: bool + insufficient_evidence: bool = False + + +class ClassificationResult(BaseModel): + assessments: list[RiskAssessment] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + model_error: str = "" + classification_mode: str = "deterministic_baseline" + model_provider_id: str = "deterministic_baseline" + model_name: str = "local-keyword" + llm_used: bool = False + deterministic_fallback_used: bool = False + + +class ClassifierConfig(BaseModel): + mode: ClassificationMode = "deterministic_baseline" + fallback_to_deterministic: bool = True + + +class ArgumentRiskClassifier: + """Taxonomy-grounded classification layer with offline and optional LLM modes.""" + + def __init__( + self, + *, + provider_profile: ProviderProfile | None = None, + llm_client: LLMClient | None = None, + config: ClassifierConfig | None = None, + ): + self.provider_profile = provider_profile + self.config = config or ClassifierConfig( + mode="llm" if provider_profile and provider_profile.provider_type != "deterministic" else "deterministic_baseline" + ) + self.llm_client = llm_client or LLMClient(provider_profile) + + def classify_claim(self, claim: str, candidates: list[TaxonomyEntry] | list[object], *, context: str = "") -> ClassificationResult: + if self.config.mode == "deterministic_baseline" or self.provider_profile is None or self.provider_profile.provider_type == "deterministic": + return self._deterministic_result(claim, candidates, context=context, fallback=False) + return self._llm_result(claim, candidates, context=context) + + def classify(self, claim: str, candidates: list[TaxonomyEntry] | list[object], *, context: str = "") -> list[RiskAssessment]: + return self.classify_claim(claim, candidates, context=context).assessments + + def _deterministic_result(self, claim: str, candidates: list[object], *, context: str, fallback: bool, warning: str = "") -> ClassificationResult: + provider_id = "deterministic_baseline" + model_name = "local-keyword" + assessments = [ + RiskAssessment(**item) + for item in classify_deterministic( + claim, + candidates, + context=context, + classification_mode="deterministic_baseline", + model_provider_id=provider_id, + model_name=model_name, + deterministic_fallback_used=fallback, + ) + ] + warnings = [warning] if warning else [] + return ClassificationResult( + assessments=assessments, + warnings=warnings, + model_error=warning, + classification_mode="deterministic_baseline", + model_provider_id=provider_id, + model_name=model_name, + llm_used=False, + deterministic_fallback_used=fallback, + ) + + def _llm_result(self, claim: str, candidates: list[object], *, context: str) -> ClassificationResult: + profile = self.provider_profile + assert profile is not None + provider_id = profile.provider_id + model_name = profile.model_name + prompt = build_classification_prompt(claim, context, candidates) + messages = build_messages(claim, context, candidates) + try: + raw = self.llm_client.classify(prompt, messages=messages) + payload = _coerce_llm_payload(raw) + assessments, warnings = _validate_llm_assessments( + payload, + claim=claim, + context=context, + candidates=candidates, + provider_id=provider_id, + model_name=model_name, + ) + if payload.get("warning"): + warnings.append(str(payload["warning"])) + return ClassificationResult( + assessments=assessments, + warnings=warnings, + classification_mode="llm", + model_provider_id=provider_id, + model_name=model_name, + llm_used=True, + deterministic_fallback_used=False, + ) + except json.JSONDecodeError as exc: + warning = f"LLM classification returned malformed JSON for provider {provider_id}: {exc.msg}" + return ClassificationResult( + assessments=[], + warnings=[warning], + model_error=warning, + classification_mode="llm", + model_provider_id=provider_id, + model_name=model_name, + llm_used=True, + deterministic_fallback_used=False, + ) + except (LLMClientError, TypeError, ValueError) as exc: + warning = f"LLM classification failed for provider {provider_id}: {exc}" + if self.config.fallback_to_deterministic: + return self._deterministic_result(claim, candidates, context=context, fallback=True, warning=warning) + return ClassificationResult( + assessments=[], + warnings=[warning], + model_error=warning, + classification_mode="llm", + model_provider_id=provider_id, + model_name=model_name, + llm_used=True, + deterministic_fallback_used=False, + ) + + +def classify_claim( + claim: str, + candidates: list[TaxonomyEntry] | list[object], + *, + context: str = "", + provider_profile: ProviderProfile | None = None, + fallback_to_deterministic: bool = True, +) -> ClassificationResult: + mode: ClassificationMode = "llm" if provider_profile and provider_profile.provider_type != "deterministic" else "deterministic_baseline" + classifier = ArgumentRiskClassifier( + provider_profile=provider_profile, + config=ClassifierConfig(mode=mode, fallback_to_deterministic=fallback_to_deterministic), + ) + return classifier.classify_claim(claim, candidates, context=context) + + +def _coerce_llm_payload(raw: object) -> dict[str, Any]: + if isinstance(raw, str): + parsed = json.loads(raw) + else: + parsed = raw + if not isinstance(parsed, dict): + raise ValueError("LLM output must be a JSON object.") + if "assessments" not in parsed: + raise ValueError("LLM output missing assessments.") + if not isinstance(parsed["assessments"], list): + raise ValueError("LLM assessments must be a list.") + return parsed + + +def _validate_llm_assessments( + payload: dict[str, Any], + *, + claim: str, + context: str, + candidates: list[object], + provider_id: str, + model_name: str, +) -> tuple[list[RiskAssessment], list[str]]: + entries = {_entry(candidate).id: _entry(candidate) for candidate in candidates if _is_entry_allowed(_entry(candidate))} + haystack = claim if not context else f"{claim}\n{context}" + assessments: list[RiskAssessment] = [] + warnings: list[str] = [] + + for index, item in enumerate(payload.get("assessments", [])): + if not isinstance(item, dict): + warnings.append(f"Dropped LLM assessment {index}: item is not an object.") + continue + if item.get("insufficient_evidence") is True: + continue + risk_id = str(item.get("risk_id") or item.get("taxonomy_id") or "") + entry = entries.get(risk_id) + if entry is None: + warnings.append(f"Dropped LLM assessment {index}: risk_id is not a supplied classification candidate.") + continue + evidence_span = str(item.get("evidence_span") or "") + start = haystack.find(evidence_span) if evidence_span else -1 + if start < 0: + warnings.append(f"Dropped LLM assessment {index}: evidence_span is not an exact substring.") + continue + confidence = _clamp_confidence(item.get("confidence", 0.0)) + if confidence <= 0: + warnings.append(f"Dropped LLM assessment {index}: confidence must be positive.") + continue + assessments.append( + RiskAssessment( + risk_id=entry.id, + category=entry.canonical_category, + label=entry.name, + severity=entry.severity.value, + confidence=confidence, + evidence_span=evidence_span, + evidence_start_char=start, + evidence_end_char=start + len(evidence_span), + explanation=str(item.get("explanation") or "LLM classification grounded in supplied taxonomy candidate."), + false_positive_warning=str(item.get("false_positive_warning") or ""), + classification_mode="llm", + model_provider_id=provider_id, + model_name=model_name, + llm_used=True, + deterministic_fallback_used=False, + insufficient_evidence=False, + ) + ) + + assessments.sort(key=lambda item: (-item.confidence, item.risk_id)) + return assessments[:3] if len(claim) <= 280 else assessments, warnings + + +def _entry(candidate: object) -> TaxonomyEntry: + return getattr(candidate, "entry", candidate) + + +def _is_entry_allowed(entry: TaxonomyEntry) -> bool: + return bool(entry.enabled_for_classification and entry.activation_status == "active") + + +def _clamp_confidence(value: object) -> float: + try: + number = float(value) + except (TypeError, ValueError): + return 0.0 + return round(min(1.0, max(0.0, number)), 3) diff --git a/engine/argument_risk_engine/classification/deterministic.py b/engine/argument_risk_engine/classification/deterministic.py index 9664ff3..b80e3cb 100644 --- a/engine/argument_risk_engine/classification/deterministic.py +++ b/engine/argument_risk_engine/classification/deterministic.py @@ -1,14 +1,183 @@ from __future__ import annotations -from argument_risk_engine.taxonomy.models import TaxonomyEntry +from argument_risk_engine.retrieval.candidate_filter import is_healthy_suppressor +from argument_risk_engine.taxonomy.models import ActivationStatus, TaxonomyEntry +MAX_RISKS_PER_SHORT_CLAIM = 3 +SHORT_CLAIM_CHAR_LIMIT = 280 -def classify_deterministic(claim: str, candidates: list[TaxonomyEntry]) -> list[dict[str, object]]: - lower = claim.lower() + +def classify_deterministic( + claim: str, + candidates: list[TaxonomyEntry] | list[object], + *, + context: str = "", + classification_mode: str = "deterministic_baseline", + model_provider_id: str = "deterministic_baseline", + model_name: str = "local-keyword", + deterministic_fallback_used: bool = False, +) -> list[dict[str, object]]: + """Conservatively classify retrieved candidates without network access. + + The deterministic baseline only emits taxonomy-grounded risks when an active, + classification-enabled candidate has an exact textual evidence span in the + claim/context and no false-positive guard dominates the match. + """ + + haystack = _evidence_haystack(claim, context) + healthy_dominates = _healthy_suppressor_dominates(candidates) results: list[dict[str, object]] = [] - for entry in candidates: - matched = [kw for kw in entry.keywords if kw.lower() in lower] - if matched: - confidence = min(0.95, 0.45 + 0.15 * len(matched)) - results.append({"taxonomy_id": entry.id, "confidence": confidence, "matched_terms": matched}) - return results + + for candidate in candidates: + entry = _entry(candidate) + if not _is_classifiable(entry): + continue + if is_healthy_suppressor(entry): + continue + if healthy_dominates and _candidate_false_positive_risk(candidate) == "high": + continue + if _exclusion_triggered(haystack, entry.exclusion_criteria): + continue + + evidence = _best_evidence_span(haystack, entry, candidate) + if evidence is None: + continue + + start, end, span, matched_terms = evidence + confidence = _confidence(candidate, matched_terms, entry) + if confidence < _minimum_confidence(entry): + continue + + results.append( + { + "risk_id": entry.id, + "taxonomy_id": entry.id, # backwards-compatible alias + "category": entry.canonical_category, + "label": entry.name, + "severity": _severity(entry, confidence), + "confidence": confidence, + "evidence_span": span, + "evidence_start_char": start, + "evidence_end_char": end, + "explanation": _explanation(entry, matched_terms, confidence), + "false_positive_warning": _false_positive_warning(entry, candidate), + "classification_mode": classification_mode, + "model_provider_id": model_provider_id, + "model_name": model_name, + "llm_used": False, + "deterministic_fallback_used": deterministic_fallback_used, + "insufficient_evidence": False, + "matched_terms": matched_terms, + } + ) + + results.sort(key=lambda item: (-float(item["confidence"]), str(item["risk_id"]))) + limit = MAX_RISKS_PER_SHORT_CLAIM if len(claim) <= SHORT_CLAIM_CHAR_LIMIT else len(results) + return results[:limit] + + +def _entry(candidate: object) -> TaxonomyEntry: + return getattr(candidate, "entry", candidate) + + +def _is_classifiable(entry: TaxonomyEntry) -> bool: + return bool( + entry.enabled_for_classification + and entry.activation_status == ActivationStatus.active.value + and not is_healthy_suppressor(entry) + ) + + +def _evidence_haystack(claim: str, context: str = "") -> str: + return claim if not context else f"{claim}\n{context}" + + +def _healthy_suppressor_dominates(candidates: list[object]) -> bool: + for candidate in candidates: + entry = _entry(candidate) + if is_healthy_suppressor(entry): + score = float(getattr(candidate, "retrieval_score", 0.0) or 0.0) + if score >= 1.0: + return True + diagnostics = getattr(candidate, "diagnostics", {}) or {} + if int(diagnostics.get("healthy_suppressor_count", 0) or 0) > 0 and _candidate_false_positive_risk(candidate) == "high": + return True + return False + + +def _candidate_false_positive_risk(candidate: object) -> str: + return str(getattr(candidate, "false_positive_risk", "medium") or "medium") + + +def _exclusion_triggered(text: str, exclusions: list[str]) -> bool: + lower = text.lower() + return any(exclusion.strip() and exclusion.lower() in lower for exclusion in exclusions) + + +def _best_evidence_span(text: str, entry: TaxonomyEntry, candidate: object) -> tuple[int, int, str, list[str]] | None: + terms = _candidate_terms(entry, candidate) + matches: list[tuple[int, int, str]] = [] + lower = text.lower() + for term in terms: + needle = term.strip() + if not needle: + continue + start = lower.find(needle.lower()) + if start >= 0: + end = start + len(needle) + matches.append((start, end, text[start:end])) + if not matches: + return None + + matches.sort(key=lambda item: (-(item[1] - item[0]), item[0])) + start, end, span = matches[0] + matched_terms = sorted({match[2] for match in matches}, key=lambda item: item.lower()) + return start, end, span, matched_terms + + +def _candidate_terms(entry: TaxonomyEntry, candidate: object) -> list[str]: + terms: list[str] = [] + terms.extend(str(term) for term in getattr(candidate, "matched_terms", []) or []) + terms.extend(entry.signals) + terms.extend(entry.trigger_patterns) + return sorted({term.strip() for term in terms if term and str(term).strip()}, key=len, reverse=True) + + +def _confidence(candidate: object, matched_terms: list[str], entry: TaxonomyEntry) -> float: + score = float(getattr(candidate, "retrieval_score", 0.0) or 0.0) + if score > 0: + confidence = min(0.92, 0.42 + (score * 0.12) + (0.04 * len(matched_terms))) + else: + confidence = min(0.86, 0.45 + (0.15 * len(matched_terms))) + if entry.requires_context or entry.requires_human_judgment: + confidence -= 0.12 + if _candidate_false_positive_risk(candidate) == "high": + confidence -= 0.10 + return round(max(0.0, confidence), 3) + + +def _minimum_confidence(entry: TaxonomyEntry) -> float: + if entry.requires_context or entry.false_positive_sensitivity == "high": + return 0.62 + return 0.50 + + +def _severity(entry: TaxonomyEntry, confidence: float) -> str: + severity = entry.severity.value + if severity == "high" and confidence < 0.58: + return "medium" + return severity + + +def _explanation(entry: TaxonomyEntry, matched_terms: list[str], confidence: float) -> str: + terms = ", ".join(matched_terms[:4]) if matched_terms else "the cited evidence" + return f"Matched {entry.name} using exact evidence ({terms}) with conservative confidence {confidence:.2f}." + + +def _false_positive_warning(entry: TaxonomyEntry, candidate: object) -> str: + warnings: list[str] = [] + if entry.common_false_positives: + warnings.append("Common false positives: " + "; ".join(entry.common_false_positives[:2])) + if _candidate_false_positive_risk(candidate) == "high": + warnings.append("High false-positive risk; human review recommended.") + return " ".join(warnings) diff --git a/engine/argument_risk_engine/classification/llm_client.py b/engine/argument_risk_engine/classification/llm_client.py index 9966b75..b68a2d2 100644 --- a/engine/argument_risk_engine/classification/llm_client.py +++ b/engine/argument_risk_engine/classification/llm_client.py @@ -11,6 +11,10 @@ from pydantic import BaseModel +class LLMClientError(RuntimeError): + """Raised when an optional model provider cannot produce a response.""" + + class ProviderTestResult(BaseModel): provider_id: str status: str @@ -24,9 +28,50 @@ class LLMClient: def __init__(self, profile: ProviderProfile | None = None): self.profile = profile - def classify(self, prompt: str) -> dict[str, str]: - provider_id = self.profile.provider_id if self.profile else "deterministic_baseline" - return {"provider": provider_id, "response": "LLM providers are optional; deterministic baseline is available."} + def classify(self, prompt: str, *, messages: list[dict[str, str]] | None = None) -> dict[str, Any]: + """Run an OpenAI-compatible chat completion and parse its JSON content.""" + + content = self.generate(prompt, messages=messages) + parsed = json.loads(content) + if not isinstance(parsed, dict): + raise LLMClientError("LLM JSON response must be an object.") + return parsed + + def generate(self, prompt: str, *, messages: list[dict[str, str]] | None = None) -> str: + if self.profile is None or self.profile.provider_type == "deterministic": + raise LLMClientError("No LLM provider is configured; deterministic baseline is available offline.") + if not self.profile.enabled: + raise LLMClientError(f"Provider {self.profile.provider_id} is disabled.") + if not self.profile.base_url: + raise LLMClientError(f"Provider {self.profile.provider_id} has no base_url configured.") + if not self.profile.model_name: + raise LLMClientError(f"Provider {self.profile.provider_id} has no model_name configured.") + + api_key = os.environ.get(self.profile.api_key_env_var, "") if self.profile.api_key_env_var else "" + body: dict[str, Any] = { + "model": self.profile.model_name, + "messages": messages or [{"role": "user", "content": prompt}], + "temperature": self.profile.temperature, + "stream": False, + } + if self.profile.max_tokens: + body["max_tokens"] = self.profile.max_tokens + if self.profile.supports_json_mode is True: + body["response_format"] = {"type": "json_object"} + + status, payload, warning = _request_json( + "POST", + _join_url(self.profile.base_url, "chat/completions"), + api_key, + body, + self.profile.timeout_seconds, + ) + if status != "ok": + raise LLMClientError(warning or "LLM provider request failed.") + try: + return str(payload["choices"][0]["message"]["content"]) + except (KeyError, IndexError, TypeError) as exc: + raise LLMClientError("LLM provider response did not contain choices[0].message.content.") from exc def test_provider(self) -> ProviderTestResult: if self.profile is None or self.profile.provider_type == "deterministic": @@ -66,7 +111,7 @@ def _get_models(profile: ProviderProfile, api_key: str) -> tuple[str, str, list[ status, payload, warning = _request_json("GET", url, api_key, None, profile.timeout_seconds) if status != "ok": return status, warning, [] - data = payload.get("data", payload if isinstance(payload, list) else []) + data = payload.get("data", payload if isinstance(payload, list) else []) if isinstance(payload, dict | list) else [] models = [str(item.get("id", item)) for item in data if item] return "ok", "", models diff --git a/engine/argument_risk_engine/classification/prompts.py b/engine/argument_risk_engine/classification/prompts.py index d3a80b1..7580e8e 100644 --- a/engine/argument_risk_engine/classification/prompts.py +++ b/engine/argument_risk_engine/classification/prompts.py @@ -1 +1,72 @@ -CLASSIFICATION_PROMPT = "Classify only against supplied taxonomy entries and cite evidence spans." +from __future__ import annotations + +import json +from typing import Any + +CLASSIFICATION_SYSTEM_PROMPT = """You are a taxonomy-grounded argument risk classifier. +Return strict JSON only. Do not include markdown, prose, or comments. + +Rules: +- Use only candidate taxonomy entries supplied in the prompt. +- Do not invent taxonomy labels, IDs, categories, or severities. +- Do not determine factual truth. +- Do not judge author intent. +- Do not classify personality or psychology. +- Do not classify without textual evidence. +- Evidence spans must be exact substrings of the claim or context. +- Prefer insufficient_evidence when uncertain. +""" + +CLASSIFICATION_PROMPT = CLASSIFICATION_SYSTEM_PROMPT + + +def build_classification_prompt(claim: str, context: str, candidates: list[object]) -> str: + payload = { + "task": "classify_argument_risks", + "claim": claim, + "context": context, + "candidate_taxonomy_entries": [_candidate_payload(candidate) for candidate in candidates], + "required_json_schema": { + "assessments": [ + { + "risk_id": "candidate id only", + "category": "candidate canonical_category only", + "label": "candidate name only", + "severity": "low|medium|high from candidate guidance only", + "confidence": "number 0..1", + "evidence_span": "exact substring of claim or context", + "explanation": "brief taxonomy-grounded reason", + "false_positive_warning": "brief warning or empty string", + "insufficient_evidence": "boolean", + } + ], + "warning": "optional warning string", + }, + } + return json.dumps(payload, ensure_ascii=False, sort_keys=True) + + +def build_messages(claim: str, context: str, candidates: list[object]) -> list[dict[str, str]]: + return [ + {"role": "system", "content": CLASSIFICATION_SYSTEM_PROMPT}, + {"role": "user", "content": build_classification_prompt(claim, context, candidates)}, + ] + + +def _candidate_payload(candidate: object) -> dict[str, Any]: + entry = getattr(candidate, "entry", candidate) + return { + "id": entry.id, + "name": entry.name, + "canonical_category": entry.canonical_category, + "short_definition": entry.short_definition, + "minimum_evidence_requirement": entry.minimum_evidence_requirement, + "signals": entry.signals, + "trigger_patterns": entry.trigger_patterns, + "exclusion_criteria": entry.exclusion_criteria, + "common_false_positives": entry.common_false_positives, + "severity": entry.severity.value, + "severity_guidance": entry.severity_guidance, + "retrieval_score": getattr(candidate, "retrieval_score", None), + "matched_terms": getattr(candidate, "matched_terms", []), + } diff --git a/tests/test_classifier.py b/tests/test_classifier.py index 6710266..355c23c 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -1,7 +1,147 @@ +from argument_risk_engine.classification.classifier import ArgumentRiskClassifier, ClassifierConfig from argument_risk_engine.classification.deterministic import classify_deterministic -from argument_risk_engine.taxonomy.models import default_taxonomy_pack +from argument_risk_engine.classification.llm_client import LLMClientError +from argument_risk_engine.classification.model_provider import ProviderProfile +from argument_risk_engine.taxonomy.models import TaxonomyEntry, default_taxonomy_pack + + +def _entry(**overrides): + data = { + "id": "risk", + "name": "Risk", + "canonical_category": "fallacy", + "signals": ["always"], + "enabled_for_classification": True, + "activation_status": "active", + "severity_guidance": ["medium"], + } + data.update(overrides) + return TaxonomyEntry(**data) + + +class FakeLLMClient: + def __init__(self, payload): + self.payload = payload + self.calls = [] + + def classify(self, prompt, *, messages=None): + self.calls.append((prompt, messages)) + if isinstance(self.payload, Exception): + raise self.payload + return self.payload def test_classifier_returns_match(): entry = default_taxonomy_pack().entries[0] assert classify_deterministic("everyone always", [entry])[0]["taxonomy_id"] == entry.id + + +def test_deterministic_baseline_works_without_api_key_and_validates_evidence(): + entry = _entry(id="overgeneralization", name="Overgeneralization", signals=["always"]) + + result = ArgumentRiskClassifier().classify_claim("People always do this.", [entry]) + + assert result.assessments[0].risk_id == "overgeneralization" + assert result.assessments[0].evidence_span == "always" + assert result.assessments[0].llm_used is False + assert result.assessments[0].model_provider_id == "deterministic_baseline" + + +def test_deterministic_ignores_inactive_excluded_and_neutral_claims(): + inactive = _entry(id="inactive", activation_status="review_required") + excluded = _entry(id="excluded", exclusion_criteria=["legitimate quantified claims"]) + + assert classify_deterministic("People always do this.", [inactive]) == [] + assert classify_deterministic("This is about legitimate quantified claims and always includes evidence.", [excluded]) == [] + assert classify_deterministic("The meeting starts at noon.", [_entry()]) == [] + + +def test_llm_mode_uses_selected_provider_and_rejects_invented_labels(): + profile = ProviderProfile( + provider_id="selected_provider", + label="Selected Provider", + provider_type="openai_compatible", + base_url="http://example.test/v1", + model_name="selected-model", + enabled=True, + ) + entry = _entry(id="allowed", name="Allowed", signals=["always"]) + llm = FakeLLMClient( + { + "assessments": [ + {"risk_id": "invented", "confidence": 0.99, "evidence_span": "always"}, + {"risk_id": "allowed", "confidence": 0.71, "evidence_span": "always", "explanation": "Exact evidence."}, + ] + } + ) + + result = ArgumentRiskClassifier(provider_profile=profile, llm_client=llm).classify_claim("People always do this.", [entry]) + + assert len(result.assessments) == 1 + assert result.assessments[0].risk_id == "allowed" + assert result.assessments[0].model_provider_id == "selected_provider" + assert result.assessments[0].model_name == "selected-model" + assert result.assessments[0].llm_used is True + assert result.warnings + assert llm.calls + + +def test_llm_evidence_spans_are_validated(): + profile = ProviderProfile( + provider_id="selected_provider", + label="Selected Provider", + provider_type="openai_compatible", + base_url="http://example.test/v1", + model_name="selected-model", + enabled=True, + ) + entry = _entry(id="allowed", name="Allowed", signals=["always"]) + llm = FakeLLMClient({"assessments": [{"risk_id": "allowed", "confidence": 0.8, "evidence_span": "not in claim"}]}) + + result = ArgumentRiskClassifier(provider_profile=profile, llm_client=llm).classify_claim("People always do this.", [entry]) + + assert result.assessments == [] + assert any("evidence_span" in warning for warning in result.warnings) + + +def test_llm_provider_failure_falls_back_without_crashing(): + profile = ProviderProfile( + provider_id="selected_provider", + label="Selected Provider", + provider_type="openai_compatible", + base_url="http://example.test/v1", + model_name="selected-model", + enabled=True, + ) + entry = _entry(id="allowed", name="Allowed", signals=["always"]) + llm = FakeLLMClient(LLMClientError("provider unavailable")) + + result = ArgumentRiskClassifier(provider_profile=profile, llm_client=llm).classify_claim("People always do this.", [entry]) + + assert result.assessments[0].risk_id == "allowed" + assert result.deterministic_fallback_used is True + assert result.assessments[0].deterministic_fallback_used is True + assert result.model_error + + +def test_llm_malformed_output_returns_empty_warning_without_crashing(): + profile = ProviderProfile( + provider_id="selected_provider", + label="Selected Provider", + provider_type="openai_compatible", + base_url="http://example.test/v1", + model_name="selected-model", + enabled=True, + ) + classifier = ArgumentRiskClassifier( + provider_profile=profile, + llm_client=FakeLLMClient("not json"), + config=ClassifierConfig(mode="llm", fallback_to_deterministic=True), + ) + + result = classifier.classify_claim("People always do this.", [_entry(id="allowed")]) + + assert result.assessments == [] + assert result.warnings + assert "malformed JSON" in result.model_error + assert result.deterministic_fallback_used is False