Skip to content

fix(audit): surface tracking and evaluation failures #518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Nicola Franco (franconicola) merged 1 commit into from
Jul 26, 2026
Merged

fix(audit): surface tracking and evaluation failures #518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,22 @@ We follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0

Commit messages should be structured as follows:

### Python Styleguide

Audit-bearing code must never silently swallow an exception. This includes
run/result/trace persistence, router dispatch, orchestration, and evaluator
code that contributes to a security report.

When handling an exception in these paths, use one of the following patterns:

* Catch the specific exception type and recover without losing audit data.
* Log the exception with `exc_info=True` and re-raise it.
* Persist a structured failure containing at least
`{"step": ..., "status": "failed", "error": ...}` on the run or result.

Do not add `except Exception: pass`, and do not turn a partially tracked or
partially evaluated run into a successful "no findings" result.

## License

By contributing to HackAgent, you agree that your contributions will be licensed under its [Apache License 2.0](LICENSE).
10 changes: 9 additions & 1 deletion hackagent/attacks/evaluator/evaluation_step.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ def _sync_metrics_to_backend_structured(self, summary: Dict[str, Any]):
self.logger.warning(
"Failed to recompute summary from persisted results: %s",
e,
exc_info=True,
)

merged_run_config: Dict[str, Any] = {}
Expand All @@ -333,6 +334,10 @@ def _sync_metrics_to_backend_structured(self, summary: Dict[str, Any]):
if isinstance(existing_run.run_config, dict):
merged_run_config = dict(existing_run.run_config)
except Exception:
self.logger.warning(
"Failed to read existing run config before metrics sync",
exc_info=True,
)
merged_run_config = {}

merged_run_config["evaluation_summary"] = summary_to_store
Expand All @@ -346,7 +351,10 @@ def _sync_metrics_to_backend_structured(self, summary: Dict[str, Any]):
self.logger.warning("No tracking client available; cannot sync metrics")

except Exception as e:
self.logger.warning(f"Failed to sync structured metrics: {e}")
self.logger.warning(
f"Failed to sync structured metrics: {e}",
exc_info=True,
)

def resolve_agent_type(self, agent_type_value: Any) -> AgentTypeEnum:
"""Convert a string, enum, or ``None`` into an ``AgentTypeEnum``."""
Expand Down
10 changes: 8 additions & 2 deletions hackagent/attacks/evaluator/inline_step_judge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ def __init__(
)
self._judges.append((judge_type, judge_range, evaluator))
except Exception as exc:
logger.warning(f"Could not initialise judge '{judge_type}': {exc}")
logger.warning(
f"Could not initialise judge '{judge_type}': {exc}",
exc_info=True,
)

if not self._judges:
logger.warning("No valid judges initialised for inline evaluation")
Expand Down Expand Up @@ -188,7 +191,10 @@ def is_jailbreak(
except (TypeError, ValueError):
pass
except Exception as exc:
self.logger.warning(f"Judge '{judge_type}' failed on candidate: {exc}")
self.logger.warning(
f"Judge '{judge_type}' failed on candidate: {exc}",
exc_info=True,
)

if not success_votes:
return False, best_score, judge_cols
Expand Down
7 changes: 6 additions & 1 deletion hackagent/attacks/evaluator/sync.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ def update_single_result(
)
merged_metadata = {**base, **metadata_updates}
except Exception:
log.warning(
"Could not read existing result metadata for %s",
result_id,
exc_info=True,
)
merged_metadata = dict(metadata_updates)

backend.update_result(
Expand All @@ -151,7 +156,7 @@ def update_single_result(
return True

except Exception as e:
log.error(f"Exception updating result {result_id}: {e}")
log.error(f"Exception updating result {result_id}: {e}", exc_info=True)
return False


Expand Down
126 changes: 115 additions & 11 deletions hackagent/attacks/orchestrator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import httpx

from hackagent.errors import HackAgentError
from hackagent.router.tracking.audit import record_run_audit_failure
from hackagent.attacks.techniques.config import (
DEFAULT_CATEGORY_CLASSIFIER_AGENT_TYPE,
DEFAULT_CATEGORY_CLASSIFIER_ENDPOINT,
Expand Down Expand Up @@ -252,6 +253,7 @@ def _backend_api_key_for_role_defaults(self) -> Optional[str]:
try:
api_key = getter()
except Exception:
logger.debug("Configured API-key getter failed", exc_info=True)
return None

if isinstance(api_key, str) and api_key.strip():
Expand Down Expand Up @@ -456,7 +458,7 @@ def _create_server_run_record(
def safe_uuid(val: str) -> UUID:
try:
return UUID(val)
except Exception:
except (AttributeError, TypeError, ValueError):
# Log warning and fallback to a new UUID
logger.warning(f"Invalid UUID '{val}', generating fallback UUID")
return uuid4()
Expand Down Expand Up @@ -680,6 +682,7 @@ def _autopull_missing_ollama_targets(self, targets: List[Dict[str, Any]]) -> Non
try:
installed = self._get_installed_ollama_models()
except Exception:
logger.debug("Unable to inspect installed Ollama models", exc_info=True)
return
seen: set[str] = set()
for model in candidates:
Expand Down Expand Up @@ -728,6 +731,10 @@ def _validate_default_category_classifier_requirements(
try:
installed_models = self._get_installed_ollama_models()
except Exception:
logger.warning(
"Unable to verify the pulled Ollama model",
exc_info=True,
)
installed_models = set()
pulled = self._is_ollama_model_present(required_model, installed_models)
if not pulled:
Expand Down Expand Up @@ -998,6 +1005,10 @@ def _register_target(
try:
agent_instance = router_obj.get_agent_instance(registration_key)
except Exception:
logger.debug(
"Unable to resolve registered agent for preflight",
exc_info=True,
)
agent_instance = None

model_name = (
Expand Down Expand Up @@ -1116,6 +1127,10 @@ def _probe_router_registration(
try:
agent = router.get_agent_instance(registration_key)
except Exception:
logger.debug(
"Unable to resolve registered agent during health check",
exc_info=True,
)
agent = None
probe_ready = getattr(agent, "probe_ready", None)
if callable(probe_ready):
Expand Down Expand Up @@ -1504,7 +1519,9 @@ def _execute_local_attack(
)
except Exception as e:
logger.warning(
"Failed to apply max_tokens override to target adapter: %s", e
"Failed to apply max_tokens override to target adapter: %s",
e,
exc_info=True,
)

# One monotonic start timestamp shared by all sub-runs/workers so
Expand Down Expand Up @@ -1801,7 +1818,12 @@ def execute(
status=StatusEnum.RUNNING.value,
)
except Exception as e:
logger.warning(f"Failed to update run status to RUNNING: {e}")
logger.error(
f"Failed to update run status to RUNNING: {e}",
exc_info=True,
)
if fail_on_run_error:
raise HackAgentError(f"Failed to start audit run {run_id}: {e}") from e

if goal_labels_by_index:
attack_config = {
Expand Down Expand Up @@ -1853,6 +1875,7 @@ def execute(
# =========================
# RUN EVALUATION PIPELINE
# =========================
evaluation_error: Optional[Exception] = None
try:
base_eval_config = {
**attack_config,
Expand Down Expand Up @@ -1910,7 +1933,15 @@ def execute(
logger.info("Evaluation pipeline completed")

except Exception as e:
logger.warning(f"Evaluation failed: {e}", exc_info=True)
evaluation_error = e
logger.error(f"Evaluation failed: {e}", exc_info=True)
record_run_audit_failure(
backend=self.hackagent_agent.backend,
run_id=run_id,
step="Evaluation Pipeline",
error=e,
logger=logger,
)
final_results = results # fallback
if _tui_event_bus is not None:
_tui_event_bus.emit(
Expand All @@ -1930,23 +1961,83 @@ def execute(
# ⏱ timing AFTER evaluation
_total_elapsed = round(time.perf_counter() - _total_t0, 3)
logger.info(f"Total run time: {_total_elapsed:.1f}s")

# A tracking failure may already have marked the run FAILED while
# the attack logic continued. Reading the run also flushes queued
# remote audit writes, so COMPLETED is only possible after all
# audit artifacts have been persisted successfully.
final_status = (
StatusEnum.FAILED
if evaluation_error is not None
else StatusEnum.COMPLETED
)
if final_status is StatusEnum.COMPLETED:
try:
run_uuid = UUID(run_id)
except (AttributeError, TypeError, ValueError):
# Some custom/test backends use opaque run identifiers.
# Their update_run implementation remains authoritative.
logger.debug(
"Skipping final audit-status read for non-UUID run id %r",
run_id,
)
run_uuid = None
try:
if run_uuid is not None:
persisted_run = self.hackagent_agent.backend.get_run(run_uuid)
persisted_status = str(
getattr(persisted_run, "status", "") or ""
).upper()
if persisted_status == StatusEnum.FAILED.value:
final_status = StatusEnum.FAILED
except Exception as status_error:
logger.error(
"Failed to verify final audit status for run %s: %s",
run_id,
status_error,
exc_info=True,
)
record_run_audit_failure(
backend=self.hackagent_agent.backend,
run_id=run_id,
step="Verify final audit status",
error=status_error,
logger=logger,
)
final_status = StatusEnum.FAILED

if _tui_event_bus is not None:
_tui_event_bus.emit(
"step_ended",
step_name="Attack Execution",
success=True,
success=final_status is StatusEnum.COMPLETED,
elapsed_s=_total_elapsed,
error=(
str(evaluation_error) if evaluation_error is not None else None
),
)

# ✅ Update run status to COMPLETED
# Only trustworthy, fully evaluated runs may be marked completed.
try:
logger.info(f"Updating run {run_id} status to COMPLETED")
logger.info(
"Updating run %s status to %s",
run_id,
final_status.value,
)
self.hackagent_agent.backend.update_run(
UUID(run_id),
status=StatusEnum.COMPLETED.value,
status=final_status.value,
)
Comment on lines 2027 to 2030
except Exception as e:
logger.warning(f"Failed to update run status to COMPLETED: {e}")
logger.error(
"Failed to update run %s status to %s: %s",
run_id,
final_status.value,
e,
exc_info=True,
)
if fail_on_run_error:
raise

return final_results

Expand All @@ -1960,7 +2051,10 @@ def execute(
run_notes=f"Execution failed: {str(e)}",
)
except Exception as update_error:
logger.warning(f"Failed to update run status to FAILED: {update_error}")
logger.critical(
f"Failed to update run status to FAILED: {update_error}",
exc_info=True,
)
if _tui_event_bus is not None:
_tui_event_bus.emit(
"step_ended",
Expand All @@ -1977,7 +2071,17 @@ def execute(
try:
flush()
except Exception as flush_error: # noqa: BLE001
logger.warning(f"Failed to flush backend writes: {flush_error}")
logger.error(
f"Failed to flush backend writes: {flush_error}",
exc_info=True,
)
record_run_audit_failure(
backend=self.hackagent_agent.backend,
run_id=run_id,
step="Flush audit writes",
error=flush_error,
logger=logger,
)

# ========================================================================
# HTTP Response Helpers
Expand Down
Loading
Loading