diff --git a/hackagent/agent.py b/hackagent/agent.py index 8aa3bd33..af485db1 100644 --- a/hackagent/agent.py +++ b/hackagent/agent.py @@ -69,6 +69,7 @@ def __init__( thinking: Optional[bool] = None, before_guardrail: Optional[Dict[str, Any]] = None, after_guardrail: Optional[Dict[str, Any]] = None, + backend: Optional[Any] = None, ): """ Initializes the HackAgent client and prepares it for interaction. @@ -110,11 +111,21 @@ def __init__( When set to `False`, requests sent through the target OLLAMA adapter include `think: false` to disable thinking output. Ignored for non-OLLAMA target agent types. + backend: Optional pre-built ``StorageBackend`` to persist runs and + results through. When omitted, a backend is selected from the + resolved API key (remote) or a default local SQLite database. + Supplying one lets an embedding host — e.g. the local dashboard + — reuse its own already-open backend. """ resolved_auth_token = utils.resolve_api_token(direct_api_key_param=api_key) - if resolved_auth_token: + if backend is not None: + self.backend = backend + logger.info( + "HackAgent using caller-provided backend %s", type(backend).__name__ + ) + elif resolved_auth_token: from hackagent.server.client import AuthenticatedClient from hackagent.server.storage.remote import RemoteBackend diff --git a/hackagent/server/dashboard/_attack_builder_mixin.py b/hackagent/server/dashboard/_attack_builder_mixin.py new file mode 100644 index 00000000..c4836dc7 --- /dev/null +++ b/hackagent/server/dashboard/_attack_builder_mixin.py @@ -0,0 +1,558 @@ +# Copyright 2026 - AI4I. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Attack builder panel — assemble and launch an attack from the dashboard. + +Provides ``DashboardAttackBuilderMixin``, the first place in +``hackagent/server/dashboard/`` where an attack is *launched* rather than +reviewed. It renders a small canvas of blocks that map 1:1 onto the config +sections the CLI already builds: + + Target block → ``agent-name`` / ``agent-type`` / ``endpoint`` + Goals block → ``goals`` list or a ``dataset`` section + Attack block(s) → one per ``ATTACK_CATALOG`` entry (drag from the palette) + Guardrail blocks → optional before/after, same three fields as the CLI + +Two or more attack blocks form a fallback ladder and are submitted through +``HackAgent.hack_chain()``; a single block goes through ``HackAgent.hack()``. +The translation itself lives in :mod:`._builder_config` so it stays testable +without a browser. + +Local vs remote: the builder always runs against the dashboard's own +``StorageBackend``. Since ``hackagent web`` redirects to the cloud dashboard +whenever an API key is configured, that backend is in practice the local +SQLite one, and submitted runs show up in the existing Runs/History/Reports +panels with no special-casing. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any, Dict, List, Optional + +from nicegui import ui + +from ._builder_config import ( + AGENT_TYPES, + CanvasValidationError, + attack_label, + attack_palette, + build_run_payload, + canvas_summary, + new_canvas, +) + +logger = logging.getLogger("hackagent.server.dashboard.builder") + +_MAX_LOG_LINES = 300 + + +class DashboardAttackBuilderMixin: + """Canvas-style attack builder panel and its in-process run submission.""" + + # ── Panel skeleton ──────────────────────────────────────────────────────── + + def _build_builder_panel(self, panel: ui.column) -> None: + with panel: + with ui.row().classes("w-full items-start gap-4 flex-nowrap"): + self._build_builder_palette() + with ui.column().classes("flex-1 gap-4 min-w-0"): + self._build_builder_toolbar() + self._build_builder_target_card() + self._build_builder_goals_card() + self._build_builder_chain_card() + self._build_builder_guardrails_card() + self._build_builder_progress_card() + self._builder_render_chain() + + def _build_builder_palette(self) -> None: + with ui.card().classes("w-64 shrink-0"): + ui.label("Attack palette").classes("font-semibold text-sm") + ui.label("Drag onto the chain, or click to append.").classes( + "text-xs text-grey-6 mb-2" + ) + with ui.column().classes("w-full gap-2"): + for entry in attack_palette(): + attack_type = entry["attack_type"] + card = ( + ui.card() + .classes("w-full p-2 cursor-grab hover:shadow-md") + .props('draggable="true"') + ) + with card: + ui.label(entry["label"]).classes("text-sm font-medium") + ui.label(entry["description"]).classes( + "text-xs text-grey-6 whitespace-normal" + ) + card.on( + "dragstart", + lambda _e, t=attack_type: self._builder_set_drag(t), + ) + card.on( + "click", lambda _e, t=attack_type: self._builder_add_attack(t) + ) + + def _build_builder_toolbar(self) -> None: + with ui.card().classes("w-full"): + with ui.row().classes("w-full items-center gap-3 flex-wrap"): + self._builder_name_input = ( + ui.input("Draft name", value=self._builder_canvas["name"]) + .props("dense outlined") + .classes("w-56") + ) + self._builder_drafts_select = ( + ui.select({}, label="Saved drafts") + .props("dense outlined") + .classes("w-56") + ) + ui.button("Save draft", icon="save", on_click=self._builder_save_draft) + ui.button( + "Open", icon="folder_open", on_click=self._builder_load_draft + ).props("outline") + ui.button( + "Delete", icon="delete", on_click=self._builder_delete_draft + ).props("outline color=grey-7") + ui.space() + ui.button( + "Preview config", icon="code", on_click=self._builder_preview + ).props("outline") + self._builder_run_btn = ui.button( + "Run attack", icon="play_arrow", on_click=self._builder_run + ).props("color=primary") + + def _build_builder_target_card(self) -> None: + target = self._builder_canvas["target"] + with ui.card().classes("w-full"): + ui.label("Target").classes("font-semibold text-sm") + with ui.row().classes("w-full gap-3 flex-wrap"): + ui.input("Agent name", value=target["agent_name"]).props( + "dense outlined" + ).classes("w-56").bind_value(target, "agent_name") + ui.select( + AGENT_TYPES, label="Agent type", value=target["agent_type"] + ).props("dense outlined").classes("w-48").bind_value( + target, "agent_type" + ) + ui.input("Endpoint URL", value=target["endpoint"]).props( + "dense outlined" + ).classes("flex-1 min-w-72").bind_value(target, "endpoint") + ui.number("Timeout (s)", value=self._builder_canvas["timeout"]).props( + "dense outlined" + ).classes("w-32").bind_value(self._builder_canvas, "timeout") + + def _build_builder_goals_card(self) -> None: + goals_block = self._builder_canvas["goals"] + with ui.card().classes("w-full"): + ui.label("Goals / Dataset").classes("font-semibold text-sm") + toggle = ui.toggle( + {"goals": "Goal list", "dataset": "Dataset"}, + value=goals_block["mode"], + ).props("dense no-caps") + toggle.bind_value(goals_block, "mode") + + goals_area = ui.column().classes("w-full gap-2") + goals_area.bind_visibility_from(goals_block, "mode", value="goals") + with goals_area: + self._builder_goals_text = ( + ui.textarea( + "Goals (one per line)", + value="\n".join(goals_block.get("goals") or []), + ) + .props("dense outlined autogrow") + .classes("w-full") + ) + + dataset = goals_block.setdefault("dataset", {}) + dataset_area = ui.row().classes("w-full gap-3 flex-wrap") + dataset_area.bind_visibility_from(goals_block, "mode", value="dataset") + with dataset_area: + for label, key, width in ( + ("Preset", "preset", "w-44"), + ("Provider", "provider", "w-40"), + ("Path / URL", "path", "w-64"), + ("Goal field", "goal_field", "w-40"), + ("Split", "split", "w-32"), + ): + dataset.setdefault(key, "") + ui.input(label, value=dataset[key]).props("dense outlined").classes( + width + ).bind_value(dataset, key) + dataset.setdefault("limit", None) + ui.number("Limit", value=dataset["limit"]).props( + "dense outlined" + ).classes("w-28").bind_value(dataset, "limit") + + def _build_builder_chain_card(self) -> None: + with ui.card().classes("w-full"): + with ui.row().classes("w-full items-center justify-between"): + ui.label("Attack chain").classes("font-semibold text-sm") + ui.label( + "Blocks run as a fallback ladder: a mitigated goal escalates " + "to the next block." + ).classes("text-xs text-grey-6") + drop_zone = ui.column().classes( + "w-full gap-2 min-h-24 p-2 rounded border-2 border-dashed" + ) + drop_zone.on("dragover.prevent", lambda _e: None) + drop_zone.on("drop.prevent", lambda _e: self._builder_drop()) + self._builder_chain_area = drop_zone + + def _build_builder_guardrails_card(self) -> None: + guardrails = self._builder_canvas["guardrails"] + with ui.card().classes("w-full"): + ui.label("Guardrails (optional)").classes("font-semibold text-sm") + for slot, title in (("before", "Before"), ("after", "After")): + block = guardrails.get(slot) or { + "identifier": "", + "agent_type": "", + "endpoint": "", + } + guardrails[slot] = block + with ui.row().classes("w-full gap-3 items-center flex-wrap"): + ui.label(title).classes("text-xs text-grey-6 w-14") + ui.input("Identifier", value=block["identifier"]).props( + "dense outlined" + ).classes("w-56").bind_value(block, "identifier") + ui.input("Agent type", value=block["agent_type"]).props( + "dense outlined" + ).classes("w-40").bind_value(block, "agent_type") + ui.input("Endpoint", value=block["endpoint"]).props( + "dense outlined" + ).classes("flex-1 min-w-64").bind_value(block, "endpoint") + + def _build_builder_progress_card(self) -> None: + with ui.card().classes("w-full"): + with ui.row().classes("w-full items-center gap-3"): + ui.label("Progress").classes("font-semibold text-sm") + self._builder_status_label = ui.label("Idle").classes( + "text-xs text-grey-6" + ) + self._builder_spinner = ui.spinner(size="sm") + self._builder_spinner.set_visibility(False) + self._builder_log = ( + ui.log(max_lines=_MAX_LOG_LINES) + .classes("w-full h-56 text-xs font-mono") + .style("white-space: pre-wrap") + ) + # Bus events arrive on the attack's worker thread; they are queued + # and flushed here so widgets are only touched from the event loop. + ui.timer(0.4, self._builder_flush_events) + + def _builder_flush_events(self) -> None: + """Move queued bus lines into the log widget (runs on the event loop).""" + if self._builder_log is None: + return + while self._builder_event_queue: + self._builder_log.push(self._builder_event_queue.popleft()) + + # ── Canvas mutation ─────────────────────────────────────────────────────── + + def _builder_set_drag(self, attack_type: str) -> None: + self._builder_drag_type = attack_type + + def _builder_drop(self) -> None: + if self._builder_drag_type: + self._builder_add_attack(self._builder_drag_type) + self._builder_drag_type = None + + def _builder_add_attack(self, attack_type: str) -> None: + self._builder_canvas["attacks"].append( + {"attack_type": attack_type, "params": ""} + ) + self._builder_render_chain() + + def _builder_move_attack(self, index: int, delta: int) -> None: + blocks = self._builder_canvas["attacks"] + target = index + delta + if 0 <= target < len(blocks): + blocks[index], blocks[target] = blocks[target], blocks[index] + self._builder_render_chain() + + def _builder_remove_attack(self, index: int) -> None: + blocks = self._builder_canvas["attacks"] + if 0 <= index < len(blocks): + blocks.pop(index) + self._builder_render_chain() + + def _builder_render_chain(self) -> None: + if self._builder_chain_area is None: + return + blocks = self._builder_canvas["attacks"] + self._builder_chain_area.clear() + with self._builder_chain_area: + if not blocks: + ui.label( + "Drag an attack from the palette here to start the chain." + ).classes("text-sm text-grey-6 p-2") + return + for index, block in enumerate(blocks): + with ui.card().classes("w-full p-2"): + with ui.row().classes("w-full items-center gap-2 flex-wrap"): + ui.badge(str(index + 1)).props("color=primary") + ui.label(attack_label(block["attack_type"])).classes( + "text-sm font-medium" + ) + ui.label(block["attack_type"]).classes( + "text-xs font-mono text-grey-6" + ) + ui.space() + ui.button( + icon="arrow_upward", + on_click=lambda _e, i=index: self._builder_move_attack( + i, -1 + ), + ).props("flat dense round") + ui.button( + icon="arrow_downward", + on_click=lambda _e, i=index: self._builder_move_attack( + i, 1 + ), + ).props("flat dense round") + ui.button( + icon="close", + on_click=lambda _e, i=index: self._builder_remove_attack(i), + ).props("flat dense round color=grey-7") + ui.textarea( + "Parameters (JSON, optional)", value=block.get("params") or "" + ).props("dense outlined autogrow").classes("w-full").bind_value( + block, "params" + ) + if index < len(blocks) - 1: + ui.icon("south").classes("text-grey-6 self-center") + + def _builder_collect_canvas(self) -> Dict[str, Any]: + """Return the canvas with widget-only values folded back in.""" + canvas = self._builder_canvas + if self._builder_name_input is not None: + canvas["name"] = ( + self._builder_name_input.value or "Untitled attack" + ).strip() + if self._builder_goals_text is not None: + canvas["goals"]["goals"] = [ + line.strip() + for line in (self._builder_goals_text.value or "").splitlines() + if line.strip() + ] + return canvas + + def _builder_apply_canvas(self, canvas: Dict[str, Any]) -> None: + """Replace the live canvas and rebuild the whole panel from it.""" + merged = new_canvas() + merged.update({k: v for k, v in canvas.items() if v is not None}) + self._builder_canvas = merged + panel = self.all_panels.get("builder") + if panel is None: + return + panel.clear() + self._build_builder_panel(panel) + + # ── Drafts ──────────────────────────────────────────────────────────────── + + def _builder_backend_supports_drafts(self) -> bool: + return hasattr(self.backend, "save_builder_draft") + + async def _builder_save_draft(self) -> None: + if not self._builder_backend_supports_drafts(): + ui.notify("This storage backend cannot save drafts.", type="warning") + return + canvas = self._builder_collect_canvas() + try: + draft = await asyncio.to_thread( + self.backend.save_builder_draft, + canvas["name"], + json.loads(json.dumps(canvas)), # detach from live widget state + self._builder_draft_id, + ) + except Exception as exc: + logger.error("Failed to save builder draft", exc_info=True) + ui.notify(f"Could not save draft: {exc}", type="negative") + return + self._builder_draft_id = draft["id"] + await self._builder_refresh_drafts() + ui.notify(f"Draft '{draft['name']}' saved.", type="positive") + + async def _builder_refresh_drafts(self) -> None: + if self._builder_drafts_select is None or not ( + self._builder_backend_supports_drafts() + ): + return + try: + drafts: List[Dict[str, Any]] = await asyncio.to_thread( + self.backend.list_builder_drafts + ) + except Exception: + logger.error("Failed to list builder drafts", exc_info=True) + return + self._builder_drafts_select.set_options( + { + draft["id"]: f"{draft['name']} — {canvas_summary(draft['canvas'])}" + for draft in drafts + }, + value=self._builder_draft_id, + ) + + def _builder_selected_draft_id(self) -> Optional[str]: + """Selected draft id, or None after notifying why there is none.""" + if not self._builder_backend_supports_drafts(): + ui.notify("This backend cannot store attack drafts.", type="warning") + return None + draft_id = self._builder_drafts_select and self._builder_drafts_select.value + if not draft_id: + ui.notify("Select a saved draft first.", type="warning") + return None + return draft_id + + async def _builder_load_draft(self) -> None: + draft_id = self._builder_selected_draft_id() + if not draft_id: + return + try: + draft = await asyncio.to_thread(self.backend.get_builder_draft, draft_id) + except Exception as exc: + logger.error("Failed to open builder draft", exc_info=True) + ui.notify(f"Could not open draft: {exc}", type="negative") + return + if not draft: + ui.notify("That draft no longer exists.", type="warning") + return + self._builder_draft_id = draft["id"] + self._builder_apply_canvas(draft["canvas"]) + await self._builder_refresh_drafts() + ui.notify(f"Opened draft '{draft['name']}'.", type="positive") + + async def _builder_delete_draft(self) -> None: + draft_id = self._builder_selected_draft_id() + if not draft_id: + return + try: + await asyncio.to_thread(self.backend.delete_builder_draft, draft_id) + except Exception as exc: + logger.error("Failed to delete builder draft", exc_info=True) + ui.notify(f"Could not delete draft: {exc}", type="negative") + return + if self._builder_draft_id == draft_id: + self._builder_draft_id = None + await self._builder_refresh_drafts() + ui.notify("Draft deleted.", type="positive") + + # ── Submission ──────────────────────────────────────────────────────────── + + def _builder_payload(self) -> Optional[Dict[str, Any]]: + try: + return build_run_payload(self._builder_collect_canvas()) + except CanvasValidationError as exc: + ui.notify(str(exc), type="warning") + return None + + def _builder_preview(self) -> None: + payload = self._builder_payload() + if payload is None: + return + preview = { + key: payload[key] + for key in ("target", "mode", "attack_config", "attacks", "timeout") + if key in payload + } + with ui.dialog() as dialog, ui.card().classes("w-full max-w-3xl"): + ui.label("Generated attack configuration").classes("font-semibold") + ui.code(json.dumps(preview, indent=2), language="json").classes("w-full") + ui.button("Close", on_click=dialog.close).props("flat") + dialog.open() + + async def _builder_run(self) -> None: + if self._builder_running: + ui.notify("An attack is already running.", type="warning") + return + payload = self._builder_payload() + if payload is None: + return + + self._builder_running = True + self._builder_run_btn.disable() + self._builder_spinner.set_visibility(True) + self._builder_status_label.text = "Running…" + self._builder_log.clear() + self._builder_event_queue.clear() + chain = ( + payload["attack_config"]["attack_type"] + if payload["mode"] == "single" + else " → ".join(step["attack_type"] for step in payload["attacks"]) + ) + self._builder_log.push(f"▶ {payload['target']['agent_name']}: {chain}") + + try: + await asyncio.to_thread(self._builder_execute, payload) + except Exception as exc: + logger.error("Attack builder run failed", exc_info=True) + self._builder_log.push(f"✖ {exc}") + self._builder_status_label.text = "Failed" + ui.notify(f"Attack failed: {exc}", type="negative") + else: + self._builder_status_label.text = "Completed" + self._builder_log.push("✔ Attack finished — see History for results.") + ui.notify("Attack completed. Results are in History.", type="positive") + finally: + self._builder_running = False + self._builder_run_btn.enable() + self._builder_spinner.set_visibility(False) + + await self.refresh_view() + + def _builder_execute(self, payload: Dict[str, Any]) -> Any: + """Blocking in-process attack submission — runs on a worker thread.""" + from hackagent import HackAgent + from hackagent.cli.tui.events import TUIEventBus + + bus = TUIEventBus() + bus.subscribe(self._builder_on_event) + + target = payload["target"] + agent = HackAgent( + name=target["agent_name"], + endpoint=target["endpoint"], + agent_type=target["agent_type"], + before_guardrail=payload.get("before_guardrail"), + after_guardrail=payload.get("after_guardrail"), + backend=self.backend, + ) + run_config_override = {"timeout": payload["timeout"]} + if payload["mode"] == "single": + return agent.hack( + attack_config=payload["attack_config"], + run_config_override=run_config_override, + fail_on_run_error=True, + _tui_event_bus=bus, + ) + return agent.hack_chain( + attacks=payload["attacks"], + goals=payload.get("goals"), + run_config_override=run_config_override, + fail_on_run_error=True, + _tui_event_bus=bus, + ) + + def _builder_on_event(self, event) -> None: + """Bus subscriber — called on the attack's worker thread. + + Only appends to a queue; :meth:`_builder_flush_events` renders it. + """ + line = self._builder_format_event(event) + if line: + self._builder_event_queue.append(line) + + @staticmethod + def _builder_format_event(event) -> str: + payload = event.payload or {} + detail = ( + payload.get("message") + or payload.get("goal") + or payload.get("step") + or payload.get("status") + or "" + ) + detail = str(detail) + if len(detail) > 160: + detail = detail[:157] + "…" + return f"{event.event_type}: {detail}" if detail else str(event.event_type) diff --git a/hackagent/server/dashboard/_builder_config.py b/hackagent/server/dashboard/_builder_config.py new file mode 100644 index 00000000..fbcab689 --- /dev/null +++ b/hackagent/server/dashboard/_builder_config.py @@ -0,0 +1,271 @@ +# Copyright 2026 - AI4I. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pure canvas ↔ attack-config translation for the dashboard attack builder. + +The dashboard's attack builder is a thin UI over the configuration shapes the +CLI already builds: a *canvas* is a plain dict of blocks (target, goals/dataset, +an ordered list of attack blocks, optional guardrails) and this module turns it +into exactly the payload ``HackAgent.hack()`` / ``HackAgent.hack_chain()`` +consume — the same dicts ``hackagent.cli.commands.attack.config`` produces. + +Nothing here touches NiceGUI, the network or the storage backend, so the whole +translation and its validation rules are directly unit-testable. + +Canvas shape:: + + { + "name": "my draft", + "target": {"agent_name": ..., "agent_type": ..., "endpoint": ...}, + "goals": { + "mode": "goals" | "dataset", + "goals": ["...", ...], + "dataset": {"preset": ..., "provider": ..., "path": ..., + "goal_field": ..., "split": ..., "limit": ...}, + }, + "attacks": [{"attack_type": "pair", "params": {...}}, ...], + "guardrails": {"before": {...} | None, "after": {...} | None}, + "timeout": 300, + } +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from hackagent.cli.commands.attack.catalog import ATTACK_CATALOG + +# Agent types offered by the Target block — mirrors ``--agent-type`` in +# ``hackagent/cli/commands/attack/options.py``. +AGENT_TYPES: List[str] = [ + "google-adk", + "litellm", + "langchain", + "openai-sdk", + "ollama", + "mcp", + "a2a", + "other", +] + +DEFAULT_TIMEOUT = 300 + + +class CanvasValidationError(ValueError): + """Raised when a canvas cannot be turned into a runnable attack config.""" + + +def attack_palette() -> List[Dict[str, str]]: + """Return the draggable attack blocks, derived from ``ATTACK_CATALOG``. + + A newly supported CLI technique appears on the canvas with no dashboard + code change. + """ + return [ + { + "attack_type": attack_type, + "label": entry.get("label", attack_type), + "description": entry.get("description", ""), + } + for attack_type, entry in sorted( + ATTACK_CATALOG.items(), key=lambda kv: kv[1].get("label", kv[0]).lower() + ) + ] + + +def attack_label(attack_type: str) -> str: + """Human-facing label for *attack_type*, falling back to the raw key.""" + return ATTACK_CATALOG.get(attack_type, {}).get("label", attack_type) + + +def new_canvas() -> Dict[str, Any]: + """Return an empty canvas with every block present but unfilled.""" + return { + "name": "Untitled attack", + "target": {"agent_name": "", "agent_type": "other", "endpoint": ""}, + "goals": {"mode": "goals", "goals": [], "dataset": {}}, + "attacks": [], + "guardrails": {"before": None, "after": None}, + "timeout": DEFAULT_TIMEOUT, + } + + +def _clean(value: Any) -> Any: + return value.strip() if isinstance(value, str) else value + + +def _build_guardrail(block: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Mirror ``_build_guardrail_config``: no identifier ⇒ no guardrail.""" + if not block: + return None + identifier = _clean(block.get("identifier") or block.get("name") or "") + if not identifier: + return None + return { + "identifier": identifier, + "agent_type": _clean(block.get("agent_type")) or None, + "endpoint": _clean(block.get("endpoint")) or None, + } + + +def _build_dataset(block: Dict[str, Any]) -> Dict[str, Any]: + """Drop empty fields from a dataset block so provider defaults apply.""" + dataset: Dict[str, Any] = {} + for key in ("preset", "provider", "path", "goal_field", "split", "url", "name"): + value = _clean(block.get(key)) + if value: + dataset[key] = value + limit = block.get("limit") + if limit not in (None, ""): + try: + dataset["limit"] = int(limit) + except (TypeError, ValueError) as exc: + raise CanvasValidationError( + "Dataset limit must be a whole number." + ) from exc + return dataset + + +def _goal_section(canvas: Dict[str, Any]) -> Dict[str, Any]: + """Build the ``goals``/``dataset`` half of an attack config.""" + block = canvas.get("goals") or {} + mode = block.get("mode", "goals") + if mode == "dataset": + dataset = _build_dataset(block.get("dataset") or {}) + if not dataset.get("preset") and not ( + dataset.get("provider") and (dataset.get("path") or dataset.get("url")) + ): + raise CanvasValidationError( + "Dataset block needs a preset, or a provider plus a path/url." + ) + return {"dataset": dataset} + + goals = [ + _clean(goal) for goal in (block.get("goals") or []) if _clean(goal) + ] # keep order, drop blanks + if not goals: + raise CanvasValidationError( + "Add at least one goal, or switch the Goals block to a dataset." + ) + return {"goals": goals} + + +def _validate_target(canvas: Dict[str, Any]) -> Dict[str, Any]: + target = canvas.get("target") or {} + agent_name = _clean(target.get("agent_name")) + endpoint = _clean(target.get("endpoint")) + if not agent_name: + raise CanvasValidationError("Target block needs an agent name.") + if not endpoint: + raise CanvasValidationError("Target block needs an endpoint URL.") + return { + "agent_name": agent_name, + "agent_type": _clean(target.get("agent_type")) or "other", + "endpoint": endpoint, + } + + +def _attack_steps(canvas: Dict[str, Any]) -> List[Dict[str, Any]]: + blocks = canvas.get("attacks") or [] + if not blocks: + raise CanvasValidationError("Drop at least one attack block on the canvas.") + steps: List[Dict[str, Any]] = [] + for index, block in enumerate(blocks): + attack_type = _clean((block or {}).get("attack_type")) + if attack_type not in ATTACK_CATALOG: + raise CanvasValidationError( + f"Attack block {index + 1} has an unknown attack type: {attack_type!r}." + ) + step: Dict[str, Any] = {"attack_type": attack_type} + params = (block or {}).get("params") or {} + if isinstance(params, str): + params = _parse_params_text(params, index) + if not isinstance(params, dict): + raise CanvasValidationError( + f"Attack block {index + 1} parameters must be a JSON object." + ) + step.update(params) + step["attack_type"] = attack_type # params must not override the block type + steps.append(step) + return steps + + +def _parse_params_text(text: str, index: int) -> Dict[str, Any]: + text = text.strip() + if not text: + return {} + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise CanvasValidationError( + f"Attack block {index + 1} parameters are not valid JSON: {exc.msg}." + ) from exc + if not isinstance(parsed, dict): + raise CanvasValidationError( + f"Attack block {index + 1} parameters must be a JSON object." + ) + return parsed + + +def build_run_payload(canvas: Dict[str, Any]) -> Dict[str, Any]: + """Translate *canvas* into everything needed to launch the run. + + Returns a dict with: + ``target``: kwargs for ``HackAgent(...)`` (name/endpoint/agent_type). + ``before_guardrail`` / ``after_guardrail``: ``HackAgent`` kwargs or None. + ``mode``: ``"single"`` for one attack block, ``"chain"`` for two or more. + ``attack_config``: single-mode payload for ``HackAgent.hack()``. + ``attacks``: chain-mode ordered list for ``HackAgent.hack_chain()``. + ``timeout``: seconds, for ``run_config_override``. + + Raises: + CanvasValidationError: if a required block is missing or malformed. + """ + target = _validate_target(canvas) + goal_section = _goal_section(canvas) + steps = _attack_steps(canvas) + + guardrails = canvas.get("guardrails") or {} + timeout = canvas.get("timeout") or DEFAULT_TIMEOUT + try: + timeout = int(timeout) + except (TypeError, ValueError) as exc: + raise CanvasValidationError( + "Timeout must be a whole number of seconds." + ) from exc + + payload: Dict[str, Any] = { + "target": target, + "before_guardrail": _build_guardrail(guardrails.get("before")), + "after_guardrail": _build_guardrail(guardrails.get("after")), + "timeout": timeout, + } + + if len(steps) == 1: + # Single block → a plain hack() call, same dict _build_attack_config makes. + attack_config = dict(steps[0]) + attack_config.update(goal_section) + payload["mode"] = "single" + payload["attack_config"] = attack_config + else: + # Chained blocks are a fallback ladder: only the first step carries the + # goal source, exactly as ``hackagent eval chain`` documents. + first = dict(steps[0]) + first.update(goal_section) + payload["mode"] = "chain" + payload["attacks"] = [first] + [dict(step) for step in steps[1:]] + # Only goal-list mode has an explicit pool to hand to ``hack_chain()``; + # in dataset mode the goal source stays on the first step. + if "goals" in goal_section: + payload["goals"] = goal_section["goals"] + + return payload + + +def canvas_summary(canvas: Dict[str, Any]) -> str: + """Short one-line description of a canvas, used in the drafts list.""" + steps = canvas.get("attacks") or [] + chain = " → ".join(attack_label(str(s.get("attack_type"))) for s in steps) + target = (canvas.get("target") or {}).get("agent_name") or "no target" + return f"{target} · {chain or 'no attacks'}" diff --git a/hackagent/server/dashboard/_data_mixin.py b/hackagent/server/dashboard/_data_mixin.py index 4649f5a5..3bc13bd0 100644 --- a/hackagent/server/dashboard/_data_mixin.py +++ b/hackagent/server/dashboard/_data_mixin.py @@ -217,6 +217,8 @@ async def refresh_view(self) -> None: await self._load_agents() elif _v == "runs": await self._load_runs() + elif _v == "builder": + await self._builder_refresh_drafts() except Exception as exc: ui.notify(f"Failed to load data: {exc}", type="negative") finally: diff --git a/hackagent/server/dashboard/_layout_mixin.py b/hackagent/server/dashboard/_layout_mixin.py index 4293bb53..948b05de 100644 --- a/hackagent/server/dashboard/_layout_mixin.py +++ b/hackagent/server/dashboard/_layout_mixin.py @@ -94,6 +94,7 @@ def _build_sidebar(self) -> ui.left_drawer: ("dashboard", "Home", "dashboard"), ("agents", "Targets", "smart_toy"), ("runs", "History", "assignment"), + ("builder", "Attack Builder", "build"), ] for view_id, label, icon_name in nav_items: btn = ( @@ -150,11 +151,13 @@ def _build_panels(self) -> None: dashboard_panel = ui.column().classes("w-full gap-6") agents_panel = ui.column().classes("w-full gap-4") runs_panel = ui.column().classes("w-full gap-4") + builder_panel = ui.column().classes("w-full gap-4") self.all_panels = { "dashboard": dashboard_panel, "agents": agents_panel, "runs": runs_panel, + "builder": builder_panel, } for panel in self.all_panels.values(): panel.set_visibility(False) @@ -163,6 +166,7 @@ def _build_panels(self) -> None: self._build_dashboard_panel(dashboard_panel) self._build_agents_panel(agents_panel) self._build_runs_panel(runs_panel) + self._build_builder_panel(builder_panel) def _build_dashboard_panel(self, panel: ui.column) -> None: with panel: diff --git a/hackagent/server/dashboard/_page.py b/hackagent/server/dashboard/_page.py index 5fafca16..a958aa6d 100644 --- a/hackagent/server/dashboard/_page.py +++ b/hackagent/server/dashboard/_page.py @@ -16,6 +16,7 @@ - DashboardReportsMixin — History/Reports views and goal rows. - DashboardResultDetailMixin — single-result detail tabs. - DashboardRunsMixin — run selection, compare, export, delete. + - DashboardAttackBuilderMixin — canvas attack builder and run submission. - DashboardAnalysisDataMixin — widget-free aggregation and metrics. - DashboardTraceAnalysisMixin — trace classification/synthesis. - DashboardTraceRenderMixin / DashboardTapTraceMixin — trace rendering. @@ -28,6 +29,7 @@ from __future__ import annotations +from collections import deque from nicegui import app as _fastapi_app from nicegui import ui @@ -49,11 +51,13 @@ ) +from ._builder_config import new_canvas as _new_canvas from ._layout_mixin import DashboardLayoutMixin from ._reports_mixin import DashboardReportsMixin from ._result_detail_mixin import DashboardResultDetailMixin from ._data_mixin import DashboardDataMixin from ._runs_mixin import DashboardRunsMixin +from ._attack_builder_mixin import DashboardAttackBuilderMixin from ._analysis_data_mixin import DashboardAnalysisDataMixin from ._trace_analysis_mixin import DashboardTraceAnalysisMixin from ._trace_render_mixin import DashboardTraceRenderMixin @@ -80,6 +84,7 @@ class DashboardPage( DashboardResultDetailMixin, DashboardDataMixin, DashboardRunsMixin, + DashboardAttackBuilderMixin, DashboardAnalysisDataMixin, DashboardTraceAnalysisMixin, DashboardTraceRenderMixin, @@ -205,6 +210,22 @@ def __init__(self, backend) -> None: self._runs_export_btn: ui.button | None = None self._attacks_delete_btn: ui.button | None = None + # Attack builder panel + self._builder_canvas: dict = _new_canvas() + self._builder_draft_id: str | None = None + self._builder_drag_type: str | None = None + self._builder_running: bool = False + # Filled from the attack worker thread, drained on the event loop. + self._builder_event_queue: deque[str] = deque() + self._builder_name_input: ui.input | None = None + self._builder_drafts_select: ui.select | None = None + self._builder_goals_text: ui.textarea | None = None + self._builder_chain_area: ui.column | None = None + self._builder_status_label: ui.label | None = None + self._builder_spinner: ui.spinner | None = None + self._builder_run_btn: ui.button | None = None + self._builder_log: ui.log | None = None + # Comparison dialog self._compare_dialog: ui.dialog | None = None self._compare_dialog_body: ui.column | None = None diff --git a/hackagent/server/storage/local.py b/hackagent/server/storage/local.py index 8901743c..f40c4fd0 100644 --- a/hackagent/server/storage/local.py +++ b/hackagent/server/storage/local.py @@ -101,6 +101,14 @@ created_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS attack_builder_drafts ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + canvas_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + CREATE INDEX IF NOT EXISTS idx_attacks_agent ON attacks(agent_id); CREATE INDEX IF NOT EXISTS idx_runs_attack ON runs(attack_id); CREATE INDEX IF NOT EXISTS idx_results_run ON results(run_id); @@ -718,3 +726,88 @@ def list_traces(self, result_id: UUID) -> List[TraceRecord]: ) for r in rows ] + + # ── Attack builder drafts ───────────────────────────────────────────────── + + def save_builder_draft( + self, + name: str, + canvas: Dict[str, Any], + draft_id: Optional[str] = None, + ) -> Dict[str, Any]: + """Insert or update a dashboard attack-builder draft canvas. + + Args: + name: User-facing draft name. + canvas: Canvas dict (node layout plus the underlying config). + draft_id: Existing draft id to overwrite; a new one is created + when omitted or unknown. + + Returns: + The stored draft as ``{id, name, canvas, created_at, updated_at}``. + """ + now = _now_str() + canvas_json = json.dumps(canvas) + with self._lock: + row = None + if draft_id: + row = self._conn.execute( + "SELECT created_at FROM attack_builder_drafts WHERE id=? LIMIT 1", + (draft_id,), + ).fetchone() + if row: + created_at = row["created_at"] + self._conn.execute( + "UPDATE attack_builder_drafts SET name=?, canvas_json=?, updated_at=? WHERE id=?", + (name, canvas_json, now, draft_id), + ) + else: + draft_id = draft_id or str(uuid.uuid4()) + created_at = now + self._conn.execute( + "INSERT INTO attack_builder_drafts (id, name, canvas_json, created_at, updated_at) VALUES (?,?,?,?,?)", + (draft_id, name, canvas_json, now, now), + ) + self._conn.commit() + return { + "id": draft_id, + "name": name, + "canvas": canvas, + "created_at": created_at, + "updated_at": now, + } + + def list_builder_drafts(self) -> List[Dict[str, Any]]: + """Return all saved builder drafts, most recently updated first.""" + with self._lock: + rows = self._conn.execute( + "SELECT * FROM attack_builder_drafts ORDER BY updated_at DESC" + ).fetchall() + return [self._row_to_draft(r) for r in rows] + + def get_builder_draft(self, draft_id: str) -> Optional[Dict[str, Any]]: + """Return one draft by id, or None if it does not exist.""" + with self._lock: + row = self._conn.execute( + "SELECT * FROM attack_builder_drafts WHERE id=? LIMIT 1", + (draft_id,), + ).fetchone() + return self._row_to_draft(row) if row else None + + def delete_builder_draft(self, draft_id: str) -> None: + """Delete a draft by id. No-op if it does not exist.""" + with self._lock: + self._conn.execute( + "DELETE FROM attack_builder_drafts WHERE id=?", (draft_id,) + ) + self._conn.commit() + + @staticmethod + def _row_to_draft(row) -> Dict[str, Any]: + return { + "id": row["id"], + "name": row["name"], + "canvas": json.loads(row["canvas_json"] or "{}"), + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/tests/unit/server/dashboard/test_builder_config.py b/tests/unit/server/dashboard/test_builder_config.py new file mode 100644 index 00000000..cb1c3fbb --- /dev/null +++ b/tests/unit/server/dashboard/test_builder_config.py @@ -0,0 +1,181 @@ +# Copyright 2026 - AI4I. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the dashboard attack-builder canvas serializer.""" + +import unittest + +from hackagent.cli.commands.attack.catalog import ATTACK_CATALOG +from hackagent.server.dashboard._builder_config import ( + CanvasValidationError, + attack_palette, + build_run_payload, + canvas_summary, + new_canvas, +) + + +def _canvas(**overrides): + canvas = new_canvas() + canvas["target"] = { + "agent_name": "victim", + "agent_type": "litellm", + "endpoint": "http://localhost:8000/v1", + } + canvas["goals"] = {"mode": "goals", "goals": ["do bad thing"], "dataset": {}} + canvas["attacks"] = [{"attack_type": "pair", "params": ""}] + canvas.update(overrides) + return canvas + + +class TestAttackPalette(unittest.TestCase): + def test_palette_covers_full_catalog(self): + palette = attack_palette() + self.assertEqual( + {entry["attack_type"] for entry in palette}, set(ATTACK_CATALOG) + ) + self.assertTrue(all(entry["label"] for entry in palette)) + + +class TestBuildRunPayload(unittest.TestCase): + def test_single_attack_builds_hack_config(self): + payload = build_run_payload(_canvas()) + self.assertEqual(payload["mode"], "single") + self.assertEqual( + payload["attack_config"], + {"attack_type": "pair", "goals": ["do bad thing"]}, + ) + self.assertNotIn("attacks", payload) + self.assertEqual(payload["target"]["agent_name"], "victim") + self.assertEqual(payload["timeout"], 300) + + def test_chained_attacks_build_fallback_ladder(self): + canvas = _canvas( + attacks=[ + {"attack_type": "pair", "params": ""}, + {"attack_type": "tap", "params": '{"branching_factor": 2}'}, + {"attack_type": "bon", "params": {}}, + ] + ) + payload = build_run_payload(canvas) + self.assertEqual(payload["mode"], "chain") + self.assertEqual( + [step["attack_type"] for step in payload["attacks"]], + ["pair", "tap", "bon"], + ) + # Only the first step carries the goal source, as ``eval chain`` documents. + self.assertEqual(payload["attacks"][0]["goals"], ["do bad thing"]) + self.assertNotIn("goals", payload["attacks"][1]) + self.assertEqual(payload["attacks"][1]["branching_factor"], 2) + self.assertEqual(payload["goals"], ["do bad thing"]) + + def test_chained_dataset_mode_omits_goal_pool(self): + canvas = _canvas( + goals={ + "mode": "dataset", + "goals": [], + "dataset": {"preset": "advbench"}, + }, + attacks=[{"attack_type": "pair"}, {"attack_type": "tap"}], + ) + payload = build_run_payload(canvas) + self.assertEqual(payload["attacks"][0]["dataset"], {"preset": "advbench"}) + self.assertNotIn("dataset", payload["attacks"][1]) + self.assertNotIn("goals", payload) + + def test_params_cannot_override_block_attack_type(self): + canvas = _canvas( + attacks=[{"attack_type": "pair", "params": '{"attack_type": "tap"}'}] + ) + self.assertEqual( + build_run_payload(canvas)["attack_config"]["attack_type"], "pair" + ) + + def test_dataset_mode_builds_dataset_section(self): + canvas = _canvas( + goals={ + "mode": "dataset", + "goals": [], + "dataset": {"preset": "advbench", "limit": "25", "split": ""}, + } + ) + payload = build_run_payload(canvas) + self.assertEqual( + payload["attack_config"]["dataset"], {"preset": "advbench", "limit": 25} + ) + self.assertNotIn("goals", payload["attack_config"]) + + def test_guardrails_mirror_cli_shape(self): + canvas = _canvas( + guardrails={ + "before": { + "identifier": "openai/guard", + "agent_type": "openai-sdk", + "endpoint": "http://guard/v1", + }, + "after": {"identifier": "", "agent_type": "", "endpoint": ""}, + } + ) + payload = build_run_payload(canvas) + self.assertEqual( + payload["before_guardrail"], + { + "identifier": "openai/guard", + "agent_type": "openai-sdk", + "endpoint": "http://guard/v1", + }, + ) + self.assertIsNone(payload["after_guardrail"]) + + +class TestCanvasValidation(unittest.TestCase): + def test_missing_target_fields(self): + for field in ("agent_name", "endpoint"): + canvas = _canvas() + canvas["target"][field] = "" + with self.assertRaises(CanvasValidationError): + build_run_payload(canvas) + + def test_no_attack_blocks(self): + with self.assertRaises(CanvasValidationError): + build_run_payload(_canvas(attacks=[])) + + def test_unknown_attack_type(self): + with self.assertRaises(CanvasValidationError): + build_run_payload(_canvas(attacks=[{"attack_type": "nope"}])) + + def test_no_goals_and_no_dataset(self): + canvas = _canvas(goals={"mode": "goals", "goals": [" "], "dataset": {}}) + with self.assertRaises(CanvasValidationError): + build_run_payload(canvas) + + def test_incomplete_dataset_block(self): + canvas = _canvas( + goals={"mode": "dataset", "goals": [], "dataset": {"provider": "file"}} + ) + with self.assertRaises(CanvasValidationError): + build_run_payload(canvas) + + def test_invalid_params_json(self): + canvas = _canvas(attacks=[{"attack_type": "pair", "params": "{oops"}]) + with self.assertRaises(CanvasValidationError): + build_run_payload(canvas) + + def test_invalid_timeout(self): + with self.assertRaises(CanvasValidationError): + build_run_payload(_canvas(timeout="soon")) + + +class TestCanvasSummary(unittest.TestCase): + def test_summary_uses_catalog_labels(self): + canvas = _canvas( + attacks=[{"attack_type": "pair"}, {"attack_type": "tap"}], + ) + self.assertEqual(canvas_summary(canvas), "victim · PAIR → TAP") + + def test_summary_of_empty_canvas(self): + self.assertEqual(canvas_summary(new_canvas()), "no target · no attacks") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/server/dashboard/test_layout_mixin.py b/tests/unit/server/dashboard/test_layout_mixin.py index 002656f3..e9a02ead 100644 --- a/tests/unit/server/dashboard/test_layout_mixin.py +++ b/tests/unit/server/dashboard/test_layout_mixin.py @@ -89,13 +89,16 @@ def test_floating_button_reopens_the_sidebar(self): def test_nav_buttons_are_registered_for_every_view(self): page, mock_ui, _ = self._build() - self.assertEqual(set(page.nav_buttons), {"dashboard", "agents", "runs"}) + self.assertEqual( + set(page.nav_buttons), {"dashboard", "agents", "runs", "builder"} + ) nav_calls = { call.args[0]: call.kwargs["on_click"] for call in mock_ui.button.call_args_list - if call.args and call.args[0] in ("Home", "Targets", "History") + if call.args + and call.args[0] in ("Home", "Targets", "History", "Attack Builder") } - self.assertEqual(len(nav_calls), 3) + self.assertEqual(len(nav_calls), 4) nav_calls["Home"]() page.navigate.assert_called_once_with("dashboard") diff --git a/tests/unit/server/storage/test_local_backend.py b/tests/unit/server/storage/test_local_backend.py index d71a2354..27c08d9e 100644 --- a/tests/unit/server/storage/test_local_backend.py +++ b/tests/unit/server/storage/test_local_backend.py @@ -645,5 +645,58 @@ def create_result(i): backend.close() +class TestLocalBackendBuilderDrafts(unittest.TestCase): + """Test attack-builder draft persistence.""" + + def test_save_list_get_delete_roundtrip(self): + with tempfile.TemporaryDirectory() as tmp: + backend = _make_backend(tmp) + try: + canvas = {"target": {"agent_name": "victim"}, "attacks": [{"a": 1}]} + draft = backend.save_builder_draft("my draft", canvas) + self.assertEqual(draft["name"], "my draft") + self.assertEqual(draft["canvas"], canvas) + + self.assertEqual(len(backend.list_builder_drafts()), 1) + fetched = backend.get_builder_draft(draft["id"]) + self.assertEqual(fetched["canvas"], canvas) + + backend.delete_builder_draft(draft["id"]) + self.assertEqual(backend.list_builder_drafts(), []) + self.assertIsNone(backend.get_builder_draft(draft["id"])) + finally: + backend.close() + + def test_save_with_existing_id_updates_in_place(self): + with tempfile.TemporaryDirectory() as tmp: + backend = _make_backend(tmp) + try: + first = backend.save_builder_draft("v1", {"attacks": []}) + second = backend.save_builder_draft( + "v2", {"attacks": [{"attack_type": "pair"}]}, first["id"] + ) + self.assertEqual(second["id"], first["id"]) + self.assertEqual(second["created_at"], first["created_at"]) + drafts = backend.list_builder_drafts() + self.assertEqual(len(drafts), 1) + self.assertEqual(drafts[0]["name"], "v2") + finally: + backend.close() + + def test_drafts_persist_across_instances(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "test.db") + b1 = LocalBackend(db_path=path) + b1.save_builder_draft("kept", {"attacks": []}) + b1.close() + b2 = LocalBackend(db_path=path) + try: + self.assertEqual( + [d["name"] for d in b2.list_builder_drafts()], ["kept"] + ) + finally: + b2.close() + + if __name__ == "__main__": unittest.main()