From 190b3fafaf44f4a1d53f220f59991004b8705f06 Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:40:04 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20outline=20initial=20p?= =?UTF-8?q?lan=20for=20multi-judge=20TUI=20integration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From fbf791be39b2c0aa9956d6dbf9c23d5e48ad4e2f Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:44:22 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20feat(tui):=20add=20per-judge=20?= =?UTF-8?q?strictness=20and=20bias=20gap=20to=20run=20summary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/AISecurityLab/hackagent/sessions/25caf2e3-9c98-45d1-95a2-77300db3bc42 Co-authored-by: franconicola <51865029+franconicola@users.noreply.github.com> --- hackagent/cli/tui/views/results.py | 29 +++++ .../tui/results/test_widget_lifecycle.py | 102 ++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/hackagent/cli/tui/views/results.py b/hackagent/cli/tui/views/results.py index 93f956f9..e1b20100 100644 --- a/hackagent/cli/tui/views/results.py +++ b/hackagent/cli/tui/views/results.py @@ -1408,6 +1408,8 @@ def _show_result_summary(self, run: Any) -> None: asr = float(eval_summary.get("overall_success_rate", 0.0) or 0.0) * 100.0 mv_asr = float(eval_summary.get("majority_vote_asr", 0.0) or 0.0) * 100.0 fleiss = eval_summary.get("fleiss_kappa") + strictness = eval_summary.get("per_judge_strictness") + is_multi_judge = bool(eval_summary.get("is_multi_judge")) summary = ( f"[bold cyan]▌ Selected Run[/bold cyan]\n" @@ -1428,6 +1430,33 @@ def _show_result_summary(self, run: Any) -> None: except (TypeError, ValueError): summary += f" Fleiss κ: [bold]{_escape(str(fleiss))}[/bold]" summary += "\n" + + if is_multi_judge and isinstance(strictness, dict): + judge_keys = [k for k in strictness.keys() if k != "bias_gap"] + if judge_keys: + parts = [] + for jk in sorted(judge_keys): + try: + val = float(strictness.get(jk, 0.0) or 0.0) + judge_name = _escape( + jk.replace("eval_", "").replace("_", " ") + ) + parts.append(f"{judge_name}: [bold]{val:.3f}[/bold]") + except (TypeError, ValueError): + continue + bias_gap = strictness.get("bias_gap") + bias_gap_str = "" + if bias_gap is not None: + try: + bias_gap_str = ( + f" Bias gap: [bold]{float(bias_gap):.3f}[/bold]" + ) + except (TypeError, ValueError): + pass + summary += ( + f" [dim]Strictness — {' '.join(parts)}[/dim]" + f"{bias_gap_str}\n" + ) else: summary += "\n[dim]No evaluation summary synced yet for this run.[/dim]\n" diff --git a/tests/integration/tui/results/test_widget_lifecycle.py b/tests/integration/tui/results/test_widget_lifecycle.py index a4c34853..b59e3721 100644 --- a/tests/integration/tui/results/test_widget_lifecycle.py +++ b/tests/integration/tui/results/test_widget_lifecycle.py @@ -503,6 +503,108 @@ def compose(self): assert len(container.children) == 0 +class TestResultsRunSummary: + """ + Test suite for the run summary header panel, including multi-judge + metrics (Majority ASR, Fleiss Kappa, per-judge strictness, bias gap). + """ + + @pytest.mark.asyncio + async def test_show_result_summary_includes_multi_judge_metrics( + self, cli_config + ): + """ + Test that _show_result_summary renders multi-judge metrics. + + When the run's evaluation_summary indicates multiple judges were + used, the header should display Majority ASR, Fleiss Kappa, + per-judge strictness values, and the bias gap. + """ + + class TestApp(App): + def compose(self): + yield ResultsTab(cli_config) + + app = TestApp() + async with app.run_test() as _: + tab = app.query_one(ResultsTab) + + run = Mock() + run.id = uuid4() + run.status = Mock(value="COMPLETED") + run.timestamp = datetime(2026, 1, 19, 11, 0, 0) + run.run_config = { + "evaluation_summary": { + "total_attacks": 10, + "overall_success_rate": 0.5, + "majority_vote_asr": 0.4, + "fleiss_kappa": 0.75, + "is_multi_judge": True, + "per_judge_strictness": { + "eval_judge_a": 0.2, + "eval_judge_b": 0.6, + "bias_gap": 0.4, + }, + } + } + + tab._show_result_summary(run) + + header = tab.query_one("#run-header-static", Static) + rendered = str(header.render()) + + assert "Majority ASR" in rendered + assert "40.0%" in rendered + assert "Fleiss" in rendered + assert "0.750" in rendered + assert "Strictness" in rendered + assert "judge a" in rendered + assert "judge b" in rendered + assert "Bias gap" in rendered + assert "0.400" in rendered + + @pytest.mark.asyncio + async def test_show_result_summary_hides_strictness_for_single_judge( + self, cli_config + ): + """ + Test that per-judge strictness/bias gap are omitted when only a + single judge is present (is_multi_judge is False). + """ + + class TestApp(App): + def compose(self): + yield ResultsTab(cli_config) + + app = TestApp() + async with app.run_test() as _: + tab = app.query_one(ResultsTab) + + run = Mock() + run.id = uuid4() + run.status = Mock(value="COMPLETED") + run.timestamp = datetime(2026, 1, 19, 11, 0, 0) + run.run_config = { + "evaluation_summary": { + "total_attacks": 5, + "overall_success_rate": 0.2, + "majority_vote_asr": 0.2, + "fleiss_kappa": 1.0, + "is_multi_judge": False, + "per_judge_strictness": {"bias_gap": 0.0}, + } + } + + tab._show_result_summary(run) + + header = tab.query_one("#run-header-static", Static) + rendered = str(header.render()) + + assert "Majority ASR" in rendered + assert "Strictness" not in rendered + assert "Bias gap" not in rendered + + class TestResultsPagination: """ Test suite for pagination functionality. From 848eaf220d7f58da7b208c1607fd6e78117f5bca Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:45:22 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=90=9B=20fix(tui):=20guard=20against?= =?UTF-8?q?=20empty=20strictness=20parts=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/AISecurityLab/hackagent/sessions/25caf2e3-9c98-45d1-95a2-77300db3bc42 Co-authored-by: franconicola <51865029+franconicola@users.noreply.github.com> --- hackagent/cli/tui/views/results.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/hackagent/cli/tui/views/results.py b/hackagent/cli/tui/views/results.py index e1b20100..6bd2d78f 100644 --- a/hackagent/cli/tui/views/results.py +++ b/hackagent/cli/tui/views/results.py @@ -1444,19 +1444,20 @@ def _show_result_summary(self, run: Any) -> None: parts.append(f"{judge_name}: [bold]{val:.3f}[/bold]") except (TypeError, ValueError): continue - bias_gap = strictness.get("bias_gap") - bias_gap_str = "" - if bias_gap is not None: - try: - bias_gap_str = ( - f" Bias gap: [bold]{float(bias_gap):.3f}[/bold]" - ) - except (TypeError, ValueError): - pass - summary += ( - f" [dim]Strictness — {' '.join(parts)}[/dim]" - f"{bias_gap_str}\n" - ) + if parts: + bias_gap = strictness.get("bias_gap") + bias_gap_str = "" + if bias_gap is not None: + try: + bias_gap_str = ( + f" Bias gap: [bold]{float(bias_gap):.3f}[/bold]" + ) + except (TypeError, ValueError): + pass + summary += ( + f" [dim]Strictness — {' '.join(parts)}[/dim]" + f"{bias_gap_str}\n" + ) else: summary += "\n[dim]No evaluation summary synced yet for this run.[/dim]\n" From 0166dc9bbb2ee8b6ff33bd713b04bb85094738d1 Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:46:09 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=93=9D=20docs(tui):=20document=20judg?= =?UTF-8?q?e=20key=20naming=20convention=20in=20summary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/AISecurityLab/hackagent/sessions/25caf2e3-9c98-45d1-95a2-77300db3bc42 Co-authored-by: franconicola <51865029+franconicola@users.noreply.github.com> --- hackagent/cli/tui/views/results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hackagent/cli/tui/views/results.py b/hackagent/cli/tui/views/results.py index 6bd2d78f..818c95d5 100644 --- a/hackagent/cli/tui/views/results.py +++ b/hackagent/cli/tui/views/results.py @@ -1435,6 +1435,10 @@ def _show_result_summary(self, run: Any) -> None: judge_keys = [k for k in strictness.keys() if k != "bias_gap"] if judge_keys: parts = [] + # Judge columns follow the "eval_" naming + # convention (see _is_canonical_eval_vote_column in + # hackagent/attacks/evaluator/metrics.py); sorted for a + # stable, deterministic display order. for jk in sorted(judge_keys): try: val = float(strictness.get(jk, 0.0) or 0.0) From 430b6e6ba6ab1783bbb3b0338da7472a41f01e14 Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:24:08 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=8E=A8=20style(tests):=20apply=20ruff?= =?UTF-8?q?=20format=20to=20multi-judge=20summary=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/AISecurityLab/hackagent/sessions/63505e61-b0d7-4dbd-bcfe-2066953a0def Co-authored-by: franconicola <51865029+franconicola@users.noreply.github.com> --- tests/integration/tui/results/test_widget_lifecycle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/tui/results/test_widget_lifecycle.py b/tests/integration/tui/results/test_widget_lifecycle.py index b59e3721..dfeccbb1 100644 --- a/tests/integration/tui/results/test_widget_lifecycle.py +++ b/tests/integration/tui/results/test_widget_lifecycle.py @@ -510,9 +510,7 @@ class TestResultsRunSummary: """ @pytest.mark.asyncio - async def test_show_result_summary_includes_multi_judge_metrics( - self, cli_config - ): + async def test_show_result_summary_includes_multi_judge_metrics(self, cli_config): """ Test that _show_result_summary renders multi-judge metrics.