From 119a2ec403229f42db930487d1f843725f0c9796 Mon Sep 17 00:00:00 2001 From: Bryant Date: Sat, 18 Jul 2026 06:55:29 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=85=20(test):=20Drive=20real=20pydant?= =?UTF-8?q?ic=5Fai=20AbstractToolset.call=5Ftool=20hook=20in=20conformance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned/shipped Pydantic AI (>=0.3.0) intercepts tool execution at AbstractToolset.call_tool, not the <0.3.0 Tool._run hook. The conformance cell drove _apply_tool_run_patch, exercising a wrapper the shipped framework never runs. Point it at _apply_toolset_call_tool_patch with the real (self, name, tool_args, ctx, tool) signature and a fake toolset. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/unit/adapters/failopen_conformance.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/unit/adapters/failopen_conformance.py b/test/unit/adapters/failopen_conformance.py index c03a27ab..46211b09 100644 --- a/test/unit/adapters/failopen_conformance.py +++ b/test/unit/adapters/failopen_conformance.py @@ -319,19 +319,27 @@ async def _invoke(_ctx: Any, _tool_input: Any) -> dict[str, object]: async def _drive_pydantic_ai(interceptor: object, ran: list[bool]) -> None: from agent_assembly.adapters.pydantic_ai import patch as pydantic_ai_patch - class FakeTool: - name = "conformance_tool" - - async def _run(self, _ctx: Any, _args: Any, **_kwargs: Any) -> dict[str, object]: + # The pinned/shipped Pydantic AI line (>=0.3.0) routes tool execution through + # ``AbstractToolset.call_tool`` — the <0.3.0 ``Tool._run`` hook this cell used to + # drive does NOT exist there, so it exercised a wrapper the shipped framework never + # runs. Drive the toolset hook (``_apply_toolset_call_tool_patch``) with its real + # ``(self, name, tool_args, ctx, tool)`` signature so the cell tests what ships. + class FakeToolset: + async def call_tool(self, name: Any, tool_args: Any, ctx: Any, tool: Any, **_kwargs: Any) -> dict[str, object]: ran[0] = True return {"ok": True} - pydantic_ai_patch._apply_tool_run_patch(FakeTool, interceptor) + pydantic_ai_patch._apply_toolset_call_tool_patch(FakeToolset, interceptor) ctx = SimpleNamespace( deps=SimpleNamespace(assembly_agent_id="conformance-agent"), run_id="run-1", ) - await FakeTool()._run(ctx, {"topic": "x"}) + await FakeToolset().call_tool( + "conformance_tool", + {"topic": "x"}, + ctx, + SimpleNamespace(name="conformance_tool"), + ) async def _drive_google_adk(interceptor: object, ran: list[bool]) -> None: From 9ec7071a21879adbc1bba751538cecf05fe3cb24 Mon Sep 17 00:00:00 2001 From: Bryant Date: Sat, 18 Jul 2026 06:56:08 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=85=20(test):=20Exercise=20agno=20aex?= =?UTF-8?q?ecute=20in=20fail-open=20conformance=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agno driver drove only the sync FunctionCall.execute wrapper, leaving the async aexecute chokepoint — agno's primary async tool path — unexercised. Drive both entry points and add _assert_sync_and_async_agree so an async-only fail-open regression cannot hide behind a passing sync path. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/unit/adapters/failopen_conformance.py | 43 +++++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test/unit/adapters/failopen_conformance.py b/test/unit/adapters/failopen_conformance.py index 46211b09..ed26dbbc 100644 --- a/test/unit/adapters/failopen_conformance.py +++ b/test/unit/adapters/failopen_conformance.py @@ -188,6 +188,31 @@ def make_fake_interceptor(scenario: str, *, enforce: bool) -> object: Driver = Callable[[object, list[bool]], Awaitable[None]] +def _assert_sync_and_async_agree(adapter: str, *, sync_ran: bool, async_ran: bool) -> bool: + """Require an adapter's sync and async tool wrappers to reach the same verdict. + + Adapters that gate both a sync and an async chokepoint (agno + ``execute``/``aexecute``, llamaindex ``call``/``acall``) wrap the *same* + governance decision on both paths. A divergence means one path fails open (or + closed) where the other does not — precisely the bug class this matrix guards + against — so surface it loudly instead of letting a single shared ``ran`` flag + mask an async-only regression behind a passing sync path. + + Args: + adapter: Adapter name, for the failure message. + sync_ran: Whether the sync wrapper let the tool body run. + async_ran: Whether the async wrapper let the tool body run. + + Returns: + The agreed-upon tool-ran outcome (both paths concur). + """ + assert sync_ran == async_ran, ( + f"{adapter}: sync and async tool wrappers disagreed (sync ran={sync_ran}, " + f"async ran={async_ran}) — one path diverges from the other's governance verdict" + ) + return async_ran + + async def _drive_crewai(interceptor: object, ran: list[bool]) -> None: from agent_assembly.adapters.crewai import patch as crewai_patch @@ -205,17 +230,31 @@ def run(self, *_args: Any, **_kwargs: Any) -> dict[str, object]: async def _drive_agno(interceptor: object, ran: list[bool]) -> None: from agent_assembly.adapters.agno import patch as agno_patch + # ``_apply_execute_patch`` governs BOTH the sync ``execute`` and the async + # ``aexecute`` chokepoint (agno's primary async tool path). Drive each so the cell + # exercises the async wrapper, not just its sync sibling. class FakeFunctionCall: def __init__(self) -> None: self.function = SimpleNamespace(name="conformance_tool") self.arguments = {"amount": 1} + self.ran = False def execute(self, *_args: Any, **_kwargs: Any) -> str: - ran[0] = True + self.ran = True + return "ok" + + async def aexecute(self, *_args: Any, **_kwargs: Any) -> str: + self.ran = True return "ok" agno_patch._apply_execute_patch(FakeFunctionCall, interceptor) - FakeFunctionCall().execute() + + sync_call = FakeFunctionCall() + sync_call.execute() + async_call = FakeFunctionCall() + await async_call.aexecute() + + ran[0] = _assert_sync_and_async_agree("agno", sync_ran=sync_call.ran, async_ran=async_call.ran) async def _drive_haystack(interceptor: object, ran: list[bool]) -> None: From adb075a9fafe911dedf4e4945c5be8c625b74634 Mon Sep 17 00:00:00 2001 From: Bryant Date: Sat, 18 Jul 2026 06:56:52 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=85=20(test):=20Exercise=20llamaindex?= =?UTF-8?q?=20acall=20in=20fail-open=20conformance=20driver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The llamaindex driver drove only the sync FunctionTool.call wrapper; acall — the primary modern path awaited by FunctionAgent / ReActAgent via AgentWorkflow — went unexercised. Apply _apply_tool_acall_patch and drive both, reusing the sync/async agreement guard so the async path is genuinely governed. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/unit/adapters/failopen_conformance.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/unit/adapters/failopen_conformance.py b/test/unit/adapters/failopen_conformance.py index ed26dbbc..10147584 100644 --- a/test/unit/adapters/failopen_conformance.py +++ b/test/unit/adapters/failopen_conformance.py @@ -293,20 +293,31 @@ def __init__(self, name: str) -> None: def get_name(self) -> str: return self._name + # ``acall`` is the primary modern path (FunctionAgent / ReActAgent via + # AgentWorkflow await ``tool.acall(...)``); ``call`` is the legacy/sync path. Both + # are governed by their own patch, so drive each rather than only the sync sibling. class FakeFunctionTool: def __init__(self) -> None: self.metadata = _Meta("conformance_tool") + self.ran = False def call(self, *_args: Any, **_kwargs: Any) -> dict[str, object]: - ran[0] = True + self.ran = True return {"ok": True} async def acall(self, *_args: Any, **_kwargs: Any) -> dict[str, object]: - ran[0] = True + self.ran = True return {"ok": True} llamaindex_patch._apply_tool_call_patch(FakeFunctionTool, interceptor) - FakeFunctionTool().call(param="x") + llamaindex_patch._apply_tool_acall_patch(FakeFunctionTool, interceptor) + + sync_tool = FakeFunctionTool() + sync_tool.call(param="x") + async_tool = FakeFunctionTool() + await async_tool.acall(param="x") + + ran[0] = _assert_sync_and_async_agree("llamaindex", sync_ran=sync_tool.ran, async_ran=async_tool.ran) async def _drive_smolagents(interceptor: object, ran: list[bool]) -> None: