Return tool-call images instead of dropping them

docs/api_to_audit.md

@@ -5,6 +5,18 @@ entry here (see [AGENTS.md](../AGENTS.md), "Plugin API"). Dropping the prefix
 is the deliberate stabilization step: audit the entry, rename project-wide,
 and delete the entry in the same change.
 
+## `experimental_buildBridgeToolCallContent`
+
+**What it does.** Converts a decoded bb tool-call response into the ordered
+text and inline-image content blocks accepted by MCP and Pi tool result
+contracts. It preserves a legacy aggregate text/images input while first-party
+bridges migrate to ordered `contentBlocks`.
+
+**Audit before stabilizing.** Confirm that MCP and Pi continue sharing this
+content-block vocabulary; decide whether legacy aggregate fields still need to
+be accepted; and define any image MIME validation, decoding, or payload-size
+policy at the server boundary before making the helper stable.
+
 ## Host plugin foundation (`bb.hosts.experimental_client`, `ExperimentalHostClient.experimental_onWorkerExit`, `ExperimentalHostClient.experimental_onSignal`, `ExperimentalHostRpcContext.experimental_retainWorker`, `experimental_defineHostEntry`, and `experimental_createHostEntryHarness`)
 
 **What it does.** Lets one plugin package declare a singular `bb.host` Node

packages/agent-runtime/src/pi/bridge/__tests__/tool-proxy.test.ts

@@ -2,6 +2,53 @@ import { describe, expect, it } from "vitest";
 import { buildDynamicTools } from "../tool-proxy.js";
 
 describe("tool-proxy", () => {
+  it("returns ordered image and text content to Pi", async () => {
+    const [tool] = buildDynamicTools(
+      [
+        {
+          name: "browser_screenshot",
+          description: "Capture the browser.",
+          inputSchema: { type: "object" },
+        },
+      ],
+      async () => ({
+        content: "after",
+        contentBlocks: [
+          { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
+          { type: "text", text: "after" },
+        ],
+        images: [{ data: "iVBORw0KGgo=", mimeType: "image/png" }],
+      }),
+    );
+
+    const result = await Reflect.apply(tool.execute, tool, [
+      "call-1",
+      {},
+      undefined,
+    ]);
+    expect(result.content).toEqual([
+      { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
+      { type: "text", text: "after" },
+    ]);
+  });
+
+  it("throws forwarded failures so Pi marks the tool result as an error", async () => {
+    const [tool] = buildDynamicTools(
+      [
+        {
+          name: "broken_tool",
+          description: "Fail.",
+          inputSchema: { type: "object" },
+        },
+      ],
+      async () => ({ content: "permission denied", isError: true }),
+    );
+
+    await expect(
+      Reflect.apply(tool.execute, tool, ["call-1", {}, undefined]),
+    ).rejects.toThrow("permission denied");
+  });
+
   it("preserves required and optional scalar fields", () => {
     const [tool] = buildDynamicTools(
       [

packages/agent-runtime/src/pi/bridge/tool-proxy.ts

@@ -1,5 +1,10 @@
 import { Type, type TSchema } from "@earendil-works/pi-ai";
 import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import {
+  buildBridgeToolCallContent,
+  type BridgeToolCallContent,
+  type BridgeToolCallImage,
+} from "@bb/provider-bridge-protocol/bridge-kit";
 
 export interface DynamicToolDefinition {
   name: string;
@@ -10,7 +15,12 @@ export interface DynamicToolDefinition {
 export type ToolCallForwarder = (
   toolName: string,
   args: Record<string, unknown>,
-) => Promise<{ content: string; isError?: boolean }>;
+) => Promise<{
+  content: string;
+  contentBlocks?: BridgeToolCallContent[];
+  images?: BridgeToolCallImage[];
+  isError?: boolean;
+}>;
 
 /**
  * Builds Pi-compatible ToolDefinition objects from dynamic tool definitions
@@ -33,10 +43,14 @@ export function buildDynamicTools(
         _signal: AbortSignal | undefined,
       ) {
         const result = await forwardToolCall(def.name, params);
+        if (result.isError) {
+          // Pi marks a tool result as failed only when execute throws; an
+          // extra isError property on AgentToolResult is ignored.
+          throw new Error(result.content || "Tool call failed");
+        }
         return {
-          content: [{ type: "text" as const, text: result.content }],
+          content: buildBridgeToolCallContent(result),
           details: {},
-          ...(result.isError ? { isError: true } : {}),
         };
       },
     } as ToolDefinition;

packages/plugin-sdk/src/provider-bridge.ts

@@ -112,6 +112,7 @@ export type {
 export {
   bashArgsSchema,
   bridgeRequestEnvelopeSchema,
+  buildBridgeToolCallContent as experimental_buildBridgeToolCallContent,
   buildShellEnvOverrides,
   createBridgeIo,
   createBridgeLineHandler,

packages/provider-bridge-protocol/src/bridge-kit/bridge-tool-calls.test.ts

@@ -0,0 +1,223 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildBridgeToolCallContent,
+  decodeToolCallResponsePayload,
+} from "./bridge-tool-calls.js";
+
+const PNG = "iVBORw0KGgo=";
+
+describe("decodeToolCallResponsePayload", () => {
+  it("keeps text results unchanged", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          { type: "inputText", text: "first" },
+          { type: "inputText", text: "second" },
+        ],
+      }),
+    ).toEqual({
+      content: "first\nsecond",
+      contentBlocks: [
+        { type: "text", text: "first" },
+        { type: "text", text: "second" },
+      ],
+      images: [],
+      isError: false,
+    });
+  });
+
+  // The bug: an image-only result decoded to the literal "OK" with the image
+  // dropped, so browser_screenshot reported success and returned nothing.
+  it("decodes an image-only result into an image rather than OK", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          { type: "inputImage", imageUrl: `data:image/png;base64,${PNG}` },
+        ],
+      }),
+    ).toEqual({
+      content: "",
+      contentBlocks: [{ type: "image", data: PNG, mimeType: "image/png" }],
+      images: [{ data: PNG, mimeType: "image/png" }],
+      isError: false,
+    });
+  });
+
+  it("keeps both halves of a mixed text and image result", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          { type: "inputText", text: "captured" },
+          { type: "inputImage", imageUrl: `data:image/jpeg;base64,${PNG}` },
+        ],
+      }),
+    ).toEqual({
+      content: "captured",
+      contentBlocks: [
+        { type: "text", text: "captured" },
+        { type: "image", data: PNG, mimeType: "image/jpeg" },
+      ],
+      images: [{ data: PNG, mimeType: "image/jpeg" }],
+      isError: false,
+    });
+  });
+
+  it("preserves interleaved text and image order", () => {
+    const decoded = decodeToolCallResponsePayload({
+      success: true,
+      contentItems: [
+        { type: "inputImage", imageUrl: `data:image/png;base64,${PNG}` },
+        { type: "inputText", text: "between" },
+        { type: "inputImage", imageUrl: `data:image/jpeg;base64,${PNG}` },
+      ],
+    });
+
+    expect(buildBridgeToolCallContent(decoded)).toEqual([
+      { type: "image", data: PNG, mimeType: "image/png" },
+      { type: "text", text: "between" },
+      { type: "image", data: PNG, mimeType: "image/jpeg" },
+    ]);
+  });
+
+  it("reports an image result that failed as an error", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: false,
+        contentItems: [
+          { type: "inputImage", imageUrl: `data:image/png;base64,${PNG}` },
+        ],
+      }).isError,
+    ).toBe(true);
+  });
+
+  // A tool result contract carries inline base64 only, so a remote reference
+  // has to survive as text; dropping it is what this fix exists to stop.
+  it("keeps a non-data image url as text", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          { type: "inputImage", imageUrl: "https://example.com/a.png" },
+        ],
+      }),
+    ).toEqual({
+      content: "https://example.com/a.png",
+      contentBlocks: [{ type: "text", text: "https://example.com/a.png" }],
+      images: [],
+      isError: false,
+    });
+  });
+
+  it("keeps a data url with an empty payload as text", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          { type: "inputImage", imageUrl: "data:image/png;base64," },
+        ],
+      }),
+    ).toEqual({
+      content: "data:image/png;base64,",
+      contentBlocks: [{ type: "text", text: "data:image/png;base64," }],
+      images: [],
+      isError: false,
+    });
+  });
+
+  it("falls back to OK only when there is neither text nor image", () => {
+    expect(
+      decodeToolCallResponsePayload({ success: true, contentItems: [] }),
+    ).toEqual({
+      content: "OK",
+      contentBlocks: [{ type: "text", text: "OK" }],
+      images: [],
+      isError: false,
+    });
+  });
+
+  it("surfaces empty failures and malformed payloads as errors", () => {
+    expect(
+      decodeToolCallResponsePayload({ success: false, contentItems: [] }),
+    ).toEqual({
+      content: "Tool call failed",
+      contentBlocks: [{ type: "text", text: "Tool call failed" }],
+      images: [],
+      isError: true,
+    });
+    expect(decodeToolCallResponsePayload({ nope: true })).toEqual({
+      content: "Invalid tool call response",
+      contentBlocks: [{ type: "text", text: "Invalid tool call response" }],
+      images: [],
+      isError: true,
+    });
+  });
+
+  it("accepts MIME parameters in an inline image data URL", () => {
+    expect(
+      decodeToolCallResponsePayload({
+        success: true,
+        contentItems: [
+          {
+            type: "inputImage",
+            imageUrl: `data:image/svg+xml;charset=utf-8;base64,${PNG}`,
+          },
+        ],
+      }).images,
+    ).toEqual([{ data: PNG, mimeType: "image/svg+xml;charset=utf-8" }]);
+  });
+});
+
+describe("buildBridgeToolCallContent", () => {
+  it("emits an image block alone when there is no text", () => {
+    expect(
+      buildBridgeToolCallContent({
+        content: "",
+        images: [{ data: PNG, mimeType: "image/png" }],
+      }),
+    ).toEqual([{ type: "image", data: PNG, mimeType: "image/png" }]);
+  });
+
+  it("keeps text first when a result carries both", () => {
+    expect(
+      buildBridgeToolCallContent({
+        content: "captured",
+        images: [{ data: PNG, mimeType: "image/png" }],
+      }),
+    ).toEqual([
+      { type: "text", text: "captured" },
+      { type: "image", data: PNG, mimeType: "image/png" },
+    ]);
+  });
+
+  it("prefers ordered content blocks when present", () => {
+    expect(
+      buildBridgeToolCallContent({
+        content: "legacy text",
+        contentBlocks: [
+          { type: "image", data: PNG, mimeType: "image/png" },
+          { type: "text", text: "after" },
+        ],
+        images: [{ data: PNG, mimeType: "image/png" }],
+      }),
+    ).toEqual([
+      { type: "image", data: PNG, mimeType: "image/png" },
+      { type: "text", text: "after" },
+    ]);
+  });
+
+  it("emits a lone text block for a text result", () => {
+    expect(buildBridgeToolCallContent({ content: "OK", images: [] })).toEqual([
+      { type: "text", text: "OK" },
+    ]);
+  });
+
+  // The pending-call failure paths resolve without an images key.
+  it("tolerates a result with no images key", () => {
+    expect(buildBridgeToolCallContent({ content: "transport closed" })).toEqual(
+      [{ type: "text", text: "transport closed" }],
+    );
+  });
+});