diff --git a/.env.example b/.env.example index cdaf0f1..80833e1 100644 --- a/.env.example +++ b/.env.example @@ -117,6 +117,9 @@ AGENT_COMPUTER_URL=http://localhost:4100 # without this value and refuses every request that does not present it. Use a long random value; # `scripts/start.sh` sets a development one for you. COMPUTER_TOKEN= +# Browser implementation inside each computer. `playwright` is the default. `cua-driver` uses Cua +# Driver's native SDK, isolated named profiles, semantic browser snapshots, and native takeover. +# COMPUTER_BACKEND=cua-driver # Local only. Lets a Bot browse this machine's own services; never set this in a deployment. AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true # @@ -163,7 +166,8 @@ AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true # `EGRESS_PROXY_DEFAULT` covers the rest, and absent means the browser goes out directly. # # Credentials may be in the URL and are split out before Playwright sees them. Only the HOST is ever -# reported back on the admin page or the API, so the password does not end up on a screen. +# reported back on the admin page or the API, so the password does not end up on a screen. Egress +# proxy settings currently apply only to the default Playwright backend. # # This is attribution, not anonymity, and it is not a boundary by itself: it gives a security team a # per-Bot address for network rules alongside AGENT_COMPUTER_POLICY. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b03d0cc..d4497d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,10 @@ jobs: - run: bun run format:check - run: bun run lint - run: bun run typecheck + # agent-computer is intentionally outside the root workspaces because it is built as its own + # image. Install and typecheck its native backend against its separate lockfile explicitly. + - run: bun install --frozen-lockfile && bun run typecheck + working-directory: agent-computer test: name: tests @@ -83,3 +87,5 @@ jobs: bun-version: 1.3.14 - run: bun install --frozen-lockfile - run: bun run build + - run: docker build -f agent-computer/Dockerfile -t openbot-agent-computer:test . + - run: docker build -t openbot:test . diff --git a/Dockerfile b/Dockerfile index 0e29dc6..3b16a14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ FROM mcr.microsoft.com/playwright:v1.62.1-noble AS base -# unzip is not in the Playwright image and bun's installer needs it. +# unzip is not in the Playwright image and bun's installer needs it. Openbox gives Cua Driver's +# native input route a foreground X11 window without adding a desktop environment. # Bun is pinned. The installer takes whatever is newest otherwise, so the runtime drifts from the # one the lockfile was resolved against and an image built next month is not the image built today. ARG BUN_VERSION=1.3.14 @@ -28,7 +29,7 @@ ARG BUN_VERSION=1.3.14 # root's home. Set before the install, or the installer has already chosen the wrong directory. ENV BUN_INSTALL=/usr/local ENV PATH="/usr/local/bin:${PATH}" -RUN apt-get update && apt-get install -y --no-install-recommends unzip xz-utils \ +RUN apt-get update && apt-get install -y --no-install-recommends openbox unzip xz-utils \ && rm -rf /var/lib/apt/lists/* \ && curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" @@ -45,8 +46,8 @@ COPY server/package.json server/package.json COPY worker/package.json worker/package.json RUN bun install --frozen-lockfile -COPY agent-computer/package.json agent-computer/package.json -RUN cd agent-computer && bun install +COPY agent-computer/package.json agent-computer/bun.lock agent-computer/ +RUN cd agent-computer && bun install --frozen-lockfile # A second tree with the build-time dependencies left out, for the runtime stage to take. Vite, # biome and the test tooling are a gigabyte that nothing in a running container imports. @@ -172,7 +173,8 @@ ENV AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true # The two directories the browser writes are its workspace and its profile, the second being what # keeps a Bot signed in between turns. Owned here, because a non-root process cannot create them at # the root of the filesystem and the failure surfaces as EACCES on the first navigation. -RUN mkdir -p /workspace /profiles \ +RUN mkdir -p /workspace /profiles /tmp/.X11-unix \ + && chmod 1777 /tmp/.X11-unix \ && chown -R pwuser:pwuser /workspace /profiles /app # Where the embedded database answers, when there is one. Overridden by whatever you set, so an diff --git a/README.md b/README.md index 27d13b0..ebadcee 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ as one replica for now. ## Features -- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it. +- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. Playwright is the default browser backend; set `COMPUTER_BACKEND=cua-driver` to use Cua Driver's semantic browser and native computer-use stack. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it. - **A shell, not just a browser**: a Bot can run a command in its workspace, install what it needs, and process a file it saved. Through the same gate as everything else, so a rule can refuse a shell outright or refuse particular commands, and the command is on the record either way. - **The gateway is the only way in**: it resolves the target from a server-held snapshot, evaluates the policy, writes the audit row, and only then calls the computer. There is no path that acts without the record existing first. - **CEL policy, fail closed**: rules can inspect `tool.name`, `intent`, `bot.id`, `actor.id`, `page.url`, `page.host`, `element.*`, `key`, `file.*` and `mcp.*`. Deny is evaluated before allow, a missing policy permits nothing, and a broken rule refuses rather than opens. @@ -194,6 +194,7 @@ Settings worth knowing: | `OPENAI_BASE_URL` | Answers the OpenAI-shaped calls from somewhere else: a gateway, a proxy. | | `ANTHROPIC_BASE_URL`, `GOOGLE_GENERATIVE_AI_BASE_URL` | The same, for those two APIs. | | `COMPUTER_TOKEN` | Secret every Bot computer request must present. `start.sh` sets one. | +| `COMPUTER_BACKEND` | `playwright` (default) or `cua-driver`. | | `SUPERVISOR_TOKEN` | Secret the supervisor requires. `start.sh` sets one. | | `COMPUTER_SUPERVISOR_URL` | Gives each Bot a computer of its own instead of one shared computer. | | `COMPUTER_RUNTIME` | Set to `runsc` to run computers under gVisor, where the host has it. | diff --git a/agent-computer/Dockerfile b/agent-computer/Dockerfile index 594db89..fc2b160 100644 --- a/agent-computer/Dockerfile +++ b/agent-computer/Dockerfile @@ -1,18 +1,20 @@ -# The Bot's computer uses Playwright's image so Chromium and its system libraries stay matched. +# The Bot's computer uses Playwright's image so Chromium, Xvfb, and their system libraries stay +# matched. Xvfb is needed by Cua Driver's optional native computer backend. # # The image tag and Playwright dependency must be pinned to the same exact version. Bump both or # neither. FROM mcr.microsoft.com/playwright:v1.62.1-noble -# unzip is not in the Playwright image and bun's installer needs it. -RUN apt-get update && apt-get install -y --no-install-recommends unzip \ +# unzip is not in the Playwright image and bun's installer needs it. Openbox gives Cua Driver's +# foreground input route an active X11 window without adding a desktop environment. +RUN apt-get update && apt-get install -y --no-install-recommends openbox unzip \ && rm -rf /var/lib/apt/lists/* \ && curl -fsSL https://bun.sh/install | bash ENV PATH="/root/.bun/bin:${PATH}" WORKDIR /app -COPY agent-computer/package.json ./ -RUN bun install +COPY agent-computer/package.json agent-computer/bun.lock ./ +RUN bun install --frozen-lockfile COPY agent-computer/src ./src @@ -21,9 +23,15 @@ RUN mkdir -p /workspace ENV WORKSPACE_DIR=/workspace ENV PORT=4100 +ENV DISPLAY=:99 +# The image runs Chromium as root inside its container, matching the existing Playwright launch. +# Cua Driver otherwise refuses that browser before OpenBot can apply its own container boundary. +ENV CUA_E2E_BROWSER_NO_SANDBOX=1 EXPOSE 4100 # Readiness is exposed through Docker health so the supervisor need not know the network route. HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=30 \ CMD bun -e "const r = await fetch('http://localhost:4100/health'); process.exit(r.ok ? 0 : 1)" -CMD ["bun", "src/index.ts"] +# Cua needs a private display and window manager for native keyboard/takeover input. Playwright stays +# on its existing headless path. Fail startup if Xvfb dies instead of waiting forever for its socket. +CMD ["sh", "-c", "if [ \"${COMPUTER_BACKEND:-playwright}\" != cua-driver ]; then exec bun src/index.ts; fi; Xvfb :99 -screen 0 1280x1024x24 -nolisten tcp & display_pid=$!; until [ -S /tmp/.X11-unix/X99 ]; do kill -0 \"$display_pid\" 2>/dev/null || exit 1; sleep 0.05; done; openbox --sm-disable & export CUA_DRIVER_BROWSER_PROFILE_ROOT=\"${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver}\"; exec bun src/index.ts"] diff --git a/agent-computer/bun.lock b/agent-computer/bun.lock index 929dce1..25d0d45 100644 --- a/agent-computer/bun.lock +++ b/agent-computer/bun.lock @@ -5,6 +5,7 @@ "": { "name": "@openbot/agent-computer", "dependencies": { + "@trycua/cua-driver": "0.20.0", "playwright": "1.62.1", "spiffe": "^0.5.1", "yaml": "^2.9.0", @@ -73,8 +74,42 @@ "@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="], + "@trycua/cua-driver": ["@trycua/cua-driver@0.20.0", "", { "dependencies": { "@ubjs/core": "0.31.0-3", "@ubjs/node": "0.31.0-3" }, "optionalDependencies": { "@trycua/cua-driver-darwin-arm64": "0.20.0", "@trycua/cua-driver-darwin-x64": "0.20.0", "@trycua/cua-driver-linux-arm64-gnu": "0.20.0", "@trycua/cua-driver-linux-x64-gnu": "0.20.0", "@trycua/cua-driver-win32-arm64-msvc": "0.20.0", "@trycua/cua-driver-win32-x64-msvc": "0.20.0" } }, "sha512-PYNA9zbZX46LLObcPSNUm37tIlP0/klphRaSyuJPA3NdaaLiglfw3HcWM/C0wB2KTb8nIS17hb2qCxB8F1VC4Q=="], + + "@trycua/cua-driver-darwin-arm64": ["@trycua/cua-driver-darwin-arm64@0.20.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-LiEZ3Mku2BI83UJD7MWNoGw+PYIyo6hfmqA5T3zWle7Rti3Jhvic+Tnu+KZ9i+HXBZEQ84GUR91S+zUTdchh+Q=="], + + "@trycua/cua-driver-darwin-x64": ["@trycua/cua-driver-darwin-x64@0.20.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-4T2+qPYvW8ZUi6XYJM9r6mjfllw3zyEZUUdYvy9Q1WQA6HjcY4mXhXFlEfLjWE0wOeXYrNEXJ8JyYEL0jlzmtw=="], + + "@trycua/cua-driver-linux-arm64-gnu": ["@trycua/cua-driver-linux-arm64-gnu@0.20.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-ya4Cc3ZO1x3HvCIAcouvVnhWDP5f7ZDWczeG1ym3qWST02AP5EtWXrqoPxRKbTV7y/yqovPO/Bk3mG+3F92AoA=="], + + "@trycua/cua-driver-linux-x64-gnu": ["@trycua/cua-driver-linux-x64-gnu@0.20.0", "", { "os": "linux", "cpu": "x64" }, "sha512-NjCt19AoCTqe148FdNAK6DmYEuFz9oW+/zh0OzCpfrGNvOvqnOkBT5wVFl7NFQIaNIlERV9f3+SG0Z6eL2QBkA=="], + + "@trycua/cua-driver-win32-arm64-msvc": ["@trycua/cua-driver-win32-arm64-msvc@0.20.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-16a1berZU8BdIA5NDg7ZWM+6r+LTTrOUcS3inUfPeWTxYiqlYhLthDVKR4PZCJhkx+w07ySSviS5O47XQx/M9A=="], + + "@trycua/cua-driver-win32-x64-msvc": ["@trycua/cua-driver-win32-x64-msvc@0.20.0", "", { "os": "win32", "cpu": "x64" }, "sha512-COOfQZJIcbhWSrfvcT98F7TFbdmwMqqP9AUQIzvPo82VxzXb/nG2KrZohinelR5dAdJ6YBGRw8hIMW/QsgTp9g=="], + "@types/node": ["@types/node@26.2.0", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg=="], + "@ubjs/core": ["@ubjs/core@0.31.0-3", "", {}, "sha512-39XrJgUZ2VVb561sSnkXPhczNoeBsNiSRArecsV0JE7CJq69ajFkcn9/tBAUS2NpgHkLIDU+z6Ks2+1wXnboxg=="], + + "@ubjs/node": ["@ubjs/node@0.31.0-3", "", { "optionalDependencies": { "@ubjs/node-darwin-arm64": "0.31.0-3", "@ubjs/node-darwin-x64": "0.31.0-3", "@ubjs/node-linux-arm64-gnu": "0.31.0-3", "@ubjs/node-linux-arm64-musl": "0.31.0-3", "@ubjs/node-linux-x64-gnu": "0.31.0-3", "@ubjs/node-linux-x64-musl": "0.31.0-3", "@ubjs/node-win32-arm64-msvc": "0.31.0-3", "@ubjs/node-win32-x64-msvc": "0.31.0-3" } }, "sha512-qNMpi2LICNwxGXZyRF8fSDBSpbezyZbEsydrbiMPJOmtOWr4tmZIEl7jkWGHVGShoBvHfFo4eHp5B4UVP928Cg=="], + + "@ubjs/node-darwin-arm64": ["@ubjs/node-darwin-arm64@0.31.0-3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-GGQVPLkVo4Gc8qVLW4IGvS8bjl8eHXyeP4a97ntGmsAdXwE5gsS29o8xUEFROjhwHKD+9sgVeblCSWVG3CpHsw=="], + + "@ubjs/node-darwin-x64": ["@ubjs/node-darwin-x64@0.31.0-3", "", { "os": "darwin", "cpu": "x64" }, "sha512-2sc47u4XFYOsbmP5EW+Gx8m/yGrYnfFDFQm6+kz7goSWTNg84eEiz3COs9HKJDVuNJ5Khv5XipTO8CFadMLXCw=="], + + "@ubjs/node-linux-arm64-gnu": ["@ubjs/node-linux-arm64-gnu@0.31.0-3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YStVXhYz/5jvlWf/p4fhiVT72unYAbGugifFC9QmO/+hnroQDAQ5t8SARbsc15G4olMcamdIB+GETiUB7gmaYg=="], + + "@ubjs/node-linux-arm64-musl": ["@ubjs/node-linux-arm64-musl@0.31.0-3", "", { "os": "linux", "cpu": "arm64" }, "sha512-Izp4nvfy/LmibzFowAztkoDOksCR2fb2zl6fh1ojR1HEsg0rAGruxtI8d3fn8DI0lBXwqmn6SF///oD4mFNJPQ=="], + + "@ubjs/node-linux-x64-gnu": ["@ubjs/node-linux-x64-gnu@0.31.0-3", "", { "os": "linux", "cpu": "x64" }, "sha512-Xdm21blyg5U/kW6s7OMvgrr8coGTkUlt26DVR9x8gKISif+E3YwdEskbFScWqystAiJnfjw7xHEc8UMu0Qlz7Q=="], + + "@ubjs/node-linux-x64-musl": ["@ubjs/node-linux-x64-musl@0.31.0-3", "", { "os": "linux", "cpu": "x64" }, "sha512-fFQ9BWS6i2LUH9SJgD9oEiKXXo/say59vHy7usFe1t7C2xvwvP34f5SuxXmWP9R8fHyA0aC4kIZ+TTwyHSv1Kw=="], + + "@ubjs/node-win32-arm64-msvc": ["@ubjs/node-win32-arm64-msvc@0.31.0-3", "", { "os": "win32", "cpu": "arm64" }, "sha512-ID6rSz1NmPsWTNBBNAw4OnJ5Dj8pcbtNJtdPB3OxcGigLBd/e0x7buhSI7os6Mo5iYtEdCynBRQHFJwub5XSPg=="], + + "@ubjs/node-win32-x64-msvc": ["@ubjs/node-win32-x64-msvc@0.31.0-3", "", { "os": "win32", "cpu": "x64" }, "sha512-wevs+Y+szwcCUT8IJFbB4/1nfxyRv/51l8oG7FGUPWD1xLPyuHfJBG27C+PDeC7KRzes/2n6aLo4DGZbr/LYTw=="], + "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], diff --git a/agent-computer/package.json b/agent-computer/package.json index 4a21501..2dc0715 100644 --- a/agent-computer/package.json +++ b/agent-computer/package.json @@ -4,11 +4,12 @@ "private": true, "type": "module", "scripts": { - "dev": "bun --watch src/index.ts", - "start": "bun src/index.ts", + "dev": "CUA_DRIVER_BROWSER_PROFILE_ROOT=${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver} bun --watch src/index.ts", + "start": "CUA_DRIVER_BROWSER_PROFILE_ROOT=${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver} bun src/index.ts", "typecheck": "tsc --noEmit" }, "dependencies": { + "@trycua/cua-driver": "0.20.0", "playwright": "1.62.1", "spiffe": "^0.5.1", "yaml": "^2.9.0" diff --git a/agent-computer/src/browser.ts b/agent-computer/src/browser.ts new file mode 100644 index 0000000..6365411 --- /dev/null +++ b/agent-computer/src/browser.ts @@ -0,0 +1,120 @@ +import type { InputMessage } from "./screencast"; +import type { ProfileSummary } from "./profiles"; + +export { VIEWPORT } from "./profiles"; + +export type BrowserElement = { + ref: string; + role: string; + name: string; + value?: string; + type?: string; + disabled?: boolean; + checked?: boolean; +}; + +export type PageRead = { + url: string; + title: string; + text: string; + truncated: boolean; +}; + +export type BrowserSnapshot = { + snapshotId: number; + url: string; + title: string; + elements: BrowserElement[]; + truncated: boolean; +}; + +export type BrowserScreenshot = { + base64: string; + width: number; + height: number; + capturedAt: string; + url: string; +}; + +export type BrowserAction = { + action: "click" | "type" | "key" | "scroll"; + ref?: string; + characters?: number; + submitted?: boolean; + key?: string; + deltaY?: number; + url: string; +}; + +export type HumanAction = { + action: "human_click" | "human_type" | "human_key" | "human_scroll"; + characters?: number; + key?: string; + deltaY?: number; + url: string; +}; + +export type FrameMessage = { + type: "frame"; + data: string; + width: number; + height: number; + mimeType?: string; +}; + +export type BrowserStream = { + stop: () => Promise; + send: (message: InputMessage) => Promise; +}; + +export class StaleSnapshotError extends Error { + constructor(message: string) { + super(message); + this.name = "StaleSnapshotError"; + } +} + +export interface BrowserComputer { + navigate(url: string, signal?: AbortSignal): Promise; + read(): Promise; + screenshot(): Promise; + snapshot(): Promise; + click( + ref: string, + snapshotId?: number, + signal?: AbortSignal, + ): Promise; + type( + ref: string, + text: string, + submit: boolean, + snapshotId?: number, + signal?: AbortSignal, + ): Promise; + key( + key: string, + ref?: string, + snapshotId?: number, + signal?: AbortSignal, + ): Promise; + scroll(deltaY: number, signal?: AbortSignal): Promise; + enterSecret( + ref: string, + text: string, + ): Promise<{ characters: number; url: string }>; + humanClick(x: number, y: number): Promise; + humanType(text: string): Promise; + humanKey(key: string): Promise; + humanScroll(deltaY: number): Promise; + startStream(onFrame: (frame: FrameMessage) => void): Promise; +} + +export interface BrowserManager { + backend: "playwright" | "cua-driver"; + computer(botId: string): BrowserComputer; + known(): Promise; + summary(botIds: string[]): ProfileSummary[]; + stop(botId: string): Promise; + reset(botId: string): Promise; + closeAll(): Promise; +} diff --git a/agent-computer/src/cua-browser.ts b/agent-computer/src/cua-browser.ts new file mode 100644 index 0000000..d4e1564 --- /dev/null +++ b/agent-computer/src/cua-browser.ts @@ -0,0 +1,801 @@ +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + BrowserAction, + BrowserComputer, + BrowserElement, + BrowserManager, + BrowserSnapshot, + HumanAction, +} from "./browser"; +import { StaleSnapshotError } from "./browser"; +import type { InputMessage } from "./screencast"; + +type CuaToolResult = { + text: string; + images: { mimeType: string; dataBase64: string }[]; + structuredJson?: string; + isError: boolean; + errorCode?: string; + rawJson: string; +}; + +export type CuaDriver = { + callTool( + name: string, + argumentsJson: string, + options?: { signal: AbortSignal }, + ): Promise; + shutdown(): Promise; + uniffiDestroy?: () => void; +}; + +type BrowserState = { + session: string; + pid: number; + windowId: number; + targetId: string; + tabId: string; + url: string; + title: string; + snapshotId: number; + startedAt: string; + lastRead?: { + text: string; + truncated: boolean; + }; +}; + +type Structured = Record; + +const PROFILE_NAME = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; +const TEXT_EXTRACT_LIMIT = 6000; +const SNAPSHOT_ELEMENT_LIMIT = 200; +const STALE_CODES = new Set(["browser_binding_stale", "browser_ref_stale"]); + +const CUA_KEY_NAMES: Record = { + ArrowDown: "down", + ArrowLeft: "left", + ArrowRight: "right", + ArrowUp: "up", + Esc: "escape", + Meta: "super", + PageDown: "pagedown", + PageUp: "pageup", + " ": "space", + Spacebar: "space", +}; + +function cuaKeyName(key: string): string { + return CUA_KEY_NAMES[key] ?? key.toLowerCase(); +} + +function structured(result: CuaToolResult): Structured { + if (!result.structuredJson) return {}; + try { + return JSON.parse(result.structuredJson) as Structured; + } catch { + return {}; + } +} + +function field(value: unknown): Structured { + return value !== null && typeof value === "object" + ? (value as Structured) + : {}; +} + +function string(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function boolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +function list(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function staleMessage(expected: number, current: number): string { + return `That list of elements is out of date: it was taken for snapshot ${expected} and the page is now at ${current}. Take a new snapshot and use the refs from it.`; +} + +export function createCuaBrowserManager( + root: string, + driver: CuaDriver, + browserExecutable: string, + options: { + seedPid?: () => Promise<{ pid: number; close: () => Promise }>; + sleep?: (ms: number) => Promise; + } = {}, +): BrowserManager { + const states = new Map(); + const starting = new Map>(); + const computers = new Map(); + + const call = async ( + name: string, + args: Record, + signal?: AbortSignal, + ): Promise<{ result: CuaToolResult; data: Structured }> => { + const result = await driver.callTool( + name, + JSON.stringify(args), + signal ? { signal } : undefined, + ); + const data = structured(result); + if (result.isError) { + const refusal = field(data.refusal); + const code = + result.errorCode ?? string(refusal.code) ?? string(data.code) ?? ""; + const message = result.text || `Cua Driver ${name} failed.`; + if ( + STALE_CODES.has(code) || + /\bbrowser_(?:binding|ref)_stale\b/.test(message) + ) { + throw new StaleSnapshotError(message); + } + throw new Error(message); + } + return { result, data }; + }; + + const pause = options.sleep ?? sleep; + const launchSeed = async (): Promise<{ + pid: number; + close: () => Promise; + }> => { + const profile = await mkdtemp(join(tmpdir(), "openbot-cua-seed-")); + const process = Bun.spawn({ + cmd: [ + browserExecutable, + "--headless", + "--no-sandbox", + "--disable-dev-shm-usage", + "--remote-debugging-port=0", + `--user-data-dir=${profile}`, + "about:blank", + ], + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }); + await pause(100); + return { + pid: process.pid, + async close() { + process.kill(); + await process.exited.catch(() => undefined); + await rm(profile, { recursive: true, force: true }); + }, + }; + }; + + const windowFor = async (pid: number): Promise => { + for (let attempt = 0; attempt < 80; attempt += 1) { + const { data } = await call("list_windows", { + pid, + on_screen_only: true, + }); + const windows = list(data.windows) + .map(field) + .filter((window) => number(window.window_id) !== undefined) + .sort( + (left, right) => + (number(right.z_index) ?? Number.NEGATIVE_INFINITY) - + (number(left.z_index) ?? Number.NEGATIVE_INFINITY), + ); + const selected = windows[0]; + if (selected) return number(selected.window_id) as number; + await pause(100); + } + throw new Error( + "Cua Driver launched Chromium but no browser window appeared.", + ); + }; + + const start = async (botId: string): Promise => { + if (!PROFILE_NAME.test(botId)) { + throw new Error( + "A Cua Driver computer id may contain only letters, digits, hyphen and underscore, start with a letter or digit, and be at most 64 characters.", + ); + } + const session = `openbot-${botId}`; + try { + await call("start_session", { session }); + const seed = await (options.seedPid ?? launchSeed)(); + let prepared: Structured; + try { + ({ data: prepared } = await call("browser_prepare", { + pid: seed.pid, + allow_launch: true, + profile: { mode: "isolated_named", name: botId }, + session, + })); + } finally { + await seed.close(); + } + + const pid = number(prepared.prepared_pid); + if (!pid) { + throw new Error( + "Cua Driver did not report the prepared browser process.", + ); + } + const windowId = await windowFor(pid); + const { data: bound } = await call("get_browser_state", { + pid, + window_id: windowId, + session, + }); + if ( + bound.binding_quality !== "exact" || + bound.mutation_allowed !== true + ) { + throw new Error( + "Cua Driver could not bind the isolated browser exactly.", + ); + } + const targetId = string(bound.target_id); + const tabs = list(bound.tabs).map(field); + const selected = + tabs.find((tab) => tab.active === true) ?? + tabs.find((tab) => tab.url === "about:blank") ?? + tabs[0]; + const tabId = string(selected?.tab_id); + if (!targetId || !tabId) { + throw new Error("Cua Driver did not return a browser target and tab."); + } + const state: BrowserState = { + session, + pid, + windowId, + targetId, + tabId, + url: string(selected?.url) ?? "about:blank", + title: string(selected?.title) ?? "", + snapshotId: 0, + startedAt: new Date().toISOString(), + }; + states.set(botId, state); + return state; + } catch (error) { + await call("end_session", { session }).catch(() => undefined); + throw error; + } + }; + + const ensure = async (botId: string): Promise => { + const existing = states.get(botId); + if (existing) { + // Explicit sessions expire after five idle minutes. Starting the same name refreshes or + // revives it; checking the exact native window detects the managed browser cleanup that an + // expiry performs, so the next request can reopen the durable profile instead of pinning this + // process to a dead pid. + await call("start_session", { session: existing.session }); + const alive = await call("list_windows", { + pid: existing.pid, + on_screen_only: false, + }) + .then(({ data }) => + list(data.windows) + .map(field) + .some((window) => number(window.window_id) === existing.windowId), + ) + .catch(() => false); + if (alive) { + return existing; + } + await call("end_session", { session: existing.session }).catch( + () => undefined, + ); + states.delete(botId); + } + const pending = starting.get(botId); + if (pending) return pending; + const created = start(botId); + starting.set(botId, created); + try { + return await created; + } finally { + starting.delete(botId); + } + }; + + const refreshBinding = async (state: BrowserState): Promise => { + const previousUrl = state.url; + const { data } = await call("get_browser_state", { + pid: state.pid, + window_id: state.windowId, + session: state.session, + }); + if (data.binding_quality !== "exact" || data.mutation_allowed !== true) { + throw new Error( + "Cua Driver could not refresh the isolated browser binding.", + ); + } + const tabs = list(data.tabs).map(field); + const selected = + tabs.find((tab) => tab.tab_id === state.tabId) ?? + tabs.find((tab) => tab.active === true) ?? + tabs[0]; + const targetId = string(data.target_id); + const tabId = string(selected?.tab_id); + if (!targetId || !tabId) { + throw new Error("Cua Driver did not return a browser target and tab."); + } + state.targetId = targetId; + state.tabId = tabId; + state.url = string(selected?.url) ?? state.url; + state.title = string(selected?.title) ?? state.title; + if (state.url !== previousUrl) state.lastRead = undefined; + }; + + const observe = async ( + state: BrowserState, + ): Promise<{ + elements: BrowserElement[]; + text: string; + truncated: boolean; + }> => { + await refreshBinding(state); + const { data } = await call("get_browser_state", { + target_id: state.targetId, + tab_id: state.tabId, + session: state.session, + snapshot_format: "semantic_v2", + include_screenshot: false, + }); + const page = field(data.page); + state.url = string(page.url) ?? state.url; + const observedTitle = string(page.title); + if ( + observedTitle !== undefined && + !(observedTitle === "about:blank" && state.url !== "about:blank") + ) { + state.title = observedTitle; + } + state.snapshotId += 1; + const actionable = list(data.refs) + .map(field) + .filter((ref) => { + const actions = list(ref.actions); + return ( + Boolean(string(ref.ref)) && + (actions.includes("click") || actions.includes("type")) + ); + }); + const elements = actionable.slice(0, SNAPSHOT_ELEMENT_LIMIT).map((ref) => { + const states = field(ref.states); + const role = string(ref.role) ?? "unknown"; + const checked = boolean(states.checked); + return { + ref: string(ref.ref) as string, + role, + name: (string(ref.name) ?? "").slice(0, 200), + ...(string(ref.value) !== undefined + ? { value: (string(ref.value) as string).slice(0, 200) } + : {}), + ...(string(states.type) !== undefined + ? { type: string(states.type) } + : {}), + ...(boolean(states.disabled) !== undefined + ? { disabled: boolean(states.disabled) } + : {}), + ...(checked !== undefined ? { checked } : {}), + } satisfies BrowserElement; + }); + const outline = string(data.outline) ?? ""; + const snapshot = field(data.snapshot); + const observed = { + elements, + text: outline.slice(0, TEXT_EXTRACT_LIMIT), + truncated: + outline.length > TEXT_EXTRACT_LIMIT || + actionable.length > SNAPSHOT_ELEMENT_LIMIT || + snapshot.complete === false || + string(snapshot.continuation) !== undefined, + }; + state.lastRead = { + text: observed.text, + truncated: observed.truncated, + }; + return observed; + }; + + const assertSnapshot = (state: BrowserState, expected?: number) => { + if (expected !== undefined && expected !== state.snapshotId) { + throw new StaleSnapshotError(staleMessage(expected, state.snapshotId)); + } + }; + + const windowTarget = (state: BrowserState) => ({ + kind: "window", + pid: state.pid, + window_id: state.windowId, + }); + + const currentUrl = (state: BrowserState): string => state.url; + + const nativeKey = async ( + state: BrowserState, + key: string, + signal?: AbortSignal, + ) => { + await call( + "press_key", + { + key: cuaKeyName(key), + target: windowTarget(state), + session: state.session, + delivery_mode: "foreground", + }, + signal, + ); + }; + + const captureWindow = async (state: BrowserState) => { + const { result, data } = await call("get_window_state", { + pid: state.pid, + window_id: state.windowId, + session: state.session, + }); + const image = result.images[0]; + if (!image) throw new Error("Cua Driver returned no window screenshot."); + return { + base64: image.dataBase64, + width: number(data.screenshot_width) ?? 0, + height: number(data.screenshot_height) ?? 0, + capturedAt: new Date().toISOString(), + url: currentUrl(state), + }; + }; + + const build = (botId: string): BrowserComputer => { + const computer: BrowserComputer = { + async navigate(url, signal) { + const state = await ensure(botId); + await call( + "browser_navigate", + { + target_id: state.targetId, + tab_id: state.tabId, + url, + session: state.session, + }, + signal, + ); + state.url = url; + const observed = await observe(state); + return { + url: state.url, + title: state.title, + text: observed.text, + truncated: observed.truncated, + }; + }, + + async read() { + const state = await ensure(botId); + await refreshBinding(state); + const observed = state.lastRead ?? (await observe(state)); + return { + url: state.url, + title: state.title, + text: observed.text, + truncated: observed.truncated, + }; + }, + + async screenshot() { + return captureWindow(await ensure(botId)); + }, + + async snapshot(): Promise { + const state = await ensure(botId); + const observed = await observe(state); + return { + snapshotId: state.snapshotId, + url: state.url, + title: state.title, + elements: observed.elements, + truncated: observed.truncated, + }; + }, + + async click(ref, expected, signal): Promise { + const state = await ensure(botId); + assertSnapshot(state, expected); + await call( + "browser_click", + { + target_id: state.targetId, + tab_id: state.tabId, + ref, + input_route: "dom_event", + session: state.session, + }, + signal, + ); + state.lastRead = undefined; + return { action: "click", ref, url: currentUrl(state) }; + }, + + async type(ref, text, submit, expected, signal): Promise { + const state = await ensure(botId); + assertSnapshot(state, expected); + await call( + "browser_type", + { + target_id: state.targetId, + tab_id: state.tabId, + ref, + text: submit ? `${text}\n` : text, + replace: true, + ...(submit ? { mode: "keystrokes" } : {}), + session: state.session, + }, + signal, + ); + state.lastRead = undefined; + return { + action: "type", + ref, + characters: text.length, + submitted: submit, + url: currentUrl(state), + }; + }, + + async key(key, ref, expected, signal): Promise { + const state = await ensure(botId); + assertSnapshot(state, expected); + if (ref) { + const refText = + key === "Enter" ? "\n" : key.length === 1 ? key : null; + if (refText === null) { + throw new Error( + "Cua Driver supports ref-scoped key presses only for Enter and printable characters; omit the ref for native keys such as arrows or Backspace.", + ); + } + await call( + "browser_type", + { + target_id: state.targetId, + tab_id: state.tabId, + ref, + text: refText, + mode: "keystrokes", + session: state.session, + }, + signal, + ); + } else { + await nativeKey(state, key, signal); + } + state.lastRead = undefined; + return { action: "key", key, ref, url: currentUrl(state) }; + }, + + async scroll(deltaY, signal): Promise { + const state = await ensure(botId); + const shot = await captureWindow(state); + await call( + "scroll", + { + x: shot.width / 2, + y: shot.height / 2, + direction: deltaY < 0 ? "up" : "down", + by: "line", + amount: Math.max(1, Math.round(Math.abs(deltaY) / 100)), + target: windowTarget(state), + session: state.session, + delivery_mode: "foreground", + }, + signal, + ); + state.lastRead = undefined; + return { action: "scroll", deltaY, url: currentUrl(state) }; + }, + + async enterSecret(ref, text) { + const state = await ensure(botId); + await call("browser_type", { + target_id: state.targetId, + tab_id: state.tabId, + ref, + text, + replace: true, + session: state.session, + }); + state.lastRead = undefined; + return { characters: text.length, url: currentUrl(state) }; + }, + + async humanClick(x, y): Promise { + const state = await ensure(botId); + const shot = await captureWindow(state); + await call("click", { + x: Math.min(Math.max(x, 0), Math.max(shot.width - 1, 0)), + y: Math.min(Math.max(y, 0), Math.max(shot.height - 1, 0)), + target: windowTarget(state), + session: state.session, + button: "left", + count: 1, + delivery_mode: "foreground", + }); + state.lastRead = undefined; + return { action: "human_click", url: currentUrl(state) }; + }, + + async humanType(text): Promise { + const state = await ensure(botId); + await call("type_text", { + text, + target: windowTarget(state), + session: state.session, + delivery_mode: "foreground", + }); + state.lastRead = undefined; + return { + action: "human_type", + characters: text.length, + url: currentUrl(state), + }; + }, + + async humanKey(key): Promise { + const state = await ensure(botId); + await nativeKey(state, key); + state.lastRead = undefined; + return { action: "human_key", key, url: currentUrl(state) }; + }, + + async humanScroll(deltaY): Promise { + const state = await ensure(botId); + const shot = await captureWindow(state); + await call("scroll", { + x: shot.width / 2, + y: shot.height / 2, + direction: deltaY < 0 ? "up" : "down", + by: "line", + amount: Math.max(1, Math.round(Math.abs(deltaY) / 100)), + target: windowTarget(state), + session: state.session, + delivery_mode: "foreground", + }); + state.lastRead = undefined; + return { action: "human_scroll", deltaY, url: currentUrl(state) }; + }, + + async startStream(onFrame) { + let stopped = false; + let previous = ""; + const loop = async () => { + while (!stopped) { + try { + const shot = await captureWindow(await ensure(botId)); + if (shot.base64 !== previous) { + previous = shot.base64; + onFrame({ + type: "frame", + data: shot.base64, + width: shot.width, + height: shot.height, + mimeType: "image/png", + }); + } + } catch { + // A later iteration can recover after the browser starts or reconnects. + } + await pause(500); + } + }; + void loop(); + const send = async (message: InputMessage) => { + if (message.type === "text") { + await computer.humanType(message.text); + } else if (message.type === "wheel") { + await computer.humanScroll(message.deltaY); + } else if (message.type === "key" && message.event === "down") { + if (message.text) await computer.humanType(message.text); + else await computer.humanKey(message.key); + } else if (message.type === "mouse" && message.event === "released") { + await computer.humanClick(message.x, message.y); + } + }; + return { + async stop() { + stopped = true; + }, + send, + }; + }, + }; + return computer; + }; + + return { + backend: "cua-driver", + computer(botId) { + const existing = computers.get(botId); + if (existing) return existing; + const created = build(botId); + computers.set(botId, created); + return created; + }, + async known() { + const entries = await readdir(root, { withFileTypes: true }).catch( + () => [], + ); + return [ + ...new Set([ + ...entries + .filter( + (entry) => entry.isDirectory() && PROFILE_NAME.test(entry.name), + ) + .map((entry) => entry.name), + ...states.keys(), + ]), + ].sort(); + }, + summary(botIds) { + const known = new Set([...botIds, ...states.keys()]); + return [...known].sort().map((botId) => { + const live = states.get(botId); + return { + botId, + running: Boolean(live), + startedAt: live?.startedAt ?? null, + egress: null, + }; + }); + }, + async stop(botId) { + const state = states.get(botId); + if (!state) return false; + await call("end_session", { session: state.session }).catch( + () => undefined, + ); + states.delete(botId); + computers.delete(botId); + return true; + }, + async reset(botId) { + const state = states.get(botId); + if (state) { + await call("end_session", { session: state.session }).catch( + () => undefined, + ); + } + states.delete(botId); + computers.delete(botId); + if (!PROFILE_NAME.test(botId)) throw new Error("Invalid computer id."); + await rm(join(root, botId), { recursive: true, force: true }); + }, + async closeAll() { + await Promise.all( + [...states.values()].map((state) => + call("end_session", { session: state.session }).catch( + () => undefined, + ), + ), + ); + states.clear(); + await driver.shutdown(); + driver.uniffiDestroy?.(); + }, + }; +} diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.ts index 350fe81..4ee7f2f 100644 --- a/agent-computer/src/index.ts +++ b/agent-computer/src/index.ts @@ -1,7 +1,12 @@ import { serve } from "bun"; -import type { Page } from "playwright"; -import { parseAriaSnapshot, type SnapshotElement } from "./aria-snapshot"; +import { join } from "node:path"; import { isOpenPath, matchesToken, offeredToken } from "./authorisation"; +import { + type BrowserComputer, + type BrowserManager, + type BrowserStream, + StaleSnapshotError, +} from "./browser"; import { type Control, ControlError, @@ -10,13 +15,10 @@ import { NO_SECRET_PENDING, TAKE_CONTROL_FIRST, } from "./control"; +import { createCuaBrowserManager, type CuaDriver } from "./cua-browser"; import { identity } from "./identity"; -import { createProfiles, VIEWPORT } from "./profiles"; -import { - type InputMessage, - type Screencast, - startScreencast, -} from "./screencast"; +import { createPlaywrightBrowserManager } from "./playwright-browser"; +import type { InputMessage } from "./screencast"; import { createWorkspace, WorkspaceFileError, @@ -87,36 +89,13 @@ const ACTION_TIMEOUT_MS = Number.parseInt( 10, ); -/** - * How much page text a navigation hands back. - * - * Bounded because a page can be megabytes and the text goes into a model's context, where a single - * unbounded page can push the rest of the conversation out. Generous enough that the visible part of - * an ordinary page arrives whole, which is what the answer is usually made of. - */ -const TEXT_EXTRACT_LIMIT = 6000; - -/** - * Which snapshot the caller's refs came from. - * - * Kept as a caller-facing guard even though Playwright enforces the real thing underneath. The - * published tool contract says an action carries the `snapshotId` it got, and a mismatch is answered - * with "take a new snapshot", which is a clearer message for a model than an element that merely fails - * to resolve. Playwright's `aria-ref` engine is the runtime enforcement: it resolves a ref only against - * the most recent snapshot, only while the element is still connected to the document, and it mints a - * new ref if an element's role or accessible name changed, so a recycled node cannot inherit an old one. - */ /** Per-Bot browser-control state. Profiles are isolated, but this process is not a security boundary. */ type BotSession = { control: Control; - /** This Bot's snapshot generation. See the note above on staleness. */ - snapshotId: number; /** The one live screen viewer for this Bot, if a person is watching. */ viewer?: { socket: unknown; - cast: Screencast; - /** Stops the loop that keeps the cast pointed at whatever page the Bot is actually on. */ - follow?: ReturnType; + cast: BrowserStream; }; }; @@ -125,7 +104,7 @@ const sessions = new Map(); function sessionFor(botId: string): BotSession { const existing = sessions.get(botId); if (existing) return existing; - const created: BotSession = { control: createControl(), snapshotId: 0 }; + const created: BotSession = { control: createControl() }; sessions.set(botId, created); return created; } @@ -159,13 +138,6 @@ const workspace = createWorkspace(process.env.WORKSPACE_DIR ?? "/workspace"); * The state machine lives in `control.ts` so it can be tested without importing Playwright. */ -/** - * The Bot's browser and the profile that outlives it. See profiles.ts. - * - * `chromium.launch()` gives a fresh anonymous profile every time. Persistent profiles live on a - * mounted volume so sign-in state survives the container. - */ -const profiles = createProfiles(process.env.PROFILES_DIR ?? "/profiles"); // Rooted in the same workspace the file tools use, so a command and a written file see one // directory rather than two. const shell = createShell(process.env.WORKSPACE_DIR ?? "/workspace"); @@ -177,118 +149,35 @@ const shell = createShell(process.env.WORKSPACE_DIR ?? "/workspace"); */ const DEFAULT_BOT_ID = process.env.COMPUTER_BOT_ID ?? "shared"; -async function currentPage(botId: string): Promise { - return profiles.page(botId); -} - -/** - * The page as text, the way a reader sees it. - * - * Script and style bodies are dropped rather than included: they are the bulk of a modern page and - * none of it is what anybody asked about, so leaving them in spends the extract on noise and pushes - * the actual article past the limit. - */ -async function readablePageText( - target: Page, -): Promise<{ text: string; truncated: boolean }> { - const raw = await target.evaluate(() => { - const clone = document.body?.cloneNode(true) as HTMLElement | undefined; - if (!clone) return ""; - for (const node of clone.querySelectorAll("script, style, noscript, svg")) { - node.remove(); - } - return clone.innerText ?? ""; +const profileRoot = process.env.PROFILES_DIR ?? "/profiles"; +const requestedBackend = process.env.COMPUTER_BACKEND?.trim() || "playwright"; +let browsers: BrowserManager; +if (requestedBackend === "cua-driver") { + const cuaProfileRoot = + process.env.CUA_DRIVER_BROWSER_PROFILE_ROOT?.trim() || + join(profileRoot, "cua-driver"); + // The native library reads this variable directly. The package scripts and Docker entrypoint set + // it before Bun starts; assigning it here also keeps Node-compatible hosts and injected SDKs + // aligned with the directory used for list/reset operations. + process.env.CUA_DRIVER_BROWSER_PROFILE_ROOT = cuaProfileRoot; + const [{ CuaDriver: NativeCuaDriver }, { chromium }] = await Promise.all([ + import("@trycua/cua-driver"), + import("playwright"), + ]); + browsers = createCuaBrowserManager( + cuaProfileRoot, + NativeCuaDriver.create(undefined) as CuaDriver, + chromium.executablePath(), + ); +} else if (requestedBackend === "playwright") { + browsers = createPlaywrightBrowserManager(profileRoot, { + actionTimeoutMs: ACTION_TIMEOUT_MS, + navigationTimeoutMs: NAVIGATION_TIMEOUT_MS, }); - - const collapsed = raw.replace(/\n{3,}/g, "\n\n").trim(); - return { - text: collapsed.slice(0, TEXT_EXTRACT_LIMIT), - truncated: collapsed.length > TEXT_EXTRACT_LIMIT, - }; -} - -/** - * Describe everything on the page a Bot can act on. - * - * Uses Playwright's AI snapshot rather than stamping attributes into the DOM. `ariaSnapshot` keeps - * refs outside the page, survives framework re-renders, resolves accessible names, reports current - * values and checked state, filters to actionable elements and descends into iframes. - */ -async function snapshotPage( - session: BotSession, - target: Page, -): Promise<{ - snapshotId: number; - url: string; - title: string; - elements: SnapshotElement[]; - truncated: boolean; -}> { - session.snapshotId += 1; - const yaml = await target.ariaSnapshot({ mode: "ai" }); - return { - snapshotId: session.snapshotId, - url: target.url(), - title: await target.title(), - ...parseAriaSnapshot(yaml), - }; -} - -/** - * Resolve a ref to a locator, refusing anything from a superseded snapshot. - * - * `aria-ref=` is a first-party Playwright selector engine, and it is the same one its MCP server uses. - * The generation check here is the caller-facing half; see the note on `snapshotId` for why both exist. - */ -function locateRef( - session: BotSession, - target: Page, - ref: string, - expectedSnapshotId: number | undefined, -) { - if ( - expectedSnapshotId !== undefined && - expectedSnapshotId !== session.snapshotId - ) { - throw new StaleSnapshotError( - `That list of elements is out of date: it was taken for snapshot ${expectedSnapshotId} and the page is now at ${session.snapshotId}. Take a new snapshot and use the refs from it.`, - ); - } - return target.locator(`aria-ref=${ref}`); -} - -/** - * The element, or a refusal that says what to do about it. - * - * A generation check is not an existence check. A ref from - * the current snapshot that names nothing on the page, because a model invented it or because the - * page moved on without a new snapshot being taken, passes `locateRef` and then simply waits. The - * action times out, and the caller gets a generic failure carrying Playwright's internal call log - * instead of the actionable answer: take a fresh snapshot. - * - * `count()` resolves immediately rather than waiting, so a ref that names nothing is refused in - * milliseconds instead of holding the action open for the full timeout. - */ -async function resolveRef( - session: BotSession, - target: Page, - ref: string, - expectedSnapshotId: number | undefined, -) { - const locator = locateRef(session, target, ref, expectedSnapshotId); - if ((await locator.count()) === 0) { - throw new StaleSnapshotError( - `Nothing on this page has the ref ${ref}. Take a new snapshot and use the refs it returns.`, - ); - } - return locator; -} - -class StaleSnapshotError extends Error { - constructor(message: string) { - super(message); - this.name = "StaleSnapshotError"; - } +} else { + throw new Error( + `COMPUTER_BACKEND must be "playwright" or "cua-driver", got ${JSON.stringify(requestedBackend)}.`, + ); } function json(body: unknown, status = 200): Response { @@ -308,13 +197,9 @@ function json(body: unknown, status = 200): Response { async function stopViewer(session: BotSession): Promise { const current = session.viewer; session.viewer = undefined; - if (current?.follow) clearInterval(current.follow); await current?.cast.stop(); } -/** How often the cast checks that it is still showing the page the Bot is on. */ -const FOLLOW_INTERVAL_MS = 1_000; - /** What a live-screen socket carries: the Bot whose screen it is showing. */ type StreamData = { botId: string }; @@ -339,31 +224,12 @@ serve({ try { ws.send(JSON.stringify(frame)); } catch { - void stopViewer(session); + if (session.viewer?.socket === ws) void stopViewer(session); } }; - /* - * The cast follows the Bot's current page. Re-checking also handles a page being closed - * underneath us without a listener per page. - */ - let casting: Page | undefined; - const attach = async () => { - const target = await currentPage(ws.data.botId); - if (target === casting) return; - const previous = session.viewer; - const cast = await startScreencast(target, send); - casting = target; - session.viewer = { socket: ws, cast, follow: previous?.follow }; - // The old cast stops after the replacement is running, so the screen does not go blank. - await previous?.cast.stop().catch(() => undefined); - }; - - await attach(); - const follow = setInterval(() => { - void attach().catch(() => undefined); - }, FOLLOW_INTERVAL_MS); - if (session.viewer) session.viewer.follow = follow; + const cast = await browsers.computer(ws.data.botId).startStream(send); + session.viewer = { socket: ws, cast }; } catch (error) { ws.send( JSON.stringify({ @@ -377,7 +243,7 @@ serve({ async message(ws, raw) { const session = sessionFor(ws.data.botId); - if (!session.viewer) return; + if (!session.viewer || session.viewer.socket !== ws) return; let message: InputMessage; try { message = JSON.parse(String(raw)) as InputMessage; @@ -415,7 +281,8 @@ serve({ }, async close(ws) { - await stopViewer(sessionFor(ws.data.botId)); + const session = sessionFor(ws.data.botId); + if (session.viewer?.socket === ws) await stopViewer(session); }, }, async fetch(request, server) { @@ -523,26 +390,16 @@ serve({ return json({ error: "A value is required." }, 400); } try { - const target = await currentPage(botId); - // Focus the field the Bot named, and let this throw if it cannot be found. A secret must not - // be reported as delivered unless a field receives it. - // - // No generation check here: a Bot may take another snapshot after asking for a secret, while - // the ref remains protected by Playwright's `aria-ref` rules. - // - // `aria-ref` resolves a ref only against the most recent snapshot, only while the element is - // still connected, and mints a - // new ref when an element's role or accessible name changes, so a recycled node cannot - // inherit an old one. If the ref resolves, it is the field the Bot meant. If it does not, - // nothing is typed, which is the outcome the generation check existed to guarantee. - const field = locateRef(session, target, pending.ref, undefined); - await field.click({ timeout: ACTION_TIMEOUT_MS }); - await field.fill(body.text, { timeout: ACTION_TIMEOUT_MS }); - const characters = body.text.length; + // No generation check here: a Bot may take another snapshot after asking for a secret. Both + // backends still resolve the ref against the current page and fail without typing if the + // target is gone. + const result = await browsers + .computer(botId) + .enterSecret(pending.ref, body.text); // Cleared only after it actually landed, so a failure leaves the request open and the person // can try again rather than being told to start over. session.control.secretSupplied(); - return json({ supplied: true, characters, url: target.url() }); + return json({ supplied: true, ...result }); } catch (error) { if (error instanceof StaleSnapshotError) { return json({ error: error.message, stale: true }, 409); @@ -586,19 +443,25 @@ serve({ unknown > | null; try { - const target = await currentPage(botId); - return json(await performHumanInput(target, url.pathname, body ?? {})); + return json( + await performHumanInput( + browsers.computer(botId), + url.pathname, + body ?? {}, + ), + ); } catch (error) { return json({ error: describe(error, "That did not work.") }, 502); } } if (url.pathname === "/health") { - const [profile] = profiles.summary([botId]); + const [profile] = browsers.summary([botId]); return json({ status: "ok", // `browser` kept as it was: it is in the published contract and start.sh reads it. browser: profile?.running ?? false, + backend: browsers.backend, profile, // Which Bot this computer can prove it is, when the deployment runs SPIRE. Null is a // deployment without it, not a failure, and it is reported rather than omitted so the @@ -613,7 +476,7 @@ serve({ * for it this second. */ if (url.pathname === "/computers" && request.method === "GET") { - return json({ computers: profiles.summary(await profiles.known()) }); + return json({ computers: browsers.summary(await browsers.known()) }); } /** @@ -624,7 +487,7 @@ serve({ * start, so there is no second way for a browser to come into existence. */ if (url.pathname === "/computers/stop" && request.method === "POST") { - const wasRunning = await profiles.stop(botId); + const wasRunning = await browsers.stop(botId); // The wheel goes back to the Bot because the controlled browser no longer exists. session.control.release(); return json({ stopped: true, wasRunning }); @@ -638,7 +501,7 @@ serve({ * to discard a login by mistyping a parameter. */ if (url.pathname === "/computers/reset" && request.method === "POST") { - await profiles.reset(botId); + await browsers.reset(botId); // Reset releases control because any previous browser session and pending secret request are gone. session.control.release(); return json({ reset: true, botId }); @@ -655,21 +518,11 @@ serve({ const startedAt = Date.now(); try { session.control.assertBotMayAct(); - const target = await currentPage(botId); - await target.goto(body.url, { - waitUntil: "domcontentloaded", - timeout: NAVIGATION_TIMEOUT_MS, - }); - // A new document wipes every stamp, so every ref handed out before now is meaningless. - // Bumping the generation makes an action carrying one fail with "take a new snapshot" rather - // than fall through to a selector that matches nothing and read as a missing element. - session.snapshotId += 1; - const extract = await readablePageText(target); + const result = await browsers + .computer(botId) + .navigate(body.url, request.signal); return json({ - url: target.url(), - title: await target.title(), - text: extract.text, - truncated: extract.truncated, + ...result, elapsedMs: Date.now() - startedAt, }); } catch (error) { @@ -691,20 +544,7 @@ serve({ if (url.pathname === "/screenshot" && request.method === "GET") { try { - const target = await currentPage(botId); - const buffer = await target.screenshot({ type: "png" }); - const size = target.viewportSize() ?? { width: 1280, height: 800 }; - return json({ - base64: buffer.toString("base64"), - width: size.width, - height: size.height, - capturedAt: new Date().toISOString(), - // Which page this is a picture of. A browser that has not been sent anywhere sits on - // `about:blank`, and a screenshot of that is a valid, entirely white PNG, indistinguishable - // from a real page to anything looking only at the bytes. The transcript needs to tell - // those apart to avoid presenting a blank browser as though it were a loaded page. - url: target.url(), - }); + return json(await browsers.computer(botId).screenshot()); } catch (error) { return json( { @@ -814,14 +654,7 @@ serve({ // confirmation said. "I clicked the button" is not an answer to what happened. if (url.pathname === "/read" && request.method === "GET") { try { - const target = await currentPage(botId); - const extract = await readablePageText(target); - return json({ - url: target.url(), - title: await target.title(), - text: extract.text, - truncated: extract.truncated, - }); + return json(await browsers.computer(botId).read()); } catch (error) { return json( { error: describe(error, "Reading the page failed.") }, @@ -835,7 +668,7 @@ serve({ // caches and prefetchers eventually punish. if (url.pathname === "/snapshot" && request.method === "POST") { try { - return json(await snapshotPage(session, await currentPage(botId))); + return json(await browsers.computer(botId).snapshot()); } catch (error) { return json({ error: describe(error, "Snapshot failed.") }, 502); } @@ -852,10 +685,8 @@ serve({ const startedAt = Date.now(); try { session.control.assertBotMayAct(); - const target = await currentPage(botId); const detail = await performAction( - session, - target, + browsers.computer(botId), url.pathname, body, // The caller going away is the stop signal: the surface aborts its request, the server @@ -934,7 +765,7 @@ const HUMAN_INPUT = new Set([ * never returned and never logged below. */ async function performHumanInput( - target: Page, + computer: BrowserComputer, action: string, body: Record, ): Promise> { @@ -944,18 +775,12 @@ async function performHumanInput( if (!Number.isFinite(x) || !Number.isFinite(y)) { throw new Error("A click needs an x and a y inside the page."); } - // Clamped rather than rejected. A click a pixel outside the viewport is a rounding artefact of - // scaling the screenshot, not a mistake worth refusing. - return { - x: Math.min(Math.max(x, 0), VIEWPORT.width - 1), - y: Math.min(Math.max(y, 0), VIEWPORT.height - 1), - }; + return { x, y }; }; if (action === "/human/click") { const { x, y } = at(); - await target.mouse.click(x, y); - return { action: "human_click", url: target.url() }; + return computer.humanClick(x, y); } if (action === "/human/type") { @@ -964,34 +789,26 @@ async function performHumanInput( } // `insertText` rather than per-key typing: a person pasting a one-time code should not have it // arrive one character at a time into a field that reformats as you go. - await target.keyboard.insertText(body.text); - // Length only, never the value. See the note above about the model not being on this path. - return { - action: "human_type", - characters: body.text.length, - url: target.url(), - }; + return computer.humanType(body.text); } if (action === "/human/key") { if (typeof body.key !== "string" || !body.key) { throw new Error("A key press needs a key name."); } - await target.keyboard.press(body.key); - return { action: "human_key", key: body.key, url: target.url() }; + return computer.humanKey(body.key); } const deltaY = typeof body.deltaY === "number" ? body.deltaY : 400; - await target.mouse.wheel(0, deltaY); - return { action: "human_scroll", deltaY, url: target.url() }; + return computer.humanScroll(deltaY); } /** * Carry out one action on the page. * - * Every action that addresses an element goes through {@link locateRef}, so the staleness check - * cannot be forgotten at a call site. `/key` and `/scroll` may omit a ref and act on the page itself, - * which is how a Bot presses Enter to submit or scrolls to bring more of a long form into view. + * Every action that addresses an element goes through the selected backend's ref resolver, so the + * staleness check cannot be forgotten at a call site. `/key` and `/scroll` may omit a ref and act on + * the page itself, which is how a Bot presses Enter or scrolls through a long form. * * Stop has to reach the browser. `signal` is the caller's request going away, the person pressed * Stop, and the abort travels from the surface, through the server, to here. Without passing it on, @@ -999,22 +816,18 @@ async function performHumanInput( * landing on a live page. Stop must reach the browser before a high-impact click lands. */ async function performAction( - session: BotSession, - target: Page, + computer: BrowserComputer, action: string, body: ActionBody, signal?: AbortSignal, ): Promise> { - // Passed to every Playwright call below. It does not disable the timeout, which still applies. - const acting = { timeout: ACTION_TIMEOUT_MS, ...(signal ? { signal } : {}) }; const expected = typeof body.snapshotId === "number" ? body.snapshotId : undefined; const ref = typeof body.ref === "string" && body.ref ? body.ref : undefined; if (action === "/click") { if (!ref) throw new Error("A click needs the ref of an element to click."); - await (await resolveRef(session, target, ref, expected)).click(acting); - return { action: "click", ref, url: target.url() }; + return computer.click(ref, expected, signal); } if (action === "/type") { @@ -1022,46 +835,26 @@ async function performAction( if (typeof body.text !== "string") { throw new Error("Typing needs the text to enter."); } - const field = await resolveRef(session, target, ref, expected); - // `fill` rather than keystrokes: it clears the field first, which is what "put this value in - // this box" means. Typing into a field a previous attempt half-filled otherwise appends, and the - // form ends up with "AlicAlice" in it. - await field.fill(body.text, acting); - if (body.submit === true) { - await field.press("Enter", acting); - } - // The text itself is deliberately NOT returned. It is echoed nowhere: this response is read by - // the model and logged by the server, and a value typed into a form is exactly where a password - // or a card number lives. The caller already knows what it sent. - return { - action: "type", + return computer.type( ref, - characters: body.text.length, - submitted: body.submit === true, - url: target.url(), - }; + body.text, + body.submit === true, + expected, + signal, + ); } if (action === "/key") { if (typeof body.key !== "string" || !body.key) { throw new Error("A key press needs a key name, such as Enter or Tab."); } - if (ref) { - await (await resolveRef(session, target, ref, expected)).press( - body.key, - acting, - ); - } else { - await target.keyboard.press(body.key); - } - return { action: "key", key: body.key, ref, url: target.url() }; + return computer.key(body.key, ref, expected, signal); } // Scroll. A plain wheel event on the page, which is what moves a long form, rather than scrolling a // specific element into view: the Bot asked to see further down, not to hunt for one control. const deltaY = typeof body.deltaY === "number" ? body.deltaY : 600; - await target.mouse.wheel(0, deltaY); - return { action: "scroll", deltaY, url: target.url() }; + return computer.scroll(deltaY, signal); } function describe(error: unknown, fallback: string): string { @@ -1082,7 +875,9 @@ function fileStatus(error: unknown): 400 | 403 | 500 { return 500; } -console.info(`agent-computer listening on http://localhost:${PORT}`); +console.info( + `agent-computer listening on http://localhost:${PORT} with ${browsers.backend}`, +); /** * Hand the profile back before dying. @@ -1097,7 +892,7 @@ for (const signal of ["SIGTERM", "SIGINT"] as const) { process.on(signal, () => { void (async () => { console.info(`${signal}: closing the browser so its profile is flushed`); - await profiles.closeAll(); + await browsers.closeAll(); process.exit(0); })(); }); diff --git a/agent-computer/src/playwright-browser.ts b/agent-computer/src/playwright-browser.ts new file mode 100644 index 0000000..f2572bb --- /dev/null +++ b/agent-computer/src/playwright-browser.ts @@ -0,0 +1,282 @@ +import type { Page } from "playwright"; +import { parseAriaSnapshot } from "./aria-snapshot"; +import { + type BrowserAction, + type BrowserComputer, + type BrowserManager, + type BrowserSnapshot, + type HumanAction, + StaleSnapshotError, +} from "./browser"; +import { createProfiles, VIEWPORT } from "./profiles"; +import { startScreencast } from "./screencast"; + +const TEXT_EXTRACT_LIMIT = 6000; +const FOLLOW_INTERVAL_MS = 1_000; + +async function readablePageText(target: Page) { + const raw = await target.evaluate(() => { + const clone = document.body?.cloneNode(true) as HTMLElement | undefined; + if (!clone) return ""; + for (const node of clone.querySelectorAll("script, style, noscript, svg")) { + node.remove(); + } + return clone.innerText ?? ""; + }); + const collapsed = raw.replace(/\n{3,}/g, "\n\n").trim(); + return { + text: collapsed.slice(0, TEXT_EXTRACT_LIMIT), + truncated: collapsed.length > TEXT_EXTRACT_LIMIT, + }; +} + +function at(body: Record): { x: number; y: number } { + const x = typeof body.x === "number" ? body.x : Number.NaN; + const y = typeof body.y === "number" ? body.y : Number.NaN; + if (!Number.isFinite(x) || !Number.isFinite(y)) { + throw new Error("A click needs an x and a y inside the page."); + } + return { + x: Math.min(Math.max(x, 0), VIEWPORT.width - 1), + y: Math.min(Math.max(y, 0), VIEWPORT.height - 1), + }; +} + +export function createPlaywrightBrowserManager( + root: string, + options: { actionTimeoutMs: number; navigationTimeoutMs: number }, +): BrowserManager { + const profiles = createProfiles(root); + const computers = new Map(); + + const build = (botId: string): BrowserComputer => { + let snapshotId = 0; + const currentPage = (): Promise => profiles.page(botId); + + const locateRef = ( + target: Page, + ref: string, + expectedSnapshotId: number | undefined, + ) => { + if ( + expectedSnapshotId !== undefined && + expectedSnapshotId !== snapshotId + ) { + throw new StaleSnapshotError( + `That list of elements is out of date: it was taken for snapshot ${expectedSnapshotId} and the page is now at ${snapshotId}. Take a new snapshot and use the refs from it.`, + ); + } + return target.locator(`aria-ref=${ref}`); + }; + + const resolveRef = async ( + target: Page, + ref: string, + expectedSnapshotId: number | undefined, + ) => { + const locator = locateRef(target, ref, expectedSnapshotId); + if ((await locator.count()) === 0) { + throw new StaleSnapshotError( + `Nothing on this page has the ref ${ref}. Take a new snapshot and use the refs it returns.`, + ); + } + return locator; + }; + + return { + async navigate(destination) { + const target = await currentPage(); + await target.goto(destination, { + waitUntil: "domcontentloaded", + timeout: options.navigationTimeoutMs, + }); + snapshotId += 1; + const extract = await readablePageText(target); + return { + url: target.url(), + title: await target.title(), + ...extract, + }; + }, + + async read() { + const target = await currentPage(); + return { + url: target.url(), + title: await target.title(), + ...(await readablePageText(target)), + }; + }, + + async screenshot() { + const target = await currentPage(); + const buffer = await target.screenshot({ type: "png" }); + const size = target.viewportSize() ?? VIEWPORT; + return { + base64: buffer.toString("base64"), + width: size.width, + height: size.height, + capturedAt: new Date().toISOString(), + url: target.url(), + }; + }, + + async snapshot(): Promise { + const target = await currentPage(); + snapshotId += 1; + const yaml = await target.ariaSnapshot({ mode: "ai" }); + return { + snapshotId, + url: target.url(), + title: await target.title(), + ...parseAriaSnapshot(yaml), + }; + }, + + async click(ref, expected, signal): Promise { + const target = await currentPage(); + await (await resolveRef(target, ref, expected)).click({ + timeout: options.actionTimeoutMs, + ...(signal ? { signal } : {}), + }); + return { action: "click", ref, url: target.url() }; + }, + + async type(ref, text, submit, expected, signal): Promise { + const target = await currentPage(); + const field = await resolveRef(target, ref, expected); + const acting = { + timeout: options.actionTimeoutMs, + ...(signal ? { signal } : {}), + }; + await field.fill(text, acting); + if (submit) await field.press("Enter", acting); + return { + action: "type", + ref, + characters: text.length, + submitted: submit, + url: target.url(), + }; + }, + + async key(key, ref, expected, signal): Promise { + const target = await currentPage(); + if (ref) { + await (await resolveRef(target, ref, expected)).press(key, { + timeout: options.actionTimeoutMs, + ...(signal ? { signal } : {}), + }); + } else { + await target.keyboard.press(key); + } + return { action: "key", key, ref, url: target.url() }; + }, + + async scroll(deltaY): Promise { + const target = await currentPage(); + await target.mouse.wheel(0, deltaY); + return { action: "scroll", deltaY, url: target.url() }; + }, + + async enterSecret(ref, text) { + const target = await currentPage(); + const field = locateRef(target, ref, undefined); + await field.click({ timeout: options.actionTimeoutMs }); + await field.fill(text, { timeout: options.actionTimeoutMs }); + return { characters: text.length, url: target.url() }; + }, + + async humanClick(x, y): Promise { + const target = await currentPage(); + const point = at({ x, y }); + await target.mouse.click(point.x, point.y); + return { action: "human_click", url: target.url() }; + }, + + async humanType(text): Promise { + const target = await currentPage(); + await target.keyboard.insertText(text); + return { + action: "human_type", + characters: text.length, + url: target.url(), + }; + }, + + async humanKey(key): Promise { + const target = await currentPage(); + await target.keyboard.press(key); + return { action: "human_key", key, url: target.url() }; + }, + + async humanScroll(deltaY): Promise { + const target = await currentPage(); + await target.mouse.wheel(0, deltaY); + return { action: "human_scroll", deltaY, url: target.url() }; + }, + + async startStream(onFrame) { + let stopped = false; + let attaching = false; + let casting = await currentPage(); + let cast = await startScreencast(casting, onFrame); + const follow = setInterval(() => { + if (stopped || attaching) return; + attaching = true; + void (async () => { + try { + const target = await currentPage(); + if (target === casting) return; + const replacement = await startScreencast(target, onFrame); + if (stopped) { + await replacement.stop().catch(() => undefined); + return; + } + const previous = cast; + casting = target; + cast = replacement; + await previous.stop().catch(() => undefined); + } finally { + attaching = false; + } + })().catch(() => undefined); + }, FOLLOW_INTERVAL_MS); + return { + async stop() { + if (stopped) return; + stopped = true; + clearInterval(follow); + await cast.stop(); + }, + send(message) { + return cast.send(message); + }, + }; + }, + }; + }; + + return { + backend: "playwright", + computer(botId) { + const existing = computers.get(botId); + if (existing) return existing; + const created = build(botId); + computers.set(botId, created); + return created; + }, + known: profiles.known, + summary: profiles.summary, + async stop(botId) { + const stopped = await profiles.stop(botId); + computers.delete(botId); + return stopped; + }, + async reset(botId) { + await profiles.reset(botId); + computers.delete(botId); + }, + closeAll: profiles.closeAll, + }; +} diff --git a/agent-computer/tests/cua-browser.test.ts b/agent-computer/tests/cua-browser.test.ts new file mode 100644 index 0000000..1a62dd2 --- /dev/null +++ b/agent-computer/tests/cua-browser.test.ts @@ -0,0 +1,300 @@ +import { describe, expect, test } from "bun:test"; +import { StaleSnapshotError } from "../src/browser"; +import { createCuaBrowserManager, type CuaDriver } from "../src/cua-browser"; + +type Call = { name: string; args: Record }; + +function response( + data: Record = {}, + images: { mimeType: string; dataBase64: string }[] = [], +) { + return { + text: "ok", + images, + structuredJson: JSON.stringify(data), + isError: false, + rawJson: "{}", + }; +} + +function refusal(code: string, text: string) { + return { + text, + images: [], + structuredJson: JSON.stringify({ refusal: { code } }), + isError: true, + errorCode: code, + rawJson: "{}", + }; +} + +function fixture( + override?: ( + call: Call, + ) => ReturnType | ReturnType | undefined, +) { + const calls: Call[] = []; + let shutdown = false; + let currentUrl = "about:blank"; + let currentTitle = ""; + const driver: CuaDriver = { + async callTool(name, argumentsJson) { + const args = JSON.parse(argumentsJson) as Record; + calls.push({ name, args }); + const overridden = override?.({ name, args }); + if (overridden) return overridden; + if (name === "browser_navigate") { + currentUrl = new URL(String(args.url)).href; + currentTitle = "Example Domain"; + return response(); + } + if (name === "browser_prepare") return response({ prepared_pid: 222 }); + if (name === "list_windows") { + return response({ windows: [{ window_id: 333, z_index: 1 }] }); + } + if (name === "get_browser_state" && "pid" in args) { + return response({ + binding_quality: "exact", + mutation_allowed: true, + target_id: "target-1", + tabs: [ + { + tab_id: "tab-1", + active: true, + url: currentUrl, + title: currentTitle, + }, + ], + }); + } + if (name === "get_browser_state") { + currentUrl = "https://example.com/"; + currentTitle = "Example Domain"; + return response({ + page: { url: currentUrl, title: currentTitle }, + outline: "Example Domain\nThis domain is for examples.", + snapshot: { complete: true }, + refs: [ + { + ref: "p1:1", + role: "link", + name: "More information", + actions: ["click"], + states: {}, + }, + { + ref: "p1:2", + role: "checkbox", + name: "Remember me", + value: "yes", + actions: ["click"], + states: { disabled: false }, + }, + { + ref: "p1:3", + role: "heading", + name: "Example Domain", + actions: [], + states: {}, + }, + ], + }); + } + if (name === "get_window_state") { + return response({ screenshot_width: 900, screenshot_height: 640 }, [ + { mimeType: "image/png", dataBase64: "cG5n" }, + ]); + } + return response(); + }, + async shutdown() { + shutdown = true; + }, + }; + const manager = createCuaBrowserManager( + "/tmp/openbot-cua-test", + driver, + "/chrome", + { + async seedPid() { + return { pid: 111, close: async () => undefined }; + }, + sleep: async () => undefined, + }, + ); + return { calls, manager, shutdown: () => shutdown }; +} + +describe("Cua Driver browser backend", () => { + test("prepares a durable isolated profile and maps semantic refs", async () => { + const { calls, manager } = fixture(); + const computer = manager.computer("sales-bot"); + + const page = await computer.navigate("https://example.com"); + expect(page).toMatchObject({ + url: "https://example.com/", + title: "Example Domain", + truncated: false, + }); + const snapshot = await computer.snapshot(); + expect(snapshot.elements).toEqual([ + { + ref: "p1:1", + role: "link", + name: "More information", + }, + { + ref: "p1:2", + role: "checkbox", + name: "Remember me", + value: "yes", + disabled: false, + }, + ]); + + expect( + calls.find((call) => call.name === "browser_prepare")?.args, + ).toMatchObject({ + pid: 111, + allow_launch: true, + profile: { mode: "isolated_named", name: "sales-bot" }, + session: "openbot-sales-bot", + }); + }); + + test("uses current refs and an explicit synthetic background click", async () => { + const { calls, manager } = fixture(); + const computer = manager.computer("sales-bot"); + const snapshot = await computer.snapshot(); + + await expect( + computer.click("p1:1", snapshot.snapshotId), + ).resolves.toMatchObject({ + action: "click", + ref: "p1:1", + }); + expect( + calls.find((call) => call.name === "browser_click")?.args, + ).toMatchObject({ + target_id: "target-1", + tab_id: "tab-1", + ref: "p1:1", + input_route: "dom_event", + }); + await expect( + computer.click("p1:1", snapshot.snapshotId - 1), + ).rejects.toBeInstanceOf(StaleSnapshotError); + }); + + test("reading a just-snapshotted page preserves its refs", async () => { + const { calls, manager } = fixture(); + const computer = manager.computer("sales-bot"); + const snapshot = await computer.snapshot(); + const semanticReads = () => + calls.filter( + (call) => + call.name === "get_browser_state" && + call.args.snapshot_format === "semantic_v2", + ).length; + + expect(semanticReads()).toBe(1); + await expect(computer.read()).resolves.toMatchObject({ + text: "Example Domain\nThis domain is for examples.", + }); + expect(semanticReads()).toBe(1); + await expect( + computer.click("p1:1", snapshot.snapshotId), + ).resolves.toMatchObject({ action: "click" }); + }); + + test("translates browser key names to Cua Driver key names", async () => { + const { calls, manager } = fixture(); + const computer = manager.computer("sales-bot"); + const snapshot = await computer.snapshot(); + + await computer.type("p1:2", "yes", true, snapshot.snapshotId); + await computer.key("ArrowDown"); + await computer.humanKey("Backspace"); + + expect( + calls.find( + (call) => call.name === "browser_type" && call.args.text === "yes\n", + )?.args, + ).toMatchObject({ mode: "keystrokes", replace: true }); + expect( + calls + .filter((call) => call.name === "press_key") + .map((call) => call.args.key), + ).toEqual(["down", "backspace"]); + expect( + calls + .filter((call) => call.name === "press_key") + .every((call) => call.args.delivery_mode === "foreground"), + ).toBe(true); + }); + + test("uses browser keystrokes for ref-scoped Enter and refuses ambiguous native keys", async () => { + const { calls, manager } = fixture(); + const computer = manager.computer("sales-bot"); + const snapshot = await computer.snapshot(); + + await computer.key("Enter", "p1:2", snapshot.snapshotId); + expect( + calls.find( + (call) => call.name === "browser_type" && call.args.text === "\n", + )?.args, + ).toMatchObject({ mode: "keystrokes", ref: "p1:2" }); + await expect( + computer.key("Backspace", "p1:2", snapshot.snapshotId), + ).rejects.toThrow("omit the ref"); + }); + + test("only maps the driver's exact stale codes to stale snapshots", async () => { + const wrongTarget = fixture(({ name }) => + name === "browser_click" + ? refusal("browser_wrong_target_refused", "wrong target refused") + : undefined, + ); + const wrongTargetComputer = wrongTarget.manager.computer("sales-bot"); + const wrongTargetSnapshot = await wrongTargetComputer.snapshot(); + const wrongTargetError = await wrongTargetComputer + .click("p1:1", wrongTargetSnapshot.snapshotId) + .catch((error) => error); + expect(wrongTargetError).toBeInstanceOf(Error); + expect(wrongTargetError).not.toBeInstanceOf(StaleSnapshotError); + + const stale = fixture(({ name }) => + name === "browser_click" + ? refusal("browser_ref_stale", "browser_ref_stale") + : undefined, + ); + const staleComputer = stale.manager.computer("sales-bot"); + const staleSnapshot = await staleComputer.snapshot(); + await expect( + staleComputer.click("p1:1", staleSnapshot.snapshotId), + ).rejects.toBeInstanceOf(StaleSnapshotError); + }); + + test("returns native screenshots for takeover and closes sessions", async () => { + const { calls, manager, shutdown } = fixture(); + const computer = manager.computer("sales-bot"); + + await expect(computer.screenshot()).resolves.toMatchObject({ + base64: "cG5n", + width: 900, + height: 640, + url: "about:blank", + }); + expect(await manager.stop("sales-bot")).toBe(true); + expect(calls.some((call) => call.name === "end_session")).toBe(true); + await manager.closeAll(); + expect(shutdown()).toBe(true); + }); + + test("rejects profile names that cannot be confined", async () => { + const { manager } = fixture(); + await expect(manager.computer("../other-bot").read()).rejects.toThrow( + "only letters, digits, hyphen and underscore", + ); + }); +}); diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index bbc9893..dc42f4e 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -69,6 +69,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { data?: string; width?: number; height?: number; + mimeType?: string; error?: string; }; try { @@ -102,7 +103,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { c.charCodeAt(0), ); const bitmap = await createImageBitmap( - new Blob([binary], { type: "image/jpeg" }), + new Blob([binary], { type: message.mimeType ?? "image/jpeg" }), ); if (closed) { bitmap.close(); diff --git a/docker-compose.yml b/docker-compose.yml index 9f38be9..1b6e98a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,7 @@ services: environment: # The secret every caller must present. The container refuses to start without it. COMPUTER_TOKEN: ${COMPUTER_TOKEN:-} + COMPUTER_BACKEND: ${COMPUTER_BACKEND:-playwright} volumes: - agent-workspace:/workspace # Chromium's user-data directory is volume-backed so logins survive container restarts. @@ -122,6 +123,7 @@ services: SUPERVISOR_TOKEN: ${SUPERVISOR_TOKEN:-} # Handed to every computer this creates, so the server and the computers share one secret. COMPUTER_TOKEN: ${COMPUTER_TOKEN:-} + COMPUTER_BACKEND: ${COMPUTER_BACKEND:-playwright} COMPUTER_IMAGE: ${COMPUTER_IMAGE:-openbot-agent-computer:latest} # Which deployment the computers it creates belong to, so two stacks on one Docker host never # derive the same container and volume names for the same Bot. diff --git a/docker/s6/s6-rc.d/computer/run b/docker/s6/s6-rc.d/computer/run index 77989bd..50e0c4c 100755 --- a/docker/s6/s6-rc.d/computer/run +++ b/docker/s6/s6-rc.d/computer/run @@ -6,4 +6,57 @@ cd /app/agent-computer export PORT=4100 export WORKSPACE_DIR=/workspace -exec s6-setuidgid pwuser /usr/local/bin/bun src/index.ts + +if [ "${COMPUTER_BACKEND:-playwright}" != "cua-driver" ]; then + exec s6-setuidgid pwuser /usr/local/bin/bun src/index.ts +fi + +# Cua Driver's foreground keyboard and takeover paths need a window manager. Keep the display and +# browser under the same unprivileged account, and stop every child if s6 asks this service to exit. +exec s6-setuidgid pwuser sh -c ' + display_pid="" + openbox_pid="" + computer_pid="" + + stop_display() { + [ -z "$openbox_pid" ] || kill "$openbox_pid" 2>/dev/null || true + [ -z "$display_pid" ] || kill "$display_pid" 2>/dev/null || true + [ -z "$openbox_pid" ] || wait "$openbox_pid" 2>/dev/null || true + [ -z "$display_pid" ] || wait "$display_pid" 2>/dev/null || true + } + terminate() { + trap "" HUP INT TERM + if [ -n "$computer_pid" ]; then + kill "$computer_pid" 2>/dev/null || true + wait "$computer_pid" 2>/dev/null || true + computer_pid="" + fi + stop_display + exit 143 + } + trap terminate HUP INT TERM + + export HOME=/home/pwuser + export USER=pwuser + export LOGNAME=pwuser + export DISPLAY=:99 + export CUA_DRIVER_BROWSER_PROFILE_ROOT="${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver}" + if [ "${COMPUTER_SANDBOX:-off}" != "on" ]; then + export CUA_E2E_BROWSER_NO_SANDBOX=1 + fi + Xvfb :99 -screen 0 1280x1024x24 -nolisten tcp & + display_pid=$! + until [ -S /tmp/.X11-unix/X99 ]; do + kill -0 "$display_pid" 2>/dev/null || exit 1 + sleep 0.05 + done + openbox --sm-disable & + openbox_pid=$! + /usr/local/bin/bun src/index.ts & + computer_pid=$! + wait "$computer_pid" + computer_status=$? + computer_pid="" + stop_display + exit "$computer_status" +' diff --git a/docs/architecture.md b/docs/architecture.md index 90b52c1..ba6b1d2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,6 +69,13 @@ startup. `agent-computer` requires `COMPUTER_TOKEN` and permits only `/health` without it. Docker Compose binds it to `127.0.0.1:4100`. +The computer exposes one backend-neutral HTTP and WebSocket contract. Playwright remains the +default. With `COMPUTER_BACKEND=cua-driver`, the same gateway uses Cua Driver's in-process native SDK +for isolated persistent browser profiles, semantic page refs, browser actions, screenshots, and +human takeover. Its profiles live in a separate `cua-driver` subdirectory, so changing backends does +not reinterpret or overwrite an existing Playwright profile. Policy and audit stay in the server in +front of either backend. + With `COMPUTER_SUPERVISOR_URL`, each Bot gets its own computer container, workspace volume, and browser profile. Without it, all Bots share `AGENT_COMPUTER_URL`. The supervisor exposes only ensure, stop, reset, and list operations. It holds the Docker socket, so do not expose it outside the deployment network. Set `COMPUTER_RUNTIME=runsc` to run computers under gVisor on hosts that support it. diff --git a/docs/configuration.md b/docs/configuration.md index ad3cf63..08650fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -97,6 +97,7 @@ Google OAuth client id and secret must be configured together. If Google OAuth i | ------------------------------------ | ----------------------------------------------------------------------------------------- | | `AGENT_COMPUTER_URL` | Shared computer URL. If absent, computer routes are not mounted. | | `COMPUTER_TOKEN` | Secret every computer request must present. The computer refuses to start without it. | +| `COMPUTER_BACKEND` | Browser backend: `playwright` (default) or `cua-driver`. | | `COMPUTER_SUPERVISOR_URL` | Supervisor URL for per-Bot computers. If absent, Bots share `AGENT_COMPUTER_URL`. | | `SUPERVISOR_TOKEN` | Bearer token required by the supervisor. | | `AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS` | Local-only private-host browsing when `true`. Cloud metadata addresses are still refused. | @@ -113,6 +114,15 @@ Google OAuth client id and secret must be configured together. If Google OAuth i - `EGRESS_PROXY_DEFAULT` - `EGRESS_PROXY_` +The Cua Driver backend stores reusable isolated profiles under `PROFILES_DIR/cua-driver`, separate +from existing Playwright profiles, and preserves the same HTTP, policy, audit, takeover, and per-Bot +container boundaries. Its Linux click route is an explicit synthetic DOM event so it can remain +background-safe; controls that require a trusted browser gesture may refuse it. Per-Bot egress proxy +variables currently apply only to Playwright. Native takeover input briefly foregrounds the browser +inside its private computer container, where no other Bot or desktop session shares the display. +Ref-scoped Cua key actions support Enter and printable characters; arrows and editing keys use the +native page-level route without a ref. + The supervisor also reads: - `COMPUTER_IMAGE` diff --git a/docs/deployment.md b/docs/deployment.md index 47da5cf..9879ebf 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,7 +18,9 @@ docker run -p 3001:3001 --env-file .env \ **In it:** the built app, the API, and Chromium. One port, 3001. The browser listens on 4100 inside the container and is deliberately not published: it holds real logins and its only caller is the -process beside it. +process beside it. Playwright remains the default browser backend. Set +`COMPUTER_BACKEND=cua-driver` to use Cua Driver; the image starts its private X11 display and window +manager only for that backend so native takeover input never reaches a host desktop session. **PostgreSQL, if you ask for it.** `EMBEDDED_POSTGRES=on` starts one inside the container, creates the database and the `vector` extension the first time, and runs the migrations on every start. It diff --git a/supervisor/src/index.ts b/supervisor/src/index.ts index 78575a0..989da10 100644 --- a/supervisor/src/index.ts +++ b/supervisor/src/index.ts @@ -68,6 +68,7 @@ function environmentFor(botId: string): string[] { * to set. */ const computerToken = process.env.COMPUTER_TOKEN; + const computerBackend = process.env.COMPUTER_BACKEND; return [ // Which Bot this container is. Read by the computer as the Bot to assume when a request does not // name one. It is normally named per request, so this is the fallback, and for a container that @@ -75,6 +76,7 @@ function environmentFor(botId: string): string[] { `COMPUTER_BOT_ID=${botId}`, // Without this the computer refuses to start; it must never answer an unauthenticated caller. ...(computerToken ? [`COMPUTER_TOKEN=${computerToken}`] : []), + ...(computerBackend ? [`COMPUTER_BACKEND=${computerBackend}`] : []), // Where to ask what it is. Absent, the computer reports no identity and carries on. ...(spireSocketVolume ? ["SPIFFE_ENDPOINT_SOCKET=/tmp/spire-agent/public/api.sock"]