From 929046d93c1816c095dc2daa6b281ed0d437c5c3 Mon Sep 17 00:00:00 2001 From: dbarr5 Date: Wed, 19 Aug 2026 11:23:06 -0400 Subject: [PATCH] =?UTF-8?q?test(brain):=20canary=207=20=E2=80=94=20local/O?= =?UTF-8?q?llama=20parity,=20and=20the=20seam=20that=20makes=20it=20possib?= =?UTF-8?q?le?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last of the seven release canaries, and the one that stayed unwritable longest. LocalBrain spawned a Python module with no injectable transport, so nothing in this repo could drive it: a search for `new LocalBrain` across test/ returned nothing, because it was not constructible under test. Adds a BrainSpawner seam. LocalBrain already speaks a newline-delimited JSON protocol over stdio, so a fake that speaks the same protocol is indistinguishable from the real child. Default behaviour is unchanged — the option simply defaults to node's spawn. What parity means here is deliberately NOT "identical events". Ollama is a pure-TypeScript loop; the Python path has a far richer vocabulary (stage, skill, telemetry, checkpoint). Requiring identical output would be a false contract. It means a host driving either brain sees the same SHAPE: the same tool calls in the same order, each carrying an id it can reply to, terminated by exactly one done. A host loop must not need to know which brain it is talking to. The test normalizes both transcripts to that view and requires them equal. Four assertions: both brains present the same shape to a host loop every tool_call from either brain carries a replyable id each brain terminates with exactly one done a brain that dies without a done still ends its stream The last one guards a hang rather than a wrong value: a host awaiting a terminal event that never arrives waits forever. Worth recording, because it nearly produced a false finding: the first run of that fourth test hung for ten minutes. The cause was the fake, not the product. LocalBrain ends its stream on the child's close event, and the fake's kill() closed stdout without emitting close — something a real child always does. A fake that is unfaithful about its termination signal turns a correct product into a hanging test. Fixed in the fake; the product path was right. Gates at this commit: npm run typecheck exit 0 npm test 1117 pass / 0 fail All seven canaries now run on every build: 1 denied mutation test/release_canaries.test.ts 2 live-child cancel test/process_tree.test.ts 3 reconnect replay test/release_canaries.test.ts 4 remote freshness test/worktree.test.ts 5 fake-gh ship test/ship_rail.test.ts 6 cap across reconnect test/release_canaries.test.ts 7 brain parity HERE --- src/core/brain_local.ts | 24 +++++- test/brain_parity.test.ts | 173 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 test/brain_parity.test.ts diff --git a/src/core/brain_local.ts b/src/core/brain_local.ts index c42346b..571d0e7 100644 --- a/src/core/brain_local.ts +++ b/src/core/brain_local.ts @@ -20,7 +20,22 @@ import { } from "./brain_protocol.js"; import type { ToolResult } from "./tool_executor.js"; +/** + * How the brain subprocess is started. Injected so the local path is testable + * without a Python installation: LocalBrain speaks a line protocol over stdio, + * and a fake that speaks the same protocol is indistinguishable from the real + * child. Without this seam nothing could drive LocalBrain at all, which is why + * the local-vs-Ollama parity canary could not be written. + */ +export type BrainSpawner = ( + command: string, + args: readonly string[], + options: { cwd: string; env: NodeJS.ProcessEnv }, +) => ChildProcessWithoutNullStreams; + export interface LocalBrainOptions { + /** Override how the child is started. Defaults to node's spawn. */ + spawn?: BrainSpawner; /** Python interpreter (default: $AETHER_PYTHON or "python"). */ python?: string; /** Module to run as the brain (default: aether_agent.headless). */ @@ -44,13 +59,16 @@ export class LocalBrain implements Brain { // fall back to "python", or spawn() throws "argument 'file' cannot be empty". const python = this.opts.python || process.env["AETHER_PYTHON"] || "python"; const mod = this.opts.module ?? "aether_agent.headless"; - const child = spawn(python, ["-m", mod], { + const start: BrainSpawner = + this.opts.spawn ?? + ((command, args, options) => + spawn(command, [...args], { ...options, stdio: ["pipe", "pipe", "pipe"] }) as ChildProcessWithoutNullStreams); + const child = start(python, ["-m", mod], { cwd: task.cwd, // PYTHONUTF8 belt-and-suspenders alongside the ASCII-escaped wire: the // child's stdio never falls back to cp1252 on Windows. env: { ...process.env, PYTHONUTF8: "1", ...(this.opts.env ?? {}) }, - stdio: ["pipe", "pipe", "pipe"], - }) as ChildProcessWithoutNullStreams; + }); this.child = child; // A dead child's stdin raises EPIPE asynchronously; without a listener diff --git a/test/brain_parity.test.ts b/test/brain_parity.test.ts new file mode 100644 index 0000000..87d204e --- /dev/null +++ b/test/brain_parity.test.ts @@ -0,0 +1,173 @@ +// Canary 7 — local/Ollama brain parity. +// +// The last of the seven, and the one that stayed unwritable longest: LocalBrain +// spawned a Python module with no injectable transport, so no test could drive +// it. With a spawner seam it becomes a fake child speaking the same line +// protocol, and the two local brains can finally be compared. +// +// What parity means here is NOT that they emit identical events — Ollama is a +// pure-TypeScript loop and the Python path has a much richer vocabulary. It +// means a host driving either one sees the same SHAPE: the same tool calls in +// the same order, each carrying an id it can reply to, terminated by exactly +// one done. A host loop must not need to know which brain it is talking to. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { LocalBrain } from "../src/core/brain_local.js"; +import { OllamaBrain } from "../src/core/brain_ollama.js"; +import type { Brain, TaskCommand } from "../src/core/brain.js"; +import type { BrainEvent } from "../src/core/brain_protocol.js"; +import type { ChatMessage, ChatReply } from "../src/core/ollama.js"; + +const task: TaskCommand = { type: "task", text: "fix the bug", cwd: process.cwd(), poolGb: 5 }; + +/** + * A fake brain subprocess: reads host commands off stdin and answers on stdout + * in the same newline-delimited JSON the Python brain speaks, so LocalBrain + * cannot tell it apart from the real child. + */ +function scriptedChild(script: (line: string, say: (event: unknown) => void) => void) { + const child = new EventEmitter() as EventEmitter & { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + killed: boolean; + kill(): void; + }; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.killed = false; + child.kill = (): void => { + child.killed = true; + child.stdout.end(); + // A real child emits close when it exits, and LocalBrain ends its stream on + // that. A fake that skips it hangs the consumer — which is the bug this + // test exists to catch, so the fake has to be faithful about it. + queueMicrotask(() => child.emit("close", 0, null)); + }; + + const say = (event: unknown): void => { + child.stdout.write(JSON.stringify(event) + "\n"); + }; + + let buffered = ""; + child.stdin.on("data", (chunk: Buffer) => { + buffered += chunk.toString("utf8"); + let at = buffered.indexOf("\n"); + while (at !== -1) { + const line = buffered.slice(0, at); + buffered = buffered.slice(at + 1); + if (line.trim()) script(line, say); + at = buffered.indexOf("\n"); + } + }); + + return child; +} + +/** The normalized transcript a host sees: what it must act on, nothing else. */ +interface HostView { + toolCalls: Array<{ name: string; hasId: boolean }>; + doneCount: number; + finalOk: boolean; +} + +async function driveHost(brain: Brain): Promise { + const view: HostView = { toolCalls: [], doneCount: 0, finalOk: false }; + for await (const event of brain.run(task)) { + if (event.type === "tool_call") { + view.toolCalls.push({ name: event.name, hasId: typeof event.id === "string" && event.id.length > 0 }); + brain.sendToolResult(event.id, { output: "[exit 0]\nok", exitCode: 0 }); + } + if (event.type === "done") { + view.doneCount += 1; + view.finalOk = event.ok; + } + } + brain.close(); + return view; +} + +async function viaLocal(): Promise { + const child = scriptedChild((line, say) => { + const cmd = JSON.parse(line) as { type: string }; + if (cmd.type === "task") { + say({ type: "tool_call", id: "c1", name: "read_file", args: { path: "a.ts" } }); + return; + } + if (cmd.type === "tool_result") { + say({ type: "done", ok: true, result: "done", remaining: 0, reason: "" }); + } + }); + return driveHost(new LocalBrain({ spawn: () => child as never })); +} + +async function viaOllama(): Promise { + const replies: ChatReply[] = [ + { + role: "assistant", + content: "", + tool_calls: [{ id: "c1", type: "function", function: { name: "read_file", arguments: '{"path":"a.ts"}' } }], + }, + { role: "assistant", content: "done" }, + ]; + let i = 0; + const chat = async (_m: readonly ChatMessage[]): Promise => + replies[Math.min(i++, replies.length - 1)] ?? { role: "assistant", content: "" }; + return driveHost(new OllamaBrain({ chat })); +} + +test("canary 7: both local brains present the same shape to a host loop", async () => { + const local = await viaLocal(); + const ollama = await viaOllama(); + + assert.deepEqual( + local.toolCalls.map((call) => call.name), + ollama.toolCalls.map((call) => call.name), + "the same tool calls, in the same order", + ); + assert.deepEqual(local, ollama, "a host must not be able to tell the two apart from its own transcript"); +}); + +test("canary 7: every tool_call from either brain carries a replyable id", async () => { + for (const [label, view] of [ + ["local", await viaLocal()], + ["ollama", await viaOllama()], + ] as const) { + assert.ok(view.toolCalls.length > 0, `${label} produced no tool call`); + for (const call of view.toolCalls) { + assert.equal(call.hasId, true, `${label} emitted a tool_call with no id to reply to`); + } + } +}); + +test("canary 7: each brain terminates with exactly one done", async () => { + for (const [label, view] of [ + ["local", await viaLocal()], + ["ollama", await viaOllama()], + ] as const) { + assert.equal(view.doneCount, 1, `${label} did not terminate with exactly one done`); + } +}); + +test("canary 7: a brain that dies without a done still ends its stream", async () => { + // The failure this guards is a hang, not a wrong value: a host awaiting a + // terminal event that never arrives waits forever. + const child = scriptedChild(() => { + /* answers nothing */ + }); + const brain = new LocalBrain({ spawn: () => child as never }); + const events: BrainEvent[] = []; + const drain = (async (): Promise => { + for await (const event of brain.run(task)) events.push(event); + })(); + child.kill(); + await drain; // must settle rather than hang + assert.equal( + events.some((event) => event.type === "tool_call"), + false, + ); +});