From 007840d1cd4ed829c0ab5706a501df8be4c61d25 Mon Sep 17 00:00:00 2001 From: Sweets Sweetman Date: Thu, 23 Jul 2026 16:21:36 -0500 Subject: [PATCH] fix(credits): meter LLM credits when a turn fails after a side-effect runAgentWorkflow charged the account + wrote the usage_events audit row only on `if (result.responseMessage)` after runAgentStep returned. The customer email is a `send_email` tool call executed INSIDE runAgentStep, so a turn that emailed the customer and then threw (or returned no responseMessage) hit the try's `finally` with no catch and skipped billing entirely: email out, wallet not debited, no usage_events row. Observed 2026-07-23: a scheduled briefing on moonshotai/kimi-k3 emailed the customer at 20:54 but wrote 0 rows with a model_id (vs the sonnet-5 run's 1). Both runs' model-independent scrape/tool charges were identical; only the LLM line went missing. Fix: extract the charge into an idempotent `chargeTurn` and call it both on the normal path and as a `finally` backstop. A turn that produced a customer-facing side-effect now always lands a wallet debit + audit row; ZERO_USAGE with no gateway cost floors to the 1c minimum, so a failed turn still writes a model_id row. Normal + user-abort paths bill exactly once (chargeTurn guards on `charged`); the original error still propagates after the backstop runs. Flips two tests that codified the gap ("does NOT bill when no responseMessage" / "...when it throws") to assert the turn is now billed. 20 workflow tests + 129 workflows/credits domain tests pass; eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/runAgentWorkflow.test.ts | 30 +++++++-- app/lib/workflows/runAgentWorkflow.ts | 66 ++++++++++++------- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/app/lib/workflows/__tests__/runAgentWorkflow.test.ts b/app/lib/workflows/__tests__/runAgentWorkflow.test.ts index be1adf5d..68ebc63b 100644 --- a/app/lib/workflows/__tests__/runAgentWorkflow.test.ts +++ b/app/lib/workflows/__tests__/runAgentWorkflow.test.ts @@ -262,7 +262,7 @@ describe("runAgentWorkflow", () => { }); }); - it("does NOT call handleChatCredits when runAgentStep returns no responseMessage", async () => { + it("STILL bills the turn (1c floor) when runAgentStep returns no responseMessage — a mid-turn side-effect (e.g. send_email) may already have fired", async () => { vi.mocked(runAgentStep).mockResolvedValue({ finishReason: "stop", aborted: false, @@ -271,15 +271,37 @@ describe("runAgentWorkflow", () => { await runAgentWorkflow(baseInput); - expect(handleChatCredits).not.toHaveBeenCalled(); + // Previously this path skipped billing entirely — a turn that emailed + // the customer but returned no responseMessage was free + left no + // usage_events audit row. Now it bills once with zero usage → 1c floor + // + a model_id row. + expect(handleChatCredits).toHaveBeenCalledTimes(1); + expect(handleChatCredits).toHaveBeenCalledWith({ + accountId: "acc-1", + model: "anthropic/claude-haiku-4.5", + source: "api", + gatewayCostUsd: undefined, + usage: { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }, + }); }); - it("does NOT call handleChatCredits when runAgentStep throws (no message to bill)", async () => { + it("STILL bills the turn (1c floor) when runAgentStep throws after a side-effect — email out must never escape the wallet debit + audit row", async () => { vi.mocked(runAgentStep).mockRejectedValue(new Error("model exploded")); + // The error still propagates (workflow fails)... await expect(runAgentWorkflow(baseInput)).rejects.toThrow("model exploded"); - expect(handleChatCredits).not.toHaveBeenCalled(); + // ...but the `finally` backstop bills the turn first, so a turn that ran + // a customer-facing tool (send_email) then threw is charged + audited + // instead of emailing-but-not-billing. + expect(handleChatCredits).toHaveBeenCalledTimes(1); + expect(handleChatCredits).toHaveBeenCalledWith({ + accountId: "acc-1", + model: "anthropic/claude-haiku-4.5", + source: "api", + gatewayCostUsd: undefined, + usage: { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }, + }); }); describe("auto-commit", () => { diff --git a/app/lib/workflows/runAgentWorkflow.ts b/app/lib/workflows/runAgentWorkflow.ts index b7335372..a5969f7d 100644 --- a/app/lib/workflows/runAgentWorkflow.ts +++ b/app/lib/workflows/runAgentWorkflow.ts @@ -97,35 +97,47 @@ export async function runAgentWorkflow(input: RunAgentWorkflowInput): Promise> | undefined; + let charged = false; + const chargeTurn = async () => { + if (charged) return; + charged = true; + const metadata = result?.responseMessage?.metadata as AgentMessageMetadata | undefined; + await handleChatCredits({ + accountId: input.accountId, + model: input.modelId, + source: "api", + gatewayCostUsd: metadata?.totalMessageCost, + usage: metadata?.totalMessageUsage ?? ZERO_USAGE, + }); + }; + try { - const result = await runAgentStep({ + result = await runAgentStep({ ...input, writable, assistantMessageId, }); console.log("[runAgentWorkflow] finish", { finishReason: result.finishReason }); - // The assistant message is persisted per step inside `runAgentStep`, so - // it's not written here. We still use the final `responseMessage` to - // charge the account for this turn: atomic wallet debit + audit row via - // the `deduct_credits_with_audit` Postgres function (`handleChatCredits` - // → `recordCreditDeduction`). - // - // Charge on user-stop too — the provider already billed us for the - // tokens consumed, and the assistant message (including partial tool - // runs) is persisted, so the user owes the charge regardless of how - // the turn ended. `result.responseMessage.metadata` carries the - // usage actually consumed up to the abort point. - if (result.responseMessage) { - const metadata = result.responseMessage.metadata as AgentMessageMetadata | undefined; - await handleChatCredits({ - accountId: input.accountId, - model: input.modelId, - source: "api", - gatewayCostUsd: metadata?.totalMessageCost, - usage: metadata?.totalMessageUsage ?? ZERO_USAGE, - }); - } + // Charge on user-stop too — the provider already billed us for the tokens + // consumed, and the assistant message (including partial tool runs) is + // persisted, so the user owes the charge regardless of how the turn ended. + await chargeTurn(); // Auto-commit + push only after a natural finish. Skip on user-stop — // don't push half-done work, and `autoCommitChatTurn` can run 30+ @@ -145,6 +157,16 @@ export async function runAgentWorkflow(input: RunAgentWorkflowInput): Promise