diff --git a/CHANGELOG.md b/CHANGELOG.md index 97fdb83..86364ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to Sentinel are documented here. The format follows ## [Unreleased] +### Added + +- Per-request **cost tracking**: set a `pricing` map (USD per 1K tokens, per model) in `sentinel.config.json` and every trace records a `costUsd`; the dashboard shows total spend, spend saved by the cache, and cost over time. + ### Fixed - The gateway production build (`pnpm build`) no longer pulls dashboard sources into the Node build; CI now runs `pnpm build` on every PR so it can't silently break again. diff --git a/README.md b/README.md index be7921f..31b1edd 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Sentinel never hardcodes a machine or a model. Set `OLLAMA_BASE_URL` in `.env` t ## Observability -Every request is traced with OpenTelemetry — provider, model, status, latency, and token usage — and persisted to a queryable store (SQLite by default; set `TRACE_DB=memory` for an ephemeral one). Spans also export to any OTLP collector (Jaeger, etc.) when `OTEL_EXPORTER_OTLP_ENDPOINT` is set. +Every request is traced with OpenTelemetry — provider, model, status, latency, token usage, and (when a `pricing` map is configured) estimated USD cost — and persisted to a queryable store (SQLite by default; set `TRACE_DB=memory` for an ephemeral one). Spans also export to any OTLP collector (Jaeger, etc.) when `OTEL_EXPORTER_OTLP_ENDPOINT` is set. Read recent traces through the admin-gated API (set `SENTINEL_ADMIN_KEY` to enable it): @@ -164,7 +164,7 @@ Sentinel v0.1.0 is a **single-node, self-hosted** gateway. Honest boundaries tod - **The async judge needs a local Ollama** with `JUDGE_MODEL` pulled. Without it, judging degrades to `unscored` (never a false pass). The bundled benchmarks run against **mock upstreams**, so the headline catch-rate is the _deterministic guardrail_ rate; the LLM judge is covered by unit tests. - **Inline guardrails apply to non-streaming responses.** Streamed responses are judged from their buffered output _after_ completion (inline blocking of a live stream is on the roadmap). - **Providers are OpenAI-compatible.** OpenAI, Groq, Gemini's OpenAI endpoint, Ollama, and other OpenAI-API providers work today; a native **Anthropic** (Messages API) adapter is planned. -- **Cost reduction is measured by request volume** (cache hits × avoided upstream calls) on a repeat-heavy workload; per-request dollar accounting is planned. +- **Per-request dollar cost requires a `pricing` map** in `sentinel.config.json` (USD per 1K tokens, per model). With it, every trace records a `costUsd` and the dashboard shows spend + cache savings; without it, cost is unknown (`null`) and only request-volume cache savings are visible. ## Development diff --git a/docs/dashboard.png b/docs/dashboard.png index f6e11fb..909a3ec 100644 Binary files a/docs/dashboard.png and b/docs/dashboard.png differ diff --git a/packages/dashboard/e2e/dashboard.spec.ts b/packages/dashboard/e2e/dashboard.spec.ts index 91fac41..cbb99aa 100644 --- a/packages/dashboard/e2e/dashboard.spec.ts +++ b/packages/dashboard/e2e/dashboard.spec.ts @@ -13,6 +13,7 @@ function sampleTrace(over: Record): Record { promptTokens: 10, completionTokens: 5, totalTokens: 15, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, @@ -32,9 +33,16 @@ function sampleTrace(over: Record): Record { } const traces = [ - sampleTrace({ id: 'a', status: 200, cacheHit: true, judgeScore: 5 }), - sampleTrace({ id: 'b', status: 500, errorType: 'upstream_error' }), - sampleTrace({ id: 'c', status: 200, provider: 'groq', fallbackUsed: true, judgeScore: 3 }), + sampleTrace({ id: 'a', status: 200, cacheHit: true, judgeScore: 5, costUsd: 0.01 }), + sampleTrace({ id: 'b', status: 500, errorType: 'upstream_error', costUsd: 0.02 }), + sampleTrace({ + id: 'c', + status: 200, + provider: 'groq', + fallbackUsed: true, + judgeScore: 3, + costUsd: 0.03, + }), ]; test('renders aggregated stats from the trace API', async ({ page }) => { @@ -54,4 +62,5 @@ test('renders aggregated stats from the trace API', async ({ page }) => { await expect(page.getByTitle('groq')).toBeVisible(); await expect(page.getByRole('heading', { name: 'Recent requests' })).toBeVisible(); await expect(page.getByTestId('stat-judge')).toContainText('4'); // mean of [5,3] + await expect(page.getByTestId('stat-cost')).toContainText('$0.05'); // 0.02 + 0.03 (cache hit excluded) }); diff --git a/packages/dashboard/e2e/screenshot.spec.ts b/packages/dashboard/e2e/screenshot.spec.ts index 29fde41..afb2717 100644 --- a/packages/dashboard/e2e/screenshot.spec.ts +++ b/packages/dashboard/e2e/screenshot.spec.ts @@ -34,6 +34,7 @@ function trace(over: Record): Record { promptTokens: 40, completionTokens: 20, totalTokens: 60, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, @@ -96,6 +97,7 @@ function buildTraces(): Record[] { promptTokens: prompt, completionTokens: completion, totalTokens: prompt + completion, + costUsd: Math.round((prompt * 0.00002 + completion * 0.00006) * 1e6) / 1e6, cacheHit, fallbackUsed, routedProvider: fallbackUsed ? 'ollama' : null, diff --git a/packages/dashboard/src/App.tsx b/packages/dashboard/src/App.tsx index bbb39cc..7f01158 100644 --- a/packages/dashboard/src/App.tsx +++ b/packages/dashboard/src/App.tsx @@ -19,6 +19,10 @@ function pct(value: number): string { return `${String(Math.round(value * 1000) / 10)}%`; } +function usd(value: number): string { + return `$${value.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 4 })}`; +} + export function App() { const [baseUrl, setBaseUrl] = useState(() => localStorage.getItem(KEY_BASE) ?? ''); const [adminKey, setAdminKey] = useState(() => localStorage.getItem(KEY_ADMIN) ?? ''); @@ -124,6 +128,12 @@ export function App() { sub={`avg ${String(stats.avgLatencyMs)} ms`} /> + + b.costUsd} /> diff --git a/packages/dashboard/src/aggregate.test.ts b/packages/dashboard/src/aggregate.test.ts index 1e5d596..f395c92 100644 --- a/packages/dashboard/src/aggregate.test.ts +++ b/packages/dashboard/src/aggregate.test.ts @@ -15,6 +15,7 @@ function trace(over: Partial): TraceRecord { promptTokens: 10, completionTokens: 5, totalTokens: 15, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, @@ -91,4 +92,27 @@ describe('computeStats', () => { expect(stats.overTime[0]?.count).toBe(2); expect(stats.overTime[1]?.count).toBe(1); }); + + it('sums cost spent vs saved by cache (null cost ignored)', () => { + const stats = computeStats([ + trace({ costUsd: 0.01, cacheHit: false }), + trace({ costUsd: 0.02, cacheHit: false }), + trace({ costUsd: 0.05, cacheHit: true }), + trace({ costUsd: null, cacheHit: false }), + ]); + expect(stats.totalCostUsd).toBe(0.03); + expect(stats.savedCostUsd).toBe(0.05); + }); + + it('buckets cost over time, excluding cache hits', () => { + const stats = computeStats( + [ + trace({ timestamp: 0, costUsd: 0.01, cacheHit: false }), + trace({ timestamp: 30_000, costUsd: 0.02, cacheHit: false }), + trace({ timestamp: 30_000, costUsd: 0.04, cacheHit: true }), + ], + 60_000, + ); + expect(stats.overTime[0]?.costUsd).toBeCloseTo(0.03, 6); + }); }); diff --git a/packages/dashboard/src/aggregate.ts b/packages/dashboard/src/aggregate.ts index 4501381..a10f379 100644 --- a/packages/dashboard/src/aggregate.ts +++ b/packages/dashboard/src/aggregate.ts @@ -10,6 +10,8 @@ export interface TimeBucket { count: number; tokens: number; errors: number; + /** Actual upstream spend in this window (USD; cache hits excluded). */ + costUsd: number; } export interface ScoreBin { @@ -26,6 +28,10 @@ export interface Stats { fallbacks: number; fallbackRate: number; totalTokens: number; + /** Actual upstream spend (USD) over non-cache-hit requests. */ + totalCostUsd: number; + /** Spend avoided by cache hits (USD). */ + savedCostUsd: number; avgLatencyMs: number; p95LatencyMs: number; judgeScoredCount: number; @@ -49,6 +55,8 @@ export const EMPTY_STATS: Stats = { fallbacks: 0, fallbackRate: 0, totalTokens: 0, + totalCostUsd: 0, + savedCostUsd: 0, avgLatencyMs: 0, p95LatencyMs: 0, judgeScoredCount: 0, @@ -97,6 +105,8 @@ export function computeStats(traces: TraceRecord[], bucketMs = 60_000): Stats { let cacheHits = 0; let fallbacks = 0; let totalTokens = 0; + let totalCostUsd = 0; + let savedCostUsd = 0; const latencies: number[] = []; const judgeScores: number[] = []; const histogram = new Map([ @@ -114,6 +124,9 @@ export function computeStats(traces: TraceRecord[], bucketMs = 60_000): Stats { if (t.cacheHit) cacheHits++; if (t.fallbackUsed) fallbacks++; totalTokens += t.totalTokens ?? 0; + const cost = t.costUsd ?? 0; + if (t.cacheHit) savedCostUsd += cost; + else totalCostUsd += cost; latencies.push(t.durationMs); if (t.judgeScore !== null) { judgeScores.push(t.judgeScore); @@ -121,9 +134,10 @@ export function computeStats(traces: TraceRecord[], bucketMs = 60_000): Stats { if (histogram.has(rounded)) histogram.set(rounded, (histogram.get(rounded) ?? 0) + 1); } const key = Math.floor(t.timestamp / bucketMs) * bucketMs; - const bucket = buckets.get(key) ?? { bucket: key, count: 0, tokens: 0, errors: 0 }; + const bucket = buckets.get(key) ?? { bucket: key, count: 0, tokens: 0, errors: 0, costUsd: 0 }; bucket.count++; bucket.tokens += t.totalTokens ?? 0; + if (!t.cacheHit) bucket.costUsd += cost; if (isError) bucket.errors++; buckets.set(key, bucket); } @@ -143,6 +157,8 @@ export function computeStats(traces: TraceRecord[], bucketMs = 60_000): Stats { fallbacks, fallbackRate: fallbacks / total, totalTokens, + totalCostUsd: Math.round(totalCostUsd * 1e6) / 1e6, + savedCostUsd: Math.round(savedCostUsd * 1e6) / 1e6, avgLatencyMs: Math.round(avgLatency * 100) / 100, p95LatencyMs: Math.round(percentile(latencies, 95) * 100) / 100, judgeScoredCount: judgeScores.length, diff --git a/packages/dashboard/src/components.tsx b/packages/dashboard/src/components.tsx index 81456da..1763bb5 100644 --- a/packages/dashboard/src/components.tsx +++ b/packages/dashboard/src/components.tsx @@ -58,15 +58,20 @@ export function Histogram(props: { title: string; bins: ScoreBin[] }) { ); } -export function Sparkline(props: { title: string; buckets: TimeBucket[] }) { +export function Sparkline(props: { + title: string; + buckets: TimeBucket[]; + value?: (b: TimeBucket) => number; +}) { const pts = props.buckets; + const value = props.value ?? ((b: TimeBucket) => b.count); const width = 280; const height = 60; - const max = Math.max(1, ...pts.map((p) => p.count)); + const max = Math.max(1, ...pts.map((p) => value(p))); const path = pts .map((p, i) => { const x = pts.length > 1 ? (i / (pts.length - 1)) * width : 0; - const y = height - (p.count / max) * height; + const y = height - (value(p) / max) * height; return `${i === 0 ? 'M' : 'L'}${String(Math.round(x))},${String(Math.round(y))}`; }) .join(' '); diff --git a/packages/dashboard/src/types.ts b/packages/dashboard/src/types.ts index f38da36..dc10028 100644 --- a/packages/dashboard/src/types.ts +++ b/packages/dashboard/src/types.ts @@ -15,6 +15,7 @@ export interface TraceRecord { promptTokens: number | null; completionTokens: number | null; totalTokens: number | null; + costUsd: number | null; errorType: string | null; errorMessage: string | null; apiKeyHash: string | null; diff --git a/packages/gateway/src/config.test.ts b/packages/gateway/src/config.test.ts index cb345eb..ec6f5e3 100644 --- a/packages/gateway/src/config.test.ts +++ b/packages/gateway/src/config.test.ts @@ -95,6 +95,26 @@ describe('loadConfig', () => { expect(cfg.defaultProvider).toBe('ollama'); }); + it('parses the pricing map and defaults it to empty', () => { + const withPricing = JSON.stringify({ + providers: { ollama: { type: 'openai-compatible', baseUrlEnv: 'OLLAMA_BASE_URL' } }, + models: { 'llama3.2': 'ollama' }, + pricing: { 'gpt-4o-mini': { inputPer1k: 0.15, outputPer1k: 0.6 } }, + }); + const cfg = loadConfig({ path: 'x', env, readFile: () => withPricing }); + expect(cfg.pricing.get('gpt-4o-mini')).toEqual({ inputPer1k: 0.15, outputPer1k: 0.6 }); + expect(loadConfig({ path: 'x', env, readFile: () => validConfig }).pricing.size).toBe(0); + }); + + it('rejects negative pricing', () => { + const bad = JSON.stringify({ + providers: { ollama: { type: 'openai-compatible', baseUrlEnv: 'OLLAMA_BASE_URL' } }, + models: {}, + pricing: { m: { inputPer1k: -1, outputPer1k: 0 } }, + }); + expect(() => loadConfig({ path: 'x', env, readFile: () => bad })).toThrow(ConfigError); + }); + it('throws on invalid JSON', () => { expect(() => loadConfig({ path: 'x', env, readFile: () => '{ not json' })).toThrow(ConfigError); }); diff --git a/packages/gateway/src/config.ts b/packages/gateway/src/config.ts index 8407ca4..ccb0e98 100644 --- a/packages/gateway/src/config.ts +++ b/packages/gateway/src/config.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs'; import { z } from 'zod'; import { ConfigError } from './errors.js'; +import type { ModelPricing } from './cost.js'; // ─────────────────────────── Server environment ─────────────────────────── @@ -129,6 +130,12 @@ const guardrailsConfigSchema = z.object({ requireJson: z.boolean().optional(), }); +// USD per 1,000 tokens, per model. Used to attribute a cost to every traced request. +const modelPricingSchema = z.object({ + inputPer1k: z.number().nonnegative(), + outputPer1k: z.number().nonnegative(), +}); + const sentinelConfigSchema = z .object({ providers: z.record(z.string(), providerConfigSchema), @@ -136,6 +143,7 @@ const sentinelConfigSchema = z defaultProvider: z.string().optional(), routing: routingConfigSchema.optional(), guardrails: guardrailsConfigSchema.optional(), + pricing: z.record(z.string(), modelPricingSchema).optional(), }) .superRefine((cfg, ctx) => { const names = new Set(Object.keys(cfg.providers)); @@ -183,6 +191,8 @@ export interface ResolvedConfig { defaultProvider: string | undefined; routing?: ResolvedRouting; guardrails?: ResolvedGuardrails; + /** model name → USD-per-1K-token pricing (empty when no `pricing` block is configured). */ + pricing: Map; } export interface LoadConfigOptions { @@ -228,6 +238,7 @@ export function loadConfig(options: LoadConfigOptions): ResolvedConfig { defaultProvider: parsed.data.defaultProvider, ...(parsed.data.routing ? { routing: parsed.data.routing } : {}), ...(parsed.data.guardrails ? { guardrails: parsed.data.guardrails } : {}), + pricing: new Map(Object.entries(parsed.data.pricing ?? {})), }; } diff --git a/packages/gateway/src/cost.test.ts b/packages/gateway/src/cost.test.ts new file mode 100644 index 0000000..a362026 --- /dev/null +++ b/packages/gateway/src/cost.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect } from 'vitest'; +import { computeCostUsd } from './cost.js'; +import type { ModelPricing } from './cost.js'; + +const pricing = new Map([ + ['gpt-4o-mini', { inputPer1k: 0.15, outputPer1k: 0.6 }], + ['free-local', { inputPer1k: 0, outputPer1k: 0 }], + ['dusty', { inputPer1k: 0.1, outputPer1k: 0.2 }], +]); + +describe('computeCostUsd', () => { + it('prices input and output tokens from the per-1K rates', () => { + // 1000 prompt × 0.15/1K + 500 completion × 0.6/1K = 0.15 + 0.30 = 0.45 + expect( + computeCostUsd('gpt-4o-mini', { promptTokens: 1000, completionTokens: 500 }, pricing), + ).toBe(0.45); + }); + + it('returns null for a model that is not in the price map', () => { + expect( + computeCostUsd('mystery-model', { promptTokens: 100, completionTokens: 100 }, pricing), + ).toBeNull(); + }); + + it('returns null when no usage is available (both sides null)', () => { + expect( + computeCostUsd('gpt-4o-mini', { promptTokens: null, completionTokens: null }, pricing), + ).toBeNull(); + }); + + it('treats a missing side as zero when the other side is known', () => { + expect( + computeCostUsd('gpt-4o-mini', { promptTokens: 2000, completionTokens: null }, pricing), + ).toBe(0.3); + expect( + computeCostUsd('gpt-4o-mini', { promptTokens: null, completionTokens: 1000 }, pricing), + ).toBe(0.6); + }); + + it('returns 0 (not null) for a priced-but-free model with real usage', () => { + expect( + computeCostUsd('free-local', { promptTokens: 500, completionTokens: 500 }, pricing), + ).toBe(0); + }); + + it('rounds away binary float dust (0.1 + 0.2)', () => { + // 1000/1K × 0.1 + 1000/1K × 0.2 = 0.30000000000000004 in IEEE-754 → rounded to 0.3 + expect(computeCostUsd('dusty', { promptTokens: 1000, completionTokens: 1000 }, pricing)).toBe( + 0.3, + ); + }); +}); diff --git a/packages/gateway/src/cost.ts b/packages/gateway/src/cost.ts new file mode 100644 index 0000000..ad0bb87 --- /dev/null +++ b/packages/gateway/src/cost.ts @@ -0,0 +1,38 @@ +/** + * Per-request cost accounting. Pure functions over token usage and a per-model + * price map — no I/O. The price map comes from `sentinel.config.json` (`pricing`). + */ + +/** USD price for one model, per 1,000 tokens. */ +export interface ModelPricing { + /** USD per 1K prompt (input) tokens. */ + inputPer1k: number; + /** USD per 1K completion (output) tokens. */ + outputPer1k: number; +} + +/** Token counts from a (real or cached) completion; either side may be unknown. */ +export interface TokenUsage { + promptTokens: number | null; + completionTokens: number | null; +} + +/** + * Computes the USD cost of a completion from its token usage and a price map. + * Returns `null` — never a guess — when the model is not priced or no usage is + * available, so an unpriced request is recorded as "unknown cost", not as `0`. + */ +export function computeCostUsd( + model: string, + usage: TokenUsage, + pricing: ReadonlyMap, +): number | null { + const price = pricing.get(model); + if (price === undefined) return null; + if (usage.promptTokens === null && usage.completionTokens === null) return null; + const prompt = usage.promptTokens ?? 0; + const completion = usage.completionTokens ?? 0; + const cost = (prompt / 1000) * price.inputPer1k + (completion / 1000) * price.outputPer1k; + // Round to micro-dollars; avoids float dust like 0.000_000_000_2 in traces. + return Math.round(cost * 1e6) / 1e6; +} diff --git a/packages/gateway/src/main.ts b/packages/gateway/src/main.ts index da82194..a007296 100644 --- a/packages/gateway/src/main.ts +++ b/packages/gateway/src/main.ts @@ -63,6 +63,7 @@ async function main(): Promise { traceStore: store, adminKey: env.adminKey, cache, + pricing: config.pricing, ...(clientThrottle ? { clientThrottle } : {}), routing: { config: config.routing, diff --git a/packages/gateway/src/providers/registry.test.ts b/packages/gateway/src/providers/registry.test.ts index 7bbb055..9ea7709 100644 --- a/packages/gateway/src/providers/registry.test.ts +++ b/packages/gateway/src/providers/registry.test.ts @@ -10,6 +10,7 @@ const config: ResolvedConfig = { ]), models: new Map([['qwen2.5:7b', 'ollama']]), defaultProvider: 'openai', + pricing: new Map(), }; describe('createRegistry', () => { @@ -33,6 +34,7 @@ describe('createRegistry', () => { providers: new Map(), models: new Map([['m', 'ghost']]), defaultProvider: undefined, + pricing: new Map(), }; expect(() => createRegistry(broken).resolve('m')).toThrow(ModelNotFoundError); }); diff --git a/packages/gateway/src/server.test.ts b/packages/gateway/src/server.test.ts index 6e9a86f..5e599ca 100644 --- a/packages/gateway/src/server.test.ts +++ b/packages/gateway/src/server.test.ts @@ -14,6 +14,7 @@ import type { SemanticCache } from './cache/cache.js'; import type { Embedder } from './cache/embedder.js'; import { createVerifier } from './verify/verifier.js'; import type { Judge } from './verify/judge.js'; +import type { ModelPricing } from './cost.js'; function makeProvider(overrides: Partial = {}): Provider { return { @@ -48,7 +49,12 @@ function makeMultiRegistry(map: Record): ProviderRegistry { function buildTestServer( registry: ProviderRegistry, - opts: { store?: TraceStore; adminKey?: string; cache?: SemanticCache } = {}, + opts: { + store?: TraceStore; + adminKey?: string; + cache?: SemanticCache; + pricing?: ReadonlyMap; + } = {}, ) { return buildServer({ registry, @@ -57,6 +63,7 @@ function buildTestServer( traceStore: opts.store ?? new InMemoryTraceStore(), adminKey: opts.adminKey, cache: opts.cache, + ...(opts.pricing ? { pricing: opts.pricing } : {}), }); } @@ -437,6 +444,25 @@ describe('tracing & /traces', () => { expect(last?.completionTokens).toBe(7); }); + it('attributes a USD cost to the trace when the model is priced', async () => { + const chatProvider = makeProvider({ + chat: () => + Promise.resolve({ + id: 'cmpl', + choices: [], + usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, + }), + }); + const app = buildTestServer(makeRegistry(chatProvider), { + store: sink, + pricing: new Map([['m', { inputPer1k: 1, outputPer1k: 2 }]]), + }); + await app.inject({ method: 'POST', url, headers: auth, payload: body }); + await app.close(); + // 5/1K × $1 + 7/1K × $2 = 0.005 + 0.014 = 0.019 + expect(sink.query()[0]?.costUsd).toBe(0.019); + }); + it('requires the admin key on /traces', async () => { const app = buildTestServer(makeRegistry(makeProvider()), { store: sink, @@ -474,6 +500,7 @@ describe('tracing & /traces', () => { promptTokens: null, completionTokens: null, totalTokens: null, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, @@ -518,6 +545,7 @@ describe('tracing & /traces', () => { promptTokens: null, completionTokens: null, totalTokens: null, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, @@ -713,6 +741,7 @@ describe('tracing & /traces', () => { promptTokens: null, completionTokens: null, totalTokens: null, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, diff --git a/packages/gateway/src/server.ts b/packages/gateway/src/server.ts index f8a7d86..9c80d3a 100644 --- a/packages/gateway/src/server.ts +++ b/packages/gateway/src/server.ts @@ -3,6 +3,8 @@ import type { FastifyServerOptions } from 'fastify'; import { SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; import type { Span } from '@opentelemetry/api'; import { chatCompletionRequestSchema } from './schemas.js'; +import { computeCostUsd } from './cost.js'; +import type { ModelPricing, TokenUsage } from './cost.js'; import { createAuthHook, extractBearerToken, hashApiKey } from './auth.js'; import { GatewayError, @@ -48,6 +50,8 @@ export interface ServerDeps { cache?: SemanticCache | undefined; routing?: RoutingDeps | undefined; verifier?: Verifier | undefined; + /** model → USD-per-1K-token pricing, for per-request cost attribution on the trace. */ + pricing?: ReadonlyMap | undefined; /** Per-API-key inbound rate-limit buckets (built from CLIENT_RPM). */ clientThrottle?: BucketRegistry | undefined; logger?: FastifyServerOptions['logger']; @@ -103,6 +107,32 @@ function setUsageAttributes(span: Span | undefined, result: unknown): void { span.setAttribute('gen_ai.usage.total_tokens', u.total_tokens); } +/** Extracts OpenAI-style token usage as plain numbers (null when a field is absent). */ +function extractUsage(result: unknown): TokenUsage { + if (typeof result !== 'object' || result === null) + return { promptTokens: null, completionTokens: null }; + const usage = (result as { usage?: unknown }).usage; + if (typeof usage !== 'object' || usage === null) + return { promptTokens: null, completionTokens: null }; + const u = usage as Record; + return { + promptTokens: typeof u.prompt_tokens === 'number' ? u.prompt_tokens : null, + completionTokens: typeof u.completion_tokens === 'number' ? u.completion_tokens : null, + }; +} + +/** Records the request's USD cost from usage × the price map (no-op if unpriced/unknown). */ +function applyCost( + span: Span | undefined, + model: string, + result: unknown, + pricing: ReadonlyMap | undefined, +): void { + if (span === undefined || pricing === undefined || pricing.size === 0) return; + const cost = computeCostUsd(model, extractUsage(result), pricing); + if (cost !== null) span.setAttribute('sentinel.cost_usd', cost); +} + /** Records which provider/model actually served the request after routing/fallback. */ function applyRouteAttributes(span: Span | undefined, outcome: RouteOutcome): void { span?.setAttribute('sentinel.provider', outcome.routedProvider); @@ -226,6 +256,8 @@ export function buildServer(deps: ServerDeps) { const cached = await deps.cache.get(chatRequest, apiKeyHash); if (cached?.kind === 'json') { setUsageAttributes(span, cached.body); + // Cost recorded on a hit = the upstream spend this cache avoided. + applyCost(span, chatRequest.model, cached.body, deps.pricing); span?.setAttribute('sentinel.cache_hit', true); endSpan(request, 200); return reply.status(200).send(cached.body); @@ -234,6 +266,7 @@ export function buildServer(deps: ServerDeps) { const { result, outcome } = await runChat(candidates, chatRequest, execOptions); setUsageAttributes(span, result); applyRouteAttributes(span, outcome); + applyCost(span, outcome.routedModel, result, deps.pricing); let responseText: string | undefined; if (deps.verifier !== undefined) { diff --git a/packages/gateway/src/telemetry/store.sqlite.ts b/packages/gateway/src/telemetry/store.sqlite.ts index 5e93369..8ceb554 100644 --- a/packages/gateway/src/telemetry/store.sqlite.ts +++ b/packages/gateway/src/telemetry/store.sqlite.ts @@ -14,6 +14,7 @@ const SCHEMA = ` prompt_tokens INTEGER, completion_tokens INTEGER, total_tokens INTEGER, + cost_usd REAL, error_type TEXT, error_message TEXT, api_key_hash TEXT, @@ -50,6 +51,7 @@ interface TraceRow { prompt_tokens: number | null; completion_tokens: number | null; total_tokens: number | null; + cost_usd: number | null; error_type: string | null; error_message: string | null; api_key_hash: string | null; @@ -74,6 +76,15 @@ export class SqliteTraceStore implements TraceStore { this.db = new Database(path); this.db.pragma('journal_mode = WAL'); this.db.exec(SCHEMA); + this.migrate(); + } + + /** Adds columns introduced after the initial schema to a pre-existing DB. */ + private migrate(): void { + const columns = new Set( + (this.db.prepare('PRAGMA table_info(traces)').all() as { name: string }[]).map((c) => c.name), + ); + if (!columns.has('cost_usd')) this.db.exec('ALTER TABLE traces ADD COLUMN cost_usd REAL'); } record(trace: TraceRecord): void { @@ -81,13 +92,13 @@ export class SqliteTraceStore implements TraceStore { .prepare( `INSERT OR REPLACE INTO traces (id, trace_id, timestamp, duration_ms, model, provider, stream, status, - prompt_tokens, completion_tokens, total_tokens, error_type, error_message, api_key_hash, + prompt_tokens, completion_tokens, total_tokens, cost_usd, error_type, error_message, api_key_hash, cache_hit, routed_provider, routed_model, fallback_used, retry_count, guardrail_status, guardrail_violations, judge_score, judge_reason, judge_error, prompt_fingerprint) VALUES (@id, @traceId, @timestamp, @durationMs, @model, @provider, @stream, @status, - @promptTokens, @completionTokens, @totalTokens, @errorType, @errorMessage, @apiKeyHash, + @promptTokens, @completionTokens, @totalTokens, @costUsd, @errorType, @errorMessage, @apiKeyHash, @cacheHit, @routedProvider, @routedModel, @fallbackUsed, @retryCount, @guardrailStatus, @guardrailViolations, @judgeScore, @judgeReason, @judgeError, @promptFingerprint)`, @@ -104,6 +115,7 @@ export class SqliteTraceStore implements TraceStore { promptTokens: trace.promptTokens, completionTokens: trace.completionTokens, totalTokens: trace.totalTokens, + costUsd: trace.costUsd, errorType: trace.errorType, errorMessage: trace.errorMessage, apiKeyHash: trace.apiKeyHash, @@ -224,6 +236,7 @@ function rowToRecord(row: TraceRow): TraceRecord { promptTokens: row.prompt_tokens, completionTokens: row.completion_tokens, totalTokens: row.total_tokens, + costUsd: row.cost_usd, errorType: row.error_type, errorMessage: row.error_message, apiKeyHash: row.api_key_hash, diff --git a/packages/gateway/src/telemetry/store.test.ts b/packages/gateway/src/telemetry/store.test.ts index 72dd46e..ba4a779 100644 --- a/packages/gateway/src/telemetry/store.test.ts +++ b/packages/gateway/src/telemetry/store.test.ts @@ -1,3 +1,7 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import Database from 'better-sqlite3'; import { describe, it, expect } from 'vitest'; import type { TraceRecord, TraceStore } from './trace.js'; import { InMemoryTraceStore } from './store.memory.js'; @@ -17,6 +21,7 @@ function sample(over: Partial = {}): TraceRecord { promptTokens: 10, completionTokens: 20, totalTokens: 30, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: 'abc', @@ -149,9 +154,46 @@ for (const [name, makeStore] of backends) { expect(got?.retryCount).toBe(3); store.close(); }); + + it('round-trips costUsd (present and null)', () => { + const store = makeStore(); + store.record(sample({ id: 'cost', costUsd: 0.0123 })); + store.record(sample({ id: 'free', costUsd: null })); + expect(store.get('cost')?.costUsd).toBe(0.0123); + expect(store.get('free')?.costUsd).toBeNull(); + store.close(); + }); }); } +describe('SqliteTraceStore migration', () => { + it('adds the cost_usd column to a DB created before it existed', () => { + const dir = mkdtempSync(join(tmpdir(), 'sentinel-mig-')); + const file = join(dir, 'legacy.db'); + // A DB written by a pre-cost build: the full schema minus the cost_usd column. + const legacy = new Database(file); + legacy.exec( + `CREATE TABLE traces ( + id TEXT PRIMARY KEY, trace_id TEXT NOT NULL, timestamp INTEGER NOT NULL, + duration_ms REAL NOT NULL, model TEXT, provider TEXT, stream INTEGER NOT NULL, + status INTEGER NOT NULL, prompt_tokens INTEGER, completion_tokens INTEGER, + total_tokens INTEGER, error_type TEXT, error_message TEXT, api_key_hash TEXT, + cache_hit INTEGER NOT NULL DEFAULT 0, routed_provider TEXT, routed_model TEXT, + fallback_used INTEGER NOT NULL DEFAULT 0, retry_count INTEGER NOT NULL DEFAULT 0, + guardrail_status TEXT, guardrail_violations TEXT, judge_score REAL, judge_reason TEXT, + judge_error TEXT, prompt_fingerprint TEXT + )`, + ); + legacy.close(); + + const store = new SqliteTraceStore(file); + store.record(sample({ id: 'm', costUsd: 0.5 })); + expect(store.get('m')?.costUsd).toBe(0.5); + store.close(); + rmSync(dir, { recursive: true, force: true }); + }); +}); + describe('createTraceStore', () => { it('builds the requested backend', () => { const mem = createTraceStore({ kind: 'memory' }); diff --git a/packages/gateway/src/telemetry/trace.ts b/packages/gateway/src/telemetry/trace.ts index cc8f1a3..c75dcbc 100644 --- a/packages/gateway/src/telemetry/trace.ts +++ b/packages/gateway/src/telemetry/trace.ts @@ -14,6 +14,8 @@ export interface TraceRecord { promptTokens: number | null; completionTokens: number | null; totalTokens: number | null; + /** Estimated USD cost from token usage × the per-model price map (null = unpriced/unknown). */ + costUsd: number | null; errorType: string | null; errorMessage: string | null; apiKeyHash: string | null; @@ -105,6 +107,7 @@ export function spanToTraceRecord(span: ReadableSpan): TraceRecord { promptTokens: asNumber(attrs['gen_ai.usage.input_tokens']), completionTokens: asNumber(attrs['gen_ai.usage.output_tokens']), totalTokens: asNumber(attrs['gen_ai.usage.total_tokens']), + costUsd: asNumber(attrs['sentinel.cost_usd']), errorType: asString(attrs['error.type']), errorMessage: isError ? (span.status.message ?? null) : null, apiKeyHash: asString(attrs['sentinel.api_key_hash']), diff --git a/packages/gateway/src/verify/regression.test.ts b/packages/gateway/src/verify/regression.test.ts index b7944d8..d37d684 100644 --- a/packages/gateway/src/verify/regression.test.ts +++ b/packages/gateway/src/verify/regression.test.ts @@ -15,6 +15,7 @@ function record(over: Partial): TraceRecord { promptTokens: null, completionTokens: null, totalTokens: null, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, diff --git a/packages/gateway/src/verify/verifier.test.ts b/packages/gateway/src/verify/verifier.test.ts index fdc1903..78da968 100644 --- a/packages/gateway/src/verify/verifier.test.ts +++ b/packages/gateway/src/verify/verifier.test.ts @@ -20,6 +20,7 @@ function record(id: string): TraceRecord { promptTokens: null, completionTokens: null, totalTokens: null, + costUsd: null, errorType: null, errorMessage: null, apiKeyHash: null, diff --git a/sentinel.config.example.json b/sentinel.config.example.json index 353513c..f910720 100644 --- a/sentinel.config.example.json +++ b/sentinel.config.example.json @@ -37,5 +37,10 @@ "blocklist": ["ignore previous instructions"], "pii": ["pii.email", "pii.credit_card", "pii.ssn"], "requireJson": true + }, + "pricing": { + "gpt-4o-mini": { "inputPer1k": 0.00015, "outputPer1k": 0.0006 }, + "gemini-2.0-flash": { "inputPer1k": 0.0001, "outputPer1k": 0.0004 }, + "llama-3.3-70b-versatile": { "inputPer1k": 0.00059, "outputPer1k": 0.00079 } } }