From 98419b977168b19ecbd1d8b74707e58a39fc8b58 Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Tue, 18 Aug 2026 13:45:37 -0400 Subject: [PATCH 1/7] feat(metrics): add `wizard metrics` program for application metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flat skill command mirroring ai-observability: no fixed skillId — the agent loads the context-mill `metrics` category (PostHog/context-mill#342) and picks the platform variant (python, nodejs, javascript, kubernetes, other/OTLP) from the project manifest. Ships a metrics-specific intro screen, registry + switchboard binding, and program tests. Generated-By: PostHog Desktop Task-Id: 495fbacb-61e8-4fdb-8c0f-7f69cd7a3fd6 --- bin.ts | 2 + src/commands/metrics.ts | 16 ++++ src/lib/agent/runner/switchboard/index.ts | 1 + .../__tests__/metrics-program.test.ts | 64 +++++++++++++ src/lib/programs/metrics/index.ts | 70 ++++++++++++++ src/lib/programs/program-registry.ts | 3 + src/ui/tui/screen-registry.tsx | 2 + src/ui/tui/screen-sequences.ts | 1 + src/ui/tui/screens/MetricsIntroScreen.tsx | 91 +++++++++++++++++++ 9 files changed, 250 insertions(+) create mode 100644 src/commands/metrics.ts create mode 100644 src/lib/programs/__tests__/metrics-program.test.ts create mode 100644 src/lib/programs/metrics/index.ts create mode 100644 src/ui/tui/screens/MetricsIntroScreen.tsx diff --git a/bin.ts b/bin.ts index ec8b7f0c..899253a7 100644 --- a/bin.ts +++ b/bin.ts @@ -50,6 +50,7 @@ import { basicIntegrationCommand } from './src/commands/basic-integration'; import { mcpCommand } from './src/commands/mcp'; import { mcpAnalyticsCommand } from './src/commands/mcp-analytics'; import { aiObservabilityCommand } from './src/commands/ai-observability'; +import { metricsCommand } from './src/commands/metrics'; import { auditCommand } from './src/commands/audit'; import { doctorCommand } from './src/commands/doctor'; import { migrateCommand } from './src/commands/migrate'; @@ -81,6 +82,7 @@ Wizard.use(basicIntegrationCommand) .use(mcpCommand) .use(mcpAnalyticsCommand) .use(aiObservabilityCommand) + .use(metricsCommand) .use(cliCommand) .use(auditCommand) .use(doctorCommand) diff --git a/src/commands/metrics.ts b/src/commands/metrics.ts new file mode 100644 index 00000000..a889bd24 --- /dev/null +++ b/src/commands/metrics.ts @@ -0,0 +1,16 @@ +import { metricsConfig } from '@lib/programs/metrics/index'; + +import type { Command } from './command'; +import { nativeCommandFactory } from './factories/native-command-factory'; + +/** + * `wizard metrics` — flat skill command, wire PostHog application metrics + * (counters, gauges, histograms via \`posthog.metrics\`) into a project. + * + * The `metrics` context-mill skill has one variant per platform (python, + * nodejs, javascript, kubernetes, other/OTLP); the agent picks the right one + * at run time by scanning the project's manifest and (when ambiguous) asking + * the user via `wizard_ask`. Stays flat while a single "add metrics to a + * project" flow is the only action. + */ +export const metricsCommand: Command = nativeCommandFactory(metricsConfig); diff --git a/src/lib/agent/runner/switchboard/index.ts b/src/lib/agent/runner/switchboard/index.ts index eefec92d..922fd753 100644 --- a/src/lib/agent/runner/switchboard/index.ts +++ b/src/lib/agent/runner/switchboard/index.ts @@ -135,6 +135,7 @@ export const PROGRAM_BINDINGS: Partial> = { 'mcp-remove': DEFAULT_BINDING, 'mcp-tutorial': DEFAULT_BINDING, 'mcp-analytics': DEFAULT_BINDING, + metrics: DEFAULT_BINDING, 'ai-observability': { sequence: Sequence.linear, harness: Harness.anthropic, diff --git a/src/lib/programs/__tests__/metrics-program.test.ts b/src/lib/programs/__tests__/metrics-program.test.ts new file mode 100644 index 00000000..30fcbd21 --- /dev/null +++ b/src/lib/programs/__tests__/metrics-program.test.ts @@ -0,0 +1,64 @@ +import { AGENT_SKILL_STEPS } from '@lib/programs/agent-skill/index'; +import { getProgramConfig, Program } from '@lib/programs/program-registry'; +import { metricsConfig } from '@lib/programs/metrics/index'; +import type { ProgramRun } from '@lib/agent/agent-runner'; +import type { WizardSession } from '@lib/wizard-session'; + +import { metricsCommand } from '../../../commands/metrics'; + +function staticRun(config: typeof metricsConfig): ProgramRun { + if (typeof config.run === 'function') { + throw new Error('expected a static ProgramRun, got a function'); + } + if (!config.run) throw new Error('expected a ProgramRun'); + return config.run; +} + +describe('metrics program', () => { + it('is registered as a flat top-level `metrics` command', () => { + const config = getProgramConfig('metrics'); + expect(config).toBe(metricsConfig); + expect(config.command).toBe('metrics'); + expect(config.parentCommand).toBeUndefined(); + expect(Program.Metrics).toBe('metrics'); + }); + + it('uses the agent-skill steps with a metrics-specific intro', () => { + const [intro, ...rest] = metricsConfig.steps; + expect(intro.id).toBe('intro'); + expect(intro.screenId).toBe('metrics-intro'); + expect(rest).toEqual(AGENT_SKILL_STEPS.slice(1)); + }); + + it('has no fixed skillId — the agent picks the variant from the menu', () => { + const run = staticRun(metricsConfig); + expect(run.skillId).toBeUndefined(); + + const prompt = run.customPrompt?.({} as WizardSession); + expect(prompt).toContain('load_skill_menu'); + expect(prompt).toContain('"metrics"'); + // Every published variant the prompt teaches the agent to choose from. + for (const variant of [ + 'metrics-python', + 'metrics-nodejs', + 'metrics-javascript', + 'metrics-kubernetes', + 'metrics-other', + ]) { + expect(prompt).toContain(variant); + } + }); + + it('points the outro at the metrics docs and report file', () => { + const run = staticRun(metricsConfig); + expect(run.docsUrl).toBe('https://posthog.com/docs/metrics'); + expect(run.reportFile).toBe('posthog-metrics-report.md'); + expect(metricsConfig.reportFile).toBe(run.reportFile); + }); + + it('is exposed as a yargs command via nativeCommandFactory', () => { + expect(metricsCommand.name).toBe('metrics'); + expect(metricsCommand.description).toBe(metricsConfig.description); + expect(typeof metricsCommand.handler).toBe('function'); + }); +}); diff --git a/src/lib/programs/metrics/index.ts b/src/lib/programs/metrics/index.ts new file mode 100644 index 00000000..aa5eb537 --- /dev/null +++ b/src/lib/programs/metrics/index.ts @@ -0,0 +1,70 @@ +import type { ProgramConfig, ProgramStep } from '@lib/programs/program-step'; +import { AGENT_SKILL_STEPS } from '@lib/programs/agent-skill/index'; +import { getContentBlocks } from '@lib/programs/agent-skill/content/index'; + +const METRICS_STEPS: ProgramStep[] = AGENT_SKILL_STEPS.map((step) => + step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, +); + +const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; + +/** + * `wizard metrics` — instrument the project with PostHog application metrics + * (`posthog.metrics` counters, gauges, and histograms). + * + * No `run.skillId`: the context-mill `metrics` group ships one variant per + * platform (python, nodejs, javascript, kubernetes, other/OTLP) and the wizard + * does no platform detection — the agent loads the menu, matches the project's + * manifest, and installs the right variant itself (see `customPrompt`), the + * same shape as `ai-observability`. Stays flat while a single "add metrics to + * a project" flow is the only action. + */ +export const metricsConfig: ProgramConfig = { + command: 'metrics', + description: 'Add PostHog application metrics to your project', + id: 'metrics', + steps: METRICS_STEPS, + reportFile: METRICS_REPORT_FILE, + getContentBlocks, + run: { + integrationLabel: 'metrics', + // No `skillId`: the agent must load the menu and install the right + // variant itself. The prompt below tells it how. + customPrompt: + () => `Instrument this project with PostHog application metrics. + +This flow has no pre-installed skill — you install the right one yourself: + +1. Call \`load_skill_menu\` with \`category: "metrics"\`. The menu is the + source of truth: one variant per platform. + +2. Pick the variant that matches the project: + - Python tooling (\`pyproject.toml\`, \`requirements.txt\`, \`Pipfile\`) → + \`metrics-python\` (needs \`posthog\` >= 7.23.0) + - \`package.json\` with server-side Node code → \`metrics-nodejs\` + (needs \`posthog-node\` >= 5.43.0) + - \`package.json\` that is browser-only → \`metrics-javascript\` + (needs \`posthog-js\` >= 1.399.0) + - Kubernetes manifests / Helm charts and the user wants cluster-level + scraping → \`metrics-kubernetes\` + - Any other language → \`metrics-other\` (plain OTLP exporter) + A full-stack app (e.g. Next.js) usually wants the server variant — metrics + measure service work, not user actions. Genuinely ambiguous → + \`wizard_ask\` with a multi-choice picker. + +3. Call \`install_skill\` with the picked variant id. Then follow that skill's + \`SKILL.md\` and references end-to-end — it covers where to place metrics + (middleware, background jobs, external calls, business commit sites) and + the low-cardinality attribute rules. + +Make only additive changes — reuse an existing PostHog client by adding the +\`metrics\` config to it rather than constructing a second client, and do not +touch existing identify calls, event capture, or dashboards. The final report +is written to ./${METRICS_REPORT_FILE}.`, + successMessage: `Application metrics configured! View the report at ./${METRICS_REPORT_FILE}`, + reportFile: METRICS_REPORT_FILE, + docsUrl: 'https://posthog.com/docs/metrics', + spinnerMessage: 'Setting up application metrics...', + estimatedDurationMinutes: 5, + }, +}; diff --git a/src/lib/programs/program-registry.ts b/src/lib/programs/program-registry.ts index 4c4e416d..5723e71a 100644 --- a/src/lib/programs/program-registry.ts +++ b/src/lib/programs/program-registry.ts @@ -31,6 +31,7 @@ import { } from './mcp/index.js'; import { mcpAnalyticsConfig } from './mcp-analytics/index.js'; import { aiObservabilityConfig } from './ai-observability/index.js'; +import { metricsConfig } from './metrics/index.js'; import { slackConnectConfig } from './slack/index.js'; // Generic skill program — runs an arbitrary context-mill skill chosen at @@ -80,6 +81,7 @@ export const PROGRAM_REGISTRY = [ mcpTutorialConfig, mcpAnalyticsConfig, aiObservabilityConfig, + metricsConfig, slackConnectConfig, ] as const satisfies readonly ProgramConfig[]; @@ -105,6 +107,7 @@ export const Program = { McpTutorial: mcpTutorialConfig.id, McpAnalytics: mcpAnalyticsConfig.id, AiObservability: aiObservabilityConfig.id, + Metrics: metricsConfig.id, SlackConnect: slackConnectConfig.id, } as const; diff --git a/src/ui/tui/screen-registry.tsx b/src/ui/tui/screen-registry.tsx index 380b1629..42765dee 100644 --- a/src/ui/tui/screen-registry.tsx +++ b/src/ui/tui/screen-registry.tsx @@ -29,6 +29,7 @@ import { SourceMapsDetectScreen } from './screens/SourceMapsDetectScreen.js'; import { SourceMapsOutroScreen } from './screens/SourceMapsOutroScreen.js'; import { AgentSkillIntroScreen } from './screens/AgentSkillIntroScreen.js'; import { AiObservabilityIntroScreen } from './screens/AiObservabilityIntroScreen.js'; +import { MetricsIntroScreen } from './screens/MetricsIntroScreen.js'; import { SelfDrivingIntroScreen } from './screens/SelfDrivingIntroScreen.js'; import { SelfDrivingIntegrationCheckScreen } from './screens/SelfDrivingIntegrationCheckScreen.js'; import { SelfDrivingIntegrationDetectScreen } from './screens/SelfDrivingIntegrationDetectScreen.js'; @@ -93,6 +94,7 @@ export function createScreens( [ScreenId.AiObservabilityIntro]: ( ), + [ScreenId.MetricsIntro]: , [ScreenId.SelfDrivingIntro]: , [ScreenId.SelfDrivingIntegrationCheck]: ( diff --git a/src/ui/tui/screen-sequences.ts b/src/ui/tui/screen-sequences.ts index 7f842bfb..8131b3f7 100644 --- a/src/ui/tui/screen-sequences.ts +++ b/src/ui/tui/screen-sequences.ts @@ -25,6 +25,7 @@ export enum ScreenId { MigrationIntro = 'migration-intro', AgentSkillIntro = 'agent-skill-intro', AiObservabilityIntro = 'ai-observability-intro', + MetricsIntro = 'metrics-intro', SelfDrivingIntro = 'self-driving-intro', SelfDrivingIntegrationCheck = 'self-driving-integration-check', SelfDrivingIntegrationDetect = 'self-driving-integration-detect', diff --git a/src/ui/tui/screens/MetricsIntroScreen.tsx b/src/ui/tui/screens/MetricsIntroScreen.tsx new file mode 100644 index 00000000..6019e852 --- /dev/null +++ b/src/ui/tui/screens/MetricsIntroScreen.tsx @@ -0,0 +1,91 @@ +import { Box, Text } from 'ink'; +import { useState, useSyncExternalStore } from 'react'; +import type { WizardStore } from '@ui/tui/store'; +import { IntroScreenLayout } from '@ui/tui/screens/IntroScreenLayout'; +import { + SkillSourceInfo, + useSkillEntry, +} from '@ui/tui/screens/SkillSourceInfo'; + +interface MetricsIntroScreenProps { + store: WizardStore; +} + +export const MetricsIntroScreen = ({ store }: MetricsIntroScreenProps) => { + useSyncExternalStore( + (cb) => store.subscribe(cb), + () => store.getSnapshot(), + ); + + const [showingMoreInfo, setShowingMoreInfo] = useState(false); + const { session } = store; + // metrics picks its skill variant at run time (python, nodejs, javascript, + // kubernetes, other), so there's no pre-seeded skillId here. Fall back to + // the group id for the "more info" lookup. + const skillId = session.skillId ?? 'metrics'; + const { skillEntry, fetchFailed } = useSkillEntry(skillId, session.localMcp); + + const body = showingMoreInfo ? ( + + + + The wizard is an agent that executes PostHog tasks. Its code is open + source: https://github.com/PostHog/wizard + + + + + The{' '} + + metrics + {' '} + program instruments your service with PostHog application metrics — + counters, gauges, and histograms via{' '} + posthog.metrics — at operational choke + points: request middleware, background jobs, external calls, and + business commit sites. Supports Python, Node.js, web JavaScript, + Kubernetes, and any other language via OTLP. + + + + + + ) : ( + + + Let's instrument your service with PostHog application metrics. + + + ); + + const menuOptions = showingMoreInfo + ? [{ label: 'Back', value: 'back' }] + : [ + { label: 'Continue', value: 'continue' }, + { label: 'More info', value: 'more-info' }, + { label: 'Cancel', value: 'cancel' }, + ]; + + const handleSelect = (value: string) => { + if (value === 'cancel') process.exit(0); + else if (value === 'more-info') setShowingMoreInfo(true); + else if (value === 'back') setShowingMoreInfo(false); + else store.completeSetup(); + }; + + return ( + + ); +}; From ee6ab19c9d750b9da22c8abc0bc39d11ce95d36b Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Tue, 18 Aug 2026 14:23:31 -0400 Subject: [PATCH 2/7] fix(metrics): make the metrics intro e2e-drivable Adding a new intro screen requires registering it in two harness allow-lists; without them a headless run stalls on the intro forever (the flow trace repeated metrics-intro for all 40 guard iterations). Registers ScreenId.MetricsIntro in the action registry and decideE2eAction, declares the program's e2e path in test/e2e.json, and adds a metrics flow snapshot so the walkthrough is regression-covered. Pre-existing, unrelated: the action-registry exhaustiveness test still reports `task-notice` as uncovered on main; left alone here. Generated-By: PostHog Desktop Task-Id: 495fbacb-61e8-4fdb-8c0f-7f69cd7a3fd6 --- .../e2e-flow-snapshot.test.ts.snap | 40 +++++++++++++++++++ .../__tests__/e2e-flow-snapshot.test.ts | 23 +++++++++++ e2e-harness/action-registry.ts | 1 + e2e-harness/e2e-profile.ts | 1 + e2e-harness/profiles.ts | 3 ++ src/lib/programs/metrics/test/e2e.json | 35 ++++++++++++++++ 6 files changed, 103 insertions(+) create mode 100644 src/lib/programs/metrics/test/e2e.json diff --git a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap index a6da13c8..f250e82a 100644 --- a/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap +++ b/e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap @@ -40,6 +40,46 @@ exports[`e2e flow snapshot — ai-observability > walks intro → health → aut } `; +exports[`e2e flow snapshot — metrics > walks intro → health → auth → run → outro → skills 1`] = ` +{ + "profile": { + "ask": "first", + "healthCheck": "dismiss", + "mcp": "skip", + "setup": "first", + "skills": "delete", + "slack": "skip", + }, + "program": "metrics", + "trace": [ + { + "action": "confirm_setup", + "screen": "metrics-intro", + }, + { + "action": "dismiss_outage", + "screen": "health-check", + }, + { + "action": "(external)", + "screen": "auth", + }, + { + "action": "(external)", + "screen": "run", + }, + { + "action": "dismiss_outro", + "screen": "outro", + }, + { + "action": "keep_skills", + "screen": "keep-skills", + }, + ], +} +`; + exports[`e2e flow snapshot — posthog-integration > Next.js (with a setup question) walks a stable path 1`] = ` { "profile": { diff --git a/e2e-harness/__tests__/e2e-flow-snapshot.test.ts b/e2e-harness/__tests__/e2e-flow-snapshot.test.ts index 2a38fc68..0cfca999 100644 --- a/e2e-harness/__tests__/e2e-flow-snapshot.test.ts +++ b/e2e-harness/__tests__/e2e-flow-snapshot.test.ts @@ -106,3 +106,26 @@ describe('e2e flow snapshot — ai-observability', () => { }).toMatchSnapshot(); }); }); + +describe('e2e flow snapshot — metrics', () => { + it('walks intro → health → auth → run → outro → skills', () => { + expect({ + program: 'metrics', + profile: profileFor(Program.Metrics), + trace: traceFlow(Integration.javascriptNode, Program.Metrics), + }).toMatchSnapshot(); + }); + + it('reaches a terminal decision instead of stalling on the intro', () => { + const trace = traceFlow(Integration.javascriptNode, Program.Metrics); + // A screen with no `decideE2eAction` case yields `(external)` forever, so + // the guard loop runs its full 40 iterations on one screen. The metrics + // intro must be drivable. + expect(trace[0]).toEqual({ + screen: 'metrics-intro', + action: 'confirm_setup', + }); + expect(trace.at(-1)?.action).toBe('keep_skills'); + expect(trace.length).toBeLessThan(40); + }); +}); diff --git a/e2e-harness/action-registry.ts b/e2e-harness/action-registry.ts index efc62d68..36cc28a8 100644 --- a/e2e-harness/action-registry.ts +++ b/e2e-harness/action-registry.ts @@ -100,6 +100,7 @@ export const ACTION_REGISTRY: Partial> = { [ScreenId.MigrationIntro]: [confirmSetupAction], [ScreenId.AgentSkillIntro]: [confirmSetupAction], [ScreenId.AiObservabilityIntro]: [confirmSetupAction], + [ScreenId.MetricsIntro]: [confirmSetupAction], [ScreenId.AuditIntro]: [confirmSetupAction], [ScreenId.DoctorIntro]: [confirmSetupAction], [ScreenId.WarehouseIntro]: [confirmSetupAction], diff --git a/e2e-harness/e2e-profile.ts b/e2e-harness/e2e-profile.ts index c4d37d21..fd9d5f6a 100644 --- a/e2e-harness/e2e-profile.ts +++ b/e2e-harness/e2e-profile.ts @@ -92,6 +92,7 @@ export function decideE2eAction( case ScreenId.MigrationIntro: case ScreenId.AgentSkillIntro: case ScreenId.AiObservabilityIntro: + case ScreenId.MetricsIntro: case ScreenId.AuditIntro: case ScreenId.SourceMapsIntro: case ScreenId.DoctorIntro: diff --git a/e2e-harness/profiles.ts b/e2e-harness/profiles.ts index 34edd66f..74962c19 100644 --- a/e2e-harness/profiles.ts +++ b/e2e-harness/profiles.ts @@ -17,11 +17,13 @@ import { } from './e2e-profile.js'; import posthogIntegrationE2e from '@lib/programs/posthog-integration/test/e2e.json'; import aiObservabilityE2e from '@lib/programs/ai-observability/test/e2e.json'; +import metricsE2e from '@lib/programs/metrics/test/e2e.json'; const PROFILES: Partial> = { [Program.PostHogIntegration]: posthogIntegrationE2e.profile as WizardE2eProfile, [Program.AiObservability]: aiObservabilityE2e.profile as WizardE2eProfile, + [Program.Metrics]: metricsE2e.profile as WizardE2eProfile, }; const VARIATIONS: Partial> = { @@ -29,6 +31,7 @@ const VARIATIONS: Partial> = { posthogIntegrationE2e.variations as WizardE2eVariation[], [Program.AiObservability]: aiObservabilityE2e.variations as WizardE2eVariation[], + [Program.Metrics]: metricsE2e.variations as WizardE2eVariation[], }; /** The e2e profile for a program, or the happy-path default if none is set. */ diff --git a/src/lib/programs/metrics/test/e2e.json b/src/lib/programs/metrics/test/e2e.json new file mode 100644 index 00000000..fb19ca08 --- /dev/null +++ b/src/lib/programs/metrics/test/e2e.json @@ -0,0 +1,35 @@ +{ + "program": "metrics", + "summary": "Happy path: confirm intro, push past health issues, let the agent pick + install the platform variant and instrument the service, delete installed skills.", + "profile": { + "setup": "first", + "healthCheck": "dismiss", + "mcp": "skip", + "slack": "skip", + "skills": "delete", + "ask": "first" + }, + "variations": [ + { + "name": "default", + "summary": "linear / anthropic / sonnet — parity with main" + } + ], + "path": [ + { "screen": "metrics-intro", "auto": "confirm & continue" }, + { + "screen": "health-check", + "auto": "dismiss outage — proceed even if the readiness probe flags an issue" + }, + { "screen": "auth", "auto": "(external) — the runner injects credentials" }, + { + "screen": "run", + "auto": "(external) — the agent loads the skill menu, picks the metrics-* variant matching the project's platform (wizard_ask answered with the first option if ambiguous), installs it, and instruments the service's choke points" + }, + { "screen": "outro", "auto": "dismiss" }, + { + "screen": "keep-skills", + "auto": "delete — leave nothing behind (terminal: the run's done-signal)" + } + ] +} From eae2c3dce1c4782981765fc4b5da4af0f2f759c6 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" <29069505+gewenyu99@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:02:49 -0400 Subject: [PATCH 3/7] feat(metrics): move the metrics program to the orchestrator sequence (#1129) Co-authored-by: Claude Fable 5 --- .../runner/__tests__/switchboard.test.ts | 18 +++++++- .../__tests__/variant-resolution.test.ts | 35 +++++++++++++++ .../switchboard/flags/__tests__/flags.test.ts | 6 +++ src/lib/agent/runner/switchboard/index.ts | 9 +++- .../__tests__/metrics-program.test.ts | 9 +++- src/lib/programs/metrics/index.ts | 44 +++++++++++++++---- 6 files changed, 107 insertions(+), 14 deletions(-) diff --git a/src/lib/agent/runner/__tests__/switchboard.test.ts b/src/lib/agent/runner/__tests__/switchboard.test.ts index 97655fe0..03fad0bf 100644 --- a/src/lib/agent/runner/__tests__/switchboard.test.ts +++ b/src/lib/agent/runner/__tests__/switchboard.test.ts @@ -61,6 +61,7 @@ describe('switchboard PROGRAM_BINDINGS', () => { it('resolves every program, unflagged, to the same default binding', () => { for (const program of PROGRAM_IDS) { if (program === 'ai-observability') continue; // pinned below + if (program === 'metrics') continue; // pinned below if (program === 'replay-vision') continue; // pinned below expect(resolveBinding({ program, flags: {} })).toEqual(DEFAULT_RESOLVED); } @@ -78,6 +79,17 @@ describe('switchboard PROGRAM_BINDINGS', () => { }, trace: { harness: 'binding', model: 'binding', sequence: 'binding' }, }, + { + name: 'binds metrics to the orchestrator on pi; stage models come from the flow frontmatter', + ctx: { program: 'metrics', flags: {} }, + binding: { + sequence: Sequence.orchestrator, + harness: Harness.pi, + model: DEFAULT_AGENT_MODEL, + thinkingLevel: undefined, + }, + trace: { harness: 'binding', model: 'binding', sequence: 'binding' }, + }, { name: 'binds replay-vision to the orchestrator sequence', ctx: { program: 'replay-vision', flags: {} }, @@ -194,13 +206,15 @@ describe('switchboard composed clamp', () => { trace: {}, }; // The flag routes posthog-integration's harness to pi; the composed - // clamp holds every sequence at linear; other programs keep their - // bindings (sonnet 5 for ai-observability, the default elsewhere). + // clamp holds every sequence at linear — the orchestrator bindings + // (metrics, replay-vision) included; other axes keep their bindings. expect(resolveBinding(ctx)).toEqual( program === 'posthog-integration' ? { ...DEFAULT_RESOLVED, harness: Harness.pi } : program === 'ai-observability' ? { ...DEFAULT_RESOLVED, model: SONNET_5_MODEL } + : program === 'metrics' + ? { ...DEFAULT_RESOLVED, harness: Harness.pi } : DEFAULT_RESOLVED, ); expect(ctx.trace?.sequence).toBe('composed'); diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts index b3f302b3..544cc342 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts @@ -102,6 +102,41 @@ const MENU: SkillEntry[] = [ ...expandBundleEntry(CAPTURE_BUNDLE_ENTRY), ]; +// Pinned from the built metrics menu: one entry per framework, same variant id. +const METRICS_ENTRIES = [ + { id: 'metrics-python', framework: 'python' }, + { id: 'metrics-python', framework: 'django' }, + { id: 'metrics-python', framework: 'flask' }, + { id: 'metrics-python', framework: 'fastapi' }, + { id: 'metrics-nodejs', framework: 'javascript_node' }, + { id: 'metrics-nodejs', framework: 'nextjs' }, + { id: 'metrics-javascript', framework: 'javascript_web' }, +].map( + (e): SkillEntry => ({ + ...e, + group: 'metrics', + name: e.id, + downloadUrl: `https://example.test/${e.id}.zip`, + }), +); + +describe('resolveSkillVariantId — multi-framework platform variants (metrics)', () => { + it('resolves the metrics family through the menu framework entries', () => { + expect(resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'flask')).toBe( + 'metrics-python', + ); + expect(resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'nextjs')).toBe( + 'metrics-nodejs', + ); + expect( + resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'javascript_web'), + ).toBe('metrics-javascript'); + expect( + resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'swift'), + ).toBeUndefined(); + }); +}); + describe('resolveSkillVariantId — menu-declared framework resolution', () => { it('resolves a bare single-variant skill id to itself', () => { expect(resolveSkillVariantId(MENU, 'integration-v2-build', 'django')).toBe( diff --git a/src/lib/agent/runner/switchboard/flags/__tests__/flags.test.ts b/src/lib/agent/runner/switchboard/flags/__tests__/flags.test.ts index 75139146..cdf40854 100644 --- a/src/lib/agent/runner/switchboard/flags/__tests__/flags.test.ts +++ b/src/lib/agent/runner/switchboard/flags/__tests__/flags.test.ts @@ -277,6 +277,12 @@ describe('isolation — everything on at once', () => { ...LINEAR_ANTHROPIC_DEFAULT, model: SONNET_5_MODEL, }); + } else if (program === 'metrics') { + // Orchestrator + pi from its OWN binding, not the flag; stage models + // are pinned context-mill side in the flow frontmatter. + expect(resolved).toEqual({ + ...ORCHESTRATOR_PI_DEFAULT, + }); } else if (program === 'replay-vision') { // Orchestrator from its OWN binding, not the flag — the // wizard-orchestrator experiment does not cover this program, so it diff --git a/src/lib/agent/runner/switchboard/index.ts b/src/lib/agent/runner/switchboard/index.ts index 251769e9..35c232ae 100644 --- a/src/lib/agent/runner/switchboard/index.ts +++ b/src/lib/agent/runner/switchboard/index.ts @@ -135,7 +135,14 @@ export const PROGRAM_BINDINGS: Partial> = { 'mcp-remove': DEFAULT_BINDING, 'mcp-tutorial': DEFAULT_BINDING, 'mcp-analytics': DEFAULT_BINDING, - metrics: DEFAULT_BINDING, + // Orchestrator on pi. The binding routes only; every stage's model and + // effort are pinned context-mill side in the flow's frontmatter + // (`model_pi`/`effort_pi`: terra seed, sol tasks, luna report). + metrics: { + sequence: Sequence.orchestrator, + harness: Harness.pi, + model: DEFAULT_AGENT_MODEL, + }, 'replay-vision': { sequence: Sequence.orchestrator, harness: Harness.anthropic, diff --git a/src/lib/programs/__tests__/metrics-program.test.ts b/src/lib/programs/__tests__/metrics-program.test.ts index 30fcbd21..8ef7197b 100644 --- a/src/lib/programs/__tests__/metrics-program.test.ts +++ b/src/lib/programs/__tests__/metrics-program.test.ts @@ -23,13 +23,18 @@ describe('metrics program', () => { expect(Program.Metrics).toBe('metrics'); }); - it('uses the agent-skill steps with a metrics-specific intro', () => { - const [intro, ...rest] = metricsConfig.steps; + it('detects the framework, then runs the agent-skill steps with a metrics-specific intro', () => { + const [detect, intro, ...rest] = metricsConfig.steps; + expect(detect.id).toBe('detect'); expect(intro.id).toBe('intro'); expect(intro.screenId).toBe('metrics-intro'); expect(rest).toEqual(AGENT_SKILL_STEPS.slice(1)); }); + it('runs the metrics agent flow on the orchestrator', () => { + expect(metricsConfig.agentFlow).toBe('metrics'); + }); + it('has no fixed skillId — the agent picks the variant from the menu', () => { const run = staticRun(metricsConfig); expect(run.skillId).toBeUndefined(); diff --git a/src/lib/programs/metrics/index.ts b/src/lib/programs/metrics/index.ts index aa5eb537..9977c19f 100644 --- a/src/lib/programs/metrics/index.ts +++ b/src/lib/programs/metrics/index.ts @@ -1,10 +1,31 @@ -import type { ProgramConfig, ProgramStep } from '@lib/programs/program-step'; +import type { + ProgramConfig, + ProgramReadyContext, + ProgramStep, +} from '@lib/programs/program-step'; import { AGENT_SKILL_STEPS } from '@lib/programs/agent-skill/index'; import { getContentBlocks } from '@lib/programs/agent-skill/content/index'; +import { detectPostHogIntegration } from '@lib/programs/posthog-integration/detect'; -const METRICS_STEPS: ProgramStep[] = AGENT_SKILL_STEPS.map((step) => - step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, -); +/** + * Framework detection ahead of the run, as on replay-vision: the orchestrator + * resolves each task's `metrics` skill variant against the detected framework + * in preflight, so the session must carry it before the run arm starts. + */ +const DETECT_STEP: ProgramStep = { + id: 'detect', + label: 'Detecting framework', + onReady: async (ctx: ProgramReadyContext) => { + await detectPostHogIntegration(ctx); + }, +}; + +const METRICS_STEPS: ProgramStep[] = [ + DETECT_STEP, + ...AGENT_SKILL_STEPS.map((step) => + step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, + ), +]; const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; @@ -13,16 +34,21 @@ const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; * (`posthog.metrics` counters, gauges, and histograms). * * No `run.skillId`: the context-mill `metrics` group ships one variant per - * platform (python, nodejs, javascript, kubernetes, other/OTLP) and the wizard - * does no platform detection — the agent loads the menu, matches the project's - * manifest, and installs the right variant itself (see `customPrompt`), the - * same shape as `ai-observability`. Stays flat while a single "add metrics to - * a project" flow is the only action. + * platform (python, nodejs, javascript, kubernetes, other/OTLP), and the menu + * maps each detected framework to its platform variant. On the orchestrator + * the runner resolves it from the detect step's framework; on a linear + * (composed) run the agent picks from the menu via `customPrompt`. Stays flat + * while a single "add metrics to a project" flow is the only action. */ export const metricsConfig: ProgramConfig = { command: 'metrics', description: 'Add PostHog application metrics to your project', id: 'metrics', + // Orchestrator flow (context-mill `context/agents/metrics`): the seed queues + // verify-sdk → instrument-metrics → report; each task's `metrics` skill + // resolves to the platform variant through the menu's framework entries. + // Explicit so renaming the program can't silently detach the flow. + agentFlow: 'metrics', steps: METRICS_STEPS, reportFile: METRICS_REPORT_FILE, getContentBlocks, From cc731bed487b8c0b1e46a4fc4dbe288fdb8a1660 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Fri, 21 Aug 2026 16:32:04 -0400 Subject: [PATCH 4/7] =?UTF-8?q?refactor(metrics):=20one=20collapsed=20skil?= =?UTF-8?q?l=20=E2=80=94=20drop=20the=20detect=20step=20and=20variant=20ma?= =?UTF-8?q?chinery=20on=20the=20wizard=20side=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../__tests__/variant-resolution.test.ts | 42 +++------ .../__tests__/metrics-program.test.ts | 24 ++---- src/lib/programs/metrics/index.ts | 86 +++++-------------- 3 files changed, 39 insertions(+), 113 deletions(-) diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts index 544cc342..dbe10739 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts @@ -102,38 +102,22 @@ const MENU: SkillEntry[] = [ ...expandBundleEntry(CAPTURE_BUNDLE_ENTRY), ]; -// Pinned from the built metrics menu: one entry per framework, same variant id. -const METRICS_ENTRIES = [ - { id: 'metrics-python', framework: 'python' }, - { id: 'metrics-python', framework: 'django' }, - { id: 'metrics-python', framework: 'flask' }, - { id: 'metrics-python', framework: 'fastapi' }, - { id: 'metrics-nodejs', framework: 'javascript_node' }, - { id: 'metrics-nodejs', framework: 'nextjs' }, - { id: 'metrics-javascript', framework: 'javascript_web' }, -].map( - (e): SkillEntry => ({ - ...e, - group: 'metrics', - name: e.id, - downloadUrl: `https://example.test/${e.id}.zip`, - }), -); +// Pinned from the built metrics menu: one collapsed skill, exact-id resolution. +const METRICS_ENTRY: SkillEntry = { + id: 'metrics', + group: 'metrics', + name: 'metrics', + downloadUrl: 'https://example.test/metrics.zip', +}; -describe('resolveSkillVariantId — multi-framework platform variants (metrics)', () => { - it('resolves the metrics family through the menu framework entries', () => { - expect(resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'flask')).toBe( - 'metrics-python', +describe('resolveSkillVariantId — collapsed single-skill group (metrics)', () => { + it('resolves by exact menu id, whatever the framework', () => { + expect(resolveSkillVariantId([METRICS_ENTRY], 'metrics', 'flask')).toBe( + 'metrics', ); - expect(resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'nextjs')).toBe( - 'metrics-nodejs', + expect(resolveSkillVariantId([METRICS_ENTRY], 'metrics', undefined)).toBe( + 'metrics', ); - expect( - resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'javascript_web'), - ).toBe('metrics-javascript'); - expect( - resolveSkillVariantId(METRICS_ENTRIES, 'metrics', 'swift'), - ).toBeUndefined(); }); }); diff --git a/src/lib/programs/__tests__/metrics-program.test.ts b/src/lib/programs/__tests__/metrics-program.test.ts index 8ef7197b..0a61ca88 100644 --- a/src/lib/programs/__tests__/metrics-program.test.ts +++ b/src/lib/programs/__tests__/metrics-program.test.ts @@ -23,9 +23,8 @@ describe('metrics program', () => { expect(Program.Metrics).toBe('metrics'); }); - it('detects the framework, then runs the agent-skill steps with a metrics-specific intro', () => { - const [detect, intro, ...rest] = metricsConfig.steps; - expect(detect.id).toBe('detect'); + it('uses the agent-skill steps with a metrics-specific intro', () => { + const [intro, ...rest] = metricsConfig.steps; expect(intro.id).toBe('intro'); expect(intro.screenId).toBe('metrics-intro'); expect(rest).toEqual(AGENT_SKILL_STEPS.slice(1)); @@ -35,23 +34,12 @@ describe('metrics program', () => { expect(metricsConfig.agentFlow).toBe('metrics'); }); - it('has no fixed skillId — the agent picks the variant from the menu', () => { + it('runs the one collapsed metrics skill', () => { const run = staticRun(metricsConfig); - expect(run.skillId).toBeUndefined(); - + expect(run.skillId).toBe('metrics'); const prompt = run.customPrompt?.({} as WizardSession); - expect(prompt).toContain('load_skill_menu'); - expect(prompt).toContain('"metrics"'); - // Every published variant the prompt teaches the agent to choose from. - for (const variant of [ - 'metrics-python', - 'metrics-nodejs', - 'metrics-javascript', - 'metrics-kubernetes', - 'metrics-other', - ]) { - expect(prompt).toContain(variant); - } + expect(prompt).toContain('metrics'); + expect(prompt).toContain('installation reference'); }); it('points the outro at the metrics docs and report file', () => { diff --git a/src/lib/programs/metrics/index.ts b/src/lib/programs/metrics/index.ts index 9977c19f..7410e5e5 100644 --- a/src/lib/programs/metrics/index.ts +++ b/src/lib/programs/metrics/index.ts @@ -1,31 +1,10 @@ -import type { - ProgramConfig, - ProgramReadyContext, - ProgramStep, -} from '@lib/programs/program-step'; +import type { ProgramConfig, ProgramStep } from '@lib/programs/program-step'; import { AGENT_SKILL_STEPS } from '@lib/programs/agent-skill/index'; import { getContentBlocks } from '@lib/programs/agent-skill/content/index'; -import { detectPostHogIntegration } from '@lib/programs/posthog-integration/detect'; -/** - * Framework detection ahead of the run, as on replay-vision: the orchestrator - * resolves each task's `metrics` skill variant against the detected framework - * in preflight, so the session must carry it before the run arm starts. - */ -const DETECT_STEP: ProgramStep = { - id: 'detect', - label: 'Detecting framework', - onReady: async (ctx: ProgramReadyContext) => { - await detectPostHogIntegration(ctx); - }, -}; - -const METRICS_STEPS: ProgramStep[] = [ - DETECT_STEP, - ...AGENT_SKILL_STEPS.map((step) => - step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, - ), -]; +const METRICS_STEPS: ProgramStep[] = AGENT_SKILL_STEPS.map((step) => + step.id === 'intro' ? { ...step, screenId: 'metrics-intro' } : step, +); const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; @@ -33,60 +12,35 @@ const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; * `wizard metrics` — instrument the project with PostHog application metrics * (`posthog.metrics` counters, gauges, and histograms). * - * No `run.skillId`: the context-mill `metrics` group ships one variant per - * platform (python, nodejs, javascript, kubernetes, other/OTLP), and the menu - * maps each detected framework to its platform variant. On the orchestrator - * the runner resolves it from the detect step's framework; on a linear - * (composed) run the agent picks from the menu via `customPrompt`. Stays flat - * while a single "add metrics to a project" flow is the only action. + * One `metrics` skill for every platform: its reference files carry the + * per-platform installation docs and the agent reads the one matching the + * project. Tasks declare `skills: [metrics]` and resolve by exact menu id — + * no framework detection, no variant machinery. Stays flat while a single + * "add metrics to a project" flow is the only action. */ export const metricsConfig: ProgramConfig = { command: 'metrics', description: 'Add PostHog application metrics to your project', id: 'metrics', // Orchestrator flow (context-mill `context/agents/metrics`): the seed queues - // verify-sdk → instrument-metrics → report; each task's `metrics` skill - // resolves to the platform variant through the menu's framework entries. - // Explicit so renaming the program can't silently detach the flow. + // verify-sdk → instrument-metrics → report. Explicit so renaming the + // program can't silently detach the flow. agentFlow: 'metrics', steps: METRICS_STEPS, reportFile: METRICS_REPORT_FILE, getContentBlocks, run: { integrationLabel: 'metrics', - // No `skillId`: the agent must load the menu and install the right - // variant itself. The prompt below tells it how. + skillId: 'metrics', customPrompt: - () => `Instrument this project with PostHog application metrics. - -This flow has no pre-installed skill — you install the right one yourself: - -1. Call \`load_skill_menu\` with \`category: "metrics"\`. The menu is the - source of truth: one variant per platform. - -2. Pick the variant that matches the project: - - Python tooling (\`pyproject.toml\`, \`requirements.txt\`, \`Pipfile\`) → - \`metrics-python\` (needs \`posthog\` >= 7.23.0) - - \`package.json\` with server-side Node code → \`metrics-nodejs\` - (needs \`posthog-node\` >= 5.43.0) - - \`package.json\` that is browser-only → \`metrics-javascript\` - (needs \`posthog-js\` >= 1.399.0) - - Kubernetes manifests / Helm charts and the user wants cluster-level - scraping → \`metrics-kubernetes\` - - Any other language → \`metrics-other\` (plain OTLP exporter) - A full-stack app (e.g. Next.js) usually wants the server variant — metrics - measure service work, not user actions. Genuinely ambiguous → - \`wizard_ask\` with a multi-choice picker. - -3. Call \`install_skill\` with the picked variant id. Then follow that skill's - \`SKILL.md\` and references end-to-end — it covers where to place metrics - (middleware, background jobs, external calls, business commit sites) and - the low-cardinality attribute rules. - -Make only additive changes — reuse an existing PostHog client by adding the -\`metrics\` config to it rather than constructing a second client, and do not -touch existing identify calls, event capture, or dashboards. The final report -is written to ./${METRICS_REPORT_FILE}.`, + () => `Instrument this project with PostHog application metrics by +running the \`metrics\` skill end-to-end. Identify the platform from the +project's manifest and read exactly the matching installation reference — the +skill's SKILL.md explains how to choose. Make only additive changes — reuse an +existing PostHog client by adding the \`metrics\` config to it rather than +constructing a second client, and do not touch existing identify calls, event +capture, or dashboards. The final report is written to +./${METRICS_REPORT_FILE}.`, successMessage: `Application metrics configured! View the report at ./${METRICS_REPORT_FILE}`, reportFile: METRICS_REPORT_FILE, docsUrl: 'https://posthog.com/docs/metrics', From 46d6332ed0a7fbab420d2096a2ebc6b6937a6d79 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Fri, 21 Aug 2026 16:41:38 -0400 Subject: [PATCH 5/7] =?UTF-8?q?revert(metrics):=20back=20to=20the=20five?= =?UTF-8?q?=20platform=20variants=20=E2=80=94=20tasks=20and=20the=20linear?= =?UTF-8?q?=20prompt=20pull=20the=20matching=20one=20themselves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../__tests__/agent-prompt-loader.test.ts | 12 ++++ src/lib/agent/agent-prompt-loader.ts | 6 ++ .../__tests__/variant-resolution.test.ts | 19 ------- .../__tests__/metrics-program.test.ts | 19 +++++-- src/lib/programs/metrics/index.ts | 56 +++++++++++++------ 5 files changed, 73 insertions(+), 39 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index 683ef2b3..9ece1795 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -192,6 +192,18 @@ describe('agentRunTools', () => { 'mcp__wizard-tools__wizard_ask', ]); }); + + it('qualifies the skill-menu tools against the wizard-tools server', () => { + const p = parseAgentPrompt( + '---\nallowedTools: [Read, load_skill_menu, install_skill]\n---\nx', + 't', + ); + expect(agentRunTools(p).allowedTools).toEqual([ + 'Read', + 'mcp__wizard-tools__load_skill_menu', + 'mcp__wizard-tools__install_skill', + ]); + }); }); describe('buildRegistry', () => { diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index ed3de2ff..576dcc2b 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -165,6 +165,10 @@ const ORCHESTRATOR_TOOLS = new Set([ /** The one tool that stops a task until a person answers. Named short in frontmatter. */ export const ASK_TOOL = 'wizard_ask'; +/** The skill-menu tools, as agents ask for them in frontmatter. */ +const SKILL_MENU_TOOL = 'load_skill_menu'; +const INSTALL_SKILL_TOOL = 'install_skill'; + /** * The PostHog MCP, as an agent asks for it in frontmatter. Every tool a task * gets is granted by its own prompt, this one included: a task that never names @@ -309,6 +313,8 @@ interface AgentMenu { /** A native tool passes through; an MCP tool gets its fully-qualified name. */ function expandToolName(name: string): string { if (name === ASK_TOOL) return WIZARD_TOOL_NAMES.wizardAsk; + if (name === SKILL_MENU_TOOL) return WIZARD_TOOL_NAMES.loadSkillMenu; + if (name === INSTALL_SKILL_TOOL) return WIZARD_TOOL_NAMES.installSkill; if (name === POSTHOG_MCP_TOOL) return POSTHOG_MCP_SDK_TOOL; return ORCHESTRATOR_TOOLS.has(name) ? `${ORCHESTRATOR_TOOL_PREFIX}${name}` diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts index dbe10739..b3f302b3 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts @@ -102,25 +102,6 @@ const MENU: SkillEntry[] = [ ...expandBundleEntry(CAPTURE_BUNDLE_ENTRY), ]; -// Pinned from the built metrics menu: one collapsed skill, exact-id resolution. -const METRICS_ENTRY: SkillEntry = { - id: 'metrics', - group: 'metrics', - name: 'metrics', - downloadUrl: 'https://example.test/metrics.zip', -}; - -describe('resolveSkillVariantId — collapsed single-skill group (metrics)', () => { - it('resolves by exact menu id, whatever the framework', () => { - expect(resolveSkillVariantId([METRICS_ENTRY], 'metrics', 'flask')).toBe( - 'metrics', - ); - expect(resolveSkillVariantId([METRICS_ENTRY], 'metrics', undefined)).toBe( - 'metrics', - ); - }); -}); - describe('resolveSkillVariantId — menu-declared framework resolution', () => { it('resolves a bare single-variant skill id to itself', () => { expect(resolveSkillVariantId(MENU, 'integration-v2-build', 'django')).toBe( diff --git a/src/lib/programs/__tests__/metrics-program.test.ts b/src/lib/programs/__tests__/metrics-program.test.ts index 0a61ca88..27dbd898 100644 --- a/src/lib/programs/__tests__/metrics-program.test.ts +++ b/src/lib/programs/__tests__/metrics-program.test.ts @@ -34,12 +34,23 @@ describe('metrics program', () => { expect(metricsConfig.agentFlow).toBe('metrics'); }); - it('runs the one collapsed metrics skill', () => { + it('has no fixed skillId — the agent picks the variant from the menu', () => { const run = staticRun(metricsConfig); - expect(run.skillId).toBe('metrics'); + expect(run.skillId).toBeUndefined(); + const prompt = run.customPrompt?.({} as WizardSession); - expect(prompt).toContain('metrics'); - expect(prompt).toContain('installation reference'); + expect(prompt).toContain('load_skill_menu'); + expect(prompt).toContain('"metrics"'); + // Every published variant the prompt teaches the agent to choose from. + for (const variant of [ + 'metrics-python', + 'metrics-nodejs', + 'metrics-javascript', + 'metrics-kubernetes', + 'metrics-other', + ]) { + expect(prompt).toContain(variant); + } }); it('points the outro at the metrics docs and report file', () => { diff --git a/src/lib/programs/metrics/index.ts b/src/lib/programs/metrics/index.ts index 7410e5e5..859eb6d1 100644 --- a/src/lib/programs/metrics/index.ts +++ b/src/lib/programs/metrics/index.ts @@ -12,35 +12,59 @@ const METRICS_REPORT_FILE = 'posthog-metrics-report.md'; * `wizard metrics` — instrument the project with PostHog application metrics * (`posthog.metrics` counters, gauges, and histograms). * - * One `metrics` skill for every platform: its reference files carry the - * per-platform installation docs and the agent reads the one matching the - * project. Tasks declare `skills: [metrics]` and resolve by exact menu id — - * no framework detection, no variant machinery. Stays flat while a single - * "add metrics to a project" flow is the only action. + * No `run.skillId`: the context-mill `metrics` group ships one variant per + * platform (python, nodejs, javascript, kubernetes, other/OTLP) and the agent + * pulls the matching one itself — the flow's tasks and the linear + * `customPrompt` both pick from the menu and install it. Stays flat while a + * single "add metrics to a project" flow is the only action. */ export const metricsConfig: ProgramConfig = { command: 'metrics', description: 'Add PostHog application metrics to your project', id: 'metrics', // Orchestrator flow (context-mill `context/agents/metrics`): the seed queues - // verify-sdk → instrument-metrics → report. Explicit so renaming the - // program can't silently detach the flow. + // verify-sdk → instrument-metrics → report; the tasks install the matching + // platform variant themselves. Explicit so renaming the program can't + // silently detach the flow. agentFlow: 'metrics', steps: METRICS_STEPS, reportFile: METRICS_REPORT_FILE, getContentBlocks, run: { integrationLabel: 'metrics', - skillId: 'metrics', + // No `skillId`: the agent must load the menu and install the right + // variant itself. The prompt below tells it how. customPrompt: - () => `Instrument this project with PostHog application metrics by -running the \`metrics\` skill end-to-end. Identify the platform from the -project's manifest and read exactly the matching installation reference — the -skill's SKILL.md explains how to choose. Make only additive changes — reuse an -existing PostHog client by adding the \`metrics\` config to it rather than -constructing a second client, and do not touch existing identify calls, event -capture, or dashboards. The final report is written to -./${METRICS_REPORT_FILE}.`, + () => `Instrument this project with PostHog application metrics. + +This flow has no pre-installed skill — you install the right one yourself: + +1. Call \`load_skill_menu\` with \`category: "metrics"\`. The menu is the + source of truth: one variant per platform. + +2. Pick the variant that matches the project: + - Python tooling (\`pyproject.toml\`, \`requirements.txt\`, \`Pipfile\`) → + \`metrics-python\` (needs \`posthog\` >= 7.23.0) + - \`package.json\` with server-side Node code → \`metrics-nodejs\` + (needs \`posthog-node\` >= 5.43.0) + - \`package.json\` that is browser-only → \`metrics-javascript\` + (needs \`posthog-js\` >= 1.399.0) + - Kubernetes manifests / Helm charts and the user wants cluster-level + scraping → \`metrics-kubernetes\` + - Any other language → \`metrics-other\` (plain OTLP exporter) + A full-stack app (e.g. Next.js) usually wants the server variant — metrics + measure service work, not user actions. Genuinely ambiguous → + \`wizard_ask\` with a multi-choice picker. + +3. Call \`install_skill\` with the picked variant id. Then follow that skill's + \`SKILL.md\` and references end-to-end — it covers where to place metrics + (middleware, background jobs, external calls, business commit sites) and + the low-cardinality attribute rules. + +Make only additive changes — reuse an existing PostHog client by adding the +\`metrics\` config to it rather than constructing a second client, and do not +touch existing identify calls, event capture, or dashboards. The final report +is written to ./${METRICS_REPORT_FILE}.`, successMessage: `Application metrics configured! View the report at ./${METRICS_REPORT_FILE}`, reportFile: METRICS_REPORT_FILE, docsUrl: 'https://posthog.com/docs/metrics', From 6f2d3521890d05291385043c13e62cfa06b1c355 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Fri, 21 Aug 2026 17:04:10 -0400 Subject: [PATCH 6/7] fix(pi): grant the skill-menu tool pair to tasks whose frontmatter allows them Co-Authored-By: Claude Fable 5 --- .../harness/pi/__tests__/task-tools.test.ts | 14 ++++++++++++++ src/lib/agent/runner/harness/pi/task.ts | 18 ++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/lib/agent/runner/harness/pi/__tests__/task-tools.test.ts b/src/lib/agent/runner/harness/pi/__tests__/task-tools.test.ts index 7dafae8b..4a19bac3 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/task-tools.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/task-tools.test.ts @@ -35,6 +35,20 @@ describe('allowedPiWizardTools', () => { it('withholds wizard_ask when a task states no tools at all', () => { expect(allowedPiWizardTools(undefined).has('wizard_ask')).toBe(false); }); + + it('grants the skill-menu pair only to a task whose prompt allows them', () => { + const granted = allowedPiWizardTools([ + 'Read', + 'load_skill_menu', + 'install_skill', + ]); + expect(granted.has('load_skill_menu')).toBe(true); + expect(granted.has('install_skill')).toBe(true); + + const withheld = allowedPiWizardTools(['Read', 'Edit']); + expect(withheld.has('load_skill_menu')).toBe(false); + expect(withheld.has('install_skill')).toBe(false); + }); }); describe('allowedPiCodingTools', () => { diff --git a/src/lib/agent/runner/harness/pi/task.ts b/src/lib/agent/runner/harness/pi/task.ts index 056f411f..0e9dcd60 100644 --- a/src/lib/agent/runner/harness/pi/task.ts +++ b/src/lib/agent/runner/harness/pi/task.ts @@ -89,9 +89,10 @@ export function allowedOrchestratorTools( /** * The wizard tools a task gets. Four are always on — their handlers are fenced - * and the coding tasks depend on them. `wizard_ask` is opt-in per task: it - * stops the run until a person answers, so only a task whose prompt asks for it - * may open that overlay. + * and the coding tasks depend on them. The rest are opt-in per task through + * its frontmatter: `wizard_ask` stops the run until a person answers, and the + * skill-menu pair (`load_skill_menu`, `install_skill`) lets a task pull its + * own skill variant. */ const ALWAYS_ON_WIZARD_TOOLS = [ 'check_env_keys', @@ -100,15 +101,16 @@ const ALWAYS_ON_WIZARD_TOOLS = [ 'publish_handoff', ]; +const OPT_IN_WIZARD_TOOLS = ['wizard_ask', 'load_skill_menu', 'install_skill']; + export function allowedPiWizardTools( allowedTools: readonly string[] | undefined, ): Set { const allowed = (allowedTools ?? []).map(shortToolName); - return new Set( - allowed.includes('wizard_ask') - ? [...ALWAYS_ON_WIZARD_TOOLS, 'wizard_ask'] - : ALWAYS_ON_WIZARD_TOOLS, - ); + return new Set([ + ...ALWAYS_ON_WIZARD_TOOLS, + ...OPT_IN_WIZARD_TOOLS.filter((tool) => allowed.includes(tool)), + ]); } /** From c5d6f23bc216bfc467cc44c0e7e4fbdff3b1b80a Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Fri, 21 Aug 2026 18:07:09 -0400 Subject: [PATCH 7/7] chore: metrics program owned by @PostHog/apm Co-Authored-By: Claude Fable 5 --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d598b25b..164f6823 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,7 @@ /src/lib/programs/ai-observability/ @PostHog/team-ai-observability /src/lib/programs/error-tracking-upload-source-maps/ @PostHog/team-error-tracking /src/lib/programs/mcp-analytics/ @PostHog/team-mcp-analytics +/src/lib/programs/metrics/ @PostHog/apm /src/lib/programs/posthog-integration/ @PostHog/team-wizard-docs /src/lib/programs/replay-vision/ @PostHog/team-replay /src/lib/programs/revenue-analytics/ @PostHog/team-web-analytics