diff --git a/.ai/typed-data-access.adf b/.ai/typed-data-access.adf index e610144..06564fa 100644 --- a/.ai/typed-data-access.adf +++ b/.ai/typed-data-access.adf @@ -3,47 +3,35 @@ ADF: 0.1 🎯 TASK: Typed data access and ontology enforcement policy 📋 CONTEXT: - - Business concepts (tenant, user, subscription, quota, credit, mrr, etc.) are defined in a canonical data registry — the single source of truth for ownership, sensitivity, and access shape across the ecosystem - - Reference registry location: Stackbilt-dev/stackbilt_llc/policies/data-registry.yaml (22+ concepts, 6 sensitivity tiers) - - Each concept declares: owner service, D1 table, sensitivity tier, definition, aliases, rpc_method, mcp_tool - - Consumer services derive their KNOWN_CONCEPTS and alias maps from the registry at build time (see aegis-web/src/lib/data-registry.ts pattern — compiled-const snapshot) - - AEGIS disambiguation firewall halts on undefined concepts rather than guessing - - CodeBeast DATA_AUTHORITY sensitivity class escalates raw D1 access to owned tables - - This charter policy module ties the three mechanisms together: registry → enforcement → disambiguation + - Business concepts (tenant, user, subscription, quota, credit, etc.) are defined in a canonical data registry — the single source of truth for ownership, sensitivity, and access shape across services + - Each concept declares: owner service, storage table, sensitivity tier, definition, aliases, and the accessor (RPC method, tool, or API) non-owning services must use + - Consumer services derive their known-concepts and alias maps from the registry at build time (a compiled-const snapshot pattern, not a runtime fetch) + - A disambiguation step halts on undefined concepts rather than guessing shape, ownership, or sensitivity + - This module ties together three generic mechanisms: registry → access enforcement → disambiguation 🔐 SENSITIVITY TIERS [load-bearing]: - public — readable from any service, no auth required (e.g., blog_post) - - service_internal — readable/writable only by the owning service, raw D1 access is fine within the owner (e.g., conversation, memory, llm_trace) - - cross_service_rpc — accessible via declared RPC method or Service Binding, never raw D1 from a non-owning service (e.g., tenant, quota, generation) - - pii_scoped — accessible only via owning service + audit_log entry required at the call site (e.g., user) - - billing_critical — writable only by the owning service plus the Stripe webhook handler; never leaves the owning service boundary even via RPC (e.g., subscription, mrr) + - service_internal — readable/writable only by the owning service, raw storage access is fine within the owner (e.g., conversation, memory, log) + - cross_service_rpc — accessible via a declared RPC method or service binding, never raw storage access from a non-owning service (e.g., tenant, quota, generation) + - pii_scoped — accessible only via the owning service + an audit-log entry required at the call site (e.g., user) + - billing_critical — writable only by the owning service plus its designated payment-webhook handler; never leaves the owning service boundary even via RPC (e.g., subscription, invoice) - secrets — never leaves the owning service boundary under any circumstance (e.g., api_key) ⚠ CONSTRAINTS [load-bearing]: - New code referencing a business concept MUST check the canonical registry first; terms not in the registry or its aliases MUST be added before the code lands - - Non-owning services reading or writing `cross_service_rpc` concepts MUST use the declared rpc_method or mcp_tool — raw D1 binding access to another service's table is a DATA_AUTHORITY violation - - `pii_scoped` access requires an audit_log entry at the call site — no silent reads + - Non-owning services reading or writing `cross_service_rpc` concepts MUST use the declared accessor — raw storage access to another service's table is a violation + - `pii_scoped` access requires an audit-log entry at the call site — no silent reads - `billing_critical` and `secrets` tiers NEVER cross the owning service boundary, even via RPC — treat as in-process only - When encountering an undefined data concept in requirements, tasks, or user prompts, HALT and ask for clarification rather than guessing shape, ownership, or sensitivity - Aliases (e.g., "credits" for "quota") are semantically equivalent; prefer the canonical form in new code, accept aliases in user-facing copy - Registry updates MUST come before consumer code updates — the source of truth leads, consumers follow - - When promoting a concept to a higher sensitivity tier (e.g., service_internal → cross_service_rpc), all existing consumers of raw D1 access to that concept must migrate to RPC in the same change set + - When promoting a concept to a higher sensitivity tier (e.g., service_internal → cross_service_rpc), all existing consumers of raw storage access to that concept must migrate to the declared accessor in the same change set 📖 ADVISORY: - Check the registry before reaching for a new type definition: the concept may already exist with a canonical shape - - Use `charter surface --format json` to discover what D1 tables a service currently exposes — cross-reference against registry ownership + - Use `charter surface --format json` to discover what tables/resources a service currently exposes — cross-reference against registry ownership - Unregistered-concept warnings from `charter validate` are early signals that ontology drift is beginning; address them immediately - - The disambiguation protocol is load-bearing for autonomous agents (AEGIS, cc-taskrunner, etc.) — these systems cannot safely guess business term semantics, and ambiguity compounds across sessions + - The disambiguation step is load-bearing for autonomous agents — they cannot safely guess business-term semantics, and ambiguity compounds across sessions 📊 METRICS: - REGISTRY_PATH: stackbilt_llc/policies/data-registry.yaml - REGISTRY_REPO: Stackbilt-dev/stackbilt_llc SENSITIVITY_TIERS: 6 - DOCUMENTED_CONCEPTS: 22 - CONSUMER_PATTERN: web/src/lib/data-registry.ts (compile-from-yaml snapshot) - -🔗 REFERENCES: - - Stackbilt-dev/charter#69 — typed data access policy umbrella issue - - codebeast#9 — DATA_AUTHORITY sensitivity class (enforcement side) - - Stackbilt-dev/aegis#344 — disambiguation firewall (runtime halt mechanism) - - Stackbilt-dev/aegis#334 — adversarial reasoning (complementary quality layer) diff --git a/packages/adf/package.json b/packages/adf/package.json index 27c0cf9..d5ec77d 100644 --- a/packages/adf/package.json +++ b/packages/adf/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/adf", "sideEffects": false, - "version": "1.7.0", + "version": "1.8.0", "description": "ADF (Attention-Directed Format) — AST-backed context format for AI agents", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/adf/src/bundler.ts b/packages/adf/src/bundler.ts index eebcaf7..382acda 100644 --- a/packages/adf/src/bundler.ts +++ b/packages/adf/src/bundler.ts @@ -17,7 +17,7 @@ import { parseManifest, resolveModules, buildTriggerReport } from './manifest'; import { mergeDocuments, estimateTokens } from './merger'; // Re-export for backward compatibility -export { parseManifest, resolveModules } from './manifest'; +export { parseManifest, resolveModules, buildTriggerReport, formatTriggerEntry } from './manifest'; // ============================================================================ // Bundle Orchestration diff --git a/packages/adf/src/index.ts b/packages/adf/src/index.ts index 4990ee4..e5b3f91 100644 --- a/packages/adf/src/index.ts +++ b/packages/adf/src/index.ts @@ -1,7 +1,7 @@ export { parseAdf } from './parser'; export { formatAdf } from './formatter'; export { applyPatches } from './patcher'; -export { parseManifest, resolveModules, bundleModules } from './bundler'; +export { parseManifest, resolveModules, buildTriggerReport, formatTriggerEntry, bundleModules } from './bundler'; export { validateConstraints, computeWeightSummary } from './validator'; export { evaluateLocBudgets, resolveBudgetStatus, matchPath } from './loc-budget'; export { evaluateEvidence } from './evidence'; diff --git a/packages/adf/src/manifest.ts b/packages/adf/src/manifest.ts index 647aa70..b62f63c 100644 --- a/packages/adf/src/manifest.ts +++ b/packages/adf/src/manifest.ts @@ -143,6 +143,20 @@ function parseTriggerEntry(entry: string): ManifestModule { return mod; } +/** + * Format a ManifestModule back into an ON_DEMAND bullet string — the + * inverse of parseTriggerEntry. Single source of truth for the bullet + * format so writers (e.g. `adf suggest --emit-ops`) can't drift from what + * parseTriggerEntry actually accepts. + */ +export function formatTriggerEntry(mod: ManifestModule): string { + const budgetSuffix = mod.tokenBudget !== undefined ? ` [budget: ${mod.tokenBudget}]` : ''; + if (mod.triggers.length === 0) { + return `${mod.path}${budgetSuffix}`; + } + return `${mod.path} (Triggers on: ${mod.triggers.join(', ')})${budgetSuffix}`; +} + // ============================================================================ // Module Resolution // ============================================================================ diff --git a/packages/cli/package.json b/packages/cli/package.json index 14b68d4..ff5312b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/cli", "sideEffects": false, - "version": "1.7.0", + "version": "1.8.0", "description": "Charter CLI — repo-level governance toolkit", "bin": { "charter": "./dist/bin.js" diff --git a/packages/cli/src/__tests__/adf-suggest.test.ts b/packages/cli/src/__tests__/adf-suggest.test.ts new file mode 100644 index 0000000..04c3cd2 --- /dev/null +++ b/packages/cli/src/__tests__/adf-suggest.test.ts @@ -0,0 +1,610 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CLIOptions } from '../index'; +import { adfSuggestCommand, buildSuggestReport } from '../commands/adf-suggest'; +import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; + +const originalCwd = process.cwd(); +const tempDirs: string[] = []; + +afterEach(() => { + process.chdir(originalCwd); + while (tempDirs.length > 0) { + const dir = tempDirs.pop(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + } + vi.restoreAllMocks(); +}); + +function tmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'charter-adf-suggest-test-')); + tempDirs.push(dir); + return dir; +} + +function resolutionEvent(overrides: Partial = {}): AdfResolutionEvent { + return { + version: 1, + eventType: 'adf.resolution', + timestamp: new Date().toISOString(), + sessionId: null, + resolutionId: 'r-1', + source: 'bundle', + keywords: [], + candidateModules: [], + resolvedModules: [], + triggerMatches: [], + ...overrides, + }; +} + +function commandEvent(overrides: Partial = {}): CliTelemetryEvent { + return { + version: 1, + eventType: 'command', + timestamp: new Date().toISOString(), + commandPath: 'validate', + flags: [], + format: 'json', + ciMode: true, + durationMs: 10, + exitCode: 0, + success: true, + ...overrides, + }; +} + +describe('buildSuggestReport', () => { + it('flags a dead module once it appears enough times with zero matches', () => { + const events = Array.from({ length: 3 }, () => + resolutionEvent({ + triggerMatches: [{ module: 'classifier.adf', trigger: 'classify', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.deadModules).toEqual([{ module: 'classifier.adf', occurrences: 3 }]); + }); + + it('does not flag a module below the occurrence threshold', () => { + const events = [ + resolutionEvent({ + triggerMatches: [{ module: 'classifier.adf', trigger: 'classify', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }), + ]; + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.deadModules).toHaveLength(0); + }); + + it('does not flag a module that matched at least once', () => { + const events = Array.from({ length: 4 }, (_, i) => + resolutionEvent({ + triggerMatches: [ + { module: 'governance.adf', trigger: 'audit', matched: i === 0, matchedKeywords: i === 0 ? ['audit'] : [], loadReason: 'trigger' }, + ], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.deadModules).toHaveLength(0); + }); + + it('reports a recurring keyword that never matches any trigger', () => { + const events = Array.from({ length: 3 }, () => + resolutionEvent({ + keywords: ['widget'], + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords).toEqual([{ keyword: 'widget', occurrences: 3, correlatedFailures: 0 }]); + }); + + it('filters generic stopwords out of the near-miss keyword report', () => { + const events = Array.from({ length: 4 }, () => + resolutionEvent({ keywords: ['the', 'widget'], triggerMatches: [] }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords.map((f) => f.keyword)).toEqual(['widget']); + }); + + it('excludes a keyword that matched a trigger at least once, even if unmatched elsewhere', () => { + const events = [ + resolutionEvent({ keywords: ['audit'], triggerMatches: [{ module: 'governance.adf', trigger: 'audit', matched: true, matchedKeywords: ['audit'], loadReason: 'trigger' }] }), + resolutionEvent({ keywords: ['audit'], triggerMatches: [] }), + resolutionEvent({ keywords: ['audit'], triggerMatches: [] }), + ]; + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords.find((f) => f.keyword === 'audit')).toBeUndefined(); + }); + + it('joins a resolution to a same-session failing command and counts it as a correlated failure', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + sessionId: 'sess-1', + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ sessionId: 'sess-1', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords).toEqual([{ keyword: 'widget', occurrences: 3, correlatedFailures: 3 }]); + }); + + it('joins by time window when no sessionId is present on either side', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ success: false, exitCode: 1, timestamp: new Date(now.getTime() + 5 * 60_000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords[0].correlatedFailures).toBe(3); + }); + + it('does not join when both sides have sessionIds but they differ, even within the time window', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + sessionId: 'sess-A', + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ sessionId: 'sess-B', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords[0].correlatedFailures).toBe(0); + }); + + it('falls through to the time window when only one side has a sessionId', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + sessionId: null, + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ sessionId: 'sess-only-here', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords[0].correlatedFailures).toBe(3); + }); + + it('respects a widened --window-minutes for the time-window fallback', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ success: false, exitCode: 1, timestamp: new Date(now.getTime() + 120 * 60_000).toISOString() })]; + const reportNarrow = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(reportNarrow.nearMissKeywords[0].correlatedFailures).toBe(0); + const reportWide = buildSuggestReport(events, cmds, 3, 180, 'events.ndjson'); + expect(reportWide.nearMissKeywords[0].correlatedFailures).toBe(3); + }); + + it('normalizes keyword case so a capitalized keyword matches its lowercase trigger match', () => { + const events = [ + resolutionEvent({ keywords: ['Audit'], triggerMatches: [{ module: 'governance.adf', trigger: 'audit', matched: true, matchedKeywords: ['audit'], loadReason: 'trigger' }] }), + resolutionEvent({ keywords: ['Audit'], triggerMatches: [] }), + resolutionEvent({ keywords: ['audit'], triggerMatches: [] }), + ]; + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords.find((f) => f.keyword === 'audit' || f.keyword === 'Audit')).toBeUndefined(); + }); + + it('does not join a command event that falls outside the time window', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ success: false, exitCode: 1, timestamp: new Date(now.getTime() + 120 * 60_000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords[0].correlatedFailures).toBe(0); + }); + + it('does not join a command event that precedes the resolution event', () => { + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const events = [res, res, res]; + const cmds = [commandEvent({ success: false, exitCode: 1, timestamp: new Date(now.getTime() - 1000).toISOString() })]; + const report = buildSuggestReport(events, cmds, 3, 60, 'events.ndjson'); + expect(report.nearMissKeywords[0].correlatedFailures).toBe(0); + }); + + it('reports loaded-but-violated for a module that resolved yet a downstream command failed', () => { + const now = new Date(); + const res = resolutionEvent({ + sessionId: 'sess-2', + timestamp: now.toISOString(), + resolvedModules: ['governance.adf'], + triggerMatches: [], + }); + const cmds = [commandEvent({ sessionId: 'sess-2', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; + const report = buildSuggestReport([res], cmds, 1, 60, 'events.ndjson'); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 1 }]); + }); + + it('does not report loaded-but-violated when nothing downstream failed', () => { + const res = resolutionEvent({ resolvedModules: ['governance.adf'], triggerMatches: [] }); + const cmds = [commandEvent({ success: true, exitCode: 0 })]; + const report = buildSuggestReport([res], cmds, 1, 60, 'events.ndjson'); + expect(report.loadedButViolated).toHaveLength(0); + }); + + it('flags insufficientData when resolution events are below the threshold', () => { + const report = buildSuggestReport([], [], 3, 60, 'events.ndjson'); + expect(report.insufficientData).toBe(true); + expect(report.deadModules).toHaveLength(0); + expect(report.nearMissKeywords).toHaveLength(0); + expect(report.loadedButViolated).toHaveLength(0); + expect(report.coOccurrenceCandidates).toHaveLength(0); + }); + + it('proposes a co-occurrence candidate when an unmatched keyword recurs alongside a genuinely triggered module', () => { + const events = Array.from({ length: 3 }, () => + resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.coOccurrenceCandidates).toEqual([ + { keyword: 'widget', module: 'frontend.adf', coOccurrences: 3, keywordOccurrences: 3, ambiguous: false }, + ]); + }); + + it('does not propose a co-occurrence candidate for a module that only default-loaded (not genuinely triggered)', () => { + const events = Array.from({ length: 3 }, () => + resolutionEvent({ + keywords: ['widget'], + resolvedModules: ['core.adf'], + triggerMatches: [ + { module: 'core.adf', trigger: 'n/a', matched: true, matchedKeywords: [], loadReason: 'default' }, + ], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.coOccurrenceCandidates).toHaveLength(0); + }); + + it('marks a co-occurrence candidate ambiguous when two modules tie for the top count', () => { + const events = Array.from({ length: 3 }, () => + resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + { module: 'backend.adf', trigger: 'api', matched: true, matchedKeywords: ['api'], loadReason: 'trigger' }, + ], + }), + ); + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.coOccurrenceCandidates).toHaveLength(2); + expect(report.coOccurrenceCandidates.every((c) => c.ambiguous)).toBe(true); + }); + + it('does not propose a co-occurrence candidate below the occurrence threshold', () => { + const events = [ + resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }), + ]; + const report = buildSuggestReport(events, [], 3, 60, 'events.ndjson'); + expect(report.coOccurrenceCandidates).toHaveLength(0); + }); +}); + +describe('adfSuggestCommand', () => { + const baseOptions: CLIOptions = { + configPath: '.charter', + format: 'json', + ciMode: false, + yes: false, + }; + + it('reports insufficientData for a missing telemetry file without crashing', () => { + const tmp = tmpDir(); + process.chdir(tmp); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, []); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { insufficientData: boolean; resolutionEvents: number }; + expect(report.insufficientData).toBe(true); + expect(report.resolutionEvents).toBe(0); + }); + + it('reads real events.ndjson and applies --min-occurrences override', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const res = resolutionEvent({ + triggerMatches: [{ module: 'analysis.adf', trigger: 'blast', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), [res, res].map((e) => JSON.stringify(e)).join('\n') + '\n'); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, ['--min-occurrences', '2']); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { deadModules: Array<{ module: string }> }; + expect(report.deadModules).toEqual([{ module: 'analysis.adf', occurrences: 2 }]); + }); + + it('applies --window-minutes to widen the CLI-level join fallback', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const now = new Date(); + const res = resolutionEvent({ + keywords: ['widget'], + timestamp: now.toISOString(), + triggerMatches: [{ module: 'frontend.adf', trigger: 'react', matched: false, matchedKeywords: [], loadReason: 'trigger' }], + }); + const cmd = commandEvent({ success: false, exitCode: 1, timestamp: new Date(now.getTime() + 120 * 60_000).toISOString() }); + const lines = [res, res, res, cmd].map((e) => JSON.stringify(e)).join('\n') + '\n'; + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), lines); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, ['--window-minutes', '180']); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { nearMissKeywords: Array<{ keyword: string; correlatedFailures: number }> }; + expect(report.nearMissKeywords.find((f) => f.keyword === 'widget')?.correlatedFailures).toBe(3); + }); + + it('treats legacy events without an eventType discriminator as command events', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + // Pre-eventType telemetry event, exactly as recordTelemetryEvent wrote + // it before this feature existed — no eventType field at all. + const legacyEvent = { + version: 1, + timestamp: new Date().toISOString(), + commandPath: 'validate', + flags: [], + format: 'json', + ciMode: true, + durationMs: 50, + exitCode: 1, + success: false, + }; + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), `${JSON.stringify(legacyEvent)}\n`); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, []); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { resolutionEvents: number; commandEvents: number }; + expect(report.commandEvents).toBe(1); + expect(report.resolutionEvents).toBe(0); + }); + + it('ignores malformed lines in events.ndjson instead of crashing', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), 'not json\n{"incomplete":\n'); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, []); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { resolutionEvents: number; commandEvents: number }; + expect(report.resolutionEvents).toBe(0); + expect(report.commandEvents).toBe(0); + }); + + it('prints a human-readable report in text format', () => { + const tmp = tmpDir(); + process.chdir(tmp); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand({ ...baseOptions, format: 'text' }, []); + expect(exitCode).toBe(0); + expect(logs.join('\n')).toContain('charter adf suggest'); + expect(logs.join('\n')).toContain('Dead modules'); + }); + + const MANIFEST_ADF = `ADF: 0.1 + +📦 DEFAULT_LOAD: + - core.adf + +📂 ON_DEMAND: + - frontend.adf (Triggers on: React, CSS, UI) + - backend.adf (Triggers on: API, Node, DB) [budget: 800] +`; + + function writeTelemetry(tmp: string, events: unknown[]): void { + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), events.map((e) => JSON.stringify(e)).join('\n') + '\n'); + } + + it('--emit-ops writes a REPLACE_BULLET op for a non-ambiguous co-occurrence candidate', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, '.ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, '.ai', 'manifest.adf'), MANIFEST_ADF); + + const res = resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const opsFile = path.join(tmp, 'ops.json'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + + const exitCode = adfSuggestCommand(baseOptions, ['--emit-ops', opsFile]); + expect(exitCode).toBe(0); + + const ops = JSON.parse(fs.readFileSync(opsFile, 'utf-8')) as Array<{ op: string; section: string; index: number; value: string }>; + expect(ops).toEqual([ + { op: 'REPLACE_BULLET', section: 'ON_DEMAND', index: 0, value: 'frontend.adf (Triggers on: React, CSS, UI, widget)' }, + ]); + }); + + it('merges two different keywords proposed for the same module into a single op, not two ops at the same index', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, '.ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, '.ai', 'manifest.adf'), MANIFEST_ADF); + + // "gizmo" and "widget" both recur alongside frontend.adf's "react" trigger — + // applying two separate REPLACE_BULLET ops at the same index in sequence + // would make the second clobber the first (this regressed once already). + const res = resolutionEvent({ + keywords: ['gizmo', 'widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const opsFile = path.join(tmp, 'ops.json'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + + const exitCode = adfSuggestCommand(baseOptions, ['--emit-ops', opsFile]); + expect(exitCode).toBe(0); + + const ops = JSON.parse(fs.readFileSync(opsFile, 'utf-8')) as Array<{ op: string; section: string; index: number; value: string }>; + expect(ops).toHaveLength(1); + expect(ops[0]).toEqual({ op: 'REPLACE_BULLET', section: 'ON_DEMAND', index: 0, value: 'frontend.adf (Triggers on: React, CSS, UI, gizmo, widget)' }); + }); + + it('preserves an existing [budget: n] suffix when emitting an op for that module', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, '.ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, '.ai', 'manifest.adf'), MANIFEST_ADF); + + const res = resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'backend.adf', trigger: 'api', matched: true, matchedKeywords: ['api'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const opsFile = path.join(tmp, 'ops.json'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + + adfSuggestCommand(baseOptions, ['--emit-ops', opsFile]); + + const ops = JSON.parse(fs.readFileSync(opsFile, 'utf-8')) as Array<{ value: string }>; + expect(ops[0].value).toBe('backend.adf (Triggers on: API, Node, DB, widget) [budget: 800]'); + }); + + it('writes an empty ops array and reports the skip reason for an ambiguous candidate', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, '.ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, '.ai', 'manifest.adf'), MANIFEST_ADF); + + const res = resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + { module: 'backend.adf', trigger: 'api', matched: true, matchedKeywords: ['api'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const opsFile = path.join(tmp, 'ops.json'); + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, ['--emit-ops', opsFile]); + expect(exitCode).toBe(0); + + const ops = JSON.parse(fs.readFileSync(opsFile, 'utf-8')) as unknown[]; + expect(ops).toEqual([]); + + const report = JSON.parse(logs[0]) as { skippedCandidates: Array<{ reason: string }> }; + expect(report.skippedCandidates.length).toBe(2); + expect(report.skippedCandidates.every((s) => s.reason.includes('ambiguous'))).toBe(true); + }); + + it('does not write an ops file when --emit-ops is not passed, only shows the preview', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, '.ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, '.ai', 'manifest.adf'), MANIFEST_ADF); + + const res = resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand({ ...baseOptions, format: 'text' }, []); + expect(exitCode).toBe(0); + expect(logs.join('\n')).toContain('co-occurs'); + expect(fs.existsSync(path.join(tmp, 'ops.json'))).toBe(false); + }); + + it('respects a custom --ai-dir when locating manifest.adf for --emit-ops', () => { + const tmp = tmpDir(); + process.chdir(tmp); + fs.mkdirSync(path.join(tmp, 'custom-ai'), { recursive: true }); + fs.writeFileSync(path.join(tmp, 'custom-ai', 'manifest.adf'), MANIFEST_ADF); + + const res = resolutionEvent({ + keywords: ['widget'], + triggerMatches: [ + { module: 'frontend.adf', trigger: 'react', matched: true, matchedKeywords: ['react'], loadReason: 'trigger' }, + ], + }); + writeTelemetry(tmp, [res, res, res]); + + const opsFile = path.join(tmp, 'ops.json'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + + adfSuggestCommand(baseOptions, ['--ai-dir', 'custom-ai', '--emit-ops', opsFile]); + + const ops = JSON.parse(fs.readFileSync(opsFile, 'utf-8')) as unknown[]; + expect(ops).toHaveLength(1); + }); +}); diff --git a/packages/cli/src/__tests__/named-scaffolds.test.ts b/packages/cli/src/__tests__/named-scaffolds.test.ts index db1dea4..4fd046a 100644 --- a/packages/cli/src/__tests__/named-scaffolds.test.ts +++ b/packages/cli/src/__tests__/named-scaffolds.test.ts @@ -26,9 +26,18 @@ describe('NAMED_MODULE_SCAFFOLDS registry', () => { expect(scaffold).toContain('secrets'); }); - it('typed-data-access scaffold references the canonical registry path', () => { + it('typed-data-access scaffold references a canonical data registry, generically', () => { const scaffold = NAMED_MODULE_SCAFFOLDS['typed-data-access']; - expect(scaffold).toContain('stackbilt_llc/policies/data-registry.yaml'); + expect(scaffold).toContain('canonical data registry'); + }); + + it('typed-data-access scaffold contains no product-specific internal references', () => { + const scaffold = NAMED_MODULE_SCAFFOLDS['typed-data-access']; + expect(scaffold).not.toMatch(/stackbilt/i); + expect(scaffold).not.toMatch(/aegis/i); + expect(scaffold).not.toMatch(/codebeast/i); + expect(scaffold).not.toMatch(/\bD1\b/); + expect(scaffold).not.toContain('DATA_AUTHORITY'); }); it('typed-data-access scaffold includes load-bearing disambiguation constraint', () => { @@ -55,7 +64,6 @@ describe('NAMED_MODULE_DEFAULT_TRIGGERS registry', () => { it('typed-data-access triggers include sensitivity and policy keywords', () => { const triggers = NAMED_MODULE_DEFAULT_TRIGGERS['typed-data-access']; expect(triggers).toContain('sensitivity'); - expect(triggers).toContain('DATA_AUTHORITY'); expect(triggers).toContain('disambiguation'); }); diff --git a/packages/cli/src/commands/adf-bundle.ts b/packages/cli/src/commands/adf-bundle.ts index bb94d65..ad7ff6c 100644 --- a/packages/cli/src/commands/adf-bundle.ts +++ b/packages/cli/src/commands/adf-bundle.ts @@ -16,6 +16,7 @@ import { import type { CLIOptions } from '../index'; import { CLIError, EXIT_CODE } from '../index'; import { getFlag, tokenizeTask } from '../flags'; +import { recordAdfResolutionEvent } from '../telemetry'; export function adfBundle(options: CLIOptions, args: string[]): number { const task = getFlag(args, '--task'); @@ -63,6 +64,15 @@ export function adfBundle(options: CLIOptions, args: string[]): number { try { const result = bundleModules(aiDir, loadableModulePaths, readFile, keywords, manifest); + recordAdfResolutionEvent(options.configPath, { + source: 'bundle', + keywords, + candidateModules: manifest.onDemand.map(m => m.path), + resolvedModules: result.resolvedModules, + triggerMatches: result.triggerMatches, + tokenEstimate: result.tokenEstimate, + }); + if (options.format === 'json') { const jsonOut: Record = { task, diff --git a/packages/cli/src/commands/adf-context.ts b/packages/cli/src/commands/adf-context.ts index ffb4587..49f11d3 100644 --- a/packages/cli/src/commands/adf-context.ts +++ b/packages/cli/src/commands/adf-context.ts @@ -12,11 +12,13 @@ import { parseManifest, resolveModules, bundleModules, + buildTriggerReport, formatAdf, } from '@stackbilt/adf'; import type { CLIOptions } from '../index'; import { CLIError, EXIT_CODE } from '../index'; import { getFlag, tokenizeTask } from '../flags'; +import { recordAdfResolutionEvent } from '../telemetry'; /** * Extract keywords from file paths using extension and directory signals. @@ -80,6 +82,7 @@ export function adfContextCommand(options: CLIOptions, args: string[]): number { const manifest = parseManifest(manifestDoc); const resolvedModules = resolveModules(manifest, dedupKeywords); + const candidateModules = manifest.onDemand.map(m => m.path); if (bundle) { // Full bundle output @@ -90,6 +93,15 @@ export function adfContextCommand(options: CLIOptions, args: string[]): number { const result = bundleModules(aiDir, resolvedModules, readFile, dedupKeywords, manifest); + recordAdfResolutionEvent(options.configPath, { + source: 'context', + keywords: dedupKeywords, + candidateModules, + resolvedModules: result.resolvedModules, + triggerMatches: result.triggerMatches, + tokenEstimate: result.tokenEstimate, + }); + if (options.format === 'json') { console.log(JSON.stringify({ keywords: dedupKeywords, @@ -103,6 +115,14 @@ export function adfContextCommand(options: CLIOptions, args: string[]): number { } } else { // Module list only + recordAdfResolutionEvent(options.configPath, { + source: 'context', + keywords: dedupKeywords, + candidateModules, + resolvedModules, + triggerMatches: buildTriggerReport(manifest, resolvedModules, dedupKeywords), + }); + if (options.format === 'json') { console.log(JSON.stringify({ keywords: dedupKeywords, diff --git a/packages/cli/src/commands/adf-named-scaffolds.ts b/packages/cli/src/commands/adf-named-scaffolds.ts index 8307eaf..7207c66 100644 --- a/packages/cli/src/commands/adf-named-scaffolds.ts +++ b/packages/cli/src/commands/adf-named-scaffolds.ts @@ -19,62 +19,52 @@ */ /** - * Typed data access and ontology enforcement policy (Stackbilt-dev/charter#69). + * Typed data access and ontology enforcement policy. * - * Codifies the cross-repo policy for how services reference business concepts - * (tenant, user, subscription, quota, etc.) — derived from the canonical data - * registry at Stackbilt-dev/stackbilt_llc/policies/data-registry.yaml. + * Codifies a generic cross-service policy for how services reference + * business concepts (tenant, user, subscription, quota, etc.) — derived + * from a canonical data registry that is the single source of truth for + * ownership, sensitivity, and access shape. * - * Declares six sensitivity tiers, the disambiguation protocol, and RPC - * boundary rules. Consumed by charter validate / codebeast DATA_AUTHORITY / - * AEGIS disambiguation firewall as the single source of truth for data - * access policy across the ecosystem. + * Declares six sensitivity tiers, a disambiguation step, and cross-service + * access-boundary rules. Framework-generic content only — keep this in + * sync with the repo-local `.ai/typed-data-access.adf` policy module. */ export const TYPED_DATA_ACCESS_SCAFFOLD = `ADF: 0.1 \u{1F3AF} TASK: Typed data access and ontology enforcement policy \u{1F4CB} CONTEXT: - - Business concepts (tenant, user, subscription, quota, credit, mrr, etc.) are defined in a canonical data registry — the single source of truth for ownership, sensitivity, and access shape across the ecosystem - - Reference registry location: Stackbilt-dev/stackbilt_llc/policies/data-registry.yaml (22+ concepts, 6 sensitivity tiers) - - Each concept declares: owner service, D1 table, sensitivity tier, definition, aliases, rpc_method, mcp_tool - - Consumer services derive their KNOWN_CONCEPTS and alias maps from the registry at build time (compiled-const snapshot) - - Disambiguation protocol halts on undefined concepts rather than guessing - - CodeBeast DATA_AUTHORITY sensitivity class escalates raw D1 access to owned tables + - Business concepts (tenant, user, subscription, quota, credit, etc.) are defined in a canonical data registry \u2014 the single source of truth for ownership, sensitivity, and access shape across services + - Each concept declares: owner service, storage table, sensitivity tier, definition, aliases, and the accessor (RPC method, tool, or API) non-owning services must use + - Consumer services derive their known-concepts and alias maps from the registry at build time (a compiled-const snapshot pattern, not a runtime fetch) + - A disambiguation step halts on undefined concepts rather than guessing shape, ownership, or sensitivity \u{1F510} SENSITIVITY TIERS [load-bearing]: - public \u2014 readable from any service, no auth required (e.g., blog_post) - - service_internal \u2014 readable/writable only by the owning service, raw D1 access is fine within the owner - - cross_service_rpc \u2014 accessible via declared rpc_method or Service Binding, never raw D1 from a non-owning service - - pii_scoped \u2014 accessible only via owning service + audit_log entry required at the call site - - billing_critical \u2014 writable only by the owning service plus the Stripe webhook handler; never leaves the owning service boundary even via RPC + - service_internal \u2014 readable/writable only by the owning service, raw storage access is fine within the owner + - cross_service_rpc \u2014 accessible via a declared RPC method or service binding, never raw storage access from a non-owning service + - pii_scoped \u2014 accessible only via the owning service + an audit-log entry required at the call site + - billing_critical \u2014 writable only by the owning service plus its designated payment-webhook handler; never leaves the owning service boundary even via RPC - secrets \u2014 never leaves the owning service boundary under any circumstance \u26A0\uFE0F CONSTRAINTS [load-bearing]: - New code referencing a business concept MUST check the canonical registry first; terms not in the registry or its aliases MUST be added before the code lands - - Non-owning services reading or writing cross_service_rpc concepts MUST use the declared rpc_method or mcp_tool \u2014 raw D1 access to another service's table is a DATA_AUTHORITY violation - - pii_scoped access requires an audit_log entry at the call site \u2014 no silent reads + - Non-owning services reading or writing cross_service_rpc concepts MUST use the declared accessor \u2014 raw storage access to another service's table is a violation + - pii_scoped access requires an audit-log entry at the call site \u2014 no silent reads - billing_critical and secrets tiers NEVER cross the owning service boundary, even via RPC - When encountering an undefined data concept in requirements, tasks, or user prompts, HALT and ask for clarification rather than guessing shape, ownership, or sensitivity - Registry updates MUST come before consumer code updates \u2014 the source of truth leads, consumers follow - - When promoting a concept to a higher sensitivity tier, all existing consumers of raw D1 access must migrate to RPC in the same change set + - When promoting a concept to a higher sensitivity tier, all existing consumers of raw storage access must migrate to the declared accessor in the same change set \u{1F4D6} ADVISORY: - Check the registry before reaching for a new type definition \u2014 the concept may already exist with a canonical shape - - Use charter surface --format json to discover what D1 tables a service currently exposes; cross-reference against registry ownership + - Use charter surface --format json to discover what tables/resources a service currently exposes; cross-reference against registry ownership - Aliases (e.g., "credits" for "quota") are semantically equivalent; prefer the canonical form in new code, accept aliases in user-facing copy - - The disambiguation protocol is load-bearing for autonomous agents \u2014 these systems cannot safely guess business term semantics + - The disambiguation step is load-bearing for autonomous agents \u2014 these systems cannot safely guess business-term semantics \u{1F4CA} METRICS: - REGISTRY_PATH: stackbilt_llc/policies/data-registry.yaml - REGISTRY_REPO: Stackbilt-dev/stackbilt_llc SENSITIVITY_TIERS: 6 - DOCUMENTED_CONCEPTS: 22 - -\u{1F517} REFERENCES: - - Stackbilt-dev/charter#69 \u2014 typed data access policy umbrella issue - - codebeast#9 \u2014 DATA_AUTHORITY sensitivity class (enforcement side) - - Stackbilt-dev/aegis#344 \u2014 disambiguation firewall (runtime halt mechanism) `; /** @@ -98,14 +88,12 @@ export const NAMED_MODULE_DEFAULT_TRIGGERS: Record = { 'subscription', 'quota', 'credit', - 'mrr', 'pii', 'sensitivity', 'data registry', 'ontology', 'disambiguation', - 'DATA_AUTHORITY', - 'raw D1', + 'raw storage access', 'service boundary', 'auth_scoped', 'billing_critical', diff --git a/packages/cli/src/commands/adf-suggest.ts b/packages/cli/src/commands/adf-suggest.ts new file mode 100644 index 0000000..c777f07 --- /dev/null +++ b/packages/cli/src/commands/adf-suggest.ts @@ -0,0 +1,534 @@ +/** + * charter adf suggest + * + * Diagnostics over .charter/telemetry/events.ndjson: which ADF modules + * never fire, which task keywords never match a trigger anywhere, and + * which modules load yet a downstream command still fails. + * + * By default this is entirely report-only. Attributing a *failure* to a + * specific module's trigger list would require knowing which module's + * constraint was violated, and nothing in current telemetry (validate/ + * drift/evidence check commit trailers, CLAUDE.md sync, and METRICS + * ceilings respectively) records that — so no op is ever generated from + * the failure-correlation signal alone. + * + * `--emit-ops` adds one narrower, self-contained mechanism on top: a + * co-occurrence heuristic. If an unmatched keyword frequently appears + * alongside keywords that already trigger some module M (in the same + * task), M is a *candidate* to adopt that keyword as a new trigger. This + * is weaker evidence than a failure correlation — it's "these words tend + * to show up together," not "this caused a violation" — so it only ever + * proposes a REPLACE_BULLET op for --emit-ops to write out; nothing is + * ever applied automatically. Ambiguous candidates (tied top module) are + * reported but never turned into an op. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { parseAdf, parseManifest, formatTriggerEntry } from '@stackbilt/adf'; +import type { PatchOperation } from '@stackbilt/adf'; +import type { CLIOptions } from '../index'; +import { EXIT_CODE } from '../index'; +import { getFlag } from '../flags'; +import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; + +const DEFAULT_MIN_OCCURRENCES = 3; +const DEFAULT_WINDOW_MINUTES = 60; + +/** + * Generic English function words filtered out of the near-miss keyword + * detector. Without this, "the"/"a"/"and" dominate the report purely on + * frequency, drowning out keywords that are actually candidate triggers. + */ +const STOPWORDS = new Set([ + 'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'for', 'with', + 'is', 'are', 'was', 'were', 'be', 'been', 'this', 'that', 'these', 'those', + 'it', 'its', 'as', 'at', 'by', 'from', 'into', 'about', 'my', 'our', 'your', +]); + +export interface DeadModuleFinding { + module: string; + occurrences: number; +} + +export interface NearMissKeywordFinding { + keyword: string; + occurrences: number; + correlatedFailures: number; +} + +export interface LoadedButViolatedFinding { + module: string; + occurrences: number; + correlatedFailures: number; +} + +/** + * A candidate trigger-keyword-to-module pairing based purely on + * co-occurrence: `keyword` never matches any trigger itself, but shows up + * in the same task alongside a keyword that already triggers `module`. + * Heuristic, not failure-backed — see file header. `ambiguous` is true + * when another module tied for the top co-occurrence count, in which case + * no op is ever generated for this candidate. + */ +export interface CoOccurrenceCandidate { + keyword: string; + module: string; + coOccurrences: number; + keywordOccurrences: number; + ambiguous: boolean; +} + +export interface SuggestReport { + resolutionEvents: number; + commandEvents: number; + minOccurrences: number; + windowMinutes: number; + insufficientData: boolean; + deadModules: DeadModuleFinding[]; + nearMissKeywords: NearMissKeywordFinding[]; + loadedButViolated: LoadedButViolatedFinding[]; + coOccurrenceCandidates: CoOccurrenceCandidate[]; + sourceFile: string; +} + +export interface EmitOpsResult { + ops: PatchOperation[]; + diffLines: string[]; + skipped: Array<{ keyword: string; module: string; reason: string }>; +} + +export function adfSuggestCommand(options: CLIOptions, args: string[]): number { + const minOccurrences = parsePositiveInt(getFlag(args, '--min-occurrences')) ?? DEFAULT_MIN_OCCURRENCES; + const windowMinutes = parsePositiveInt(getFlag(args, '--window-minutes')) ?? DEFAULT_WINDOW_MINUTES; + const aiDir = getFlag(args, '--ai-dir') || '.ai'; + const emitOpsFile = getFlag(args, '--emit-ops'); + + const telemetryFile = path.join(options.configPath, 'telemetry', 'events.ndjson'); + const { resolutionEvents, commandEvents } = readEvents(telemetryFile); + const report = buildSuggestReport(resolutionEvents, commandEvents, minOccurrences, windowMinutes, telemetryFile); + const emitResult = buildEmitOps(report.coOccurrenceCandidates, aiDir); + + if (emitOpsFile) { + fs.writeFileSync(emitOpsFile, `${JSON.stringify(emitResult.ops, null, 2)}\n`); + } + + if (options.format === 'json') { + console.log(JSON.stringify({ + ...report, + opsFile: emitOpsFile ?? null, + opsEmitted: emitOpsFile ? emitResult.ops.length : 0, + skippedCandidates: emitResult.skipped, + }, null, 2)); + } else { + printReport(report); + printEmitOpsSection(emitResult, emitOpsFile); + } + + return EXIT_CODE.SUCCESS; +} + +/** + * Turn non-ambiguous co-occurrence candidates into REPLACE_BULLET ops + * against manifest.adf's ON_DEMAND section. Always computed (so the diff + * preview shows even without --emit-ops); writing the ops file is gated + * on the caller passing --emit-ops. + */ +function buildEmitOps(candidates: CoOccurrenceCandidate[], aiDir: string): EmitOpsResult { + const ops: PatchOperation[] = []; + const diffLines: string[] = []; + const skipped: Array<{ keyword: string; module: string; reason: string }> = []; + + if (candidates.length === 0) { + return { ops, diffLines, skipped }; + } + + const manifestPath = path.join(aiDir, 'manifest.adf'); + let manifestContent: string; + try { + manifestContent = fs.readFileSync(manifestPath, 'utf-8'); + } catch { + for (const c of candidates) { + skipped.push({ keyword: c.keyword, module: c.module, reason: `manifest.adf not found at ${manifestPath}` }); + } + return { ops, diffLines, skipped }; + } + + const doc = parseAdf(manifestContent); + const manifest = parseManifest(doc); + const onDemandSection = doc.sections.find((s) => s.key === 'ON_DEMAND'); + const rawItems = onDemandSection && onDemandSection.content.type === 'list' ? onDemandSection.content.items : []; + + // Group by target module first: two candidates proposing the same module + // (e.g. "gizmo" and "component" both -> frontend.adf) must become ONE + // REPLACE_BULLET op with both keywords appended, not two ops at the same + // index — applying them in sequence would make the second silently + // clobber the first (adf patch applies against the evolving document). + const byModule = new Map(); + for (const candidate of candidates) { + if (candidate.ambiguous) { + skipped.push({ keyword: candidate.keyword, module: candidate.module, reason: 'ambiguous: tied with another module for top co-occurrence count' }); + continue; + } + const group = byModule.get(candidate.module) ?? []; + group.push(candidate); + byModule.set(candidate.module, group); + } + + for (const [modulePath, groupCandidates] of byModule) { + const modIndex = manifest.onDemand.findIndex((m) => m.path === modulePath); + if (modIndex === -1 || modIndex >= rawItems.length) { + for (const c of groupCandidates) { + skipped.push({ keyword: c.keyword, module: c.module, reason: 'module not found in manifest ON_DEMAND section' }); + } + continue; + } + + const mod = manifest.onDemand[modIndex]; + const existingLower = new Set(mod.triggers.map((t) => t.toLowerCase())); + const newKeywords: string[] = []; + for (const c of groupCandidates) { + if (existingLower.has(c.keyword)) { + skipped.push({ keyword: c.keyword, module: c.module, reason: 'already a trigger for this module' }); + continue; + } + newKeywords.push(c.keyword); + } + if (newKeywords.length === 0) continue; + + const before = rawItems[modIndex]; + const after = formatTriggerEntry({ ...mod, triggers: [...mod.triggers, ...newKeywords] }); + + ops.push({ op: 'REPLACE_BULLET', section: 'ON_DEMAND', index: modIndex, value: after }); + for (const c of groupCandidates) { + if (!newKeywords.includes(c.keyword)) continue; + diffLines.push(` [+] ${modulePath}: add trigger "${c.keyword}" (co-occurs ${c.coOccurrences}x with an existing trigger)`); + } + diffLines.push(` before: ${before}`); + diffLines.push(` after: ${after}`); + } + + return { ops, diffLines, skipped }; +} + +function parsePositiveInt(raw: string | undefined): number | undefined { + if (!raw) return undefined; + const n = parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : undefined; +} + +function readEvents(telemetryFile: string): { + resolutionEvents: AdfResolutionEvent[]; + commandEvents: CliTelemetryEvent[]; +} { + const resolutionEvents: AdfResolutionEvent[] = []; + const commandEvents: CliTelemetryEvent[] = []; + + let raw: string; + try { + raw = fs.readFileSync(telemetryFile, 'utf-8'); + } catch { + return { resolutionEvents, commandEvents }; + } + + for (const line of raw.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + continue; + } + if (!parsed || typeof parsed !== 'object') continue; + + const record = parsed as { eventType?: string; timestamp?: unknown; commandPath?: unknown; exitCode?: unknown }; + if (typeof record.timestamp !== 'string') continue; + + if (record.eventType === 'adf.resolution') { + resolutionEvents.push(parsed as AdfResolutionEvent); + } else if (record.eventType === 'command' || record.eventType === undefined) { + // Events written before eventType existed have no discriminator; + // treat those as command events for backward compatibility. Require + // the fields a real command event always has, matching the + // validation in telemetry.ts's own readEvents, so a malformed or + // truncated line isn't silently counted as a valid command outcome. + if (typeof record.commandPath !== 'string' || typeof record.exitCode !== 'number') continue; + commandEvents.push(parsed as CliTelemetryEvent); + } + } + + return { resolutionEvents, commandEvents }; +} + +/** + * Whether a command event falls within the join window of a resolution + * event: same sessionId (high confidence) if both are stamped, else a + * forward-only time-window fallback (heuristic). + */ +function isJoined(resolution: AdfResolutionEvent, command: CliTelemetryEvent, windowMinutes: number): boolean { + if (resolution.sessionId && command.sessionId) { + return resolution.sessionId === command.sessionId; + } + + const resTime = Date.parse(resolution.timestamp); + const cmdTime = Date.parse(command.timestamp); + if (!Number.isFinite(resTime) || !Number.isFinite(cmdTime)) return false; + + const deltaMs = cmdTime - resTime; + return deltaMs >= 0 && deltaMs <= windowMinutes * 60_000; +} + +function hasCorrelatedFailure( + resolution: AdfResolutionEvent, + failedCommandEvents: CliTelemetryEvent[], + windowMinutes: number, +): boolean { + return failedCommandEvents.some((c) => isJoined(resolution, c, windowMinutes)); +} + +export function buildSuggestReport( + resolutionEvents: AdfResolutionEvent[], + commandEvents: CliTelemetryEvent[], + minOccurrences: number, + windowMinutes: number, + sourceFile: string, +): SuggestReport { + const moduleOccurrences = new Map(); + const moduleMatched = new Map(); + const globallyMatchedKeywords = new Set(); + const loadedOccurrences = new Map(); + const loadedCorrelatedFailures = new Map(); + const keywordOccurrences = new Map(); + const keywordCorrelatedFailures = new Map(); + // keyword -> module -> co-occurrence count (module genuinely ON_DEMAND-triggered + // in the same event, not just default-loaded). + const keywordModuleCoOccurrence = new Map>(); + + // Only failed commands can ever satisfy hasCorrelatedFailure/isJoined, so + // scanning just this (typically much smaller) subset per resolution event + // avoids an O(resolutions x commands) scan over the full command history. + const failedCommandEvents = commandEvents.filter((c) => c.success === false); + + for (const res of resolutionEvents) { + // triggerMatches has one entry per (module, trigger) pair, so a module + // declared with multiple triggers must be deduped per event here — + // otherwise it would count as multiple "candidate resolutions" from a + // single real invocation, reaching --min-occurrences too early relative + // to modules with fewer declared triggers. + const modulesSeenThisEvent = new Set(); + const modulesMatchedThisEvent = new Set(); + for (const tm of res.triggerMatches) { + modulesSeenThisEvent.add(tm.module); + if (tm.matched) modulesMatchedThisEvent.add(tm.module); + for (const kw of tm.matchedKeywords) { + globallyMatchedKeywords.add(kw); + } + } + for (const mod of modulesSeenThisEvent) { + moduleOccurrences.set(mod, (moduleOccurrences.get(mod) ?? 0) + 1); + } + for (const mod of modulesMatchedThisEvent) { + moduleMatched.set(mod, (moduleMatched.get(mod) ?? 0) + 1); + } + for (const mod of res.resolvedModules) { + loadedOccurrences.set(mod, (loadedOccurrences.get(mod) ?? 0) + 1); + } + } + + // Second pass: needs globallyMatchedKeywords fully populated first. + for (const res of resolutionEvents) { + const failed = hasCorrelatedFailure(res, failedCommandEvents, windowMinutes); + + // Modules genuinely triggered on-demand in this event (excludes + // default-load modules, which "resolve" regardless of keywords and so + // can't be evidence of a keyword/module co-occurrence). + const triggeredModulesThisEvent = new Set( + res.triggerMatches.filter((tm) => tm.matched && tm.loadReason === 'trigger').map((tm) => tm.module), + ); + + // A single task can repeat a word (or an upstream producer can hand back + // a non-deduplicated keyword array) — dedupe per event so one invocation + // with a repeated word can't alone cross --min-occurrences. + const seenKeywordsThisEvent = new Set(); + const unmatchedKeywordsThisEvent: string[] = []; + for (const rawKw of res.keywords) { + // matchedKeywords (buildTriggerReport) are always lowercased; normalize + // here too so "Audit" and "audit" aren't treated as different keywords + // and a capitalized keyword that DID match isn't falsely reported as a miss. + const kw = rawKw.toLowerCase(); + if (seenKeywordsThisEvent.has(kw)) continue; + seenKeywordsThisEvent.add(kw); + if (globallyMatchedKeywords.has(kw) || STOPWORDS.has(kw)) continue; + keywordOccurrences.set(kw, (keywordOccurrences.get(kw) ?? 0) + 1); + unmatchedKeywordsThisEvent.push(kw); + if (failed) { + keywordCorrelatedFailures.set(kw, (keywordCorrelatedFailures.get(kw) ?? 0) + 1); + } + } + + if (triggeredModulesThisEvent.size > 0) { + for (const kw of unmatchedKeywordsThisEvent) { + let moduleCounts = keywordModuleCoOccurrence.get(kw); + if (!moduleCounts) { + moduleCounts = new Map(); + keywordModuleCoOccurrence.set(kw, moduleCounts); + } + for (const mod of triggeredModulesThisEvent) { + moduleCounts.set(mod, (moduleCounts.get(mod) ?? 0) + 1); + } + } + } + + if (failed) { + for (const mod of res.resolvedModules) { + loadedCorrelatedFailures.set(mod, (loadedCorrelatedFailures.get(mod) ?? 0) + 1); + } + } + } + + const deadModules: DeadModuleFinding[] = [...moduleOccurrences.entries()] + .filter(([module, occurrences]) => occurrences >= minOccurrences && (moduleMatched.get(module) ?? 0) === 0) + .map(([module, occurrences]) => ({ module, occurrences })) + .sort((a, b) => b.occurrences - a.occurrences); + + const nearMissKeywords: NearMissKeywordFinding[] = [...keywordOccurrences.entries()] + .filter(([, occurrences]) => occurrences >= minOccurrences) + .map(([keyword, occurrences]) => ({ + keyword, + occurrences, + correlatedFailures: keywordCorrelatedFailures.get(keyword) ?? 0, + })) + .sort((a, b) => b.correlatedFailures - a.correlatedFailures || b.occurrences - a.occurrences); + + const loadedButViolated: LoadedButViolatedFinding[] = [...loadedCorrelatedFailures.entries()] + .filter(([, correlatedFailures]) => correlatedFailures > 0) + .map(([module, correlatedFailures]) => ({ + module, + occurrences: loadedOccurrences.get(module) ?? 0, + correlatedFailures, + })) + .sort((a, b) => b.correlatedFailures - a.correlatedFailures); + + const nearMissKeywordSet = new Set(nearMissKeywords.map((f) => f.keyword)); + const coOccurrenceCandidates: CoOccurrenceCandidate[] = []; + for (const [keyword, moduleCounts] of keywordModuleCoOccurrence) { + if (!nearMissKeywordSet.has(keyword)) continue; // below minOccurrences overall + let topModules: string[] = []; + let topCount = 0; + for (const [module, count] of moduleCounts) { + if (count > topCount) { + topCount = count; + topModules = [module]; + } else if (count === topCount) { + topModules.push(module); + } + } + if (topCount < minOccurrences) continue; + for (const module of topModules) { + coOccurrenceCandidates.push({ + keyword, + module, + coOccurrences: topCount, + keywordOccurrences: keywordOccurrences.get(keyword) ?? 0, + ambiguous: topModules.length > 1, + }); + } + } + coOccurrenceCandidates.sort((a, b) => b.coOccurrences - a.coOccurrences); + + return { + resolutionEvents: resolutionEvents.length, + commandEvents: commandEvents.length, + minOccurrences, + windowMinutes, + insufficientData: resolutionEvents.length < minOccurrences, + deadModules, + nearMissKeywords, + loadedButViolated, + coOccurrenceCandidates, + sourceFile: sourceFile.replace(/\\/g, '/'), + }; +} + +function printReport(report: SuggestReport): void { + console.log(''); + console.log(' charter adf suggest — diagnostics over ADF module resolution telemetry'); + console.log(''); + console.log(` Resolution events: ${report.resolutionEvents}, command events: ${report.commandEvents}`); + console.log(` Thresholds: min-occurrences=${report.minOccurrences}, window-minutes=${report.windowMinutes}`); + if (report.insufficientData) { + console.log(' [!] Insufficient data — keep using the CLI/MCP server, then re-run.'); + } + console.log(''); + + console.log(' Dead modules (0 matches across enough candidate resolutions):'); + if (report.deadModules.length === 0) { + console.log(' (none)'); + } else { + for (const f of report.deadModules) { + console.log(` [!] ${f.module} — 0 matches / ${f.occurrences} candidate resolutions`); + } + } + console.log(''); + + console.log(' Recurring unmatched keywords (no trigger, anywhere, matches this keyword):'); + if (report.nearMissKeywords.length === 0) { + console.log(' (none)'); + } else { + for (const f of report.nearMissKeywords) { + const corr = f.correlatedFailures > 0 ? `, ${f.correlatedFailures} correlated with a downstream failure` : ''; + console.log(` [?] "${f.keyword}" — seen ${f.occurrences}x${corr}`); + } + console.log(' Note: failure correlation alone never attributes a keyword to a module — see co-occurrence candidates below for the (weaker) heuristic that does.'); + } + console.log(''); + + console.log(' Loaded-but-violated (module resolved, yet a downstream failure still occurred):'); + if (report.loadedButViolated.length === 0) { + console.log(' (none)'); + } else { + for (const f of report.loadedButViolated) { + console.log(` [!] ${f.module} — resolved ${f.occurrences}x, ${f.correlatedFailures} correlated with a downstream failure`); + } + console.log(' Note: routing worked; rule content may be weak. Needs human/strong-model rewrite, not a trigger change.'); + } + console.log(''); + + console.log(' Co-occurrence trigger candidates (heuristic — not failure-backed, review before applying):'); + if (report.coOccurrenceCandidates.length === 0) { + console.log(' (none)'); + } else { + for (const c of report.coOccurrenceCandidates) { + const status = c.ambiguous ? ' [ambiguous — tied with another module, no op generated]' : ''; + console.log(` [?] "${c.keyword}" -> ${c.module} — co-occurs ${c.coOccurrences}/${c.keywordOccurrences}x${status}`); + } + console.log(' Run with --emit-ops to write non-ambiguous candidates as REPLACE_BULLET ops for `charter adf patch --ops-file`.'); + } + console.log(''); + console.log(` Source: ${report.sourceFile}`); + console.log(''); +} + +function printEmitOpsSection(emitResult: EmitOpsResult, emitOpsFile: string | undefined): void { + if (emitResult.diffLines.length > 0) { + console.log(' Proposed ops (preview):'); + for (const line of emitResult.diffLines) { + console.log(line); + } + console.log(''); + } + if (emitResult.skipped.length > 0) { + console.log(' Skipped candidates:'); + for (const s of emitResult.skipped) { + console.log(` [-] "${s.keyword}" -> ${s.module}: ${s.reason}`); + } + console.log(''); + } + if (emitOpsFile) { + console.log(` [ok] Wrote ${emitResult.ops.length} op(s) to ${emitOpsFile}`); + console.log(` Review, then apply with: charter adf patch .ai/manifest.adf --ops-file ${emitOpsFile}`); + console.log(''); + } +} diff --git a/packages/cli/src/commands/adf.ts b/packages/cli/src/commands/adf.ts index e2c1619..5f37e24 100644 --- a/packages/cli/src/commands/adf.ts +++ b/packages/cli/src/commands/adf.ts @@ -26,6 +26,7 @@ import { adfTidyCommand } from './adf-tidy'; import { adfPopulateCommand } from './adf-populate'; import { adfContextCommand } from './adf-context'; import { adfCompileCommand } from './adf-compile'; +import { adfSuggestCommand } from './adf-suggest'; import { NAMED_MODULE_SCAFFOLDS, NAMED_MODULE_DEFAULT_TRIGGERS, @@ -265,8 +266,10 @@ export async function adfCommand(options: CLIOptions, args: string[]): Promise] [--window-minutes ] [--ai-dir ]'); + console.log(' [--emit-ops ]'); + console.log(' Diagnostics over .charter/telemetry/events.ndjson: dead modules,'); + console.log(' recurring unmatched keywords, and modules that load yet a downstream'); + console.log(' command still fails — all report-only, no module attribution guessed.'); + console.log(' Also reports co-occurrence trigger candidates (a weaker, non-failure-'); + console.log(' backed heuristic: keyword X often appears alongside a keyword that'); + console.log(' already triggers module M). --emit-ops writes non-ambiguous candidates'); + console.log(' as REPLACE_BULLET ops to , for review then `charter adf patch'); + console.log(' --ops-file ` — nothing is ever applied automatically.'); + console.log(' --min-occurrences: threshold before a signal is reported (default: 3)'); + console.log(' --window-minutes: time-window fallback for joining a resolution to a'); + console.log(' later command outcome when no sessionId is set (default: 60)'); + console.log(''); } function parseTriggers(raw: string | undefined): string[] { diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 46eb424..ded00be 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -13,6 +13,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; @@ -39,6 +40,7 @@ import type { CLIOptions } from '../index'; import { CLIError, EXIT_CODE } from '../index'; import { getFlag } from '../flags'; import { detectTsconfigAliases } from './blast'; +import { getSessionId, recordAdfResolutionEvent } from '../telemetry'; // ============================================================================ // Constants @@ -94,6 +96,7 @@ export async function serveCommand(options: CLIOptions, args: string[]): Promise } const projectName = customName ?? inferProjectName(aiDir); + const sessionId = getSessionId() ?? randomUUID(); // Lazy-import both MCP SDK modules here — after all guards — so the SDK's // stdin handle is never acquired on the error paths above. @@ -107,7 +110,7 @@ export async function serveCommand(options: CLIOptions, args: string[]): Promise version: '1.0.0', }); - registerTools(server, aiDir, options); + registerTools(server, aiDir, options, sessionId); registerResources(server, aiDir); if (options.format !== 'json') { @@ -125,7 +128,7 @@ export async function serveCommand(options: CLIOptions, args: string[]): Promise // Tool Registration // ============================================================================ -function registerTools(server: McpServer, aiDir: string, options: CLIOptions): void { +function registerTools(server: McpServer, aiDir: string, options: CLIOptions, sessionId: string): void { (server.registerTool as Function)( 'charter_context', @@ -159,7 +162,7 @@ function registerTools(server: McpServer, aiDir: string, options: CLIOptions): v async ({ task }: { task?: string }) => { try { const manifest = loadManifest(aiDir); - const keywords = task ? task.toLowerCase().split(/\s+/) : []; + const keywords = task ? [...new Set(task.toLowerCase().split(/\s+/))] : []; const modulePaths = resolveModules(manifest, keywords); const bundle = bundleModules( aiDir, @@ -168,6 +171,15 @@ function registerTools(server: McpServer, aiDir: string, options: CLIOptions): v keywords, manifest, ); + recordAdfResolutionEvent(options.configPath, { + source: 'mcp.getProjectContext', + keywords, + candidateModules: manifest.onDemand.map(m => m.path), + resolvedModules: bundle.resolvedModules, + triggerMatches: bundle.triggerMatches, + tokenEstimate: bundle.tokenEstimate, + sessionId, + }); return { content: [{ type: 'text' as const, text: formatAdf(bundle.mergedDocument) }] }; } catch (err) { return { content: [{ type: 'text' as const, text: `Error: ${err instanceof Error ? err.message : String(err)}` }], isError: true }; diff --git a/packages/cli/src/flags.ts b/packages/cli/src/flags.ts index 535e17a..4d8b8be 100644 --- a/packages/cli/src/flags.ts +++ b/packages/cli/src/flags.ts @@ -32,10 +32,14 @@ export function readFlagFile(filePath: string, flagName: string): string { /** * Tokenize a task prompt into keywords for ADF module resolution. + * Deduplicated — a repeated word in the prompt (or a caller passing the + * same task twice) shouldn't count as multiple independent keyword + * occurrences downstream (e.g. in adf suggest's telemetry analysis). */ export function tokenizeTask(task: string): string[] { - return task + const tokens = task .split(/[\s,;:()[\]{}]+/) .filter(w => w.length > 1) .map(w => w.replace(/[^a-zA-Z0-9]/g, '')); + return [...new Set(tokens)]; } diff --git a/packages/cli/src/telemetry.ts b/packages/cli/src/telemetry.ts index 8428680..5f6977f 100644 --- a/packages/cli/src/telemetry.ts +++ b/packages/cli/src/telemetry.ts @@ -7,9 +7,12 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type { BundleResult } from '@stackbilt/adf'; export interface CliTelemetryEvent { version: 1; + eventType?: 'command'; timestamp: string; commandPath: string; flags: string[]; @@ -19,6 +22,50 @@ export interface CliTelemetryEvent { exitCode: number; success: boolean; errorName?: string; + sessionId?: string; +} + +/** + * Sibling event recording which ADF modules were candidates/resolved for a + * task, and the trigger-match detail behind that resolution. Joinable + * against subsequent command events (same sessionId, or a time-window + * fallback) to detect misses: triggers that never fire, or modules that + * load but whose constraints get violated anyway. + */ +export interface AdfResolutionEvent { + version: 1; + eventType: 'adf.resolution'; + timestamp: string; + sessionId: string | null; + resolutionId: string; + source: 'bundle' | 'context' | 'mcp.getProjectContext'; + keywords: string[]; + candidateModules: string[]; + resolvedModules: string[]; + triggerMatches: BundleResult['triggerMatches']; + tokenEstimate?: number; +} + +export interface RecordAdfResolutionInput { + source: AdfResolutionEvent['source']; + keywords: string[]; + candidateModules: string[]; + resolvedModules: string[]; + triggerMatches: BundleResult['triggerMatches']; + tokenEstimate?: number; + /** Override the CHARTER_SESSION_ID env lookup — used by long-lived processes (e.g. `charter serve`) that mint one session id at startup. */ + sessionId?: string | null; +} + +/** + * Read the ambient session id from the environment, treating an empty + * string the same as unset — a wrapper script that exports + * CHARTER_SESSION_ID="" should not produce a literal empty-string session + * id. Exported so long-lived processes (e.g. `charter serve`) that mint + * their own session id can share this same empty-string handling. + */ +export function getSessionId(): string | undefined { + return process.env.CHARTER_SESSION_ID || undefined; } export interface RecordEventInput { @@ -38,6 +85,7 @@ export function recordTelemetryEvent(configPath: string, input: RecordEventInput const event: CliTelemetryEvent = { version: 1, + eventType: 'command', timestamp: new Date().toISOString(), commandPath: inferCommandPath(input.args), flags: extractFlagNames(input.args), @@ -47,6 +95,37 @@ export function recordTelemetryEvent(configPath: string, input: RecordEventInput exitCode: input.exitCode, success: input.exitCode === 0, errorName: input.errorName, + sessionId: getSessionId(), + }; + + fs.appendFileSync(telemetryFile, `${JSON.stringify(event)}\n`); + } catch { + // Telemetry is best-effort and must never block command execution. + } +} + +/** + * Persist an ADF module-resolution event (best-effort, never throws). + * Sibling to recordTelemetryEvent — same file, discriminated by eventType. + */ +export function recordAdfResolutionEvent(configPath: string, input: RecordAdfResolutionInput): void { + try { + const telemetryDir = path.join(configPath, 'telemetry'); + const telemetryFile = path.join(telemetryDir, 'events.ndjson'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const event: AdfResolutionEvent = { + version: 1, + eventType: 'adf.resolution', + timestamp: new Date().toISOString(), + sessionId: input.sessionId ?? getSessionId() ?? null, + resolutionId: randomUUID(), + source: input.source, + keywords: input.keywords, + candidateModules: input.candidateModules, + resolvedModules: input.resolvedModules, + triggerMatches: input.triggerMatches, + tokenEstimate: input.tokenEstimate, }; fs.appendFileSync(telemetryFile, `${JSON.stringify(event)}\n`);