diff --git a/packages/adf/package.json b/packages/adf/package.json index d5ec77d..97c1cdc 100644 --- a/packages/adf/package.json +++ b/packages/adf/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/adf", "sideEffects": false, - "version": "1.8.0", + "version": "1.9.0", "description": "ADF (Attention-Directed Format) — AST-backed context format for AI agents", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/adf/src/__tests__/bundler.test.ts b/packages/adf/src/__tests__/bundler.test.ts index f8bf8f5..6dacd66 100644 --- a/packages/adf/src/__tests__/bundler.test.ts +++ b/packages/adf/src/__tests__/bundler.test.ts @@ -236,6 +236,12 @@ describe('bundleModules', () => { } }); + it('populates documents with each resolved module\'s parsed doc, keyed by path', () => { + const result = bundleModules('/ai', ['core.adf', 'state.adf'], readFile); + expect(Object.keys(result.documents ?? {})).toEqual(['core.adf', 'state.adf']); + expect(result.documents?.['core.adf'].sections.some(s => s.key === 'CONSTRAINTS')).toBe(true); + }); + it('returns token estimate > 0', () => { const result = bundleModules('/ai', ['core.adf'], readFile); expect(result.tokenEstimate).toBeGreaterThan(0); diff --git a/packages/adf/src/__tests__/evidence.test.ts b/packages/adf/src/__tests__/evidence.test.ts index a314bf4..9ffcf63 100644 --- a/packages/adf/src/__tests__/evidence.test.ts +++ b/packages/adf/src/__tests__/evidence.test.ts @@ -188,4 +188,88 @@ describe('evaluateEvidence', () => { const report = evaluateEvidence(makeBundleResult()); expect(report.staleBaselines).toEqual([]); }); + + it('attributes constraints to their owning module when bundle.documents is present', () => { + const bundle = makeBundleResult({ + resolvedModules: ['frontend.adf', 'backend.adf'], + documents: { + 'frontend.adf': { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'entry_loc', value: 300, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + 'backend.adf': { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'handler_loc', value: 50, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }, + }); + const report = evaluateEvidence(bundle); + expect(report.constraints).toHaveLength(2); + const failing = report.constraints.find((c) => c.metric === 'entry_loc'); + expect(failing?.module).toBe('frontend.adf'); + expect(failing?.status).toBe('fail'); + const passing = report.constraints.find((c) => c.metric === 'handler_loc'); + expect(passing?.module).toBe('backend.adf'); + expect(passing?.status).toBe('pass'); + expect(report.failCount).toBe(1); + expect(report.allPassing).toBe(false); + }); + + it('falls back to unattributed merged-document validation when bundle.documents is absent', () => { + const report = evaluateEvidence( + makeBundleResult({ + mergedDocument: { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }), + ); + expect(report.constraints).toHaveLength(1); + expect(report.constraints[0].module).toBeUndefined(); + }); + + it('falls back to merged-document validation when documents is present but missing an entry for a resolved module', () => { + // `documents: {}` is truthy but incomplete -- must not be treated the + // same as "per-module data available," or a resolved module with no + // matching document would silently produce zero constraints instead of + // falling back. + const report = evaluateEvidence( + makeBundleResult({ + resolvedModules: ['core.adf'], + documents: {}, + mergedDocument: { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'loc', value: 250, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }), + ); + expect(report.constraints).toHaveLength(1); + expect(report.constraints[0].status).toBe('fail'); + expect(report.constraints[0].module).toBeUndefined(); + expect(report.allPassing).toBe(false); + }); }); diff --git a/packages/adf/src/bundler.ts b/packages/adf/src/bundler.ts index 382acda..4c7b842 100644 --- a/packages/adf/src/bundler.ts +++ b/packages/adf/src/bundler.ts @@ -40,7 +40,7 @@ export function bundleModules( const manifest = preloadedManifest ?? loadAndParseManifest(basePath, readFile); const triggerMatches = buildTriggerReport(manifest, modulePaths, taskKeywords); - const documents: AdfDocument[] = []; + const documentsByPath: Record = {}; const perModuleTokens: Record = {}; const advisoryOnlyModules: string[] = []; const defaultLoadSet = new Set(manifest.defaultLoad); @@ -54,7 +54,7 @@ export function bundleModules( throw new AdfBundleError(`Module not found: ${modPath}`, modPath); } const doc = parseAdf(content); - documents.push(doc); + documentsByPath[modPath] = doc; perModuleTokens[modPath] = estimateTokens(doc); // Flag on-demand modules with no load-bearing sections @@ -66,7 +66,10 @@ export function bundleModules( } } - const merged = mergeDocuments(documents); + // Module paths are never integer-index-like strings, so Object.values + // preserves insertion order — safe to derive the merge input from + // documentsByPath instead of maintaining a second parallel array. + const merged = mergeDocuments(Object.values(documentsByPath)); const tokenEstimate = estimateTokens(merged); const tokenBudget = manifest.tokenBudget ?? null; const tokenUtilization = tokenBudget !== null ? tokenEstimate / tokenBudget : null; @@ -103,6 +106,7 @@ export function bundleModules( triggerMatches, unmatchedModules, advisoryOnlyModules, + documents: documentsByPath, }; } diff --git a/packages/adf/src/evidence.ts b/packages/adf/src/evidence.ts index 0966bbc..c22ca5b 100644 --- a/packages/adf/src/evidence.ts +++ b/packages/adf/src/evidence.ts @@ -10,9 +10,10 @@ import type { AdfDocument, BundleResult, + ConstraintResult, EvidenceResult, } from './types'; -import { validateConstraints } from './validator'; +import { computeConstraints, computeWeightSummary } from './validator'; // ============================================================================ // Types @@ -58,7 +59,18 @@ export function evaluateEvidence( context?: Record, staleThreshold?: number, ): EvidenceReport { - const evidence = validateConstraints(bundle.mergedDocument, context); + // When per-module documents are available (the real bundleModules() path), + // validate each one separately so ConstraintResults are attributed back to + // their owning module. Falls back to the merged document (unattributed, + // matching pre-attribution behavior) for hand-built BundleResults that + // don't populate `documents` — checked for genuine full coverage, not bare + // truthiness, since `documents: {}` is truthy but has nothing to validate. + const constraints = hasCompleteDocuments(bundle) + ? gatherPerModuleConstraints(bundle.documents, bundle.resolvedModules, context) + : computeConstraints(bundle.mergedDocument, context); + const failCount = constraints.filter((c) => c.status === 'fail').length; + const warnCount = constraints.filter((c) => c.status === 'warn').length; + const staleBaselines = detectStaleBaselines( bundle.mergedDocument, context, @@ -66,7 +78,11 @@ export function evaluateEvidence( ); return { - ...evidence, + constraints, + weightSummary: computeWeightSummary(bundle.mergedDocument), + allPassing: failCount === 0, + failCount, + warnCount, tokenEstimate: bundle.tokenEstimate, tokenBudget: bundle.tokenBudget, tokenUtilization: bundle.tokenUtilization, @@ -77,6 +93,40 @@ export function evaluateEvidence( }; } +/** + * True only when `documents` has a parsed document for every resolved + * module — not merely present. A bare truthiness check on `bundle.documents` + * would treat `{}` (present but empty/incomplete) as "per-module data + * available," silently validating nothing for any module missing a document + * instead of falling back to the merged-document path. + */ +function hasCompleteDocuments( + bundle: BundleResult, +): bundle is BundleResult & { documents: Record } { + const documents = bundle.documents; + if (documents === undefined) return false; + return bundle.resolvedModules.every((modPath) => modPath in documents); +} + +/** + * Validate each resolved module's document separately and tag results with + * their module path, instead of validating the merged document where module + * identity is already lost. + */ +function gatherPerModuleConstraints( + documents: Record, + resolvedModules: string[], + context?: Record, +): ConstraintResult[] { + const constraints: ConstraintResult[] = []; + for (const modPath of resolvedModules) { + const doc = documents[modPath]; + if (!doc) continue; + constraints.push(...computeConstraints(doc, context, modPath)); + } + return constraints; +} + // ============================================================================ // Stale Baseline Detection // ============================================================================ diff --git a/packages/adf/src/types/bundle.ts b/packages/adf/src/types/bundle.ts index 4a17862..45c102f 100644 --- a/packages/adf/src/types/bundle.ts +++ b/packages/adf/src/types/bundle.ts @@ -27,4 +27,11 @@ export interface BundleResult { }>; unmatchedModules: string[]; advisoryOnlyModules: string[]; + /** + * Parsed document for each resolved module, keyed by module path. Populated + * by `bundleModules()` (absent on hand-built BundleResults, e.g. in tests) + * so `evaluateEvidence` can validate per-module and attribute constraint + * results back to their owning module instead of only the merged document. + */ + documents?: Record; } diff --git a/packages/adf/src/types/validation.ts b/packages/adf/src/types/validation.ts index d7a8377..6776ba0 100644 --- a/packages/adf/src/types/validation.ts +++ b/packages/adf/src/types/validation.ts @@ -13,6 +13,8 @@ export interface ConstraintResult { status: ConstraintStatus; message: string; source: 'metric' | 'context'; + /** Path of the `.ai/*.adf` module this constraint was checked against, when known. */ + module?: string; } export interface WeightSummary { diff --git a/packages/adf/src/validator.ts b/packages/adf/src/validator.ts index 8c2ce68..e84fce0 100644 --- a/packages/adf/src/validator.ts +++ b/packages/adf/src/validator.ts @@ -23,11 +23,39 @@ import type { * * @param doc - Parsed ADF document * @param context - Optional external metric overrides (e.g., actual LOC counts) + * @param module - Path of the source module this document was loaded from, when known. + * Stamped onto each ConstraintResult for failure attribution (e.g. `charter adf suggest`). */ export function validateConstraints( doc: AdfDocument, context?: Record, + module?: string, ): EvidenceResult { + const constraints = computeConstraints(doc, context, module); + const failCount = constraints.filter(c => c.status === 'fail').length; + const warnCount = constraints.filter(c => c.status === 'warn').length; + + return { + constraints, + weightSummary: computeWeightSummary(doc), + allPassing: failCount === 0, + failCount, + warnCount, + }; +} + +/** + * Compute per-entry ConstraintResults for a single document's METRICS + * sections, without the aggregate weightSummary/failCount/warnCount wrapper. + * Shared by validateConstraints and evidence.ts's per-module attribution + * path, so validating N modules doesn't also compute (and discard) N + * redundant weight summaries and pass/fail counts. + */ +export function computeConstraints( + doc: AdfDocument, + context?: Record, + module?: string, +): ConstraintResult[] { const constraints: ConstraintResult[] = []; for (const section of doc.sections) { @@ -48,20 +76,12 @@ export function validateConstraints( status, message: `${entry.key}: ${value} / ${entry.ceiling} [${entry.unit}] -- ${statusLabel}`, source: hasContext ? 'context' : 'metric', + ...(module !== undefined ? { module } : {}), }); } } - const failCount = constraints.filter(c => c.status === 'fail').length; - const warnCount = constraints.filter(c => c.status === 'warn').length; - - return { - constraints, - weightSummary: computeWeightSummary(doc), - allPassing: failCount === 0, - failCount, - warnCount, - }; + return constraints; } /** diff --git a/packages/cli/package.json b/packages/cli/package.json index ff5312b..13b2b82 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/cli", "sideEffects": false, - "version": "1.8.0", + "version": "1.9.0", "description": "Charter CLI — repo-level governance toolkit", "bin": { "charter": "./dist/bin.js" diff --git a/packages/cli/src/__tests__/adf-suggest.test.ts b/packages/cli/src/__tests__/adf-suggest.test.ts index 04c3cd2..a5edb89 100644 --- a/packages/cli/src/__tests__/adf-suggest.test.ts +++ b/packages/cli/src/__tests__/adf-suggest.test.ts @@ -4,7 +4,7 @@ import * as path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import type { CLIOptions } from '../index'; import { adfSuggestCommand, buildSuggestReport } from '../commands/adf-suggest'; -import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; +import type { AdfConstraintEvent, AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; const originalCwd = process.cwd(); const tempDirs: string[] = []; @@ -56,6 +56,19 @@ function commandEvent(overrides: Partial = {}): CliTelemetryE }; } +function constraintEvent(overrides: Partial = {}): AdfConstraintEvent { + return { + version: 1, + eventType: 'adf.constraint', + timestamp: new Date().toISOString(), + sessionId: null, + module: 'governance.adf', + metric: 'entry_loc', + status: 'fail', + ...overrides, + }; +} + describe('buildSuggestReport', () => { it('flags a dead module once it appears enough times with zero matches', () => { const events = Array.from({ length: 3 }, () => @@ -234,7 +247,7 @@ describe('buildSuggestReport', () => { }); const cmds = [commandEvent({ sessionId: 'sess-2', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; const report = buildSuggestReport([res], cmds, 1, 60, 'events.ndjson'); - expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 1 }]); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 1, exactFailures: 0 }]); }); it('does not report loaded-but-violated when nothing downstream failed', () => { @@ -244,6 +257,46 @@ describe('buildSuggestReport', () => { expect(report.loadedButViolated).toHaveLength(0); }); + it('reports exact loaded-but-violated attribution from adf.constraint fail events, joined by sessionId', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 1 }]); + }); + + it('does not multiply exactFailures by the number of resolution events that share a session with one real failure', () => { + const resolutions = [1, 2, 3, 4, 5].map(() => + resolutionEvent({ sessionId: 'sess-fanout', resolvedModules: ['governance.adf'], triggerMatches: [] }), + ); + const constraints = [constraintEvent({ sessionId: 'sess-fanout', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport(resolutions, [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 5, correlatedFailures: 0, exactFailures: 1 }]); + }); + + it('counts exactFailures as the number of distinct fail events for a module, not the number of resolutions', () => { + const res = resolutionEvent({ sessionId: 'sess-multi', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [ + constraintEvent({ sessionId: 'sess-multi', module: 'governance.adf', status: 'fail' }), + constraintEvent({ sessionId: 'sess-multi', module: 'governance.adf', status: 'fail' }), + ]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 2 }]); + }); + + it('does not attribute an adf.constraint fail event to a module that was not resolved in that session', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['frontend.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toHaveLength(0); + }); + + it('ignores adf.constraint events with a pass/warn status', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'pass' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toHaveLength(0); + }); + it('flags insufficientData when resolution events are below the threshold', () => { const report = buildSuggestReport([], [], 3, 60, 'events.ndjson'); expect(report.insufficientData).toBe(true); @@ -426,6 +479,25 @@ describe('adfSuggestCommand', () => { expect(report.commandEvents).toBe(0); }); + it('reads adf.constraint fail events from events.ndjson and surfaces exact loaded-but-violated attribution', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const res = resolutionEvent({ sessionId: 'sess-e2e', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraint = constraintEvent({ sessionId: 'sess-e2e', module: 'governance.adf', metric: 'entry_loc', status: 'fail' }); + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), [res, constraint].map((e) => JSON.stringify(e)).join('\n') + '\n'); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, []); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { loadedButViolated: Array<{ module: string; exactFailures: number; correlatedFailures: number }> }; + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 1 }]); + }); + it('prints a human-readable report in text format', () => { const tmp = tmpDir(); process.chdir(tmp); diff --git a/packages/cli/src/__tests__/integration/evidence-contracts.test.ts b/packages/cli/src/__tests__/integration/evidence-contracts.test.ts index 082830c..1eb6e6b 100644 --- a/packages/cli/src/__tests__/integration/evidence-contracts.test.ts +++ b/packages/cli/src/__tests__/integration/evidence-contracts.test.ts @@ -210,4 +210,43 @@ describe('adf evidence CI contracts (integration)', () => { expect(output.staleBaselineCount).toBe(0); expect(output.staleBaselines).toBeUndefined(); // field omitted when empty }); + + // ── Module attribution: constraints know which module they came from ── + it('attributes each constraint to its owning module (core.adf)', () => { + const tmp = makeTempDir('module-attribution'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 79, /* baseline */ 80, /* ceiling */ 150); + + const { output } = captureJson(adfEvidence, jsonOptions, ['--auto-measure']); + const constraints = output.constraints as Array<{ metric: string; module?: string }>; + expect(constraints[0].module).toBe('core.adf'); + }); + + // ── Exact-attribution telemetry: adf.constraint fail events on disk ──── + it('records an adf.constraint fail event for a breached ceiling', () => { + const tmp = makeTempDir('constraint-event'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 160, /* baseline */ 100, /* ceiling */ 150); + + captureJson(adfEvidence, ciOptions, ['--auto-measure']); + + const telemetryFile = path.join(tmp, '.charter', 'telemetry', 'events.ndjson'); + const lines = fs.readFileSync(telemetryFile, 'utf-8').trim().split('\n'); + const events = lines.map((l) => JSON.parse(l) as Record); + const constraintEvent = events.find((e) => e.eventType === 'adf.constraint'); + expect(constraintEvent).toMatchObject({ module: 'core.adf', metric: 'app_loc', status: 'fail' }); + }); + + it('does not record an adf.constraint event when all constraints pass', () => { + const tmp = makeTempDir('constraint-event-pass'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 79, /* baseline */ 80, /* ceiling */ 150); + + captureJson(adfEvidence, ciOptions, ['--auto-measure']); + + // A run with no failing constraints has nothing to attribute, so + // recordAdfConstraintEvents never even creates the telemetry file. + const telemetryFile = path.join(tmp, '.charter', 'telemetry', 'events.ndjson'); + expect(fs.existsSync(telemetryFile)).toBe(false); + }); }); diff --git a/packages/cli/src/commands/adf-evidence.ts b/packages/cli/src/commands/adf-evidence.ts index cbe202c..c22eb0f 100644 --- a/packages/cli/src/commands/adf-evidence.ts +++ b/packages/cli/src/commands/adf-evidence.ts @@ -18,6 +18,7 @@ import type { CLIOptions } from '../index'; import { CLIError, EXIT_CODE } from '../index'; import { getFlag, readFlagFile, tokenizeTask } from '../flags'; import { hashContent, loadLockFile } from './adf-sync'; +import { recordAdfConstraintEvents } from '../telemetry'; interface AutoMeasurement { metric: string; @@ -94,6 +95,7 @@ export function adfEvidence(options: CLIOptions, args: string[]): number { try { const bundle = bundleModules(aiDir, modulePaths, readFile, keywords, manifest); const report: EvidenceReport = evaluateEvidence(bundle, context, staleThreshold); + recordAdfConstraintEvents(options.configPath, { results: report.constraints }); // Check sync status const lockFile = path.join(aiDir, '.adf.lock'); diff --git a/packages/cli/src/commands/adf-suggest.ts b/packages/cli/src/commands/adf-suggest.ts index c777f07..d073c8d 100644 --- a/packages/cli/src/commands/adf-suggest.ts +++ b/packages/cli/src/commands/adf-suggest.ts @@ -3,14 +3,20 @@ * * Diagnostics over .charter/telemetry/events.ndjson: which ADF modules * never fire, which task keywords never match a trigger anywhere, and - * which modules load yet a downstream command still fails. + * which modules load yet still get violated. * - * By default this is entirely report-only. Attributing a *failure* to a - * specific module's trigger list would require knowing which module's - * constraint was violated, and nothing in current telemetry (validate/ - * drift/evidence check commit trailers, CLAUDE.md sync, and METRICS - * ceilings respectively) records that — so no op is ever generated from - * the failure-correlation signal alone. + * `loadedButViolated` now has two tiers of evidence: + * - Exact: `charter adf evidence` records an `adf.constraint` event for + * every failing METRICS ceiling, tagged with the specific module it + * belongs to (see `validateConstraints`'s `module` param). Joining + * resolution events against these by module + session/time gives a + * real failure attribution, not a guess. + * - Correlated (fallback): for constraint checks that don't emit + * module-tagged evidence (commit-trailer validation, blessed-stack + * drift scans — neither of which is module-shaped), or for telemetry + * predating this event, we fall back to "some downstream command in + * this session/window failed" — a much weaker signal, kept only so the + * detector still says something in the absence of exact evidence. * * `--emit-ops` adds one narrower, self-contained mechanism on top: a * co-occurrence heuristic. If an unmatched keyword frequently appears @@ -30,7 +36,7 @@ import type { PatchOperation } from '@stackbilt/adf'; import type { CLIOptions } from '../index'; import { EXIT_CODE } from '../index'; import { getFlag } from '../flags'; -import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; +import type { AdfConstraintEvent, AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; const DEFAULT_MIN_OCCURRENCES = 3; const DEFAULT_WINDOW_MINUTES = 60; @@ -60,7 +66,10 @@ export interface NearMissKeywordFinding { export interface LoadedButViolatedFinding { module: string; occurrences: number; + /** Downstream command in the session/window failed — no confirmed link to this module's own constraints. */ correlatedFailures: number; + /** This module's own METRICS ceiling was confirmed to fail (adf.constraint events), joined by session/window. */ + exactFailures: number; } /** @@ -105,8 +114,8 @@ export function adfSuggestCommand(options: CLIOptions, args: string[]): number { const emitOpsFile = getFlag(args, '--emit-ops'); const telemetryFile = path.join(options.configPath, 'telemetry', 'events.ndjson'); - const { resolutionEvents, commandEvents } = readEvents(telemetryFile); - const report = buildSuggestReport(resolutionEvents, commandEvents, minOccurrences, windowMinutes, telemetryFile); + const { resolutionEvents, commandEvents, constraintEvents } = readEvents(telemetryFile); + const report = buildSuggestReport(resolutionEvents, commandEvents, minOccurrences, windowMinutes, telemetryFile, constraintEvents); const emitResult = buildEmitOps(report.coOccurrenceCandidates, aiDir); if (emitOpsFile) { @@ -164,16 +173,12 @@ function buildEmitOps(candidates: CoOccurrenceCandidate[], aiDir: string): EmitO // REPLACE_BULLET op with both keywords appended, not two ops at the same // index — applying them in sequence would make the second silently // clobber the first (adf patch applies against the evolving document). - const byModule = new Map(); for (const candidate of candidates) { if (candidate.ambiguous) { skipped.push({ keyword: candidate.keyword, module: candidate.module, reason: 'ambiguous: tied with another module for top co-occurrence count' }); - continue; } - const group = byModule.get(candidate.module) ?? []; - group.push(candidate); - byModule.set(candidate.module, group); } + const byModule = groupBy(candidates.filter((c) => !c.ambiguous), (c) => c.module); for (const [modulePath, groupCandidates] of byModule) { const modIndex = manifest.onDemand.findIndex((m) => m.path === modulePath); @@ -220,15 +225,17 @@ function parsePositiveInt(raw: string | undefined): number | undefined { function readEvents(telemetryFile: string): { resolutionEvents: AdfResolutionEvent[]; commandEvents: CliTelemetryEvent[]; + constraintEvents: AdfConstraintEvent[]; } { const resolutionEvents: AdfResolutionEvent[] = []; const commandEvents: CliTelemetryEvent[] = []; + const constraintEvents: AdfConstraintEvent[] = []; let raw: string; try { raw = fs.readFileSync(telemetryFile, 'utf-8'); } catch { - return { resolutionEvents, commandEvents }; + return { resolutionEvents, commandEvents, constraintEvents }; } for (const line of raw.split(/\r?\n/)) { @@ -243,11 +250,14 @@ function readEvents(telemetryFile: string): { } if (!parsed || typeof parsed !== 'object') continue; - const record = parsed as { eventType?: string; timestamp?: unknown; commandPath?: unknown; exitCode?: unknown }; + const record = parsed as { eventType?: string; timestamp?: unknown; commandPath?: unknown; exitCode?: unknown; module?: unknown; status?: unknown }; if (typeof record.timestamp !== 'string') continue; if (record.eventType === 'adf.resolution') { resolutionEvents.push(parsed as AdfResolutionEvent); + } else if (record.eventType === 'adf.constraint') { + if (typeof record.module !== 'string' || typeof record.status !== 'string') continue; + constraintEvents.push(parsed as AdfConstraintEvent); } else if (record.eventType === 'command' || record.eventType === undefined) { // Events written before eventType existed have no discriminator; // treat those as command events for backward compatibility. Require @@ -259,24 +269,29 @@ function readEvents(telemetryFile: string): { } } - return { resolutionEvents, commandEvents }; + return { resolutionEvents, commandEvents, constraintEvents }; +} + +interface Joinable { + sessionId?: string | null; + timestamp: string; } /** - * Whether a command event falls within the join window of a resolution - * event: same sessionId (high confidence) if both are stamped, else a - * forward-only time-window fallback (heuristic). + * Whether two events fall within the join window of each other: same + * sessionId (high confidence) if both are stamped, else a forward-only + * time-window fallback (heuristic). */ -function isJoined(resolution: AdfResolutionEvent, command: CliTelemetryEvent, windowMinutes: number): boolean { - if (resolution.sessionId && command.sessionId) { - return resolution.sessionId === command.sessionId; +function isJoined(a: Joinable, b: Joinable, windowMinutes: number): boolean { + if (a.sessionId && b.sessionId) { + return a.sessionId === b.sessionId; } - const resTime = Date.parse(resolution.timestamp); - const cmdTime = Date.parse(command.timestamp); - if (!Number.isFinite(resTime) || !Number.isFinite(cmdTime)) return false; + const aTime = Date.parse(a.timestamp); + const bTime = Date.parse(b.timestamp); + if (!Number.isFinite(aTime) || !Number.isFinite(bTime)) return false; - const deltaMs = cmdTime - resTime; + const deltaMs = bTime - aTime; return deltaMs >= 0 && deltaMs <= windowMinutes * 60_000; } @@ -288,29 +303,63 @@ function hasCorrelatedFailure( return failedCommandEvents.some((c) => isJoined(resolution, c, windowMinutes)); } +/** Group items by a derived key, preserving each group's insertion order. */ +function groupBy(items: T[], keyFn: (item: T) => string): Map { + const groups = new Map(); + for (const item of items) { + const key = keyFn(item); + const group = groups.get(key) ?? []; + group.push(item); + groups.set(key, group); + } + return groups; +} + +interface LoadedModuleStats { + occurrences: number; + correlatedFailures: number; + exactFailures: number; +} + export function buildSuggestReport( resolutionEvents: AdfResolutionEvent[], commandEvents: CliTelemetryEvent[], minOccurrences: number, windowMinutes: number, sourceFile: string, + constraintEvents: AdfConstraintEvent[] = [], ): SuggestReport { const moduleOccurrences = new Map(); const moduleMatched = new Map(); const globallyMatchedKeywords = new Set(); - const loadedOccurrences = new Map(); - const loadedCorrelatedFailures = new Map(); + const loadedModuleStats = new Map(); const keywordOccurrences = new Map(); const keywordCorrelatedFailures = new Map(); // keyword -> module -> co-occurrence count (module genuinely ON_DEMAND-triggered // in the same event, not just default-loaded). const keywordModuleCoOccurrence = new Map>(); + function getLoadedStats(module: string): LoadedModuleStats { + let stats = loadedModuleStats.get(module); + if (!stats) { + stats = { occurrences: 0, correlatedFailures: 0, exactFailures: 0 }; + loadedModuleStats.set(module, stats); + } + return stats; + } + // Only failed commands can ever satisfy hasCorrelatedFailure/isJoined, so // scanning just this (typically much smaller) subset per resolution event // avoids an O(resolutions x commands) scan over the full command history. const failedCommandEvents = commandEvents.filter((c) => c.success === false); + // Fail-status constraint events, grouped by the module they were checked + // against — the exact-attribution counterpart to failedCommandEvents. + const failedConstraintEventsByModule = groupBy( + constraintEvents.filter((ev) => ev.status === 'fail'), + (ev) => ev.module, + ); + for (const res of resolutionEvents) { // triggerMatches has one entry per (module, trigger) pair, so a module // declared with multiple triggers must be deduped per event here — @@ -333,7 +382,7 @@ export function buildSuggestReport( moduleMatched.set(mod, (moduleMatched.get(mod) ?? 0) + 1); } for (const mod of res.resolvedModules) { - loadedOccurrences.set(mod, (loadedOccurrences.get(mod) ?? 0) + 1); + getLoadedStats(mod).occurrences += 1; } } @@ -383,11 +432,26 @@ export function buildSuggestReport( if (failed) { for (const mod of res.resolvedModules) { - loadedCorrelatedFailures.set(mod, (loadedCorrelatedFailures.get(mod) ?? 0) + 1); + getLoadedStats(mod).correlatedFailures += 1; } } } + // Exact attribution: each confirmed constraint failure counts once, + // regardless of how many resolution events happened to load that module + // (a module resolved 5x with 1 real failure must report exactFailures: 1, + // not 5 — joining per resolution event, as correlatedFailures does, would + // multiply a single failure by however many times the module was loaded). + // Only attributed to modules that appear as "loaded" at all: an + // adf.constraint event for a module never resolved via adf bundle/context/ + // serve can't be reported as loaded-but-violated by definition. + for (const [module, events] of failedConstraintEventsByModule) { + const stats = loadedModuleStats.get(module); + if (stats) { + stats.exactFailures = events.length; + } + } + const deadModules: DeadModuleFinding[] = [...moduleOccurrences.entries()] .filter(([module, occurrences]) => occurrences >= minOccurrences && (moduleMatched.get(module) ?? 0) === 0) .map(([module, occurrences]) => ({ module, occurrences })) @@ -402,14 +466,15 @@ export function buildSuggestReport( })) .sort((a, b) => b.correlatedFailures - a.correlatedFailures || b.occurrences - a.occurrences); - const loadedButViolated: LoadedButViolatedFinding[] = [...loadedCorrelatedFailures.entries()] - .filter(([, correlatedFailures]) => correlatedFailures > 0) - .map(([module, correlatedFailures]) => ({ + const loadedButViolated: LoadedButViolatedFinding[] = [...loadedModuleStats.entries()] + .filter(([, stats]) => stats.correlatedFailures > 0 || stats.exactFailures > 0) + .map(([module, stats]) => ({ module, - occurrences: loadedOccurrences.get(module) ?? 0, - correlatedFailures, + occurrences: stats.occurrences, + correlatedFailures: stats.correlatedFailures, + exactFailures: stats.exactFailures, })) - .sort((a, b) => b.correlatedFailures - a.correlatedFailures); + .sort((a, b) => b.exactFailures - a.exactFailures || b.correlatedFailures - a.correlatedFailures); const nearMissKeywordSet = new Set(nearMissKeywords.map((f) => f.keyword)); const coOccurrenceCandidates: CoOccurrenceCandidate[] = []; @@ -490,7 +555,10 @@ function printReport(report: SuggestReport): void { console.log(' (none)'); } else { for (const f of report.loadedButViolated) { - console.log(` [!] ${f.module} — resolved ${f.occurrences}x, ${f.correlatedFailures} correlated with a downstream failure`); + const parts: string[] = []; + if (f.exactFailures > 0) parts.push(`${f.exactFailures} confirmed constraint failure${f.exactFailures === 1 ? '' : 's'}`); + if (f.correlatedFailures > 0) parts.push(`${f.correlatedFailures} correlated with a downstream failure`); + console.log(` [!] ${f.module} — resolved ${f.occurrences}x, ${parts.join(', ')}`); } console.log(' Note: routing worked; rule content may be weak. Needs human/strong-model rewrite, not a trigger change.'); } diff --git a/packages/cli/src/telemetry.ts b/packages/cli/src/telemetry.ts index 5f6977f..287c656 100644 --- a/packages/cli/src/telemetry.ts +++ b/packages/cli/src/telemetry.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { randomUUID } from 'node:crypto'; -import type { BundleResult } from '@stackbilt/adf'; +import type { BundleResult, ConstraintStatus } from '@stackbilt/adf'; export interface CliTelemetryEvent { version: 1; @@ -46,6 +46,72 @@ export interface AdfResolutionEvent { tokenEstimate?: number; } +/** + * Sibling event recording a single failed METRICS constraint, attributed to + * the specific `.ai/*.adf` module it was checked against (see + * `validateConstraints`'s `module` parameter). Unlike AdfResolutionEvent's + * command-success correlation, this is exact evidence: `charter adf + * suggest`'s `loadedButViolated` detector can join this directly on module + * (plus sessionId/time-window) instead of guessing from a downstream + * command's exit code. + */ +export interface AdfConstraintEvent { + version: 1; + eventType: 'adf.constraint'; + timestamp: string; + sessionId: string | null; + module: string; + metric: string; + status: ConstraintStatus; +} + +export interface RecordAdfConstraintInput { + results: Array<{ module?: string; metric: string; status: ConstraintStatus }>; + /** Override the CHARTER_SESSION_ID env lookup, same as RecordAdfResolutionInput. */ + sessionId?: string | null; +} + +/** + * Persist one event per failing, module-attributed constraint (best-effort, + * never throws). Passing/warning constraints aren't recorded — only `fail` + * is a "violation" in the sense `loadedButViolated` cares about — and + * results with no `module` are skipped since there's nothing to attribute. + */ +export function recordAdfConstraintEvents(configPath: string, input: RecordAdfConstraintInput): void { + try { + const violated = input.results.filter( + (r): r is { module: string; metric: string; status: ConstraintStatus } => + r.module !== undefined && r.status === 'fail', + ); + if (violated.length === 0) return; + + const telemetryDir = path.join(configPath, 'telemetry'); + const telemetryFile = path.join(telemetryDir, 'events.ndjson'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const timestamp = new Date().toISOString(); + const sessionId = input.sessionId ?? getSessionId() ?? null; + const lines = violated + .map((r) => { + const event: AdfConstraintEvent = { + version: 1, + eventType: 'adf.constraint', + timestamp, + sessionId, + module: r.module, + metric: r.metric, + status: r.status, + }; + return JSON.stringify(event); + }) + .join('\n'); + + fs.appendFileSync(telemetryFile, `${lines}\n`); + } catch { + // Telemetry is best-effort and must never block command execution. + } +} + export interface RecordAdfResolutionInput { source: AdfResolutionEvent['source']; keywords: string[];