From 30be189ed1c5539ae843716f8eb9ac0d0533d331 Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Sat, 4 Jul 2026 04:26:48 -0500 Subject: [PATCH 1/2] feat(adf): real module-attribution for METRICS constraint failures charter adf suggest's loadedButViolated detector previously could only correlate "some downstream command failed nearby" with a loaded module (session/time-window heuristic) -- it had no way to know which module's own constraint actually failed. Scope note / premise correction: earlier work in this area assumed validate/drift/evidence all needed module-attribution. That's wrong -- validate checks git commit trailers and drift checks blessed-stack anti-patterns in source files; neither is module-shaped. Only adf-evidence's METRICS ceiling checks (and serve.ts's getProjectState, left untouched since it already groups output by module) actually check per-module ADF constraints. This closes that gap only. Changes (all additive, no breaking changes): - ConstraintResult gains an optional `module` field; validateConstraints takes an optional third `module` param to stamp it. - BundleResult gains an optional `documents` map (module path -> parsed doc), populated by bundleModules(). evaluateEvidence validates each module's document separately when available, attributing constraints to their module -- falling back to the old merged-document validation when `documents` is absent, so existing BundleResult fixtures/tests are unaffected. - New AdfConstraintEvent telemetry type + recordAdfConstraintEvents(): records one event per failing, module-attributed constraint. Wired into `charter adf evidence`. - adf-suggest's loadedButViolated now reports exactFailures (joined on module + session/time against adf.constraint fail events) alongside the existing correlatedFailures (kept as the fallback signal for non-module-shaped checks and pre-existing telemetry). Verified end-to-end with real CLI invocations (adf bundle -> adf evidence --auto-measure with a breached ceiling -> adf suggest), not just hand-written telemetry fixtures, to confirm the join actually fires on a real command sequence. packages/adf and packages/cli bumped 1.8.0 -> 1.9.0. Co-Authored-By: Claude Sonnet 5 --- packages/adf/package.json | 2 +- packages/adf/src/__tests__/bundler.test.ts | 6 ++ packages/adf/src/__tests__/evidence.test.ts | 57 ++++++++++ packages/adf/src/bundler.ts | 3 + packages/adf/src/evidence.ts | 40 ++++++- packages/adf/src/types/bundle.ts | 7 ++ packages/adf/src/types/validation.ts | 2 + packages/adf/src/validator.ts | 4 + packages/cli/package.json | 2 +- .../cli/src/__tests__/adf-suggest.test.ts | 57 +++++++++- .../integration/evidence-contracts.test.ts | 39 +++++++ packages/cli/src/commands/adf-evidence.ts | 2 + packages/cli/src/commands/adf-suggest.ts | 100 +++++++++++++----- packages/cli/src/telemetry.ts | 68 +++++++++++- 14 files changed, 352 insertions(+), 37 deletions(-) diff --git a/packages/adf/package.json b/packages/adf/package.json index d5ec77d..97c1cdc 100644 --- a/packages/adf/package.json +++ b/packages/adf/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/adf", "sideEffects": false, - "version": "1.8.0", + "version": "1.9.0", "description": "ADF (Attention-Directed Format) — AST-backed context format for AI agents", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/adf/src/__tests__/bundler.test.ts b/packages/adf/src/__tests__/bundler.test.ts index f8bf8f5..6dacd66 100644 --- a/packages/adf/src/__tests__/bundler.test.ts +++ b/packages/adf/src/__tests__/bundler.test.ts @@ -236,6 +236,12 @@ describe('bundleModules', () => { } }); + it('populates documents with each resolved module\'s parsed doc, keyed by path', () => { + const result = bundleModules('/ai', ['core.adf', 'state.adf'], readFile); + expect(Object.keys(result.documents ?? {})).toEqual(['core.adf', 'state.adf']); + expect(result.documents?.['core.adf'].sections.some(s => s.key === 'CONSTRAINTS')).toBe(true); + }); + it('returns token estimate > 0', () => { const result = bundleModules('/ai', ['core.adf'], readFile); expect(result.tokenEstimate).toBeGreaterThan(0); diff --git a/packages/adf/src/__tests__/evidence.test.ts b/packages/adf/src/__tests__/evidence.test.ts index a314bf4..cb36b45 100644 --- a/packages/adf/src/__tests__/evidence.test.ts +++ b/packages/adf/src/__tests__/evidence.test.ts @@ -188,4 +188,61 @@ describe('evaluateEvidence', () => { const report = evaluateEvidence(makeBundleResult()); expect(report.staleBaselines).toEqual([]); }); + + it('attributes constraints to their owning module when bundle.documents is present', () => { + const bundle = makeBundleResult({ + resolvedModules: ['frontend.adf', 'backend.adf'], + documents: { + 'frontend.adf': { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'entry_loc', value: 300, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + 'backend.adf': { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'handler_loc', value: 50, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }, + }); + const report = evaluateEvidence(bundle); + expect(report.constraints).toHaveLength(2); + const failing = report.constraints.find((c) => c.metric === 'entry_loc'); + expect(failing?.module).toBe('frontend.adf'); + expect(failing?.status).toBe('fail'); + const passing = report.constraints.find((c) => c.metric === 'handler_loc'); + expect(passing?.module).toBe('backend.adf'); + expect(passing?.status).toBe('pass'); + expect(report.failCount).toBe(1); + expect(report.allPassing).toBe(false); + }); + + it('falls back to unattributed merged-document validation when bundle.documents is absent', () => { + const report = evaluateEvidence( + makeBundleResult({ + mergedDocument: { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'loc', value: 100, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }), + ); + expect(report.constraints).toHaveLength(1); + expect(report.constraints[0].module).toBeUndefined(); + }); }); diff --git a/packages/adf/src/bundler.ts b/packages/adf/src/bundler.ts index 382acda..bd3a72e 100644 --- a/packages/adf/src/bundler.ts +++ b/packages/adf/src/bundler.ts @@ -41,6 +41,7 @@ export function bundleModules( const triggerMatches = buildTriggerReport(manifest, modulePaths, taskKeywords); const documents: AdfDocument[] = []; + const documentsByPath: Record = {}; const perModuleTokens: Record = {}; const advisoryOnlyModules: string[] = []; const defaultLoadSet = new Set(manifest.defaultLoad); @@ -55,6 +56,7 @@ export function bundleModules( } const doc = parseAdf(content); documents.push(doc); + documentsByPath[modPath] = doc; perModuleTokens[modPath] = estimateTokens(doc); // Flag on-demand modules with no load-bearing sections @@ -103,6 +105,7 @@ export function bundleModules( triggerMatches, unmatchedModules, advisoryOnlyModules, + documents: documentsByPath, }; } diff --git a/packages/adf/src/evidence.ts b/packages/adf/src/evidence.ts index 0966bbc..9c42ba8 100644 --- a/packages/adf/src/evidence.ts +++ b/packages/adf/src/evidence.ts @@ -10,9 +10,10 @@ import type { AdfDocument, BundleResult, + ConstraintResult, EvidenceResult, } from './types'; -import { validateConstraints } from './validator'; +import { validateConstraints, computeWeightSummary } from './validator'; // ============================================================================ // Types @@ -58,7 +59,17 @@ export function evaluateEvidence( context?: Record, staleThreshold?: number, ): EvidenceReport { - const evidence = validateConstraints(bundle.mergedDocument, context); + // When per-module documents are available (the real bundleModules() path), + // validate each one separately so ConstraintResults are attributed back to + // their owning module. Falls back to the merged document (unattributed, + // matching pre-attribution behavior) for hand-built BundleResults that + // don't populate `documents`. + const constraints = bundle.documents + ? gatherPerModuleConstraints(bundle.documents, bundle.resolvedModules, context) + : validateConstraints(bundle.mergedDocument, context).constraints; + const failCount = constraints.filter((c) => c.status === 'fail').length; + const warnCount = constraints.filter((c) => c.status === 'warn').length; + const staleBaselines = detectStaleBaselines( bundle.mergedDocument, context, @@ -66,7 +77,11 @@ export function evaluateEvidence( ); return { - ...evidence, + constraints, + weightSummary: computeWeightSummary(bundle.mergedDocument), + allPassing: failCount === 0, + failCount, + warnCount, tokenEstimate: bundle.tokenEstimate, tokenBudget: bundle.tokenBudget, tokenUtilization: bundle.tokenUtilization, @@ -77,6 +92,25 @@ export function evaluateEvidence( }; } +/** + * Validate each resolved module's document separately and tag results with + * their module path, instead of validating the merged document where module + * identity is already lost. + */ +function gatherPerModuleConstraints( + documents: Record, + resolvedModules: string[], + context?: Record, +): ConstraintResult[] { + const constraints: ConstraintResult[] = []; + for (const modPath of resolvedModules) { + const doc = documents[modPath]; + if (!doc) continue; + constraints.push(...validateConstraints(doc, context, modPath).constraints); + } + return constraints; +} + // ============================================================================ // Stale Baseline Detection // ============================================================================ diff --git a/packages/adf/src/types/bundle.ts b/packages/adf/src/types/bundle.ts index 4a17862..45c102f 100644 --- a/packages/adf/src/types/bundle.ts +++ b/packages/adf/src/types/bundle.ts @@ -27,4 +27,11 @@ export interface BundleResult { }>; unmatchedModules: string[]; advisoryOnlyModules: string[]; + /** + * Parsed document for each resolved module, keyed by module path. Populated + * by `bundleModules()` (absent on hand-built BundleResults, e.g. in tests) + * so `evaluateEvidence` can validate per-module and attribute constraint + * results back to their owning module instead of only the merged document. + */ + documents?: Record; } diff --git a/packages/adf/src/types/validation.ts b/packages/adf/src/types/validation.ts index d7a8377..6776ba0 100644 --- a/packages/adf/src/types/validation.ts +++ b/packages/adf/src/types/validation.ts @@ -13,6 +13,8 @@ export interface ConstraintResult { status: ConstraintStatus; message: string; source: 'metric' | 'context'; + /** Path of the `.ai/*.adf` module this constraint was checked against, when known. */ + module?: string; } export interface WeightSummary { diff --git a/packages/adf/src/validator.ts b/packages/adf/src/validator.ts index 8c2ce68..d216c65 100644 --- a/packages/adf/src/validator.ts +++ b/packages/adf/src/validator.ts @@ -23,10 +23,13 @@ import type { * * @param doc - Parsed ADF document * @param context - Optional external metric overrides (e.g., actual LOC counts) + * @param module - Path of the source module this document was loaded from, when known. + * Stamped onto each ConstraintResult for failure attribution (e.g. `charter adf suggest`). */ export function validateConstraints( doc: AdfDocument, context?: Record, + module?: string, ): EvidenceResult { const constraints: ConstraintResult[] = []; @@ -48,6 +51,7 @@ export function validateConstraints( status, message: `${entry.key}: ${value} / ${entry.ceiling} [${entry.unit}] -- ${statusLabel}`, source: hasContext ? 'context' : 'metric', + ...(module !== undefined ? { module } : {}), }); } } diff --git a/packages/cli/package.json b/packages/cli/package.json index ff5312b..13b2b82 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@stackbilt/cli", "sideEffects": false, - "version": "1.8.0", + "version": "1.9.0", "description": "Charter CLI — repo-level governance toolkit", "bin": { "charter": "./dist/bin.js" diff --git a/packages/cli/src/__tests__/adf-suggest.test.ts b/packages/cli/src/__tests__/adf-suggest.test.ts index 04c3cd2..36d0028 100644 --- a/packages/cli/src/__tests__/adf-suggest.test.ts +++ b/packages/cli/src/__tests__/adf-suggest.test.ts @@ -4,7 +4,7 @@ import * as path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import type { CLIOptions } from '../index'; import { adfSuggestCommand, buildSuggestReport } from '../commands/adf-suggest'; -import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; +import type { AdfConstraintEvent, AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; const originalCwd = process.cwd(); const tempDirs: string[] = []; @@ -56,6 +56,19 @@ function commandEvent(overrides: Partial = {}): CliTelemetryE }; } +function constraintEvent(overrides: Partial = {}): AdfConstraintEvent { + return { + version: 1, + eventType: 'adf.constraint', + timestamp: new Date().toISOString(), + sessionId: null, + module: 'governance.adf', + metric: 'entry_loc', + status: 'fail', + ...overrides, + }; +} + describe('buildSuggestReport', () => { it('flags a dead module once it appears enough times with zero matches', () => { const events = Array.from({ length: 3 }, () => @@ -234,7 +247,7 @@ describe('buildSuggestReport', () => { }); const cmds = [commandEvent({ sessionId: 'sess-2', success: false, exitCode: 1, timestamp: new Date(now.getTime() + 1000).toISOString() })]; const report = buildSuggestReport([res], cmds, 1, 60, 'events.ndjson'); - expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 1 }]); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 1, exactFailures: 0 }]); }); it('does not report loaded-but-violated when nothing downstream failed', () => { @@ -244,6 +257,27 @@ describe('buildSuggestReport', () => { expect(report.loadedButViolated).toHaveLength(0); }); + it('reports exact loaded-but-violated attribution from adf.constraint fail events, joined by sessionId', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 1 }]); + }); + + it('does not attribute an adf.constraint fail event to a module that was not resolved in that session', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['frontend.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toHaveLength(0); + }); + + it('ignores adf.constraint events with a pass/warn status', () => { + const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'pass' })]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toHaveLength(0); + }); + it('flags insufficientData when resolution events are below the threshold', () => { const report = buildSuggestReport([], [], 3, 60, 'events.ndjson'); expect(report.insufficientData).toBe(true); @@ -426,6 +460,25 @@ describe('adfSuggestCommand', () => { expect(report.commandEvents).toBe(0); }); + it('reads adf.constraint fail events from events.ndjson and surfaces exact loaded-but-violated attribution', () => { + const tmp = tmpDir(); + process.chdir(tmp); + const telemetryDir = path.join(tmp, '.charter', 'telemetry'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const res = resolutionEvent({ sessionId: 'sess-e2e', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraint = constraintEvent({ sessionId: 'sess-e2e', module: 'governance.adf', metric: 'entry_loc', status: 'fail' }); + fs.writeFileSync(path.join(telemetryDir, 'events.ndjson'), [res, constraint].map((e) => JSON.stringify(e)).join('\n') + '\n'); + + const logs: string[] = []; + vi.spyOn(console, 'log').mockImplementation((msg: string) => logs.push(msg)); + + const exitCode = adfSuggestCommand(baseOptions, []); + expect(exitCode).toBe(0); + const report = JSON.parse(logs[0]) as { loadedButViolated: Array<{ module: string; exactFailures: number; correlatedFailures: number }> }; + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 1 }]); + }); + it('prints a human-readable report in text format', () => { const tmp = tmpDir(); process.chdir(tmp); diff --git a/packages/cli/src/__tests__/integration/evidence-contracts.test.ts b/packages/cli/src/__tests__/integration/evidence-contracts.test.ts index 082830c..1eb6e6b 100644 --- a/packages/cli/src/__tests__/integration/evidence-contracts.test.ts +++ b/packages/cli/src/__tests__/integration/evidence-contracts.test.ts @@ -210,4 +210,43 @@ describe('adf evidence CI contracts (integration)', () => { expect(output.staleBaselineCount).toBe(0); expect(output.staleBaselines).toBeUndefined(); // field omitted when empty }); + + // ── Module attribution: constraints know which module they came from ── + it('attributes each constraint to its owning module (core.adf)', () => { + const tmp = makeTempDir('module-attribution'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 79, /* baseline */ 80, /* ceiling */ 150); + + const { output } = captureJson(adfEvidence, jsonOptions, ['--auto-measure']); + const constraints = output.constraints as Array<{ metric: string; module?: string }>; + expect(constraints[0].module).toBe('core.adf'); + }); + + // ── Exact-attribution telemetry: adf.constraint fail events on disk ──── + it('records an adf.constraint fail event for a breached ceiling', () => { + const tmp = makeTempDir('constraint-event'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 160, /* baseline */ 100, /* ceiling */ 150); + + captureJson(adfEvidence, ciOptions, ['--auto-measure']); + + const telemetryFile = path.join(tmp, '.charter', 'telemetry', 'events.ndjson'); + const lines = fs.readFileSync(telemetryFile, 'utf-8').trim().split('\n'); + const events = lines.map((l) => JSON.parse(l) as Record); + const constraintEvent = events.find((e) => e.eventType === 'adf.constraint'); + expect(constraintEvent).toMatchObject({ module: 'core.adf', metric: 'app_loc', status: 'fail' }); + }); + + it('does not record an adf.constraint event when all constraints pass', () => { + const tmp = makeTempDir('constraint-event-pass'); + process.chdir(tmp); + writeEvidenceFixture(tmp, /* actual */ 79, /* baseline */ 80, /* ceiling */ 150); + + captureJson(adfEvidence, ciOptions, ['--auto-measure']); + + // A run with no failing constraints has nothing to attribute, so + // recordAdfConstraintEvents never even creates the telemetry file. + const telemetryFile = path.join(tmp, '.charter', 'telemetry', 'events.ndjson'); + expect(fs.existsSync(telemetryFile)).toBe(false); + }); }); diff --git a/packages/cli/src/commands/adf-evidence.ts b/packages/cli/src/commands/adf-evidence.ts index cbe202c..c22eb0f 100644 --- a/packages/cli/src/commands/adf-evidence.ts +++ b/packages/cli/src/commands/adf-evidence.ts @@ -18,6 +18,7 @@ import type { CLIOptions } from '../index'; import { CLIError, EXIT_CODE } from '../index'; import { getFlag, readFlagFile, tokenizeTask } from '../flags'; import { hashContent, loadLockFile } from './adf-sync'; +import { recordAdfConstraintEvents } from '../telemetry'; interface AutoMeasurement { metric: string; @@ -94,6 +95,7 @@ export function adfEvidence(options: CLIOptions, args: string[]): number { try { const bundle = bundleModules(aiDir, modulePaths, readFile, keywords, manifest); const report: EvidenceReport = evaluateEvidence(bundle, context, staleThreshold); + recordAdfConstraintEvents(options.configPath, { results: report.constraints }); // Check sync status const lockFile = path.join(aiDir, '.adf.lock'); diff --git a/packages/cli/src/commands/adf-suggest.ts b/packages/cli/src/commands/adf-suggest.ts index c777f07..2fb7180 100644 --- a/packages/cli/src/commands/adf-suggest.ts +++ b/packages/cli/src/commands/adf-suggest.ts @@ -3,14 +3,20 @@ * * Diagnostics over .charter/telemetry/events.ndjson: which ADF modules * never fire, which task keywords never match a trigger anywhere, and - * which modules load yet a downstream command still fails. + * which modules load yet still get violated. * - * By default this is entirely report-only. Attributing a *failure* to a - * specific module's trigger list would require knowing which module's - * constraint was violated, and nothing in current telemetry (validate/ - * drift/evidence check commit trailers, CLAUDE.md sync, and METRICS - * ceilings respectively) records that — so no op is ever generated from - * the failure-correlation signal alone. + * `loadedButViolated` now has two tiers of evidence: + * - Exact: `charter adf evidence` records an `adf.constraint` event for + * every failing METRICS ceiling, tagged with the specific module it + * belongs to (see `validateConstraints`'s `module` param). Joining + * resolution events against these by module + session/time gives a + * real failure attribution, not a guess. + * - Correlated (fallback): for constraint checks that don't emit + * module-tagged evidence (commit-trailer validation, blessed-stack + * drift scans — neither of which is module-shaped), or for telemetry + * predating this event, we fall back to "some downstream command in + * this session/window failed" — a much weaker signal, kept only so the + * detector still says something in the absence of exact evidence. * * `--emit-ops` adds one narrower, self-contained mechanism on top: a * co-occurrence heuristic. If an unmatched keyword frequently appears @@ -30,7 +36,7 @@ import type { PatchOperation } from '@stackbilt/adf'; import type { CLIOptions } from '../index'; import { EXIT_CODE } from '../index'; import { getFlag } from '../flags'; -import type { AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; +import type { AdfConstraintEvent, AdfResolutionEvent, CliTelemetryEvent } from '../telemetry'; const DEFAULT_MIN_OCCURRENCES = 3; const DEFAULT_WINDOW_MINUTES = 60; @@ -60,7 +66,10 @@ export interface NearMissKeywordFinding { export interface LoadedButViolatedFinding { module: string; occurrences: number; + /** Downstream command in the session/window failed — no confirmed link to this module's own constraints. */ correlatedFailures: number; + /** This module's own METRICS ceiling was confirmed to fail (adf.constraint events), joined by session/window. */ + exactFailures: number; } /** @@ -105,8 +114,8 @@ export function adfSuggestCommand(options: CLIOptions, args: string[]): number { const emitOpsFile = getFlag(args, '--emit-ops'); const telemetryFile = path.join(options.configPath, 'telemetry', 'events.ndjson'); - const { resolutionEvents, commandEvents } = readEvents(telemetryFile); - const report = buildSuggestReport(resolutionEvents, commandEvents, minOccurrences, windowMinutes, telemetryFile); + const { resolutionEvents, commandEvents, constraintEvents } = readEvents(telemetryFile); + const report = buildSuggestReport(resolutionEvents, commandEvents, minOccurrences, windowMinutes, telemetryFile, constraintEvents); const emitResult = buildEmitOps(report.coOccurrenceCandidates, aiDir); if (emitOpsFile) { @@ -220,15 +229,17 @@ function parsePositiveInt(raw: string | undefined): number | undefined { function readEvents(telemetryFile: string): { resolutionEvents: AdfResolutionEvent[]; commandEvents: CliTelemetryEvent[]; + constraintEvents: AdfConstraintEvent[]; } { const resolutionEvents: AdfResolutionEvent[] = []; const commandEvents: CliTelemetryEvent[] = []; + const constraintEvents: AdfConstraintEvent[] = []; let raw: string; try { raw = fs.readFileSync(telemetryFile, 'utf-8'); } catch { - return { resolutionEvents, commandEvents }; + return { resolutionEvents, commandEvents, constraintEvents }; } for (const line of raw.split(/\r?\n/)) { @@ -243,11 +254,14 @@ function readEvents(telemetryFile: string): { } if (!parsed || typeof parsed !== 'object') continue; - const record = parsed as { eventType?: string; timestamp?: unknown; commandPath?: unknown; exitCode?: unknown }; + const record = parsed as { eventType?: string; timestamp?: unknown; commandPath?: unknown; exitCode?: unknown; module?: unknown; status?: unknown }; if (typeof record.timestamp !== 'string') continue; if (record.eventType === 'adf.resolution') { resolutionEvents.push(parsed as AdfResolutionEvent); + } else if (record.eventType === 'adf.constraint') { + if (typeof record.module !== 'string' || typeof record.status !== 'string') continue; + constraintEvents.push(parsed as AdfConstraintEvent); } else if (record.eventType === 'command' || record.eventType === undefined) { // Events written before eventType existed have no discriminator; // treat those as command events for backward compatibility. Require @@ -259,24 +273,29 @@ function readEvents(telemetryFile: string): { } } - return { resolutionEvents, commandEvents }; + return { resolutionEvents, commandEvents, constraintEvents }; +} + +interface Joinable { + sessionId?: string | null; + timestamp: string; } /** - * Whether a command event falls within the join window of a resolution - * event: same sessionId (high confidence) if both are stamped, else a - * forward-only time-window fallback (heuristic). + * Whether two events fall within the join window of each other: same + * sessionId (high confidence) if both are stamped, else a forward-only + * time-window fallback (heuristic). */ -function isJoined(resolution: AdfResolutionEvent, command: CliTelemetryEvent, windowMinutes: number): boolean { - if (resolution.sessionId && command.sessionId) { - return resolution.sessionId === command.sessionId; +function isJoined(a: Joinable, b: Joinable, windowMinutes: number): boolean { + if (a.sessionId && b.sessionId) { + return a.sessionId === b.sessionId; } - const resTime = Date.parse(resolution.timestamp); - const cmdTime = Date.parse(command.timestamp); - if (!Number.isFinite(resTime) || !Number.isFinite(cmdTime)) return false; + const aTime = Date.parse(a.timestamp); + const bTime = Date.parse(b.timestamp); + if (!Number.isFinite(aTime) || !Number.isFinite(bTime)) return false; - const deltaMs = cmdTime - resTime; + const deltaMs = bTime - aTime; return deltaMs >= 0 && deltaMs <= windowMinutes * 60_000; } @@ -294,12 +313,14 @@ export function buildSuggestReport( minOccurrences: number, windowMinutes: number, sourceFile: string, + constraintEvents: AdfConstraintEvent[] = [], ): SuggestReport { const moduleOccurrences = new Map(); const moduleMatched = new Map(); const globallyMatchedKeywords = new Set(); const loadedOccurrences = new Map(); const loadedCorrelatedFailures = new Map(); + const loadedExactFailures = new Map(); const keywordOccurrences = new Map(); const keywordCorrelatedFailures = new Map(); // keyword -> module -> co-occurrence count (module genuinely ON_DEMAND-triggered @@ -311,6 +332,16 @@ export function buildSuggestReport( // avoids an O(resolutions x commands) scan over the full command history. const failedCommandEvents = commandEvents.filter((c) => c.success === false); + // Fail-status constraint events, grouped by the module they were checked + // against — the exact-attribution counterpart to failedCommandEvents. + const failedConstraintEventsByModule = new Map(); + for (const ev of constraintEvents) { + if (ev.status !== 'fail') continue; + const list = failedConstraintEventsByModule.get(ev.module) ?? []; + list.push(ev); + failedConstraintEventsByModule.set(ev.module, list); + } + for (const res of resolutionEvents) { // triggerMatches has one entry per (module, trigger) pair, so a module // declared with multiple triggers must be deduped per event here — @@ -386,6 +417,13 @@ export function buildSuggestReport( loadedCorrelatedFailures.set(mod, (loadedCorrelatedFailures.get(mod) ?? 0) + 1); } } + + for (const mod of new Set(res.resolvedModules)) { + const failedEvents = failedConstraintEventsByModule.get(mod); + if (failedEvents && failedEvents.some((ev) => isJoined(res, ev, windowMinutes))) { + loadedExactFailures.set(mod, (loadedExactFailures.get(mod) ?? 0) + 1); + } + } } const deadModules: DeadModuleFinding[] = [...moduleOccurrences.entries()] @@ -402,14 +440,15 @@ export function buildSuggestReport( })) .sort((a, b) => b.correlatedFailures - a.correlatedFailures || b.occurrences - a.occurrences); - const loadedButViolated: LoadedButViolatedFinding[] = [...loadedCorrelatedFailures.entries()] - .filter(([, correlatedFailures]) => correlatedFailures > 0) - .map(([module, correlatedFailures]) => ({ + const violatedModules = new Set([...loadedCorrelatedFailures.keys(), ...loadedExactFailures.keys()]); + const loadedButViolated: LoadedButViolatedFinding[] = [...violatedModules] + .map((module) => ({ module, occurrences: loadedOccurrences.get(module) ?? 0, - correlatedFailures, + correlatedFailures: loadedCorrelatedFailures.get(module) ?? 0, + exactFailures: loadedExactFailures.get(module) ?? 0, })) - .sort((a, b) => b.correlatedFailures - a.correlatedFailures); + .sort((a, b) => b.exactFailures - a.exactFailures || b.correlatedFailures - a.correlatedFailures); const nearMissKeywordSet = new Set(nearMissKeywords.map((f) => f.keyword)); const coOccurrenceCandidates: CoOccurrenceCandidate[] = []; @@ -490,7 +529,10 @@ function printReport(report: SuggestReport): void { console.log(' (none)'); } else { for (const f of report.loadedButViolated) { - console.log(` [!] ${f.module} — resolved ${f.occurrences}x, ${f.correlatedFailures} correlated with a downstream failure`); + const parts: string[] = []; + if (f.exactFailures > 0) parts.push(`${f.exactFailures} confirmed constraint failure${f.exactFailures === 1 ? '' : 's'}`); + if (f.correlatedFailures > 0) parts.push(`${f.correlatedFailures} correlated with a downstream failure`); + console.log(` [!] ${f.module} — resolved ${f.occurrences}x, ${parts.join(', ')}`); } console.log(' Note: routing worked; rule content may be weak. Needs human/strong-model rewrite, not a trigger change.'); } diff --git a/packages/cli/src/telemetry.ts b/packages/cli/src/telemetry.ts index 5f6977f..287c656 100644 --- a/packages/cli/src/telemetry.ts +++ b/packages/cli/src/telemetry.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { randomUUID } from 'node:crypto'; -import type { BundleResult } from '@stackbilt/adf'; +import type { BundleResult, ConstraintStatus } from '@stackbilt/adf'; export interface CliTelemetryEvent { version: 1; @@ -46,6 +46,72 @@ export interface AdfResolutionEvent { tokenEstimate?: number; } +/** + * Sibling event recording a single failed METRICS constraint, attributed to + * the specific `.ai/*.adf` module it was checked against (see + * `validateConstraints`'s `module` parameter). Unlike AdfResolutionEvent's + * command-success correlation, this is exact evidence: `charter adf + * suggest`'s `loadedButViolated` detector can join this directly on module + * (plus sessionId/time-window) instead of guessing from a downstream + * command's exit code. + */ +export interface AdfConstraintEvent { + version: 1; + eventType: 'adf.constraint'; + timestamp: string; + sessionId: string | null; + module: string; + metric: string; + status: ConstraintStatus; +} + +export interface RecordAdfConstraintInput { + results: Array<{ module?: string; metric: string; status: ConstraintStatus }>; + /** Override the CHARTER_SESSION_ID env lookup, same as RecordAdfResolutionInput. */ + sessionId?: string | null; +} + +/** + * Persist one event per failing, module-attributed constraint (best-effort, + * never throws). Passing/warning constraints aren't recorded — only `fail` + * is a "violation" in the sense `loadedButViolated` cares about — and + * results with no `module` are skipped since there's nothing to attribute. + */ +export function recordAdfConstraintEvents(configPath: string, input: RecordAdfConstraintInput): void { + try { + const violated = input.results.filter( + (r): r is { module: string; metric: string; status: ConstraintStatus } => + r.module !== undefined && r.status === 'fail', + ); + if (violated.length === 0) return; + + const telemetryDir = path.join(configPath, 'telemetry'); + const telemetryFile = path.join(telemetryDir, 'events.ndjson'); + fs.mkdirSync(telemetryDir, { recursive: true }); + + const timestamp = new Date().toISOString(); + const sessionId = input.sessionId ?? getSessionId() ?? null; + const lines = violated + .map((r) => { + const event: AdfConstraintEvent = { + version: 1, + eventType: 'adf.constraint', + timestamp, + sessionId, + module: r.module, + metric: r.metric, + status: r.status, + }; + return JSON.stringify(event); + }) + .join('\n'); + + fs.appendFileSync(telemetryFile, `${lines}\n`); + } catch { + // Telemetry is best-effort and must never block command execution. + } +} + export interface RecordAdfResolutionInput { source: AdfResolutionEvent['source']; keywords: string[]; From 6548df96307a54b385d50f9c2414b5ec9b2170ac Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Sat, 4 Jul 2026 05:17:54 -0500 Subject: [PATCH 2/2] fix(adf): address code-review findings on PR #246 1. exactFailures was counting joined resolution events, not distinct constraint failures -- a module resolved 5x in one session with a single real ceiling breach reported "5 confirmed constraint failures" instead of 1. Fixed by attributing exactFailures directly from the number of fail events per module (grouped once), decoupled from how many times the module happened to be resolved. 2. evaluateEvidence used a bare truthiness check on bundle.documents, so a present-but-incomplete `documents: {}` would silently validate zero constraints (hiding real violations) instead of falling back to merged-document validation. Fixed with a hasCompleteDocuments guard that confirms every resolved module has a matching document. 3. Constraint aggregation (weightSummary/failCount/warnCount) was duplicated between validator.ts and evidence.ts's per-module path, and computeWeightSummary ran twice on the same document in the merged-document fallback. Extracted computeConstraints() as the shared per-entry logic; validateConstraints wraps it with the aggregate fields, evidence.ts uses the lighter helper directly. 4. bundler.ts maintained two parallel structures (a documents array and a documentsByPath map) built from the same loop. Removed the array; mergeDocuments now takes Object.values(documentsByPath). 5. adf-suggest.ts's loadedButViolated was built from three separate Maps plus a Set-union; consolidated into one Map. Also added a shared groupBy() helper, used by both the new failedConstraintEventsByModule grouping and the pre-existing co-occurrence byModule grouping in buildEmitOps. Added regression tests for (1) and (2). 497/497 tests pass, full workspace build/typecheck clean. Verified (1) live with a real CLI sequence: 5x `adf bundle` + 1 real ceiling breach now reports occurrences: 5, exactFailures: 1 (previously would have reported 5). Co-Authored-By: Claude Sonnet 5 --- packages/adf/src/__tests__/evidence.test.ts | 27 ++++++ packages/adf/src/bundler.ts | 7 +- packages/adf/src/evidence.ts | 26 ++++-- packages/adf/src/validator.ts | 36 +++++--- .../cli/src/__tests__/adf-suggest.test.ts | 19 +++++ packages/cli/src/commands/adf-suggest.ts | 82 ++++++++++++------- 6 files changed, 151 insertions(+), 46 deletions(-) diff --git a/packages/adf/src/__tests__/evidence.test.ts b/packages/adf/src/__tests__/evidence.test.ts index cb36b45..9ffcf63 100644 --- a/packages/adf/src/__tests__/evidence.test.ts +++ b/packages/adf/src/__tests__/evidence.test.ts @@ -245,4 +245,31 @@ describe('evaluateEvidence', () => { expect(report.constraints).toHaveLength(1); expect(report.constraints[0].module).toBeUndefined(); }); + + it('falls back to merged-document validation when documents is present but missing an entry for a resolved module', () => { + // `documents: {}` is truthy but incomplete -- must not be treated the + // same as "per-module data available," or a resolved module with no + // matching document would silently produce zero constraints instead of + // falling back. + const report = evaluateEvidence( + makeBundleResult({ + resolvedModules: ['core.adf'], + documents: {}, + mergedDocument: { + version: '0.1', + sections: [ + { + key: 'METRICS', + decoration: null, + content: { type: 'metric', entries: [{ key: 'loc', value: 250, ceiling: 200, unit: 'lines' }] }, + }, + ], + }, + }), + ); + expect(report.constraints).toHaveLength(1); + expect(report.constraints[0].status).toBe('fail'); + expect(report.constraints[0].module).toBeUndefined(); + expect(report.allPassing).toBe(false); + }); }); diff --git a/packages/adf/src/bundler.ts b/packages/adf/src/bundler.ts index bd3a72e..4c7b842 100644 --- a/packages/adf/src/bundler.ts +++ b/packages/adf/src/bundler.ts @@ -40,7 +40,6 @@ export function bundleModules( const manifest = preloadedManifest ?? loadAndParseManifest(basePath, readFile); const triggerMatches = buildTriggerReport(manifest, modulePaths, taskKeywords); - const documents: AdfDocument[] = []; const documentsByPath: Record = {}; const perModuleTokens: Record = {}; const advisoryOnlyModules: string[] = []; @@ -55,7 +54,6 @@ export function bundleModules( throw new AdfBundleError(`Module not found: ${modPath}`, modPath); } const doc = parseAdf(content); - documents.push(doc); documentsByPath[modPath] = doc; perModuleTokens[modPath] = estimateTokens(doc); @@ -68,7 +66,10 @@ export function bundleModules( } } - const merged = mergeDocuments(documents); + // Module paths are never integer-index-like strings, so Object.values + // preserves insertion order — safe to derive the merge input from + // documentsByPath instead of maintaining a second parallel array. + const merged = mergeDocuments(Object.values(documentsByPath)); const tokenEstimate = estimateTokens(merged); const tokenBudget = manifest.tokenBudget ?? null; const tokenUtilization = tokenBudget !== null ? tokenEstimate / tokenBudget : null; diff --git a/packages/adf/src/evidence.ts b/packages/adf/src/evidence.ts index 9c42ba8..c22ca5b 100644 --- a/packages/adf/src/evidence.ts +++ b/packages/adf/src/evidence.ts @@ -13,7 +13,7 @@ import type { ConstraintResult, EvidenceResult, } from './types'; -import { validateConstraints, computeWeightSummary } from './validator'; +import { computeConstraints, computeWeightSummary } from './validator'; // ============================================================================ // Types @@ -63,10 +63,11 @@ export function evaluateEvidence( // validate each one separately so ConstraintResults are attributed back to // their owning module. Falls back to the merged document (unattributed, // matching pre-attribution behavior) for hand-built BundleResults that - // don't populate `documents`. - const constraints = bundle.documents + // don't populate `documents` — checked for genuine full coverage, not bare + // truthiness, since `documents: {}` is truthy but has nothing to validate. + const constraints = hasCompleteDocuments(bundle) ? gatherPerModuleConstraints(bundle.documents, bundle.resolvedModules, context) - : validateConstraints(bundle.mergedDocument, context).constraints; + : computeConstraints(bundle.mergedDocument, context); const failCount = constraints.filter((c) => c.status === 'fail').length; const warnCount = constraints.filter((c) => c.status === 'warn').length; @@ -92,6 +93,21 @@ export function evaluateEvidence( }; } +/** + * True only when `documents` has a parsed document for every resolved + * module — not merely present. A bare truthiness check on `bundle.documents` + * would treat `{}` (present but empty/incomplete) as "per-module data + * available," silently validating nothing for any module missing a document + * instead of falling back to the merged-document path. + */ +function hasCompleteDocuments( + bundle: BundleResult, +): bundle is BundleResult & { documents: Record } { + const documents = bundle.documents; + if (documents === undefined) return false; + return bundle.resolvedModules.every((modPath) => modPath in documents); +} + /** * Validate each resolved module's document separately and tag results with * their module path, instead of validating the merged document where module @@ -106,7 +122,7 @@ function gatherPerModuleConstraints( for (const modPath of resolvedModules) { const doc = documents[modPath]; if (!doc) continue; - constraints.push(...validateConstraints(doc, context, modPath).constraints); + constraints.push(...computeConstraints(doc, context, modPath)); } return constraints; } diff --git a/packages/adf/src/validator.ts b/packages/adf/src/validator.ts index d216c65..e84fce0 100644 --- a/packages/adf/src/validator.ts +++ b/packages/adf/src/validator.ts @@ -31,6 +31,31 @@ export function validateConstraints( context?: Record, module?: string, ): EvidenceResult { + const constraints = computeConstraints(doc, context, module); + const failCount = constraints.filter(c => c.status === 'fail').length; + const warnCount = constraints.filter(c => c.status === 'warn').length; + + return { + constraints, + weightSummary: computeWeightSummary(doc), + allPassing: failCount === 0, + failCount, + warnCount, + }; +} + +/** + * Compute per-entry ConstraintResults for a single document's METRICS + * sections, without the aggregate weightSummary/failCount/warnCount wrapper. + * Shared by validateConstraints and evidence.ts's per-module attribution + * path, so validating N modules doesn't also compute (and discard) N + * redundant weight summaries and pass/fail counts. + */ +export function computeConstraints( + doc: AdfDocument, + context?: Record, + module?: string, +): ConstraintResult[] { const constraints: ConstraintResult[] = []; for (const section of doc.sections) { @@ -56,16 +81,7 @@ export function validateConstraints( } } - const failCount = constraints.filter(c => c.status === 'fail').length; - const warnCount = constraints.filter(c => c.status === 'warn').length; - - return { - constraints, - weightSummary: computeWeightSummary(doc), - allPassing: failCount === 0, - failCount, - warnCount, - }; + return constraints; } /** diff --git a/packages/cli/src/__tests__/adf-suggest.test.ts b/packages/cli/src/__tests__/adf-suggest.test.ts index 36d0028..a5edb89 100644 --- a/packages/cli/src/__tests__/adf-suggest.test.ts +++ b/packages/cli/src/__tests__/adf-suggest.test.ts @@ -264,6 +264,25 @@ describe('buildSuggestReport', () => { expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 1 }]); }); + it('does not multiply exactFailures by the number of resolution events that share a session with one real failure', () => { + const resolutions = [1, 2, 3, 4, 5].map(() => + resolutionEvent({ sessionId: 'sess-fanout', resolvedModules: ['governance.adf'], triggerMatches: [] }), + ); + const constraints = [constraintEvent({ sessionId: 'sess-fanout', module: 'governance.adf', status: 'fail' })]; + const report = buildSuggestReport(resolutions, [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 5, correlatedFailures: 0, exactFailures: 1 }]); + }); + + it('counts exactFailures as the number of distinct fail events for a module, not the number of resolutions', () => { + const res = resolutionEvent({ sessionId: 'sess-multi', resolvedModules: ['governance.adf'], triggerMatches: [] }); + const constraints = [ + constraintEvent({ sessionId: 'sess-multi', module: 'governance.adf', status: 'fail' }), + constraintEvent({ sessionId: 'sess-multi', module: 'governance.adf', status: 'fail' }), + ]; + const report = buildSuggestReport([res], [], 1, 60, 'events.ndjson', constraints); + expect(report.loadedButViolated).toEqual([{ module: 'governance.adf', occurrences: 1, correlatedFailures: 0, exactFailures: 2 }]); + }); + it('does not attribute an adf.constraint fail event to a module that was not resolved in that session', () => { const res = resolutionEvent({ sessionId: 'sess-9', resolvedModules: ['frontend.adf'], triggerMatches: [] }); const constraints = [constraintEvent({ sessionId: 'sess-9', module: 'governance.adf', status: 'fail' })]; diff --git a/packages/cli/src/commands/adf-suggest.ts b/packages/cli/src/commands/adf-suggest.ts index 2fb7180..d073c8d 100644 --- a/packages/cli/src/commands/adf-suggest.ts +++ b/packages/cli/src/commands/adf-suggest.ts @@ -173,16 +173,12 @@ function buildEmitOps(candidates: CoOccurrenceCandidate[], aiDir: string): EmitO // REPLACE_BULLET op with both keywords appended, not two ops at the same // index — applying them in sequence would make the second silently // clobber the first (adf patch applies against the evolving document). - const byModule = new Map(); for (const candidate of candidates) { if (candidate.ambiguous) { skipped.push({ keyword: candidate.keyword, module: candidate.module, reason: 'ambiguous: tied with another module for top co-occurrence count' }); - continue; } - const group = byModule.get(candidate.module) ?? []; - group.push(candidate); - byModule.set(candidate.module, group); } + const byModule = groupBy(candidates.filter((c) => !c.ambiguous), (c) => c.module); for (const [modulePath, groupCandidates] of byModule) { const modIndex = manifest.onDemand.findIndex((m) => m.path === modulePath); @@ -307,6 +303,24 @@ function hasCorrelatedFailure( return failedCommandEvents.some((c) => isJoined(resolution, c, windowMinutes)); } +/** Group items by a derived key, preserving each group's insertion order. */ +function groupBy(items: T[], keyFn: (item: T) => string): Map { + const groups = new Map(); + for (const item of items) { + const key = keyFn(item); + const group = groups.get(key) ?? []; + group.push(item); + groups.set(key, group); + } + return groups; +} + +interface LoadedModuleStats { + occurrences: number; + correlatedFailures: number; + exactFailures: number; +} + export function buildSuggestReport( resolutionEvents: AdfResolutionEvent[], commandEvents: CliTelemetryEvent[], @@ -318,15 +332,22 @@ export function buildSuggestReport( const moduleOccurrences = new Map(); const moduleMatched = new Map(); const globallyMatchedKeywords = new Set(); - const loadedOccurrences = new Map(); - const loadedCorrelatedFailures = new Map(); - const loadedExactFailures = new Map(); + const loadedModuleStats = new Map(); const keywordOccurrences = new Map(); const keywordCorrelatedFailures = new Map(); // keyword -> module -> co-occurrence count (module genuinely ON_DEMAND-triggered // in the same event, not just default-loaded). const keywordModuleCoOccurrence = new Map>(); + function getLoadedStats(module: string): LoadedModuleStats { + let stats = loadedModuleStats.get(module); + if (!stats) { + stats = { occurrences: 0, correlatedFailures: 0, exactFailures: 0 }; + loadedModuleStats.set(module, stats); + } + return stats; + } + // Only failed commands can ever satisfy hasCorrelatedFailure/isJoined, so // scanning just this (typically much smaller) subset per resolution event // avoids an O(resolutions x commands) scan over the full command history. @@ -334,13 +355,10 @@ export function buildSuggestReport( // Fail-status constraint events, grouped by the module they were checked // against — the exact-attribution counterpart to failedCommandEvents. - const failedConstraintEventsByModule = new Map(); - for (const ev of constraintEvents) { - if (ev.status !== 'fail') continue; - const list = failedConstraintEventsByModule.get(ev.module) ?? []; - list.push(ev); - failedConstraintEventsByModule.set(ev.module, list); - } + const failedConstraintEventsByModule = groupBy( + constraintEvents.filter((ev) => ev.status === 'fail'), + (ev) => ev.module, + ); for (const res of resolutionEvents) { // triggerMatches has one entry per (module, trigger) pair, so a module @@ -364,7 +382,7 @@ export function buildSuggestReport( moduleMatched.set(mod, (moduleMatched.get(mod) ?? 0) + 1); } for (const mod of res.resolvedModules) { - loadedOccurrences.set(mod, (loadedOccurrences.get(mod) ?? 0) + 1); + getLoadedStats(mod).occurrences += 1; } } @@ -414,15 +432,23 @@ export function buildSuggestReport( if (failed) { for (const mod of res.resolvedModules) { - loadedCorrelatedFailures.set(mod, (loadedCorrelatedFailures.get(mod) ?? 0) + 1); + getLoadedStats(mod).correlatedFailures += 1; } } + } - for (const mod of new Set(res.resolvedModules)) { - const failedEvents = failedConstraintEventsByModule.get(mod); - if (failedEvents && failedEvents.some((ev) => isJoined(res, ev, windowMinutes))) { - loadedExactFailures.set(mod, (loadedExactFailures.get(mod) ?? 0) + 1); - } + // Exact attribution: each confirmed constraint failure counts once, + // regardless of how many resolution events happened to load that module + // (a module resolved 5x with 1 real failure must report exactFailures: 1, + // not 5 — joining per resolution event, as correlatedFailures does, would + // multiply a single failure by however many times the module was loaded). + // Only attributed to modules that appear as "loaded" at all: an + // adf.constraint event for a module never resolved via adf bundle/context/ + // serve can't be reported as loaded-but-violated by definition. + for (const [module, events] of failedConstraintEventsByModule) { + const stats = loadedModuleStats.get(module); + if (stats) { + stats.exactFailures = events.length; } } @@ -440,13 +466,13 @@ export function buildSuggestReport( })) .sort((a, b) => b.correlatedFailures - a.correlatedFailures || b.occurrences - a.occurrences); - const violatedModules = new Set([...loadedCorrelatedFailures.keys(), ...loadedExactFailures.keys()]); - const loadedButViolated: LoadedButViolatedFinding[] = [...violatedModules] - .map((module) => ({ + const loadedButViolated: LoadedButViolatedFinding[] = [...loadedModuleStats.entries()] + .filter(([, stats]) => stats.correlatedFailures > 0 || stats.exactFailures > 0) + .map(([module, stats]) => ({ module, - occurrences: loadedOccurrences.get(module) ?? 0, - correlatedFailures: loadedCorrelatedFailures.get(module) ?? 0, - exactFailures: loadedExactFailures.get(module) ?? 0, + occurrences: stats.occurrences, + correlatedFailures: stats.correlatedFailures, + exactFailures: stats.exactFailures, })) .sort((a, b) => b.exactFailures - a.exactFailures || b.correlatedFailures - a.correlatedFailures);