From fc683041ab0e4b80349e800a2048a323e373f89d Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Thu, 9 Jul 2026 21:15:51 +0530 Subject: [PATCH 01/10] =?UTF-8?q?week=5F5:=20Module=20C=20(The=20Librarian?= =?UTF-8?q?)=20=E2=80=94=20C.3=20confidence=20calibration=20(temperature?= =?UTF-8?q?=20scaling)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/librarian/temperature_test.py | 169 ++++++++++++++ application/utils/librarian/__init__.py | 4 +- .../utils/librarian/calibration/__init__.py | 13 ++ .../librarian/calibration/temperature.py | 207 ++++++++++++++++++ scripts/evaluate_librarian.py | 126 +++++++++-- 5 files changed, 499 insertions(+), 20 deletions(-) create mode 100644 application/tests/librarian/temperature_test.py create mode 100644 application/utils/librarian/calibration/__init__.py create mode 100644 application/utils/librarian/calibration/temperature.py diff --git a/application/tests/librarian/temperature_test.py b/application/tests/librarian/temperature_test.py new file mode 100644 index 000000000..4b4dee1cd --- /dev/null +++ b/application/tests/librarian/temperature_test.py @@ -0,0 +1,169 @@ +"""Tests for C.3 temperature-scaling calibration (Week 5). + +Hermetic and deterministic: synthetic candidate-logit shortlists + 0/1 labels +only — no cross-encoder, DB, or embedding key. Covers the softmax-over-shortlist +confidence, temperature flatten/sharpen behaviour, the NLL fit recovering a known +T and reducing NLL, ECE on perfectly- and mis-calibrated data (hand-checked), and +every guard. +""" + +import math +import unittest + +import numpy as np +from scipy.special import softmax + +from application.utils.librarian.calibration.temperature import ( + CALIBRATOR_NAME, + CalibrationError, + DegenerateLabelsError, + TemperatureScaler, + expected_calibration_error, + fit_temperature, + negative_log_likelihood, +) + + +class TemperatureScalerTest(unittest.TestCase): + def test_confidence_is_softmax_top_mass(self) -> None: + logits = [2.0, 1.0, 0.0] + expected = float(softmax(np.array(logits)).max()) + self.assertAlmostEqual(TemperatureScaler(1.0).confidence(logits), expected, 9) + + def test_probabilities_sum_to_one_and_peak_at_top(self) -> None: + p = TemperatureScaler(1.0).probabilities([3.0, 1.0, -2.0]) + self.assertAlmostEqual(float(p.sum()), 1.0, places=9) + self.assertEqual(int(np.argmax(p)), 0) # highest logit -> highest prob + + def test_high_temperature_flattens_toward_uniform(self) -> None: + logits = [3.0, 1.0, 0.0] + hot = TemperatureScaler(1000.0).confidence(logits) + base = TemperatureScaler(1.0).confidence(logits) + self.assertLess(hot, base) + self.assertAlmostEqual(hot, 1.0 / 3.0, places=2) # ~uniform over 3 + + def test_low_temperature_sharpens_toward_one(self) -> None: + logits = [3.0, 1.0, 0.0] + cold = TemperatureScaler(0.1).confidence(logits) + base = TemperatureScaler(1.0).confidence(logits) + self.assertGreater(cold, base) + self.assertGreater(cold, 0.99) + + def test_single_candidate_confidence_is_one(self) -> None: + # softmax over one element is always 1.0, at any temperature. + self.assertAlmostEqual(TemperatureScaler(3.0).confidence([0.42]), 1.0, 9) + + def test_empty_shortlist_rejected(self) -> None: + with self.assertRaises(CalibrationError): + TemperatureScaler(1.0).confidence([]) + + def test_non_positive_or_nonfinite_temperature_rejected(self) -> None: + for bad in (0.0, -1.0, float("inf"), float("nan")): + with self.assertRaises(CalibrationError): + TemperatureScaler(bad) + + +class NegativeLogLikelihoodTest(unittest.TestCase): + def test_matches_hand_computed_single_pair(self) -> None: + # one shortlist, top-1 correct: loss = -log(softmax([2,0]).max()) + conf = float(softmax(np.array([2.0, 0.0])).max()) + self.assertAlmostEqual( + negative_log_likelihood([[2.0, 0.0]], [1.0], 1.0), -math.log(conf), 6 + ) + + def test_length_mismatch_rejected(self) -> None: + with self.assertRaises(CalibrationError): + negative_log_likelihood([[1.0, 0.0], [2.0, 0.0]], [1.0], 1.0) + + def test_non_positive_temperature_rejected(self) -> None: + with self.assertRaises(CalibrationError): + negative_log_likelihood([[1.0, 0.0]], [1.0], 0.0) + + +class FitTemperatureTest(unittest.TestCase): + def _synthetic(self, true_t: float, n: int = 4000, k: int = 5): + """Shortlists whose top-1 correctness is drawn from softmax(logits/true_t). + + Data generated with true_t flattens the softmax, so the raw (T=1) + confidence is over-confident and the fit should recover T ~ true_t. + Seeded -> stable. + """ + rng = np.random.default_rng(0) + sets, labels = [], [] + for _ in range(n): + z = rng.normal(0.0, 3.0, size=k) + z[::-1].sort() # descending so index 0 is the top-1 (argmax) + p_correct = softmax(z / true_t).max() + sets.append(z.tolist()) + labels.append(1.0 if rng.random() < p_correct else 0.0) + return sets, labels + + def test_recovers_known_temperature(self) -> None: + sets, labels = self._synthetic(true_t=2.0) + scaler = fit_temperature(sets, labels) + self.assertAlmostEqual(scaler.temperature, 2.0, delta=0.5) + + def test_fit_reduces_nll_versus_T1(self) -> None: + sets, labels = self._synthetic(true_t=2.0) + scaler = fit_temperature(sets, labels) + self.assertLess( + negative_log_likelihood(sets, labels, scaler.temperature), + negative_log_likelihood(sets, labels, 1.0), + ) + + def test_single_class_labels_raise_degenerate(self) -> None: + with self.assertRaises(DegenerateLabelsError): + fit_temperature([[2.0, 0.0], [3.0, 1.0]], [1.0, 1.0]) + + def test_non_binary_labels_rejected(self) -> None: + with self.assertRaises(CalibrationError): + fit_temperature([[2.0, 0.0], [3.0, 1.0]], [0.0, 2.0]) + + +class ExpectedCalibrationErrorTest(unittest.TestCase): + def test_perfectly_calibrated_is_near_zero(self) -> None: + # bin [0.5,0.6): five correct, five wrong -> acc 0.5 == conf 0.5. + self.assertAlmostEqual( + expected_calibration_error([0.5] * 10, [1.0, 0.0] * 5), 0.0, places=6 + ) + + def test_confidently_wrong_is_large(self) -> None: + self.assertAlmostEqual( + expected_calibration_error([0.9] * 10, [0.0] * 10), 0.9, places=6 + ) + + def test_hand_checked_two_bin_value(self) -> None: + # [0.2,0.3): conf .2 acc 0 gap .2 ; [0.8,0.9): conf .8 acc 1 gap .2 + # ECE = 0.5*0.2 + 0.5*0.2 = 0.20 + self.assertAlmostEqual( + expected_calibration_error([0.2, 0.2, 0.8, 0.8], [0.0, 0.0, 1.0, 1.0]), + 0.20, + places=6, + ) + + def test_length_mismatch_and_empty_rejected(self) -> None: + with self.assertRaises(CalibrationError): + expected_calibration_error([0.5, 0.6], [1.0]) + with self.assertRaises(CalibrationError): + expected_calibration_error([], []) + with self.assertRaises(CalibrationError): + expected_calibration_error([0.5], [1.0], n_bins=0) + + +class EndToEndTest(unittest.TestCase): + def test_flat_shortlists_are_low_confidence(self) -> None: + # A near-tie shortlist (bad match) -> low top-1 confidence. + self.assertLess(TemperatureScaler(1.0).confidence([0.1, 0.0, -0.1]), 0.45) + + def test_peaked_shortlist_is_high_confidence(self) -> None: + # A clear winner -> high top-1 confidence. + self.assertGreater(TemperatureScaler(1.0).confidence([6.0, -1.0, -3.0]), 0.9) + + +class MetadataTest(unittest.TestCase): + def test_calibrator_name_is_versioned(self) -> None: + self.assertEqual(CALIBRATOR_NAME, "temperature-scaling/0.2.0") + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/librarian/__init__.py b/application/utils/librarian/__init__.py index f168f2bb4..1a23cc3cb 100644 --- a/application/utils/librarian/__init__.py +++ b/application/utils/librarian/__init__.py @@ -16,7 +16,9 @@ explicit-link resolution). W3 (C.1): candidate retriever (in-memory + pgvector) + pipeline switch. W4 (C.2): cross-encoder reranker — re-sorts the C.1 shortlist, fills reranked[]. -Calibration + decision routing (C.3-C.4, W5-W6) onward is not built yet. + W5 (C.3): confidence calibration — temperature scaling maps a rerank logit to + an honest probability (fit by NLL on the golden set, gated ECE < 0.10). +Decision routing (C.4, W6) onward is not built yet. Vendored RFC JSON schemas live under ``_rfc_schemas/``. They are pinned to upstream/owasp-graph @ 2b1437987768d5ed20fe9ee721ab9a898c4b84af (PR #734). diff --git a/application/utils/librarian/calibration/__init__.py b/application/utils/librarian/calibration/__init__.py new file mode 100644 index 000000000..305586a76 --- /dev/null +++ b/application/utils/librarian/calibration/__init__.py @@ -0,0 +1,13 @@ +"""Module C.3 — confidence calibration (Week 5). + +C.2 (the cross-encoder, W4) emits a raw ranking logit per candidate — great for +ordering, meaningless as confidence (a +1.5 is not "82% sure"). C.3 turns that +logit into an honest probability via **temperature scaling**: ``p = sigmoid(z/T)`` +with a single scalar ``T`` fit by negative-log-likelihood on the golden set, and +proves the result honest with **ECE < 0.10**. + +The W6 decision engine thresholds that probability (auto-link vs. human review), +so calibration is what makes the threshold trustworthy. Kept dependency-light +(numpy + scipy) and model-free so it stays hermetically testable — mirrors the +C.1/C.2 seams. +""" diff --git a/application/utils/librarian/calibration/temperature.py b/application/utils/librarian/calibration/temperature.py new file mode 100644 index 000000000..928dacf7a --- /dev/null +++ b/application/utils/librarian/calibration/temperature.py @@ -0,0 +1,207 @@ +"""Module C.3 — temperature-scaling calibration (Week 5). The truth-teller. + +C.2 hands up a reranked shortlist of candidate CREs, each with a raw cross-encoder +logit (unbounded, higher = better match). We need one honest number: "how likely +is the top candidate the correct CRE?" — so Week 6 can threshold auto-link vs. +human review. + +The naive attempt — ``sigmoid(top1_logit / T)`` on the single absolute logit — +does not work, and the golden set proves why: a cross-encoder's absolute logit has +no fixed zero point (its 50/50 boundary is not at z=0), and dividing by a single +temperature can only *squash* toward 0.5, never *shift* the boundary. So no T +makes it honest. + +The fix is **temperature scaling in the Guo et al. sense**: calibrate the *softmax +over the whole shortlist*, not one absolute logit. The candidates' *relative* +logits are what a cross-encoder's scores actually mean, so + + p = softmax(logits / T) confidence = p for the top-1 candidate + +is a genuine probability distribution over "which candidate is right", and its +top-1 mass answers exactly the question Week 6 asks. It is still **one knob T**: +T = 1 leaves the distribution unchanged, T > 1 flattens it (less confident), T < 1 +sharpens it. T is fit once by minimising negative-log-likelihood of "is the top-1 +correct?" on the golden set; honesty is measured by Expected Calibration Error, +and the Week 5 gate is ECE < 0.10. + +Like C.1/C.2 this is a thin, model-free seam (numpy + scipy only) so every branch +is hermetically testable. The fitted ``TemperatureScaler`` is a frozen, shareable +artifact the W6 decision engine loads to turn a reranked shortlist into the +confidence it thresholds on. +""" + +from dataclasses import dataclass +from typing import Sequence + +import numpy as np +from scipy.optimize import minimize_scalar +from scipy.special import softmax + +# Identify the calibrator in the RFC audit trail (mirrors RETRIEVER_NAME / +# RERANKER_NAME). 0.2.0 = softmax-over-shortlist (the single-logit sigmoid of +# 0.1.0 could not be calibrated by temperature alone — see module docstring). +CALIBRATOR_NAME = "temperature-scaling/0.2.0" + +# Clip probabilities off {0, 1} so the log in NLL stays finite. +_EPS = 1e-7 + + +class CalibrationError(ValueError): + """Base class for calibration construction/usage failures.""" + + +class DegenerateLabelsError(CalibrationError): + """Labels are single-class — temperature is unidentifiable from NLL. + + With only correct (or only incorrect) top-1s, NLL is monotonic in ``T`` and + the optimum runs to a bound: the fit is meaningless. The calibration set must + contain both outcomes (in the harness: the ``positive`` slice supplies + correct top-1s, ``hard_negative`` supplies incorrect ones). + """ + + +def _softmax_top(logits: Sequence[float], temperature: float) -> float: + """Top-1 probability mass of ``softmax(logits / T)`` over one shortlist.""" + z = np.asarray(list(logits), dtype=float) + if z.size == 0: + raise CalibrationError("cannot calibrate an empty candidate shortlist") + return float(softmax(z / temperature).max()) + + +def _validate_temperature(temperature: float) -> None: + if not np.isfinite(temperature) or temperature <= 0: + raise CalibrationError(f"temperature must be finite and > 0, got {temperature}") + + +def _paired(logit_sets: Sequence[Sequence[float]], labels: Sequence[float]): + """Validate matched (shortlist, label) inputs: non-empty and equal length.""" + sets = list(logit_sets) + y = np.asarray(list(labels), dtype=float) + if y.ndim != 1: + raise CalibrationError(f"labels must be 1-D, got shape {y.shape}") + if len(sets) != y.shape[0]: + raise CalibrationError( + f"{len(sets)} logit-sets and {y.shape[0]} labels must be the same length" + ) + if not sets: + raise CalibrationError("need at least one (shortlist, label) pair") + return sets, y + + +@dataclass(frozen=True) +class TemperatureScaler: + """Maps a reranked shortlist to a calibrated top-1 confidence. + + ``temperature`` is the single learned scalar; build one with + ``fit_temperature``. Frozen so a fitted scaler is a stable, shareable value + (like the retriever/reranker being constructed once and reused). + """ + + temperature: float + + def __post_init__(self) -> None: + _validate_temperature(self.temperature) + + def probabilities(self, logits: Sequence[float]) -> np.ndarray: + """The full ``softmax(logits / T)`` distribution over one shortlist.""" + z = np.asarray(list(logits), dtype=float) + if z.size == 0: + raise CalibrationError("cannot calibrate an empty candidate shortlist") + return softmax(z / self.temperature) + + def confidence(self, logits: Sequence[float]) -> float: + """P(the top candidate is correct) — the top-1 mass of the softmax. + + This is the number the W6 decision engine thresholds on. + """ + return _softmax_top(logits, self.temperature) + + +def negative_log_likelihood( + logit_sets: Sequence[Sequence[float]], + labels: Sequence[float], + temperature: float, +) -> float: + """Binary cross-entropy of the top-1 confidence against 0/1 correctness. + + For each shortlist the confidence is ``softmax(logits / T)``'s top-1 mass; + the label is 1 iff that top-1 candidate is the correct CRE. This is the + objective ``fit_temperature`` minimises over ``T``; exposed so the fit is + testable and a caller can compare NLL at ``T=1`` vs the fitted T. + """ + _validate_temperature(temperature) + sets, y = _paired(logit_sets, labels) + p = np.clip( + np.array([_softmax_top(s, temperature) for s in sets]), _EPS, 1.0 - _EPS + ) + return float(-np.sum(y * np.log(p) + (1.0 - y) * np.log(1.0 - p))) + + +def fit_temperature( + logit_sets: Sequence[Sequence[float]], + labels: Sequence[float], + *, + bounds: tuple = (1e-2, 1e2), +) -> TemperatureScaler: + """Fit ``T`` by minimising NLL over (shortlist, is-top1-correct) pairs. + + ``labels`` must be 0/1 and contain both outcomes (else + ``DegenerateLabelsError``). One free parameter over a bounded interval, so + the 1-D ``minimize_scalar`` fit is fast and barely over-fits — the golden set + is the calibration set by design. + """ + sets, y = _paired(logit_sets, labels) + values = set(np.unique(y).tolist()) + if not values <= {0.0, 1.0}: + raise CalibrationError(f"labels must be 0/1, got values {sorted(values)}") + if len(values) < 2: + raise DegenerateLabelsError( + "labels are single-class; temperature is unidentifiable — the " + "calibration set needs both correct and incorrect top-1s" + ) + + result = minimize_scalar( + lambda t: negative_log_likelihood(sets, y, t), + bounds=bounds, + method="bounded", + ) + if not result.success: + raise CalibrationError(f"temperature fit did not converge: {result.message}") + return TemperatureScaler(temperature=float(result.x)) + + +def expected_calibration_error( + confidences: Sequence[float], labels: Sequence[float], *, n_bins: int = 10 +) -> float: + """ECE — sample-weighted mean ``|accuracy - confidence|`` across equal bins. + + Partition [0, 1] into ``n_bins`` equal-width bins. Per bin, ``confidence`` is + the mean predicted top-1 probability and ``accuracy`` is the fraction whose + top-1 was actually correct; ECE weights each bin's gap by its share of the + sample. 0 = perfectly honest; the Week 5 gate is < 0.10. + """ + if n_bins < 1: + raise CalibrationError(f"n_bins must be >= 1, got {n_bins}") + p = np.asarray(list(confidences), dtype=float) + y = np.asarray(list(labels), dtype=float) + if p.shape != y.shape: + raise CalibrationError( + f"confidences {p.shape} and labels {y.shape} must be the same length" + ) + if p.size == 0: + raise CalibrationError("need at least one (confidence, label) pair") + + edges = np.linspace(0.0, 1.0, n_bins + 1) + idx = np.clip(np.digitize(p, edges[1:-1], right=False), 0, n_bins - 1) + + n = p.size + ece = 0.0 + for b in range(n_bins): + mask = idx == b + count = int(mask.sum()) + if count == 0: + continue + confidence = float(p[mask].mean()) + accuracy = float(y[mask].mean()) + ece += (count / n) * abs(accuracy - confidence) + return ece diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 89afa773d..14d4a1df8 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -94,28 +94,20 @@ def predict(section: Section, registry: Set[str], hub: List[HubRep]) -> List[str return [] -def report_retrieval_recall( - rows: List[GoldenDatasetRow], +def _build_live_pipeline( cache_file: str, top_k: int, threshold: float, top_n_rerank: int, crossencoder_model: str, -) -> None: - """Measure the live C.1 -> C.2 pipeline over the positive slice (v1). - - Two metrics, both live — there is no honest offline value: the candidate - pool must be the real CRE-node vectors, and seeding it from the golden text - is exactly the leakage the hub firewall strips. +): + """Construct the live C.1 retriever + C.2 reranker against the OpenCRE DB. - - retrieval recall@k (C.1): does the expected CRE id make it into the top-K - shortlist the reranker will see? A miss here is unrecoverable downstream. - - rerank top-1 (C.2): after the cross-encoder re-reads each pair and re-sorts - the shortlist, is the #1 candidate an expected CRE? This is the first - end-to-end accuracy number for the search path (W4 target >= 0.80). + Live deps are imported lazily so the offline harness needs neither a DB, an + embedding model, nor the cross-encoder stack. Shared by every live report + (recall/top-1 and calibration) so the heavy hub + model load happens once + per report and the id-space translation stays in one place. """ - # Live deps are imported lazily so the offline harness needs neither a DB, - # an embedding model, nor the cross-encoder stack. from application.cmd.cre_main import db_connect from application.defs import cre_defs from application.prompt_client import prompt_client @@ -164,6 +156,32 @@ def _to_ext(mapping): database.get_embedding_contents_by_doc_type(cre_defs.Credoctypes.CRE.value) ), ) + return retriever, reranker + + +def report_retrieval_recall( + rows: List[GoldenDatasetRow], + cache_file: str, + top_k: int, + threshold: float, + top_n_rerank: int, + crossencoder_model: str, +) -> None: + """Measure the live C.1 -> C.2 pipeline over the positive slice (v1). + + Two metrics, both live — there is no honest offline value: the candidate + pool must be the real CRE-node vectors, and seeding it from the golden text + is exactly the leakage the hub firewall strips. + + - retrieval recall@k (C.1): does the expected CRE id make it into the top-K + shortlist the reranker will see? A miss here is unrecoverable downstream. + - rerank top-1 (C.2): after the cross-encoder re-reads each pair and re-sorts + the shortlist, is the #1 candidate an expected CRE? This is the first + end-to-end accuracy number for the search path (W4 target >= 0.80). + """ + retriever, reranker = _build_live_pipeline( + cache_file, top_k, threshold, top_n_rerank, crossencoder_model + ) positives = [r for r in rows if r.slice.value == "positive" and r.expected.cre_ids] if not positives: @@ -193,6 +211,66 @@ def _to_ext(mapping): ) +def report_calibration( + rows: List[GoldenDatasetRow], + cache_file: str, + top_k: int, + threshold: float, + top_n_rerank: int, + crossencoder_model: str, +) -> int: + """Fit temperature on the golden set and report the Week 5 ECE gate (< 0.10). + + Builds a (shortlist, label) calibration set from the live C.1 -> C.2 pipeline + over the positive + hard_negative slices: each row's *reranked shortlist* of + logits, labelled 1 iff its top-1 candidate is an expected CRE (hard_negatives + expect none, so they contribute the 0 class). Both slices are needed so the + fit sees both outcomes (else it is degenerate). Confidence is the top-1 mass + of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on + ECE < 0.10; returns 1 on a failed gate so a live run can fail. + """ + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + expected_calibration_error, + fit_temperature, + ) + + retriever, reranker = _build_live_pipeline( + cache_file, top_k, threshold, top_n_rerank, crossencoder_model + ) + cal_rows = [r for r in rows if r.slice.value in ("positive", "hard_negative")] + logit_sets: List[List[float]] = [] + labels: List[float] = [] + for row in cal_rows: + audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + reranked = [c for c in audit.reranked if c.score_rerank is not None] + if not reranked: + continue + expected = set(row.expected.cre_ids or []) + logit_sets.append([float(c.score_rerank) for c in reranked]) + labels.append(1.0 if reranked[0].cre_id in expected else 0.0) + + if len(set(labels)) < 2: + print( + "calibration (C.3): need both outcomes in the selection (positive + " + "hard_negative slices) to fit temperature; skipped" + ) + return 0 + + scaler = fit_temperature(logit_sets, labels) + conf_raw = [TemperatureScaler(1.0).confidence(s) for s in logit_sets] + conf_cal = [scaler.confidence(s) for s in logit_sets] + ece_raw = expected_calibration_error(conf_raw, labels) + ece_cal = expected_calibration_error(conf_cal, labels) + gate_ok = ece_cal < 0.10 + print( + f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; " + f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " + f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" + ) + return 0 if gate_ok else 1 + + def main(argv: List[str]) -> int: cfg = load_config() parser = argparse.ArgumentParser(description="Module C eval harness (W2: C.0)") @@ -275,6 +353,7 @@ def main(argv: List[str]) -> int: ) if not gate_ok: return 1 + calib_status = 0 if args.use_live_embeddings: report_retrieval_recall( rows, @@ -284,14 +363,23 @@ def main(argv: List[str]) -> int: args.top_k_rerank, cfg.crossencoder_model, ) + calib_status = report_calibration( + rows, + args.cache_file, + args.top_k_retrieval, + args.threshold, + args.top_k_rerank, + cfg.crossencoder_model, + ) else: print( - "semantic pipeline (C.1 retrieve + C.2 rerank): wired; recall@k and " - "rerank top-1 need --use_live_embeddings (no CRE vectors offline — " - "seeding from golden text would be leakage)" + "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3): " + "wired; recall@k, rerank top-1, and the ECE gate need " + "--use_live_embeddings (no CRE vectors offline — seeding from golden " + "text would be leakage)" ) print(f"correct overall (semantic path still stubbed): {correct}/{len(rows)}") - return 0 + return calib_status if __name__ == "__main__": From 68a07ee19de1f0f08ffd74a7aea3d77523ddcb81 Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Thu, 9 Jul 2026 23:19:31 +0530 Subject: [PATCH 02/10] =?UTF-8?q?week=5F5:=20address=20CodeRabbit=20findin?= =?UTF-8?q?g=20on=20#974=20=E2=80=94=20build=20live=20pipeline=20once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report_retrieval_recall and report_calibration each built the live pipeline (DB + embedding model + cross-encoder) independently, loading it twice and reranking every positive row twice per --use_live_embeddings run. Build it once in main and pass (retriever, reranker) into both reports, matching _build_live_pipeline's stated intent. Behavior-preserving: recall@20 285/292, rerank top-1 220/292, ECE 0.046 PASS unchanged. --- scripts/evaluate_librarian.py | 57 +++++++++++++++-------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 14d4a1df8..201497b23 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -161,17 +161,18 @@ def _to_ext(mapping): def report_retrieval_recall( rows: List[GoldenDatasetRow], - cache_file: str, + retriever, + reranker, top_k: int, - threshold: float, top_n_rerank: int, - crossencoder_model: str, ) -> None: """Measure the live C.1 -> C.2 pipeline over the positive slice (v1). - Two metrics, both live — there is no honest offline value: the candidate - pool must be the real CRE-node vectors, and seeding it from the golden text - is exactly the leakage the hub firewall strips. + Takes a prebuilt ``retriever``/``reranker`` (built once in ``main``) so the + DB, embedding model, and cross-encoder load once per run and are shared with + ``report_calibration``. Two metrics, both live — there is no honest offline + value: the candidate pool must be the real CRE-node vectors, and seeding it + from the golden text is exactly the leakage the hub firewall strips. - retrieval recall@k (C.1): does the expected CRE id make it into the top-K shortlist the reranker will see? A miss here is unrecoverable downstream. @@ -179,10 +180,6 @@ def report_retrieval_recall( the shortlist, is the #1 candidate an expected CRE? This is the first end-to-end accuracy number for the search path (W4 target >= 0.80). """ - retriever, reranker = _build_live_pipeline( - cache_file, top_k, threshold, top_n_rerank, crossencoder_model - ) - positives = [r for r in rows if r.slice.value == "positive" and r.expected.cre_ids] if not positives: print("retrieval recall: no positive rows with expected ids in this selection") @@ -213,20 +210,19 @@ def report_retrieval_recall( def report_calibration( rows: List[GoldenDatasetRow], - cache_file: str, - top_k: int, - threshold: float, - top_n_rerank: int, - crossencoder_model: str, + retriever, + reranker, ) -> int: """Fit temperature on the golden set and report the Week 5 ECE gate (< 0.10). - Builds a (shortlist, label) calibration set from the live C.1 -> C.2 pipeline - over the positive + hard_negative slices: each row's *reranked shortlist* of - logits, labelled 1 iff its top-1 candidate is an expected CRE (hard_negatives - expect none, so they contribute the 0 class). Both slices are needed so the - fit sees both outcomes (else it is degenerate). Confidence is the top-1 mass - of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on + Takes the prebuilt ``retriever``/``reranker`` shared with + ``report_retrieval_recall`` (built once in ``main``). Builds a + (shortlist, label) calibration set from the live C.1 -> C.2 pipeline over the + positive + hard_negative slices: each row's *reranked shortlist* of logits, + labelled 1 iff its top-1 candidate is an expected CRE (hard_negatives expect + none, so they contribute the 0 class). Both slices are needed so the fit sees + both outcomes (else it is degenerate). Confidence is the top-1 mass of + softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on ECE < 0.10; returns 1 on a failed gate so a live run can fail. """ from application.utils.librarian.calibration.temperature import ( @@ -235,9 +231,6 @@ def report_calibration( fit_temperature, ) - retriever, reranker = _build_live_pipeline( - cache_file, top_k, threshold, top_n_rerank, crossencoder_model - ) cal_rows = [r for r in rows if r.slice.value in ("positive", "hard_negative")] logit_sets: List[List[float]] = [] labels: List[float] = [] @@ -355,22 +348,20 @@ def main(argv: List[str]) -> int: return 1 calib_status = 0 if args.use_live_embeddings: - report_retrieval_recall( - rows, + # Build the live pipeline once (DB + embedding model + cross-encoder) and + # share it across both live reports, so the heavy load and per-row rerank + # happen a single time per run. + retriever, reranker = _build_live_pipeline( args.cache_file, args.top_k_retrieval, args.threshold, args.top_k_rerank, cfg.crossencoder_model, ) - calib_status = report_calibration( - rows, - args.cache_file, - args.top_k_retrieval, - args.threshold, - args.top_k_rerank, - cfg.crossencoder_model, + report_retrieval_recall( + rows, retriever, reranker, args.top_k_retrieval, args.top_k_rerank ) + calib_status = report_calibration(rows, retriever, reranker) else: print( "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3): " From fccfaabc4cbf2c247ca977843a50dd86f65677ac Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Sat, 18 Jul 2026 20:12:43 +0530 Subject: [PATCH 03/10] =?UTF-8?q?week=5F6:=20Module=20C=20(The=20Librarian?= =?UTF-8?q?)=20=E2=80=94=20C.4=20decision=20engine=20+=20golden-set=20deci?= =?UTF-8?q?sion=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C.3 (Week 5) produces one honest, calibrated confidence per chunk; C.4 turns it into the action — auto-link into the OpenCRE graph, or route to a human — which is the accuracy gate of the whole pipeline. - decision_engine.py: pure `decide(confidence, candidates, *, threshold, adversarial, update_ambiguous) -> DecisionResult`. Links the top-1 iff confidence >= threshold AND candidates exist AND no blocking flag; otherwise reviews with a reason_code. Reason precedence NO_CANDIDATES > ADVERSARIAL_FLAG > UPDATE_AMBIGUOUS > BELOW_THRESHOLD. Frozen result, versioned ENGINE_NAME, custom DecisionError — mirrors the C.1/C.2/C.3 model-free seams. Does not import the C.3 scaler (confidence-in -> decision-out), so it is hermetically testable. - decision_engine_test.py: 14 hermetic tests — table-driven over every confidence/flag combination, the inclusive >= boundary, all four reason codes, precedence order, and the input guards. - evaluate_librarian.py: additive report_decision_accuracy — fits T on positive+hard_negative, runs the live C.1->C.4 decision over the golden set, and reports overall agreement plus auto-link recall vs review recall (a single accuracy hides that at tau=0.80 the softmax top-1 mass of a correct-but-close winner is often ~0.5, so correct positives route to review — the safe direction; W7 tunes tau). Informational, not a gate: the SafetyGuard flags are not wired yet, so flag-based reason codes lag until that lands. Emitters (LinkProposal/ReviewItem writers) and the C.0->C.4 pipeline glue follow in a stacked week_6b PR. --- .../tests/librarian/decision_engine_test.py | 99 +++++++++++++++++++ .../utils/librarian/decision_engine.py | 99 +++++++++++++++++++ scripts/evaluate_librarian.py | 95 ++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 application/tests/librarian/decision_engine_test.py create mode 100644 application/utils/librarian/decision_engine.py diff --git a/application/tests/librarian/decision_engine_test.py b/application/tests/librarian/decision_engine_test.py new file mode 100644 index 000000000..f9e1e630e --- /dev/null +++ b/application/tests/librarian/decision_engine_test.py @@ -0,0 +1,99 @@ +"""Hermetic tests for C.4 — the decision engine (Week 6). + +Table-driven over every (confidence, candidates, flag) combination the rule can +see, plus reason-code precedence and the input guards. No key, DB, or model. +""" + +import dataclasses +import math +import unittest + +from application.utils.librarian.decision_engine import ( + ENGINE_NAME, + DecisionError, + DecisionResult, + decide, +) +from application.utils.librarian.schemas import Decision, ReasonCode + +TAU = 0.8 +CANDS = ("616-305", "764-507", "611-909") + + +class DecideTest(unittest.TestCase): + def test_links_when_confident_and_unflagged(self): + r = decide(0.95, CANDS, threshold=TAU) + self.assertEqual(r.decision, Decision.linked) + self.assertIsNone(r.reason_code) + self.assertEqual(r.cre_ids, ("616-305",)) # only the top-1 is linked + + def test_confidence_exactly_at_threshold_links(self): + # link iff confidence >= threshold — the boundary is inclusive. + r = decide(TAU, CANDS, threshold=TAU) + self.assertEqual(r.decision, Decision.linked) + self.assertIsNone(r.reason_code) + + def test_just_below_threshold_reviews(self): + r = decide(TAU - 1e-9, CANDS, threshold=TAU) + self.assertEqual(r.decision, Decision.review) + self.assertEqual(r.reason_code, ReasonCode.below_threshold) + self.assertEqual(r.cre_ids, ("616-305",)) # best-guess suggestion kept + + def test_no_candidates_reviews_even_when_confident(self): + r = decide(0.99, (), threshold=TAU) + self.assertEqual(r.decision, Decision.review) + self.assertEqual(r.reason_code, ReasonCode.no_candidates) + self.assertEqual(r.cre_ids, ()) # nothing to suggest + + def test_adversarial_flag_reviews_even_when_confident(self): + r = decide(0.99, CANDS, threshold=TAU, adversarial=True) + self.assertEqual(r.decision, Decision.review) + self.assertEqual(r.reason_code, ReasonCode.adversarial_flag) + + def test_update_ambiguous_flag_reviews_even_when_confident(self): + r = decide(0.99, CANDS, threshold=TAU, update_ambiguous=True) + self.assertEqual(r.decision, Decision.review) + self.assertEqual(r.reason_code, ReasonCode.update_ambiguous) + + def test_precedence_no_candidates_beats_everything(self): + # empty shortlist + a flag + high confidence -> still NO_CANDIDATES. + r = decide(0.99, (), threshold=TAU, adversarial=True, update_ambiguous=True) + self.assertEqual(r.reason_code, ReasonCode.no_candidates) + + def test_precedence_adversarial_beats_below_threshold(self): + r = decide(0.10, CANDS, threshold=TAU, adversarial=True) + self.assertEqual(r.reason_code, ReasonCode.adversarial_flag) + + def test_precedence_adversarial_beats_update_ambiguous(self): + r = decide(0.99, CANDS, threshold=TAU, adversarial=True, update_ambiguous=True) + self.assertEqual(r.reason_code, ReasonCode.adversarial_flag) + + def test_confidence_is_carried_through(self): + for conf in (0.0, 0.42, 0.8, 1.0): + self.assertEqual(decide(conf, CANDS, threshold=TAU).confidence, conf) + + +class GuardTest(unittest.TestCase): + def test_bad_threshold_rejected(self): + for bad in (-0.1, 1.1, math.nan, math.inf): + with self.assertRaises(DecisionError): + decide(0.5, CANDS, threshold=bad) + + def test_bad_confidence_rejected(self): + for bad in (-0.1, 1.1, math.nan, math.inf): + with self.assertRaises(DecisionError): + decide(bad, CANDS, threshold=TAU) + + +class ResultTest(unittest.TestCase): + def test_engine_name_is_versioned(self): + self.assertRegex(ENGINE_NAME, r"^decision-engine/\d+\.\d+\.\d+$") + + def test_result_is_frozen(self): + r = decide(0.95, CANDS, threshold=TAU) + with self.assertRaises(dataclasses.FrozenInstanceError): + r.confidence = 0.1 # type: ignore[misc] + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/librarian/decision_engine.py b/application/utils/librarian/decision_engine.py new file mode 100644 index 000000000..c34a3ef6e --- /dev/null +++ b/application/utils/librarian/decision_engine.py @@ -0,0 +1,99 @@ +"""Module C.4 — the decision engine (Week 6). The gatekeeper. + +C.3 hands up one calibrated confidence: "how likely is the top reranked candidate +the correct CRE?" C.4 turns that honest number into an action — **auto-link** the +chunk into the OpenCRE graph, or **route it to a human** for review. That choice is +the accuracy gate of the whole pipeline, so the rule is deliberately small and total: + + - no candidate at all -> review (NO_CANDIDATES) + - a blocking safety flag -> review (ADVERSARIAL_FLAG / UPDATE_AMBIGUOUS) + - confidence below the threshold -> review (BELOW_THRESHOLD) + - otherwise -> auto-link the top-1 candidate + +Like C.1/C.2/C.3 this is a thin, model-free seam: a pure function of +``(confidence, candidates, flags, threshold)`` -> ``DecisionResult``. It does **not** +import the C.3 ``TemperatureScaler`` — it consumes the confidence that scaler already +produced — so it is hermetically testable and agnostic to how the number was made. +Turning a ``DecisionResult`` into the RFC ``LinkProposal`` / ``ReviewItem`` envelope +(which needs the full chunk context) is the emitter's job, wired in the pipeline. + +Reason-code precedence when several conditions hold at once: +``NO_CANDIDATES > ADVERSARIAL_FLAG > UPDATE_AMBIGUOUS > BELOW_THRESHOLD``. A safety +flag is surfaced to the reviewer ahead of a mere low-confidence note, because it is +the more important thing for a human to see; you cannot link nothing, so the empty +shortlist dominates everything. +""" + +import math +from dataclasses import dataclass +from typing import Optional, Sequence, Tuple + +from application.utils.librarian.schemas import Decision, ReasonCode + +# Identify the engine in the RFC audit trail (mirrors RETRIEVER_NAME / +# RERANKER_NAME / CALIBRATOR_NAME). +ENGINE_NAME = "decision-engine/0.1.0" + + +class DecisionError(ValueError): + """Raised on decision-engine misuse (bad threshold or confidence).""" + + +@dataclass(frozen=True) +class DecisionResult: + """The verdict for one chunk. Frozen so it is a stable, loggable value. + + ``cre_ids`` is the top-1 candidate: the CRE that gets linked when + ``decision == linked``, or the reviewer's best-guess suggestion when + ``decision == review`` (empty only when there were no candidates at all). + ``reason_code`` is set iff ``decision == review``. + """ + + decision: Decision + confidence: float + cre_ids: Tuple[str, ...] + reason_code: Optional[ReasonCode] = None + + +def _validate(confidence: float, threshold: float) -> None: + if not math.isfinite(threshold) or not 0.0 <= threshold <= 1.0: + raise DecisionError(f"threshold must be finite in [0, 1], got {threshold}") + if not math.isfinite(confidence) or not 0.0 <= confidence <= 1.0: + raise DecisionError(f"confidence must be finite in [0, 1], got {confidence}") + + +def decide( + confidence: float, + candidate_cre_ids: Sequence[str], + *, + threshold: float, + adversarial: bool = False, + update_ambiguous: bool = False, +) -> DecisionResult: + """Apply the auto-link rule to one chunk's calibrated confidence. + + ``candidate_cre_ids`` is the reranked shortlist, best first (may be empty). + ``threshold`` is the auto-link bar τ (``LibrarianConfig.link_threshold``); a + chunk links only when ``confidence >= threshold``. ``adversarial`` / + ``update_ambiguous`` are blocking flags from the SafetyGuard (both default + False until it is wired) — either one forces review regardless of confidence. + """ + _validate(confidence, threshold) + + top = tuple(candidate_cre_ids[:1]) + + if not candidate_cre_ids: + return DecisionResult(Decision.review, confidence, (), ReasonCode.no_candidates) + if adversarial: + return DecisionResult( + Decision.review, confidence, top, ReasonCode.adversarial_flag + ) + if update_ambiguous: + return DecisionResult( + Decision.review, confidence, top, ReasonCode.update_ambiguous + ) + if confidence < threshold: + return DecisionResult( + Decision.review, confidence, top, ReasonCode.below_threshold + ) + return DecisionResult(Decision.linked, confidence, top, None) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 201497b23..9e3e95937 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -264,6 +264,100 @@ def report_calibration( return 0 if gate_ok else 1 +def report_decision_accuracy( + rows: List[GoldenDatasetRow], + retriever, + reranker, + threshold: float, +) -> int: + """Run the full C.1 -> C.4 decision over the golden set and measure how often + ``decide()`` lands on the expected auto-link-vs-review call. + + Fits temperature on the positive + hard_negative slices (as in + ``report_calibration``), then for every golden row carrying an expected + decision: retrieve -> rerank -> C.3 confidence (top-1 softmax mass) -> + ``decide()`` at the auto-link threshold. Reports the linked-vs-review accuracy + (the meaningful C.4 number at this fixed threshold) and, for expected-review + rows, how often the ``reason_code`` matches too. + + Informational — it does not fail the run: the SafetyGuard flags (adversarial / + update_ambiguous) are not wired yet, so ``decide()`` sees them as False here and + reason codes that depend on them lag until that lands; and tuning the threshold + itself is the Week 7 experiment, so hard-gating it now would be premature. + """ + from application.utils.librarian.calibration.temperature import fit_temperature + from application.utils.librarian.decision_engine import decide + from application.utils.librarian.schemas import Decision + + # Fit T on the same positive + hard_negative calibration set as C.3. + cal_rows = [r for r in rows if r.slice.value in ("positive", "hard_negative")] + logit_sets: List[List[float]] = [] + labels: List[float] = [] + for row in cal_rows: + audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + reranked = [c for c in audit.reranked if c.score_rerank is not None] + if not reranked: + continue + expected = set(row.expected.cre_ids or []) + logit_sets.append([float(c.score_rerank) for c in reranked]) + labels.append(1.0 if reranked[0].cre_id in expected else 0.0) + if len(set(labels)) < 2: + print("decision (C.4): need both outcomes to fit temperature; skipped") + return 0 + scaler = fit_temperature(logit_sets, labels) + + graded = [r for r in rows if r.expected.decision is not None] + if not graded: + print("decision (C.4): no rows with an expected decision in this selection") + return 0 + + dec_match = reason_match = 0 + link_total = link_correct = review_total = review_correct = 0 + for row in graded: + audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + reranked = [c for c in audit.reranked if c.score_rerank is not None] + logits = [float(c.score_rerank) for c in reranked] + cre_ids = [c.cre_id for c in reranked] + confidence = scaler.confidence(logits) if logits else 0.0 + result = decide(confidence, cre_ids, threshold=threshold) + matched = result.decision == row.expected.decision + if matched: + dec_match += 1 + if row.expected.decision == Decision.linked: + link_total += 1 + link_correct += matched + elif row.expected.decision == Decision.review: + review_total += 1 + review_correct += matched + if result.reason_code == row.expected.reason_code: + reason_match += 1 + + # Overall agreement plus the two directions split out, because a single + # accuracy hides the story at an untuned threshold: at tau=0.80 the softmax + # top-1 mass of a correct-but-close winner is often ~0.5, so many correct + # positives fall *below* the bar and route to review (the safe direction). + # Auto-link recall vs review recall makes that visible; W7 tunes tau. + n = len(graded) + print( + f"decision (C.4, {n} rows @ tau={threshold:.2f}): " + f"overall {dec_match}/{n} ({dec_match / n:.0%})" + ) + if link_total: + print( + f" auto-link recall (expected-linked rows): " + f"{link_correct}/{link_total} ({link_correct / link_total:.0%})" + ) + if review_total: + print( + f" review recall (expected-review rows): " + f"{review_correct}/{review_total} ({review_correct / review_total:.0%}); " + f"reason_code match {reason_match}/{review_total} " + f"({reason_match / review_total:.0%}) " + f"(SafetyGuard flags not wired — flag-based codes lag)" + ) + return 0 + + def main(argv: List[str]) -> int: cfg = load_config() parser = argparse.ArgumentParser(description="Module C eval harness (W2: C.0)") @@ -362,6 +456,7 @@ def main(argv: List[str]) -> int: rows, retriever, reranker, args.top_k_retrieval, args.top_k_rerank ) calib_status = report_calibration(rows, retriever, reranker) + report_decision_accuracy(rows, retriever, reranker, args.threshold) else: print( "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3): " From 001bd6e7dc224c1b27504a499ba9242cc35ceeaa Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Fri, 24 Jul 2026 20:38:00 +0530 Subject: [PATCH 04/10] =?UTF-8?q?week=5F6:=20address=20CodeRabbit=20on=20#?= =?UTF-8?q?990=20=E2=80=94=20fix=20calibration=20package=20docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The calibration/__init__.py docstring still described the rejected single-logit `p = sigmoid(z/T)` approach. temperature.py actually calibrates the softmax over the whole shortlist (`p = softmax(logits / T)`, confidence = top-1 mass) — the sigmoid-on-one-logit approach cannot be calibrated by temperature alone. Docstring now matches the implementation. --- application/utils/librarian/calibration/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/application/utils/librarian/calibration/__init__.py b/application/utils/librarian/calibration/__init__.py index 305586a76..54b774038 100644 --- a/application/utils/librarian/calibration/__init__.py +++ b/application/utils/librarian/calibration/__init__.py @@ -1,10 +1,12 @@ """Module C.3 — confidence calibration (Week 5). C.2 (the cross-encoder, W4) emits a raw ranking logit per candidate — great for -ordering, meaningless as confidence (a +1.5 is not "82% sure"). C.3 turns that -logit into an honest probability via **temperature scaling**: ``p = sigmoid(z/T)`` -with a single scalar ``T`` fit by negative-log-likelihood on the golden set, and -proves the result honest with **ECE < 0.10**. +ordering, meaningless as confidence (a +1.5 is not "82% sure"). C.3 turns those +logits into an honest probability by calibrating the **softmax over the whole +shortlist**, ``p = softmax(logits / T)``, with the confidence being the top-1 +candidate's mass — a single scalar ``T`` fit by negative-log-likelihood on the +golden set. It proves the result honest with **ECE < 0.10**. (Calibrating the +single top-1 logit with ``sigmoid(z/T)`` cannot work; see ``temperature.py``.) The W6 decision engine thresholds that probability (auto-link vs. human review), so calibration is what makes the threshold trustworthy. Kept dependency-light From 86d3d5b54a89f4b8440aa9cdae535912a844b3fa Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Mon, 3 Aug 2026 10:43:52 +0530 Subject: [PATCH 05/10] =?UTF-8?q?week=5F6:=20address=20maintainer=20review?= =?UTF-8?q?=20on=20#990=20=E2=80=94=20refresh=20package=20docstring=20scop?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The application/utils/librarian package docstring still said "Decision routing (C.4, W6) onward is not built yet", but W6 adds the C.4 decision engine (decide()) in this package. Add the W6 (C.4) scope line and move the not-yet marker to the W6b emitter/pipeline glue and the W8 queue/graph writers. --- application/utils/librarian/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/application/utils/librarian/__init__.py b/application/utils/librarian/__init__.py index 1a23cc3cb..869b807fa 100644 --- a/application/utils/librarian/__init__.py +++ b/application/utils/librarian/__init__.py @@ -18,7 +18,10 @@ W4 (C.2): cross-encoder reranker — re-sorts the C.1 shortlist, fills reranked[]. W5 (C.3): confidence calibration — temperature scaling maps a rerank logit to an honest probability (fit by NLL on the golden set, gated ECE < 0.10). -Decision routing (C.4, W6) onward is not built yet. + W6 (C.4): decision engine — thresholds the calibrated confidence to auto-link + (LinkProposal) or route to human review (ReviewItem), with a reason. +Envelope emitter + pipeline glue (C.4, W6b) and the queue/graph writers (W8) are +not built yet. Vendored RFC JSON schemas live under ``_rfc_schemas/``. They are pinned to upstream/owasp-graph @ 2b1437987768d5ed20fe9ee721ab9a898c4b84af (PR #734). From ae6bb69cdb1b9567673999c895983ed875dc1de5 Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Tue, 4 Aug 2026 17:17:35 +0530 Subject: [PATCH 06/10] =?UTF-8?q?week=5F5:=20address=20review=20nitpicks?= =?UTF-8?q?=20on=20#974/#990=20=E2=80=94=20share=20live=20shortlists,=20de?= =?UTF-8?q?dupe=20softmax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three nitpicks from the bot review, none of which changed a metric: - evaluate_librarian: the live retrieve+rerank was recomputed per report. The models were already built once in main, but report_retrieval_recall and report_calibration each re-ran the pipeline over the positive slice, so every positive row paid for two cross-encoder passes. live_audits() now retrieves and reranks each row once, keyed by golden row id, and both reports read the same shortlists. Rows without an audit no longer count toward a report's denominator, so the printed fractions cannot divide by unscored rows. - temperature: _softmax_top duplicated the empty-shortlist guard and the softmax already in TemperatureScaler.probabilities. Both now route through one _softmax_at helper, and confidence() derives from probabilities() so the two can never disagree. - temperature is now clean under the --strict mypy the coding guidelines ask for: annotated _paired's return and the bounds tuple, re-asserted the array type over untyped scipy, and hoisted the label conversion out of the fit objective (it was re-validated on every optimiser iteration anyway). Behaviour is unchanged: 113 librarian tests pass and the hermetic harness run still exits 0. (cherry picked from commit 2bbc76e57463d82ec90a9ac9f8a4395419812469) --- .../librarian/calibration/temperature.py | 38 +++++---- scripts/evaluate_librarian.py | 79 +++++++++++++------ 2 files changed, 79 insertions(+), 38 deletions(-) diff --git a/application/utils/librarian/calibration/temperature.py b/application/utils/librarian/calibration/temperature.py index 928dacf7a..8ded3abc3 100644 --- a/application/utils/librarian/calibration/temperature.py +++ b/application/utils/librarian/calibration/temperature.py @@ -31,7 +31,7 @@ """ from dataclasses import dataclass -from typing import Sequence +from typing import List, Sequence, Tuple import numpy as np from scipy.optimize import minimize_scalar @@ -60,12 +60,18 @@ class DegenerateLabelsError(CalibrationError): """ -def _softmax_top(logits: Sequence[float], temperature: float) -> float: - """Top-1 probability mass of ``softmax(logits / T)`` over one shortlist.""" +def _softmax_at(logits: Sequence[float], temperature: float) -> np.ndarray: + """``softmax(logits / T)`` over one shortlist. + + The single place the empty-shortlist guard lives: ``TemperatureScaler``'s + ``probabilities``/``confidence`` and the free-``T`` NLL objective all route + their softmax through here, so the distribution is only defined once. + """ z = np.asarray(list(logits), dtype=float) if z.size == 0: raise CalibrationError("cannot calibrate an empty candidate shortlist") - return float(softmax(z / temperature).max()) + # scipy is untyped, so re-assert the array type for --strict. + return np.asarray(softmax(z / temperature), dtype=float) def _validate_temperature(temperature: float) -> None: @@ -73,7 +79,9 @@ def _validate_temperature(temperature: float) -> None: raise CalibrationError(f"temperature must be finite and > 0, got {temperature}") -def _paired(logit_sets: Sequence[Sequence[float]], labels: Sequence[float]): +def _paired( + logit_sets: Sequence[Sequence[float]], labels: Sequence[float] +) -> Tuple[List[Sequence[float]], np.ndarray]: """Validate matched (shortlist, label) inputs: non-empty and equal length.""" sets = list(logit_sets) y = np.asarray(list(labels), dtype=float) @@ -104,17 +112,16 @@ def __post_init__(self) -> None: def probabilities(self, logits: Sequence[float]) -> np.ndarray: """The full ``softmax(logits / T)`` distribution over one shortlist.""" - z = np.asarray(list(logits), dtype=float) - if z.size == 0: - raise CalibrationError("cannot calibrate an empty candidate shortlist") - return softmax(z / self.temperature) + return _softmax_at(logits, self.temperature) def confidence(self, logits: Sequence[float]) -> float: """P(the top candidate is correct) — the top-1 mass of the softmax. - This is the number the W6 decision engine thresholds on. + Derived from ``probabilities`` rather than recomputing the softmax, so + the two can never disagree. This is the number the W6 decision engine + thresholds on. """ - return _softmax_top(logits, self.temperature) + return float(self.probabilities(logits).max()) def negative_log_likelihood( @@ -132,7 +139,7 @@ def negative_log_likelihood( _validate_temperature(temperature) sets, y = _paired(logit_sets, labels) p = np.clip( - np.array([_softmax_top(s, temperature) for s in sets]), _EPS, 1.0 - _EPS + np.array([_softmax_at(s, temperature).max() for s in sets]), _EPS, 1.0 - _EPS ) return float(-np.sum(y * np.log(p) + (1.0 - y) * np.log(1.0 - p))) @@ -141,7 +148,7 @@ def fit_temperature( logit_sets: Sequence[Sequence[float]], labels: Sequence[float], *, - bounds: tuple = (1e-2, 1e2), + bounds: Tuple[float, float] = (1e-2, 1e2), ) -> TemperatureScaler: """Fit ``T`` by minimising NLL over (shortlist, is-top1-correct) pairs. @@ -160,8 +167,11 @@ def fit_temperature( "calibration set needs both correct and incorrect top-1s" ) + # Hoisted out of the objective: the labels are re-validated on every NLL call, + # so convert once rather than per optimiser iteration. + y_list: List[float] = [float(v) for v in y.tolist()] result = minimize_scalar( - lambda t: negative_log_likelihood(sets, y, t), + lambda t: negative_log_likelihood(sets, y_list, t), bounds=bounds, method="bounded", ) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 9e3e95937..7ec1f8bea 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -19,7 +19,7 @@ import os import sys from collections import Counter -from typing import List, Set +from typing import Any, Dict, List, Set # Bootstrap project root onto sys.path so this runs as a standalone script. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) @@ -104,9 +104,10 @@ def _build_live_pipeline( """Construct the live C.1 retriever + C.2 reranker against the OpenCRE DB. Live deps are imported lazily so the offline harness needs neither a DB, an - embedding model, nor the cross-encoder stack. Shared by every live report - (recall/top-1 and calibration) so the heavy hub + model load happens once - per report and the id-space translation stays in one place. + embedding model, nor the cross-encoder stack. Called once per run from + ``main`` and shared by every live report (recall/top-1 and calibration), so + the heavy hub + model load happens a single time and the id-space translation + stays in one place. """ from application.cmd.cre_main import db_connect from application.defs import cre_defs @@ -159,20 +160,39 @@ def _to_ext(mapping): return retriever, reranker -def report_retrieval_recall( +def live_audits( rows: List[GoldenDatasetRow], retriever, reranker, +) -> Dict[str, Any]: + """Retrieve + rerank every row once, keyed by golden row id. + + Every live report wants the same thing per row: the reranked shortlist. The + cross-encoder pass is the expensive step (one inference per candidate pair), + and the positive slice is read by more than one report, so computing the + audits here means a live run pays for exactly one retrieve + rerank per row + regardless of how many reports consume it. + """ + return { + row.id: reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + for row in rows + } + + +def report_retrieval_recall( + rows: List[GoldenDatasetRow], + audits: Dict[str, Any], top_k: int, top_n_rerank: int, ) -> None: """Measure the live C.1 -> C.2 pipeline over the positive slice (v1). - Takes a prebuilt ``retriever``/``reranker`` (built once in ``main``) so the - DB, embedding model, and cross-encoder load once per run and are shared with - ``report_calibration``. Two metrics, both live — there is no honest offline - value: the candidate pool must be the real CRE-node vectors, and seeding it - from the golden text is exactly the leakage the hub firewall strips. + Reads the shared ``audits`` from ``live_audits`` (computed once in ``main``) + so the DB, embedding model, and cross-encoder load once per run and no row is + retrieved or reranked twice. Two metrics, both live — there is no honest + offline value: the candidate pool must be the real CRE-node vectors, and + seeding it from the golden text is exactly the leakage the hub firewall + strips. - retrieval recall@k (C.1): does the expected CRE id make it into the top-K shortlist the reranker will see? A miss here is unrecoverable downstream. @@ -180,14 +200,19 @@ def report_retrieval_recall( the shortlist, is the #1 candidate an expected CRE? This is the first end-to-end accuracy number for the search path (W4 target >= 0.80). """ - positives = [r for r in rows if r.slice.value == "positive" and r.expected.cre_ids] + # Only rows that were actually audited count toward the denominator, so the + # printed fractions never silently divide by rows no report ever scored. + positives = [ + r + for r in rows + if r.slice.value == "positive" and r.expected.cre_ids and r.id in audits + ] if not positives: print("retrieval recall: no positive rows with expected ids in this selection") return any_hit = all_hit = top1_hit = 0 for row in positives: - audit = retriever.retrieve(row.input.text) - audit = reranker.rerank(row.input.text, audit) + audit = audits[row.id] retrieved = {c.cre_id for c in audit.candidates} expected = set(row.expected.cre_ids or []) if expected & retrieved: @@ -210,13 +235,13 @@ def report_retrieval_recall( def report_calibration( rows: List[GoldenDatasetRow], - retriever, - reranker, + audits: Dict[str, Any], ) -> int: """Fit temperature on the golden set and report the Week 5 ECE gate (< 0.10). - Takes the prebuilt ``retriever``/``reranker`` shared with - ``report_retrieval_recall`` (built once in ``main``). Builds a + Reads the shared ``audits`` from ``live_audits``, the same shortlists + ``report_retrieval_recall`` scores, so the positive slice is not retrieved and + reranked a second time just to calibrate on it. Builds a (shortlist, label) calibration set from the live C.1 -> C.2 pipeline over the positive + hard_negative slices: each row's *reranked shortlist* of logits, labelled 1 iff its top-1 candidate is an expected CRE (hard_negatives expect @@ -235,7 +260,9 @@ def report_calibration( logit_sets: List[List[float]] = [] labels: List[float] = [] for row in cal_rows: - audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + audit = audits.get(row.id) + if audit is None: + continue reranked = [c for c in audit.reranked if c.score_rerank is not None] if not reranked: continue @@ -442,9 +469,10 @@ def main(argv: List[str]) -> int: return 1 calib_status = 0 if args.use_live_embeddings: - # Build the live pipeline once (DB + embedding model + cross-encoder) and - # share it across both live reports, so the heavy load and per-row rerank - # happen a single time per run. + # Build the live pipeline once (DB + embedding model + cross-encoder), then + # retrieve + rerank each row once, so the heavy model load *and* the + # per-pair cross-encoder inference each happen a single time per run no + # matter how many reports read the shortlists. retriever, reranker = _build_live_pipeline( args.cache_file, args.top_k_retrieval, @@ -452,10 +480,13 @@ def main(argv: List[str]) -> int: args.top_k_rerank, cfg.crossencoder_model, ) - report_retrieval_recall( - rows, retriever, reranker, args.top_k_retrieval, args.top_k_rerank + audits = live_audits( + [r for r in rows if r.slice.value in ("positive", "hard_negative")], + retriever, + reranker, ) - calib_status = report_calibration(rows, retriever, reranker) + report_retrieval_recall(rows, audits, args.top_k_retrieval, args.top_k_rerank) + calib_status = report_calibration(rows, audits) report_decision_accuracy(rows, retriever, reranker, args.threshold) else: print( From cb5577a8bab0dd2c83a12ae5e8fafbd2687d4619 Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Mon, 3 Aug 2026 10:42:24 +0530 Subject: [PATCH 07/10] =?UTF-8?q?week=5F5:=20address=20maintainer=20review?= =?UTF-8?q?=20on=20#974=20=E2=80=94=20degenerate=20calibration=20gate=20mu?= =?UTF-8?q?st=20fail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report_calibration returned 0 ("skipped") when the live calibration set was degenerate (single-class labels, empty after dropping empty shortlists, or a --slice with one class). Under --use_live_embeddings that let a run exit 0 without the ECE < 0.10 gate ever running, so CI could greenwash a live run in which calibration was never checked. A skipped gate now returns 1 (fail), with a message stating the row/class counts, so exit 0 means the gate actually ran and passed. (cherry picked from commit 62abf2d2c66201aee916bad649fbfb45397276bb) --- scripts/evaluate_librarian.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 7ec1f8bea..7d643906a 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -248,7 +248,8 @@ def report_calibration( none, so they contribute the 0 class). Both slices are needed so the fit sees both outcomes (else it is degenerate). Confidence is the top-1 mass of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on - ECE < 0.10; returns 1 on a failed gate so a live run can fail. + ECE < 0.10. Returns 1 on a failed gate, and also on a degenerate calibration + set, so a live run can never exit 0 without the gate actually having run. """ from application.utils.librarian.calibration.temperature import ( TemperatureScaler, @@ -271,11 +272,17 @@ def report_calibration( labels.append(1.0 if reranked[0].cre_id in expected else 0.0) if len(set(labels)) < 2: + # Degenerate calibration set: single-class labels, or nothing left after + # dropping rows with an empty shortlist. Either way the ECE gate did not + # run, so this must not exit 0 — a skipped gate reported as success lets + # CI greenwash a live run in which calibration was never checked. print( "calibration (C.3): need both outcomes in the selection (positive + " - "hard_negative slices) to fit temperature; skipped" + "hard_negative slices) to fit temperature; got " + f"{len(labels)} row(s) covering {len(set(labels))} class(es); " + "FAILED (gate did not run)" ) - return 0 + return 1 scaler = fit_temperature(logit_sets, labels) conf_raw = [TemperatureScaler(1.0).confidence(s) for s in logit_sets] From 6ef78651892149429671f66c1c6a73522c83afed Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Tue, 4 Aug 2026 17:25:17 +0530 Subject: [PATCH 08/10] =?UTF-8?q?week=5F6:=20address=20review=20nitpick=20?= =?UTF-8?q?on=20#991=20=E2=80=94=20fit=20T=20once,=20and=20cover=20the=20l?= =?UTF-8?q?ive=20reports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report_decision_accuracy rebuilt the positive + hard_negative calibration set from its own retrieve+rerank pass and fit its own temperature, duplicating what report_calibration had already done a few lines earlier. On a live run that meant a third pipeline pass over the calibration slices and two independent fits of the same T, with nothing guaranteeing the two agreed. Now there is one of each: - calibration_set() extracts the (shortlist, label) derivation, so the C.3 gate and the C.4 report read the same pairs off the same shared audits. - report_calibration returns (status, scaler); report_decision_accuracy takes the fitted scaler instead of fitting its own, so C.4 thresholds on exactly the T the ECE gate measured. - A degenerate set yields (1, None) and C.4 is skipped with a message: no fitted T means no honest confidence to threshold, and the run has already failed. - The live audit set now covers expected-decision rows too. C.4 grades those and they are not confined to the positive/hard_negative slices, so keying them off the calibration slices alone would have dropped them. Adds evaluate_harness_test.py. The live reports only run under --use_live_embeddings, so nothing exercised their wiring — which is exactly the code that has to share one pipeline pass and one T across three reports. A counting stub asserts the pipeline is called once per row and not once per report, that only the two calibration slices enter the fit, and that a degenerate set returns status 1 with no scaler rather than reporting success. Verified the last one fails if the gate is flipped back to 0. 134 librarian tests pass; the hermetic harness run still exits 0. --- .../tests/librarian/evaluate_harness_test.py | 251 ++++++++++++++++++ scripts/evaluate_librarian.py | 132 +++++---- 2 files changed, 327 insertions(+), 56 deletions(-) create mode 100644 application/tests/librarian/evaluate_harness_test.py diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py new file mode 100644 index 000000000..b321e2f5e --- /dev/null +++ b/application/tests/librarian/evaluate_harness_test.py @@ -0,0 +1,251 @@ +"""Hermetic tests for the live-report plumbing in ``scripts/evaluate_librarian.py``. + +The live reports (recall/top-1, the C.3 ECE gate, the C.4 decision accuracy) only +run under ``--use_live_embeddings``, which needs a populated DB, an embedding +model, and the cross-encoder — so nothing exercised their wiring. That is exactly +the code that has to share one retrieve+rerank pass and one fitted ``T`` across +three reports, so the sharing is asserted here against stub seams instead: + +- ``live_audits`` must call the pipeline once per row, never once per report. +- ``calibration_set`` must draw only the positive + hard_negative slices. +- ``report_calibration`` must hand back the fitted scaler, and must fail (status + 1, no scaler) on a degenerate set rather than reporting success. +- ``report_decision_accuracy`` must consume that scaler and the shared audits + without touching the retriever or reranker again. +""" + +import importlib.util +import os +import unittest +from typing import List, Optional + +from application.utils.librarian.schemas import CreCandidate, RetrievalAudit + +# The harness is a standalone script, not an importable package module. +_HARNESS_PATH = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "scripts", "evaluate_librarian.py" +) +_spec = importlib.util.spec_from_file_location("evaluate_librarian", _HARNESS_PATH) +assert _spec and _spec.loader +harness = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(harness) + + +def _golden_row( + row_id: str, + slice_name: str, + text: str, + cre_ids: List[str], + reason_code: Optional[str] = None, +): + """Build a GoldenDatasetRow through the real validator, not a stub. + + ``expected.decision`` is required by the schema, and ``linked`` requires + ``cre_ids`` while ``review`` requires a ``reason_code``, so both are derived: + rows with expected ids are linked, rows without route to review below the bar. + """ + from application.utils.librarian.schemas import GoldenDatasetRow + + expected: dict = { + "decision": "linked" if cre_ids else "review", + "cre_ids": cre_ids or None, + } + if not cre_ids: + expected["reason_code"] = reason_code or "BELOW_THRESHOLD" + elif reason_code is not None: + expected["reason_code"] = reason_code + source_input: dict = {"text": text, "source_standard": "ASVS"} + if slice_name == "explicit": + # The schema ties the explicit slice to a cited CRE id. + source_input["explicit_cre_ref"] = (cre_ids or ["616-305"])[0] + return GoldenDatasetRow.model_validate( + { + "id": row_id, + "schema_version": "0.1.0", + "slice": slice_name, + "input": source_input, + "expected": expected, + "provenance": { + "section_path": f"{row_id}.md", + "ground_truth_source": "synthesised for the harness plumbing tests", + }, + } + ) + + +class CountingPipeline: + """Stub retriever+reranker that records how many passes it was asked for.""" + + def __init__(self, shortlists): + # shortlists: row text -> list of (cre_id, logit), best first + self._shortlists = shortlists + self.retrieve_calls = 0 + self.rerank_calls = 0 + + def retrieve(self, text: str) -> RetrievalAudit: + self.retrieve_calls += 1 + pairs = self._shortlists.get(text, []) + return RetrievalAudit( + retriever="stub/1.0.0", + candidates=[CreCandidate(cre_id=c, score_vector=0.5) for c, _ in pairs], + reranked=[], + threshold=0.0, + ) + + def rerank(self, text: str, audit: RetrievalAudit) -> RetrievalAudit: + self.rerank_calls += 1 + pairs = self._shortlists.get(text, []) + return audit.model_copy( + update={ + "reranked": [ + CreCandidate(cre_id=c, score_rerank=logit) for c, logit in pairs + ] + } + ) + + +class LiveAuditsTest(unittest.TestCase): + def test_pipeline_runs_once_per_row_not_once_per_report(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + ] + pipe = CountingPipeline({"alpha": [("616-305", 4.0)], "beta": [("111-111", 3.0)]}) + + audits = harness.live_audits(rows, pipe, pipe) + + self.assertEqual(pipe.retrieve_calls, 2) + self.assertEqual(pipe.rerank_calls, 2) + self.assertEqual(set(audits), {"p1", "n1"}) + + # Three reports read the same audits; none of them may re-run the pipeline. + harness.report_retrieval_recall(rows, audits, 10, 5) + status, scaler = harness.report_calibration(rows, audits) + self.assertIsNotNone(scaler) + harness.report_decision_accuracy(rows, audits, scaler, 0.80) + self.assertEqual(pipe.retrieve_calls, 2) + self.assertEqual(pipe.rerank_calls, 2) + + +class CalibrationSetTest(unittest.TestCase): + def test_draws_only_the_two_calibration_slices(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("a1", "ambiguous", "gamma", ["616-305"]), + _golden_row("e1", "explicit", "delta", ["616-305"]), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 4.0)], + "beta": [("111-111", 3.0)], + "gamma": [("616-305", 2.0)], + "delta": [("616-305", 1.0)], + } + ) + audits = harness.live_audits(rows, pipe, pipe) + + logit_sets, labels = harness.calibration_set(rows, audits) + + # ambiguous/explicit rows are audited but must not enter the fit. + self.assertEqual(len(logit_sets), 2) + self.assertEqual(sorted(labels), [0.0, 1.0]) + + def test_skips_rows_with_no_audit_and_empty_shortlists(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + ] + pipe = CountingPipeline({"alpha": [("616-305", 4.0)], "beta": [("111-111", 3.0)]}) + # "empty" yields no candidates; p3 is never audited at all. + audits = harness.live_audits(rows, pipe, pipe) + + logit_sets, labels = harness.calibration_set(rows, audits) + self.assertEqual(len(logit_sets), 2) + self.assertEqual(len(labels), 2) + + +class ReportCalibrationTest(unittest.TestCase): + def test_returns_status_and_fitted_scaler(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("n2", "hard_negative", "beta2", []), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "beta2": [("111-111", 2.0), ("222-222", 1.9)], + } + ) + audits = harness.live_audits(rows, pipe, pipe) + + status, scaler = harness.report_calibration(rows, audits) + + self.assertIn(status, (0, 1)) # gate outcome depends on the stub logits + self.assertIsNotNone(scaler) + self.assertGreater(scaler.temperature, 0.0) + + def test_degenerate_set_fails_and_yields_no_scaler(self) -> None: + # Single-class labels: every top-1 is correct, so T is unidentifiable. + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + ] + pipe = CountingPipeline( + {"alpha": [("616-305", 5.0)], "alpha2": [("616-305", 4.0)]} + ) + audits = harness.live_audits(rows, pipe, pipe) + + status, scaler = harness.report_calibration(rows, audits) + + self.assertEqual(status, 1, "a skipped gate must not report success") + self.assertIsNone(scaler) + + +class ReportDecisionAccuracyTest(unittest.TestCase): + def test_grades_expected_decision_rows_off_shared_audits(self) -> None: + from application.utils.librarian.calibration.temperature import TemperatureScaler + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row( + "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD" + ), + ] + pipe = CountingPipeline( + { + # A dominant top-1 clears tau; a near-tie falls below it. + "alpha": [("616-305", 20.0), ("999-999", 0.0)], + "beta": [("111-111", 1.0), ("222-222", 0.99)], + } + ) + audits = harness.live_audits(rows, pipe, pipe) + before = (pipe.retrieve_calls, pipe.rerank_calls) + + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + + self.assertEqual(status, 0, "the C.4 report is informational, never a gate") + self.assertEqual((pipe.retrieve_calls, pipe.rerank_calls), before) + + def test_no_graded_rows_is_not_an_error(self) -> None: + from application.utils.librarian.calibration.temperature import TemperatureScaler + + rows = [_golden_row("p1", "positive", "alpha", ["616-305"])] + pipe = CountingPipeline({"alpha": [("616-305", 4.0)]}) + audits = harness.live_audits(rows, pipe, pipe) + + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + self.assertEqual(status, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 7d643906a..54bbf5529 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -19,7 +19,7 @@ import os import sys from collections import Counter -from typing import Any, Dict, List, Set +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple # Bootstrap project root onto sys.path so this runs as a standalone script. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) @@ -38,6 +38,9 @@ section_from_queue_row, ) +if TYPE_CHECKING: # the live calibration deps are imported lazily below + from application.utils.librarian.calibration.temperature import TemperatureScaler + # Harness-only synthetic provenance: golden rows are not queue rows, so we # synthesize the minimum B-shaped row needed to exercise the C.0 boundary. _SYNTHETIC_SHA = "0" * 40 @@ -233,34 +236,26 @@ def report_retrieval_recall( ) -def report_calibration( +def calibration_set( rows: List[GoldenDatasetRow], audits: Dict[str, Any], -) -> int: - """Fit temperature on the golden set and report the Week 5 ECE gate (< 0.10). +) -> Tuple[List[List[float]], List[float]]: + """The (shortlist, is-top1-correct) pairs temperature is fit on. - Reads the shared ``audits`` from ``live_audits``, the same shortlists - ``report_retrieval_recall`` scores, so the positive slice is not retrieved and - reranked a second time just to calibrate on it. Builds a - (shortlist, label) calibration set from the live C.1 -> C.2 pipeline over the - positive + hard_negative slices: each row's *reranked shortlist* of logits, - labelled 1 iff its top-1 candidate is an expected CRE (hard_negatives expect - none, so they contribute the 0 class). Both slices are needed so the fit sees - both outcomes (else it is degenerate). Confidence is the top-1 mass of - softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on - ECE < 0.10. Returns 1 on a failed gate, and also on a degenerate calibration - set, so a live run can never exit 0 without the gate actually having run. - """ - from application.utils.librarian.calibration.temperature import ( - TemperatureScaler, - expected_calibration_error, - fit_temperature, - ) + Drawn from the positive + hard_negative slices: each row contributes its + *reranked shortlist* of logits, labelled 1 iff its top-1 candidate is an + expected CRE (hard_negatives expect none, so they supply the 0 class). Both + slices are needed or the fit is degenerate. - cal_rows = [r for r in rows if r.slice.value in ("positive", "hard_negative")] + Split out so the C.3 gate and the C.4 decision report derive the calibration + set exactly once from the same shared audits, rather than each rebuilding it + and fitting its own ``T`` off a separate rerank pass. + """ logit_sets: List[List[float]] = [] labels: List[float] = [] - for row in cal_rows: + for row in rows: + if row.slice.value not in ("positive", "hard_negative"): + continue audit = audits.get(row.id) if audit is None: continue @@ -270,6 +265,34 @@ def report_calibration( expected = set(row.expected.cre_ids or []) logit_sets.append([float(c.score_rerank) for c in reranked]) labels.append(1.0 if reranked[0].cre_id in expected else 0.0) + return logit_sets, labels + + +def report_calibration( + rows: List[GoldenDatasetRow], + audits: Dict[str, Any], +) -> Tuple[int, Optional["TemperatureScaler"]]: + """Fit temperature on the golden set and report the Week 5 ECE gate (< 0.10). + + Reads the shared ``audits`` from ``live_audits``, the same shortlists + ``report_retrieval_recall`` scores, so the positive slice is not retrieved and + reranked a second time just to calibrate on it. Confidence is the top-1 mass + of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on + ECE < 0.10. + + Returns ``(status, scaler)``. Status is 1 on a failed gate, and also on a + degenerate calibration set, so a live run can never exit 0 without the gate + actually having run. The fitted scaler is handed back (``None`` when the set + was degenerate) so the C.4 report thresholds on this same ``T`` instead of + fitting its own. + """ + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + expected_calibration_error, + fit_temperature, + ) + + logit_sets, labels = calibration_set(rows, audits) if len(set(labels)) < 2: # Degenerate calibration set: single-class labels, or nothing left after @@ -282,7 +305,7 @@ def report_calibration( f"{len(labels)} row(s) covering {len(set(labels))} class(es); " "FAILED (gate did not run)" ) - return 1 + return 1, None scaler = fit_temperature(logit_sets, labels) conf_raw = [TemperatureScaler(1.0).confidence(s) for s in logit_sets] @@ -295,52 +318,36 @@ def report_calibration( f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" ) - return 0 if gate_ok else 1 + return (0 if gate_ok else 1), scaler def report_decision_accuracy( rows: List[GoldenDatasetRow], - retriever, - reranker, + audits: Dict[str, Any], + scaler: "TemperatureScaler", threshold: float, ) -> int: """Run the full C.1 -> C.4 decision over the golden set and measure how often ``decide()`` lands on the expected auto-link-vs-review call. - Fits temperature on the positive + hard_negative slices (as in - ``report_calibration``), then for every golden row carrying an expected - decision: retrieve -> rerank -> C.3 confidence (top-1 softmax mass) -> - ``decide()`` at the auto-link threshold. Reports the linked-vs-review accuracy - (the meaningful C.4 number at this fixed threshold) and, for expected-review - rows, how often the ``reason_code`` matches too. + Takes the ``scaler`` already fitted by ``report_calibration`` and the shared + ``audits``, so the calibration set is derived once and ``T`` is fit once per + run: this report re-uses both rather than rebuilding the set and fitting its + own ``T`` off a second rerank pass. For every golden row carrying an expected + decision: C.3 confidence (top-1 softmax mass) -> ``decide()`` at the auto-link + threshold. Reports the linked-vs-review accuracy (the meaningful C.4 number at + this fixed threshold) and, for expected-review rows, how often the + ``reason_code`` matches too. Informational — it does not fail the run: the SafetyGuard flags (adversarial / update_ambiguous) are not wired yet, so ``decide()`` sees them as False here and reason codes that depend on them lag until that lands; and tuning the threshold itself is the Week 7 experiment, so hard-gating it now would be premature. """ - from application.utils.librarian.calibration.temperature import fit_temperature from application.utils.librarian.decision_engine import decide from application.utils.librarian.schemas import Decision - # Fit T on the same positive + hard_negative calibration set as C.3. - cal_rows = [r for r in rows if r.slice.value in ("positive", "hard_negative")] - logit_sets: List[List[float]] = [] - labels: List[float] = [] - for row in cal_rows: - audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) - reranked = [c for c in audit.reranked if c.score_rerank is not None] - if not reranked: - continue - expected = set(row.expected.cre_ids or []) - logit_sets.append([float(c.score_rerank) for c in reranked]) - labels.append(1.0 if reranked[0].cre_id in expected else 0.0) - if len(set(labels)) < 2: - print("decision (C.4): need both outcomes to fit temperature; skipped") - return 0 - scaler = fit_temperature(logit_sets, labels) - - graded = [r for r in rows if r.expected.decision is not None] + graded = [r for r in rows if r.expected.decision is not None and r.id in audits] if not graded: print("decision (C.4): no rows with an expected decision in this selection") return 0 @@ -348,7 +355,7 @@ def report_decision_accuracy( dec_match = reason_match = 0 link_total = link_correct = review_total = review_correct = 0 for row in graded: - audit = reranker.rerank(row.input.text, retriever.retrieve(row.input.text)) + audit = audits[row.id] reranked = [c for c in audit.reranked if c.score_rerank is not None] logits = [float(c.score_rerank) for c in reranked] cre_ids = [c.cre_id for c in reranked] @@ -487,14 +494,27 @@ def main(argv: List[str]) -> int: args.top_k_rerank, cfg.crossencoder_model, ) + # Union of what the reports read: the calibration slices, plus any row + # carrying an expected decision — C.4 grades those and they are not + # confined to positive/hard_negative. audits = live_audits( - [r for r in rows if r.slice.value in ("positive", "hard_negative")], + [ + r + for r in rows + if r.slice.value in ("positive", "hard_negative") + or r.expected.decision is not None + ], retriever, reranker, ) report_retrieval_recall(rows, audits, args.top_k_retrieval, args.top_k_rerank) - calib_status = report_calibration(rows, audits) - report_decision_accuracy(rows, retriever, reranker, args.threshold) + calib_status, scaler = report_calibration(rows, audits) + if scaler is not None: + report_decision_accuracy(rows, audits, scaler, args.threshold) + else: + # No fitted T means no honest confidence for C.4 to threshold on. + # report_calibration has already failed the run. + print("decision (C.4): skipped — calibration produced no fitted T") else: print( "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3): " From 8bb865a3e0585db4e1622be7a262d7e8187e7016 Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Tue, 4 Aug 2026 23:15:58 +0530 Subject: [PATCH 09/10] week_5: reject duplicate golden row ids in load_dataset Follow-up to the shared-audit refactor. live_audits keys the reranked shortlists by row.id and the reports read them back with audits[row.id], so two rows sharing an id would collapse in that dict: the surviving audit gets reused for the earlier row and its scores are reported against the wrong text. Before the refactor each report recomputed per row, so a duplicate id was harmless; keying by it turned a harmless quirk into a silently wrong number. GoldenDatasetRow only requires an id to be non-empty, so uniqueness is enforced at load time and the harness refuses the file instead of printing a wrong metric. The committed dataset has 319 distinct ids, so nothing changes today; this closes the trap the refactor opened. Adds a test that a forced collision raises and names the offending id. (cherry picked from commit f352dfcc0ce0c03dbd9bc83091fda428fd5687c1) --- application/tests/librarian/dataset_test.py | 41 +++++++++++++++++++++ scripts/evaluate_librarian.py | 18 ++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/application/tests/librarian/dataset_test.py b/application/tests/librarian/dataset_test.py index 0dab83f17..d7ee3284d 100644 --- a/application/tests/librarian/dataset_test.py +++ b/application/tests/librarian/dataset_test.py @@ -101,6 +101,47 @@ def test_ids_are_unique(self): self.assertEqual(len(ids), len(set(ids))) +class TestLoadDatasetRejectsDuplicateIds(unittest.TestCase): + """The harness keys its shared retrieval audits by row id. + + Two rows sharing an id would collapse in that dict and one row would be + scored against the other's shortlist, so ``load_dataset`` must refuse the + file rather than let a wrong number through. The schema only requires an id + to be non-empty, which is why this is checked at load time. + """ + + def _load_harness(self): + import importlib.util + + path = os.path.join(_REPO_ROOT, "scripts", "evaluate_librarian.py") + spec = importlib.util.spec_from_file_location("evaluate_librarian", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_committed_dataset_loads(self): + harness = self._load_harness() + self.assertEqual(len(harness.load_dataset(_DATASET)), len(_load(_DATASET))) + + def test_duplicate_id_is_rejected(self): + import tempfile + + harness = self._load_harness() + rows = _load(_DATASET)[:2] + rows[1] = dict(rows[1], id=rows[0]["id"]) # force a collision + with tempfile.NamedTemporaryFile( + "w", suffix=".json", delete=False, encoding="utf-8" + ) as fh: + json.dump(rows, fh) + tmp = fh.name + try: + with self.assertRaises(ValueError) as ctx: + harness.load_dataset(tmp) + self.assertIn(rows[0]["id"], str(ctx.exception)) + finally: + os.unlink(tmp) + + class TestDatasetDeterminism(unittest.TestCase): """The committed JSON must re-derive identically from the DB.""" diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 54bbf5529..53628b87e 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -48,9 +48,25 @@ def load_dataset(path: str) -> List[GoldenDatasetRow]: + """Load and validate the golden set, rejecting duplicate row ids. + + The live reports key their shared audits by ``row.id``, so two rows sharing an + id would collapse in that dict and one row would be scored against the other's + shortlist. The schema only requires an id to be non-empty, so uniqueness is + enforced here rather than discovered as a wrong number downstream. + """ with open(path, encoding="utf-8") as fh: raw = json.load(fh) - return [GoldenDatasetRow.model_validate(row) for row in raw] + rows = [GoldenDatasetRow.model_validate(row) for row in raw] + duplicates = sorted( + row_id for row_id, n in Counter(r.id for r in rows).items() if n > 1 + ) + if duplicates: + raise ValueError( + f"golden dataset {path} has duplicate row ids: {', '.join(duplicates)}; " + "ids key the shared retrieval audits and must be unique" + ) + return rows def queue_row_from_golden(row: GoldenDatasetRow) -> dict: From 90992f5f8152da640e1416dc0450a01dc81b698a Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Tue, 4 Aug 2026 23:17:16 +0530 Subject: [PATCH 10/10] week_6: describe the C.4 live report accurately, drop an unused binding Two follow-ups from the bot review of the last push, neither behavioural: - The live-path descriptions still predated C.4. The module docstring claimed the semantic path was stubbed, _build_live_pipeline named only recall and calibration as its consumers, --use_live_embeddings help listed only recall and top-1, and the offline message omitted the decision report. All four now say what the run actually does, including that C.3 is the one live report that sets the exit status (a failed or skipped gate returns nonzero) while C.4 is informational until SafetyGuard and tau tuning land. - evaluate_harness_test bound the calibration status it never asserted (Ruff RUF059). Bound to _status: the test is about the pipeline not being re-run and the scaler coming back, and the gate outcome on stub logits is not a meaningful assertion. ruff check is clean on both files. 136 librarian tests pass; the hermetic harness run still exits 0. --- .../tests/librarian/evaluate_harness_test.py | 2 +- scripts/evaluate_librarian.py | 41 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index b321e2f5e..b9b840fe1 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -120,7 +120,7 @@ def test_pipeline_runs_once_per_row_not_once_per_report(self) -> None: # Three reports read the same audits; none of them may re-run the pipeline. harness.report_retrieval_recall(rows, audits, 10, 5) - status, scaler = harness.report_calibration(rows, audits) + _status, scaler = harness.report_calibration(rows, audits) self.assertIsNotNone(scaler) harness.report_decision_accuracy(rows, audits, scaler, 0.80) self.assertEqual(pipe.retrieve_calls, 2) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 53628b87e..66ee69b1f 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -"""Module C regression harness — Week 2: C.0 deterministic input boundary. +"""Module C regression harness — C.0 through C.4 over the golden set. -On top of the W1 skeleton (golden dataset + scorer + TRACT hub-firewall), -the harness now runs every golden row through the C.0 boundary: +On top of the W1 skeleton (golden dataset + scorer + TRACT hub-firewall), every +golden row runs through the C.0 boundary, and these two reports are offline: 1. SectionValidator — each row is adapted to a synthetic knowledge_queue row and must validate into an internal ``Section``; the harness prints the @@ -10,8 +10,19 @@ 2. ExplicitLinkResolver — sections citing a CRE id resolve deterministically (no ML); the explicit slice is gated at 100% correctness. -The semantic path (retriever W3, cross-encoder W4) is still stubbed: rows -without an explicit reference yield no predictions. +The semantic path needs ``--use_live_embeddings``, because there is no honest +offline value: the candidate pool must be the real CRE-node vectors, and seeding +it from golden text is the leakage the hub firewall exists to strip. Under that +flag the run retrieves and reranks each row once (``live_audits``) and three +reports share those shortlists: + +3. C.1 retrieval recall@k and C.2 rerank top-1 over the positive slice. +4. C.3 temperature calibration — fits one ``T`` and gates on ECE < 0.10. This is + the only live report that sets the exit status: a failed *or* skipped gate + returns nonzero, so a live run cannot pass without calibration having run. +5. C.4 decision accuracy — thresholds that same fitted ``T`` through ``decide()``. + Informational only, since the SafetyGuard flags are not wired until W8 and + tuning tau is the W7 experiment. """ import argparse @@ -124,9 +135,9 @@ def _build_live_pipeline( Live deps are imported lazily so the offline harness needs neither a DB, an embedding model, nor the cross-encoder stack. Called once per run from - ``main`` and shared by every live report (recall/top-1 and calibration), so - the heavy hub + model load happens a single time and the id-space translation - stays in one place. + ``main`` and shared by every live report (recall/top-1, the C.3 ECE gate, and + the C.4 decision accuracy), so the heavy hub + model load happens a single + time and the id-space translation stays in one place. """ from application.cmd.cre_main import db_connect from application.defs import cre_defs @@ -432,8 +443,10 @@ def main(argv: List[str]) -> int: parser.add_argument( "--use_live_embeddings", action="store_true", - help="connect to the OpenCRE DB + embedding model and measure the live " - "C.1 retrieval recall@k and C.2 rerank top-1 over the positive slice " + help="connect to the OpenCRE DB + embedding model and run every live " + "report: C.1 retrieval recall@k and C.2 rerank top-1 over the positive " + "slice, the C.3 ECE gate (which sets a nonzero exit status when it fails " + "or cannot run), and the informational C.4 decision accuracy " "(needs an LLM + populated DB)", ) parser.add_argument( @@ -533,10 +546,10 @@ def main(argv: List[str]) -> int: print("decision (C.4): skipped — calibration produced no fitted T") else: print( - "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3): " - "wired; recall@k, rerank top-1, and the ECE gate need " - "--use_live_embeddings (no CRE vectors offline — seeding from golden " - "text would be leakage)" + "semantic pipeline (C.1 retrieve + C.2 rerank) + calibration (C.3) + " + "decision (C.4): wired; recall@k, rerank top-1, the ECE gate, and the " + "decision accuracy all need --use_live_embeddings (no CRE vectors " + "offline — seeding from golden text would be leakage)" ) print(f"correct overall (semantic path still stubbed): {correct}/{len(rows)}") return calib_status