diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de08d4934..d45a7594d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,7 +353,8 @@ jobs: python -m pytest -q \ MonteCarloMarginalizeCode/Code/test/test_av_empty_live_volume.py \ MonteCarloMarginalizeCode/Code/test/test_l0_rescue_seed.py \ - MonteCarloMarginalizeCode/Code/test/test_portfolio_fairdraw_backend.py + MonteCarloMarginalizeCode/Code/test/test_portfolio_fairdraw_backend.py \ + MonteCarloMarginalizeCode/Code/test/test_tail_deficit_gate.py - name: Run test scripts run: | . .travis/test-coord.sh diff --git a/MonteCarloMarginalizeCode/Code/bin/util_CIPCompositionReweightWrapper.sh b/MonteCarloMarginalizeCode/Code/bin/util_CIPCompositionReweightWrapper.sh new file mode 100755 index 000000000..0aa2c6c2f --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/util_CIPCompositionReweightWrapper.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# +# util_CIPCompositionReweightWrapper.sh -- GATED drop-in CIP wrapper (opt in from +# util_RIFT_pseudo_pipe.py via --internal-cip-composition-reweight, or pass directly to +# create_event_parameter_pipeline_BasicIteration --cip-exe). Default pipeline behaviour is +# untouched: nothing uses this wrapper unless explicitly requested. +# +# WHAT IT DOES (detect -> repair, two CIP passes): +# pass 1 run the REAL CIP on the ORIGINAL training set, with the sample export bumped to +# >= 20000 and redirected to temp files (the real output paths are never touched +# by pass 1); +# gate util_CIPTailDeficitGate.py on (training set, pass-1 posterior): FIRE only on a +# SEVERE tail deficit (R < 0.32, calibrated in this exact fresh-CIP channel) AND +# only when the implied tail mass is resolvable (validity floor >= 50/n_post -- +# MANDATORY, the gate tool has no bypass); +# final FIRE -> util_CompositionReweightNet.py thins the training set and the final +# CIP runs on the thinned set (original argv otherwise verbatim); +# NO-FIRE / ABSTAIN / any failure -> the final CIP runs on the ORIGINAL argv +# verbatim, and the decision (with R and which condition decided it) +# is logged LOUDLY on stderr -- a no-op is always visible in the log. +# +# COST: CIP_REWEIGHT_GATE_REPS extra CIP passes per invocation when enabled (default 5). +# CIP is the cheap CPU stage. Why N>1: single-rep R noise is shot-dominated (measured +# sigma ~0.02-0.03 at 20k samples on mid-band events) against a healthy-control margin of +# ~0.06 to the threshold; deciding on the mean of N reps shrinks sigma by 1/sqrt(N). +# +# SCOPE (measured; details in the gate tool's docstring): repairs SEVERE deficits only +# (~half the affected low-mass events); mild deficit and healthy width are not separable in +# the mid-R band and are left alone, loudly. Safety is the strongly supported side (zero +# false fires in-sample and on known-truth healthy-narrow benchmarks). +# +# FAIL-SAFE THROUGHOUT: any tool error, unreadable input, or degenerate case -> the final +# CIP runs on the original argv unchanged, with a loud warning. The pipeline never breaks. +# +# Test hooks: CIP_REWEIGHT_REAL_CIP= substitutes the CIP executable; +# CIP_REWEIGHT_CONVERT= substitutes convert_output_format_ile2inference. + +set -u + +TOOL=$(command -v util_CompositionReweightNet.py || true) +GATE=$(command -v util_CIPTailDeficitGate.py || true) +CONVERT="${CIP_REWEIGHT_CONVERT:-$(command -v convert_output_format_ile2inference || true)}" +CIP_EXE="${CIP_REWEIGHT_REAL_CIP:-$(command -v util_ConstructIntrinsicPosterior_GenericCoordinates.py || true)}" +GATE_MIN_EXPORT=20000 +GATE_REPS="${CIP_REWEIGHT_GATE_REPS:-5}" +[[ "$GATE_REPS" =~ ^[0-9]+$ ]] && ((GATE_REPS >= 1)) || GATE_REPS=5 +TAGP="util_CIPCompositionReweightWrapper" + +if [[ -z "$CIP_EXE" ]]; then + echo "$TAGP: FATAL: real CIP not on PATH" >&2 + exit 1 +fi + +loud() { + echo "**************************************************************************" >&2 + echo "$TAGP: $1" >&2 + echo "**************************************************************************" >&2 +} + +# --- parse argv: training file, output-sample / integral paths, export count --- +args=("$@") +n=${#args[@]} +fname=""; out_samples=""; n_output="" +for ((i = 0; i < n; i++)); do + a="${args[$i]}" + case "$a" in + --fname=*) fname="${a#--fname=}" ;; + --fname) ((i + 1 < n)) && fname="${args[$((i + 1))]}" ;; + --fname-output-samples=*) out_samples="${a#--fname-output-samples=}" ;; + --fname-output-samples) ((i + 1 < n)) && out_samples="${args[$((i + 1))]}" ;; + --n-output-samples=*) n_output="${a#--n-output-samples=}" ;; + --n-output-samples) ((i + 1 < n)) && n_output="${args[$((i + 1))]}" ;; + esac +done + +run_original() { + # single exit path for every no-op / fail-safe branch: original argv, verbatim + exec "$CIP_EXE" "${args[@]}" +} + +if [[ -z "$fname" || -z "$out_samples" || -z "$TOOL" || -z "$GATE" || -z "$CONVERT" ]]; then + missing="" + [[ -z "$fname" ]] && missing="$missing --fname-not-in-argv" + [[ -z "$out_samples" ]] && missing="$missing --fname-output-samples-not-in-argv" + [[ -z "$TOOL" ]] && missing="$missing util_CompositionReweightNet.py-not-on-PATH" + [[ -z "$GATE" ]] && missing="$missing util_CIPTailDeficitGate.py-not-on-PATH" + [[ -z "$CONVERT" ]] && missing="$missing converter-not-on-PATH" + loud "cannot gate ($missing); running CIP UNCHANGED on the original training set" + run_original +fi + +dir=$(dirname "$fname") +tag="$(date +%Y%m%d%H%M%S)_p$$" +gatejson="$dir/gate_decision_${tag}.json" +comp="$dir/all_comp_${tag}.net" +cleanup() { rm -f "$dir/gatepass1_${tag}"_r*.xml.gz "$dir/gatepass1_${tag}"_r*_intg* \ + "$dir/gatepass1_${tag}"_r*.dat "$comp"; } +trap cleanup EXIT + +# --- detect passes: original training set, export bumped, temp outputs, GATE_REPS times --- +p1dats=() +for ((rep = 1; rep <= GATE_REPS; rep++)); do + p1base="$dir/gatepass1_${tag}_r${rep}" + p1=() + skip=0 + for ((i = 0; i < n; i++)); do + if ((skip)); then skip=0; continue; fi + a="${args[$i]}" + case "$a" in + --fname-output-samples=*) p1+=("--fname-output-samples=$p1base") ;; + --fname-output-samples) p1+=("--fname-output-samples" "$p1base"); skip=1 ;; + --fname-output-integral=*) p1+=("--fname-output-integral=${p1base}_intg") ;; + --fname-output-integral) p1+=("--fname-output-integral" "${p1base}_intg"); skip=1 ;; + --n-output-samples=*) nv="${a#--n-output-samples=}" + ((nv < GATE_MIN_EXPORT)) && nv=$GATE_MIN_EXPORT + p1+=("--n-output-samples=$nv") ;; + --n-output-samples) nv="${args[$((i + 1))]}" + ((nv < GATE_MIN_EXPORT)) && nv=$GATE_MIN_EXPORT + p1+=("--n-output-samples" "$nv"); skip=1 ;; + *) p1+=("$a") ;; + esac + done + echo "$TAGP: detect pass $rep/$GATE_REPS (gate-quality CIP on the ORIGINAL training set)" >&2 + if ! "$CIP_EXE" "${p1[@]}" 1>&2 || [[ ! -s "$p1base.xml.gz" ]]; then + echo "$TAGP: WARNING: detect pass $rep FAILED; continuing with the other reps" >&2 + continue + fi + if ! "$CONVERT" "$p1base.xml.gz" > "$p1base.dat" 2>/dev/null || [[ ! -s "$p1base.dat" ]]; then + echo "$TAGP: WARNING: detect pass $rep conversion FAILED; continuing" >&2 + continue + fi + p1dats+=("$p1base.dat") +done +if ((${#p1dats[@]} == 0)); then + loud "ALL detect passes FAILED; falling back: final CIP on the ORIGINAL training set" + run_original +fi +if ((${#p1dats[@]} < GATE_REPS)); then + echo "$TAGP: WARNING: deciding on ${#p1dats[@]}/$GATE_REPS detect reps (noise is larger)" >&2 +fi + +# --- gate: one decision on the MEAN R over the detect reps --- +decision_line=$("$GATE" "$fname" "${p1dats[@]}" --json "$gatejson" | tail -1) +if [[ "$decision_line" != GATE\ DECISION=* ]]; then + loud "gate tool FAILED to evaluate; falling back: final CIP on the ORIGINAL training set" + run_original +fi +echo "$TAGP: $decision_line (record: $gatejson)" >&2 + +use="$fname" +case "$decision_line" in + "GATE DECISION=FIRE"*) + echo "$TAGP: SEVERE tail deficit detected -> composition-reweight thinning" >&2 + if "$TOOL" "$fname" --output "$comp" --seed 0 --stats-json "$dir/comp_stats_${tag}.json" 1>&2 \ + && [[ -s "$comp" ]]; then + use="$comp" + echo "$TAGP: final CIP will train on $comp (original preserved at $fname)" >&2 + else + loud "reweight tool FAILED after a FIRE; falling back to the ORIGINAL training set" + fi + ;; + "GATE DECISION=NO-FIRE"*) + echo "$TAGP: NO-OP by threshold: no severe deficit ($decision_line)" >&2 + echo "$TAGP: (mild deficits above the threshold are NOT separable from healthy" >&2 + echo "$TAGP: width and are deliberately left alone -- see util_CIPTailDeficitGate.py)" >&2 + ;; + "GATE DECISION=ABSTAIN-FLOOR"*) + echo "$TAGP: NO-OP by validity floor: implied tail mass unresolvable at this sample" >&2 + echo "$TAGP: size ($decision_line); raising --n-output-samples would sharpen the gate" >&2 + ;; +esac + +# --- final CIP: original argv verbatim, fname swapped only on a successful FIRE --- +out=() +skip=0 +for ((i = 0; i < n; i++)); do + if ((skip)); then skip=0; continue; fi + a="${args[$i]}" + if [[ "$a" == --fname=* ]]; then + out+=("--fname=$use") + elif [[ "$a" == "--fname" ]] && ((i + 1 < n)); then + out+=("--fname" "$use") + skip=1 + else + out+=("$a") + fi +done + +"$CIP_EXE" "${out[@]}" +exit $? diff --git a/MonteCarloMarginalizeCode/Code/bin/util_CIPTailDeficitGate.py b/MonteCarloMarginalizeCode/Code/bin/util_CIPTailDeficitGate.py new file mode 100755 index 000000000..663186200 --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/util_CIPTailDeficitGate.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# +# util_CIPTailDeficitGate.py +# +# SEVERE-deficit gate for the CIP composition reweight (used by +# util_CIPCompositionReweightWrapper.sh). From RIFT's own products alone -- a CIP training +# set (composite/all.net) and a CIP posterior produced FROM it -- it decides whether the +# posterior returns materially less transverse-tail mass than the training likelihoods +# license: +# +# implied tail mass = sum over chi1_perp bins of prior_vol(bin ∩ tail) x _bin, +# _bin = mean exp(lnL - peak) over the bin's ILE rows; +# prior volume by MC of the analytic spin prior +# (a1 ~ U(0, chi-max), isotropic tilt), fixed seed +# delivered tail mass = fraction of the posterior's samples in the tail +# tail boundary = the training set's own chi1_perp q80 (self-quantile) +# R = delivered / implied +# +# FIRE (severe deficit) <=> implied >= floor_counts / n_post AND R < threshold +# ABSTAIN-FLOOR <=> implied < floor_counts / n_post +# NO-FIRE otherwise +# +# THE VALIDITY FLOOR IS MANDATORY AND CANNOT BE BYPASSED. When the implied tail mass is +# below what n_post samples can resolve, a perfectly correct posterior is expected to put +# ~zero samples there and R degenerates to 0 -- maximum apparent deficit exactly when the +# detector can resolve nothing, the worst possible failure direction. On known-truth +# healthy-narrow benchmarks (toybench T5/T5b/T5c, transverse confinement 8/15/20 nats) every +# chain read R = 0 and only the floor prevented a false FIRE. There is deliberately no +# option to disable it; decide() applies it before the threshold is ever consulted. +# +# THE THRESHOLD IS CHANNEL-CALIBRATED. R depends on which posterior supplies "delivered": +# a fresh single-CIP posterior (what this gate's pass-1 measures) reads mid-band healthy +# events LOWER than the production consolidated posterior by up to ~0.07, while severe- +# deficit events read the same in both. The population threshold 0.42 (production-channel +# gap 0.379-0.457 over 102 events) therefore does NOT transfer: in the deployment channel +# the healthy-control floor is 0.386 and the severe-deficit ceiling is 0.270, giving +# THRESHOLD = 0.32 (geometric midpoint; margins ~1.2x each side, outside the training-row +# bootstrap bands of the edge events). This was caught by the regression fixture, not by +# design -- see test/test_tail_deficit_gate.py. +# +# SCOPE (measured, 2026-08-19 study record: results_triage/R_TOYBENCH_VALIDATION_2026-08-19.md +# and R_POPULATION_CALIBRATION_2026-08-19.md of rift_transverse_highSNR_study): +# * This detects SEVERE deficits only (production low-mass class, chi1_perp width ratios +# ~0.62-0.69 vs reference; deployment-channel R <= 0.270). Mild deficits and healthy- +# width posteriors are NOT separable above the threshold: out-of-sample, truth-deficient +# toybench T3 chains (R 0.532-0.634) abut truth-healthy T2 chains (R 0.633-0.673), and +# two borderline production events (dead-tail S241011k at 0.358, ratio-0.794 S240512r) +# sit just above the deployment gap. Expect roughly half the affected low-mass events +# to be repaired and the mild rest to be (loudly) left alone. +# * Safety is the strongly supported side: zero false fires in 77 in-sample healthy events +# (95% bound: false-fire rate <= 3.8%) and zero on known-truth healthy-narrow toys. +# The fire side is in-sample-validated only (12 production events at 100%, 95% bound on +# the miss rate 22.1%, plus the causal 13-event paired-CIP repair). +# +# Exit codes: 0 = evaluated (decision on the LAST stdout line, machine-readable); +# 2 = could not evaluate (caller must fail safe: proceed with the original data). + +import argparse +import json +import sys + +import numpy as np + +THRESHOLD = 0.32 # calibrated IN THE DEPLOYMENT CHANNEL (fresh-CIP posterior; see SCOPE) +FLOOR_COUNTS = 50.0 # validity floor: implied must be >= FLOOR_COUNTS / n_post (pre-registered) +CHI_MAX = 0.99 +EDGES = np.array([0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.99]) +MIN_BIN = 20 +SELFQ = 0.80 +LNL_COL = 9 +N_PRIOR = 4_000_000 +PRIOR_SEED = 11 +MIN_POST = 100 # a posterior this small cannot support any decision -> error (exit 2) + + +def decide(implied, R, n_post, threshold=THRESHOLD, floor_counts=FLOOR_COUNTS): + """The one and only decision function. The validity floor is applied FIRST and + unconditionally; no caller can reach the threshold comparison without passing it.""" + floor = floor_counts / float(n_post) + if not np.isfinite(implied) or implied < floor: + return "ABSTAIN-FLOOR", floor + decision = "FIRE" if (np.isfinite(R) and R < threshold) else "NO-FIRE" + # structural guarantee, not a debug aid: a FIRE below the floor is a contract violation + assert decision != "FIRE" or implied >= floor + return decision, floor + + +def load_posterior_cp(path): + hdr = open(path).readline().lstrip('#').split() + d = np.genfromtxt(path, names=hdr, skip_header=1, invalid_raise=False) + cp = np.hypot(np.asarray(d['a1x'], float), np.asarray(d['a1y'], float)) + return cp[np.isfinite(cp)] + + +def compute_R(train_path, post_paths, chi_max=CHI_MAX): + if isinstance(post_paths, str): + post_paths = [post_paths] + a = np.loadtxt(train_path, ndmin=2) + if a.shape[1] <= LNL_COL: + raise ValueError(f"training file has {a.shape[1]} cols, need lnL at 0-based col {LNL_COL}") + lnl = a[:, LNL_COL] + cp = np.hypot(a[:, 3], a[:, 4]) + ok = np.isfinite(lnl) & np.isfinite(cp) + lnl, cp = lnl[ok], cp[ok] + if len(lnl) < 1000: + raise ValueError(f"only {len(lnl)} usable training rows") + peak = lnl.max() + b = float(np.quantile(cp, SELFQ)) + rng = np.random.default_rng(PRIOR_SEED) + am = rng.uniform(0, chi_max, N_PRIOR) + c_ = rng.uniform(-1, 1, N_PRIOR) + cpp = am * np.sqrt(1 - c_ ** 2) + imp_tail = imp_tot = 0.0 + for lo, hi in zip(EDGES[:-1], EDGES[1:]): + m = (cp >= lo) & (cp < hi) + if m.sum() < MIN_BIN: + continue + meanL = float(np.exp(lnl[m] - peak).mean()) + ib = (cpp >= lo) & (cpp < hi) + imp_tot += float(ib.mean()) * meanL + imp_tail += float((ib & (cpp > b)).mean()) * meanL + implied = imp_tail / imp_tot if imp_tot > 0 else float("nan") + dl, np_list = [], [] + for pp in post_paths: + cpo = load_posterior_cp(pp) + if len(cpo) < MIN_POST: + raise ValueError(f"{pp}: only {len(cpo)} posterior samples (< {MIN_POST})") + dl.append(float(np.mean(cpo > b))) + np_list.append(int(len(cpo))) + delivered = float(np.mean(dl)) # equal-weight mean over detect reps + R = delivered / implied if implied > 0 else float("nan") + R_reps = [d / implied if implied > 0 else float("nan") for d in dl] + # floor uses the MIN per-rep sample count: conservative, and unchanged for N=1 + return dict(boundary=b, implied=implied, delivered=delivered, R=R, + n_post=int(min(np_list)), n_train=int(len(lnl)), + n_reps=len(dl), R_per_rep=R_reps, + R_rep_sd=float(np.std(R_reps, ddof=1)) if len(dl) > 1 else float("nan")) + + +def main(): + ap = argparse.ArgumentParser( + description="Severe-tail-deficit gate from RIFT's own products (training set + its CIP " + "posterior). Detects SEVERE transverse deficits only; mild deficit and " + "healthy width are not separable in the mid-R band (see module docstring " + "for the measured scope and bounds). The sample-resolution validity floor " + "is mandatory and has no disable option: without it the detector returns " + "maximum deficit exactly where it can resolve nothing.") + ap.add_argument("training", help="composite/all.net (col 9 = lnL)") + ap.add_argument("posterior", nargs="+", + help="CIP posterior samples .dat produced from that training set (header " + "with a1x a1y columns). Give SEVERAL independent detect-pass " + "posteriors to decide on the MEAN R: single-rep R noise is " + "shot-dominated (~0.02 at 20k samples on mid-band events, measured), " + "so N reps shrink it by 1/sqrt(N).") + ap.add_argument("--threshold", type=float, default=THRESHOLD, + help="R below this (and above the validity floor) fires; default %(default)s, " + "calibrated in the DEPLOYMENT channel (fresh-CIP posterior); changing " + "it voids the recorded validation") + ap.add_argument("--floor-counts", type=float, default=FLOOR_COUNTS, + help="validity floor = this / n_post expected tail samples; default " + "%(default)s (pre-registered). May be raised (stricter); values below " + "1 are refused -- the floor cannot be turned off.") + ap.add_argument("--chi-max", type=float, default=CHI_MAX) + ap.add_argument("--json", default=None, help="write full evaluation record here") + args = ap.parse_args() + if args.floor_counts < 1.0: + sys.stderr.write("util_CIPTailDeficitGate: --floor-counts < 1 refused: the validity " + "floor is mandatory and cannot be effectively disabled\n") + sys.exit(2) + try: + r = compute_R(args.training, args.posterior, chi_max=args.chi_max) + if r["n_reps"] > 1: + sys.stderr.write(f"util_CIPTailDeficitGate: {r['n_reps']} detect reps: R per rep = " + + " ".join(f"{x:.4f}" for x in r["R_per_rep"]) + + f" (sd {r['R_rep_sd']:.4f}; deciding on the mean)\n") + except Exception as e: + sys.stderr.write(f"util_CIPTailDeficitGate: cannot evaluate: {type(e).__name__}: {e}\n") + sys.exit(2) + decision, floor = decide(r["implied"], r["R"], r["n_post"], + threshold=args.threshold, floor_counts=args.floor_counts) + r.update(decision=decision, floor=floor, threshold=args.threshold, + floor_counts=args.floor_counts) + if args.json: + try: + json.dump(r, open(args.json, "w"), indent=1, default=float) + except OSError as e: + sys.stderr.write(f"util_CIPTailDeficitGate: cannot write json: {e}\n") + reason = {"FIRE": f"R={r['R']:.4f} < threshold={args.threshold}", + "NO-FIRE": f"R={r['R']:.4f} >= threshold={args.threshold}", + "ABSTAIN-FLOOR": f"implied={r['implied']:.3e} < floor={floor:.3e} " + f"({args.floor_counts:g}/{r['n_post']} samples): tail unresolvable" + }[decision] + sys.stderr.write(f"util_CIPTailDeficitGate: {decision}: {reason} " + f"(boundary cp>{r['boundary']:.3f}, delivered={r['delivered']:.4e}, " + f"implied={r['implied']:.4e}, n_post={r['n_post']})\n") + print(f"GATE DECISION={decision} R={r['R']:.6f} implied={r['implied']:.6e} " + f"delivered={r['delivered']:.6e} n_post={r['n_post']} floor={floor:.6e} " + f"threshold={args.threshold}") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/MonteCarloMarginalizeCode/Code/bin/util_CompositionReweightNet.py b/MonteCarloMarginalizeCode/Code/bin/util_CompositionReweightNet.py new file mode 100755 index 000000000..1702031ab --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/bin/util_CompositionReweightNet.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# +# util_CompositionReweightNet.py +# +# Composition-equalising THINNING of a CIP training set (composite/all.net: 13 whitespace +# cols; 1=m1, 2=m2, 3-5=s1xyz, 6-8=s2xyz, 9=lnL). In accumulated low-mass training sets the +# fraction of near-peak rows degrades with chi1_perp, so an RF fit -- a local average -- +# regresses the transverse tail toward its many low-lnL neighbours and the recovered +# chi1_perp/a1 posterior is too narrow. This tool thins the FAR-from-peak rows per +# chi1_perp bin so every bin's near-peak fraction matches the best bin's: +# - per bin, KEEP every near-peak row, thin the far rows at random (density reweighting; +# NEVER a global lnL truncation, NEVER duplication, NEVER invented points -- the lnL +# span and peak are preserved exactly); +# - SELF-QUANTILE definitions only (no external reference, no absolute thresholds): +# near-peak: lnL > lnL.max() - NAT (NAT = 5 nats below the set's own peak) +# tail/core boundary: chi1_perp > its own q80 +# - FAIL-SAFE: on any internal error or degenerate input (too few rows, flat lnL, dead +# tail, ...) it copies input -> output UNCHANGED, warns loudly, exits 0 -- it must never +# break the pipeline. --strict makes such failures fatal (bench use). +# +# Rows are written back as the ORIGINAL INPUT LINES (values untouched); only line SELECTION +# happens here, deterministically under --seed. Opt in from util_RIFT_pseudo_pipe.py via +# --internal-cip-composition-reweight (runs through util_CIPCompositionReweightWrapper.sh). + +import argparse +import json +import shutil +import sys + +import numpy as np + +NAT = 5.0 # near-peak window, nats below the set's own peak +SELFQ = 0.80 # self-quantile tail/core boundary +LNL_COL = 9 +MIN_ROWS = 1000 # below this, thinning statistics are meaningless -> fallback +MIN_BIN = 10 # bins with fewer rows are skipped +MIN_NEAR = 100 # too few near-peak rows overall -> fallback +MIN_BINS_USED = 2 # need at least two usable bins to equalise anything +# validated bin edges (default --nbins 10 keeps these) +VERIFY_EDGES = np.array([0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.99, 10.]) + + +def comp_stats(cp, lnl): + """Self-quantile composition statistic (fully self-contained definitions).""" + peak = lnl.max() + near = lnl > peak - NAT + q80 = float(np.quantile(cp, SELFQ)) + tail_s = cp > q80 + nf_tail_s = float(near[tail_s].mean()) if tail_s.sum() >= 50 else float("nan") + nf_core_s = float(near[~tail_s].mean()) if (~tail_s).sum() else float("nan") + comp_self = nf_tail_s / nf_core_s if nf_core_s and nf_core_s > 0 else float("nan") + m05 = cp > 0.5 + nf_gt05 = float(near[m05].mean()) if m05.sum() else float("nan") + return dict(n=int(len(lnl)), peak=float(peak), n_near=int(near.sum()), + near_frac=float(near.mean()), q80=q80, + nf_tail=nf_tail_s, nf_core=nf_core_s, comp_self=comp_self, + nearfrac_cp_gt_0p5=nf_gt05, + lnl_span=float(lnl.max() - lnl.min())) + + +def fallback(args, reason, stats): + sys.stderr.write("*" * 78 + "\n") + sys.stderr.write(f"util_CompositionReweightNet WARNING: {reason}\n") + if args.strict: + sys.stderr.write(" --strict: FATAL, no output written.\n") + sys.stderr.write("*" * 78 + "\n") + _write_stats(args, dict(stats, fallback=reason, strict_fatal=True)) + sys.exit(2) + sys.stderr.write(f" FALLING BACK: copying input -> output unchanged ({args.output})\n") + sys.stderr.write("*" * 78 + "\n") + shutil.copyfile(args.input, args.output) + _write_stats(args, dict(stats, fallback=reason, strict_fatal=False)) + sys.exit(0) + + +def _write_stats(args, d): + if not args.stats_json: + return + try: + with open(args.stats_json, "w") as f: + json.dump(d, f, indent=1, default=float) + except OSError as e: + sys.stderr.write(f"util_CompositionReweightNet WARNING: cannot write stats json: {e}\n") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("input", help="composite/all.net (13 whitespace cols, col 9 = lnL)") + ap.add_argument("--output", required=True, help="thinned output file") + ap.add_argument("--seed", type=int, default=0, help="rng seed for the deterministic far-row thinning") + ap.add_argument("--exploration-parameter", default="chi1_perp", + help="binning coordinate; only chi1_perp implemented") + ap.add_argument("--nbins", type=int, default=10, + help="number of chi1_perp bins; default 10 uses the validated " + "edges [0,.1,...,.8,.99,10]; any other value uses uniform bins over the data range") + ap.add_argument("--strict", action="store_true", + help="make degenerate input / internal errors fatal (bench use); default is fail-safe copy-through") + ap.add_argument("--stats-json", default=None, help="write before/after composition stats as json") + args = ap.parse_args() + + base_stats = dict(tool="util_CompositionReweightNet", input=args.input, output=args.output, + seed=args.seed, nat=NAT, self_quantile=SELFQ, + exploration_parameter=args.exploration_parameter, nbins=args.nbins, + definitions=dict( + near_peak="lnL > lnL.max() - 5.0 (self peak, NAT=5 nats)", + tail_core_boundary="chi1_perp > quantile(chi1_perp, 0.80) of the input itself", + comp="nearfrac(tail)/nearfrac(core)", + construction="per chi1_perp bin keep ALL near-peak rows; thin far rows so " + "per-bin near-peak fraction equals the best bin's (thinning only)")) + + try: + if args.exploration_parameter != "chi1_perp": + fallback(args, f"exploration parameter '{args.exploration_parameter}' not implemented", base_stats) + + # keep original lines so selected rows are written back verbatim + lines, rows = [], [] + with open(args.input) as f: + for ln in f: + if not ln.strip() or ln.lstrip().startswith("#"): + continue + lines.append(ln) + rows.append(ln.split()) + if len(rows) < MIN_ROWS: + fallback(args, f"too few rows ({len(rows)} < {MIN_ROWS}) for composition thinning", base_stats) + ncol = len(rows[0]) + if ncol <= LNL_COL or any(len(r) != ncol for r in rows): + fallback(args, "inconsistent or too-few columns; need lnL at 0-based col 9", base_stats) + a = np.asarray(rows, dtype=float) + lnl = a[:, LNL_COL] + cp = np.hypot(a[:, 3], a[:, 4]) # chi1_perp = |s1_xy| + if not (np.isfinite(lnl).all() and np.isfinite(cp).all()): + fallback(args, "non-finite lnL or spin components in input", base_stats) + + before = comp_stats(cp, lnl) + base_stats["rows_in"] = before["n"] + base_stats["before"] = before + peak = lnl.max() + near = lnl > peak - NAT + + if before["lnl_span"] <= NAT: + fallback(args, f"lnL span {before['lnl_span']:.3f} <= NAT={NAT}: every row is 'near-peak', " + "nothing to classify (flat/degenerate likelihoods)", base_stats) + if near.sum() < MIN_NEAR: + fallback(args, f"only {near.sum()} near-peak rows (< {MIN_NEAR})", base_stats) + + # --- the composition-equalising construction --- + if args.nbins == 10: + edges = VERIFY_EDGES + else: + if args.nbins < 2: + fallback(args, f"--nbins {args.nbins} < 2", base_stats) + hi = max(cp.max() * (1 + 1e-9), 1e-6) + edges = np.linspace(0.0, hi, args.nbins + 1) + edges[-1] = max(edges[-1], 10.0) # last bin catches everything + binfo = [] + for lo, hi in zip(edges[:-1], edges[1:]): + m = (cp >= lo) & (cp < hi) + if m.sum() < MIN_BIN: + continue + binfo.append((float(lo), float(hi), m, float(near[m].mean()))) + if len(binfo) < MIN_BINS_USED: + fallback(args, f"only {len(binfo)} usable chi1_perp bins (need >= {MIN_BINS_USED})", base_stats) + + # Equalise UP to the BEST bin: keep every near-peak point everywhere, thin the FAR + # points in the poorer bins until their near-peak fraction matches. Thinning only. + target = max(b[3] for b in binfo) + if not (0 < target <= 1): + fallback(args, f"degenerate target near-peak fraction {target}", base_stats) + rng = np.random.default_rng(args.seed) + keep = np.zeros(len(a), bool) + bins_out = [] + for lo, hi, m, f in binfo: + idx = np.flatnonzero(m) + nr = idx[near[idx]] + fr = idx[~near[idx]] + n_far_needed = int(len(nr) * (1 - target) / target) + keep[nr] = True + if len(fr): + keep[rng.choice(fr, min(len(fr), n_far_needed), replace=False)] = True + bins_out.append(dict(lo=lo, hi=hi, n_before=int(m.sum()), near=int(len(nr)), + nearfrac_before=f)) + + kept_idx = np.flatnonzero(keep) + after = comp_stats(cp[keep], lnl[keep]) + for b in bins_out: + m = (cp[keep] >= b["lo"]) & (cp[keep] < b["hi"]) + nn = near[keep][m] + b["n_after"] = int(m.sum()) + b["nearfrac_after"] = float(nn.mean()) if m.sum() else float("nan") + + # sanity: this is a thinning -- never grow, never lose the peak + assert len(kept_idx) <= len(a) and near[keep].sum() == near.sum() + + with open(args.output, "w") as f: + for i in kept_idx: # original file order, original bytes + f.write(lines[i]) + + base_stats.update(rows_out=int(len(kept_idx)), target_nearfrac=float(target), + edges=[float(e) for e in edges], after=after, + bins=bins_out, fallback=None) + _write_stats(args, base_stats) + + print(f"util_CompositionReweightNet: {len(a)} -> {len(kept_idx)} rows " + f"(target per-bin nearfrac {target:.4f})") + print(f" lnL span before/after: {before['lnl_span']:.4f} / {after['lnl_span']:.4f} " + f"(peak preserved: {before['peak']:.4f})") + print(f" comp (self-q80 tail/core) before/after: {before['comp_self']:.4f} / {after['comp_self']:.4f}") + print(f" nearfrac|cp>0.5 before/after: {before['nearfrac_cp_gt_0p5']:.4f} / {after['nearfrac_cp_gt_0p5']:.4f}") + print(f" {'bin':<14}{'n_in':>8}{'nf_in':>9}{'n_out':>8}{'nf_out':>9}") + for b in bins_out: + print(f" [{b['lo']:.2f},{b['hi']:.2f}){'':<3}{b['n_before']:>8d}{b['nearfrac_before']:>9.4f}" + f"{b['n_after']:>8d}{b['nearfrac_after']:>9.4f}") + except SystemExit: + raise + except Exception as e: + fallback(args, f"internal error: {type(e).__name__}: {e}", base_stats) + + +if __name__ == "__main__": + main() diff --git a/MonteCarloMarginalizeCode/Code/bin/util_RIFT_pseudo_pipe.py b/MonteCarloMarginalizeCode/Code/bin/util_RIFT_pseudo_pipe.py index 2dcf54291..2e98ab1fd 100755 --- a/MonteCarloMarginalizeCode/Code/bin/util_RIFT_pseudo_pipe.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_RIFT_pseudo_pipe.py @@ -438,6 +438,7 @@ def run_lisa_known_sky_surface(opts): parser.add_argument("--internal-ile-buffer-after-trigger",default=2,type=float,help="Provided to allow user to change time after trigger. NOT FULLY IMPLEMENTED") parser.add_argument("--internal-ile-request-disk",help="Use if you are transferring large files, or if you otherwise expect a lot of data ") parser.add_argument("--internal-cip-request-disk",help="Use if you are transferring large files, or if you otherwise expect a lot of data ") +parser.add_argument("--internal-cip-composition-reweight",action='store_true',help="Opt-in (default OFF): run every CIP through util_CIPCompositionReweightWrapper.sh -- a gated detect-then-repair for the low-mass transverse-spin width deficit. A first gate-quality CIP pass measures the tail-deficit ratio R (util_CIPTailDeficitGate.py); ONLY a SEVERE deficit (R<0.32 in the fresh-CIP measurement channel, above a mandatory sample-resolution validity floor) triggers composition-equalising thinning of the training set (util_CompositionReweightNet.py) for the final CIP; otherwise the final CIP runs unchanged and the no-op is logged with the R value. SCOPE, measured: repairs severe deficits only (~half the affected low-mass events); mild deficit vs healthy width is NOT separable in the mid-R band; safety is the strongly supported side (zero false fires across 77 in-sample healthy events, bound 3.8%, and known-truth healthy-narrow benchmarks); fire side in-sample-validated (12/12 events, miss-rate bound 22.1%, plus a causal 13-event paired-CIP repair). Costs one extra CIP per invocation. Fail-safe throughout. Conflicts with --internal-use-amr (both set --cip-exe).") parser.add_argument("--internal-general-request-disk",help="Use if you are transferring large files, or if you otherwise expect a lot of data. Specifically for things like calmarg/surrogate h5 files ") parser.add_argument("--internal-ile-request-memory",default=4096,type=int,help="ILE memory request in Mb. Only experts should change this.") parser.add_argument("--internal-ile-n-max",default=None,type=int,help="Set maximum number of evaluations each ILE worker uses. EXPERTS ONLY") @@ -2020,6 +2021,10 @@ def approx_supports_precession(approx_name): cmd += " --comov-distance-reweighting --comov-distance-reweighting-exe `which make_uni_comov_skymap.py` --convert-ascii2h5-exe `which convert_output_format_ascii2h5.py` " if opts.use_gauss_early: cmd += " --cip-exe-G `which util_ConstructIntrinsicPosterior_GaussianResampling.py ` " +if opts.internal_cip_composition_reweight: + if opts.internal_use_amr: + raise SystemExit("pseudo_pipe: --internal-cip-composition-reweight conflicts with --internal-use-amr (both set --cip-exe)") + cmd += " --cip-exe `which util_CIPCompositionReweightWrapper.sh` " if opts.internal_use_amr: print(" AMR prototype: Using hardcoded aligned-spin settings, assembling grid, requires coinc!") if _use_hpip_pp and opts.manual_initial_grid is None: diff --git a/MonteCarloMarginalizeCode/Code/test/fixtures_tail_deficit_gate.json b/MonteCarloMarginalizeCode/Code/test/fixtures_tail_deficit_gate.json new file mode 100644 index 000000000..eb3b77328 --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/test/fixtures_tail_deficit_gate.json @@ -0,0 +1,117 @@ +{ + "provenance": "Measured 2026-08-19; rift_transverse_highSNR_study: gate0_*.json (production all.net + baseline-rerun CIP posteriors, GATED_REWEIGHT_2026-08-19.md) and toybench_R_validation/toybench_R.json (known-truth toys, R_TOYBENCH_VALIDATION_2026-08-19.md). These pin the SHIPPED gate's decisions on the events that defined its validation: the three controls are the events the UNGATED fix drove to 1.543/1.175x the reference width; the T5 family must abstain VIA THE FLOOR (each has R < threshold, so the threshold alone would false-fire).", + "cases": [ + { + "case": "S241109bn", + "kind": "production exemplar (severe deficit)", + "implied": 0.1439840092181753, + "R": 0.2696837088415274, + "n_post": 58537, + "expect": "FIRE" + }, + { + "case": "S240413p", + "kind": "production exemplar (severe deficit)", + "implied": 0.18010047217569025, + "R": 0.11220398813147393, + "n_post": 45675, + "expect": "FIRE" + }, + { + "case": "S241102br", + "kind": "production exemplar (severe deficit)", + "implied": 0.20096137001114464, + "R": 0.07031232313953607, + "n_post": 43595, + "expect": "FIRE" + }, + { + "case": "S240629by", + "kind": "production exemplar (severe deficit)", + "implied": 0.15577056176952697, + "R": 0.16373531256694002, + "n_post": 44344, + "expect": "FIRE" + }, + { + "case": "S241225c", + "kind": "production exemplar (severe deficit)", + "implied": 0.30436345947532645, + "R": 0.1706208724185756, + "n_post": 49566, + "expect": "FIRE" + }, + { + "case": "S240921cw", + "kind": "production bad-comp healthy-width control", + "implied": 0.053384219558466804, + "R": 0.5313680128937592, + "n_post": 60000, + "expect": "NO-FIRE" + }, + { + "case": "S240930aa", + "kind": "production bad-comp healthy-width control", + "implied": 0.03919360486950652, + "R": 0.3861174133820846, + "n_post": 60000, + "expect": "NO-FIRE" + }, + { + "case": "S240621dy", + "kind": "production bad-comp healthy-width control", + "implied": 0.13790937987839455, + "R": 0.43228870087905064, + "n_post": 60000, + "expect": "NO-FIRE" + }, + { + "case": "T5/shipped_s2", + "kind": "toybench truth-healthy-narrow", + "implied": 0.00011633869261157893, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + }, + { + "case": "T5/shipped_s4", + "kind": "toybench truth-healthy-narrow", + "implied": 0.00018174932964659058, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + }, + { + "case": "T5b/shipped_s1", + "kind": "toybench truth-healthy-narrow", + "implied": 5.444486851012013e-06, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + }, + { + "case": "T5b/shipped_s3", + "kind": "toybench truth-healthy-narrow", + "implied": 1.5719962998183644e-05, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + }, + { + "case": "T5b/shipped_s5", + "kind": "toybench truth-healthy-narrow", + "implied": 8.994319821827222e-05, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + }, + { + "case": "T5c/shipped_s4", + "kind": "toybench truth-healthy-narrow", + "implied": 4.2942846827743646e-05, + "R": 0.0, + "n_post": 20000, + "expect": "ABSTAIN-FLOOR" + } + ] +} \ No newline at end of file diff --git a/MonteCarloMarginalizeCode/Code/test/test_tail_deficit_gate.py b/MonteCarloMarginalizeCode/Code/test/test_tail_deficit_gate.py new file mode 100644 index 000000000..4462ee5fd --- /dev/null +++ b/MonteCarloMarginalizeCode/Code/test/test_tail_deficit_gate.py @@ -0,0 +1,225 @@ +"""Regression suite for the CIP composition-reweight gate (util_CIPTailDeficitGate.py and +its wrapper). Named in .github/workflows/ci.yml -- a test file is not in CI otherwise. + +Three layers: + 1. Measured-value regressions (fixtures_tail_deficit_gate.json): the SHIPPED gate's + decisions on the events that defined its validation. The three production controls + are the events the UNGATED fix drove to 1.543/1.175x the reference width -- they are + the permanent guard against that regression. The T5-family toys must abstain VIA THE + VALIDITY FLOOR, not the threshold (asserted: each has R < threshold). + 2. The validity floor is unbypassable: property scan over decide(), and the CLI refuses + any attempt to weaken it below one expected count. + 3. End-to-end CLI on synthetic data: severe deficit fires; healthy does not; unresolvable + tail abstains; the wrapper's fname-rebuild contract stays intact. +""" +import importlib.util +import json +import os +import subprocess +import sys + +import numpy as np +import pytest + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_BIN = os.path.join(_HERE, "..", "bin") +_GATE = os.path.abspath(os.path.join(_BIN, "util_CIPTailDeficitGate.py")) +_FIX = os.path.join(_HERE, "fixtures_tail_deficit_gate.json") + +_spec = importlib.util.spec_from_file_location("tail_deficit_gate", _GATE) +gate = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(gate) + +FIXTURES = json.load(open(_FIX))["cases"] + + +# --------------------------------------------------------------------------- layer 1 +@pytest.mark.parametrize("case", FIXTURES, ids=[c["case"] for c in FIXTURES]) +def test_shipped_gate_decision_on_measured_events(case): + """The shipped gate (default threshold + floor) must reproduce the validated decision + on every event that defined its validation.""" + decision, floor = gate.decide(case["implied"], case["R"], case["n_post"]) + assert decision == case["expect"], ( + f"{case['case']} ({case['kind']}): shipped gate says {decision}, " + f"validated decision is {case['expect']}") + + +def test_controls_are_noops_and_exemplars_fire(): + """Group-level restatement, so a failure names the physics not just one event.""" + by = {c["case"]: gate.decide(c["implied"], c["R"], c["n_post"])[0] for c in FIXTURES} + for ev in ("S240921cw", "S240930aa", "S240621dy"): + assert by[ev] == "NO-FIRE", f"control {ev} must be a NO-OP (ungated fix broke it)" + for ev in ("S241109bn", "S240413p", "S241102br", "S240629by", "S241225c"): + assert by[ev] == "FIRE", f"exemplar {ev} must fire (severe deficit)" + + +def test_t5_family_abstains_via_floor_not_threshold(): + """The reason matters: every T5-family chain has R < threshold, so the THRESHOLD alone + would false-fire on known-truth healthy-narrow cases; only the floor prevents it.""" + t5 = [c for c in FIXTURES if c["expect"] == "ABSTAIN-FLOOR"] + assert len(t5) >= 6 + for c in t5: + assert c["R"] < gate.THRESHOLD, f"{c['case']}: fixture no longer exercises the floor" + decision, floor = gate.decide(c["implied"], c["R"], c["n_post"]) + assert decision == "ABSTAIN-FLOOR" + assert c["implied"] < floor, f"{c['case']}: abstain must be BECAUSE of the floor" + + +# --------------------------------------------------------------------------- layer 2 +def test_floor_is_unbypassable_property_scan(): + """No (R, implied, n_post) with implied below the floor may ever FIRE -- including + R = 0, the degenerate maximum-deficit reading the floor exists to intercept.""" + rng = np.random.default_rng(20260819) + for _ in range(2000): + n_post = int(rng.integers(100, 200000)) + floor = gate.FLOOR_COUNTS / n_post + implied = floor * rng.uniform(0, 1.0 - 1e-12) # strictly below the floor + R = float(rng.choice([0.0, rng.uniform(0, 2.0), np.nan])) + decision, _ = gate.decide(implied, R, n_post) + assert decision == "ABSTAIN-FLOOR", (implied, R, n_post, decision) + # and the exact worst case + assert gate.decide(0.0, 0.0, 20000)[0] == "ABSTAIN-FLOOR" + assert gate.decide(float("nan"), 0.0, 20000)[0] == "ABSTAIN-FLOOR" + + +def test_fire_requires_floor_and_threshold_jointly(): + rng = np.random.default_rng(7) + for _ in range(2000): + n_post = int(rng.integers(100, 200000)) + implied = float(rng.uniform(0, 0.5)) + R = float(rng.uniform(0, 1.5)) + decision, floor = gate.decide(implied, R, n_post) + if decision == "FIRE": + assert implied >= floor and R < gate.THRESHOLD + elif decision == "NO-FIRE": + assert implied >= floor and not (R < gate.THRESHOLD) + + +def test_cli_refuses_to_disable_floor(tmp_path): + """--floor-counts below 1 is refused (exit 2), and no bypass flag exists at all.""" + train, post = _synthetic(tmp_path, deficit="severe") + r = subprocess.run([sys.executable, _GATE, str(train), str(post), "--floor-counts", "0"], + capture_output=True, text=True) + assert r.returncode == 2 and "mandatory" in r.stderr + h = subprocess.run([sys.executable, _GATE, "--help"], capture_output=True, text=True) + assert h.returncode == 0 + low = h.stdout.lower() + for bad in ("--no-floor", "--skip-floor", "--disable-floor", "--force-fire"): + assert bad not in low + + +# --------------------------------------------------------------------------- layer 3 +def _synthetic(tmp_path, deficit): + """Small synthetic (training, posterior) pair with a controllable tail state. + Training: 6000 rows, 13 cols, uniform-ish chi1_perp coverage; lnL flat near-peak in the + core. 'severe': tail lnL also near peak (implied large) but posterior confined -> + delivered tiny -> R tiny -> FIRE. 'healthy': posterior tracks the implied tail mass -> + NO-FIRE. 'confined': tail lnL ~ 25 nats down -> implied unresolvable -> ABSTAIN-FLOOR.""" + rng = np.random.default_rng(42) + n = 6000 + a1 = rng.uniform(0, 0.99, n) + ct = rng.uniform(-1, 1, n) + cp = a1 * np.sqrt(1 - ct ** 2) + lnl = 100.0 - 0.5 * rng.uniform(0, 1, n) # near-peak core everywhere + tail = cp > 0.5 + if deficit == "confined": + lnl[tail] -= 25.0 # tail is truly dead + X = np.zeros((n, 13)) + X[:, 1] = 10.0; X[:, 2] = 8.0 # m1 m2 (unused by the gate) + X[:, 3] = cp; X[:, 4] = 0.0; X[:, 5] = a1 * ct # s1x s1y s1z + X[:, 9] = lnl + train = tmp_path / f"train_{deficit}.net" + np.savetxt(train, X) + m = 20000 + if deficit == "healthy": + pa1 = rng.uniform(0, 0.99, m); pct = rng.uniform(-1, 1, m) + pcp = pa1 * np.sqrt(1 - pct ** 2) # prior-wide: delivered ~ implied + else: + pcp = rng.uniform(0, 0.10, m) # confined posterior + post = tmp_path / f"post_{deficit}.dat" + with open(post, "w") as f: + f.write("# a1x a1y lnL\n") + np.savetxt(f, np.column_stack([pcp, np.zeros(m), np.zeros(m)])) + return train, post + + +@pytest.mark.parametrize("deficit,expect", [("severe", "FIRE"), ("healthy", "NO-FIRE"), + ("confined", "ABSTAIN-FLOOR")]) +def test_cli_end_to_end_synthetic(tmp_path, deficit, expect): + train, post = _synthetic(tmp_path, deficit) + r = subprocess.run([sys.executable, _GATE, str(train), str(post), + "--json", str(tmp_path / "g.json")], + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + last = r.stdout.strip().splitlines()[-1] + assert last.startswith(f"GATE DECISION={expect} "), (last, r.stderr) + rec = json.load(open(tmp_path / "g.json")) + assert rec["decision"] == expect + # the stderr log must name the deciding condition -- silent no-ops are the bug class + assert expect.split("-")[0] in r.stderr or expect in r.stderr + + +def test_cli_multi_rep_mean_decides(tmp_path): + """With several detect-rep posteriors the gate must decide on the MEAN R, and the json + must record the per-rep values.""" + train, post_conf = _synthetic(tmp_path, "severe") # confined posterior: R ~ 0 + _, post_wide = _synthetic(tmp_path, "healthy") # prior-wide posterior: R ~ 1 + # one confined + two wide: mean R lands well above the threshold -> NO-FIRE, + # even though the confined rep alone would FIRE + r = subprocess.run([sys.executable, _GATE, str(train), str(post_conf), str(post_wide), + str(post_wide), "--json", str(tmp_path / "m.json")], + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + assert r.stdout.strip().splitlines()[-1].startswith("GATE DECISION=NO-FIRE ") + rec = json.load(open(tmp_path / "m.json")) + assert rec["n_reps"] == 3 and len(rec["R_per_rep"]) == 3 + assert rec["R_per_rep"][0] < gate.THRESHOLD < rec["R"] + assert "deciding on the mean" in r.stderr + + +def test_wrapper_gates_and_falls_back(tmp_path): + """Wrapper e2e with both hooks faked (2 detect reps): FIRE path swaps --fname to the + thinned set for the FINAL pass only; a broken gate falls back to the original loudly.""" + wrapper = os.path.abspath(os.path.join(_BIN, "util_CIPCompositionReweightWrapper.sh")) + train, post = _synthetic(tmp_path, "severe") + fake_cip = tmp_path / "fake_cip.sh" + fake_cip.write_text( + "#!/bin/bash\n" + "out=''; nxt=0\n" + "for a in \"$@\"; do\n" + " if [[ $nxt == 1 ]]; then out=$a; nxt=0; fi\n" + " [[ $a == --fname-output-samples ]] && nxt=1\n" + " [[ $a == --fname-output-samples=* ]] && out=${a#--fname-output-samples=}\n" + "done\n" + "[[ -n $out ]] && echo fake > \"$out.xml.gz\"\n" + "echo \"FAKECIP ARGS: $@\"\n") + fake_cip.chmod(0o755) + fake_conv = tmp_path / "fake_conv.sh" + fake_conv.write_text(f"#!/bin/bash\ncat {post}\n") + fake_conv.chmod(0o755) + # the wrapper invokes the gate/reweight tools via their `#!/usr/bin/env python3` + # shebangs; guarantee that python3 resolves to THIS interpreter (numpy-capable) + pybin = tmp_path / "pybin" + pybin.mkdir() + (pybin / "python3").symlink_to(sys.executable) + env = dict(os.environ, CIP_REWEIGHT_REAL_CIP=str(fake_cip), + CIP_REWEIGHT_CONVERT=str(fake_conv), CIP_REWEIGHT_GATE_REPS="2", + PATH=str(pybin) + os.pathsep + _BIN + os.pathsep + os.environ.get("PATH", "")) + r = subprocess.run(["bash", wrapper, "--fname", str(train), + "--fname-output-samples", str(tmp_path / "final"), + "--n-output-samples", "138"], + capture_output=True, text=True, env=env, timeout=600) + assert r.returncode == 0, r.stderr + assert "GATE DECISION=FIRE" in r.stderr + final_args = [ln for ln in r.stdout.splitlines() if ln.startswith("FAKECIP ARGS")][-1] + assert "all_comp_" in final_args, "final CIP must train on the thinned set after a FIRE" + # fallback: converter emits garbage -> gate cannot evaluate -> original file, loudly + fake_conv.write_text("#!/bin/bash\necho not-a-posterior\n") + r2 = subprocess.run(["bash", wrapper, "--fname", str(train), + "--fname-output-samples", str(tmp_path / "final2"), + "--n-output-samples", "138"], + capture_output=True, text=True, env=env, timeout=600) + assert r2.returncode == 0 + assert "falling back" in r2.stderr + final2 = [ln for ln in r2.stdout.splitlines() if ln.startswith("FAKECIP ARGS")][-1] + assert str(train) in final2 and "all_comp_" not in final2 diff --git a/PR_DRAFT_comp_reweight_optin.md b/PR_DRAFT_comp_reweight_optin.md new file mode 100644 index 000000000..ff27e2c29 --- /dev/null +++ b/PR_DRAFT_comp_reweight_optin.md @@ -0,0 +1,136 @@ +# DRAFT PR: opt-in gated composition reweight for the low-mass transverse-spin width deficit + +**This PR is not proposed for merge yet.** It is staged as a DRAFT while the paper-scale +demonstration (in preparation) independently establishes the problem and the fix; the merge +decision waits on that demonstration. Please do not review or mark ready until then. + +(This file mirrors the PR description; delete it at merge.) + +## What this is + +An opt-in (default OFF) severe-deficit repair for CIP. Across the O4 low-mass catalogue, +RIFT's chi1_perp/a1 posteriors are systematically narrow (median width ratio 0.800 vs bilby +over 25 events; high-mass events sit at 1.007). Root cause, established by intervention: +training-set COMPOSITION — the near-peak fraction of `all.net` rows degrades with chi1_perp, +and the RF fit (a local average) regresses the transverse tail toward its junk-diluted +neighbours. Density-equalising THINNING of the far-from-peak rows (never truncation, never +invented points; lnL span preserved exactly) repairs it: on 5 deficit exemplars, paired CIP +on identical inputs moved chi1_perp toward the reference by +0.042..+0.115 with mc/q +undamaged. + +An UNGATED version of that thinning was tested two-sided and REJECTED, twice and +independently: on real data it overshot three bad-composition events whose widths are +already correct to 1.18–1.54x the reference, and on the known-truth toybench it recovered +**2.56x / 3.32x / 2.23x truth** on genuinely-narrow transverse posteriors (T5/T5b/T5c) — +a specificity veto at every tail depth, so the true worst case of unconditional thinning is +up to **3.32x**, not the 1.54x real data happened to show. The two results reconcile +exactly: the toybench predicted the high-mass control group should widen past the reference +under unconditional reweighting, and the real-data test measured precisely that pattern — +good-composition high-mass controls were exact no-ops (+0.001, +0.013) while +bad-composition healthy events blew up. The driver is COMPOSITION, not mass: unconditional +thinning harms bad-composition events regardless of whether they are deficient, because on +a genuinely narrow posterior it deletes the far rows that teach the fit the tail is +unsupported. The T5 family is the known-truth instance of exactly the S240930aa class. +This PR ships the gated version: thin ONLY on a detected severe deficit. + +## The gate (`util_CIPTailDeficitGate.py`) + +R = delivered/implied transverse-tail posterior mass, computed from RIFT's own products +only (all.net + a CIP posterior trained on it; no external reference): +implied = per-chi1_perp-bin analytic prior volume x mean exp(lnL-peak) over the bin's real +ILE rows; boundary = the training set's own cp q80. FIRE iff R < 0.32 AND the MANDATORY +sample-resolution validity floor (implied >= 50/n_post) is satisfied; otherwise a loudly +logged no-op. + +* **The validity floor cannot be bypassed** (no flag exists; `--floor-counts < 1` is + refused; `decide()` applies it before the threshold; property-tested). Without it the + detector returns R = 0 — maximum apparent deficit — exactly where it can resolve nothing: + on known-truth healthy-narrow benchmarks (toybench T5/T5b/T5c) every chain read R = 0 and + only the floor prevented false fires. +* **The toybench's own remedy is this design**: its veto of the ungated arm concludes a + deployable version must gate on tail *support*, not row counts — R is exactly a + tail-support statistic (implied tail mass from measured lnL x prior volume), the validity + floor covers the unresolvable-support case, and on the T5 family the shipped gate was + measured to ABSTAIN via that floor. The toys independently veto the ungated tool and + independently endorse the gated one's design principle; no more than that is claimed. +* **The threshold is channel-calibrated at 0.32**, in the fresh-CIP measurement channel the + wrapper actually uses. The population calibration (102 production events, perfect + 12-vs-77 separation, gap 0.379–0.457) was measured on production consolidated posteriors; + the fresh-CIP channel reads mid-band healthy events lower by up to ~0.07, so the + population threshold 0.42 does NOT transfer — **the regression fixture caught this** + (control S240930aa: deployment-channel R 0.386 < 0.42). Deployment-channel gap: severe + deficits <= 0.270, healthy controls >= 0.386; 0.32 is the geometric midpoint. + +## The wrapper (`util_CIPCompositionReweightWrapper.sh`, drop-in `--cip-exe`) + +Detect -> repair: N detect passes (default N=5, `CIP_REWEIGHT_GATE_REPS`) run the REAL CIP +on the ORIGINAL data (exports bumped to >= 20000, temp outputs; real output paths +untouched); the gate decides ONCE on the MEAN R over the reps; the final CIP runs the +original argv verbatim, with `--fname` swapped to the thinned set only on a FIRE. Every +no-op is logged with the R value, the per-rep spread, and the deciding condition. +Fail-safe throughout: any tool failure -> final CIP on the original argv, loudly. Costs N +extra CIP passes per invocation when enabled (CIP is the cheap CPU stage). +RandomizeOverlapOrder-style modularity: nothing in CIP or the merge step changes; flag-off +is byte-identical to today. + +Opt in via `util_RIFT_pseudo_pipe.py --internal-cip-composition-reweight` (conflict-guarded +vs `--internal-use-amr`). + +## SCOPE — read before enabling + +* Repairs SEVERE deficits only (production low-mass class, width ratios ~0.62–0.69; + deployment-channel R <= 0.27). **Roughly half the affected low-mass events; the mild rest + are loudly left alone**: mild deficit and healthy width are NOT separable above the + threshold (out-of-sample, truth-deficient toybench T3 chains at R 0.532–0.634 abut + truth-healthy T2 chains at 0.633–0.673). +* Safety is the strongly supported side: zero false fires across 77 in-sample healthy + events (95% bound 3.8%) and on known-truth healthy-narrow toys. The fire side is + in-sample-validated (12/12 severe-deficit events, 95% miss-rate bound 22.1%) plus the + causal 13-event paired-CIP repair. +* What the known-truth evidence does and does not certify: the toybench could not + reproduce the pathological regime — its gate (a) FAILED (a1 = 1.381 vs required <= 0.65: + from a fresh bootstrap the loop EXPANDS 38% at iteration 1 rather than collapsing; third + independent harness to fail this, and the first running the real cepp_BasicIteration + DAG) — so it cannot certify the fix where the deficit actually develops. The known-truth + evidence certifies the SAFETY side (abstain/no-fire behaviour) only; the REPAIR side + rests entirely on the real-data paired-CIP result. +* Margin, MEASURED (16 independent detect reps on the closest healthy control S240930aa; + R_dispersion.json + the live N=5 run in the study record): deployment-channel mean + R = 0.371, single-rep sigma = 0.024, and **1 of the 16 single reps actually read below + the threshold (0.325)** -- a single-pass gate has an observed ~6%/run false-fire rate on + this event class, each fire costing up to 3.32x truth (known-truth toybench bound; + 1.54x was the real-data instance). The shipped + default N=5 mean-of-reps retires this: effective margin ~4.6 sigma (sem 0.011), and in + the live N=5 verification the sub-threshold rep occurred and the mean correctly decided + NO-FIRE. Fire side: measured exemplar sigma 0.014 (S240629by, 11 sigma single-rep); + the nearest exemplar to the threshold (S241109bn, R = 0.27) keeps ~5 sigma at N=5. + The two S240930aa readings quoted earlier (0.386 pooled, 0.357 live) decompose into a + REAL channel offset (production 0.457 vs deployment 0.371, -0.086 >> sem) plus this + single-rep noise -- both handled: channel-calibrated threshold, averaged decision. + +## Verification + +* `test/test_tail_deficit_gate.py` (named in ci.yml; 24 tests, all passing): measured-value + regressions pinning the shipped gate's decisions on the 5 exemplars (FIRE), the 3 + controls the ungated fix broke (NO-FIRE), and 6 known-truth healthy-narrow toybench + chains (ABSTAIN **via the floor** — each has R < threshold, so the threshold alone would + false-fire; the reason is asserted, not just the outcome); property scans proving no + (implied, R, n_post) below the floor can ever FIRE; CLI bypass refusal; synthetic + end-to-end FIRE/NO-FIRE/ABSTAIN; wrapper e2e (FIRE swaps fname for the final pass only; + broken gate falls back loudly to the original). +* Live end-to-end on production data (real CIP, branch as a unit): S240629by -> FIRE + (R=0.171), final CIP trained on the thinned set; S240930aa -> NO-FIRE (R=0.357), final + CIP verbatim; both rc=0 with the decision and record path in the log. + +## Evidence record + +rift_transverse_highSNR_study `results_triage/`: MULTIEVENT_REWEIGHT_2026-08-19.md (ungated +two-sided test, rejected), GATED_REWEIGHT_2026-08-19.md (13-event gate-0), +R_POPULATION_CALIBRATION_2026-08-19.md (102-event calibration; comp-contamination hypothesis +refuted), R_TOYBENCH_VALIDATION_2026-08-19.md (known-truth out-of-sample; floor load-bearing), +TOYBENCH_RESULT_2026-08-19.md (independent campaign: ungated arm VETOED at 2.56-3.32x +truth; gate (a) failure bounding what the bench can certify), BRANCH_LANDING_2026-08-19.md +(channel systematic, live e2e, measured R dispersion). + +Not applicable to the LISA fork (CIP-side change; the LISA fork diverges only in the ILE +driver).