diff --git a/audit_truncation.py b/audit_truncation.py new file mode 100644 index 0000000..48a57d8 --- /dev/null +++ b/audit_truncation.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Truncation-marker audit for codex-batch-bench runs. + +Companion to audit_ingestion.py. Addresses openai/codex#35421: tool output is +truncated at more than one layer, the layers emit different markers, and one of +them (the legacy shell capture path) discards bytes silently and then reports a +*constant* figure regardless of how much was actually lost. + +Two model-facing marker forms appear in rollout output: + + head form "Warning: truncated output (original token count: N)" + emitted by the exec / unified formatter; N tracks real size and + varies with the output. + + middle form "...N tokens truncated..." (unicode ellipsis U+2026) + emitted by the legacy shell formatter's middle elision. Once the + upstream 1 MiB capture cap has saturated, the elision input is + always exactly 1,048,576 bytes, so N collapses to a constant + (252144 for the observed ~40k-byte budget) and carries no + information about the true discarded volume. + +For every trial this counts each marker by form, records the reported figures, +and separates capture-cap saturation (a middle figure repeated as a constant) +from variable per-call-budget elisions. Reports per arm per run, mirroring +audit_ingestion.py, so the two truncation surfaces can be read side by side. + +Usage: python3 audit_truncation.py [runs_dir] +""" + +import collections +import glob +import json +import os +import re +import sys + +HEAD_RE = re.compile(r"original token count:\s*(\d+)") +MIDDLE_RE = re.compile(r"(\d+)\s+tokens truncated") + + +def iter_outputs(tdir, meta): + for rname in meta["rollouts"]: + path = os.path.join(tdir, rname) + if not os.path.exists(path): + continue + for line in open(path): + try: + d = json.loads(line) + except json.JSONDecodeError: + continue + p = d.get("payload", {}) + if isinstance(p, dict) and p.get("type") in ( + "function_call_output", "custom_tool_call_output"): + out = p.get("output") + if isinstance(out, str): + yield out + elif isinstance(out, list): + yield "".join(seg.get("text", "") for seg in out + if isinstance(seg, dict)) + + +def markers(text): + """Yield (form, reported_value) for each truncation marker in text. + + head markers are matched first and their span removed so a head marker is + never also counted as a middle marker. + """ + head = [int(m.group(1)) for m in HEAD_RE.finditer(text)] + residual = HEAD_RE.sub("", text) + middle = [int(m.group(1)) for m in MIDDLE_RE.finditer(residual)] + for v in head: + yield "head", v + for v in middle: + yield "middle", v + + +def main(): + runs_dir = sys.argv[1] if len(sys.argv) > 1 else "runs" + print(f"{'run':40s} {'arm':9s} {'trials':>6s} head mid const_mid distinct_mid") + for rd in sorted(glob.glob(os.path.join(runs_dir, "*"))): + if not os.path.isdir(rd) or os.path.basename(rd).startswith("ping-"): + continue + per_arm = {} + for tdir in sorted(glob.glob(os.path.join(rd, "t*_*"))): + meta_path = os.path.join(tdir, "trial_meta.json") + if not os.path.exists(meta_path): + continue + meta = json.loads(open(meta_path).read()) + head, mid = 0, collections.Counter() + for text in iter_outputs(tdir, meta): + for form, val in markers(text): + if form == "head": + head += 1 + else: + mid[val] += 1 + a = per_arm.setdefault( + meta["arm_name"], + {"trials": 0, "head": 0, "mid": collections.Counter()}) + a["trials"] += 1 + a["head"] += head + a["mid"].update(mid) + for arm, agg in sorted(per_arm.items()): + mid = agg["mid"] + total_mid = sum(mid.values()) + # capture-cap signature: the most common middle value, when it + # recurs, is the saturated constant. Everything else is variable + # per-call-budget elision. + const_mid = mid.most_common(1)[0][1] if mid else 0 + distinct = len(mid) + print(f"{os.path.basename(rd)[:40]:40s} {arm:9s} " + f"{agg['trials']:>6} {agg['head']:>4} {total_mid:>3} " + f"{const_mid:>8} {distinct:>11}") + print("\nhead = 'original token count: N' (exec/unified formatter; N varies)") + print("mid = 'N tokens truncated' (legacy shell middle elision)") + print("const_mid = count of the single most common middle figure; a large" + " value here is the capture-cap constant (stage-1 saturated)") + print("distinct_mid = number of distinct middle figures; 1 means every" + " middle event reported the same constant") + print("\nNote: markers are matched in model-facing output text. Content that" + " merely quotes a marker (e.g. a command printing a rollout) cannot be" + " distinguished from a formatter-emitted marker by text alone.") + + +if __name__ == "__main__": + main()