Add audit_truncation.py — classify truncation markers by layer

audit_truncation.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Truncation-marker audit for codex-batch-bench runs.
+
+Companion to audit_ingestion.py. Addresses openai/codex#35421: tool output is
+truncated at more than one layer, the layers emit different markers, and one of
+them (the legacy shell capture path) discards bytes silently and then reports a
+*constant* figure regardless of how much was actually lost.
+
+Two model-facing marker forms appear in rollout output:
+
+  head form    "Warning: truncated output (original token count: N)"
+               emitted by the exec / unified formatter; N tracks real size and
+               varies with the output.
+
+  middle form  "...N tokens truncated..." (unicode ellipsis U+2026)
+               emitted by the legacy shell formatter's middle elision. Once the
+               upstream 1 MiB capture cap has saturated, the elision input is
+               always exactly 1,048,576 bytes, so N collapses to a constant
+               (252144 for the observed ~40k-byte budget) and carries no
+               information about the true discarded volume.
+
+For every trial this counts each marker by form, records the reported figures,
+and separates capture-cap saturation (a middle figure repeated as a constant)
+from variable per-call-budget elisions. Reports per arm per run, mirroring
+audit_ingestion.py, so the two truncation surfaces can be read side by side.
+
+Usage: python3 audit_truncation.py [runs_dir]
+"""
+
+import collections
+import glob
+import json
+import os
+import re
+import sys
+
+HEAD_RE = re.compile(r"original token count:\s*(\d+)")
+MIDDLE_RE = re.compile(r"(\d+)\s+tokens truncated")
+
+
+def iter_outputs(tdir, meta):
+    for rname in meta["rollouts"]:
+        path = os.path.join(tdir, rname)
+        if not os.path.exists(path):
+            continue
+        for line in open(path):
+            try:
+                d = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            p = d.get("payload", {})
+            if isinstance(p, dict) and p.get("type") in (
+                    "function_call_output", "custom_tool_call_output"):
+                out = p.get("output")
+                if isinstance(out, str):
+                    yield out
+                elif isinstance(out, list):
+                    yield "".join(seg.get("text", "") for seg in out
+                                  if isinstance(seg, dict))
+
+
+def markers(text):
+    """Yield (form, reported_value) for each truncation marker in text.
+
+    head markers are matched first and their span removed so a head marker is
+    never also counted as a middle marker.
+    """
+    head = [int(m.group(1)) for m in HEAD_RE.finditer(text)]
+    residual = HEAD_RE.sub("", text)
+    middle = [int(m.group(1)) for m in MIDDLE_RE.finditer(residual)]
+    for v in head:
+        yield "head", v
+    for v in middle:
+        yield "middle", v
+
+
+def main():
+    runs_dir = sys.argv[1] if len(sys.argv) > 1 else "runs"
+    print(f"{'run':40s} {'arm':9s} {'trials':>6s}  head  mid  const_mid  distinct_mid")
+    for rd in sorted(glob.glob(os.path.join(runs_dir, "*"))):
+        if not os.path.isdir(rd) or os.path.basename(rd).startswith("ping-"):
+            continue
+        per_arm = {}
+        for tdir in sorted(glob.glob(os.path.join(rd, "t*_*"))):
+            meta_path = os.path.join(tdir, "trial_meta.json")
+            if not os.path.exists(meta_path):
+                continue
+            meta = json.loads(open(meta_path).read())
+            head, mid = 0, collections.Counter()
+            for text in iter_outputs(tdir, meta):
+                for form, val in markers(text):
+                    if form == "head":
+                        head += 1
+                    else:
+                        mid[val] += 1
+            a = per_arm.setdefault(
+                meta["arm_name"],
+                {"trials": 0, "head": 0, "mid": collections.Counter()})
+            a["trials"] += 1
+            a["head"] += head
+            a["mid"].update(mid)
+        for arm, agg in sorted(per_arm.items()):
+            mid = agg["mid"]
+            total_mid = sum(mid.values())
+            # capture-cap signature: the most common middle value, when it
+            # recurs, is the saturated constant. Everything else is variable
+            # per-call-budget elision.
+            const_mid = mid.most_common(1)[0][1] if mid else 0
+            distinct = len(mid)
+            print(f"{os.path.basename(rd)[:40]:40s} {arm:9s} "
+                  f"{agg['trials']:>6}  {agg['head']:>4}  {total_mid:>3}  "
+                  f"{const_mid:>8}  {distinct:>11}")
+    print("\nhead   = 'original token count: N' (exec/unified formatter; N varies)")
+    print("mid    = 'N tokens truncated' (legacy shell middle elision)")
+    print("const_mid    = count of the single most common middle figure; a large"
+          " value here is the capture-cap constant (stage-1 saturated)")
+    print("distinct_mid = number of distinct middle figures; 1 means every"
+          " middle event reported the same constant")
+    print("\nNote: markers are matched in model-facing output text. Content that"
+          " merely quotes a marker (e.g. a command printing a rollout) cannot be"
+          " distinguished from a formatter-emitted marker by text alone.")
+
+
+if __name__ == "__main__":
+    main()