Skip to content

Commit 8f07009

Browse files
adigo-proclaude
adigo-pro
and
claude
committed
Bake-off round 2: gpt-5-mini is Nemotron's complement, not its replacement
Seven more candidates. gpt-5-mini: 4/4 catches incl. docstring-trap (which every round-1 model missed) but 2 false positives on clean cases — the exact inverse of Nemotron's profile (0 false positives, 2 catch-misses). gpt-oss 5/7, kimi 5/7 (slow), qwen3.5-122b 4/7. NVIDIA free tier: mistral-large and codestral not servable via provider registration, llama-4 times out. Single-critic seat stays Nemotron (precision-first, 5x faster, free); the data now makes the two-model council case concrete. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 43e3e13 commit 8f07009

1 file changed

Lines changed: 261 additions & 0 deletions

File tree

‎docs/benchmarks/2026-07-21-critic-bakeoff-round2.json‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
{
2+
"cases": [
3+
"clean-rename",
4+
"clean-slugify",
5+
"clean-truncate",
6+
"docstring-trap",
7+
"latent-config",
8+
"latent-drift",
9+
"planted-divide"
10+
],
11+
"models": {
12+
"nvidia-nim/mistralai/mistral-large": {
13+
"score": null,
14+
"note": "PONG failed \u2014 skipped"
15+
},
16+
"nvidia-nim/meta/llama-4-maverick-17b-128e-instruct": {
17+
"score": null,
18+
"note": "PONG failed \u2014 skipped"
19+
},
20+
"nvidia-nim/mistralai/codestral-22b-instruct-v0.1": {
21+
"score": null,
22+
"note": "PONG failed \u2014 skipped"
23+
},
24+
"openrouter/openai/gpt-5-mini": {
25+
"score": 0.714,
26+
"correct": 5,
27+
"malformed": 0,
28+
"errors": 0,
29+
"mean_s": 10.5,
30+
"max_s": 16.5,
31+
"rows": [
32+
{
33+
"case": "clean-rename",
34+
"expected": "pass",
35+
"verdict": "SUGGESTION",
36+
"ok": false,
37+
"s": 15.9
38+
},
39+
{
40+
"case": "clean-slugify",
41+
"expected": "pass",
42+
"verdict": "PASS",
43+
"ok": true,
44+
"s": 6.5
45+
},
46+
{
47+
"case": "clean-truncate",
48+
"expected": "pass",
49+
"verdict": "SUGGESTION",
50+
"ok": false,
51+
"s": 10.1
52+
},
53+
{
54+
"case": "docstring-trap",
55+
"expected": "flag",
56+
"verdict": "SUGGESTION",
57+
"ok": true,
58+
"s": 9.9
59+
},
60+
{
61+
"case": "latent-config",
62+
"expected": "flag",
63+
"verdict": "SUGGESTION",
64+
"ok": true,
65+
"s": 7.6
66+
},
67+
{
68+
"case": "latent-drift",
69+
"expected": "flag",
70+
"verdict": "SUGGESTION",
71+
"ok": true,
72+
"s": 16.5
73+
},
74+
{
75+
"case": "planted-divide",
76+
"expected": "flag",
77+
"verdict": "SUGGESTION",
78+
"ok": true,
79+
"s": 7.2
80+
}
81+
]
82+
},
83+
"openrouter/openai/gpt-oss-120b": {
84+
"score": 0.714,
85+
"correct": 5,
86+
"malformed": 1,
87+
"errors": 0,
88+
"mean_s": 12.5,
89+
"max_s": 42.3,
90+
"rows": [
91+
{
92+
"case": "clean-rename",
93+
"expected": "pass",
94+
"verdict": "SUGGESTION",
95+
"ok": false,
96+
"s": 42.3
97+
},
98+
{
99+
"case": "clean-slugify",
100+
"expected": "pass",
101+
"verdict": "PASS",
102+
"ok": true,
103+
"s": 5.3
104+
},
105+
{
106+
"case": "clean-truncate",
107+
"expected": "pass",
108+
"verdict": "PASS",
109+
"ok": true,
110+
"s": 23.4
111+
},
112+
{
113+
"case": "docstring-trap",
114+
"expected": "flag",
115+
"verdict": "SUGGESTION",
116+
"ok": true,
117+
"s": 3.8
118+
},
119+
{
120+
"case": "latent-config",
121+
"expected": "flag",
122+
"verdict": "PASS",
123+
"ok": false,
124+
"s": 1.6
125+
},
126+
{
127+
"case": "latent-drift",
128+
"expected": "flag",
129+
"verdict": "SUGGESTION",
130+
"ok": true,
131+
"s": 6.9
132+
},
133+
{
134+
"case": "planted-divide",
135+
"expected": "flag",
136+
"verdict": "SUGGESTION",
137+
"ok": true,
138+
"s": 4.2
139+
}
140+
]
141+
},
142+
"openrouter/moonshotai/kimi-k2.6": {
143+
"score": 0.714,
144+
"correct": 5,
145+
"malformed": 2,
146+
"errors": 0,
147+
"mean_s": 31.9,
148+
"max_s": 62.1,
149+
"rows": [
150+
{
151+
"case": "clean-rename",
152+
"expected": "pass",
153+
"verdict": "PASS",
154+
"ok": true,
155+
"s": 10.0
156+
},
157+
{
158+
"case": "clean-slugify",
159+
"expected": "pass",
160+
"verdict": "PASS",
161+
"ok": true,
162+
"s": 36.1
163+
},
164+
{
165+
"case": "clean-truncate",
166+
"expected": "pass",
167+
"verdict": "SUGGESTION",
168+
"ok": false,
169+
"s": 39.0
170+
},
171+
{
172+
"case": "docstring-trap",
173+
"expected": "flag",
174+
"verdict": "SUGGESTION",
175+
"ok": true,
176+
"s": 44.4
177+
},
178+
{
179+
"case": "latent-config",
180+
"expected": "flag",
181+
"verdict": "SUGGESTION",
182+
"ok": true,
183+
"s": 62.1
184+
},
185+
{
186+
"case": "latent-drift",
187+
"expected": "flag",
188+
"verdict": "PASS",
189+
"ok": false,
190+
"s": 11.2
191+
},
192+
{
193+
"case": "planted-divide",
194+
"expected": "flag",
195+
"verdict": "SUGGESTION",
196+
"ok": true,
197+
"s": 20.6
198+
}
199+
]
200+
},
201+
"openrouter/qwen/qwen3.5-122b-a10b": {
202+
"score": 0.571,
203+
"correct": 4,
204+
"malformed": 0,
205+
"errors": 0,
206+
"mean_s": 6.4,
207+
"max_s": 9.9,
208+
"rows": [
209+
{
210+
"case": "clean-rename",
211+
"expected": "pass",
212+
"verdict": "PASS",
213+
"ok": true,
214+
"s": 4.3
215+
},
216+
{
217+
"case": "clean-slugify",
218+
"expected": "pass",
219+
"verdict": "PASS",
220+
"ok": true,
221+
"s": 4.4
222+
},
223+
{
224+
"case": "clean-truncate",
225+
"expected": "pass",
226+
"verdict": "SUGGESTION",
227+
"ok": false,
228+
"s": 8.4
229+
},
230+
{
231+
"case": "docstring-trap",
232+
"expected": "flag",
233+
"verdict": "PASS",
234+
"ok": false,
235+
"s": 6.6
236+
},
237+
{
238+
"case": "latent-config",
239+
"expected": "flag",
240+
"verdict": "SUGGESTION",
241+
"ok": true,
242+
"s": 8.5
243+
},
244+
{
245+
"case": "latent-drift",
246+
"expected": "flag",
247+
"verdict": "PASS",
248+
"ok": false,
249+
"s": 9.9
250+
},
251+
{
252+
"case": "planted-divide",
253+
"expected": "flag",
254+
"verdict": "SUGGESTION",
255+
"ok": true,
256+
"s": 2.6
257+
}
258+
]
259+
}
260+
}
261+
}

0 commit comments

Comments
 (0)