[algoperf] Allow scoring with target relaxations

.gitignore

@@ -5,3 +5,6 @@ __pycache__/
 # Scoring output artifacts (see README "Scoring")
 scoring_results*/
 *.egg-info/
+
+# Virtual environments (uv.lock is committed; the venv itself is not)
+.venv/

README.md

@@ -82,8 +82,10 @@ pip3 install -e .          # installs the scoring tooling (numpy, pandas, scipy,
 > definitions in [algorithmic-efficiency](https://github.com/mlcommons/algorithmic-efficiency)
 > (`scoring/generate_workload_targets.py`) and commited here. Each file is
 > frozen for one benchmark version, carrying that version's base/held-out
-> workload sets and per-workload targets. Regenerate and re-copy when a
-> benchmark version changes the workloads or targets.
+> workload sets and per-workload metric names, optimization goals, targets,
+> and step hints. Regenerate and re-copy when a benchmark version changes the
+> workloads or targets; the generator must emit `target_metric_goal` as either
+> `minimize` or `maximize` for every workload.
 
 ### Regenerating the leaderboard
 
@@ -112,6 +114,50 @@ python -m scoring.score_submissions \
   --output_dir scoring_results_v05_external
 ```
 
+### Exploring relaxed convergence targets
+
+To measure how leaderboard scores would change under easier convergence
+targets, pass `--target_relaxations`. The value is a comma-separated list of
+`SELECTOR=FRACTION` assignments. Fractions are relative: `0.05` means a 5%
+relaxation. Use `all` to select every workload, or name a workload to select
+only that workload. Naming a base workload also selects all of its held-out
+variants.
+
+```bash
+# Relax every convergence target by 5%.
+python -m scoring.score_submissions \
+  --submission_directory logs/self_tuning \
+  --compute_performance_profiles \
+  --self_tuning_ruleset \
+  --target_relaxations=all=0.05 \
+  --output_dir scoring_results_relaxed
+
+# Relax only WMT by 5% and the ImageNet ResNet family by 10%.
+python -m scoring.score_submissions \
+  --submission_directory logs/self_tuning \
+  --compute_performance_profiles \
+  --self_tuning_ruleset \
+  --target_relaxations=wmt=0.05,imagenet_resnet=0.10 \
+  --output_dir scoring_results_selected_relaxations
+
+# Apply 5% globally, with a 10% override for ImageNet ResNet.
+--target_relaxations=all=0.05,imagenet_resnet=0.10
+```
+
+For lower-is-better metrics such as loss, a relaxation increases the target;
+for higher-is-better metrics such as accuracy, it decreases the target. A 5%
+relaxation therefore changes a loss target of `0.2` to `0.21`, and an accuracy
+target of `0.8` to `0.76`.
+
+The command parses the submission logs once, then scores the official and
+relaxed workload configurations serially. Unsuffixed artifacts remain the
+official results; the exploratory artifacts use a `_relaxed` suffix. For
+example, the run produces `scores.csv` and `scores_relaxed.csv`, along with
+`time_to_targets.csv` and `time_to_targets_relaxed.csv`.
+
+Target relaxation is an exploratory analysis feature. It does not modify the
+frozen `workload_targets*.json` files or the official scoring definition.
+
 See the [scoring methodology](https://github.com/mlcommons/algorithmic-efficiency/blob/main/docs/DOCUMENTATION.md#scoring)
 in the benchmark documentation for details on how scores are computed.
 

artifacts/leaderboard_v2/ademamix_summary.csv

@@ -0,0 +1,27 @@
+,workload,trial,val target metric name,val target metric value,val target reached,best metric value on val,time to best eval on val (s),time to target on val (s),step_time (s),step_hint
+7,criteo1tb_pytorch,trial_1,validation/loss,0.123735,True,0.1236847321358005,7839.193164110184,7839.193164110184,1.0010825763927422,10666
+16,criteo1tb_pytorch,trial_1,validation/loss,0.123735,False,0.1242111736866617,13214.635874271393,inf,1.0010825763927422,10666
+24,criteo1tb_pytorch,trial_1,validation/loss,0.123735,True,0.1236501955610997,7835.540827274322,7479.550709962845,1.0010825763927422,10666
+5,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.7248104441078714,1311.1073644161224,1311.1073644161224,0.9420394021221715,18094
+14,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.7249505124067952,1517.7321906089785,1517.7321906089785,0.9420394021221715,18094
+22,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.724893770663337,1555.3687834739685,1555.3687834739685,0.9420394021221715,18094
+1,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,22.14082852319187,18025.636202573776,18025.636202573776,0.40403201664468946,72000
+10,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,22.23496029572029,18027.61917901039,18027.61917901039,0.40403201664468946,72000
+19,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,22.247438140097906,18025.89689064026,18025.89689064026,0.40403201664468946,72000
+6,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.60218,59997.68517279625,inf,0.6740027356507963,195999
+15,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.61658,73926.78414797783,inf,0.6740027356507963,195999
+23,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.61518,73948.011013031,inf,0.6740027356507963,195999
+4,imagenet_vit_pytorch,trial_1,validation/accuracy,0.77309,False,0.61544,95268.04540586472,inf,0.802424341129202,167999
+13,imagenet_vit_pytorch,trial_1,validation/accuracy,0.77309,False,0.61512,92620.5374662876,inf,0.802424341129202,167999
+2,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,True,0.0848356104861681,54136.87689781189,54136.87689781189,0.7305170534584913,76000
+11,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,True,0.0838990006276251,48894.85426735878,48894.85426735878,0.7305170534584913,76000
+20,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,True,0.0842659199536522,48888.971920251846,48888.971920251846,0.7305170534584913,76000
+8,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,False,0.1200695215565104,34737.411699295044,inf,0.7884586655586601,38400
+17,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,True,0.1185245980785014,28953.85911822319,28953.85911822319,0.7884586655586601,38400
+25,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,False,0.1215758219475691,33312.74022984505,inf,0.7884586655586601,38400
+3,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.2872654622003903,7232.420216321945,6329.966245889664,0.2304398308491926,52000
+12,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.2920392515125629,8137.10068655014,6332.58651471138,0.2304398308491926,52000
+21,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.283304967314845,5891.7984273433685,5891.7984273433685,0.2304398308491926,52000
+0,wmt_pytorch,trial_1,validation/bleu,30.8491,True,30.87605698677609,17399.639942407608,17399.639942407608,0.1538000967767499,120000
+9,wmt_pytorch,trial_1,validation/bleu,30.8491,False,30.80807424915632,19338.09857201576,inf,0.1538000967767499,120000
+18,wmt_pytorch,trial_1,validation/bleu,30.8491,True,30.867059390411267,16752.356283187866,16752.356283187866,0.1538000967767499,120000

artifacts/leaderboard_v2/cautious_nadamw_summary.csv

@@ -0,0 +1,26 @@
+,workload,trial,val target metric name,val target metric value,val target reached,best metric value on val,time to best eval on val (s),time to target on val (s),step_time (s),step_hint
+1,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1256997472383533,2863.6579899787903,inf,0.9266347767470695,10666
+9,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1258371449566668,1792.0047852993011,inf,0.9266347767470695,10666
+18,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1246944847459977,9989.461620092392,inf,0.9266347767470695,10666
+4,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7244385315181837,1608.188658952713,1608.188658952713,0.6559752236253,18094
+12,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7238933024409117,1383.1596400737762,1383.1596400737762,0.6559752236253,18094
+21,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7236819978369443,2571.815328359604,2571.815328359604,0.6559752236253,18094
+17,finewebedu_lm_jax,trial_1,validation/ppl,22.2995,True,21.63268571712524,25738.981688261032,25738.981688261032,0.44144648333990966,72000
+6,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.7466,69885.5102982521,inf,0.25149491638964216,195999
+14,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.74748,67896.18414497375,inf,0.25149491638964216,195999
+23,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.74964,61917.01934456825,inf,0.25149491638964216,195999
+5,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,False,0.7506,87452.66640520096,inf,0.49596923795226744,167999
+13,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,False,0.75158,82308.66560220718,inf,0.49596923795226744,167999
+22,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,False,0.75034,87448.36735343933,inf,0.49596923795226744,167999
+0,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1421619272920388,64988.59479641914,inf,2.589838852085477,76000
+8,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1426254043354415,65012.818663835526,inf,2.589838852085477,76000
+16,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1444986240525274,64979.35094237328,inf,2.589838852085477,76000
+2,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.2943175783324482,10338.725378990172,inf,2.7988120351912658,38400
+10,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.2872881765075074,10334.769649744034,inf,2.7988120351912658,38400
+19,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.2853473663882586,14680.395282268524,inf,2.7988120351912658,38400
+3,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,True,0.2845644139519467,5444.764040708542,4992.686820983887,0.1979359465075458,52000
+11,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,True,0.2874222775151546,5446.810508966446,4542.640103340149,0.1979359465075458,52000
+20,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,True,0.2809900831846614,4995.647005558014,4995.647005558014,0.1979359465075458,52000
+7,wmt_jax,trial_1,validation/bleu,30.8491,True,30.876108999624343,12943.72738480568,12943.72738480568,0.1255462995675322,120000
+15,wmt_jax,trial_1,validation/bleu,30.8491,True,31.02074381397432,14876.415894269943,14232.458706855774,0.1255462995675322,120000
+24,wmt_jax,trial_1,validation/bleu,30.8491,True,30.91251520933702,15517.656443834305,15517.656443834305,0.1255462995675322,120000

artifacts/leaderboard_v2/leaderboard_steps_table.tex

@@ -0,0 +1,27 @@
+\begin{table}[htbp]
+  \centering
+  \caption{Wall-clock vs.\ step-based benchmark scores. Both use the same performance-profile scoring; only the notion of training time differs (seconds vs.\ optimizer steps to target). $\Delta$ is the rank change when moving from wall-clock to step-based scoring; Cautious NAdamW and Muon (JAX) tie exactly under step-based scoring.}
+  \label{tab:leaderboard_steps}
+  \begin{tabular}{lrrrrc}
+    \toprule
+    & \multicolumn{2}{c}{Wall-clock} & \multicolumn{2}{c}{Steps} & \\
+    \cmidrule(lr){2-3}\cmidrule(lr){4-5}
+    Submission & Score & Rank & Score & Rank & $\Delta$ \\
+    \midrule
+    Schedule-Free AdamW v2 & \textbf{0.5679} & 1 & 0.5253 & 2 & $\downarrow$1 \\
+    AdEMAMix & 0.5191 & 2 & \textbf{0.5258} & 1 & $\uparrow$1 \\
+    Schedule-Free AdamW & 0.4669 & 3 & 0.4461 & 4 & $\downarrow$1 \\
+    NAdamW (Baseline v0.5) & 0.4506 & 4 & 0.4545 & 3 & $\uparrow$1 \\
+    Schedule-Free AdamW (JAX v2) & 0.4405 & 5 & 0.4125 & 6 & $\downarrow$1 \\
+    Muon (PyTorch) & 0.4231 & 6 & 0.4327 & 5 & $\uparrow$1 \\
+    Schedule-Free AdamW (JAX) & 0.4108 & 7 & 0.4091 & 7 & -- \\
+    NAdamW & 0.3844 & 8 & 0.4040 & 8 & -- \\
+    Cautious NAdamW & 0.3143 & 9 & 0.3199 & 9 & -- \\
+    Muon (JAX) & 0.2845 & 10 & 0.3199 & 9 & $\uparrow$1 \\
+    Lion & 0.2783 & 11 & 0.2733 & 11 & -- \\
+    NAdamW (ResNet) & 0.1857 & 12 & 0.1953 & 12 & -- \\
+    DiLoCo (Single Worker) & 0.1369 & 13 & 0.1863 & 13 & -- \\
+    DiLoCo v2 (Single Worker) & 0.1302 & 14 & 0.1829 & 14 & -- \\
+    \bottomrule
+  \end{tabular}
+\end{table}

artifacts/leaderboard_v2/lion_summary.csv

@@ -0,0 +1,28 @@
+,workload,trial,val target metric name,val target metric value,val target reached,best metric value on val,time to best eval on val (s),time to target on val (s),step_time (s),step_hint
+7,criteo1tb_pytorch,trial_1,validation/loss,0.123735,False,0.1241905161220999,9978.416061878204,inf,0.91592558011244,10666
+16,criteo1tb_pytorch,trial_1,validation/loss,0.123735,False,0.1242616643239586,12466.455169200895,inf,0.91592558011244,10666
+25,criteo1tb_pytorch,trial_1,validation/loss,0.123735,False,0.1242412303757036,12106.893656492231,inf,0.91592558011244,10666
+5,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.7238031751195836,619.9493722915649,619.9493722915649,0.27376442936493867,18094
+14,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.7243804158826322,1421.2225756645205,1421.2225756645205,0.27376442936493867,18094
+23,fastmri_pytorch,trial_1,validation/ssim,0.723653,True,0.7239882383845667,1394.1750195026398,1394.1750195026398,0.27376442936493867,18094
+1,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,21.261897266260053,23164.569310188293,23164.569310188293,0.38839825067830164,72000
+10,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,21.110008038411504,23177.09142231941,23177.09142231941,0.38839825067830164,72000
+19,finewebedu_lm_pytorch,trial_1,validation/ppl,22.2995,True,21.130699319951358,23172.039316177368,23172.039316177368,0.38839825067830164,72000
+6,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.61338,73980.61954474449,inf,0.6743097256486612,195999
+15,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.61892,67952.96683573723,inf,0.6743097256486612,195999
+24,imagenet_resnet_pytorch,trial_1,validation/accuracy,0.77431,False,0.62902,67951.73512744904,inf,0.6743097256486612,195999
+4,imagenet_vit_pytorch,trial_1,validation/accuracy,0.77309,False,0.6242,87448.87829089165,inf,0.8120775926593171,167999
+13,imagenet_vit_pytorch,trial_1,validation/accuracy,0.77309,False,0.63292,95191.34969496728,inf,0.8120775926593171,167999
+22,imagenet_vit_pytorch,trial_1,validation/accuracy,0.77309,False,0.6418,95165.0220131874,inf,0.8120775926593171,167999
+2,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,False,0.1153188818616327,57621.20974063873,inf,0.7231872968606753,76000
+11,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,False,0.1507845314536764,54127.30689024925,inf,0.7231872968606753,76000
+20,librispeech_conformer_pytorch,trial_1,validation/wer,0.085884,False,0.1264423309032974,57621.14528346062,inf,0.7231872968606753,76000
+8,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,False,0.1247042919905373,31862.61851024628,inf,0.7694697594617248,38400
+17,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,False,0.126799594457587,31838.147053956985,inf,0.7694697594617248,38400
+26,librispeech_deepspeech_pytorch,trial_1,validation/wer,0.119936,False,0.1238063052189446,31852.81674218177,inf,0.7694697594617248,38400
+3,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.2857323411413317,7223.987207174301,7223.987207174301,0.21286914966307793,52000
+12,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.2812007204404579,6782.790376663208,6782.790376663208,0.21286914966307793,52000
+21,ogbg_pytorch,trial_1,validation/mean_average_precision,0.28098,True,0.283014990979593,8582.123725652695,8582.123725652695,0.21286914966307793,52000
+0,wmt_pytorch,trial_1,validation/bleu,30.8491,True,30.948023339872734,15475.552730083466,15475.552730083466,0.1413890021270408,120000
+9,wmt_pytorch,trial_1,validation/bleu,30.8491,True,30.87531810470684,17401.493114233017,15474.955520391464,0.1413890021270408,120000
+18,wmt_pytorch,trial_1,validation/bleu,30.8491,False,30.76188896433939,17388.50763297081,inf,0.1413890021270408,120000

artifacts/leaderboard_v2/muon_summary.csv

@@ -0,0 +1,28 @@
+,workload,trial,val target metric name,val target metric value,val target reached,best metric value on val,time to best eval on val (s),time to target on val (s),step_time (s),step_hint
+2,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1245190224490917,13217.0788064003,inf,1.80744596371582,10666
+11,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1246482386019362,12154.283058166504,inf,1.80744596371582,10666
+20,criteo1tb_jax,trial_1,validation/loss,0.123735,False,0.1245746128447548,13221.646473169329,inf,1.80744596371582,10666
+5,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7239323209763646,1145.923245429993,1145.923245429993,0.15300968625117742,18094
+14,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7238764035681626,2594.490219116211,2594.490219116211,0.15300968625117742,18094
+23,fastmri_jax,trial_1,validation/ssim,0.723653,True,0.7247154394696117,1869.765276670456,1869.765276670456,0.15300968625117742,18094
+1,finewebedu_lm_jax,trial_1,validation/ppl,22.2995,True,21.808373188667463,20594.037749767303,20594.037749767303,0.504613462023225,72000
+10,finewebedu_lm_jax,trial_1,validation/ppl,22.2995,True,21.85239511139204,20594.19531917572,20594.19531917572,0.504613462023225,72000
+19,finewebedu_lm_jax,trial_1,validation/ppl,22.2995,True,21.82093364841247,20593.710190296173,20593.710190296173,0.504613462023225,72000
+7,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.73021996,73908.46513557434,inf,0.34063982264727855,195999
+16,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.73056,53931.644364357,inf,0.34063982264727855,195999
+25,imagenet_resnet_jax,trial_1,validation/accuracy,0.77431,False,0.71808,65941.6421945095,inf,0.34063982264727855,195999
+6,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,True,0.77518,77162.91657114029,77162.91657114029,0.5145936103030498,167999
+15,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,False,0.75193995,95183.25943279266,inf,0.5145936103030498,167999
+24,imagenet_vit_jax,trial_1,validation/accuracy,0.77309,True,0.77476,74592.82279109955,74592.82279109955,0.5145936103030498,167999
+0,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1673441799835852,64973.90770673752,inf,2.486110555855915,76000
+9,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1665524067011055,64973.39529895783,inf,2.486110555855915,76000
+18,librispeech_conformer_jax,trial_1,validation/wer,0.085884,False,0.1644281369188432,64948.560970783234,inf,2.486110555855915,76000
+3,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.174721189591078,55211.92119574547,inf,2.709749883580429,38400
+12,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.1775792980253946,55245.24317121506,inf,2.709749883580429,38400
+21,librispeech_deepspeech_jax,trial_1,validation/wer,0.119936,False,0.1763819823299377,53775.95950937271,inf,2.709749883580429,38400
+4,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,False,0.279643299276242,1828.2280414104464,inf,0.19192375124386668,52000
+13,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,False,0.2754040150725418,1378.5888230800629,inf,0.19192375124386668,52000
+22,ogbg_jax,trial_1,validation/mean_average_precision,0.28098,False,0.2761794983317486,1827.266467809677,inf,0.19192375124386668,52000
+8,wmt_jax,trial_1,validation/bleu,30.8491,False,28.997438769958425,12296.452765226364,inf,0.173934285073483,120000
+17,wmt_jax,trial_1,validation/bleu,30.8491,False,28.65403612242089,11011.619378566742,inf,0.173934285073483,120000
+26,wmt_jax,trial_1,validation/bleu,30.8491,False,28.777579127157672,12941.239448785782,inf,0.173934285073483,120000