fix(eval): stop zero-variance Cohen's d from reporting no effect

.github/workflows/ci.yml

@@ -112,13 +112,12 @@ jobs:
       # test_joint_embedding.py imports mlx.core (Apple Silicon only) and would
       # fail collection here — run it locally.
       #
-      # The three deselected cases fail on this base and are tracked separately;
-      # they are pre-existing (issue #38), reproduce on both NumPy 1.26 and 2.4, and
-      # are not caused by this workflow. The rest of their files still run.
+      # The remaining deselected cases are pre-existing failures tracked in
+      # issue #38, reproducing on both NumPy 1.26 and 2.4. They are removed one
+      # at a time as each is resolved; the rest of their files still run.
       - name: Test (eval contract suites)
         run: |
           pytest Tests/eval -v \
             --ignore=Tests/eval/test_joint_embedding.py \
-            --deselect Tests/eval/test_eval_stats.py::test_cohens_d_large_effect \
             --deselect Tests/eval/test_eval_stats.py::test_mann_whitney_u_disjoint \
             --deselect Tests/eval/test_embedding_space.py::test_cka_independent

Evaluation/scripts/analyze_results.py

@@ -14,9 +14,13 @@
 import json
 import math
 import statistics
+import sys
 from pathlib import Path
 
 import numpy as np
+
+sys.path.insert(0, str(Path(__file__).parent))
+from eval_stats import cohens_d
 import pandas as pd
 
 REPO_ROOT = Path(__file__).resolve().parent.parent.parent
@@ -39,17 +43,6 @@ def bootstrap_ci(data, confidence=0.95, n_boot=10000):
     return (lo, hi)
 
 
-def cohens_d(a, b):
-    """Cohen's d effect size (pooled SD)."""
-    a, b = np.array(a, dtype=float), np.array(b, dtype=float)
-    if len(a) < 2 or len(b) < 2:
-        return float("nan")
-    pooled_std = math.sqrt((a.var(ddof=1) + b.var(ddof=1)) / 2)
-    if pooled_std == 0:
-        return 0.0
-    return float((a.mean() - b.mean()) / pooled_std)
-
-
 def safe_mean(values):
     return float(np.nanmean(values)) if len(values) > 0 else float("nan")
 

Evaluation/scripts/embedding_analyze.py

@@ -27,6 +27,9 @@
 
 import numpy as np
 from scipy import stats as sp_stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from eval_stats import cohens_d
 from sklearn.cluster import KMeans
 from sklearn.preprocessing import StandardScaler
 
@@ -50,16 +53,6 @@ def bootstrap_ci(data, confidence=0.95, n_boot=10000):
     return (lo, hi, len(data))
 
 
-def cohens_d(a, b):
-    a, b = np.array(a, dtype=float), np.array(b, dtype=float)
-    if len(a) < 2 or len(b) < 2:
-        return float("nan")
-    pooled_std = math.sqrt((a.var(ddof=1) + b.var(ddof=1)) / 2)
-    if pooled_std == 0:
-        return 0.0
-    return float((a.mean() - b.mean()) / pooled_std)
-
-
 def mann_whitney_u(a, b):
     a, b = np.array(a, dtype=float), np.array(b, dtype=float)
     if len(a) < 2 or len(b) < 2:

Evaluation/scripts/eval_stats.py

@@ -32,14 +32,20 @@ def bootstrap_ci(data, confidence=0.95, n_boot=10000):
 
 
 def cohens_d(a, b):
-    """Cohen's d effect size between two samples."""
+    """Cohen's d effect size between two samples.
+
+    Returns NaN where d is undefined rather than a finite value that would rank
+    alongside real effects: under-sized samples, and zero pooled variance with
+    separated means. Only equal constant groups are a true zero.
+    """
     a, b = np.array(a, dtype=float), np.array(b, dtype=float)
     if len(a) < 2 or len(b) < 2:
         return float("nan")
+    mean_difference = float(a.mean() - b.mean())
     pooled_std = math.sqrt((a.var(ddof=1) + b.var(ddof=1)) / 2)
     if pooled_std == 0:
-        return 0.0
-    return float((a.mean() - b.mean()) / pooled_std)
+        return 0.0 if mean_difference == 0 else float("nan")
+    return mean_difference / pooled_std
 
 
 def mann_whitney_u(a, b):

Evaluation/scripts/statistical_analysis.py

@@ -31,6 +31,9 @@
 
 import numpy as np
 from scipy import stats as sp_stats
+
+sys.path.insert(0, str(Path(__file__).parent))
+from eval_stats import cohens_d
 from sklearn.cluster import KMeans
 from sklearn.preprocessing import StandardScaler
 
@@ -55,17 +58,6 @@ def bootstrap_ci(data, confidence=0.95, n_boot=10000):
     return (lo, hi, len(data))
 
 
-def cohens_d(a, b):
-    """Cohen's d effect size with pooled SD."""
-    a, b = np.array(a, dtype=float), np.array(b, dtype=float)
-    if len(a) < 2 or len(b) < 2:
-        return float("nan")
-    pooled_std = math.sqrt((a.var(ddof=1) + b.var(ddof=1)) / 2)
-    if pooled_std == 0:
-        return 0.0
-    return float((a.mean() - b.mean()) / pooled_std)
-
-
 def mann_whitney_u(a, b):
     """Mann-Whitney U test — non-parametric, no normality assumption."""
     a, b = np.array(a, dtype=float), np.array(b, dtype=float)

Tests/eval/test_eval_stats.py

@@ -26,9 +26,19 @@ def test_cohens_d_identical():
     assert d == 0.0
 
 
-def test_cohens_d_large_effect():
-    d = cohens_d([1, 1, 1, 1], [5, 5, 5, 5])
-    assert d > 2.0  # very large effect
+def test_cohens_d_identical_constant_groups_are_zero():
+    assert cohens_d([1, 1, 1, 1], [1, 1, 1, 1]) == 0.0
+
+
+def test_cohens_d_separated_constant_groups_are_undefined():
+    # Zero pooled variance with separated means: d is unbounded, not absent.
+    # Returning 0.0 here would rank total separation as "no effect".
+    assert np.isnan(cohens_d([1, 1, 1, 1], [5, 5, 5, 5]))
+
+
+def test_cohens_d_large_nondegenerate_effect():
+    d = cohens_d([0.9, 1.0, 1.1, 1.0], [4.9, 5.0, 5.1, 5.0])
+    assert abs(d) > 2.0
 
 
 def test_mann_whitney_u_disjoint():