Fix McNemar ZeroDivisionError when models have no discordant pairs

comparisons/mcnemar/README.md

@@ -51,7 +51,7 @@ Its arguments are:
 
 The McNemar comparison outputs two things:
 
-`stat`: The McNemar statistic.
+`stat`: The McNemar statistic. It is 0 when both models are correct and incorrect on exactly the same examples, since there are no discordant pairs to compare in that case.
 
 `p`: The p value.
 

comparisons/mcnemar/mcnemar.py

@@ -35,7 +35,7 @@
     references (`list` of `int`): Ground truth labels.
 
 Returns:
-    stat (`float`): McNemar test score.
+    stat (`float`): McNemar test score. It is 0 when both models are correct and incorrect on exactly the same examples, since there are no discordant pairs to compare in that case.
     p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference.
 
 Examples:
@@ -92,7 +92,9 @@ def _compute(self, predictions1, predictions2, references):
 
         # compute statistic
         b, c = tbl[0][1], tbl[1][0]
-        statistic = abs(b - c) ** 2 / (1.0 * (b + c))
+        # without discordant pairs both models are right and wrong on exactly the same
+        # examples, so there is no evidence of a difference between them
+        statistic = abs(b - c) ** 2 / (1.0 * (b + c)) if b + c > 0 else 0.0
         df = 1
         pvalue = chi2.sf(statistic, df)
         return {"stat": statistic, "p": pvalue}

tests/test_metric_common.py

@@ -225,3 +225,9 @@ def test_seqeval_raises_when_incorrect_scheme():
     error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}"
     with pytest.raises(ValueError, match=re.escape(error_message)):
         metric.compute(predictions=[], references=[], scheme=wrong_scheme)
+
+
+def test_mcnemar_without_discordant_pairs():
+    comparison = load(os.path.join("comparisons", "mcnemar"))
+    results = comparison.compute(references=[1, 0, 1], predictions1=[1, 0, 1], predictions2=[1, 0, 1])
+    assert results == {"stat": 0.0, "p": 1.0}