From 1cc7bb9613320853ee1015c20752ed80de978d83 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 10:15:54 +0000 Subject: [PATCH] fix: avoid ZeroDivisionError in mcnemar without discordant pairs McNemar's statistic divides by the number of discordant pairs, so comparing two models that are correct and incorrect on exactly the same examples raised a ZeroDivisionError. Report a statistic of 0.0 (and therefore a p value of 1.0) instead, since there is no evidence of a difference in that case. Co-authored-by: Tony Coder <407243179@qq.com> --- comparisons/mcnemar/README.md | 2 +- comparisons/mcnemar/mcnemar.py | 6 ++++-- tests/test_metric_common.py | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/comparisons/mcnemar/README.md b/comparisons/mcnemar/README.md index 1ceaaee21..09d06cd20 100644 --- a/comparisons/mcnemar/README.md +++ b/comparisons/mcnemar/README.md @@ -51,7 +51,7 @@ Its arguments are: The McNemar comparison outputs two things: -`stat`: The McNemar statistic. +`stat`: The McNemar statistic. It is 0 when both models are correct and incorrect on exactly the same examples, since there are no discordant pairs to compare in that case. `p`: The p value. diff --git a/comparisons/mcnemar/mcnemar.py b/comparisons/mcnemar/mcnemar.py index 86b85b5e3..160ad12d4 100644 --- a/comparisons/mcnemar/mcnemar.py +++ b/comparisons/mcnemar/mcnemar.py @@ -35,7 +35,7 @@ references (`list` of `int`): Ground truth labels. Returns: - stat (`float`): McNemar test score. + stat (`float`): McNemar test score. It is 0 when both models are correct and incorrect on exactly the same examples, since there are no discordant pairs to compare in that case. p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference. Examples: @@ -92,7 +92,9 @@ def _compute(self, predictions1, predictions2, references): # compute statistic b, c = tbl[0][1], tbl[1][0] - statistic = abs(b - c) ** 2 / (1.0 * (b + c)) + # without discordant pairs both models are right and wrong on exactly the same + # examples, so there is no evidence of a difference between them + statistic = abs(b - c) ** 2 / (1.0 * (b + c)) if b + c > 0 else 0.0 df = 1 pvalue = chi2.sf(statistic, df) return {"stat": statistic, "p": pvalue} diff --git a/tests/test_metric_common.py b/tests/test_metric_common.py index 014dc0b32..65bbc3fb0 100644 --- a/tests/test_metric_common.py +++ b/tests/test_metric_common.py @@ -225,3 +225,9 @@ def test_seqeval_raises_when_incorrect_scheme(): error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}" with pytest.raises(ValueError, match=re.escape(error_message)): metric.compute(predictions=[], references=[], scheme=wrong_scheme) + + +def test_mcnemar_without_discordant_pairs(): + comparison = load(os.path.join("comparisons", "mcnemar")) + results = comparison.compute(references=[1, 0, 1], predictions1=[1, 0, 1], predictions2=[1, 0, 1]) + assert results == {"stat": 0.0, "p": 1.0}