diff --git a/measurements/text_duplicates/text_duplicates.py b/measurements/text_duplicates/text_duplicates.py index 013646440..a1f0a85d8 100644 --- a/measurements/text_duplicates/text_duplicates.py +++ b/measurements/text_duplicates/text_duplicates.py @@ -82,7 +82,7 @@ def _compute(self, data, list_duplicates=False): if list_duplicates == True: logger.warning("This functionality can be memory-intensive for large datasets!") n_dedup = len(set([get_hash(d) for d in data])) - c = Counter(data) + c = Counter(d.strip() for d in data) duplicates = {k: v for k, v in c.items() if v > 1} return {"duplicate_fraction": 1 - (n_dedup / len(data)), "duplicates_dict": duplicates} else: diff --git a/tests/test_metric_common.py b/tests/test_metric_common.py index 014dc0b32..b6d711ec8 100644 --- a/tests/test_metric_common.py +++ b/tests/test_metric_common.py @@ -225,3 +225,11 @@ def test_seqeval_raises_when_incorrect_scheme(): error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}" with pytest.raises(ValueError, match=re.escape(error_message)): metric.compute(predictions=[], references=[], scheme=wrong_scheme) + + +def test_text_duplicates_lists_the_duplicates_it_counts(): + measurement = load(os.path.join("measurements", "text_duplicates")) + # the fraction ignores surrounding whitespace, so the listed duplicates have to as well + results = measurement.compute(data=["hello sun", "hello sun ", "hello moon"], list_duplicates=True) + assert results["duplicate_fraction"] == pytest.approx(1 / 3) + assert results["duplicates_dict"] == {"hello sun": 2}