From a2242f0dd8b2ebd99e4cf6954922d8a7a63c4027 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 10:26:54 +0000 Subject: [PATCH] Don't let generation_kwargs leak into the class-level PIPELINE_KWARGS Text2TextGenerationEvaluator.compute() updated the class attribute PIPELINE_KWARGS in place, so generation_kwargs passed to one call kept being forwarded to the pipeline on every later call and on every other instance of the evaluator. Co-authored-by: Tony Coder <407243179@qq.com> --- .../evaluator/text2text_generation.py | 5 ++-- tests/test_evaluator.py | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/evaluate/evaluator/text2text_generation.py b/src/evaluate/evaluator/text2text_generation.py index 6dfd2c035..5e6a6f4a0 100644 --- a/src/evaluate/evaluator/text2text_generation.py +++ b/src/evaluate/evaluator/text2text_generation.py @@ -127,8 +127,9 @@ def compute( label_column: str = "label", generation_kwargs: dict = None, ) -> Tuple[Dict[str, float], Any]: - if generation_kwargs is not None: - self.PIPELINE_KWARGS.update(generation_kwargs) + # `PIPELINE_KWARGS` is a class attribute, so it is shadowed with a per-call copy to keep + # `generation_kwargs` from leaking into later calls and into other evaluator instances. + self.PIPELINE_KWARGS = {**type(self).PIPELINE_KWARGS, **(generation_kwargs or {})} result = super().compute( model_or_pipeline=model_or_pipeline, diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 259b5c7b9..4d04afa28 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -63,8 +63,10 @@ class DummyText2TextGenerationPipeline: def __init__(self, prefix="generated", task="text2text-generation"): self.task = task self.prefix = prefix + self.call_kwargs = None def __call__(self, inputs, **kwargs): + self.call_kwargs = kwargs return [{f"{self.prefix}_text": "Lorem ipsum"} for _ in inputs] @@ -960,6 +962,28 @@ def test_overwrite_default_metric(self): ) self.assertEqual(results["rouge1"], 1.0) + def test_generation_kwargs(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + generation_kwargs={"max_length": 5}, + ) + self.assertEqual(self.pipe.call_kwargs, {"truncation": True, "max_length": 5}) + + def test_generation_kwargs_are_not_persisted(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + generation_kwargs={"max_length": 5}, + ) + self.assertEqual(Text2TextGenerationEvaluator.PIPELINE_KWARGS, {"truncation": True}) + + evaluator("text2text-generation").compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(self.pipe.call_kwargs, {"truncation": True}) + def test_summarization(self): pipe = DummyText2TextGenerationPipeline(task="summarization", prefix="summary") e = evaluator("summarization")