From cb6136b63280fb50bc16283407f997e21066659e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 10:57:03 +0000 Subject: [PATCH] Fix ASR generation_kwargs leaking across compute calls AutomaticSpeechRecognitionEvaluator.compute updated the class-level PIPELINE_KWARGS dict in place, so generation_kwargs from one call were still passed to the pipeline on every later call and on every other evaluator instance. Build a per-call copy instead. Co-authored-by: Tony Coder <407243179@qq.com> --- .../evaluator/automatic_speech_recognition.py | 5 ++-- tests/test_evaluator.py | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/evaluate/evaluator/automatic_speech_recognition.py b/src/evaluate/evaluator/automatic_speech_recognition.py index ee423826c..2ba90c94b 100644 --- a/src/evaluate/evaluator/automatic_speech_recognition.py +++ b/src/evaluate/evaluator/automatic_speech_recognition.py @@ -90,8 +90,9 @@ def compute( The generation kwargs are passed to the pipeline and set the text generation strategy. """ - if generation_kwargs is not None: - self.PIPELINE_KWARGS.update(generation_kwargs) + # `PIPELINE_KWARGS` is a class attribute, so it is shadowed with a per-call copy to keep + # `generation_kwargs` from leaking into later calls and into other evaluator instances. + self.PIPELINE_KWARGS = {**type(self).PIPELINE_KWARGS, **(generation_kwargs or {})} result = super().compute( model_or_pipeline=model_or_pipeline, diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 259b5c7b9..02a474437 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -125,8 +125,10 @@ def __call__(self, inputs, **kwargs): class DummyAutomaticSpeechRecognitionPipeline: def __init__(self) -> None: self.task = "automatic-speech-recognition" + self.call_kwargs = None def __call__(self, inputs, **kwargs): + self.call_kwargs = kwargs return [{"text": "Lorem ipsum"} for _ in inputs] @@ -1041,6 +1043,28 @@ def test_overwrite_default_metric(self): ) self.assertEqual(results["cer"], 0.7272727272727273) + def test_generation_kwargs(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + generation_kwargs={"max_new_tokens": 5}, + ) + self.assertEqual(self.pipe.call_kwargs, {"truncation": True, "max_new_tokens": 5}) + + def test_generation_kwargs_are_not_persisted(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + generation_kwargs={"max_new_tokens": 5}, + ) + self.assertEqual(AutomaticSpeechRecognitionEvaluator.PIPELINE_KWARGS, {"truncation": True}) + + evaluator("automatic-speech-recognition").compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(self.pipe.call_kwargs, {"truncation": True}) + class TestAudioClassificationEvaluator(TestCase): def setUp(self):