diff --git a/src/evaluate/evaluator/automatic_speech_recognition.py b/src/evaluate/evaluator/automatic_speech_recognition.py
index ee423826..2ba90c94 100644
--- a/src/evaluate/evaluator/automatic_speech_recognition.py
+++ b/src/evaluate/evaluator/automatic_speech_recognition.py
@@ -90,8 +90,9 @@ def compute(
             The generation kwargs are passed to the pipeline and set the text generation strategy.
         """
 
-        if generation_kwargs is not None:
-            self.PIPELINE_KWARGS.update(generation_kwargs)
+        # `PIPELINE_KWARGS` is a class attribute, so it is shadowed with a per-call copy to keep
+        # `generation_kwargs` from leaking into later calls and into other evaluator instances.
+        self.PIPELINE_KWARGS = {**type(self).PIPELINE_KWARGS, **(generation_kwargs or {})}
 
         result = super().compute(
             model_or_pipeline=model_or_pipeline,
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index 259b5c7b..02a47443 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -125,8 +125,10 @@ def __call__(self, inputs, **kwargs):
 class DummyAutomaticSpeechRecognitionPipeline:
     def __init__(self) -> None:
         self.task = "automatic-speech-recognition"
+        self.call_kwargs = None
 
     def __call__(self, inputs, **kwargs):
+        self.call_kwargs = kwargs
         return [{"text": "Lorem ipsum"} for _ in inputs]
 
 
@@ -1041,6 +1043,28 @@ def test_overwrite_default_metric(self):
         )
         self.assertEqual(results["cer"], 0.7272727272727273)
 
+    def test_generation_kwargs(self):
+        self.evaluator.compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+            generation_kwargs={"max_new_tokens": 5},
+        )
+        self.assertEqual(self.pipe.call_kwargs, {"truncation": True, "max_new_tokens": 5})
+
+    def test_generation_kwargs_are_not_persisted(self):
+        self.evaluator.compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+            generation_kwargs={"max_new_tokens": 5},
+        )
+        self.assertEqual(AutomaticSpeechRecognitionEvaluator.PIPELINE_KWARGS, {"truncation": True})
+
+        evaluator("automatic-speech-recognition").compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+        )
+        self.assertEqual(self.pipe.call_kwargs, {"truncation": True})
+
 
 class TestAudioClassificationEvaluator(TestCase):
     def setUp(self):