diff --git a/src/evaluate/evaluator/text2text_generation.py b/src/evaluate/evaluator/text2text_generation.py
index 6dfd2c03..5e6a6f4a 100644
--- a/src/evaluate/evaluator/text2text_generation.py
+++ b/src/evaluate/evaluator/text2text_generation.py
@@ -127,8 +127,9 @@ def compute(
         label_column: str = "label",
         generation_kwargs: dict = None,
     ) -> Tuple[Dict[str, float], Any]:
-        if generation_kwargs is not None:
-            self.PIPELINE_KWARGS.update(generation_kwargs)
+        # `PIPELINE_KWARGS` is a class attribute, so it is shadowed with a per-call copy to keep
+        # `generation_kwargs` from leaking into later calls and into other evaluator instances.
+        self.PIPELINE_KWARGS = {**type(self).PIPELINE_KWARGS, **(generation_kwargs or {})}
 
         result = super().compute(
             model_or_pipeline=model_or_pipeline,
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index 259b5c7b..4d04afa2 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -63,8 +63,10 @@ class DummyText2TextGenerationPipeline:
     def __init__(self, prefix="generated", task="text2text-generation"):
         self.task = task
         self.prefix = prefix
+        self.call_kwargs = None
 
     def __call__(self, inputs, **kwargs):
+        self.call_kwargs = kwargs
         return [{f"{self.prefix}_text": "Lorem ipsum"} for _ in inputs]
 
 
@@ -960,6 +962,28 @@ def test_overwrite_default_metric(self):
         )
         self.assertEqual(results["rouge1"], 1.0)
 
+    def test_generation_kwargs(self):
+        self.evaluator.compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+            generation_kwargs={"max_length": 5},
+        )
+        self.assertEqual(self.pipe.call_kwargs, {"truncation": True, "max_length": 5})
+
+    def test_generation_kwargs_are_not_persisted(self):
+        self.evaluator.compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+            generation_kwargs={"max_length": 5},
+        )
+        self.assertEqual(Text2TextGenerationEvaluator.PIPELINE_KWARGS, {"truncation": True})
+
+        evaluator("text2text-generation").compute(
+            model_or_pipeline=self.pipe,
+            data=self.data,
+        )
+        self.assertEqual(self.pipe.call_kwargs, {"truncation": True})
+
     def test_summarization(self):
         pipe = DummyText2TextGenerationPipeline(task="summarization", prefix="summary")
         e = evaluator("summarization")