From 44c9e602fb643a0fb688981f6f2fbdf6bc9042f7 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 17 Aug 2026 10:51:33 +0000
Subject: [PATCH] Accept pipeline task aliases in the evaluator

`evaluator("sentiment-analysis")` rejected a `text-classification` pipeline (and
vice versa) because `prepare_pipeline` compared the raw task strings, even though
`sentiment-analysis` is a documented alias of `text-classification`. Resolve both
task names through `transformers`' `TASK_ALIASES` before comparing them.

Co-authored-by: Tony Coder <407243179@qq.com>
---
 src/evaluate/evaluator/base.py | 10 +++++++++-
 tests/test_evaluator.py        | 20 ++++++++++++++++++--
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/evaluate/evaluator/base.py b/src/evaluate/evaluator/base.py
index 09de31f1..cef02241 100644
--- a/src/evaluate/evaluator/base.py
+++ b/src/evaluate/evaluator/base.py
@@ -32,6 +32,7 @@
 try:
     import transformers
     from transformers import Pipeline, pipeline
+    from transformers.pipelines import TASK_ALIASES
 
     TRANSFORMERS_AVAILABLE = True
 except ImportError:
@@ -208,6 +209,11 @@ def _infer_device() -> int:
 
         return device
 
+    @staticmethod
+    def _normalize_task(task: str) -> str:
+        """Helper function to resolve a task alias (e.g. `"sentiment-analysis"`) to its canonical task name."""
+        return TASK_ALIASES.get(task, task)
+
     @abstractmethod
     def predictions_processor(self, *args, **kwargs):
         """
@@ -471,7 +477,9 @@ def prepare_pipeline(
                 pipe = model_or_pipeline
             if tokenizer is not None and feature_extractor is not None:
                 logger.warning("Ignoring the value of the preprocessor argument (`tokenizer` or `feature_extractor`).")
-        if (pipe.task != self.task) and not (self.task == "translation" and pipe.task.startswith("translation")):
+        if (self._normalize_task(pipe.task) != self._normalize_task(self.task)) and not (
+            self.task == "translation" and pipe.task.startswith("translation")
+        ):
             raise ValueError(
                 f"Incompatible `model_or_pipeline`. Please specify `model_or_pipeline` compatible with the `{self.task}` task."
             )
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index 259b5c7b..5530d914 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -69,8 +69,8 @@ def __call__(self, inputs, **kwargs):
 
 
 class DummyTextClassificationPipeline:
-    def __init__(self, sleep_time=None):
-        self.task = "text-classification"
+    def __init__(self, sleep_time=None, task="text-classification"):
+        self.task = task
         self.sleep_time = sleep_time
 
     def __call__(self, inputs, **kwargs):
@@ -260,6 +260,22 @@ def test_pipe_init(self):
         )
         self.assertEqual(results["accuracy"], 1.0)
 
+    def test_task_alias_pipe_init(self):
+        # `sentiment-analysis` is an alias of `text-classification`, so both names have to be accepted
+        # on the evaluator side as well as on the pipeline side
+        for evaluator_task, pipe_task in [
+            ("sentiment-analysis", "text-classification"),
+            ("text-classification", "sentiment-analysis"),
+        ]:
+            results = evaluator(evaluator_task).compute(
+                model_or_pipeline=DummyTextClassificationPipeline(task=pipe_task),
+                data=self.data,
+                input_column="text",
+                label_column="label",
+                label_mapping=self.label_mapping,
+            )
+            self.assertEqual(results["accuracy"], 1.0)
+
     @slow
     def test_model_init(self):
         results = self.evaluator.compute(