fix: treat sentiment-analysis and text-classification as the same evaluator task

src/evaluate/evaluator/base.py

@@ -32,6 +32,7 @@
 try:
     import transformers
     from transformers import Pipeline, pipeline
+    from transformers.pipelines import TASK_ALIASES
 
     TRANSFORMERS_AVAILABLE = True
 except ImportError:
@@ -208,6 +209,11 @@ def _infer_device() -> int:
 
         return device
 
+    @staticmethod
+    def _normalize_task(task: str) -> str:
+        """Helper function to resolve a task alias (e.g. `"sentiment-analysis"`) to its canonical task name."""
+        return TASK_ALIASES.get(task, task)
+
     @abstractmethod
     def predictions_processor(self, *args, **kwargs):
         """
@@ -471,7 +477,9 @@ def prepare_pipeline(
                 pipe = model_or_pipeline
             if tokenizer is not None and feature_extractor is not None:
                 logger.warning("Ignoring the value of the preprocessor argument (`tokenizer` or `feature_extractor`).")
-        if (pipe.task != self.task) and not (self.task == "translation" and pipe.task.startswith("translation")):
+        if (self._normalize_task(pipe.task) != self._normalize_task(self.task)) and not (
+            self.task == "translation" and pipe.task.startswith("translation")
+        ):
             raise ValueError(
                 f"Incompatible `model_or_pipeline`. Please specify `model_or_pipeline` compatible with the `{self.task}` task."
             )

tests/test_evaluator.py

@@ -69,8 +69,8 @@ def __call__(self, inputs, **kwargs):
 
 
 class DummyTextClassificationPipeline:
-    def __init__(self, sleep_time=None):
-        self.task = "text-classification"
+    def __init__(self, sleep_time=None, task="text-classification"):
+        self.task = task
         self.sleep_time = sleep_time
 
     def __call__(self, inputs, **kwargs):
@@ -260,6 +260,22 @@ def test_pipe_init(self):
         )
         self.assertEqual(results["accuracy"], 1.0)
 
+    def test_task_alias_pipe_init(self):
+        # `sentiment-analysis` is an alias of `text-classification`, so both names have to be accepted
+        # on the evaluator side as well as on the pipeline side
+        for evaluator_task, pipe_task in [
+            ("sentiment-analysis", "text-classification"),
+            ("text-classification", "sentiment-analysis"),
+        ]:
+            results = evaluator(evaluator_task).compute(
+                model_or_pipeline=DummyTextClassificationPipeline(task=pipe_task),
+                data=self.data,
+                input_column="text",
+                label_column="label",
+                label_mapping=self.label_mapping,
+            )
+            self.assertEqual(results["accuracy"], 1.0)
+
     @slow
     def test_model_init(self):
         results = self.evaluator.compute(