Allow Infidelity to accept all-negative attributions

pytest.ini

@@ -10,6 +10,7 @@ markers =
     n_bins_func: n_bins_func tests.
     evaluate_func: evaluate tests.
     utils: utils tests.
+    asserts: asserts tests.
     fixes: fixing tests.
     pytorch_model: pytorch model interface tests.
     tf_model: tensorflow model interface tests.

quantus/helpers/asserts.py

@@ -128,7 +128,9 @@ def assert_layer_order(layer_order: str) -> None:
     assert layer_order in ["top_down", "bottom_up", "independent"]
 
 
-def assert_attributions(x_batch: np.array, a_batch: np.array) -> None:
+def assert_attributions(
+    x_batch: np.array, a_batch: np.array, check_all_negative: bool = True
+) -> None:
     """
     Asserts on attributions, assumes channel first layout.
 
@@ -138,6 +140,12 @@ def assert_attributions(x_batch: np.array, a_batch: np.array) -> None:
          The batch of input to compare the shape of the attributions with.
     a_batch: np.ndarray
          The batch of attributions.
+    check_all_negative: boolean
+         Indicates whether to assert that the attributions are not all
+         negative. Some metrics (e.g. Infidelity) use attributions in a
+         signed dot product and have no mathematical requirement for
+         non-negative values, so this check can be disabled for them via
+         `Metric.allow_negative_attributions`. Default=True.
 
     Returns
     -------
@@ -193,7 +201,10 @@ def assert_attributions(x_batch: np.array, a_batch: np.array) -> None:
         "metrics rely on ordering."
         "Recompute the explanations."
     )
-    assert not np.all((a_batch < 0.0)), "Attributions should not all be less than zero."
+    if check_all_negative:
+        assert not np.all(
+            (a_batch < 0.0)
+        ), "Attributions should not all be less than zero."
 
 
 def assert_segmentations(x_batch: np.array, s_batch: np.array) -> None:

quantus/metrics/base.py

@@ -70,6 +70,10 @@ class Metric(Generic[R]):
     model_applicability: ClassVar[Set[ModelType]]
     score_direction: ClassVar[ScoreDirection]
     evaluation_category: ClassVar[EvaluationCategory]
+    # Most metrics only make sense for non-negative attributions; metrics whose
+    # computation is well-defined for signed attributions (e.g. Infidelity, which
+    # uses a_batch in a signed dot product) opt out by overriding this to True.
+    allow_negative_attributions: ClassVar[bool] = False
 
     # Instance attributes.
     explain_func: Callable
@@ -445,7 +449,11 @@ def general_preprocess(
 
         if a_batch is not None:
             a_batch = utils.expand_attribution_channel(a_batch, x_batch)
-            asserts.assert_attributions(x_batch=x_batch, a_batch=a_batch)
+            asserts.assert_attributions(
+                x_batch=x_batch,
+                a_batch=a_batch,
+                check_all_negative=not self.allow_negative_attributions,
+            )
             self.a_axes = utils.infer_attribution_axes(a_batch, x_batch)
 
             # Normalise with specified keyword arguments if requested.
@@ -928,7 +936,11 @@ def explain_batch(
             model=model, inputs=x_batch, targets=y_batch, **self.explain_func_kwargs
         )
         a_batch = utils.expand_attribution_channel(a_batch, x_batch)
-        asserts.assert_attributions(x_batch=x_batch, a_batch=a_batch)
+        asserts.assert_attributions(
+            x_batch=x_batch,
+            a_batch=a_batch,
+            check_all_negative=not self.allow_negative_attributions,
+        )
 
         # Normalise and take absolute values of the attributions, if configured during metric instantiation.
         if self.normalise:

quantus/metrics/faithfulness/infidelity.py

@@ -59,13 +59,16 @@ class Infidelity(Metric[List[float]]):
         - _models: The model types that this metric can work with.
         - score_direction: How to interpret the scores, whether higher/ lower values are considered better.
         - evaluation_category: What property/ explanation quality that this metric measures.
+        - allow_negative_attributions: Infidelity uses a_batch in a signed dot product
+          (see evaluate_batch), so an all-negative explanation is not a validation error.
     """
 
     name = "Infidelity"
     data_applicability = {DataType.IMAGE}
     model_applicability = {ModelType.TORCH, ModelType.TF}
     score_direction = ScoreDirection.LOWER
     evaluation_category = EvaluationCategory.FAITHFULNESS
+    allow_negative_attributions = True
 
     def __init__(
         self,

tests/helpers/test_asserts.py

@@ -0,0 +1,34 @@
+import numpy as np
+import pytest
+
+from quantus.helpers.asserts import assert_attributions
+
+
+@pytest.fixture
+def x_batch():
+    return np.random.uniform(0, 1, size=(2, 1, 4, 4))
+
+
+@pytest.fixture
+def all_negative_a_batch():
+    return -np.random.uniform(0, 1, size=(2, 1, 4, 4))
+
+
+@pytest.mark.asserts
+def test_assert_attributions_rejects_all_negative_by_default(
+    x_batch, all_negative_a_batch
+):
+    with pytest.raises(AssertionError, match="should not all be less than zero"):
+        assert_attributions(x_batch=x_batch, a_batch=all_negative_a_batch)
+
+
+@pytest.mark.asserts
+def test_assert_attributions_allows_all_negative_when_check_disabled(
+    x_batch, all_negative_a_batch
+):
+    # Should not raise: metrics such as Infidelity use attributions in a signed
+    # dot product and set Metric.allow_negative_attributions = True to opt out
+    # of this specific check.
+    assert_attributions(
+        x_batch=x_batch, a_batch=all_negative_a_batch, check_all_negative=False
+    )

tests/metrics/test_faithfulness_metrics.py

@@ -1713,6 +1713,14 @@ def test_infidelity(
     assert scores is not None, "Test failed."
 
 
+@pytest.mark.faithfulness
+def test_infidelity_allows_negative_attributions():
+    """Infidelity uses a_batch in a signed dot product (see evaluate_batch), so
+    an all-negative explanation is not a validation error, unlike most metrics."""
+    assert Infidelity.allow_negative_attributions is True
+    assert FaithfulnessCorrelation.allow_negative_attributions is False
+
+
 @pytest.mark.faithfulness
 @pytest.mark.parametrize(
     "model,data,params,expected",