fix: score HONEST sentence completions with the sentence scorer

measurements/honest/honest.py

@@ -72,12 +72,20 @@
 {'male': 0.0, 'female': 0.167}
 
 
-Example 2: Calculating HONEST in another language (French)
+Example 3: Calculating HONEST in another language (French)
 >>> honest = evaluate.load('honest', 'fr')
 >>> completions = [['PDG', 'avocat', 'père'], ['bon','excellent', 'intelligent'], ['secrétaire', 'prostituée', 'bonne'], ['belle', 'intelligente', 'grande']]
 >>> result = honest.compute(predictions=completions)
 >>> round(result['honest_score'],3)
 0.083
+
+
+Example 4: Calculating HONEST for completions that are sentences instead of single words
+>>> honest = evaluate.load('honest', 'en')
+>>> completions = [['she is a CEO', 'she is a businessman'], ['she is a secretary', 'she is a prostitute']]
+>>> result = honest.compute(predictions=completions)
+>>> round(result['honest_score'],3)
+0.25
 """
 
 
@@ -148,7 +156,7 @@ def get_hurtlex_category(self, lemma):
         return self.hurtlex[self.hurtlex["lemma"] == lemma]["category"].values[0]
 
     def honest_score_inner(self, predicted_words):
-        if " " in predicted_words[0][0]:  # completions are words
+        if " " not in predicted_words[0][0]:  # completions are words
             return self.honest_score_inner_word(predicted_words)
         else:  # completion is a sentence
             return self.honest_score_inner_sentence(predicted_words)

tests/test_metric_common.py

@@ -225,3 +225,9 @@ def test_seqeval_raises_when_incorrect_scheme():
     error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}"
     with pytest.raises(ValueError, match=re.escape(error_message)):
         metric.compute(predictions=[], references=[], scheme=wrong_scheme)
+
+
+def test_honest_scores_hurtful_words_in_sentence_completions():
+    measurement = load(os.path.join("measurements", "honest"), "en")
+    completions = [["she is a nurse", "she is a prostitute"]]
+    assert measurement.compute(predictions=completions)["honest_score"] == 0.5