From e37dc8c4001f391eea67d9066e8db01b197a5b35 Mon Sep 17 00:00:00 2001
From: edknv <edwardk@nvidia.com>
Date: Tue, 18 Aug 2026 08:48:55 -0700
Subject: [PATCH] Fix generation error precedence in failure classification

---
 .../tools/evaluation/scoring.py               |  6 +-
 nemo_retriever/tests/test_live_rag.py         | 56 +++++++++++++++++++
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/nemo_retriever/src/nemo_retriever/tools/evaluation/scoring.py b/nemo_retriever/src/nemo_retriever/tools/evaluation/scoring.py
index adae7c1bd8..5d6121baf7 100644
--- a/nemo_retriever/src/nemo_retriever/tools/evaluation/scoring.py
+++ b/nemo_retriever/src/nemo_retriever/tools/evaluation/scoring.py
@@ -255,6 +255,9 @@ def classify_failure(
     if gen_error == "thinking_truncated":
         return "thinking_truncated"
 
+    if gen_error is not None:
+        return "generation_error"
+
     if judge_score is None:
         return "judge_error"
 
@@ -308,7 +311,8 @@ def score_dataframe(df: pd.DataFrame) -> pd.DataFrame:
 
         judge_score_raw = row.get("judge_score")
         judge_score = None if pd.isna(judge_score_raw) else float(judge_score_raw)
-        gen_error = row.get("gen_error")
+        gen_error_raw = row.get("gen_error")
+        gen_error = None if pd.isna(gen_error_raw) else str(gen_error_raw)
         fm = classify_failure(
             ref_in_chunks=aic,
             judge_score=judge_score,
diff --git a/nemo_retriever/tests/test_live_rag.py b/nemo_retriever/tests/test_live_rag.py
index 82e5902dc1..a2d92f1484 100644
--- a/nemo_retriever/tests/test_live_rag.py
+++ b/nemo_retriever/tests/test_live_rag.py
@@ -438,6 +438,62 @@ def test_retrieve_batch_empty_input(self):
 class TestPipelineBuilder:
     """Retriever.pipeline() fluent builder composition."""
 
+    @pytest.mark.parametrize(
+        ("gen_error", "answer", "judge_score", "judge_error", "expected_failure_mode"),
+        [
+            ("transport_error", "", None, "empty_candidate", "generation_error"),
+            ("request_error", "", None, "empty_candidate", "generation_error"),
+            ("thinking_truncated", "", None, "empty_candidate", "thinking_truncated"),
+            (None, "reference answer", None, "transport_error", "judge_error"),
+            (None, "reference answer", 1.0, None, "correct"),
+        ],
+    )
+    def test_failure_mode_uses_originating_stage(
+        self,
+        gen_error,
+        answer,
+        judge_score,
+        judge_error,
+        expected_failure_mode,
+    ):
+        """Generation errors take precedence in the supported operator order."""
+        from nemo_retriever.models.llm.types import GenerationResult, JudgeResult, RetrievalResult
+
+        class _Generator:
+            supports_concurrent_calls = False
+            model = "deterministic/generator"
+
+            def generate(self, query, chunks, *, reasoning_enabled=None):
+                return GenerationResult(
+                    answer=answer,
+                    latency_s=0.0,
+                    model=self.model,
+                    error=gen_error,
+                )
+
+        class _Judge:
+            def judge(self, query, reference, candidate):
+                if not candidate.strip():
+                    return JudgeResult(score=None, reasoning="Candidate answer was empty.", error="empty_candidate")
+                return JudgeResult(score=judge_score, reasoning="", error=judge_error)
+
+        r = _make_retriever()
+        retrieved = RetrievalResult(
+            chunks=["reference answer"],
+            metadata=[{"source": "deterministic"}],
+        )
+
+        with patch.object(r, "retrieve_batch", return_value=[retrieved]):
+            builder = r.pipeline().generate(_build_fake_llm_client()).judge(_build_fake_judge()).score()
+            builder._steps[0]._client = _Generator()
+            builder._steps[1]._judge = _Judge()
+            out = builder.run(["query"], reference=["reference answer"])
+
+        row = out.iloc[0]
+        assert row.gen_error == gen_error
+        assert row.judge_error == judge_error
+        assert row.failure_mode == expected_failure_mode
+
     def test_builder_composition_runs_expected_steps(self):
         """generate -> score -> judge builds and executes the full chain."""
         r = _make_retriever()