From 692853d81b110ccb198c3c29f8f7fd05a325150b Mon Sep 17 00:00:00 2001 From: AImindcrafter Date: Mon, 13 Jul 2026 23:59:29 +0500 Subject: [PATCH] Add Retrieval-Conditioned Confidence Metric (RCCS) for RAG evaluation --- metrics/rccs/README.md | 64 ++++++++++++++++++++ metrics/rccs/app.py | 6 ++ metrics/rccs/rccs.py | 107 ++++++++++++++++++++++++++++++++++ metrics/rccs/requirements.txt | 3 + 4 files changed, 180 insertions(+) create mode 100644 metrics/rccs/README.md create mode 100644 metrics/rccs/app.py create mode 100644 metrics/rccs/rccs.py create mode 100644 metrics/rccs/requirements.txt diff --git a/metrics/rccs/README.md b/metrics/rccs/README.md new file mode 100644 index 00000000..d80c8cdf --- /dev/null +++ b/metrics/rccs/README.md @@ -0,0 +1,64 @@ +--- +title: Retrieval-Conditioned Confidence Score +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Retrieval-Conditioned Confidence Score (RCCS) for RAG evaluation. + RCCS measures the alignment between the retrieval relevance score (R), model confidence score (C), and ground-truth correctness (A). +--- + +# Metric Card for RCCS + +## Metric Description +Retrieval-Conditioned Confidence Score (RCCS) evaluates the alignment of a model's confidence with its performance, conditioned on the quality of the retrieved information. +It evaluates the joint alignment between retrieval relevance score (R), model confidence score (C), and ground-truth correctness (A). + +## How to Use +This metric takes lists of `retrieval_score`, `confidence_score`, and `correctness` as input: + +```python +>>> rccs_metric = evaluate.load("rccs") +>>> results = rccs_metric.compute( +... retrieval_score=[0.8, 0.3, 0.5], +... confidence_score=[0.9, 0.2, 0.7], +... correctness=[1, 0, 1] +... ) +>>> print(results) +{'rccs_correlation': 0.829006943846357, 'confidence_calibration_error': 0.33000001311302185, 'mean_rc': 0.3766666650772095, 'n': 3} +``` + +### Inputs +- **retrieval_score** (`list` of `float`): Retrieval relevance score (R) per example. +- **confidence_score** (`list` of `float`): Model confidence (C) per example. Calibrated probability recommended. +- **correctness** (`list` of `int`): Ground-truth correctness (A) per example (0 or 1). + +### Output Values +- **rccs_correlation** (`float`): Pearson correlation between `(R * C)` and `A`. +- **confidence_calibration_error** (`float`): Mean absolute error between `(R * C)` and `A`. +- **mean_rc** (`float`): Mean of `R * C`. +- **n** (`int`): Number of examples. + +Output Example: +```python +{ + 'rccs_correlation': 0.829006943846357, + 'confidence_calibration_error': 0.33000001311302185, + 'mean_rc': 0.3766666650772095, + 'n': 3 +} +``` + +## Limitations and Bias +Pearson correlation is undefined and returns `NaN` when the input vectors are constant (e.g. if all answers are correct or all retrieval*confidence products are identical). + +## Citation(s) +```bibtex +``` diff --git a/metrics/rccs/app.py b/metrics/rccs/app.py new file mode 100644 index 00000000..e110f4e9 --- /dev/null +++ b/metrics/rccs/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("metrics/rccs") +launch_gradio_widget(module) diff --git a/metrics/rccs/rccs.py b/metrics/rccs/rccs.py new file mode 100644 index 00000000..7d7453a9 --- /dev/null +++ b/metrics/rccs/rccs.py @@ -0,0 +1,107 @@ +# Copyright 2026 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Retrieval-Conditioned Confidence Metric (RCCS) for RAG evaluation.""" + +import datasets +import numpy as np +from scipy.stats import pearsonr + +import evaluate + + +_DESCRIPTION = """ +Retrieval-Conditioned Confidence Score (RCCS) evaluates the alignment of a model's confidence with its performance, +conditioned on the quality of the retrieved information. +It evaluates the joint alignment between retrieval relevance score (R), model confidence score (C), and ground-truth correctness (A). +""" + + +_KWARGS_DESCRIPTION = """ +Args: + retrieval_score (`list` of `float`): Retrieval relevance score (R) per example. + confidence_score (`list` of `float`): Model confidence (C) per example. Calibrated probability recommended. + correctness (`list` of `int`): Ground-truth correctness (A) per example (0 or 1). + +Returns: + rccs_correlation (`float`): Pearson correlation between `(R * C)` and `A`. + confidence_calibration_error (`float`): Mean absolute error between `(R * C)` and `A`. + mean_rc (`float`): Mean of `R * C`. + n (`int`): Number of examples. + +Examples: + + Example 1 - A simple example using lists of scores and correctness: + >>> rccs_metric = evaluate.load("rccs") + >>> results = rccs_metric.compute(retrieval_score=[0.8, 0.3, 0.5], confidence_score=[0.9, 0.2, 0.7], correctness=[1, 0, 1]) + >>> print(results['n']) + 3 + >>> print(round(results['mean_rc'], 2)) + 0.38 + >>> print(round(results['confidence_calibration_error'], 2)) + 0.33 + >>> print(round(results['rccs_correlation'], 2)) + 0.83 +""" + + +_CITATION = """ +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class RCCS(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "retrieval_score": datasets.Value("float32"), + "confidence_score": datasets.Value("float32"), + "correctness": datasets.Value("int32"), + } + ), + reference_urls=[], + ) + + def _compute(self, retrieval_score, confidence_score, correctness): + R = np.array(retrieval_score, dtype=np.float32) + C = np.array(confidence_score, dtype=np.float32) + A = np.array(correctness, dtype=np.int32) + + if len(R) == 0: + return { + "rccs_correlation": float("nan"), + "confidence_calibration_error": float("nan"), + "mean_rc": float("nan"), + "n": 0, + } + + rc = R * C + mean_rc = float(np.mean(rc)) + confidence_calibration_error = float(np.mean(np.abs(rc - A))) + n = len(R) + + if n < 2 or np.all(rc == rc[0]) or np.all(A == A[0]): + rccs_correlation = float("nan") + else: + rccs_correlation = float(pearsonr(rc, A)[0]) + + return { + "rccs_correlation": rccs_correlation, + "confidence_calibration_error": confidence_calibration_error, + "mean_rc": mean_rc, + "n": n, + } diff --git a/metrics/rccs/requirements.txt b/metrics/rccs/requirements.txt new file mode 100644 index 00000000..d8dbdd21 --- /dev/null +++ b/metrics/rccs/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy +numpy