From 1a4342a9c54bfe0aaad0a90f2b413be8117ef9ee Mon Sep 17 00:00:00 2001 From: Julio Perez Date: Thu, 6 Aug 2026 17:23:09 +0000 Subject: [PATCH] add ragas notebook for NRL --- .../nrl_financebench_ragas_evaluation.ipynb | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 examples/nrl_financebench_ragas_evaluation.ipynb diff --git a/examples/nrl_financebench_ragas_evaluation.ipynb b/examples/nrl_financebench_ragas_evaluation.ipynb new file mode 100644 index 0000000000..974d5ab0da --- /dev/null +++ b/examples/nrl_financebench_ragas_evaluation.ipynb @@ -0,0 +1,252 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FinanceBench RAGAS evaluation with NeMo Retriever Library\n", + "\n", + "This notebook reproduces the FinanceBench evaluation flow from NVIDIA's `evaluation_01_ragas.ipynb`, while using **NeMo Retriever Library (NRL)** to ingest PDFs, retrieve contexts, and generate answers. It reports the same Ragas metrics—Answer Accuracy, Context Relevance, and Response Groundedness—and adds **document-level recall@1, recall@5, and recall@10 before answer generation**.\n", + "\n", + "FinanceBench labels each question with a `doc_name`; therefore recall here means that at least one of the top-*k* NRL hits comes from the labelled source document. This isolates retrieval quality from answer-generation quality." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Install dependencies and download FinanceBench\n", + "\n", + "Run this from the NeMo Retriever repository root. The `[llm]` extra provides NRL's LiteLLM client. Ragas is retained only to calculate the same three metrics as the reference notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -e \"./nemo_retriever[llm]\" ragas langchain-nvidia-ai-endpoints\n", + "!git clone https://github.com/patronus-ai/financebench.git data/financebench" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Configure and ingest with NRL\n", + "\n", + "Set `REBUILD_INDEX=True` only when the LanceDB table does not already contain the FinanceBench PDFs. Ingesting can take several minutes and requires the normal NRL extraction/embedding setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import os\n", + "\n", + "FINANCEBENCH_ROOT = Path(\"data/financebench\")\n", + "PDF_DIR = FINANCEBENCH_ROOT / \"pdfs\"\n", + "QA_PATH = FINANCEBENCH_ROOT / \"data\" / \"financebench_open_source.jsonl\"\n", + "LANCEDB_URI = \"lancedb-financebench\"\n", + "TABLE_NAME = \"financebench\"\n", + "EMBED_MODEL = \"nvidia/llama-nemotron-embed-1b-v2\"\n", + "MAX_QUESTIONS = 50 # Set to None for the complete FinanceBench open-source split.\n", + "RETRIEVAL_K = 10 # Must be at least max(1, 5, 10).\n", + "REBUILD_INDEX = False\n", + "\n", + "assert PDF_DIR.is_dir(), f\"Missing FinanceBench PDFs: {PDF_DIR}\"\n", + "assert QA_PATH.is_file(), f\"Missing FinanceBench labels: {QA_PATH}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if REBUILD_INDEX:\n", + " # NRL creates a LanceDB collection containing embedded document chunks.\n", + " !retriever ingest {PDF_DIR} --lancedb-uri {LANCEDB_URI} --table-name {TABLE_NAME} --embed-model-name {EMBED_MODEL}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Load questions and retrieve with NRL\n", + "\n", + "This cell performs all retrieval first. It preserves the top-10 chunks and metadata for each question; generation never performs another retrieval pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from nemo_retriever.graph.retriever import Retriever\n", + "\n", + "with QA_PATH.open() as handle:\n", + " qa_pairs = [json.loads(line) for line in handle]\n", + "if MAX_QUESTIONS is not None:\n", + " qa_pairs = qa_pairs[:MAX_QUESTIONS]\n", + "\n", + "retriever = Retriever(\n", + " vdb_kwargs={\"uri\": LANCEDB_URI, \"table_name\": TABLE_NAME},\n", + " embed_kwargs={\"model_name\": EMBED_MODEL, \"embed_model_name\": EMBED_MODEL},\n", + " top_k=RETRIEVAL_K,\n", + ")\n", + "print(f\"Evaluating {len(qa_pairs)} FinanceBench questions\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def normalise_document_name(value):\n", + " \"\"\"Compare FinanceBench doc_name values with NRL source metadata.\"\"\"\n", + " return Path(str(value or \"\")).stem.lower()\n", + "\n", + "def source_name(metadata):\n", + " \"\"\"Return the source identifier populated by NRL/LanceDB.\"\"\"\n", + " for key in (\"source\", \"source_id\", \"path\", \"source_path\"):\n", + " if metadata.get(key):\n", + " return normalise_document_name(metadata[key])\n", + " return \"\"\n", + "\n", + "def document_recall_at_k(hit_metadata, gold_doc_name, k):\n", + " gold = normalise_document_name(gold_doc_name)\n", + " return any(source_name(metadata) == gold for metadata in hit_metadata[:k])\n", + "\n", + "retrieval_rows = []\n", + "for index, qa in enumerate(qa_pairs, start=1):\n", + " result = retriever.retrieve(qa[\"question\"], top_k=RETRIEVAL_K)\n", + " retrieval_rows.append({\n", + " \"financebench_id\": qa[\"financebench_id\"],\n", + " \"query\": qa[\"question\"],\n", + " \"reference\": qa[\"answer\"],\n", + " \"gold_doc_name\": qa[\"doc_name\"],\n", + " \"retrieved_contexts\": result.chunks,\n", + " \"retrieval_metadata\": result.metadata,\n", + " \"recall_at_1\": document_recall_at_k(result.metadata, qa[\"doc_name\"], 1),\n", + " \"recall_at_5\": document_recall_at_k(result.metadata, qa[\"doc_name\"], 5),\n", + " \"recall_at_10\": document_recall_at_k(result.metadata, qa[\"doc_name\"], 10),\n", + " })\n", + " if index % 10 == 0 or index == len(qa_pairs):\n", + " print(f\"Retrieved {index}/{len(qa_pairs)} questions\")\n", + "\n", + "retrieval_df = pd.DataFrame(retrieval_rows)\n", + "recall_summary = (retrieval_df[[\"recall_at_1\", \"recall_at_5\", \"recall_at_10\"]]\n", + " .mean().rename(lambda column: column.replace(\"recall_at_\", \"Recall@\")))\n", + "display(recall_summary.to_frame(\"document_recall\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Generate answers with NRL\n", + "\n", + "Set `NVIDIA_API_KEY` (or configure an OpenAI-compatible endpoint through LiteLLM) before running. Generation receives exactly the contexts retrieved above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from nemo_retriever.models.llm import LiteLLMClient\n", + "\n", + "assert os.environ.get(\"NVIDIA_API_KEY\"), \"Set NVIDIA_API_KEY before generating answers.\"\n", + "generator = LiteLLMClient.from_kwargs(\n", + " model=\"nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", + " temperature=0.0,\n", + " max_tokens=512,\n", + ")\n", + "\n", + "answers = []\n", + "for index, row in retrieval_df.iterrows():\n", + " generated = generator.generate(row.query, row.retrieved_contexts)\n", + " answers.append(generated.answer if generated.error is None else \"\")\n", + " if (index + 1) % 10 == 0 or index + 1 == len(retrieval_df):\n", + " print(f\"Generated {index + 1}/{len(retrieval_df)} answers\")\n", + "\n", + "evaluation_records = retrieval_df.assign(response=answers)[\n", + " [\"query\", \"reference\", \"retrieved_contexts\", \"response\"]\n", + "].rename(columns={\"query\": \"user_input\"}).to_dict(\"records\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Run the same Ragas metrics as the reference notebook\n", + "\n", + "NRL supplied the documents, contexts, and answers. Ragas evaluates the identical metric set from the reference: Answer Accuracy, Context Relevance, and Response Groundedness." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_nvidia_ai_endpoints import ChatNVIDIA\n", + "from ragas import EvaluationDataset, evaluate\n", + "from ragas.llms import LangchainLLMWrapper\n", + "from ragas.metrics import AnswerAccuracy, ContextRelevance, ResponseGroundedness\n", + "from ragas.run_config import RunConfig\n", + "\n", + "judge_llm = ChatNVIDIA(model=\"openai/gpt-oss-120b\")\n", + "ragas_results = evaluate(\n", + " dataset=EvaluationDataset.from_list(evaluation_records),\n", + " metrics=[AnswerAccuracy(), ContextRelevance(), ResponseGroundedness()],\n", + " llm=LangchainLLMWrapper(judge_llm),\n", + " run_config=RunConfig(max_workers=1, max_wait=120),\n", + ")\n", + "ragas_results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Inspect recall alongside answer-quality metrics\n", + "\n", + "Recall is calculated before generation and is not affected by the generator or judge. The per-question table makes retrieval misses visible next to Ragas scores." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ragas_df = ragas_results.to_pandas()\n", + "report_df = pd.concat([\n", + " retrieval_df[[\"financebench_id\", \"gold_doc_name\", \"recall_at_1\", \"recall_at_5\", \"recall_at_10\"]].reset_index(drop=True),\n", + " ragas_df.reset_index(drop=True),\n", + "], axis=1)\n", + "\n", + "display(recall_summary.to_frame(\"document_recall\"))\n", + "display(report_df.head())\n", + "report_df.to_json(\"financebench_nrl_ragas_results.jsonl\", orient=\"records\", lines=True)" + ] + } + ], + "metadata": { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python", "version": "3.12"} + }, + "nbformat": 4, + "nbformat_minor": 5 +}