feat: improve answer quality to match llama-server behavior

paperrag/config.py

@@ -60,12 +60,12 @@
     ),
 }
 
-# Presets that need more output tokens than the 256 default.
+# Presets that need more output tokens than the 1024 default.
 PRESET_MAX_TOKENS: dict[str, int] = {
-    "default": 256,
-    "reviewer": 512,
-    "summarizer": 512,
-    "explainer": 512,
+    "default": 1024,
+    "reviewer": 2048,
+    "summarizer": 2048,
+    "explainer": 1024,
 }
 
 
@@ -141,7 +141,7 @@ class EmbedderConfig(BaseModel):
 class RetrieverConfig(BaseModel):
     """Retrieval configuration."""
 
-    top_k: int = Field(default=2, ge=1)
+    top_k: int = Field(default=5, ge=1)
     score_threshold: float = Field(
         default=0.1,
         ge=0.0,
@@ -246,9 +246,9 @@ class LLMConfig(BaseModel):
         "Be concise and cite sources."
     )
     temperature: float = 0.0
-    max_tokens: int = 256
+    max_tokens: int = 1024
     ctx_size: int = Field(
-        default=2048,
+        default=4096,
         ge=512,
         validation_alias=AliasChoices("ctx_size", "n_ctx"),
         description=(

paperrag/llm.py

@@ -69,10 +69,28 @@ def _cleanup_llama_servers() -> None:
 atexit.register(_cleanup_llama_servers)
 
 # Maximum characters per context chunk sent to the LLM.
-# Longer chunks are truncated to keep prompt size manageable for small models.
-_MAX_CHUNK_CHARS = 750
+# Set to 2000 to accommodate typical chunk sizes (1000 chars) without truncation
+# when ctx_size is 4096+. Prevents loss of important context that causes
+# inferior answers compared to llama-server's full-document approach.
+_MAX_CHUNK_CHARS = 2000
 _TRAILING_SOURCE_LINE_RE = re.compile(r"^\s*Sources?:\s*\[[0-9,\s-]+\]\s*$", re.IGNORECASE)
 
+# Suffix appended to the user's configured system prompt for follow-up questions
+# answered from conversation history only.
+_FOLLOWUP_PROMPT_SUFFIX = (
+    " Answer the follow-up question based on the conversation so far. "
+    "If the previous conversation does not contain relevant information, say so."
+)
+
+
+def _get_followup_system_prompt(config: "LLMConfig") -> str:
+    """Derive the follow-up system prompt from the user's configured system_prompt.
+
+    This ensures that tone, language, and constraints set by the user via
+    /prompt or /preset are respected even when answering from history only.
+    """
+    return config.system_prompt + _FOLLOWUP_PROMPT_SUFFIX
+
 
 def _build_prompt(question: str, context_chunks: list[str], source_labels: list[int] | None = None) -> str:
     context_lines = []
@@ -89,10 +107,19 @@ def _build_prompt(question: str, context_chunks: list[str], source_labels: list[
         "Use inline citation [1] within your answer." if n == 1
         else f"Use inline citations [1]–[{n}] within your answer. Only cite sources from [1] to [{n}]."
     )
+
+    # Use a more detailed instruction when we have substantial context
+    # (> 3000 chars ≈ 3+ full chunks, indicating rich retrieval or full-document mode)
+    total_context_chars = sum(len(c) for c in context_chunks)
+    if total_context_chars > 3000:
+        answer_style = "Answer thoroughly using ONLY the context. Provide detailed reasoning and cite specific statements."
+    else:
+        answer_style = "Answer using ONLY the context."
+
     return (
         f"Context:\n{context_block}\n\n"
         f"Question: {question}\n\n"
-        f"Answer concisely using ONLY the context. {cite_instruction} Do not add a separate 'Source:' or 'Sources:' list at the end."
+        f"{answer_style} {cite_instruction} Do not add a separate 'Source:' or 'Sources:' list at the end."
     )
 
 
@@ -459,7 +486,7 @@ def _get_or_start_llama_server(model_path: str, ctx_size: int, n_gpu_layers: int
 # ---------------------------------------------------------------------------
 
 
-def _build_messages(question: str, context_chunks: list[str], model_name: str, system_prompt: str, source_labels: list[int] | None = None, think: bool = False) -> list[dict]:
+def _build_messages(question: str, context_chunks: list[str], model_name: str, system_prompt: str, source_labels: list[int] | None = None, think: bool = False, conversation_history: list[dict] | None = None) -> list[dict]:
     """Build the chat messages list from question, context chunks, and model name."""
     user_prompt = _build_prompt(question, context_chunks, source_labels=source_labels)
 
@@ -471,23 +498,28 @@ def _build_messages(question: str, context_chunks: list[str], model_name: str, s
         if not think:
             user_prompt += " /no_think"
 
-    return [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": user_prompt},
-    ]
+    messages: list[dict] = [{"role": "system", "content": system_prompt}]
+
+    # Include conversation history for follow-up questions
+    if conversation_history:
+        messages.extend(conversation_history)
+
+    messages.append({"role": "user", "content": user_prompt})
+    return messages
 
 
 def _prepare(
     question: str,
     context_chunks: list[str],
     config: LLMConfig,
     source_labels: list[int] | None = None,
+    conversation_history: list[dict] | None = None,
 ) -> tuple:
     """Ollama-specific setup: build messages, get/cache OpenAI client, return (client, messages)."""
     global _client_cache
     from openai import OpenAI
 
-    messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, source_labels=source_labels, think=config.think)
+    messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, source_labels=source_labels, think=config.think, conversation_history=conversation_history)
 
     if _client_cache is not None:
         client = _client_cache
@@ -519,6 +551,7 @@ def generate_answer(
     question: str,
     context_chunks: list[str],
     config: LLMConfig | None = None,
+    conversation_history: list[dict] | None = None,
 ) -> str:
     """Generate an answer using the configured LLM backend (blocking).
 
@@ -528,6 +561,12 @@ def generate_answer(
       **llama.cpp** via ``llama-server`` (install: ``brew install llama-cpp``).
     * All other model names delegate to **Ollama**.
 
+    Parameters
+    ----------
+    conversation_history : list[dict] | None
+        Optional list of previous messages (role/content dicts) to provide
+        context for follow-up questions.
+
     Examples::
 
         # Ollama (unchanged)
@@ -547,7 +586,7 @@ def generate_answer(
     if _is_llama_backend(config.model_name):
         model_path = _resolve_model_path(config.model_name)
         client = _get_or_start_llama_server(model_path, config.ctx_size, config.n_gpu_layers, config.n_threads)
-        messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, think=config.think)
+        messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, think=config.think, conversation_history=conversation_history)
         logger.info(
             "Calling llama-server (model=%s, temp=%.2f)", config.model_name, config.temperature
         )
@@ -560,7 +599,7 @@ def generate_answer(
         return _strip_trailing_source_footers(response.choices[0].message.content or "")
 
     try:
-        client, messages = _prepare(question, context_chunks, config)
+        client, messages = _prepare(question, context_chunks, config, conversation_history=conversation_history)
     except ImportError:
         raise ImportError(
             "The 'openai' package is required. Install with: uv pip install openai"
@@ -581,6 +620,7 @@ def stream_answer(
     context_chunks: list[str],
     config: LLMConfig | None = None,
     source_files: list[str] | None = None,
+    conversation_history: list[dict] | None = None,
 ) -> Iterator[str]:
     """Yield text chunks as they arrive from the LLM (streaming).
 
@@ -590,6 +630,12 @@ def stream_answer(
       **llama.cpp** via ``llama-server``.
     * All other model names delegate to **Ollama**.
 
+    Parameters
+    ----------
+    conversation_history : list[dict] | None
+        Optional list of previous messages (role/content dicts) to provide
+        context for follow-up questions.
+
     Usage::
 
         for chunk in stream_answer(question, chunks, cfg.llm):
@@ -615,7 +661,7 @@ def stream_answer(
     if _is_llama_backend(config.model_name):
         model_path = _resolve_model_path(config.model_name)
         client = _get_or_start_llama_server(model_path, config.ctx_size, config.n_gpu_layers, config.n_threads)
-        messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, source_labels=source_labels, think=config.think)
+        messages = _build_messages(question, context_chunks, config.model_name, config.system_prompt, source_labels=source_labels, think=config.think, conversation_history=conversation_history)
         logger.info(
             "Calling llama-server streaming (model=%s, temp=%.2f)",
             config.model_name,
@@ -636,12 +682,91 @@ def stream_answer(
         return
 
     try:
-        client, messages = _prepare(question, context_chunks, config, source_labels=source_labels)
+        client, messages = _prepare(question, context_chunks, config, source_labels=source_labels, conversation_history=conversation_history)
+    except ImportError:
+        raise ImportError(
+            "The 'openai' package is required. Install with: uv pip install openai"
+        )
+
+    response = client.chat.completions.create(
+        model=config.model_name,
+        messages=messages,
+        temperature=config.temperature,
+        max_tokens=config.max_tokens,
+        stream=True,
+        extra_body={"num_ctx": config.ctx_size, "keep_alive": "30m"},
+    )
+    yield from _sanitize_stream(
+        delta
+        for chunk in response
+        if (delta := chunk.choices[0].delta.content)
+    )
+
+
+def stream_followup(
+    question: str,
+    conversation_history: list[dict],
+    config: LLMConfig | None = None,
+) -> Iterator[str]:
+    """Yield text chunks for a follow-up question using conversation history only.
+
+    This is used when retrieval returns no results but conversation history
+    exists, allowing the LLM to answer based on previously discussed context.
+    """
+    config = config or LLMConfig()
+
+    if not conversation_history:
+        yield "No conversation history available to answer the question."
+        return
+
+    # Build messages with conversation history but no retrieval context
+    model_lower = config.model_name.lower()
+    user_prompt = question
+    if "qwen3" in model_lower or "qwen-3" in model_lower:
+        if not config.think:
+            user_prompt += " /no_think"
+
+    messages: list[dict] = [{"role": "system", "content": _get_followup_system_prompt(config)}]
+    messages.extend(conversation_history)
+    messages.append({"role": "user", "content": user_prompt})
+
+    if _is_llama_backend(config.model_name):
+        model_path = _resolve_model_path(config.model_name)
+        client = _get_or_start_llama_server(model_path, config.ctx_size, config.n_gpu_layers, config.n_threads)
+        logger.info(
+            "Calling llama-server streaming follow-up (model=%s, temp=%.2f)",
+            config.model_name,
+            config.temperature,
+        )
+        response = client.chat.completions.create(  # type: ignore[union-attr]
+            model=os.path.basename(model_path),
+            messages=messages,
+            temperature=config.temperature,
+            max_tokens=config.max_tokens,
+            stream=True,
+        )
+        yield from _sanitize_stream(
+            delta
+            for chunk in response
+            if (delta := chunk.choices[0].delta.content)
+        )
+        return
+
+    try:
+        from openai import OpenAI
+
+        global _client_cache
+        if _client_cache is not None:
+            client = _client_cache
+        else:
+            client = OpenAI(api_key="not-needed", base_url=_OLLAMA_API_URL)
+            _client_cache = client
     except ImportError:
         raise ImportError(
             "The 'openai' package is required. Install with: uv pip install openai"
         )
 
+    logger.info("Calling Ollama LLM follow-up (model=%s, temp=%.2f)", config.model_name, config.temperature)
     response = client.chat.completions.create(
         model=config.model_name,
         messages=messages,