feat(evaluation): v2 native LLM-as-judge (ground truth) + Langfuse-free

backend/app/alembic/versions/075_add_judge_columns_to_evaluation_run.py

@@ -0,0 +1,57 @@
+"""Add native LLM-as-judge columns to evaluation_run
+
+Revision ID: 075
+Revises: 074
+Create Date: 2026-07-16 00:00:00.000000
+
+The v2 judged fast-eval run stores its judge state on the existing evaluation_run
+row rather than a new table. The chunked aggregate task only knows an eval_run_id,
+so the judge intent (is_judge_run) must be durable on the row for the aggregate to
+read at judge time. per_item_ground_truth mirrors per_item_scores as Kaapi's native
+per-row store (v2 never syncs to Langfuse). Judging is system-config only — always
+the fallback model + built-in prompt — so no per-run judge config is persisted.
+
+Both columns are nullable with default NULL: v1 and pre-feature runs carry no judge
+data, so no backfill is needed.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects.postgresql import JSONB
+
+revision = "075"
+down_revision = "074"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "evaluation_run",
+        sa.Column(
+            "is_judge_run",
+            sa.Boolean(),
+            nullable=True,
+            comment=(
+                "True for v2 runs that run the native LLM-as-judge (and skip the "
+                "Langfuse score sync). NULL/False = v1 run, cosine-only, Langfuse-synced"
+            ),
+        ),
+    )
+    op.add_column(
+        "evaluation_run",
+        sa.Column(
+            "per_item_ground_truth",
+            JSONB(),
+            nullable=True,
+            comment=(
+                "Durable {ref: score} map of the Adherence to Ground Truth judge "
+                "scores (ref = trace_id when traced, else item_id); Kaapi's own store"
+            ),
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("evaluation_run", "per_item_ground_truth")
+    op.drop_column("evaluation_run", "is_judge_run")

backend/app/api/docs/evaluation/create_evaluation_dataset_v2.md

@@ -0,0 +1,18 @@
+Upload a CSV of golden Q&A pairs for evaluation, stored natively by Kaapi with no
+Langfuse dataset (v2).
+
+Same multipart shape as the v1 dataset upload (`file`, `dataset_name`,
+`description`, `duplication_factor`), but the dataset is stored in object storage
+(S3) only: the row is created with `langfuse_dataset_id` null and the CSV holds
+exactly the **original** rows — no physical duplication. The `duplication_factor`
+is recorded in `dataset_metadata` (with a run-time-duplication marker) and applied
+when the run executes, so an 8-row CSV with factor 5 evaluates 40 rows.
+
+**Response:** `APIResponse[DatasetUploadResponse]`, same shape as v1 with
+`langfuse_dataset_id` null. `original_items` is the stored row count and
+`total_items` is `original_items × duplication_factor` (the count the run will
+produce, not stored rows). `eligible_for_fast` reflects only the unique-row size
+cap (`EVAL_FAST_MAX_UNIQUE_ROWS`).
+
+**CSV format:** required columns `question`, `answer`; optional `category`. Rows
+with a missing question or answer are skipped. Maximum upload size is 1 MB.

backend/app/api/docs/evaluation/create_evaluation_v2.md

@@ -0,0 +1,32 @@
+Start a v2 evaluation run. Replicates the v1 `POST /api/v1/evaluations` request
+body with Kaapi's native LLM-as-Judge built in — v1 is left unchanged.
+
+v2 runs are **always fast** and always judged (there is no `run_mode`; batch is
+deferred to a later phase). Every scoreable row is automatically judged (no opt-in
+flag) on **Adherence to Ground Truth**: an LLM judge scores whether the answer
+conveys the same correct information as the dataset's golden answer (0–1, with
+reasoning). Scores, the durable per-row map (`per_item_ground_truth`), and the
+`ground_truth_judge` cost stage are stored natively by Kaapi. v2 runs compute **no
+cosine similarity** and do **not** touch Langfuse.
+
+Judging is system-config only: the judge always uses the configured model
+(`EVAL_JUDGE_MODEL`, default `gpt-5-mini`) and the built-in ground-truth prompt.
+There is no per-run or ad-hoc judge configuration.
+
+## Example
+
+```json
+{
+  "dataset_id": 123,
+  "experiment_name": "judge-smoke-1",
+  "config_id": "f54f0d67-4817-4103-9fdf-b74b3d46733e",
+  "config_version": 1
+}
+```
+
+## Error responses
+
+| Status | Code | When |
+| --- | --- | --- |
+| 409 | `run_name_already_exists` | A run with the same `experiment_name` already exists for this (organization, project) |
+| 422 | — | An unsupported config type / oversized dataset |

backend/app/api/main.py

@@ -3,7 +3,6 @@
 from app.api.routes import (
     analytics,
     api_keys,
-    assessment as assessment_routes,
     assistants,
     auth,
     collection_job,
@@ -35,7 +34,13 @@
     users,
     utils,
 )
-from app.core.config import settings
+from app.api.routes import (
+    assessment as assessment_routes,
+)
+from app.api.routes.evaluations.dataset_v2 import (
+    router as evaluations_dataset_v2_router,
+)
+from app.api.routes.evaluations.evaluation_v2 import router as evaluations_v2_router
 
 api_router = APIRouter()
 api_router.include_router(analytics.router)
@@ -73,3 +78,10 @@
 api_router.include_router(private.router)
 # if settings.ENVIRONMENT in ["development", "testing"]:
 #     api_router.include_router(private.router)
+
+
+# v2 API surface (mounted at settings.API_V2_STR). Only the endpoints that differ
+# from v1 live here — currently the judged run trigger. Everything else stays v1.
+api_v2_router = APIRouter()
+api_v2_router.include_router(evaluations_v2_router)
+api_v2_router.include_router(evaluations_dataset_v2_router)

backend/app/api/routes/evaluations/dataset_v2.py

@@ -0,0 +1,62 @@
+"""v2 Langfuse-free evaluation dataset upload route.
+
+Replica of the v1 dataset upload with the same multipart shape and response, but
+it creates no Langfuse dataset and stores only the original items — duplication is
+recorded in metadata and applied at run time.
+"""
+
+import logging
+
+from fastapi import APIRouter, Depends, File, Form, UploadFile
+
+from app.api.deps import AuthContextDep, SessionDep
+from app.api.permissions import Permission, require_permission
+from app.api.routes.evaluations.dataset import _dataset_to_response
+from app.core.rate_monitor import monitor_rate
+from app.models.evaluation import DatasetUploadResponse
+from app.services.evaluations import upload_dataset_v2, validate_csv_file
+from app.utils import APIResponse, load_description
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/evaluations/datasets", tags=["Evaluation v2"])
+
+
+@router.post(
+    "",
+    description=load_description("evaluation/create_evaluation_dataset_v2.md"),
+    response_model=APIResponse[DatasetUploadResponse],
+    dependencies=[
+        Depends(require_permission(Permission.REQUIRE_PROJECT)),
+        Depends(monitor_rate("evaluations")),
+    ],
+)
+async def upload_dataset_v2_route(
+    session: SessionDep,
+    auth_context: AuthContextDep,
+    file: UploadFile = File(
+        ..., description="CSV file with 'question' and 'answer' columns"
+    ),
+    dataset_name: str = Form(..., description="Name for the dataset"),
+    description: str | None = Form(None, description="Optional dataset description"),
+    duplication_factor: int = Form(
+        default=1,
+        ge=1,
+        le=5,
+        description="Run-time duplication per item (min: 1, max: 5)",
+    ),
+) -> APIResponse[DatasetUploadResponse]:
+    """Upload a Langfuse-free evaluation dataset (v2)."""
+    csv_content = await validate_csv_file(file)
+
+    dataset = upload_dataset_v2(
+        session=session,
+        csv_content=csv_content,
+        dataset_name=dataset_name,
+        description=description,
+        duplication_factor=duplication_factor,
+        organization_id=auth_context.organization_.id,
+        project_id=auth_context.project_.id,
+    )
+
+    return APIResponse.success_response(data=_dataset_to_response(dataset))

backend/app/api/routes/evaluations/evaluation_v2.py

@@ -0,0 +1,56 @@
+"""v2 evaluation run trigger — replica of the v1 trigger plus native judging."""
+
+import logging
+from uuid import UUID
+
+from asgi_correlation_id import correlation_id
+from fastapi import APIRouter, Body, Depends
+
+from app.api.deps import AuthContextDep, SessionDep
+from app.api.permissions import Permission, require_permission
+from app.core.rate_monitor import monitor_rate
+from app.models.evaluation import EvaluationRunPublic
+from app.services.evaluations.judge import validate_and_start_judged_evaluation
+from app.utils import APIResponse, load_description
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/evaluations", tags=["Evaluation v2"])
+
+
+@router.post(
+    "",
+    description=load_description("evaluation/create_evaluation_v2.md"),
+    response_model=APIResponse[EvaluationRunPublic],
+    dependencies=[
+        Depends(require_permission(Permission.REQUIRE_PROJECT)),
+        Depends(monitor_rate("evaluations")),
+    ],
+)
+def evaluate_v2(
+    session: SessionDep,
+    auth_context: AuthContextDep,
+    dataset_id: int = Body(..., description="ID of the evaluation dataset"),
+    experiment_name: str = Body(
+        ..., description="Name for this evaluation experiment/run"
+    ),
+    config_id: UUID = Body(..., description="Stored config ID"),
+    config_version: int = Body(..., ge=1, description="Stored config version"),
+) -> APIResponse[EvaluationRunPublic]:
+    """Start a v2 evaluation run.
+
+    v2 runs are always fast and judged on Adherence to Ground Truth; there is no
+    `run_mode` — batch judging is deferred to a later phase.
+    """
+
+    eval_run = validate_and_start_judged_evaluation(
+        session=session,
+        dataset_id=dataset_id,
+        run_name=experiment_name,
+        config_id=config_id,
+        config_version=config_version,
+        organization_id=auth_context.organization_.id,
+        project_id=auth_context.project_.id,
+        trace_id=correlation_id.get() or "N/A",
+    )
+    return APIResponse.success_response(data=eval_run)

backend/app/core/config.py

@@ -33,6 +33,8 @@ class Settings(BaseSettings):
     )
 
     API_V1_STR: str = "/api/v1"
+    # v2 hosts
+    API_V2_STR: str = "/api/v2"
     SECRET_KEY: str = secrets.token_urlsafe(32)
     # 60 minutes * 24 hours * 1 days = 1 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 1
@@ -209,6 +211,12 @@ def AWS_S3_BUCKET(self) -> str:
     # task well under CELERY_TASK_SOFT_TIME_LIMIT.
     EVAL_FAST_CHUNK_SIZE: int = 50
 
+    EVAL_JUDGE_MODEL: str = "gpt-5-mini"
+
+    # Reasoning effort for the judge model. "minimal" keeps per-row judging fast.
+    # models. One of: none | minimal | low | medium | high | xhigh.
+    EVAL_JUDGE_REASONING_EFFORT: str = "minimal"
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def COMPUTED_CELERY_WORKER_CONCURRENCY(self) -> int:

backend/app/core/security.py

@@ -13,20 +13,20 @@
 import os
 import secrets
 from datetime import UTC, datetime, timedelta
-from typing import Any
+from typing import Any, Tuple
 
 import boto3
 import jwt
+from jwt.exceptions import InvalidTokenError
 from botocore.client import BaseClient
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
-from jwt.exceptions import InvalidTokenError
 from passlib.context import CryptContext
 from sqlmodel import Session, and_, select
 
 from app.core.config import settings
-from app.models import APIKey, AuthContext, Organization, Project, User
+from app.models import APIKey, User, Organization, Project, AuthContext
 
 logger = logging.getLogger(__name__)
 
@@ -263,7 +263,7 @@ class APIKeyManager:
     pwd_context = CryptContext(schemes=[HASH_ALGORITHM], deprecated="auto")
 
     @classmethod
-    def generate(cls) -> tuple[str, str, str]:
+    def generate(cls) -> Tuple[str, str, str]:
         """
         Generate a new API key with prefix and hashed value.
         Ensures exact lengths: prefix=22 chars, secret=43 chars.
@@ -288,7 +288,7 @@ def generate(cls) -> tuple[str, str, str]:
         return raw_key, key_prefix, key_hash
 
     @classmethod
-    def _extract_key_parts(cls, raw_key: str) -> tuple[str, str] | None:
+    def _extract_key_parts(cls, raw_key: str) -> Tuple[str, str] | None:
         """
         Extract prefix and secret from an API key based on its format.