feat(evals): Update prompt iteration flow

backend/app/api/docs/evaluation/improve_prompt_v2.md

@@ -0,0 +1,29 @@
+Enqueue a v2 prompt-recommendation job for the configuration evaluated by a judged run.
+
+Unlike v1 (which reads cosine similarity + correctness), this consumes the native
+three-metric judge results — **Adherence to Ground Truth**, **Adherence to Prompt**,
+and **Adherence to Knowledge Base** — each carrying a 0–1 score and the judge's
+reasoning. The worker reads both the score and the reasoning per metric, focuses the
+rewrite on rows where Adherence to Prompt / Ground Truth are low, and changes only
+`completion.params.instructions`; model, knowledge base, and all other settings are
+preserved. Low Adherence to Knowledge Base is treated as a retrieval/KB gap rather
+than a prompt fault.
+
+The run must be a **judged v2 run** (`is_judge_run = true`), `status = completed`,
+and have a `score_trace_url`. The request body is identical to v1 — `callback_url`
+only. The LLM round-trip runs on a background worker, so this returns `202 Accepted`
+with a `job_id` immediately.
+
+The success/failure result is POSTed to `callback_url` as an `APIResponse` envelope
+whose `data` is a `PromptRecommendationJobPublic`. It adds `recommendation_type`,
+which is `"prompt"` for now; knowledge-base and model recommendation types are
+deferred to a later phase but the field keeps the callback API extensible.
+
+**Validation errors (returned synchronously before the job is created):**
+- `404 evaluation_not_found` — no run with this id in the caller's project.
+- `409 evaluation_not_completed` — run is not yet completed.
+- `409 source_config_unavailable` — the run's config or config_version has been deleted.
+- `422 traces_not_available` — the run has no `score_trace_url`.
+- `422 not_a_judge_run` — the run is not a judged (v2) run.
+- `422 invalid_callback_url` — `callback_url` is missing, not `https://`, or resolves to a blocked (private/loopback/metadata) address.
+- `500 prompt_improvement_enqueue_failed` — the job could not be queued.

backend/app/api/main.py

@@ -41,6 +41,9 @@
     router as evaluations_dataset_v2_router,
 )
 from app.api.routes.evaluations.evaluation_v2 import router as evaluations_v2_router
+from app.api.routes.evaluations.prompt_improvement_v2 import (
+    router as evaluations_prompt_improvement_v2_router,
+)
 
 api_router = APIRouter()
 api_router.include_router(analytics.router)
@@ -85,3 +88,4 @@
 api_v2_router = APIRouter()
 api_v2_router.include_router(evaluations_v2_router)
 api_v2_router.include_router(evaluations_dataset_v2_router)
+api_v2_router.include_router(evaluations_prompt_improvement_v2_router)

backend/app/api/routes/evaluations/prompt_improvement_v2.py

@@ -0,0 +1,59 @@
+"""v2 prompt-improvement trigger — consumes three-metric judge results.
+
+Mirrors the v1 improve-prompt route but requires a judged (v2) run and returns a
+typed prompt recommendation. The v1 route is left unchanged.
+"""
+
+import logging
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.api.deps import AuthContextDep, SessionDep
+from app.api.permissions import Permission, require_permission
+from app.models.evaluation import ImprovePromptRequest
+from app.models.llm.response import LLMJobImmediatePublic
+from app.services.evaluations import start_prompt_improvement_job
+from app.utils import APIResponse, load_description, validate_callback_url
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/evaluations", tags=["Evaluation v2"])
+
+
+@router.post(
+    "/{evaluation_id}/improve-prompt",
+    description=load_description("evaluation/improve_prompt_v2.md"),
+    response_model=APIResponse[LLMJobImmediatePublic],
+    status_code=202,
+    dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
+)
+def improve_evaluation_prompt_v2(
+    evaluation_id: int,
+    session: SessionDep,
+    auth_context: AuthContextDep,
+    request: ImprovePromptRequest,
+) -> APIResponse[LLMJobImmediatePublic]:
+    """Enqueue a v2 prompt-recommendation job for a judged evaluation run."""
+    try:
+        validate_callback_url(str(request.callback_url))
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=f"invalid_callback_url: {exc}")
+
+    job = start_prompt_improvement_job(
+        session=session,
+        evaluation_id=evaluation_id,
+        organization_id=auth_context.organization_.id,
+        project_id=auth_context.project_.id,
+        callback_url=str(request.callback_url),
+        require_judge_run=True,
+    )
+
+    return APIResponse.success_response(
+        data=LLMJobImmediatePublic(
+            job_id=job.id,
+            status=job.status.value,
+            message="Prompt recommendation is running; the result will be delivered to your callback_url.",
+            job_inserted_at=job.inserted_at,
+            job_updated_at=job.updated_at,
+        )
+    )

backend/app/models/evaluation.py

@@ -583,3 +583,30 @@ class PromptImprovementJobPublic(SQLModel):
     status: str
     config_version: ConfigVersionPublic | None = None
     error_message: str | None = None
+
+
+class RecommendationTypeEnum(str, Enum):
+    """Kind of recommendation a v2 improvement job returns.
+
+    Only prompt rewrites ship now; KNOWLEDGE_BASE and MODEL are deferred to a
+    later phase but named here so the callback API stays extensible.
+    """
+
+    PROMPT = "prompt"
+    # KNOWLEDGE_BASE = "knowledge_base"  # deferred
+    # MODEL = "model"  # deferred
+
+
+class PromptRecommendationJobPublic(SQLModel):
+    """v2 callback payload: a typed recommendation from a judged (v2) run.
+
+    Distinct from the v1 PromptImprovementJobPublic so the v1 callback shape
+    stays byte-for-byte unchanged; adds recommendation_type as the extensibility
+    hook for future KB/model recommendations.
+    """
+
+    job_id: UUID
+    status: str
+    recommendation_type: RecommendationTypeEnum = RecommendationTypeEnum.PROMPT
+    config_version: ConfigVersionPublic | None = None
+    error_message: str | None = None