feat(Prompt Iteration): Move processing to Celery

backend/app/alembic/versions/074_add_prompt_improvement_jobtype.py

@@ -0,0 +1,27 @@
+"""add PROMPT_IMPROVEMENT jobtype enum value
+
+Revision ID: 074
+Revises: 073
+Create Date: 2026-07-09 00:00:00.000000
+
+Prompt improvement moved from a synchronous request to a Celery job, so its runs
+are tracked in the `job` table under this new jobtype. ALTER TYPE ... ADD VALUE
+must run outside a transaction, hence the autocommit block.
+"""
+
+from alembic import op
+
+revision = "074"
+down_revision = "073"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.get_context().autocommit_block():
+        op.execute("ALTER TYPE jobtype ADD VALUE IF NOT EXISTS 'PROMPT_IMPROVEMENT'")
+
+
+def downgrade() -> None:
+    # Postgres cannot drop a value from an enum type; leaving it is harmless.
+    pass

backend/app/api/docs/evaluation/improve_prompt.md

@@ -1,13 +1,59 @@
-Generate a new, AI-improved prompt iteration for the configuration that was evaluated by this run.
+Enqueue an AI prompt-improvement job for the configuration that was evaluated by this run.
 
-The run must have `status = completed` and a `score_trace_url` pointing to the stored trace file. No request body is required — the service reads the trace file directly from S3, uploads it to the Anthropic Files API, and asks Claude to identify low-performing answers (by scores and divergence from ground truth) and rewrite the system prompt to address them. Only `completion.params.instructions` is changed — model, knowledge base, and all other settings are preserved.
+The heavy LLM round-trip runs on a background worker, so this endpoint validates preconditions and returns immediately with `202 Accepted` and a `job_id`. The run must have `status = completed` and a `score_trace_url` pointing to the stored trace file. The worker reads the trace file, asks Claude to identify low-performing answers (by scores and divergence from ground truth) and rewrite the system prompt. Only `completion.params.instructions` is changed; model, knowledge base, and all other settings are preserved.
 
-The new prompt is persisted as the next `config_version` for the evaluated configuration. AI provenance and the source evaluation run id are recorded in the version's `commit_message` field (prefixed with `[AI Generated]` for auditability).
+On success the new prompt is persisted as the next `config_version` for the evaluated configuration, with AI provenance and the source evaluation run id recorded in the version's `commit_message` (prefixed with `[AI Generated]`).
 
-**Error codes:**
+**Request body (required):**
+
+```json
+{ "callback_url": "https://your-service.example.com/webhooks/prompt-improvement" }
+```
+
+- `callback_url` — HTTPS webhook that receives the result. Must be `https://` and must not resolve to a private, loopback, or cloud-metadata address.
+
+**Callback delivery:**
+
+Once the worker finishes (success or failure), it POSTs a single JSON body to `callback_url`. Delivery is **best-effort, single-attempt** — there is no retry. The `config_version` is persisted regardless of callback outcome and remains recoverable from the configuration's version list, so a failed callback loses the notification, not the work.
+
+When a webhook signing secret is configured for the project, the request carries `X-Webhook-Signature` (HMAC-SHA256 of the body) and `X-Webhook-Timestamp` headers; verify them before trusting the payload.
+
+The body is a standard `APIResponse` envelope whose `data` is a `PromptImprovementJobPublic`:
+
+```json
+{
+  "success": true,
+  "data": {
+    "job_id": "…",
+    "status": "SUCCESS",
+    "config_version": { "…": "…" },
+    "error_message": null
+  },
+  "error": null,
+  "metadata": null
+}
+```
+
+```json
+{
+  "success": false,
+  "data": {
+    "job_id": "…",
+    "status": "FAILED",
+    "config_version": null,
+    "error_message": "…"
+  },
+  "error": "…",
+  "metadata": null
+}
+```
+
+LLM and trace-download failures surface only via the failure callback, not on the `202` response.
+
+**Validation errors (returned synchronously before the job is created):**
 - `404 evaluation_not_found` — no run with this id in the caller's project.
 - `409 evaluation_not_completed` — run is not yet completed.
 - `409 source_config_unavailable` — the run's config or config_version has been deleted.
 - `422 traces_not_available` — the run has no `score_trace_url`; trace file is required.
-- `502 trace_download_failed` — the trace file could not be retrieved from storage.
-- `502 prompt_generation_failed` — the platform Anthropic key is not configured, or the LLM call failed.
+- `422 invalid_callback_url` — `callback_url` is missing, not `https://`, or resolves to a blocked (private/loopback/metadata) address.
+- `500 prompt_improvement_enqueue_failed` — the job could not be queued.

backend/app/api/routes/evaluations/evaluation.py

@@ -12,23 +12,27 @@
     Query,
 )
 
-
 from app.api.deps import AuthContextDep, SessionDep
 from app.api.permissions import Permission, require_permission
 from app.core.rate_monitor import monitor_rate
 from app.crud.evaluations import list_evaluation_runs as list_evaluation_runs_crud
 from app.crud.evaluations.core import group_traces_by_question_id
-from app.models.config.version import ConfigVersionPublic
-from app.models.evaluation import EvaluationRunPublic, RunModeEnum
+from app.models.evaluation import (
+    EvaluationRunPublic,
+    ImprovePromptRequest,
+    RunModeEnum,
+)
+from app.models.llm.response import LLMJobImmediatePublic
 from app.services.evaluations import (
     get_evaluation_with_scores,
-    improve_prompt,
+    start_prompt_improvement_job,
     validate_and_start_batch_evaluation,
     validate_and_start_fast_evaluation,
 )
 from app.utils import (
     APIResponse,
     load_description,
+    validate_callback_url,
 )
 
 logger = logging.getLogger(__name__)
@@ -208,26 +212,43 @@ def get_evaluation_run_status(
 @router.post(
     "/{evaluation_id}/improve-prompt",
     description=load_description("evaluation/improve_prompt.md"),
-    response_model=APIResponse[ConfigVersionPublic],
-    status_code=201,
+    response_model=APIResponse[LLMJobImmediatePublic],
+    status_code=202,
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
 )
 def improve_evaluation_prompt(
     evaluation_id: int,
     session: SessionDep,
     auth_context: AuthContextDep,
-) -> APIResponse[ConfigVersionPublic]:
-    """Generate an AI-improved prompt iteration from a completed evaluation run."""
+    request: ImprovePromptRequest,
+) -> APIResponse[LLMJobImmediatePublic]:
+    """Enqueue an AI prompt-improvement job for a completed evaluation run."""
     logger.info(
         f"[improve_evaluation_prompt] Starting | evaluation_id={evaluation_id} "
         f"org_id={auth_context.organization_.id} project_id={auth_context.project_.id}"
     )
 
-    new_version = improve_prompt(
+    # No global ValueError handler exists, so map the SSRF guard's ValueError to
+    # a 422 here rather than letting it surface as a 500.
+    try:
+        validate_callback_url(str(request.callback_url))
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=f"invalid_callback_url: {exc}")
+
+    job = start_prompt_improvement_job(
         session=session,
         evaluation_id=evaluation_id,
         organization_id=auth_context.organization_.id,
         project_id=auth_context.project_.id,
+        callback_url=str(request.callback_url),
     )
 
-    return APIResponse.success_response(data=new_version)
+    return APIResponse.success_response(
+        data=LLMJobImmediatePublic(
+            job_id=job.id,
+            status=job.status.value,
+            message="Prompt improvement is running; the result will be delivered to your callback_url.",
+            job_inserted_at=job.inserted_at,
+            job_updated_at=job.updated_at,
+        )
+    )

backend/app/celery/tasks/job_execution.py

@@ -246,6 +246,25 @@ def run_evaluation_batch_submission(
     )
 
 
+# Priority 6 (fast-eval tier): user-blocking interactive prompt iteration in the
+# evaluation domain, above default batch work but below core LLM call/chain jobs.
+@celery_app.task(bind=True, queue="default", priority=6)
+@gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_prompt_improvement")
+def run_prompt_improvement(self, project_id: int, job_id: str, trace_id: str, **kwargs):
+    from app.services.evaluations.prompt_improvement import execute_prompt_improvement
+
+    _set_trace(trace_id)
+    return _run_with_otel_parent(
+        self,
+        lambda: execute_prompt_improvement(
+            project_id=project_id,
+            job_id=job_id,
+            task_id=current_task.request.id,
+            **kwargs,
+        ),
+    )
+
+
 @celery_app.task(bind=True, queue="default", priority=2)
 @gevent_timeout(settings.CELERY_TASK_SOFT_TIME_LIMIT, "run_stt_batch_submission")
 def run_stt_batch_submission(

backend/app/celery/utils.py

@@ -180,6 +180,24 @@ def start_evaluation_batch_submission(
     return task_id
 
 
+def start_prompt_improvement(
+    project_id: int, job_id: str, trace_id: str = "N/A", **kwargs
+) -> str:
+    from app.celery.tasks.job_execution import run_prompt_improvement
+
+    task_id = _enqueue_with_trace_context(
+        run_prompt_improvement,
+        project_id=project_id,
+        job_id=job_id,
+        trace_id=trace_id,
+        **kwargs,
+    )
+    logger.info(
+        f"[start_prompt_improvement] Started job {job_id} with Celery task {task_id}"
+    )
+    return task_id
+
+
 def start_stt_batch_submission(
     project_id: int, job_id: str, trace_id: str = "N/A", **kwargs
 ) -> str:

backend/app/models/evaluation.py

@@ -3,12 +3,13 @@
 from typing import TYPE_CHECKING, Any, Optional
 from uuid import UUID
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, HttpUrl
 from sqlalchemy import Boolean, Column, Index, Text, UniqueConstraint, text
 from sqlalchemy.dialects.postgresql import ENUM, JSONB
 from sqlmodel import Field as SQLField, Relationship, SQLModel
 
 from app.core.util import now
+from app.models.config.version import ConfigVersionPublic
 
 if TYPE_CHECKING:
     from .batch_job import BatchJob
@@ -549,3 +550,20 @@ class EvaluationDatasetPublic(SQLModel):
     project_id: int
     inserted_at: datetime
     updated_at: datetime
+
+
+class ImprovePromptRequest(SQLModel):
+    """Body for POST /evaluations/{id}/improve-prompt."""
+
+    callback_url: HttpUrl = Field(
+        description="HTTPS webhook that receives the result once the job finishes."
+    )
+
+
+class PromptImprovementJobPublic(SQLModel):
+    """Callback payload body: the new config_version once the job succeeds."""
+
+    job_id: UUID
+    status: str
+    config_version: ConfigVersionPublic | None = None
+    error_message: str | None = None

backend/app/models/job.py

@@ -22,6 +22,7 @@ class JobType(str, Enum):
     LLM_API = "LLM_API"
     LLM_CHAIN = "LLM_CHAIN"
     LLM_GUARDRAILS = "LLM_GUARDRAILS"
+    PROMPT_IMPROVEMENT = "PROMPT_IMPROVEMENT"
 
 
 class Job(SQLModel, table=True):
@@ -67,7 +68,7 @@ class Job(SQLModel, table=True):
     job_type: JobType = Field(
         description="Type of job being executed (e.g., response, ingestion).",
         sa_column_kwargs={
-            "comment": "Type of job being executed (e.g., RESPONSE, LLM_API, LLM_CHAIN, LLM_GUARDRAILS)"
+            "comment": "Type of job being executed (e.g., RESPONSE, LLM_API, LLM_CHAIN, LLM_GUARDRAILS, PROMPT_IMPROVEMENT)"
         },
     )
     meta: dict[str, Any] | None = Field(

backend/app/services/evaluations/__init__.py

@@ -12,7 +12,9 @@
     validate_and_start_fast_evaluation,
 )
 from app.services.evaluations.prompt_improvement import (
-    improve_prompt,
+    execute_prompt_improvement,
+    start_prompt_improvement_job,
+    validate_improve_prompt,
 )
 from app.services.evaluations.validators import (
     ALLOWED_EXTENSIONS,