From eecc7e9dcdb6f820ac929f36b08d150467aae2f3 Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 10:33:10 +0300 Subject: [PATCH 1/6] test(model-onboarding): run simple, tool-call, and file payloads per path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously each path only ran file-processing cells. Now every path runs three capability payloads: - simple: plain ainvoke, assert non-empty completion - tools: full tool-calling round trip — bind get_weather, force the call, execute the tool, feed the ToolMessage back, and assert the final answer uses the result - files/: one cell per selected file attachment (unchanged behavior) files can now be empty without losing coverage: simple + tools still run. Round-trip logic verified with fake models (happy path, no-call, and ignored-result all assert correctly). README updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- testcases/model-onboarding/README.md | 22 +++- testcases/model-onboarding/src/main.py | 150 ++++++++++++++++++++----- 2 files changed, 135 insertions(+), 37 deletions(-) diff --git a/testcases/model-onboarding/README.md b/testcases/model-onboarding/README.md index 739502359..77f908e12 100644 --- a/testcases/model-onboarding/README.md +++ b/testcases/model-onboarding/README.md @@ -1,9 +1,17 @@ # model-onboarding testcase Exercises **one runtime-specified model** across the distinct `get_chat_model` -code paths it is expected to support, plus optional file attachments. Rolls -every `path × file` cell up into a single `success` boolean and asserts on both -the output and the emitted traces. +code paths it is expected to support. For **every path** it runs three +capability payloads: + +- **`simple`** — a plain text call; asserts a non-empty completion. +- **`tools`** — a full tool-calling **round trip**: bind a tool, let the model + request it, execute the tool, feed the result back, and assert the final + answer uses it. +- **`files/`** — one cell per selected file attachment (multimodal). + +Every `path × payload` cell rolls up into a single `success` boolean, asserted +alongside the emitted traces. Unlike `multimodal-invoke` (which hardcodes its model matrix), the model here is **input**. To onboard a model, edit `input.json` — no code change. @@ -31,9 +39,11 @@ Unlike `multimodal-invoke` (which hardcodes its model matrix), the model here is (and misleading) failure. - **`agenthub_config`** — AgentHub config header value; must exist in the tenant behind your `BASE_URL`. Defaults to `agentsplayground`. -- **`files`** — file attachments to test. Valid keys: `image`, `pdf`. Use `[]` - for a **text-only** model — an empty list runs a plain reachability check via - `ainvoke` instead of a multimodal call. +- **`files`** — file attachments for the `files/*` payload. Valid keys: + `image`, `pdf`. Use `[]` for a **text-only** model — the `simple` and `tools` + payloads still run; only the per-file cells are skipped. +- **`prompt`** — used by the `simple` and `files/*` payloads. The `tools` + payload uses its own fixed weather prompt. ## Prerequisites (external to the repo) diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index 303fa1a8b..ea75c286c 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -1,11 +1,19 @@ """Model-parameterized onboarding test case. Exercises a single model — supplied at runtime via ``input.json`` — across the -distinct ``get_chat_model`` code paths that model is expected to support, plus -an optional set of file attachments. Every ``path x file`` cell is invoked -independently; failures are caught per cell and rolled up into one ``success`` -boolean, mirroring the ``multimodal-invoke`` contract so this project drops into -the existing integration-test matrix unchanged. +distinct ``get_chat_model`` code paths that model is expected to support. For +every path, three capability payloads are run: + +- ``simple`` — a plain text ``ainvoke``; asserts a non-empty completion. +- ``tools`` — a full tool-calling round trip: bind a tool, let the model + request it, execute the tool, feed the ``ToolMessage`` back, and + assert the model produces a final answer that uses the result. +- ``files/*`` — one cell per selected file attachment via ``llm_call_with_files``. + +Every ``path x payload`` cell is invoked independently; failures are caught per +cell and rolled up into one ``success`` boolean, mirroring the +``multimodal-invoke`` output contract so this project drops into the existing +integration-test matrix unchanged. The model is NOT hardcoded. Edit ``input.json`` to onboard any model: @@ -20,15 +28,17 @@ } ``paths`` is explicit on purpose: a model ID is only valid on the vendor -families it actually ships on, so the caller declares which surfaces to test -rather than the code guessing from the name. ``files`` may be empty for -text-only models — an empty list runs a plain ``ainvoke`` reachability check. +families it actually ships on, so the caller declares which surfaces to test. +``files`` may be empty — the ``simple`` and ``tools`` payloads still run, only +the per-file cells are skipped. """ import logging from typing import Callable -from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.tools import tool from langgraph.checkpoint.memory import MemorySaver from langgraph.graph import END, START, MessagesState, StateGraph from pydantic import BaseModel, Field @@ -126,6 +136,28 @@ } +# --------------------------------------------------------------------------- # +# Tool-calling payload: a single deterministic tool. The prompt is written to +# force a call, and the expected answer is deterministic so the round trip can +# be asserted end to end. +# --------------------------------------------------------------------------- # +@tool +def get_weather(city: str) -> str: + """Get the current weather for a city. + + Args: + city: The city to look up the weather for. + """ + # Deterministic so the final-answer assertion is stable. + return f"The weather in {city} is 22 degrees Celsius and sunny." + + +TOOLS = [get_weather] +TOOL_PROMPT = "What is the weather in Paris? Use the get_weather tool." +# A token the final answer must contain to prove the tool result was consumed. +TOOL_ANSWER_TOKEN = "22" + + class ModelSpec(BaseModel): """Runtime specification for the model under test.""" @@ -139,8 +171,8 @@ class ModelSpec(BaseModel): ) files: list[str] = Field( default_factory=list, - description="File attachments to test; keys of FILE_REGISTRY. Empty => " - "text-only reachability check via plain ainvoke.", + description="File attachments for the 'files' payload; keys of " + "FILE_REGISTRY. Empty => only simple + tools payloads run.", ) @@ -172,17 +204,60 @@ def _build_model(path: str, model_name: str, settings: UiPathBaseSettings) -> ob return builder(model_name, settings) +async def _run_simple(model: BaseChatModel, prompt: str) -> str: + """Simple-call payload: plain ainvoke, require a non-empty completion.""" + response = await model.ainvoke([HumanMessage(content=prompt)]) + if not isinstance(response, AIMessage): + return f"✗ non-AIMessage: {type(response).__name__}" + if response.content and str(response.content).strip(): + return "✓" + return "✗ empty response" + + +async def _run_tools(model: BaseChatModel) -> str: + """Tool-calling payload: a full round trip. + + Binds the tool, forces a call, executes the tool locally, feeds the + ToolMessage back, and asserts the final answer reflects the tool result. + """ + llm = model.bind_tools(TOOLS) + messages: list = [HumanMessage(content=TOOL_PROMPT)] + + first = await llm.ainvoke(messages) + if not isinstance(first, AIMessage) or not first.tool_calls: + return "✗ no tool call requested" + + messages.append(first) + by_name = {t.name: t for t in TOOLS} + for call in first.tool_calls: + tool_obj = by_name.get(call["name"]) + if tool_obj is None: + return f"✗ unexpected tool '{call['name']}'" + result = tool_obj.invoke(call["args"]) + messages.append(ToolMessage(content=str(result), tool_call_id=call["id"])) + + final = await llm.ainvoke(messages) + if not isinstance(final, AIMessage) or not str(final.content).strip(): + return "✗ empty final answer" + if TOOL_ANSWER_TOKEN not in str(final.content): + return f"✗ final answer did not use tool result (missing '{TOOL_ANSWER_TOKEN}')" + return "✓" + + +async def _run_file( + model: BaseChatModel, prompt: str, file_info: FileInfo +) -> str: + """File-processing payload: invoke with one attached file.""" + response = await llm_call_with_files( + [HumanMessage(content=prompt)], [file_info], model + ) + if response.content and str(response.content).strip(): + return "✓" + return "✗ empty response" + + async def run_model_onboarding(state: GraphState) -> dict: spec = ModelSpec.model_validate(state["model_spec"]) - messages = [HumanMessage(content=state["prompt"])] - - # Empty files => a single text-only cell (llm_call_with_files does a plain - # ainvoke when the file list is empty). - selected_files: list[tuple[str, list[FileInfo]]] - if spec.files: - selected_files = [(name, [FILE_REGISTRY[name]]) for name in spec.files] - else: - selected_files = [("text-only", [])] try: client_settings = PlatformSettings(agenthub_config=spec.agenthub_config) @@ -214,22 +289,35 @@ async def run_model_onboarding(state: GraphState) -> dict: continue cell_results: dict[str, str] = {} - for label, files in selected_files: + + # 1. Simple call + logger.info(" simple...") + try: + cell_results["simple"] = await _run_simple(model, state["prompt"]) + except Exception as e: + cell_results["simple"] = f"✗ {str(e)[:60]}" + logger.info(f" simple: {cell_results['simple']}") + + # 2. Tool call (full round trip) + logger.info(" tools...") + try: + cell_results["tools"] = await _run_tools(model) + except Exception as e: + cell_results["tools"] = f"✗ {str(e)[:60]}" + logger.info(f" tools: {cell_results['tools']}") + + # 3. File processing — one cell per selected file + for file_name in spec.files: + label = f"files/{file_name}" logger.info(f" {label}...") try: - response: AIMessage = await llm_call_with_files( - messages, files, model + cell_results[label] = await _run_file( + model, state["prompt"], FILE_REGISTRY[file_name] ) - # Guard against an empty/blank completion counting as success. - if response.content and str(response.content).strip(): - logger.info(f" {label}: ✓") - cell_results[label] = "✓" - else: - logger.warning(f" {label}: ✗ empty response") - cell_results[label] = "✗ empty response" except Exception as e: - logger.error(f" {label}: ✗ {e}") cell_results[label] = f"✗ {str(e)[:60]}" + logger.info(f" {label}: {cell_results[label]}") + model_results[path] = cell_results summary_lines = [] From 998a5c8601c00bb6b997984f880f8b0ed17655fc Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 11:40:39 +0300 Subject: [PATCH 2/6] test(model-onboarding): pin Vertex GENERATE_CONTENT flavor explicitly The vertex path relied on autodetection to reach the Google class, so the GENERATE_CONTENT api_flavor was only covered by default, not by intent. Pin vendor_type=VERTEXAI + api_flavor=GENERATE_CONTENT so the flavor is exercised deterministically. Verified the path forwards both args to the factory. Co-Authored-By: Claude Opus 4.8 (1M context) --- testcases/model-onboarding/src/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index ea75c286c..10e18c567 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -49,6 +49,7 @@ from uipath_langchain_client.settings import ( ApiFlavor, UiPathBaseSettings, + VendorType, ) from uipath_langchain.agent.multimodal.invoke import llm_call_with_files @@ -84,10 +85,14 @@ temperature=0.0, max_tokens=200, ), - # VendorType.VERTEXAI (Google family) -> UiPathChatGoogleGenerativeAI + # VendorType.VERTEXAI (Google family) -> UiPathChatGoogleGenerativeAI. + # vendor_type + api_flavor are pinned explicitly so this path deterministically + # exercises the GENERATE_CONTENT flavor rather than relying on autodetection. "vertex": lambda model, settings: get_chat_model( model=model, client_settings=settings, + vendor_type=VendorType.VERTEXAI, + api_flavor=ApiFlavor.GENERATE_CONTENT, temperature=0.0, max_tokens=200, ), From fce4a982fa27f9776e5446d645f2f660e20c6d33 Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 14:59:44 +0300 Subject: [PATCH 3/6] feat(model-onboarding): add coded file-processing agent + agents/ structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces testcases/model-onboarding/src/agents/ as a container for coded (LangGraph) agents run by the onboarding matrix, so more agents can be added as sibling subfolders and registered in AGENT_REGISTRY. First agent: file_processing — reads a PDF/image and answers a task. It is the coded equivalent of a low-code UiPath agent (FileProcessingAgent) built and validated in Studio Web/Agent Builder; that low-code source of truth is kept alongside under file_processing/lowcode/ (agent.json + Analyze Files tool). No automated low-code->coded eject exists, so agent.py is a faithful re-implementation with the system prompt kept in sync. The file payload in main.py now routes through the coded agent. The agent receives the model built from model_spec, so model + API flavor stay configurable and the run.sh / input.json / model_spec Action contract is unchanged. Verified: graph compiles, input.json validates, and `uipath init` loads the graph (imports the agents subpackage) successfully. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../model-onboarding/src/agents/README.md | 30 +++++ .../model-onboarding/src/agents/__init__.py | 30 +++++ .../src/agents/file_processing/__init__.py | 2 + .../src/agents/file_processing/agent.py | 57 ++++++++++ .../agents/file_processing/lowcode/agent.json | 104 ++++++++++++++++++ .../file_processing/lowcode/entry-points.json | 66 +++++++++++ .../resources/Analyze Files/resource.json | 79 +++++++++++++ testcases/model-onboarding/src/main.py | 26 +++-- 8 files changed, 387 insertions(+), 7 deletions(-) create mode 100644 testcases/model-onboarding/src/agents/README.md create mode 100644 testcases/model-onboarding/src/agents/__init__.py create mode 100644 testcases/model-onboarding/src/agents/file_processing/__init__.py create mode 100644 testcases/model-onboarding/src/agents/file_processing/agent.py create mode 100644 testcases/model-onboarding/src/agents/file_processing/lowcode/agent.json create mode 100644 testcases/model-onboarding/src/agents/file_processing/lowcode/entry-points.json create mode 100644 testcases/model-onboarding/src/agents/file_processing/lowcode/resources/Analyze Files/resource.json diff --git a/testcases/model-onboarding/src/agents/README.md b/testcases/model-onboarding/src/agents/README.md new file mode 100644 index 000000000..c279fce79 --- /dev/null +++ b/testcases/model-onboarding/src/agents/README.md @@ -0,0 +1,30 @@ +# Coded agents for model-onboarding + +Each subfolder is a **coded (LangGraph) agent** exercised by the onboarding +testcase against a configurable model + API flavor (from `model_spec` in +`../../input.json`). Agents are registered in [`__init__.py`](__init__.py) via +`AGENT_REGISTRY`; every registered agent runs its payload per code path in the +onboarding grid. + +## Adding an agent + +1. Create `agents//agent.py` exposing: + - `NAME: str` — the registry key + - `async def run(model, prompt, files) -> str` — returns `"✓"` or `"✗ ..."` +2. Register it in `__init__.py`'s `AGENT_REGISTRY`. +3. Wire it into `main.py` where the payload should run. + +## `file_processing` + +Reads a single PDF or image and answers a task about it. This is the **coded +equivalent of a low-code UiPath agent** — the low-code source of truth lives in +[`file_processing/lowcode/`](file_processing/lowcode/) (`agent.json` + the +*Analyze Files* built-in tool resource), built and validated in Studio Web / +Agent Builder. + +There is no automated low-code → coded eject in the tooling, so +[`file_processing/agent.py`](file_processing/agent.py) is a faithful +re-implementation: same system prompt, same single-file analysis behavior, +expressed against `uipath_langchain`'s multimodal invoke helper. When the low-code +`agent.json` changes, update `agent.py` to match (the system prompt is the main +thing kept in sync). diff --git a/testcases/model-onboarding/src/agents/__init__.py b/testcases/model-onboarding/src/agents/__init__.py new file mode 100644 index 000000000..ffc9594eb --- /dev/null +++ b/testcases/model-onboarding/src/agents/__init__.py @@ -0,0 +1,30 @@ +"""Coded agents exercised by the model-onboarding testcase. + +Each agent is the coded (LangGraph) equivalent of a low-code UiPath agent, +re-authored here so the testcase can run it against a configurable model + +API flavor (sourced from ``model_spec`` in ``input.json``). + +Add a new coded agent by creating ``agents//agent.py`` exposing an async +``run(model, prompt, files)`` coroutine and a ``NAME`` constant, then register +it in ``AGENT_REGISTRY`` below. Every registered agent becomes a per-path cell +in the onboarding grid. +""" + +from typing import Awaitable, Callable + +from langchain_core.language_models import BaseChatModel + +from uipath_langchain.agent.multimodal.types import FileInfo + +from .file_processing.agent import NAME as FILE_PROCESSING_NAME +from .file_processing.agent import run as run_file_processing + +# An agent entry: given a built model, the prompt, and the selected files, +# returns a result string ("✓" or "✗ ..."). +AgentRunner = Callable[[BaseChatModel, str, list[FileInfo]], Awaitable[str]] + +AGENT_REGISTRY: dict[str, AgentRunner] = { + FILE_PROCESSING_NAME: run_file_processing, +} + +__all__ = ["AGENT_REGISTRY", "AgentRunner"] diff --git a/testcases/model-onboarding/src/agents/file_processing/__init__.py b/testcases/model-onboarding/src/agents/file_processing/__init__.py new file mode 100644 index 000000000..a80353db6 --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/__init__.py @@ -0,0 +1,2 @@ +"""File-processing coded agent (coded equivalent of the low-code +FileProcessingAgent built in Studio Web / Agent Builder).""" diff --git a/testcases/model-onboarding/src/agents/file_processing/agent.py b/testcases/model-onboarding/src/agents/file_processing/agent.py new file mode 100644 index 000000000..8b448af7b --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/agent.py @@ -0,0 +1,57 @@ +"""File-processing coded agent. + +Coded (LangGraph-compatible) equivalent of the low-code ``FileProcessingAgent`` +authored in Studio Web / Agent Builder. The low-code version accepts a +``job-attachment`` input plus a ``prompt`` and uses the built-in *Analyze Files* +tool to read PDF/image contents and answer the task. + +There is no automated low-code -> coded eject in the tooling, so this is a +faithful re-implementation: the same system prompt and the same single-file +analysis behavior, expressed against ``uipath_langchain``'s multimodal invoke +helper. The model (and thus API flavor) is supplied by the caller, which builds +it from ``model_spec`` in ``input.json`` — so this agent runs against whatever +model/path the onboarding matrix is exercising. + +Mirrors the low-code ``agent.json``: +- system prompt: file-processing assistant that reads the file then answers +- input: a task ``prompt`` + one file +- output: a text analysis grounded in the file +""" + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage + +from uipath_langchain.agent.multimodal.invoke import llm_call_with_files +from uipath_langchain.agent.multimodal.types import FileInfo + +NAME = "file_processing" + +# Kept in sync with the low-code agent's system message (agent.json). +SYSTEM_PROMPT = ( + "You are a file-processing assistant. You are given a single file " + "(PDF or image) and a task. Read the file's contents, then answer the task " + "concisely based only on what the file contains. If the file cannot be " + "read, say so plainly." +) + + +async def run(model: BaseChatModel, prompt: str, files: list[FileInfo]) -> str: + """Run the file-processing agent over one file. + + Args: + model: The chat model to use (already built for the target path/flavor). + prompt: The task/question to answer about the file. + files: Exactly the files to attach for this cell (one per invocation in + the onboarding grid). An empty list degrades to a plain text call. + + Returns: + ``"✓"`` on a non-empty grounded answer, otherwise ``"✗ ..."``. + """ + messages = [ + HumanMessage(content=SYSTEM_PROMPT), + HumanMessage(content=prompt), + ] + response: AIMessage = await llm_call_with_files(messages, files, model) + if response.content and str(response.content).strip(): + return "✓" + return "✗ empty response" diff --git a/testcases/model-onboarding/src/agents/file_processing/lowcode/agent.json b/testcases/model-onboarding/src/agents/file_processing/lowcode/agent.json new file mode 100644 index 000000000..aa2f463e9 --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/lowcode/agent.json @@ -0,0 +1,104 @@ +{ + "version": "1.1.0", + "settings": { + "model": "gpt-5.4", + "maxTokens": 128000, + "temperature": 0, + "engine": "basic-v2", + "maxIterations": 25, + "mode": "standard" + }, + "inputSchema": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The task or question to answer about the attached file." + }, + "fileIn": { + "$ref": "#/definitions/job-attachment" + } + }, + "required": [ + "prompt", + "fileIn" + ], + "definitions": { + "job-attachment": { + "type": "object", + "properties": { + "ID": { + "type": "string", + "description": "Orchestrator attachment key" + }, + "FullName": { + "type": "string", + "description": "File name" + }, + "MimeType": { + "type": "string", + "description": "MIME type, e.g. \"application/pdf\", \"image/png\"" + }, + "Metadata": { + "type": "object", + "description": "Dictionary of metadata", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "ID" + ], + "x-uipath-resource-kind": "JobAttachment" + } + } + }, + "outputSchema": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The agent's analysis of the attached file." + } + } + }, + "metadata": { + "storageVersion": "50.0.0", + "isConversational": false, + "showProjectCreationExperience": false, + "targetRuntime": "pythonAgent" + }, + "type": "lowCode", + "messages": [ + { + "role": "system", + "content": "You are a file-processing assistant. You are given a single file (PDF or image) and a task. Use the Analyze Files tool to read the file's contents, then answer the task concisely based only on what the file contains. If the file cannot be read, say so plainly.", + "contentTokens": [ + { + "type": "simpleText", + "rawString": "You are a file-processing assistant. You are given a single file (PDF or image) and a task. Use the Analyze Files tool to read the file's contents, then answer the task concisely based only on what the file contains. If the file cannot be read, say so plainly." + } + ] + }, + { + "role": "user", + "content": "{{input.prompt}}\n\n{{input.fileIn}}", + "contentTokens": [ + { + "type": "variable", + "rawString": "input.prompt" + }, + { + "type": "simpleText", + "rawString": "\n\n" + }, + { + "type": "variable", + "rawString": "input.fileIn" + } + ] + } + ], + "projectId": "f2a107da-b1e1-469d-9619-25a5919b657d" +} diff --git a/testcases/model-onboarding/src/agents/file_processing/lowcode/entry-points.json b/testcases/model-onboarding/src/agents/file_processing/lowcode/entry-points.json new file mode 100644 index 000000000..02c98cf88 --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/lowcode/entry-points.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://cloud.uipath.com/draft/2024-12/entry-point", + "$id": "entry-points.json", + "entryPoints": [ + { + "filePath": "/content/agent.json", + "uniqueId": "8b69f376-8da7-4380-825c-e2074874a07f", + "type": "agent", + "input": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The task or question to answer about the attached file." + }, + "fileIn": { + "$ref": "#/definitions/job-attachment" + } + }, + "required": [ + "prompt", + "fileIn" + ], + "definitions": { + "job-attachment": { + "type": "object", + "properties": { + "ID": { + "type": "string", + "description": "Orchestrator attachment key" + }, + "FullName": { + "type": "string", + "description": "File name" + }, + "MimeType": { + "type": "string", + "description": "MIME type, e.g. \"application/pdf\", \"image/png\"" + }, + "Metadata": { + "type": "object", + "description": "Dictionary of metadata", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "ID" + ], + "x-uipath-resource-kind": "JobAttachment" + } + } + }, + "output": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The agent's analysis of the attached file." + } + } + } + } + ] +} diff --git a/testcases/model-onboarding/src/agents/file_processing/lowcode/resources/Analyze Files/resource.json b/testcases/model-onboarding/src/agents/file_processing/lowcode/resources/Analyze Files/resource.json new file mode 100644 index 000000000..71a9c1a6c --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/lowcode/resources/Analyze Files/resource.json @@ -0,0 +1,79 @@ +{ + "$resourceType": "tool", + "id": "abc1b835-8dbc-4dad-bb63-16052b00157d", + "referenceKey": null, + "name": "Analyze Files", + "type": "internal", + "description": "Analyze one or more files with an LLM to extract, synthesize, or answer queries about their content.", + "isEnabled": true, + "inputSchema": { + "type": "object", + "properties": { + "attachments": { + "type": "array", + "items": { + "$ref": "#/definitions/job-attachment" + }, + "description": "Array of files, documents, images, or other attachments to process" + }, + "analysisTask": { + "type": "string", + "description": "The task, question, or instruction for processing the files" + } + }, + "required": [ + "attachments", + "analysisTask" + ], + "definitions": { + "job-attachment": { + "type": "object", + "properties": { + "ID": { + "type": "string", + "description": "Orchestrator attachment key" + }, + "FullName": { + "type": "string", + "description": "File name" + }, + "MimeType": { + "type": "string", + "description": "MIME type, e.g. \"application/pdf\", \"image/png\"" + }, + "Metadata": { + "type": "object", + "description": "Dictionary of metadata", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "ID" + ], + "x-uipath-resource-kind": "JobAttachment" + } + } + }, + "outputSchema": { + "type": "object", + "properties": { + "analysis": { + "type": "string", + "description": "Analysis result of the attachments" + } + }, + "required": [ + "analysis" + ] + }, + "settings": {}, + "guardrail": { + "policies": [] + }, + "argumentProperties": {}, + "properties": { + "toolType": "analyze-attachments" + } +} diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index 10e18c567..5a95d87c9 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -56,6 +56,18 @@ from uipath_langchain.agent.multimodal.types import FileInfo from uipath_langchain.chat.chat_model_factory import get_chat_model +# main.py is loaded as a top-level module (./src/main.py:graph), so `agents` +# is a sibling top-level package on sys.path. Fall back to loading it by path +# if the loader did not put src/ on sys.path. +try: + from agents import AGENT_REGISTRY +except ModuleNotFoundError: # pragma: no cover - defensive for alt loaders + import os + import sys + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from agents import AGENT_REGISTRY + logger = logging.getLogger(__name__) @@ -252,13 +264,13 @@ async def _run_tools(model: BaseChatModel) -> str: async def _run_file( model: BaseChatModel, prompt: str, file_info: FileInfo ) -> str: - """File-processing payload: invoke with one attached file.""" - response = await llm_call_with_files( - [HumanMessage(content=prompt)], [file_info], model - ) - if response.content and str(response.content).strip(): - return "✓" - return "✗ empty response" + """File-processing payload: delegate to the coded file_processing agent. + + The agent is the coded equivalent of the low-code FileProcessingAgent; it + receives the built model (already configured for the target path/flavor), + the task prompt, and the single file to attach. + """ + return await AGENT_REGISTRY["file_processing"](model, prompt, [file_info]) async def run_model_onboarding(state: GraphState) -> dict: From e011df890b1bb39cdfb17ae77378a94674d8ff9a Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 16:57:30 +0300 Subject: [PATCH 4/6] =?UTF-8?q?feat(model-onboarding):=20add=20is=5Ftools?= =?UTF-8?q?=20agent=20=E2=80=94=20IS=20activity=20tools=20per=20flavor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds agents/is_tools: for each configured flavor the model under test binds an Integration Service activity tool, is forced to call it, the activity executes through a real IS connection, and the result feeds back into a final answer (full round trip). Cells appear as is_tools/. Six flavors, built from live `uip is resources describe` output (paths, query/path/multipart routing, required fields all verified): azure_openai, openai, openai_v1, bedrock_converse, vertex, anthropic. Bedrock's IS connector exposes only converse. Multipart activities (openai, bedrock, vertex) use json_body_section="body"; azure carries modelId in query, vertex modelName in path. Config via model_spec.is_tools in input.json: flavors, per-flavor connection id overrides, per-flavor vendor model overrides. Defaults target the llm_gateway_automated_testing alpha tenant (connections verified Enabled via ping). Live smoke: anthropic flavor invoked end to end through the real gateway ("OK", end_turn); openai_v1 default model needs a Foundry deployment name — documented as configurable. Verified: graph compiles, input.json validates, all 6 builders produce coherent shapes, mocked round trips pass for all flavors, failure modes (no tool call / unknown flavor / IS exception) are legible. Co-Authored-By: Claude Opus 4.8 (1M context) --- testcases/model-onboarding/input.json | 12 +- .../model-onboarding/src/agents/README.md | 25 ++ .../src/agents/is_tools/__init__.py | 2 + .../src/agents/is_tools/agent.py | 303 ++++++++++++++++++ testcases/model-onboarding/src/main.py | 47 +++ 5 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 testcases/model-onboarding/src/agents/is_tools/__init__.py create mode 100644 testcases/model-onboarding/src/agents/is_tools/agent.py diff --git a/testcases/model-onboarding/input.json b/testcases/model-onboarding/input.json index b94149b73..7850b6b50 100644 --- a/testcases/model-onboarding/input.json +++ b/testcases/model-onboarding/input.json @@ -4,6 +4,16 @@ "model_name": "gpt-5.2-2025-12-11", "paths": ["azure_responses", "azure_chat_completions"], "agenthub_config": "agentsplayground", - "files": ["image", "pdf"] + "files": ["image", "pdf"], + "is_tools": { + "flavors": [ + "azure_openai", + "openai", + "openai_v1", + "bedrock_converse", + "vertex", + "anthropic" + ] + } } } diff --git a/testcases/model-onboarding/src/agents/README.md b/testcases/model-onboarding/src/agents/README.md index c279fce79..4b8a21c33 100644 --- a/testcases/model-onboarding/src/agents/README.md +++ b/testcases/model-onboarding/src/agents/README.md @@ -14,6 +14,31 @@ onboarding grid. 2. Register it in `__init__.py`'s `AGENT_REGISTRY`. 3. Wire it into `main.py` where the payload should run. +## `is_tools` + +Tests that the model under test can drive an **Integration Service activity +tool** end to end, across the BYO LLM vendor connector flavors: per selected +flavor it binds one tool wired to that flavor's IS activity, forces a call, +executes it through a real IS connection, feeds the result back, and requires +a non-empty final answer. Cells appear as `is_tools/`. + +Flavors (verified against the live tenant with `uip is resources describe`): +`azure_openai`, `openai`, `openai_v1`, `bedrock_converse`, `vertex`, +`anthropic`. Bedrock's IS connector exposes only a converse activity, so +there is no separate invoke flavor on the IS side. + +Configuration comes from `model_spec.is_tools` in `input.json` — `flavors` +(which cells run), `connections` (per-flavor IS connection id overrides), and +`models` (per-flavor vendor model/deployment for the activity payload). The +defaults in [`is_tools/agent.py`](is_tools/agent.py) target the +`llm_gateway_automated_testing` alpha tenant; **running against another org +requires overriding `connections`** or the cells fail legibly. + +Note: this agent's runner is `run_flavor(model, flavor, connection_id, +is_model)` — it needs flavor config, so it does not fit the +`(model, prompt, files)` `AGENT_REGISTRY` signature and is imported directly +by `main.py`. + ## `file_processing` Reads a single PDF or image and answers a task about it. This is the **coded diff --git a/testcases/model-onboarding/src/agents/is_tools/__init__.py b/testcases/model-onboarding/src/agents/is_tools/__init__.py new file mode 100644 index 000000000..34c16cafd --- /dev/null +++ b/testcases/model-onboarding/src/agents/is_tools/__init__.py @@ -0,0 +1,2 @@ +"""IS-tools coded agent: exercises Integration Service activity tools +across the BYO LLM vendor connector flavors.""" diff --git a/testcases/model-onboarding/src/agents/is_tools/agent.py b/testcases/model-onboarding/src/agents/is_tools/agent.py new file mode 100644 index 000000000..a8cb89499 --- /dev/null +++ b/testcases/model-onboarding/src/agents/is_tools/agent.py @@ -0,0 +1,303 @@ +"""IS-tools coded agent. + +Tests that the model under test can drive an **Integration Service activity +tool** end to end, across the BYO LLM vendor connector flavors available in +the tenant. For each selected flavor the agent: + +1. binds one LangChain tool (``ask_llm_via_gateway``) whose executor invokes + that flavor's IS activity via ``sdk.connections.invoke_activity_async``, +2. sends a prompt that forces the model to call the tool, +3. executes the activity through the flavor's IS connection, +4. feeds the ``ToolMessage`` back and requires a non-empty final answer. + +The flavor registry below was built from live ``uip is resources describe`` +output against the ``llm_gateway_automated_testing`` alpha tenant (paths, +methods, query/path/multipart routing, and required fields are all verified, +not guessed). Connection ids and vendor model names are **defaults for that +tenant** — override both per flavor via ``model_spec.is_tools`` in +``input.json`` when running against another org. + +Flavors (6 — what the IS connectors actually expose; e.g. Bedrock's connector +has only a converse activity, so there is no separate invoke flavor here): + +====================== ======================================= ============ +flavor connector / activity body style +====================== ======================================= ============ +``azure_openai`` microsoft-azureopenai / generateChat… JSON + query +``openai`` openai-openai / v2::chat::completion multipart +``openai_v1`` openai-openaiv1compliant / chatCompl… JSON +``bedrock_converse`` aws-bedrock / completion::converse multipart+qry +``vertex`` google-vertex / textCompletionUsingG… multipart+path +``anthropic`` anthropic-claude / messages JSON +====================== ======================================= ============ +""" + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Callable + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.tools import tool + +logger = logging.getLogger(__name__) + +NAME = "is_tools" + +# Prompt that forces the model under test to call the IS tool. +IS_TOOL_PROMPT = ( + "Use the ask_llm_via_gateway tool to ask this exact question: " + "'Reply with the single word OK.' Then report what the tool returned." +) + + +@dataclass +class IsFlavor: + """One IS activity flavor: how to build and invoke the activity.""" + + connector_key: str + object_path: str + default_connection_id: str + default_model: str + # Builds (activity_metadata_kwargs, activity_input) for a question. + build: Callable[[str, str], tuple[dict[str, Any], dict[str, Any]]] = field( + repr=False, default=None # type: ignore[assignment] + ) + + +def _azure_openai(model: str, question: str) -> tuple[dict, dict]: + meta = { + "object_path": "/generateChatCompletionConsolidated", + "method_name": "POST", + "content_type": "application/json", + "query_params": ["modelId", "api-version"], + "body_fields": ["prompt", "knowledge_base", "max_tokens", "temperature"], + } + body = { + "modelId": model, + "prompt": question, + "knowledge_base": False, + "max_tokens": 100, + "temperature": 0, + } + return meta, body + + +def _openai(model: str, question: str) -> tuple[dict, dict]: + meta = { + "object_path": "/v2/chat/completion", + "method_name": "POST", + "content_type": "multipart/form-data", + "multipart_params": ["body"], + "json_body_section": "body", + } + body = { + "body": { + "model": model, + "prompt": question, + "max_tokens": 100, + "temperature": 0, + } + } + return meta, body + + +def _openai_v1(model: str, question: str) -> tuple[dict, dict]: + meta = { + "object_path": "/chatCompletion", + "method_name": "POST", + "content_type": "application/json", + "body_fields": ["model", "prompt"], + } + return meta, {"model": model, "prompt": question} + + +def _bedrock_converse(model: str, question: str) -> tuple[dict, dict]: + meta = { + "object_path": "/completion/converse", + "method_name": "POST", + "content_type": "multipart/form-data", + "query_params": ["modelName"], + "multipart_params": ["body"], + "json_body_section": "body", + } + body = {"modelName": model, "body": {"prompt": question, "maxTokens": 100}} + return meta, body + + +def _vertex(model: str, question: str) -> tuple[dict, dict]: + meta = { + # modelName is a PATH parameter on this activity. + "object_path": f"/textCompletionUsingGemini/{model}", + "method_name": "POST", + "content_type": "multipart/form-data", + "multipart_params": ["body"], + "json_body_section": "body", + } + return meta, {"body": {"prompt": question, "maxOutputTokens": 100}} + + +def _anthropic(model: str, question: str) -> tuple[dict, dict]: + meta = { + "object_path": "/messages", + "method_name": "POST", + "content_type": "application/json", + "body_fields": ["model", "prompt", "maxTokens"], + } + return meta, {"model": model, "prompt": question, "maxTokens": 100} + + +# Connection ids default to the llm_gateway_automated_testing alpha tenant +# (all verified Enabled via `uip is connections ping`). Override per org via +# model_spec.is_tools.connections. +# +# default_model caveat: azure_openai and openai_v1 (Azure/Foundry-backed) +# expect a DEPLOYMENT name specific to the connection — a wrong one fails with +# DeploymentNotFound (observed live). anthropic/bedrock/vertex model ids are +# vendor-global; the anthropic default was verified live end to end. +FLAVOR_REGISTRY: dict[str, IsFlavor] = { + "azure_openai": IsFlavor( + connector_key="uipath-microsoft-azureopenai", + object_path="/generateChatCompletionConsolidated", + default_connection_id="fda2fdd1-a0ca-4a8a-bbcc-01e2a908d2ce", + default_model="gpt-4o-mini", + build=_azure_openai, + ), + "openai": IsFlavor( + connector_key="uipath-openai-openai", + object_path="/v2/chat/completion", + default_connection_id="5bc09bd6-adfa-47da-b7a6-13725b9a0404", + default_model="gpt-4o-mini", + build=_openai, + ), + "openai_v1": IsFlavor( + connector_key="uipath-openai-openaiv1compliant", + object_path="/chatCompletion", + default_connection_id="2c4118a4-f27d-4354-a4b7-bcc6e2ecaf07", + default_model="gpt-4o-mini", + build=_openai_v1, + ), + "bedrock_converse": IsFlavor( + connector_key="uipath-aws-bedrock", + object_path="/completion/converse", + default_connection_id="6452e8dc-64df-4e48-83c5-bc6e17945340", + default_model="anthropic.claude-haiku-4-5-20251001-v1:0", + build=_bedrock_converse, + ), + "vertex": IsFlavor( + connector_key="uipath-google-vertex", + object_path="/textCompletionUsingGemini/{modelName}", + default_connection_id="3cbbb133-3557-409d-8c75-7528121d6f0a", + default_model="gemini-2.5-flash", + build=_vertex, + ), + "anthropic": IsFlavor( + connector_key="uipath-anthropic-claude", + object_path="/messages", + default_connection_id="b631c4bb-8719-4cb7-823a-7aef6dab9766", + default_model="claude-haiku-4-5-20251001", + build=_anthropic, + ), +} + + +async def _invoke_is_activity( + flavor: str, connection_id: str, is_model: str, question: str +) -> str: + """Invoke one flavor's IS activity and return a compact result string. + + Isolated so tests can monkeypatch it; builds the SDK client lazily + (module-level construction breaks scaffold/introspection tooling). + """ + from uipath.platform import UiPath + from uipath.platform.connections import ( + ActivityMetadata, + ActivityParameterLocationInfo, + ) + + cfg = FLAVOR_REGISTRY[flavor] + meta_kwargs, activity_input = cfg.build(is_model, question) + + location = ActivityParameterLocationInfo( + query_params=meta_kwargs.pop("query_params", []), + path_params=meta_kwargs.pop("path_params", []), + body_fields=meta_kwargs.pop("body_fields", []), + multipart_params=meta_kwargs.pop("multipart_params", []), + ) + json_body_section = meta_kwargs.pop("json_body_section", None) + metadata = ActivityMetadata( + parameter_location_info=location, + json_body_section=json_body_section, + **meta_kwargs, + ) + + sdk = UiPath() + response = await sdk.connections.invoke_activity_async( + activity_metadata=metadata, + connection_id=connection_id, + activity_input=activity_input, + ) + # Response shapes differ per vendor; a compact JSON dump is enough for the + # model to report on, and avoids per-flavor parsing. + return json.dumps(response, default=str)[:400] + + +async def run_flavor( + model: BaseChatModel, + flavor: str, + connection_id: str | None = None, + is_model: str | None = None, +) -> str: + """Full model-driven round trip for one IS flavor. + + The model under test must request the tool; the tool executes the real IS + activity; the result is fed back; the final answer must be non-empty. + + Returns: + ``"✓"`` or ``"✗ ..."`` per the testcase cell contract. + """ + if flavor not in FLAVOR_REGISTRY: + return f"✗ unknown flavor '{flavor}'" + cfg = FLAVOR_REGISTRY[flavor] + conn_id = connection_id or cfg.default_connection_id + vendor_model = is_model or cfg.default_model + + captured: dict[str, str] = {} + + @tool + async def ask_llm_via_gateway(question: str) -> str: + """Ask a question to an LLM through the UiPath Integration Service + gateway and return its raw response. + + Args: + question: The question to send to the gateway LLM. + """ + result = await _invoke_is_activity(flavor, conn_id, vendor_model, question) + captured["is_response"] = result + return result + + llm = model.bind_tools([ask_llm_via_gateway]) + messages: list = [HumanMessage(content=IS_TOOL_PROMPT)] + + first = await llm.ainvoke(messages) + if not isinstance(first, AIMessage) or not first.tool_calls: + return "✗ no tool call requested" + + messages.append(first) + for call in first.tool_calls: + if call["name"] != "ask_llm_via_gateway": + return f"✗ unexpected tool '{call['name']}'" + try: + result = await ask_llm_via_gateway.ainvoke(call["args"]) + except Exception as e: # IS/vendor failures must surface per cell + return f"✗ IS activity failed: {str(e)[:80]}" + messages.append(ToolMessage(content=str(result), tool_call_id=call["id"])) + + if "is_response" not in captured: + return "✗ tool executed but produced no IS response" + + final = await llm.ainvoke(messages) + if not isinstance(final, AIMessage) or not str(final.content).strip(): + return "✗ empty final answer" + return "✓" diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index 5a95d87c9..eba70a3b9 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -61,12 +61,14 @@ # if the loader did not put src/ on sys.path. try: from agents import AGENT_REGISTRY + from agents.is_tools.agent import run_flavor as run_is_flavor except ModuleNotFoundError: # pragma: no cover - defensive for alt loaders import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from agents import AGENT_REGISTRY + from agents.is_tools.agent import run_flavor as run_is_flavor logger = logging.getLogger(__name__) @@ -175,6 +177,30 @@ def get_weather(city: str) -> str: TOOL_ANSWER_TOKEN = "22" +class IsToolsSpec(BaseModel): + """Configuration for the IS-tools payload (agents/is_tools). + + Each flavor binds an Integration Service activity tool to the model under + test and runs a full round trip through the tenant's IS connection. + """ + + flavors: list[str] = Field( + default_factory=list, + description="IS activity flavors to exercise; keys of " + "agents.is_tools.agent.FLAVOR_REGISTRY. Empty => payload skipped.", + ) + connections: dict[str, str] = Field( + default_factory=dict, + description="Per-flavor IS connection id overrides; defaults target " + "the llm_gateway_automated_testing alpha tenant.", + ) + models: dict[str, str] = Field( + default_factory=dict, + description="Per-flavor vendor model/deployment overrides for the " + "activity payload (NOT the model under test).", + ) + + class ModelSpec(BaseModel): """Runtime specification for the model under test.""" @@ -191,6 +217,11 @@ class ModelSpec(BaseModel): description="File attachments for the 'files' payload; keys of " "FILE_REGISTRY. Empty => only simple + tools payloads run.", ) + is_tools: IsToolsSpec = Field( + default_factory=IsToolsSpec, + description="IS activity-tool payload configuration; empty flavors " + "list => no is_tools cells.", + ) class GraphInput(BaseModel): @@ -335,6 +366,22 @@ async def run_model_onboarding(state: GraphState) -> dict: cell_results[label] = f"✗ {str(e)[:60]}" logger.info(f" {label}: {cell_results[label]}") + # 4. IS activity tools — one cell per selected flavor (model-driven + # round trip through a real Integration Service connection). + for flavor in spec.is_tools.flavors: + label = f"is_tools/{flavor}" + logger.info(f" {label}...") + try: + cell_results[label] = await run_is_flavor( + model, + flavor, + connection_id=spec.is_tools.connections.get(flavor), + is_model=spec.is_tools.models.get(flavor), + ) + except Exception as e: + cell_results[label] = f"✗ {str(e)[:60]}" + logger.info(f" {label}: {cell_results[label]}") + model_results[path] = cell_results summary_lines = [] From eed55bcd0ef42fa6c15d34891773469335e3f257 Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 18:27:15 +0300 Subject: [PATCH 5/6] refactor(model-onboarding): adopt Studio Web coded-copy format for agents Both test agents are now the coded agents generated by Studio Web's "Clone as Coded Agent" from their low-code twins, downloaded via `uip agent file get` and adapted with exactly one structural change: the module-level hardcoded llm becomes build_graph(llm, ...) so the testcase injects the model built from model_spec (model + API flavor stay configurable). The pristine generated sources are kept under each agent's coded-copy/ dir and utils.py is the generated interpolation helper, verbatim. file_processing: production ReAct stack (create_agent + Analyze Files internal tool); the adapter uploads the test file as a real platform attachment and invokes the generated graph. is_tools: replaces the hand-rolled LLM-gateway flavor registry with the generated Slack Send Message to Channel + Outlook 365 Send Email Activity tools (create_integration_tool + AgentIntegrationToolResourceConfig). IsToolsSpec becomes {connections: {slack, outlook}, prompt}; the cell is skipped when no connections are configured because a run performs real communication actions. Connection ids are injected into the generated tool configs (typed attribute path); create_agent returns an uncompiled StateGraph so adapters compile before invoking. Verified: graph compiles, input.json validates, both build_graphs construct (is_tools under stub auth env; create_integration_tool builds UiPath() at tool-creation time), guards are legible, and `uipath init` loads the graph. Co-Authored-By: Claude Opus 4.8 (1M context) --- testcases/model-onboarding/input.json | 9 +- .../model-onboarding/src/agents/README.md | 86 ++-- .../src/agents/file_processing/agent.py | 192 +++++++-- .../agents/file_processing/coded-copy/main.py | 108 +++++ .../file_processing/coded-copy/utils.py | 57 +++ .../src/agents/is_tools/agent.py | 402 ++++++------------ .../src/agents/is_tools/coded-copy/main.py | 111 +++++ .../src/agents/is_tools/coded-copy/utils.py | 57 +++ .../model-onboarding/src/agents/utils.py | 57 +++ testcases/model-onboarding/src/main.py | 56 +-- 10 files changed, 731 insertions(+), 404 deletions(-) create mode 100644 testcases/model-onboarding/src/agents/file_processing/coded-copy/main.py create mode 100644 testcases/model-onboarding/src/agents/file_processing/coded-copy/utils.py create mode 100644 testcases/model-onboarding/src/agents/is_tools/coded-copy/main.py create mode 100644 testcases/model-onboarding/src/agents/is_tools/coded-copy/utils.py create mode 100644 testcases/model-onboarding/src/agents/utils.py diff --git a/testcases/model-onboarding/input.json b/testcases/model-onboarding/input.json index 7850b6b50..d48f02a65 100644 --- a/testcases/model-onboarding/input.json +++ b/testcases/model-onboarding/input.json @@ -6,14 +6,7 @@ "agenthub_config": "agentsplayground", "files": ["image", "pdf"], "is_tools": { - "flavors": [ - "azure_openai", - "openai", - "openai_v1", - "bedrock_converse", - "vertex", - "anthropic" - ] + "connections": {} } } } diff --git a/testcases/model-onboarding/src/agents/README.md b/testcases/model-onboarding/src/agents/README.md index 4b8a21c33..74d2a77b0 100644 --- a/testcases/model-onboarding/src/agents/README.md +++ b/testcases/model-onboarding/src/agents/README.md @@ -1,55 +1,49 @@ # Coded agents for model-onboarding -Each subfolder is a **coded (LangGraph) agent** exercised by the onboarding -testcase against a configurable model + API flavor (from `model_spec` in -`../../input.json`). Agents are registered in [`__init__.py`](__init__.py) via -`AGENT_REGISTRY`; every registered agent runs its payload per code path in the -onboarding grid. +Each subfolder holds a **coded agent generated by Studio Web's "Clone as +Coded Agent"** from its low-code twin, adapted for the onboarding testcase. +The pristine generated sources are kept in each agent's `coded-copy/` +directory; the agent's `agent.py` is that file with exactly one structural +change — the module-level hardcoded `llm = get_chat_model('gpt-5.4', ...)` +becomes `build_graph(llm, ...)` so the testcase injects the model built from +`model_spec` (model + API flavor configurable). `utils.py` is the generated +interpolation helper, shared verbatim. + +Provenance (alpha tenant, org `llm_gateway_automated_testing`): + +| agent | low-code twin | coded copy project | +|---|---|---| +| `file_processing` | FileProcessingAgent (solution `1823d2d6-…`) | `e7b8598c-…` | +| `is_tools` | IsToolsAgent (solution `925fc5db-…`) | `0959f23b-…` | + +When a low-code twin changes, re-clone in Studio Web, download with +`uip agent file list/get `, refresh `coded-copy/`, and re-apply +the `build_graph` adaptation. -## Adding an agent +## `file_processing` -1. Create `agents//agent.py` exposing: - - `NAME: str` — the registry key - - `async def run(model, prompt, files) -> str` — returns `"✓"` or `"✗ ..."` -2. Register it in `__init__.py`'s `AGENT_REGISTRY`. -3. Wire it into `main.py` where the payload should run. +Reads a single PDF or image and answers a task about it, via the production +ReAct stack: `create_agent` + the *Analyze Files* internal tool +(`create_internal_tool`). The testcase adapter uploads the test file as a +real platform attachment (`sdk.attachments.upload_async`) and invokes the +generated graph. Cells: `files/`, one per selected file. ## `is_tools` -Tests that the model under test can drive an **Integration Service activity -tool** end to end, across the BYO LLM vendor connector flavors: per selected -flavor it binds one tool wired to that flavor's IS activity, forces a call, -executes it through a real IS connection, feeds the result back, and requires -a non-empty final answer. Cells appear as `is_tools/`. - -Flavors (verified against the live tenant with `uip is resources describe`): -`azure_openai`, `openai`, `openai_v1`, `bedrock_converse`, `vertex`, -`anthropic`. Bedrock's IS connector exposes only a converse activity, so -there is no separate invoke flavor on the IS side. - -Configuration comes from `model_spec.is_tools` in `input.json` — `flavors` -(which cells run), `connections` (per-flavor IS connection id overrides), and -`models` (per-flavor vendor model/deployment for the activity payload). The -defaults in [`is_tools/agent.py`](is_tools/agent.py) target the -`llm_gateway_automated_testing` alpha tenant; **running against another org -requires overriding `connections`** or the cells fail legibly. - -Note: this agent's runner is `run_flavor(model, flavor, connection_id, -is_model)` — it needs flavor config, so it does not fit the -`(model, prompt, files)` `AGENT_REGISTRY` signature and is imported directly -by `main.py`. +Tests that the model under test can drive **real Integration Service +Activity tools** — Slack *Send Message to Channel* and Outlook 365 *Send +Email* — via `create_integration_tool` + `AgentIntegrationToolResourceConfig` +exactly as generated. Cell: `is_tools` (one per path). -## `file_processing` +Configuration via `model_spec.is_tools` in `input.json`: + +- `connections` — IS connection ids by tool key (`slack`, `outlook`). + **Empty ⇒ the cell is skipped**, because a run performs real communication + actions (it actually sends the Slack message / email). +- `prompt` — the action the model must perform. -Reads a single PDF or image and answers a task about it. This is the **coded -equivalent of a low-code UiPath agent** — the low-code source of truth lives in -[`file_processing/lowcode/`](file_processing/lowcode/) (`agent.json` + the -*Analyze Files* built-in tool resource), built and validated in Studio Web / -Agent Builder. - -There is no automated low-code → coded eject in the tooling, so -[`file_processing/agent.py`](file_processing/agent.py) is a faithful -re-implementation: same system prompt, same single-file analysis behavior, -expressed against `uipath_langchain`'s multimodal invoke helper. When the low-code -`agent.json` changes, update `agent.py` to match (the system prompt is the main -thing kept in sync). +Create the Slack / Outlook 365 connections in Integration Service (or pick +them on the low-code twin in Studio Web), then put their ids in +`connections` to enable the cell. Note `create_integration_tool` requires +UiPath auth (`uipath auth` / `UIPATH_URL`) at tool-creation time — locally +unauthenticated runs fail legibly in the cell. diff --git a/testcases/model-onboarding/src/agents/file_processing/agent.py b/testcases/model-onboarding/src/agents/file_processing/agent.py index 8b448af7b..da74091b2 100644 --- a/testcases/model-onboarding/src/agents/file_processing/agent.py +++ b/testcases/model-onboarding/src/agents/file_processing/agent.py @@ -1,57 +1,161 @@ -"""File-processing coded agent. - -Coded (LangGraph-compatible) equivalent of the low-code ``FileProcessingAgent`` -authored in Studio Web / Agent Builder. The low-code version accepts a -``job-attachment`` input plus a ``prompt`` and uses the built-in *Analyze Files* -tool to read PDF/image contents and answer the task. - -There is no automated low-code -> coded eject in the tooling, so this is a -faithful re-implementation: the same system prompt and the same single-file -analysis behavior, expressed against ``uipath_langchain``'s multimodal invoke -helper. The model (and thus API flavor) is supplied by the caller, which builds -it from ``model_spec`` in ``input.json`` — so this agent runs against whatever -model/path the onboarding matrix is exercising. - -Mirrors the low-code ``agent.json``: -- system prompt: file-processing assistant that reads the file then answers -- input: a task ``prompt`` + one file -- output: a text analysis grounded in the file +"""File-processing coded agent — Studio Web "Coded copy of FileProcessingAgent". + +Source of truth: the coded agent generated by Studio Web's "Clone as Coded +Agent" from the low-code FileProcessingAgent (solution 1823d2d6-…, project +e7b8598c-…). The pristine download is kept in ``coded-copy/``; this module is +that file with exactly one structural change: the module-level hardcoded +``llm`` is replaced by ``build_graph(llm)`` so the onboarding testcase can +inject the model built from ``model_spec`` (model + API flavor configurable). + +The ``run`` adapter at the bottom uploads the test file as a real platform +attachment and invokes the generated graph — exercising the production ReAct +agent (``create_agent``) and the real *Analyze Files* internal tool. """ -from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, HumanMessage +from datetime import ( + datetime, + timezone +) +from langchain_core.messages import ( + HumanMessage, + SystemMessage +) +from pydantic import ( + BaseModel, + Field +) +from pydantic import ( + ConfigDict +) +from typing import ( + Sequence +) +from uipath.agent.models.agent import ( + AgentInternalToolResourceConfig +) +from uipath.agent.react import ( + AGENT_SYSTEM_PROMPT_TEMPLATE +) +from uipath.platform.attachments import ( + Attachment +) +from uipath_langchain.agent.react import ( + create_agent +) +from uipath_langchain.agent.tools.internal_tools import ( + create_internal_tool +) + +from agents.utils import ( + interpolate_legacy_message +) -from uipath_langchain.agent.multimodal.invoke import llm_call_with_files -from uipath_langchain.agent.multimodal.types import FileInfo +# Required alias for job attachment detection at runtime +__Job_attachment = Attachment NAME = "file_processing" -# Kept in sync with the low-code agent's system message (agent.json). -SYSTEM_PROMPT = ( - "You are a file-processing assistant. You are given a single file " - "(PDF or image) and a task. Read the file's contents, then answer the task " - "concisely based only on what the file contains. If the file cannot be " - "read, say so plainly." -) +# Input/Output Models +class AgentInput(BaseModel): + model_config = ConfigDict(extra='allow') + prompt: str = Field(..., description="The task or question to answer about the attached file.") + fileIn: Attachment -async def run(model: BaseChatModel, prompt: str, files: list[FileInfo]) -> str: - """Run the file-processing agent over one file. - Args: - model: The chat model to use (already built for the target path/flavor). - prompt: The task/question to answer about the file. - files: Exactly the files to attach for this cell (one per invocation in - the onboarding grid). An empty list degrades to a plain text call. +class AgentOutput(BaseModel): + model_config = ConfigDict(extra='allow') + content: str | None = Field(None, description="The agent's analysis of the attached file.") - Returns: - ``"✓"`` on a non-empty grounded answer, otherwise ``"✗ ..."``. - """ - messages = [ - HumanMessage(content=SYSTEM_PROMPT), - HumanMessage(content=prompt), + +# Agent Messages Function +def create_messages(state: AgentInput) -> Sequence[SystemMessage | HumanMessage]: + # Extract values safely from state + fileIn = getattr(state, 'fileIn', '') + prompt = getattr(state, 'prompt', '') + + # Apply system prompt template + current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + system_prompt_content = """You are a file-processing assistant. You are given a single file (PDF or image) and a task. Use the Analyze Files tool to read the file's contents, then answer the task concisely based only on what the file contains. If the file cannot be read, say so plainly.""" + system_prompt_content = interpolate_legacy_message(system_prompt_content, state.model_dump()) + enhanced_system_prompt = ( + AGENT_SYSTEM_PROMPT_TEMPLATE + .replace('{{systemPrompt}}', system_prompt_content) + .replace('{{currentDate}}', current_date) + .replace('{{agentName}}', 'Mr Assistant') + ) + + return [ + SystemMessage(content=enhanced_system_prompt), + HumanMessage(content=interpolate_legacy_message("""{{prompt}} + +{{fileIn}}""", state.model_dump())), ] - response: AIMessage = await llm_call_with_files(messages, files, model) - if response.content and str(response.content).strip(): + + +def build_graph(llm): + """Build the generated agent graph for an injected model. + + Mirrors the Studio Web coded copy's module body; ``llm`` replaces the + hardcoded ``get_chat_model(model='gpt-5.4', ...)`` so the testcase can + exercise any model/path from ``model_spec``. + """ + # Context Grounding Tool: analyze_files + analyze_files_config = AgentInternalToolResourceConfig( + name='Analyze Files', + description='Analyze one or more files with an LLM to extract, synthesize, or answer queries about their content.', + type='Internal', + input_schema={'type': 'object', 'properties': {'attachments': {'type': 'array', 'items': {'$ref': '#/definitions/job-attachment'}, 'description': 'Array of files, documents, images, or other attachments to process'}, 'analysisTask': {'type': 'string', 'description': 'The task, question, or instruction for processing the files'}}, 'required': ['attachments', 'analysisTask'], 'definitions': {'job-attachment': {'type': 'object', 'properties': {'ID': {'type': 'string', 'description': 'Orchestrator attachment key'}, 'FullName': {'type': 'string', 'description': 'File name'}, 'MimeType': {'type': 'string', 'description': 'MIME type, e.g. "application/pdf", "image/png"'}, 'Metadata': {'type': 'object', 'description': 'Dictionary of metadata', 'additionalProperties': {'type': 'string'}}}, 'required': ['ID'], 'x-uipath-resource-kind': 'JobAttachment'}}}, + output_schema={'type': 'object', 'properties': {'analysis': {'type': 'string', 'description': 'Analysis result of the attachments'}}, 'required': ['analysis']}, + properties={'requireConversationalConfirmation': False, 'toolType': 'analyze-attachments'}, + arguments={}, + argument_properties={} + ) + analyze_files_tool = create_internal_tool(analyze_files_config, llm) + + # Collect all tools + tools = [] + tools.append(analyze_files_tool) + + # Create agent graph + return create_agent(model=llm, messages=create_messages, tools=tools, input_schema=AgentInput, output_schema=AgentOutput) + + +async def _upload_attachment(file_info) -> Attachment: + """Fetch the test file and register it as a platform attachment.""" + import httpx + from uipath.platform import UiPath + from uipath_langchain._utils import get_httpx_client_kwargs + + async with httpx.AsyncClient(**get_httpx_client_kwargs()) as client: + response = await client.get(file_info.url) + response.raise_for_status() + content = response.content + + sdk = UiPath() + attachment_id = await sdk.attachments.upload_async( + name=file_info.name, content=content + ) + return Attachment( + id=attachment_id, + full_name=file_info.name, + mime_type=file_info.mime_type, + ) + + +async def run(model, prompt: str, files) -> str: + """Testcase adapter: run the generated graph over one attached file. + + Returns ``"✓"`` or ``"✗ ..."`` per the onboarding cell contract. + """ + if not files: + return "✗ file_processing requires a file" + attachment = await _upload_attachment(files[0]) + graph = build_graph(model) + if hasattr(graph, "compile"): # create_agent returns an uncompiled StateGraph + graph = graph.compile() + result = await graph.ainvoke(AgentInput(prompt=prompt, fileIn=attachment)) + content = (result or {}).get("content") if isinstance(result, dict) else getattr(result, "content", None) + if content and str(content).strip(): return "✓" return "✗ empty response" diff --git a/testcases/model-onboarding/src/agents/file_processing/coded-copy/main.py b/testcases/model-onboarding/src/agents/file_processing/coded-copy/main.py new file mode 100644 index 000000000..e4a564b95 --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/coded-copy/main.py @@ -0,0 +1,108 @@ +from datetime import ( + datetime, + timezone +) +from langchain_core.messages import ( + HumanMessage, + SystemMessage +) +from pydantic import ( + BaseModel, + Field +) +from pydantic import ( + ConfigDict +) +from typing import ( + Sequence +) +from uipath.agent.models.agent import ( + AgentInternalToolResourceConfig +) +from uipath.agent.react import ( + AGENT_SYSTEM_PROMPT_TEMPLATE +) +from uipath.platform.attachments import ( + Attachment +) +from uipath_langchain.agent.react import ( + create_agent +) +from uipath_langchain.agent.tools.internal_tools import ( + create_internal_tool +) +from uipath_langchain.chat.chat_model_factory import ( + get_chat_model +) +from utils import ( + interpolate_legacy_message +) + +# Required alias for job attachment detection at runtime +__Job_attachment = Attachment + + +# LLM Model Configuration +llm = get_chat_model( + model='gpt-5.4', + temperature=0.0, + max_tokens=128000, + agenthub_config="agentsruntime", +) + +# Context Grounding Tool: analyze_files +analyze_files_config = AgentInternalToolResourceConfig( + name='Analyze Files', + description='Analyze one or more files with an LLM to extract, synthesize, or answer queries about their content.', + type='Internal', + input_schema={'type': 'object', 'properties': {'attachments': {'type': 'array', 'items': {'$ref': '#/definitions/job-attachment'}, 'description': 'Array of files, documents, images, or other attachments to process'}, 'analysisTask': {'type': 'string', 'description': 'The task, question, or instruction for processing the files'}}, 'required': ['attachments', 'analysisTask'], 'definitions': {'job-attachment': {'type': 'object', 'properties': {'ID': {'type': 'string', 'description': 'Orchestrator attachment key'}, 'FullName': {'type': 'string', 'description': 'File name'}, 'MimeType': {'type': 'string', 'description': 'MIME type, e.g. "application/pdf", "image/png"'}, 'Metadata': {'type': 'object', 'description': 'Dictionary of metadata', 'additionalProperties': {'type': 'string'}}}, 'required': ['ID'], 'x-uipath-resource-kind': 'JobAttachment'}}}, + output_schema={'type': 'object', 'properties': {'analysis': {'type': 'string', 'description': 'Analysis result of the attachments'}}, 'required': ['analysis']}, + properties={'requireConversationalConfirmation': False, 'toolType': 'analyze-attachments'}, + arguments={}, + argument_properties={} +) +analyze_files_tool = create_internal_tool(analyze_files_config, llm) + + +# Collect all tools +tools = [] +tools.append(analyze_files_tool) + + +# Input/Output Models +class AgentInput(BaseModel): + model_config = ConfigDict(extra='allow') + prompt: str = Field(..., description="The task or question to answer about the attached file.") + fileIn: Attachment + + +class AgentOutput(BaseModel): + model_config = ConfigDict(extra='allow') + content: str | None = Field(None, description="The agent's analysis of the attached file.") + +# Agent Messages Function +def create_messages(state: AgentInput) -> Sequence[SystemMessage | HumanMessage]: + # Extract values safely from state + fileIn = getattr(state, 'fileIn', '') + prompt = getattr(state, 'prompt', '') + + # Apply system prompt template + current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + system_prompt_content = """You are a file-processing assistant. You are given a single file (PDF or image) and a task. Use the Analyze Files tool to read the file's contents, then answer the task concisely based only on what the file contains. If the file cannot be read, say so plainly.""" + system_prompt_content = interpolate_legacy_message(system_prompt_content, state.model_dump()) + enhanced_system_prompt = ( + AGENT_SYSTEM_PROMPT_TEMPLATE + .replace('{{systemPrompt}}', system_prompt_content) + .replace('{{currentDate}}', current_date) + .replace('{{agentName}}', 'Mr Assistant') + ) + + return [ + SystemMessage(content=enhanced_system_prompt), + HumanMessage(content=interpolate_legacy_message("""{{prompt}} + +{{fileIn}}""", state.model_dump())), + ] + +# Create agent graph +graph = create_agent(model=llm, messages=create_messages, tools=tools, input_schema=AgentInput, output_schema=AgentOutput) \ No newline at end of file diff --git a/testcases/model-onboarding/src/agents/file_processing/coded-copy/utils.py b/testcases/model-onboarding/src/agents/file_processing/coded-copy/utils.py new file mode 100644 index 000000000..f91c83f93 --- /dev/null +++ b/testcases/model-onboarding/src/agents/file_processing/coded-copy/utils.py @@ -0,0 +1,57 @@ +import json +import re +from typing import Any, Pattern + +SAFE_ARGUMENT_PATTERN: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.\[\]*]+$") + + +def safe_get_nested(data: dict[str, Any], path: str) -> Any: + """Get nested dictionary value using dot notation (e.g., "user.email").""" + keys = path.split(".") + current = data + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None + + return current + + +def serialize_argument( + value: str | int | float | bool | list[Any] | dict[str, Any] | None, +) -> str: + """Serialize value for interpolation: primitives as-is, collections as JSON.""" + if value is None: + return "" + if isinstance(value, (list, dict, bool)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def interpolate_legacy_message(content: str, input_values: dict[str, Any]) -> str: + """Replace {{variable}} placeholders with values. Supports nested paths (e.g., {{user.email}}). + + Uses whitelist pattern to prevent injection attacks. + """ + argument_placeholder_pattern = r"\{\{([^}]+)\}\}" + matches = re.findall(argument_placeholder_pattern, content) + + interpolated = content + for field_path in matches: + field_path = field_path.strip() + + # Validate field path to prevent injection + if not SAFE_ARGUMENT_PATTERN.match(field_path): + continue + + value = safe_get_nested(input_values, field_path) + + if value is not None: + placeholder = f"{{{{{field_path}}}}}" + interpolated = interpolated.replace( + placeholder, serialize_argument(str(value)) + ) + + return interpolated diff --git a/testcases/model-onboarding/src/agents/is_tools/agent.py b/testcases/model-onboarding/src/agents/is_tools/agent.py index a8cb89499..e331e3632 100644 --- a/testcases/model-onboarding/src/agents/is_tools/agent.py +++ b/testcases/model-onboarding/src/agents/is_tools/agent.py @@ -1,303 +1,147 @@ -"""IS-tools coded agent. - -Tests that the model under test can drive an **Integration Service activity -tool** end to end, across the BYO LLM vendor connector flavors available in -the tenant. For each selected flavor the agent: - -1. binds one LangChain tool (``ask_llm_via_gateway``) whose executor invokes - that flavor's IS activity via ``sdk.connections.invoke_activity_async``, -2. sends a prompt that forces the model to call the tool, -3. executes the activity through the flavor's IS connection, -4. feeds the ``ToolMessage`` back and requires a non-empty final answer. - -The flavor registry below was built from live ``uip is resources describe`` -output against the ``llm_gateway_automated_testing`` alpha tenant (paths, -methods, query/path/multipart routing, and required fields are all verified, -not guessed). Connection ids and vendor model names are **defaults for that -tenant** — override both per flavor via ``model_spec.is_tools`` in -``input.json`` when running against another org. - -Flavors (6 — what the IS connectors actually expose; e.g. Bedrock's connector -has only a converse activity, so there is no separate invoke flavor here): - -====================== ======================================= ============ -flavor connector / activity body style -====================== ======================================= ============ -``azure_openai`` microsoft-azureopenai / generateChat… JSON + query -``openai`` openai-openai / v2::chat::completion multipart -``openai_v1`` openai-openaiv1compliant / chatCompl… JSON -``bedrock_converse`` aws-bedrock / completion::converse multipart+qry -``vertex`` google-vertex / textCompletionUsingG… multipart+path -``anthropic`` anthropic-claude / messages JSON -====================== ======================================= ============ +"""IS-tools coded agent — Studio Web "Coded copy of IsToolsAgent". + +Source of truth: the coded agent generated by Studio Web's "Clone as Coded +Agent" from the low-code IsToolsAgent (solution 925fc5db-…, project +0959f23b-…). The pristine download is kept in ``coded-copy/``; this module is +that file with exactly one structural change: the module-level hardcoded +``llm`` is replaced by ``build_graph(llm, connections)`` so the onboarding +testcase can inject the model built from ``model_spec`` (model + API flavor +configurable) and the Integration Service connection ids +(``model_spec.is_tools.connections``: keys ``slack`` and ``outlook``). + +The tools are real IS Activity tools — Slack *Send Message to Channel* and +Outlook 365 *Send Email* — so a run with wired connections performs real +communication actions. Cells are skipped legibly when no connections are +configured. """ -import json -import logging -from dataclasses import dataclass, field -from typing import Any, Callable - -from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from langchain_core.tools import tool - -logger = logging.getLogger(__name__) - -NAME = "is_tools" - -# Prompt that forces the model under test to call the IS tool. -IS_TOOL_PROMPT = ( - "Use the ask_llm_via_gateway tool to ask this exact question: " - "'Reply with the single word OK.' Then report what the tool returned." +from datetime import ( + datetime, + timezone +) +from langchain_core.messages import ( + HumanMessage, + SystemMessage +) +from pydantic import ( + BaseModel, + Field +) +from pydantic import ( + ConfigDict +) +from typing import ( + Sequence +) +from uipath.agent.models.agent import ( + AgentIntegrationToolResourceConfig +) +from uipath.agent.react import ( + AGENT_SYSTEM_PROMPT_TEMPLATE +) +from uipath_langchain.agent.react import ( + create_agent +) +from uipath_langchain.agent.tools import ( + create_integration_tool +) +from agents.utils import ( + interpolate_legacy_message ) -@dataclass -class IsFlavor: - """One IS activity flavor: how to build and invoke the activity.""" - - connector_key: str - object_path: str - default_connection_id: str - default_model: str - # Builds (activity_metadata_kwargs, activity_input) for a question. - build: Callable[[str, str], tuple[dict[str, Any], dict[str, Any]]] = field( - repr=False, default=None # type: ignore[assignment] - ) - - -def _azure_openai(model: str, question: str) -> tuple[dict, dict]: - meta = { - "object_path": "/generateChatCompletionConsolidated", - "method_name": "POST", - "content_type": "application/json", - "query_params": ["modelId", "api-version"], - "body_fields": ["prompt", "knowledge_base", "max_tokens", "temperature"], - } - body = { - "modelId": model, - "prompt": question, - "knowledge_base": False, - "max_tokens": 100, - "temperature": 0, - } - return meta, body - - -def _openai(model: str, question: str) -> tuple[dict, dict]: - meta = { - "object_path": "/v2/chat/completion", - "method_name": "POST", - "content_type": "multipart/form-data", - "multipart_params": ["body"], - "json_body_section": "body", - } - body = { - "body": { - "model": model, - "prompt": question, - "max_tokens": 100, - "temperature": 0, - } - } - return meta, body - - -def _openai_v1(model: str, question: str) -> tuple[dict, dict]: - meta = { - "object_path": "/chatCompletion", - "method_name": "POST", - "content_type": "application/json", - "body_fields": ["model", "prompt"], - } - return meta, {"model": model, "prompt": question} - +# Input/Output Models +class AgentInput(BaseModel): + model_config = ConfigDict(extra='allow') + prompt: str = Field(..., description="The question to ask through the Integration Service LLM gateway tools.") -def _bedrock_converse(model: str, question: str) -> tuple[dict, dict]: - meta = { - "object_path": "/completion/converse", - "method_name": "POST", - "content_type": "multipart/form-data", - "query_params": ["modelName"], - "multipart_params": ["body"], - "json_body_section": "body", - } - body = {"modelName": model, "body": {"prompt": question, "maxTokens": 100}} - return meta, body +class AgentOutput(BaseModel): + model_config = ConfigDict(extra='allow') + content: str | None = Field(None, description="The answer assembled from the gateway tool responses.") -def _vertex(model: str, question: str) -> tuple[dict, dict]: - meta = { - # modelName is a PATH parameter on this activity. - "object_path": f"/textCompletionUsingGemini/{model}", - "method_name": "POST", - "content_type": "multipart/form-data", - "multipart_params": ["body"], - "json_body_section": "body", - } - return meta, {"body": {"prompt": question, "maxOutputTokens": 100}} +# Agent Messages Function +def create_messages(state: AgentInput) -> Sequence[SystemMessage | HumanMessage]: + # Extract values safely from state + prompt = getattr(state, 'prompt', '') -def _anthropic(model: str, question: str) -> tuple[dict, dict]: - meta = { - "object_path": "/messages", - "method_name": "POST", - "content_type": "application/json", - "body_fields": ["model", "prompt", "maxTokens"], - } - return meta, {"model": model, "prompt": question, "maxTokens": 100} + # Apply system prompt template + current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + system_prompt_content = """You are an assistant that performs communication actions using the available Integration Service tools. Send Slack messages with the 'Send Message to Channel' tool and emails with the 'Send Email' tool, filling each tool's parameters from the user's request. After acting, report the outcome of every action you took. If a tool fails, report its error plainly.""" + system_prompt_content = interpolate_legacy_message(system_prompt_content, state.model_dump()) + enhanced_system_prompt = ( + AGENT_SYSTEM_PROMPT_TEMPLATE + .replace('{{systemPrompt}}', system_prompt_content) + .replace('{{currentDate}}', current_date) + .replace('{{agentName}}', 'Mr Assistant') + ) + return [ + SystemMessage(content=enhanced_system_prompt), + HumanMessage(content=interpolate_legacy_message("""{{prompt}}""", state.model_dump())), + ] -# Connection ids default to the llm_gateway_automated_testing alpha tenant -# (all verified Enabled via `uip is connections ping`). Override per org via -# model_spec.is_tools.connections. -# -# default_model caveat: azure_openai and openai_v1 (Azure/Foundry-backed) -# expect a DEPLOYMENT name specific to the connection — a wrong one fails with -# DeploymentNotFound (observed live). anthropic/bedrock/vertex model ids are -# vendor-global; the anthropic default was verified live end to end. -FLAVOR_REGISTRY: dict[str, IsFlavor] = { - "azure_openai": IsFlavor( - connector_key="uipath-microsoft-azureopenai", - object_path="/generateChatCompletionConsolidated", - default_connection_id="fda2fdd1-a0ca-4a8a-bbcc-01e2a908d2ce", - default_model="gpt-4o-mini", - build=_azure_openai, - ), - "openai": IsFlavor( - connector_key="uipath-openai-openai", - object_path="/v2/chat/completion", - default_connection_id="5bc09bd6-adfa-47da-b7a6-13725b9a0404", - default_model="gpt-4o-mini", - build=_openai, - ), - "openai_v1": IsFlavor( - connector_key="uipath-openai-openaiv1compliant", - object_path="/chatCompletion", - default_connection_id="2c4118a4-f27d-4354-a4b7-bcc6e2ecaf07", - default_model="gpt-4o-mini", - build=_openai_v1, - ), - "bedrock_converse": IsFlavor( - connector_key="uipath-aws-bedrock", - object_path="/completion/converse", - default_connection_id="6452e8dc-64df-4e48-83c5-bc6e17945340", - default_model="anthropic.claude-haiku-4-5-20251001-v1:0", - build=_bedrock_converse, - ), - "vertex": IsFlavor( - connector_key="uipath-google-vertex", - object_path="/textCompletionUsingGemini/{modelName}", - default_connection_id="3cbbb133-3557-409d-8c75-7528121d6f0a", - default_model="gemini-2.5-flash", - build=_vertex, - ), - "anthropic": IsFlavor( - connector_key="uipath-anthropic-claude", - object_path="/messages", - default_connection_id="b631c4bb-8719-4cb7-823a-7aef6dab9766", - default_model="claude-haiku-4-5-20251001", - build=_anthropic, - ), -} +NAME = "is_tools" -async def _invoke_is_activity( - flavor: str, connection_id: str, is_model: str, question: str -) -> str: - """Invoke one flavor's IS activity and return a compact result string. +def build_graph(llm, connections): + """Build the generated agent graph for an injected model + connections. - Isolated so tests can monkeypatch it; builds the SDK client lazily - (module-level construction breaks scaffold/introspection tooling). + Mirrors the Studio Web coded copy's module body; ``llm`` replaces the + hardcoded ``get_chat_model(model='gpt-5.4', ...)`` and ``connections`` + supplies the IS connection ids the maker would pick in the designer. """ - from uipath.platform import UiPath - from uipath.platform.connections import ( - ActivityMetadata, - ActivityParameterLocationInfo, + # Integration Tool send_message_to_channel + send_message_to_channel_config = AgentIntegrationToolResourceConfig( + name='Send Message to Channel', + description='Create a(n) send_message_to_channel', + type='Integration', + input_schema={'type': 'object', 'properties': {'channel': {'type': 'string', 'title': 'Channel name/ID', 'description': 'Channel name/ID'}, 'messageToSend': {'type': 'string', 'title': 'Message', 'description': "This is the main text that people will see in the message. It's also used for notifications and in places where rich formatting isn't supported."}}, 'required': ['channel', 'messageToSend']}, + output_schema={'type': 'object', 'properties': {'app_id': {'type': 'string', 'title': 'App ID', 'description': ''}, 'blocks': {'type': 'array', 'title': 'blocks[*]', 'items': {'$ref': '#/definitions/blocks'}}, 'channel': {'type': 'string', 'title': 'Channel name/ID', 'description': 'Channel name/ID'}, 'icons.emoji': {'type': 'string', 'title': 'Icons emoji', 'description': ''}, 'message.app_id': {'type': 'string', 'title': 'Message app ID', 'description': ''}, 'message.attachments': {'type': 'array', 'title': 'message.attachments[*]', 'items': {'$ref': '#/definitions/message.attachments'}}, 'message.blocks': {'type': 'array', 'title': 'message.blocks[*]', 'items': {'$ref': '#/definitions/message.blocks'}}, 'message.bot_id': {'type': 'string', 'title': 'Bot Id', 'description': ''}, 'message.bot_profile.app_id': {'type': 'string', 'title': 'Application Id', 'description': ''}, 'message.bot_profile.deleted': {'type': 'boolean', 'title': 'Deleted', 'description': ''}, 'message.bot_profile.icons.image_36': {'type': 'string', 'title': 'Image 3 6', 'description': ''}, 'message.bot_profile.icons.image_48': {'type': 'string', 'title': 'Image 4 8', 'description': ''}, 'message.bot_profile.icons.image_72': {'type': 'string', 'title': 'Image 7 2', 'description': ''}, 'message.bot_profile.id': {'type': 'string', 'title': 'Message bot profile ID', 'description': ''}, 'message.bot_profile.name': {'type': 'string', 'title': 'Message bot profile name', 'description': ''}, 'message.bot_profile.team_id': {'type': 'string', 'title': 'Team Id', 'description': ''}, 'message.bot_profile.updated': {'type': 'integer', 'title': 'Updated', 'description': ''}, 'message.icons.emoji': {'type': 'string', 'title': 'Message icons emoji', 'description': ''}, 'message.metadata.event_payload.id': {'type': 'string', 'title': 'Message metadata event payload ID', 'description': ''}, 'message.metadata.event_payload.title': {'type': 'string', 'title': 'Message metadata event payload title', 'description': ''}, 'message.metadata.event_type': {'type': 'string', 'title': 'Message metadata event type', 'description': ''}, 'message.root.app_id': {'type': 'string', 'title': 'Message root app ID', 'description': ''}, 'message.root.blocks': {'type': 'array', 'title': 'message.root.blocks[*]', 'items': {'$ref': '#/definitions/message.root.blocks'}}, 'message.root.bot_id': {'type': 'string', 'title': 'Message root bot ID', 'description': ''}, 'message.root.icons.emoji': {'type': 'string', 'title': 'Message root icons emoji', 'description': ''}, 'message.root.is_locked': {'type': 'boolean', 'title': 'Message root is locked', 'description': ''}, 'message.root.latest_reply': {'type': 'string', 'title': 'Message root latest reply', 'description': ''}, 'message.root.metadata.event_type': {'type': 'string', 'title': 'Message root metadata event type', 'description': ''}, 'message.root.reply_count': {'type': 'integer', 'title': 'Message root reply count', 'description': ''}, 'message.root.reply_users': {'type': 'string', 'title': 'Message root reply users', 'description': ''}, 'message.root.reply_users_count': {'type': 'integer', 'title': 'Message root reply users count', 'description': ''}, 'message.root.subscribed': {'type': 'boolean', 'title': 'Message root subscribed', 'description': ''}, 'message.root.subtype': {'type': 'string', 'title': 'Message root subtype', 'description': ''}, 'message.root.text': {'type': 'string', 'title': 'Message root text', 'description': ''}, 'message.root.thread_ts': {'type': 'string', 'title': 'Message root thread ts', 'description': ''}, 'message.root.ts': {'type': 'string', 'title': 'Message root ts', 'description': ''}, 'message.root.type': {'type': 'string', 'title': 'Message root type', 'description': ''}, 'message.root.username': {'type': 'string', 'title': 'Message root username', 'description': ''}, 'message.subtype': {'type': 'string', 'title': 'Message subtype', 'description': ''}, 'message.team': {'type': 'string', 'title': 'Team', 'description': ''}, 'message.text': {'type': 'string', 'title': 'Message text', 'description': ''}, 'message.thread_ts': {'type': 'string', 'title': 'Message thread ts', 'description': ''}, 'message.ts': {'type': 'string', 'title': 'Message ts', 'description': ''}, 'message.type': {'type': 'string', 'title': 'Message type', 'description': ''}, 'message.user': {'type': 'string', 'title': 'User', 'description': ''}, 'message.username': {'type': 'string', 'title': 'Message username', 'description': ''}, 'metadata.event_payload.id': {'type': 'string', 'title': 'Metadata event payload ID', 'description': ''}, 'metadata.event_payload.title': {'type': 'string', 'title': 'Metadata event payload title', 'description': ''}, 'metadata.event_type': {'type': 'string', 'title': 'Metadata event type', 'description': ''}, 'ok': {'type': 'boolean', 'title': 'Ok', 'description': ''}, 'response_metadata.warnings': {'type': 'string', 'title': 'Response metadata warnings', 'description': ''}, 'root.app_id': {'type': 'string', 'title': 'Root app ID', 'description': ''}, 'root.blocks': {'type': 'array', 'title': 'root.blocks[*]', 'items': {'$ref': '#/definitions/root.blocks'}}, 'root.bot_id': {'type': 'string', 'title': 'Root bot ID', 'description': ''}, 'root.icons.emoji': {'type': 'string', 'title': 'Root icons emoji', 'description': ''}, 'root.is_locked': {'type': 'boolean', 'title': 'Root is locked', 'description': ''}, 'root.latest_reply': {'type': 'string', 'title': 'Root latest reply', 'description': ''}, 'root.metadata.event_type': {'type': 'string', 'title': 'Root metadata event type', 'description': ''}, 'root.reply_count': {'type': 'integer', 'title': 'Root reply count', 'description': ''}, 'root.reply_users': {'type': 'string', 'title': 'Root reply users', 'description': ''}, 'root.reply_users_count': {'type': 'integer', 'title': 'Root reply users count', 'description': ''}, 'root.subscribed': {'type': 'boolean', 'title': 'Root subscribed', 'description': ''}, 'root.subtype': {'type': 'string', 'title': 'Root subtype', 'description': ''}, 'root.text': {'type': 'string', 'title': 'Root text', 'description': ''}, 'root.thread_ts': {'type': 'string', 'title': 'Root thread ts', 'description': ''}, 'root.ts': {'type': 'string', 'title': 'Root ts', 'description': ''}, 'root.type': {'type': 'string', 'title': 'Root type', 'description': ''}, 'root.username': {'type': 'string', 'title': 'Root username', 'description': ''}, 'subtype': {'type': 'string', 'title': 'Subtype', 'description': ''}, 'thread_ts': {'type': 'string', 'title': 'Message timestamp', 'description': 'The ID (timestamp) of the message sent'}, 'ts': {'type': 'string', 'title': 'Message timestamp', 'description': 'The ID (timestamp) of the message sent'}, 'username': {'type': 'string', 'title': 'Bot name', 'description': 'Bot name'}}, 'definitions': {'blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Blocks block ID', 'description': ''}, 'text.emoji': {'type': 'boolean', 'title': 'Blocks text emoji', 'description': ''}, 'text.text': {'type': 'string', 'title': 'Blocks text text', 'description': ''}, 'text.type': {'type': 'string', 'title': 'Type', 'description': ''}, 'type': {'type': 'string', 'title': 'Blocks type', 'description': ''}}}, 'message.attachments': {'type': 'object', 'properties': {'actions.confirm.dismiss_text': {'type': 'string', 'title': 'Dismiss Text', 'description': ''}, 'actions.confirm.ok_text': {'type': 'string', 'title': 'Ok Text', 'description': ''}, 'actions.confirm.text': {'type': 'string', 'title': 'Message attachments actions confirm text', 'description': ''}, 'actions.confirm.title': {'type': 'string', 'title': 'Title', 'description': ''}, 'actions.id': {'type': 'string', 'title': 'Message attachments actions ID', 'description': ''}, 'actions.name': {'type': 'string', 'title': 'Name', 'description': ''}, 'actions.style': {'type': 'string', 'title': 'Style', 'description': ''}, 'actions.text': {'type': 'string', 'title': 'Message attachments actions text', 'description': ''}, 'actions.type': {'type': 'string', 'title': 'Message attachments actions type', 'description': ''}, 'actions.value': {'type': 'string', 'title': 'Value', 'description': ''}, 'callback_id': {'type': 'string', 'title': 'Call Back Id', 'description': ''}, 'color': {'type': 'string', 'title': 'Color', 'description': ''}, 'fallback': {'type': 'string', 'title': 'Fall Back', 'description': ''}, 'id': {'type': 'integer', 'title': 'Id', 'description': ''}, 'text': {'type': 'string', 'title': 'Message attachments text', 'description': ''}}}, 'message.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Message blocks block ID', 'description': ''}, 'text.emoji': {'type': 'boolean', 'title': 'Message blocks text emoji', 'description': ''}, 'text.text': {'type': 'string', 'title': 'Message blocks text', 'description': ''}, 'text.type': {'type': 'string', 'title': 'Message blocks text type', 'description': ''}, 'type': {'type': 'string', 'title': 'Message blocks type', 'description': ''}}}, 'message.root.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Message root blocks block ID', 'description': ''}, 'elements.elements.text': {'type': 'string', 'title': 'Message root blocks elements text', 'description': ''}, 'elements.elements.type': {'type': 'string', 'title': 'Message root blocks elements elements type', 'description': ''}, 'elements.type': {'type': 'string', 'title': 'Message root blocks elements type', 'description': ''}, 'type': {'type': 'string', 'title': 'Message root blocks type', 'description': ''}}}, 'root.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Root blocks block ID', 'description': ''}, 'elements.elements.text': {'type': 'string', 'title': 'Root blocks elements text', 'description': ''}, 'elements.elements.type': {'type': 'string', 'title': 'Root blocks elements elements type', 'description': ''}, 'elements.type': {'type': 'string', 'title': 'Root blocks elements type', 'description': ''}, 'type': {'type': 'string', 'title': 'Root blocks type', 'description': ''}}}}}, + properties={'toolPath': '/send_message_to_channel_v2', 'objectName': 'send_message_to_channel_v2', 'toolDisplayName': 'Send Message to Channel', 'toolDescription': 'Create a(n) send_message_to_channel', 'method': 'POST', 'connection': {'id': '', 'name': '', 'state': 'changed', 'apiBaseUri': '', 'elementInstanceId': 0, 'connector': {'key': 'uipath-salesforce-slack', 'name': 'Slack', 'image': 'https://alpha.uipath.com/llm_gateway_automated_testing/DefaultTenant/elements_/v3/element/elements/uipath-salesforce-slack/image', 'enabled': True}, 'isDefault': False, 'folder': {'key': '00000000-0000-0000-0000-000000000000', 'path': ''}, 'solutionProperties': {'resourceKey': '0554f0a1-40ff-4e1d-8edc-9bdc857e9fa7'}}, 'parameters': [{'name': 'send_as', 'type': 'string', 'value': 'bot', 'fieldLocation': 'query', 'displayName': 'Send as', 'description': 'Whether to send the message as the App bot i.e. using Bot token or yourself i.e. using User token?', 'position': 'primary', 'fieldVariant': 'static', 'dynamic': False, 'isCascading': False, 'sortOrder': 1, 'required': True, 'dynamicBehavior': []}, {'name': 'channel', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Channel name/ID', 'description': 'Channel name/ID', 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 2, 'required': True, 'dynamicBehavior': []}, {'name': 'messageToSend', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Message', 'description': "This is the main text that people will see in the message. It's also used for notifications and in places where rich formatting isn't supported.", 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 3, 'required': True, 'dynamicBehavior': []}], 'bodyStructure': {'contentType': 'json'}}, + settings={} ) - - cfg = FLAVOR_REGISTRY[flavor] - meta_kwargs, activity_input = cfg.build(is_model, question) - - location = ActivityParameterLocationInfo( - query_params=meta_kwargs.pop("query_params", []), - path_params=meta_kwargs.pop("path_params", []), - body_fields=meta_kwargs.pop("body_fields", []), - multipart_params=meta_kwargs.pop("multipart_params", []), - ) - json_body_section = meta_kwargs.pop("json_body_section", None) - metadata = ActivityMetadata( - parameter_location_info=location, - json_body_section=json_body_section, - **meta_kwargs, + send_message_to_channel_config.properties.connection.id = connections.get('slack', '') + send_message_to_channel_tool = create_integration_tool(send_message_to_channel_config) + + # Integration Tool send_email + send_email_config = AgentIntegrationToolResourceConfig( + name='Send Email', + description='Send the message specified in the request body using either JSON.', + type='Integration', + input_schema={'type': 'object', 'properties': {'message': {'type': 'object', 'properties': {'toRecipients': {'type': 'string', 'title': 'To', 'description': 'The main recipients of the email, separated by comma(,)'}, 'subject': {'type': 'string', 'title': 'Subject', 'description': 'The subject of the email'}, 'body': {'type': 'object', 'properties': {'content': {'type': 'string', 'title': 'Body', 'description': 'The body of the email'}}}}}}, 'required': ['message']}, + output_schema={'type': 'object', 'properties': {'status': {'type': 'string', 'title': 'Status', 'description': ''}, 'id': {'type': 'string', 'title': 'Message ID', 'description': 'A unique identifier for the sent email or saved draft.'}, 'conversationId': {'type': 'string', 'title': 'Conversation Id', 'description': 'Unique identifier for the email conversation thread. Use this with the List Emails / Get Newest Email Conversation ID filter to track incoming replies.'}, 'internetMessageId': {'type': 'string', 'title': 'Internet message Id', 'description': 'A unique identifier for the email message on the internet.'}}}, + properties={'toolPath': '/hubs/productivity/send-mail-v2', 'objectName': 'send-mail-v2', 'toolDisplayName': 'Send Email', 'toolDescription': 'Send the message specified in the request body using either JSON.', 'method': 'POST', 'connection': {'id': '', 'name': '', 'state': 'changed', 'apiBaseUri': '', 'elementInstanceId': 0, 'connector': {'key': 'uipath-microsoft-outlook365', 'name': 'Microsoft Outlook 365', 'image': 'https://alpha.uipath.com/llm_gateway_automated_testing/DefaultTenant/elements_/v3/element/elements/uipath-microsoft-outlook365/image', 'enabled': True}, 'isDefault': False, 'folder': {'key': '00000000-0000-0000-0000-000000000000', 'path': ''}, 'solutionProperties': {'resourceKey': 'c74d4a3a-257e-4752-8646-3c59663efdb8'}}, 'parameters': [{'name': 'message.toRecipients', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'To', 'description': 'The main recipients of the email, separated by comma(,)', 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 1, 'required': True, 'dynamicBehavior': []}, {'name': 'message.subject', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Subject', 'description': 'The subject of the email', 'position': 'secondary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 2, 'required': False, 'dynamicBehavior': []}, {'name': 'message.body.content', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Body', 'description': 'The body of the email', 'position': 'secondary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 3, 'required': False, 'dynamicBehavior': []}], 'bodyStructure': {'contentType': 'json'}}, + settings={} ) + send_email_config.properties.connection.id = connections.get('outlook', '') + send_email_tool = create_integration_tool(send_email_config) - sdk = UiPath() - response = await sdk.connections.invoke_activity_async( - activity_metadata=metadata, - connection_id=connection_id, - activity_input=activity_input, - ) - # Response shapes differ per vendor; a compact JSON dump is enough for the - # model to report on, and avoids per-flavor parsing. - return json.dumps(response, default=str)[:400] + # Collect all tools + tools = [] + tools.append(send_message_to_channel_tool) + tools.append(send_email_tool) + # Create agent graph + return create_agent(model=llm, messages=create_messages, tools=tools, input_schema=AgentInput, output_schema=AgentOutput) -async def run_flavor( - model: BaseChatModel, - flavor: str, - connection_id: str | None = None, - is_model: str | None = None, -) -> str: - """Full model-driven round trip for one IS flavor. - The model under test must request the tool; the tool executes the real IS - activity; the result is fed back; the final answer must be non-empty. +async def run(model, connections, prompt: str) -> str: + """Testcase adapter: one model-driven run over the real Activity tools. - Returns: - ``"✓"`` or ``"✗ ..."`` per the testcase cell contract. + Returns ``"✓"`` or ``"✗ ..."`` per the onboarding cell contract. When no + connection ids are configured the cell reports a legible skip instead of + invoking tools that cannot resolve. """ - if flavor not in FLAVOR_REGISTRY: - return f"✗ unknown flavor '{flavor}'" - cfg = FLAVOR_REGISTRY[flavor] - conn_id = connection_id or cfg.default_connection_id - vendor_model = is_model or cfg.default_model - - captured: dict[str, str] = {} - - @tool - async def ask_llm_via_gateway(question: str) -> str: - """Ask a question to an LLM through the UiPath Integration Service - gateway and return its raw response. - - Args: - question: The question to send to the gateway LLM. - """ - result = await _invoke_is_activity(flavor, conn_id, vendor_model, question) - captured["is_response"] = result - return result - - llm = model.bind_tools([ask_llm_via_gateway]) - messages: list = [HumanMessage(content=IS_TOOL_PROMPT)] - - first = await llm.ainvoke(messages) - if not isinstance(first, AIMessage) or not first.tool_calls: - return "✗ no tool call requested" - - messages.append(first) - for call in first.tool_calls: - if call["name"] != "ask_llm_via_gateway": - return f"✗ unexpected tool '{call['name']}'" - try: - result = await ask_llm_via_gateway.ainvoke(call["args"]) - except Exception as e: # IS/vendor failures must surface per cell - return f"✗ IS activity failed: {str(e)[:80]}" - messages.append(ToolMessage(content=str(result), tool_call_id=call["id"])) - - if "is_response" not in captured: - return "✗ tool executed but produced no IS response" - - final = await llm.ainvoke(messages) - if not isinstance(final, AIMessage) or not str(final.content).strip(): - return "✗ empty final answer" - return "✓" + connections = connections or {} + if not any(connections.values()): + return "✗ no IS connections configured (model_spec.is_tools.connections)" + graph = build_graph(model, connections) + if hasattr(graph, "compile"): # create_agent returns an uncompiled StateGraph + graph = graph.compile() + result = await graph.ainvoke(AgentInput(prompt=prompt)) + content = (result or {}).get("content") if isinstance(result, dict) else getattr(result, "content", None) + if content and str(content).strip(): + return "✓" + return "✗ empty response" diff --git a/testcases/model-onboarding/src/agents/is_tools/coded-copy/main.py b/testcases/model-onboarding/src/agents/is_tools/coded-copy/main.py new file mode 100644 index 000000000..bd21ee767 --- /dev/null +++ b/testcases/model-onboarding/src/agents/is_tools/coded-copy/main.py @@ -0,0 +1,111 @@ +from datetime import ( + datetime, + timezone +) +from langchain_core.messages import ( + HumanMessage, + SystemMessage +) +from pydantic import ( + BaseModel, + Field +) +from pydantic import ( + ConfigDict +) +from typing import ( + Sequence +) +from uipath.agent.models.agent import ( + AgentIntegrationToolResourceConfig +) +from uipath.agent.react import ( + AGENT_SYSTEM_PROMPT_TEMPLATE +) +from uipath_langchain.agent.react import ( + create_agent +) +from uipath_langchain.agent.tools import ( + create_integration_tool +) +from uipath_langchain.chat.chat_model_factory import ( + get_chat_model +) +from utils import ( + interpolate_legacy_message +) + + + +# LLM Model Configuration +llm = get_chat_model( + model='gpt-5.4', + temperature=0.0, + max_tokens=128000, + agenthub_config="agentsruntime", +) + +# Integration Tool send_message_to_channel +send_message_to_channel_config = AgentIntegrationToolResourceConfig( + name='Send Message to Channel', + description='Create a(n) send_message_to_channel', + type='Integration', + input_schema={'type': 'object', 'properties': {'channel': {'type': 'string', 'title': 'Channel name/ID', 'description': 'Channel name/ID'}, 'messageToSend': {'type': 'string', 'title': 'Message', 'description': "This is the main text that people will see in the message. It's also used for notifications and in places where rich formatting isn't supported."}}, 'required': ['channel', 'messageToSend']}, + output_schema={'type': 'object', 'properties': {'app_id': {'type': 'string', 'title': 'App ID', 'description': ''}, 'blocks': {'type': 'array', 'title': 'blocks[*]', 'items': {'$ref': '#/definitions/blocks'}}, 'channel': {'type': 'string', 'title': 'Channel name/ID', 'description': 'Channel name/ID'}, 'icons.emoji': {'type': 'string', 'title': 'Icons emoji', 'description': ''}, 'message.app_id': {'type': 'string', 'title': 'Message app ID', 'description': ''}, 'message.attachments': {'type': 'array', 'title': 'message.attachments[*]', 'items': {'$ref': '#/definitions/message.attachments'}}, 'message.blocks': {'type': 'array', 'title': 'message.blocks[*]', 'items': {'$ref': '#/definitions/message.blocks'}}, 'message.bot_id': {'type': 'string', 'title': 'Bot Id', 'description': ''}, 'message.bot_profile.app_id': {'type': 'string', 'title': 'Application Id', 'description': ''}, 'message.bot_profile.deleted': {'type': 'boolean', 'title': 'Deleted', 'description': ''}, 'message.bot_profile.icons.image_36': {'type': 'string', 'title': 'Image 3 6', 'description': ''}, 'message.bot_profile.icons.image_48': {'type': 'string', 'title': 'Image 4 8', 'description': ''}, 'message.bot_profile.icons.image_72': {'type': 'string', 'title': 'Image 7 2', 'description': ''}, 'message.bot_profile.id': {'type': 'string', 'title': 'Message bot profile ID', 'description': ''}, 'message.bot_profile.name': {'type': 'string', 'title': 'Message bot profile name', 'description': ''}, 'message.bot_profile.team_id': {'type': 'string', 'title': 'Team Id', 'description': ''}, 'message.bot_profile.updated': {'type': 'integer', 'title': 'Updated', 'description': ''}, 'message.icons.emoji': {'type': 'string', 'title': 'Message icons emoji', 'description': ''}, 'message.metadata.event_payload.id': {'type': 'string', 'title': 'Message metadata event payload ID', 'description': ''}, 'message.metadata.event_payload.title': {'type': 'string', 'title': 'Message metadata event payload title', 'description': ''}, 'message.metadata.event_type': {'type': 'string', 'title': 'Message metadata event type', 'description': ''}, 'message.root.app_id': {'type': 'string', 'title': 'Message root app ID', 'description': ''}, 'message.root.blocks': {'type': 'array', 'title': 'message.root.blocks[*]', 'items': {'$ref': '#/definitions/message.root.blocks'}}, 'message.root.bot_id': {'type': 'string', 'title': 'Message root bot ID', 'description': ''}, 'message.root.icons.emoji': {'type': 'string', 'title': 'Message root icons emoji', 'description': ''}, 'message.root.is_locked': {'type': 'boolean', 'title': 'Message root is locked', 'description': ''}, 'message.root.latest_reply': {'type': 'string', 'title': 'Message root latest reply', 'description': ''}, 'message.root.metadata.event_type': {'type': 'string', 'title': 'Message root metadata event type', 'description': ''}, 'message.root.reply_count': {'type': 'integer', 'title': 'Message root reply count', 'description': ''}, 'message.root.reply_users': {'type': 'string', 'title': 'Message root reply users', 'description': ''}, 'message.root.reply_users_count': {'type': 'integer', 'title': 'Message root reply users count', 'description': ''}, 'message.root.subscribed': {'type': 'boolean', 'title': 'Message root subscribed', 'description': ''}, 'message.root.subtype': {'type': 'string', 'title': 'Message root subtype', 'description': ''}, 'message.root.text': {'type': 'string', 'title': 'Message root text', 'description': ''}, 'message.root.thread_ts': {'type': 'string', 'title': 'Message root thread ts', 'description': ''}, 'message.root.ts': {'type': 'string', 'title': 'Message root ts', 'description': ''}, 'message.root.type': {'type': 'string', 'title': 'Message root type', 'description': ''}, 'message.root.username': {'type': 'string', 'title': 'Message root username', 'description': ''}, 'message.subtype': {'type': 'string', 'title': 'Message subtype', 'description': ''}, 'message.team': {'type': 'string', 'title': 'Team', 'description': ''}, 'message.text': {'type': 'string', 'title': 'Message text', 'description': ''}, 'message.thread_ts': {'type': 'string', 'title': 'Message thread ts', 'description': ''}, 'message.ts': {'type': 'string', 'title': 'Message ts', 'description': ''}, 'message.type': {'type': 'string', 'title': 'Message type', 'description': ''}, 'message.user': {'type': 'string', 'title': 'User', 'description': ''}, 'message.username': {'type': 'string', 'title': 'Message username', 'description': ''}, 'metadata.event_payload.id': {'type': 'string', 'title': 'Metadata event payload ID', 'description': ''}, 'metadata.event_payload.title': {'type': 'string', 'title': 'Metadata event payload title', 'description': ''}, 'metadata.event_type': {'type': 'string', 'title': 'Metadata event type', 'description': ''}, 'ok': {'type': 'boolean', 'title': 'Ok', 'description': ''}, 'response_metadata.warnings': {'type': 'string', 'title': 'Response metadata warnings', 'description': ''}, 'root.app_id': {'type': 'string', 'title': 'Root app ID', 'description': ''}, 'root.blocks': {'type': 'array', 'title': 'root.blocks[*]', 'items': {'$ref': '#/definitions/root.blocks'}}, 'root.bot_id': {'type': 'string', 'title': 'Root bot ID', 'description': ''}, 'root.icons.emoji': {'type': 'string', 'title': 'Root icons emoji', 'description': ''}, 'root.is_locked': {'type': 'boolean', 'title': 'Root is locked', 'description': ''}, 'root.latest_reply': {'type': 'string', 'title': 'Root latest reply', 'description': ''}, 'root.metadata.event_type': {'type': 'string', 'title': 'Root metadata event type', 'description': ''}, 'root.reply_count': {'type': 'integer', 'title': 'Root reply count', 'description': ''}, 'root.reply_users': {'type': 'string', 'title': 'Root reply users', 'description': ''}, 'root.reply_users_count': {'type': 'integer', 'title': 'Root reply users count', 'description': ''}, 'root.subscribed': {'type': 'boolean', 'title': 'Root subscribed', 'description': ''}, 'root.subtype': {'type': 'string', 'title': 'Root subtype', 'description': ''}, 'root.text': {'type': 'string', 'title': 'Root text', 'description': ''}, 'root.thread_ts': {'type': 'string', 'title': 'Root thread ts', 'description': ''}, 'root.ts': {'type': 'string', 'title': 'Root ts', 'description': ''}, 'root.type': {'type': 'string', 'title': 'Root type', 'description': ''}, 'root.username': {'type': 'string', 'title': 'Root username', 'description': ''}, 'subtype': {'type': 'string', 'title': 'Subtype', 'description': ''}, 'thread_ts': {'type': 'string', 'title': 'Message timestamp', 'description': 'The ID (timestamp) of the message sent'}, 'ts': {'type': 'string', 'title': 'Message timestamp', 'description': 'The ID (timestamp) of the message sent'}, 'username': {'type': 'string', 'title': 'Bot name', 'description': 'Bot name'}}, 'definitions': {'blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Blocks block ID', 'description': ''}, 'text.emoji': {'type': 'boolean', 'title': 'Blocks text emoji', 'description': ''}, 'text.text': {'type': 'string', 'title': 'Blocks text text', 'description': ''}, 'text.type': {'type': 'string', 'title': 'Type', 'description': ''}, 'type': {'type': 'string', 'title': 'Blocks type', 'description': ''}}}, 'message.attachments': {'type': 'object', 'properties': {'actions.confirm.dismiss_text': {'type': 'string', 'title': 'Dismiss Text', 'description': ''}, 'actions.confirm.ok_text': {'type': 'string', 'title': 'Ok Text', 'description': ''}, 'actions.confirm.text': {'type': 'string', 'title': 'Message attachments actions confirm text', 'description': ''}, 'actions.confirm.title': {'type': 'string', 'title': 'Title', 'description': ''}, 'actions.id': {'type': 'string', 'title': 'Message attachments actions ID', 'description': ''}, 'actions.name': {'type': 'string', 'title': 'Name', 'description': ''}, 'actions.style': {'type': 'string', 'title': 'Style', 'description': ''}, 'actions.text': {'type': 'string', 'title': 'Message attachments actions text', 'description': ''}, 'actions.type': {'type': 'string', 'title': 'Message attachments actions type', 'description': ''}, 'actions.value': {'type': 'string', 'title': 'Value', 'description': ''}, 'callback_id': {'type': 'string', 'title': 'Call Back Id', 'description': ''}, 'color': {'type': 'string', 'title': 'Color', 'description': ''}, 'fallback': {'type': 'string', 'title': 'Fall Back', 'description': ''}, 'id': {'type': 'integer', 'title': 'Id', 'description': ''}, 'text': {'type': 'string', 'title': 'Message attachments text', 'description': ''}}}, 'message.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Message blocks block ID', 'description': ''}, 'text.emoji': {'type': 'boolean', 'title': 'Message blocks text emoji', 'description': ''}, 'text.text': {'type': 'string', 'title': 'Message blocks text', 'description': ''}, 'text.type': {'type': 'string', 'title': 'Message blocks text type', 'description': ''}, 'type': {'type': 'string', 'title': 'Message blocks type', 'description': ''}}}, 'message.root.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Message root blocks block ID', 'description': ''}, 'elements.elements.text': {'type': 'string', 'title': 'Message root blocks elements text', 'description': ''}, 'elements.elements.type': {'type': 'string', 'title': 'Message root blocks elements elements type', 'description': ''}, 'elements.type': {'type': 'string', 'title': 'Message root blocks elements type', 'description': ''}, 'type': {'type': 'string', 'title': 'Message root blocks type', 'description': ''}}}, 'root.blocks': {'type': 'object', 'properties': {'block_id': {'type': 'string', 'title': 'Root blocks block ID', 'description': ''}, 'elements.elements.text': {'type': 'string', 'title': 'Root blocks elements text', 'description': ''}, 'elements.elements.type': {'type': 'string', 'title': 'Root blocks elements elements type', 'description': ''}, 'elements.type': {'type': 'string', 'title': 'Root blocks elements type', 'description': ''}, 'type': {'type': 'string', 'title': 'Root blocks type', 'description': ''}}}}}, + properties={'toolPath': '/send_message_to_channel_v2', 'objectName': 'send_message_to_channel_v2', 'toolDisplayName': 'Send Message to Channel', 'toolDescription': 'Create a(n) send_message_to_channel', 'method': 'POST', 'connection': {'id': '', 'name': '', 'state': 'changed', 'apiBaseUri': '', 'elementInstanceId': 0, 'connector': {'key': 'uipath-salesforce-slack', 'name': 'Slack', 'image': 'https://alpha.uipath.com/llm_gateway_automated_testing/DefaultTenant/elements_/v3/element/elements/uipath-salesforce-slack/image', 'enabled': True}, 'isDefault': False, 'folder': {'key': '00000000-0000-0000-0000-000000000000', 'path': ''}, 'solutionProperties': {'resourceKey': '0554f0a1-40ff-4e1d-8edc-9bdc857e9fa7'}}, 'parameters': [{'name': 'send_as', 'type': 'string', 'value': 'bot', 'fieldLocation': 'query', 'displayName': 'Send as', 'description': 'Whether to send the message as the App bot i.e. using Bot token or yourself i.e. using User token?', 'position': 'primary', 'fieldVariant': 'static', 'dynamic': False, 'isCascading': False, 'sortOrder': 1, 'required': True, 'dynamicBehavior': []}, {'name': 'channel', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Channel name/ID', 'description': 'Channel name/ID', 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 2, 'required': True, 'dynamicBehavior': []}, {'name': 'messageToSend', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Message', 'description': "This is the main text that people will see in the message. It's also used for notifications and in places where rich formatting isn't supported.", 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 3, 'required': True, 'dynamicBehavior': []}], 'bodyStructure': {'contentType': 'json'}}, + settings={} +) +send_message_to_channel_tool = create_integration_tool(send_message_to_channel_config) + +# Integration Tool send_email +send_email_config = AgentIntegrationToolResourceConfig( + name='Send Email', + description='Send the message specified in the request body using either JSON.', + type='Integration', + input_schema={'type': 'object', 'properties': {'message': {'type': 'object', 'properties': {'toRecipients': {'type': 'string', 'title': 'To', 'description': 'The main recipients of the email, separated by comma(,)'}, 'subject': {'type': 'string', 'title': 'Subject', 'description': 'The subject of the email'}, 'body': {'type': 'object', 'properties': {'content': {'type': 'string', 'title': 'Body', 'description': 'The body of the email'}}}}}}, 'required': ['message']}, + output_schema={'type': 'object', 'properties': {'status': {'type': 'string', 'title': 'Status', 'description': ''}, 'id': {'type': 'string', 'title': 'Message ID', 'description': 'A unique identifier for the sent email or saved draft.'}, 'conversationId': {'type': 'string', 'title': 'Conversation Id', 'description': 'Unique identifier for the email conversation thread. Use this with the List Emails / Get Newest Email Conversation ID filter to track incoming replies.'}, 'internetMessageId': {'type': 'string', 'title': 'Internet message Id', 'description': 'A unique identifier for the email message on the internet.'}}}, + properties={'toolPath': '/hubs/productivity/send-mail-v2', 'objectName': 'send-mail-v2', 'toolDisplayName': 'Send Email', 'toolDescription': 'Send the message specified in the request body using either JSON.', 'method': 'POST', 'connection': {'id': '', 'name': '', 'state': 'changed', 'apiBaseUri': '', 'elementInstanceId': 0, 'connector': {'key': 'uipath-microsoft-outlook365', 'name': 'Microsoft Outlook 365', 'image': 'https://alpha.uipath.com/llm_gateway_automated_testing/DefaultTenant/elements_/v3/element/elements/uipath-microsoft-outlook365/image', 'enabled': True}, 'isDefault': False, 'folder': {'key': '00000000-0000-0000-0000-000000000000', 'path': ''}, 'solutionProperties': {'resourceKey': 'c74d4a3a-257e-4752-8646-3c59663efdb8'}}, 'parameters': [{'name': 'message.toRecipients', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'To', 'description': 'The main recipients of the email, separated by comma(,)', 'position': 'primary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 1, 'required': True, 'dynamicBehavior': []}, {'name': 'message.subject', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Subject', 'description': 'The subject of the email', 'position': 'secondary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 2, 'required': False, 'dynamicBehavior': []}, {'name': 'message.body.content', 'type': 'string', 'value': '{{prompt}}', 'fieldLocation': 'body', 'displayName': 'Body', 'description': 'The body of the email', 'position': 'secondary', 'fieldVariant': 'dynamic', 'dynamic': True, 'isCascading': False, 'sortOrder': 3, 'required': False, 'dynamicBehavior': []}], 'bodyStructure': {'contentType': 'json'}}, + settings={} +) +send_email_tool = create_integration_tool(send_email_config) + + +# Collect all tools +tools = [] +tools.append(send_message_to_channel_tool) +tools.append(send_email_tool) + + +# Input/Output Models +class AgentInput(BaseModel): + model_config = ConfigDict(extra='allow') + prompt: str = Field(..., description="The question to ask through the Integration Service LLM gateway tools.") + + +class AgentOutput(BaseModel): + model_config = ConfigDict(extra='allow') + content: str | None = Field(None, description="The answer assembled from the gateway tool responses.") + +# Agent Messages Function +def create_messages(state: AgentInput) -> Sequence[SystemMessage | HumanMessage]: + # Extract values safely from state + prompt = getattr(state, 'prompt', '') + + # Apply system prompt template + current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + system_prompt_content = """You are an assistant that performs communication actions using the available Integration Service tools. Send Slack messages with the 'Send Message to Channel' tool and emails with the 'Send Email' tool, filling each tool's parameters from the user's request. After acting, report the outcome of every action you took. If a tool fails, report its error plainly.""" + system_prompt_content = interpolate_legacy_message(system_prompt_content, state.model_dump()) + enhanced_system_prompt = ( + AGENT_SYSTEM_PROMPT_TEMPLATE + .replace('{{systemPrompt}}', system_prompt_content) + .replace('{{currentDate}}', current_date) + .replace('{{agentName}}', 'Mr Assistant') + ) + + return [ + SystemMessage(content=enhanced_system_prompt), + HumanMessage(content=interpolate_legacy_message("""{{prompt}}""", state.model_dump())), + ] + +# Create agent graph +graph = create_agent(model=llm, messages=create_messages, tools=tools, input_schema=AgentInput, output_schema=AgentOutput) \ No newline at end of file diff --git a/testcases/model-onboarding/src/agents/is_tools/coded-copy/utils.py b/testcases/model-onboarding/src/agents/is_tools/coded-copy/utils.py new file mode 100644 index 000000000..f91c83f93 --- /dev/null +++ b/testcases/model-onboarding/src/agents/is_tools/coded-copy/utils.py @@ -0,0 +1,57 @@ +import json +import re +from typing import Any, Pattern + +SAFE_ARGUMENT_PATTERN: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.\[\]*]+$") + + +def safe_get_nested(data: dict[str, Any], path: str) -> Any: + """Get nested dictionary value using dot notation (e.g., "user.email").""" + keys = path.split(".") + current = data + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None + + return current + + +def serialize_argument( + value: str | int | float | bool | list[Any] | dict[str, Any] | None, +) -> str: + """Serialize value for interpolation: primitives as-is, collections as JSON.""" + if value is None: + return "" + if isinstance(value, (list, dict, bool)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def interpolate_legacy_message(content: str, input_values: dict[str, Any]) -> str: + """Replace {{variable}} placeholders with values. Supports nested paths (e.g., {{user.email}}). + + Uses whitelist pattern to prevent injection attacks. + """ + argument_placeholder_pattern = r"\{\{([^}]+)\}\}" + matches = re.findall(argument_placeholder_pattern, content) + + interpolated = content + for field_path in matches: + field_path = field_path.strip() + + # Validate field path to prevent injection + if not SAFE_ARGUMENT_PATTERN.match(field_path): + continue + + value = safe_get_nested(input_values, field_path) + + if value is not None: + placeholder = f"{{{{{field_path}}}}}" + interpolated = interpolated.replace( + placeholder, serialize_argument(str(value)) + ) + + return interpolated diff --git a/testcases/model-onboarding/src/agents/utils.py b/testcases/model-onboarding/src/agents/utils.py new file mode 100644 index 000000000..f91c83f93 --- /dev/null +++ b/testcases/model-onboarding/src/agents/utils.py @@ -0,0 +1,57 @@ +import json +import re +from typing import Any, Pattern + +SAFE_ARGUMENT_PATTERN: Pattern[str] = re.compile(r"^[a-zA-Z0-9_.\[\]*]+$") + + +def safe_get_nested(data: dict[str, Any], path: str) -> Any: + """Get nested dictionary value using dot notation (e.g., "user.email").""" + keys = path.split(".") + current = data + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None + + return current + + +def serialize_argument( + value: str | int | float | bool | list[Any] | dict[str, Any] | None, +) -> str: + """Serialize value for interpolation: primitives as-is, collections as JSON.""" + if value is None: + return "" + if isinstance(value, (list, dict, bool)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def interpolate_legacy_message(content: str, input_values: dict[str, Any]) -> str: + """Replace {{variable}} placeholders with values. Supports nested paths (e.g., {{user.email}}). + + Uses whitelist pattern to prevent injection attacks. + """ + argument_placeholder_pattern = r"\{\{([^}]+)\}\}" + matches = re.findall(argument_placeholder_pattern, content) + + interpolated = content + for field_path in matches: + field_path = field_path.strip() + + # Validate field path to prevent injection + if not SAFE_ARGUMENT_PATTERN.match(field_path): + continue + + value = safe_get_nested(input_values, field_path) + + if value is not None: + placeholder = f"{{{{{field_path}}}}}" + interpolated = interpolated.replace( + placeholder, serialize_argument(str(value)) + ) + + return interpolated diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index eba70a3b9..3af4ee5bd 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -61,14 +61,14 @@ # if the loader did not put src/ on sys.path. try: from agents import AGENT_REGISTRY - from agents.is_tools.agent import run_flavor as run_is_flavor + from agents.is_tools.agent import run as run_is_tools except ModuleNotFoundError: # pragma: no cover - defensive for alt loaders import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from agents import AGENT_REGISTRY - from agents.is_tools.agent import run_flavor as run_is_flavor + from agents.is_tools.agent import run as run_is_tools logger = logging.getLogger(__name__) @@ -180,24 +180,27 @@ def get_weather(city: str) -> str: class IsToolsSpec(BaseModel): """Configuration for the IS-tools payload (agents/is_tools). - Each flavor binds an Integration Service activity tool to the model under - test and runs a full round trip through the tenant's IS connection. + The agent is the Studio Web coded copy of the low-code IsToolsAgent: the + model under test drives real Integration Service Activity tools (Slack + Send Message to Channel, Outlook 365 Send Email). Running it performs + real communication actions, so the payload only runs when connection ids + are configured. """ - flavors: list[str] = Field( - default_factory=list, - description="IS activity flavors to exercise; keys of " - "agents.is_tools.agent.FLAVOR_REGISTRY. Empty => payload skipped.", - ) connections: dict[str, str] = Field( default_factory=dict, - description="Per-flavor IS connection id overrides; defaults target " - "the llm_gateway_automated_testing alpha tenant.", + description="IS connection ids by tool key ('slack', 'outlook'). " + "Empty => payload skipped (no is_tools cell).", ) - models: dict[str, str] = Field( - default_factory=dict, - description="Per-flavor vendor model/deployment overrides for the " - "activity payload (NOT the model under test).", + prompt: str = Field( + default=( + "Send a Slack message saying 'model onboarding connectivity " + "test' to the channel #model-onboarding-tests, then send an " + "email with subject 'model onboarding connectivity test' and " + "the same body to model-onboarding-tests@uipath.com." + ), + description="The communication action the model must perform via " + "the Activity tools.", ) @@ -366,21 +369,20 @@ async def run_model_onboarding(state: GraphState) -> dict: cell_results[label] = f"✗ {str(e)[:60]}" logger.info(f" {label}: {cell_results[label]}") - # 4. IS activity tools — one cell per selected flavor (model-driven - # round trip through a real Integration Service connection). - for flavor in spec.is_tools.flavors: - label = f"is_tools/{flavor}" - logger.info(f" {label}...") + # 4. IS Activity tools — the model drives real Slack/Outlook tools + # (Studio Web coded copy). Skipped when no connections are wired, + # because a run performs real communication actions. + if any(spec.is_tools.connections.values()): + logger.info(" is_tools...") try: - cell_results[label] = await run_is_flavor( - model, - flavor, - connection_id=spec.is_tools.connections.get(flavor), - is_model=spec.is_tools.models.get(flavor), + cell_results["is_tools"] = await run_is_tools( + model, spec.is_tools.connections, spec.is_tools.prompt ) except Exception as e: - cell_results[label] = f"✗ {str(e)[:60]}" - logger.info(f" {label}: {cell_results[label]}") + cell_results["is_tools"] = f"✗ {str(e)[:60]}" + logger.info(f" is_tools: {cell_results['is_tools']}") + else: + logger.info(" is_tools: skipped (no connections configured)") model_results[path] = cell_results From 3f31fb44267758fdd656c021fdfdc031e2960a5d Mon Sep 17 00:00:00 2001 From: denispetre Date: Fri, 24 Jul 2026 18:32:03 +0300 Subject: [PATCH 6/6] =?UTF-8?q?test(model-onboarding):=20make=20is=5Ftools?= =?UTF-8?q?=20call-only=20=E2=80=94=20assert=20the=20tool=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: the onboarding test only needs the model to produce the tool call, not to execute it. run(model, prompt) now binds the generated Slack + Outlook Activity tools, sends the exact production messages (create_messages), and asserts >=1 well-formed tool call back: known bound tool name (the factory underscores display names -> Send_Message_to_Channel, Send_Email) and required args present (channel+messageToSend / message.toRecipients). Tools are never executed, so no IS connections are needed, nothing is sent, and the cell now runs on every onboarding pass. IsToolsSpec drops connections (prompt only); build_graph(llm, connections) remains for deliberate full executed runs. Verified all outcomes with fake models: both-tools ✓ with names, missing-args ✗, unknown-tool ✗, no-call ✗; graph compiles and input.json validates. Co-Authored-By: Claude Opus 4.8 (1M context) --- testcases/model-onboarding/input.json | 4 +- .../model-onboarding/src/agents/README.md | 34 +++---- .../src/agents/is_tools/agent.py | 88 +++++++++++++------ testcases/model-onboarding/src/main.py | 42 ++++----- 4 files changed, 95 insertions(+), 73 deletions(-) diff --git a/testcases/model-onboarding/input.json b/testcases/model-onboarding/input.json index d48f02a65..607790434 100644 --- a/testcases/model-onboarding/input.json +++ b/testcases/model-onboarding/input.json @@ -5,8 +5,6 @@ "paths": ["azure_responses", "azure_chat_completions"], "agenthub_config": "agentsplayground", "files": ["image", "pdf"], - "is_tools": { - "connections": {} - } + "is_tools": {} } } diff --git a/testcases/model-onboarding/src/agents/README.md b/testcases/model-onboarding/src/agents/README.md index 74d2a77b0..91d1cc48e 100644 --- a/testcases/model-onboarding/src/agents/README.md +++ b/testcases/model-onboarding/src/agents/README.md @@ -30,20 +30,20 @@ generated graph. Cells: `files/`, one per selected file. ## `is_tools` -Tests that the model under test can drive **real Integration Service -Activity tools** — Slack *Send Message to Channel* and Outlook 365 *Send -Email* — via `create_integration_tool` + `AgentIntegrationToolResourceConfig` -exactly as generated. Cell: `is_tools` (one per path). - -Configuration via `model_spec.is_tools` in `input.json`: - -- `connections` — IS connection ids by tool key (`slack`, `outlook`). - **Empty ⇒ the cell is skipped**, because a run performs real communication - actions (it actually sends the Slack message / email). -- `prompt` — the action the model must perform. - -Create the Slack / Outlook 365 connections in Integration Service (or pick -them on the low-code twin in Studio Web), then put their ids in -`connections` to enable the cell. Note `create_integration_tool` requires -UiPath auth (`uipath auth` / `UIPATH_URL`) at tool-creation time — locally -unauthenticated runs fail legibly in the cell. +Tests that the model under test **produces a well-formed tool call** against +the real Integration Service Activity tools — Slack *Send Message to +Channel* and Outlook 365 *Send Email* — built via `create_integration_tool` ++ `AgentIntegrationToolResourceConfig` exactly as generated. Cell: +`is_tools` (one per path), reporting the called tools, e.g. +`✓ (Send Message to Channel, Send Email)`. + +**Call-only:** the tools are bound to the model but never executed — no IS +connections are needed and nothing is actually sent, so the cell runs on +every onboarding pass. Asserts ≥1 tool call with the right name and required +args (`channel` + `messageToSend`; `message.toRecipients`). + +`model_spec.is_tools.prompt` overrides the communication request. For a full +executed run (real Slack/email side effects), `build_graph(llm, connections)` +remains — wire real connection ids and invoke it deliberately; the testcase +never does. Note `create_integration_tool` requires UiPath auth at +tool-creation time — locally unauthenticated runs fail legibly in the cell. diff --git a/testcases/model-onboarding/src/agents/is_tools/agent.py b/testcases/model-onboarding/src/agents/is_tools/agent.py index e331e3632..2fdd94c94 100644 --- a/testcases/model-onboarding/src/agents/is_tools/agent.py +++ b/testcases/model-onboarding/src/agents/is_tools/agent.py @@ -4,15 +4,16 @@ Agent" from the low-code IsToolsAgent (solution 925fc5db-…, project 0959f23b-…). The pristine download is kept in ``coded-copy/``; this module is that file with exactly one structural change: the module-level hardcoded -``llm`` is replaced by ``build_graph(llm, connections)`` so the onboarding -testcase can inject the model built from ``model_spec`` (model + API flavor -configurable) and the Integration Service connection ids -(``model_spec.is_tools.connections``: keys ``slack`` and ``outlook``). +``llm`` is replaced by ``build_tools()`` / ``build_graph(llm)`` so the +onboarding testcase can inject the model built from ``model_spec`` (model + +API flavor configurable). The tools are real IS Activity tools — Slack *Send Message to Channel* and -Outlook 365 *Send Email* — so a run with wired connections performs real -communication actions. Cells are skipped legibly when no connections are -configured. +Outlook 365 *Send Email*. The testcase ``run`` is **call-only**: it binds the +generated tools to the model under test and asserts the model produces a +well-formed tool call. The tools are never executed, so no connections are +needed and nothing is actually sent. ``build_graph`` remains for full +executed runs (requires real connection ids; performs real actions). """ from datetime import ( @@ -85,13 +86,14 @@ def create_messages(state: AgentInput) -> Sequence[SystemMessage | HumanMessage] NAME = "is_tools" -def build_graph(llm, connections): - """Build the generated agent graph for an injected model + connections. +def build_tools(connections=None): + """Build the generated IS Activity tools (Studio Web coded copy, verbatim). - Mirrors the Studio Web coded copy's module body; ``llm`` replaces the - hardcoded ``get_chat_model(model='gpt-5.4', ...)`` and ``connections`` - supplies the IS connection ids the maker would pick in the designer. + ``connections`` optionally supplies the IS connection ids the maker would + pick in the designer. For the call-only onboarding test no connections are + needed — the tools are bound to the model but never executed. """ + connections = connections or {} # Integration Tool send_message_to_channel send_message_to_channel_config = AgentIntegrationToolResourceConfig( name='Send Message to Channel', @@ -122,26 +124,56 @@ def build_graph(llm, connections): tools = [] tools.append(send_message_to_channel_tool) tools.append(send_email_tool) + return tools + + +def build_graph(llm, connections=None): + """Build the generated agent graph (full ReAct run — executes the tools). + + Mirrors the Studio Web coded copy's module body; ``llm`` replaces the + hardcoded ``get_chat_model(model='gpt-5.4', ...)``. Only use with real + connection ids — a run performs real communication actions. + """ + tools = build_tools(connections) # Create agent graph return create_agent(model=llm, messages=create_messages, tools=tools, input_schema=AgentInput, output_schema=AgentOutput) -async def run(model, connections, prompt: str) -> str: - """Testcase adapter: one model-driven run over the real Activity tools. +# Required args the model must fill for a call to count as well-formed. +# Keys are the BOUND tool names (create_integration_tool underscores them). +_REQUIRED_ARGS = { + "Send_Message_to_Channel": lambda a: a.get("channel") and a.get("messageToSend"), + "Send_Email": lambda a: (a.get("message") or {}).get("toRecipients"), +} + + +async def run(model, prompt: str) -> str: + """Testcase adapter: call-only — assert the model produces the tool call. - Returns ``"✓"`` or ``"✗ ..."`` per the onboarding cell contract. When no - connection ids are configured the cell reports a legible skip instead of - invoking tools that cannot resolve. + Binds the generated Activity tools to the model under test, sends the + exact production messages (``create_messages``), and requires at least one + well-formed tool call back. The tools are NEVER executed, so no IS + connections are needed and nothing is actually sent. + + Returns ``"✓ (names)"`` or ``"✗ ..."`` per the onboarding cell contract. """ - connections = connections or {} - if not any(connections.values()): - return "✗ no IS connections configured (model_spec.is_tools.connections)" - graph = build_graph(model, connections) - if hasattr(graph, "compile"): # create_agent returns an uncompiled StateGraph - graph = graph.compile() - result = await graph.ainvoke(AgentInput(prompt=prompt)) - content = (result or {}).get("content") if isinstance(result, dict) else getattr(result, "content", None) - if content and str(content).strip(): - return "✓" - return "✗ empty response" + tools = build_tools() + messages = list(create_messages(AgentInput(prompt=prompt))) + llm = model.bind_tools(tools) + + response = await llm.ainvoke(messages) + calls = getattr(response, "tool_calls", None) or [] + if not calls: + return "✗ no tool call requested" + + known = {t.name for t in tools} + well_formed = [] + for call in calls: + if call["name"] not in known: + return f"✗ unexpected tool '{call['name']}'" + check = _REQUIRED_ARGS.get(call["name"]) + if check and not check(call.get("args") or {}): + return f"✗ tool call '{call['name']}' missing required args" + well_formed.append(call["name"]) + return f"✓ ({', '.join(well_formed)})" diff --git a/testcases/model-onboarding/src/main.py b/testcases/model-onboarding/src/main.py index 3af4ee5bd..878025962 100644 --- a/testcases/model-onboarding/src/main.py +++ b/testcases/model-onboarding/src/main.py @@ -181,17 +181,12 @@ class IsToolsSpec(BaseModel): """Configuration for the IS-tools payload (agents/is_tools). The agent is the Studio Web coded copy of the low-code IsToolsAgent: the - model under test drives real Integration Service Activity tools (Slack - Send Message to Channel, Outlook 365 Send Email). Running it performs - real communication actions, so the payload only runs when connection ids - are configured. + model under test is bound to real Integration Service Activity tools + (Slack Send Message to Channel, Outlook 365 Send Email) and must produce + a well-formed tool call. Call-only — the tools are never executed, so no + connections are needed and nothing is actually sent. """ - connections: dict[str, str] = Field( - default_factory=dict, - description="IS connection ids by tool key ('slack', 'outlook'). " - "Empty => payload skipped (no is_tools cell).", - ) prompt: str = Field( default=( "Send a Slack message saying 'model onboarding connectivity " @@ -199,8 +194,8 @@ class IsToolsSpec(BaseModel): "email with subject 'model onboarding connectivity test' and " "the same body to model-onboarding-tests@uipath.com." ), - description="The communication action the model must perform via " - "the Activity tools.", + description="The communication request the model must translate " + "into Activity tool calls.", ) @@ -369,20 +364,17 @@ async def run_model_onboarding(state: GraphState) -> dict: cell_results[label] = f"✗ {str(e)[:60]}" logger.info(f" {label}: {cell_results[label]}") - # 4. IS Activity tools — the model drives real Slack/Outlook tools - # (Studio Web coded copy). Skipped when no connections are wired, - # because a run performs real communication actions. - if any(spec.is_tools.connections.values()): - logger.info(" is_tools...") - try: - cell_results["is_tools"] = await run_is_tools( - model, spec.is_tools.connections, spec.is_tools.prompt - ) - except Exception as e: - cell_results["is_tools"] = f"✗ {str(e)[:60]}" - logger.info(f" is_tools: {cell_results['is_tools']}") - else: - logger.info(" is_tools: skipped (no connections configured)") + # 4. IS Activity tools — call-only: the model is bound to the real + # Slack/Outlook Activity tools (Studio Web coded copy) and must + # produce a well-formed tool call. Nothing is executed or sent. + logger.info(" is_tools...") + try: + cell_results["is_tools"] = await run_is_tools( + model, spec.is_tools.prompt + ) + except Exception as e: + cell_results["is_tools"] = f"✗ {str(e)[:60]}" + logger.info(f" is_tools: {cell_results['is_tools']}") model_results[path] = cell_results