From e4e0f7fb9d7919fb7a8b6ae38533cbe879c145a6 Mon Sep 17 00:00:00 2001 From: diego-ferrand Date: Mon, 20 Jul 2026 11:13:37 -0300 Subject: [PATCH 1/2] Add perfecto_skills tool and AI Scriptless getting-started skill --- main.py | 11 + pyproject.toml | 2 +- .../skills/perfecto-ai-scriptless/SKILL.md | 90 ++++++ .../references/ai-commands.md | 47 +++ .../ai-user-actions-best-practices.md | 69 +++++ .../ai-validations-best-practices.md | 58 ++++ .../ai-visual-comparison-best-practices.md | 32 ++ .../references/control-constructs.md | 57 ++++ .../references/execute-tests.md | 33 ++ .../references/getting-started.md | 38 +++ .../references/handle-tests.md | 53 ++++ .../references/interface.md | 63 ++++ server.py | 2 + tests/test_skills_manager_security.py | 58 ++++ tests/test_skills_utils_security.py | 110 +++++++ tools/ai_scriptless_manager.py | 2 +- tools/skills_manager.py | 266 ++++++++++++++++ tools/skills_utils.py | 289 ++++++++++++++++++ 18 files changed, 1278 insertions(+), 2 deletions(-) create mode 100644 resources/skills/perfecto-ai-scriptless/SKILL.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/ai-commands.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/ai-user-actions-best-practices.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/ai-validations-best-practices.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/ai-visual-comparison-best-practices.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/control-constructs.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/execute-tests.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/getting-started.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/handle-tests.md create mode 100644 resources/skills/perfecto-ai-scriptless/references/interface.md create mode 100644 tests/test_skills_manager_security.py create mode 100644 tests/test_skills_utils_security.py create mode 100644 tools/skills_manager.py create mode 100644 tools/skills_utils.py diff --git a/main.py b/main.py index 016e47b..87d155f 100644 --- a/main.py +++ b/main.py @@ -70,6 +70,17 @@ def run(log_level: str = "CRITICAL"): instructions = """ # Perfecto MCP Server +## Proactive Knowledge Consultation + +- **ALWAYS consult Perfecto Skills and Help tools first** before answering questions about Perfecto features, authoring AI Scriptless tests, interpreting results, troubleshooting, or providing recommendations. +- **Use `perfecto_skills`**: Access specialized knowledge about Perfecto capabilities (starting with AI Scriptless), best practices, and official getting-started guidance. +- **Use `perfecto_help`**: Consult live Perfecto documentation (category_id='perfecto', subcategory_id_list=['ide'] for AI Scriptless). +- **Golden rule**: If you're not 100% certain about something related to Perfecto, consult Skills or Help first, and if you need to search online, prioritize help.perfecto.io. + +## Important Guidelines +- **Batch Operations**: When making multiple calls to the same tool, check if that tool supports a `batch` action and use it instead of separate calls. +- **Don't assume / Don't invent**: If something is unclear, consult Skills/Help before responding. +- **Provide resources**: Include markdown-formatted links to Perfecto help documentation when relevant. """ mcp = FastMCP("perfecto-mcp", instructions=instructions, diff --git a/pyproject.toml b/pyproject.toml index 4d10b99..2a934f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ where = ["."] include = ["tools*", "config", "models", "formatters", "resources"] [tool.setuptools.package-data] -"resources" = ["*.png"] +"resources" = ["**/*"] [tool.pytest.ini_options] pythonpath = ["."] diff --git a/resources/skills/perfecto-ai-scriptless/SKILL.md b/resources/skills/perfecto-ai-scriptless/SKILL.md new file mode 100644 index 0000000..f605516 --- /dev/null +++ b/resources/skills/perfecto-ai-scriptless/SKILL.md @@ -0,0 +1,90 @@ +--- +name: perfecto-ai-scriptless +description: Getting started and authoring guidance for Perfecto AI Scriptless (Scriptless Mobile), including the workspace UI, test lifecycle, execution, control constructs, AI commands, and best practices. Use when working with AI Scriptless for (1) Understanding the interface and how to access it, (2) Creating, opening, saving, or deleting tests, (3) Adding commands, checkpoints, loops, conditions, or logical steps, (4) Executing tests on devices, (5) Writing AI user actions, validations, or visual comparisons, (6) Using Perfecto MCP perfecto_ai_scriptless tools, or any other AI Scriptless tasks. +--- + +# Perfecto AI Scriptless + +Guides for building, running, and managing Perfecto AI Scriptless tests — with MCP tool workflows for the Perfecto MCP server. + +Official docs hub: [AI Scriptless interface](https://help.perfecto.io/perfecto-help/content/perfecto/ide/scriptless-mobile-interface.htm) + +## Overview + +AI Scriptless lets you automate mobile (and, with the right licenses, desktop web) testing without writing traditional scripts. You design tests with a visual editor, predefined commands/checkpoints, and AI natural-language steps that work independently of object locators, UI language, or dynamic content. + +**License labeling:** +- With Perfecto AI + Desktop Web licenses → interface labeled **AI Scriptless** (mobile + desktop web). +- Without those licenses → labeled **Scriptless Mobile** (mobile only). +- AI commands require a Perfecto AI license (admin feature-toggle opt-in). Without it, AI commands and related MCP operations will not work. + +## Quick Start + +1. **Access**: Perfecto landing page → Scriptless Automation → **Build ai scriptless test** (optional: select a real device). +2. **UI**: Learn the [workspace layout](references/interface.md) (toolbar, sidebar, editor, devices, widgets, action panel). +3. **Author**: Create/open/save tests, add AI or classic commands, use loops/conditions/logical steps. +4. **Run**: Execute with a DUT (device under test), then open the Single Test Report. +5. **MCP**: Prefer `perfecto_skills` + `perfecto_ai_scriptless` for programmatic authoring; use `perfecto_help` (category `perfecto` / subcategory `ide`) for live docs. + +## MCP Tools Integration + +### Skills and Help + +- `perfecto_skills` — `list_skills`, `read_skill` (`perfecto-ai-scriptless`), then `list_skill_resources` / `read_skill_resource_uri` for reference files. +- `perfecto_help` — `list_help_category_content` / `read_help_info` with `category_id='perfecto'`, `subcategory_id='ide'` (or `subcategory_id_list=['ide']`). + +### AI Scriptless tool (`perfecto_ai_scriptless`) + +| Goal | Actions | +| --- | --- | +| Discover tests | `list_filter_values` → `list_tests` | +| Inspect / edit structure | `view_test_structure`, `list_commands`, `get_command_definitions`, `add_command`, `modify_command`, `delete_command`, `set_command_enabled`, `move_command` | +| Control flow | `add_logical_step`, `add_loop`, `add_condition`, `set_condition_expression` | +| Lifecycle | `create_test`, `save_test`, `save_test_as`, `delete_test`, `move_test` | +| Variables / history | `list_test_variables`, `add_test_variable`, `modify_test_variable`, `delete_test_variable`, `list_snapshots`, `view_snapshot` | +| Execute | `execute_test` (after validating device via `perfecto_devices`) | + +### Critical MCP rules + +- **Licenses**: AI Scriptless MCP actions need Perfecto AI license; desktop web authoring needs Desktop Web license. +- **step_path**: Dot-separated positional path (e.g. `0`, `2.0`, `5.b0.1`). Paths are not persisted and change after inserts/moves/deletes — always `view_test_structure` before the next structure edit. +- **Command policy**: Call `list_commands` first and follow the selection policy in `info` before `add_command`. Prefer primary AI commands (`ai_user-action`, `ai_validation`, `ai_visual-comparison`) when authoring natural-language steps. +- **UI access**: Lab entry is `{cloud_url}/lab/scriptless-mobile/` from `perfecto_user` `read_user`. There is no per-test URL; open tests in the UI by folder/name from `list_tests`. +- **Not in MCP yet**: DataTables, Scheduler, Embedded tests, Object Spy, AI Assistant chat, folder rename/restore snapshot, download as Appium, and similar advanced UI features — guide users to the lab UI / help docs. + +### Example workflow (author + run) + +1. `perfecto_skills` → `read_skill` → `perfecto-ai-scriptless` +2. `list_commands` (and `get_command_definitions` for argument shapes) +3. `create_test` or `list_tests` → `view_test_structure` +4. `add_command` / control-flow actions as needed; re-`view_test_structure` after each mutation +5. Validate device (`list_real_devices` / virtual / desktop + availability checks) +6. `execute_test` → monitor with `perfecto_execution` (`list_live_executions`, `list_report_executions`) + +## Reference Files + +### Getting started and UI +- **[getting-started.md](references/getting-started.md)**: What AI Scriptless is, licenses, how to access +- **[interface.md](references/interface.md)**: Top toolbar, left sidebar, test editor, devices pane, widgets, action panel + +### Authoring and execution +- **[handle-tests.md](references/handle-tests.md)**: New, open, save, save as, delete, snapshots/versions +- **[execute-tests.md](references/execute-tests.md)**: Run, stop, re-run, DUT selection, reports +- **[control-constructs.md](references/control-constructs.md)**: Conditions, loops, logical steps + +### AI commands +- **[ai-commands.md](references/ai-commands.md)**: AI commands overview and FAQ +- **[ai-user-actions-best-practices.md](references/ai-user-actions-best-practices.md)**: Phrasing user actions, forms, install apps +- **[ai-validations-best-practices.md](references/ai-validations-best-practices.md)**: PASS/FAIL validations, do's and don'ts +- **[ai-visual-comparison-best-practices.md](references/ai-visual-comparison-best-practices.md)**: Visual comparison failure categories + +## When to Use Each Reference + +- **Getting started / interface**: First-time users or UI orientation questions +- **Handle tests / execute / control constructs**: Classic Scriptless authoring and run flows +- **AI command references**: Natural-language steps, validations, visual comparisons, licensing FAQ +- **MCP section above**: When operating through Perfecto MCP tools instead of (or with) the lab UI + +## Official documentation + +For the full live catalog under Perfecto Help → IDE, start at [AI Scriptless](https://help.perfecto.io/perfecto-help/content/perfecto/ide/get-started-with-scriptless-mobile.htm) and use `perfecto_help` to pull any page under subcategory `ide`. diff --git a/resources/skills/perfecto-ai-scriptless/references/ai-commands.md b/resources/skills/perfecto-ai-scriptless/references/ai-commands.md new file mode 100644 index 0000000..f24a02e --- /dev/null +++ b/resources/skills/perfecto-ai-scriptless/references/ai-commands.md @@ -0,0 +1,47 @@ +# AI commands overview and FAQ + +Sources: +- [AI commands FAQ](https://help.perfecto.io/perfecto-help/content/perfecto/ide/sm-ai-commands-faq.htm) +- [AI commands](https://help.perfecto.io/perfecto-help/content/perfecto/sm-commands-and-checkpoints/ai-commands.htm) + +## What AI commands do + +AI commands (Scriptless) and AI functions (Appium) use natural language — in any language — like a conversation. Benefits: + +- Complex validations that may combine multiple traditional steps into one +- Coverage beyond fragile locator-based scripting +- Platform-agnostic steps that adapt better to UI changes + +In the Scriptless UI, AI commands live under the **AI** left-sidebar tab. Prefer Perfecto's AI Assistant in the lab when crafting prompts. + +Primary MCP command IDs (confirm via `list_commands`): + +- `ai_user-action` — perform actions on the DUT +- `ai_validation` — PASS/FAIL assertions +- `ai_visual-comparison` — screen comparison categories + +## Licensing FAQ + +- AI is **not** on by default for every Perfecto feature — only AI commands/functions. +- A **separate Perfecto AI license** and admin **feature-toggle opt-in** are required. +- If the org opts out later, AI-based commands cannot run and dependent tests fail. +- Data use: see Perforce [Generative AI Policy](https://www.perforce.com/generative-ai-policy). + +## Limits and tips + +- One AI command can contain up to **30** internal steps; execution limit is **3 minutes** per call. For best results, target **≤ 15** steps. +- Form fill / text extract with AI works on the **default tab** in web sessions (desktop browser, mobile web, WebView) — not native mobile/desktop apps, and not tabs opened mid-flow (Selenium stays on the default tab). +- If answers are inconsistent: refine with the AI Assistant suggestions, then review best-practice docs. Ambiguous prompts are the usual cause. + +## Example validation conversation pattern + +User: "Is there a sign-up here option for new user?" +Assistant may suggest: "Is there a sign-up option for new users labeled 'Sign Up Here'?" + +Use assistant suggestions as the final validation text when possible. + +## Related best practices + +- [AI user actions](ai-user-actions-best-practices.md) +- [AI validations](ai-validations-best-practices.md) +- [AI visual comparisons](ai-visual-comparison-best-practices.md) diff --git a/resources/skills/perfecto-ai-scriptless/references/ai-user-actions-best-practices.md b/resources/skills/perfecto-ai-scriptless/references/ai-user-actions-best-practices.md new file mode 100644 index 0000000..64831af --- /dev/null +++ b/resources/skills/perfecto-ai-scriptless/references/ai-user-actions-best-practices.md @@ -0,0 +1,69 @@ +# Best practices: AI user actions + +Source: [Best practices for working with AI user actions](https://help.perfecto.io/perfecto-help/content/perfecto/ide/sm-ai-user-action-best-practices.htm) + +Write the action once; run across devices/OS. Describe the **logical goal**, not locators. + +In Scriptless, use AI commands ([AI commands](https://help.perfecto.io/perfecto-help/content/perfecto/sm-commands-and-checkpoints/ai-commands.htm)). For Appium, see [AI functions](https://help.perfecto.io/perfecto-help/content/perfecto/automation-testing/ai-functions.htm). + +## Performance + +- Phrase each User Action so it resolves to **≤ 15** steps and finishes in **under 3 minutes**. +- Enable **Reasoning** when counting, sorting, or ordering needs extra reasoning resources. + +## What works well + +- Common UI: hamburger, help, notifications +- Click buttons, links, icons, FABs +- Type into empty fields or replace text +- Radio buttons, checkboxes, sliders +- Tabs, lists, navbars +- Dismiss interrupting popups/notifications +- Browser navigation (go to, back, forward, refresh, clean) +- Install mobile apps (see install section) + +Still evolving / may be unreliable: inline labels relative to fields, table cell/row actions, color pickers. Not yet fully tested: append text to existing field content. LLMs can make mistakes. + +## Why AI user actions help + +- **Write once, run everywhere** — no device-specific locators +- **Survive UI renames/moves** — goal-based instructions +- **Wait instead of fail** — instruct wait/retry/dismiss modals/progress +- **Focus** — “focus on / zoom into …” for dense UIs + +## Do's and don'ts + +| Don't | Do | Why | +| --- | --- | --- | +| Move mouse to Cart button bottom-right | On the profile page, open the shopping cart | Logical goal, not coordinates/labels | +| Register a new account | Fill the form: Enter "Jill A. Smith" in the name fields… | Enough concrete detail | +| Click the red shoes | From search results, select the red sneakers that cost $30 | Disambiguate similar items | +| Pay by card | Fill the form: In the drop down, select credit card… | Triggers form-fill tooling for dropdowns | + +## Specialized knowledge (`ai.txt`) + +For domain UIs (seat maps, dashboards, canvases): + +1. Write instructions in plain text (max **2,000** characters; larger files are ignored). +2. Save as `ai.txt`. +3. Upload to the **root** of the Perfecto Repository. +4. Reloads on login; applied as general user instructions for AI assistants/commands. + +## Install mobile apps via User Action + +Example prompts: `Install public:/path/myapp.ipa`, `Install ${myApp} on the device, open it and log in`. + +Configure instrumentation in the UI widget (Sensor, WebView, Secured Screen, Resign). iOS notes: + +- Virtual iOS: use `appname.zip` +- Real iOS: Resign Application on by default (mandatory unless WebView changes rules) + +See [Install a mobile app using AI User Actions](https://help.perfecto.io/perfecto-help/content/perfecto/ide/sm-ai-user-action-install-app.htm). + +## Fill forms + +Prefer **one** prompt that fills many fields, then a separate submit action. + +Supported: text/textarea, native `