diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a87e973 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# OpenRouter identifies requests in its App column through X-Title. +OPENROUTER_API_KEY= +OPENROUTER_APP_NAME=testql +OPENROUTER_SITE_URL= +LLM_MODEL=openrouter/z-ai/glm-5.2 diff --git a/README.md b/README.md index 261c4af..f6cba4f 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,19 @@ print(result.to_dict()) # testql.verification-result.v1 Packaged JSON Schemas are available through `verification_contract_schema()`. +### Versioned LLM contracts + +The live `nlp2env` path accepts only the versioned `ToolCall 1.0.0` response. +Its GBNF, Protobuf, JSON Schema and boundary manifest are packaged under +`testql/contracts/nlp2env/v1`. OpenRouter receives the schema through +`response_format`, Ollama through `format`, and TestQL validates the complete +JSON response again before invoking MCP. Markdown fences, surrounding prose, +unknown tools, literal password fields and contract-version drift fail closed. + +Set `OPENROUTER_APP_NAME` to identify TestQL in OpenRouter logs. If it is not +set, TestQL uses the current project folder name. See `.env.example` for the +GLM 5.2 defaults. + ## Artifact Discovery, Topology, and Web Inspection diff --git a/pyproject.toml b/pyproject.toml index 6737d47..d822e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "click>=8.0", "rich>=13.0", "pyyaml>=6.0", + "jsonschema>=4.21", "goal>=2.1.0", "costs>=0.1.20", "pfix>=0.1.60", @@ -79,7 +80,13 @@ dev = ["pytest", "pytest-asyncio", "pytest-cov", include = ["testql*"] [tool.setuptools.package-data] -testql = ["data/*.json", "interpreter/*.cjs"] +testql = [ + "contracts/nlp2env/v1/*.gbnf", + "contracts/nlp2env/v1/*.json", + "contracts/nlp2env/v1/*.proto", + "data/*.json", + "interpreter/*.cjs", +] [tool.uv.sources] vdisplay = { path = "../../wronai/vdisplay", editable = true } diff --git a/testql/contracts/__init__.py b/testql/contracts/__init__.py new file mode 100644 index 0000000..609bb49 --- /dev/null +++ b/testql/contracts/__init__.py @@ -0,0 +1 @@ +"""Versioned contracts for TestQL's LLM-facing boundaries.""" diff --git a/testql/contracts/nlp2env/__init__.py b/testql/contracts/nlp2env/__init__.py new file mode 100644 index 0000000..383a429 --- /dev/null +++ b/testql/contracts/nlp2env/__init__.py @@ -0,0 +1,51 @@ +"""Runtime binding for the nlp2env LLM-to-MCP ToolCall contract.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from jsonschema import Draft202012Validator + +CONTRACT_VERSION = "1.0.0" + + +def _contract_file(name: str): + return files(__package__).joinpath("v1", name) + + +def load_schema() -> dict[str, Any]: + return json.loads( + _contract_file("tool-call.schema.json").read_text(encoding="utf-8") + ) + + +def validate_tool_call(payload: object) -> None: + validator = Draft202012Validator(load_schema()) + errors = sorted(validator.iter_errors(payload), key=lambda error: list(error.path)) + if errors: + first = errors[0] + location = ".".join(str(part) for part in first.absolute_path) or "$" + raise ValueError( + f"LLM response violates nlp2env ToolCall v1 at {location}: {first.message}" + ) + + +def openai_response_format() -> dict[str, Any]: + return { + "type": "json_schema", + "json_schema": { + "name": "nlp2env_tool_call_v1", + "strict": True, + "schema": load_schema(), + }, + } + + +__all__ = [ + "CONTRACT_VERSION", + "load_schema", + "openai_response_format", + "validate_tool_call", +] diff --git a/testql/contracts/nlp2env/v1/manifest.json b/testql/contracts/nlp2env/v1/manifest.json new file mode 100644 index 0000000..86bfc3d --- /dev/null +++ b/testql/contracts/nlp2env/v1/manifest.json @@ -0,0 +1,19 @@ +{ + "contract": "testql.nlp2env.ToolCall", + "version": "1.0.0", + "boundary": "testql.nlp2env.llm.translate_nl_to_mcp", + "mediaType": "application/json", + "artifacts": { + "grammar": "tool-call.gbnf", + "protobuf": "tool-call.proto", + "schema": "tool-call.schema.json" + }, + "providers": { + "openrouter": "response_format.json_schema", + "ollama": "format" + }, + "runtime": { + "parser": "testql.nlp2env.llm._extract_json_object", + "validator": "testql.contracts.nlp2env.validate_tool_call" + } +} diff --git a/testql/contracts/nlp2env/v1/tool-call.gbnf b/testql/contracts/nlp2env/v1/tool-call.gbnf new file mode 100644 index 0000000..193264f --- /dev/null +++ b/testql/contracts/nlp2env/v1/tool-call.gbnf @@ -0,0 +1,17 @@ +root ::= ws "{" ws version ws "," ws call ws "}" ws +version ::= "\"contractVersion\"" ws ":" ws "\"1.0.0\"" +call ::= set-email | status | list + +set-email ::= "\"tool\"" ws ":" ws "\"nlp2env_set_email\"" ws "," ws "\"arguments\"" ws ":" ws "{" ws email-pair (ws "," ws email-pair)* ws "}" +status ::= "\"tool\"" ws ":" ws "\"nlp2env_email_status\"" ws "," ws empty-arguments +list ::= "\"tool\"" ws ":" ws "\"nlp2env_list\"" ws "," ws empty-arguments +empty-arguments ::= "\"arguments\"" ws ":" ws "{" ws "}" +email-pair ::= regular-pair | password-pair +regular-pair ::= regular-key ws ":" ws string +regular-key ::= "\"host\"" | "\"user\"" | "\"port\"" | "\"from_addr\"" +password-pair ::= "\"password_env\"" ws ":" ws "\"SMTP_PASSWORD\"" + +string ::= "\"" char* "\"" +char ::= [^"\\\x00-\x1f] | "\\" (["\\/bfnrt] | "u" hex hex hex hex) +hex ::= [0-9a-fA-F] +ws ::= [ \t\n\r]* diff --git a/testql/contracts/nlp2env/v1/tool-call.proto b/testql/contracts/nlp2env/v1/tool-call.proto new file mode 100644 index 0000000..c3f9532 --- /dev/null +++ b/testql/contracts/nlp2env/v1/tool-call.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package testql.contracts.nlp2env.v1; + +message ToolCall { + string contract_version = 1 [json_name = "contractVersion"]; + Tool tool = 2; + Arguments arguments = 3; +} + +enum Tool { + tool_unspecified = 0; + nlp2env_set_email = 1; + nlp2env_email_status = 2; + nlp2env_list = 3; +} + +message Arguments { + string host = 1; + string user = 2; + string port = 3; + string from_addr = 4; + string password_env = 5; +} diff --git a/testql/contracts/nlp2env/v1/tool-call.schema.json b/testql/contracts/nlp2env/v1/tool-call.schema.json new file mode 100644 index 0000000..8333fbd --- /dev/null +++ b/testql/contracts/nlp2env/v1/tool-call.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://testql.dev/contracts/nlp2env/v1/tool-call.schema.json", + "title": "nlp2env ToolCall v1", + "type": "object", + "required": ["contractVersion", "tool", "arguments"], + "properties": { + "contractVersion": { "const": "1.0.0" }, + "tool": { + "enum": ["nlp2env_set_email", "nlp2env_email_status", "nlp2env_list"] + }, + "arguments": { "type": "object" } + }, + "oneOf": [ + { + "properties": { + "tool": { "const": "nlp2env_set_email" }, + "arguments": { + "type": "object", + "minProperties": 1, + "properties": { + "host": { "type": "string", "minLength": 1, "maxLength": 253 }, + "user": { "type": "string", "minLength": 1, "maxLength": 320 }, + "port": { "type": "string", "pattern": "^[0-9]{1,5}$" }, + "from_addr": { "type": "string", "minLength": 1, "maxLength": 320 }, + "password_env": { "const": "SMTP_PASSWORD" } + }, + "additionalProperties": false + } + } + }, + { + "properties": { + "tool": { "enum": ["nlp2env_email_status", "nlp2env_list"] }, + "arguments": { "type": "object", "maxProperties": 0 } + } + } + ], + "additionalProperties": false +} diff --git a/testql/nlp2env/llm.py b/testql/nlp2env/llm.py index 0edd8f1..6895b49 100644 --- a/testql/nlp2env/llm.py +++ b/testql/nlp2env/llm.py @@ -4,15 +4,21 @@ import json import os -import re import urllib.error import urllib.request -from typing import Any +from pathlib import Path +from typing import Any, cast + +from testql.contracts.nlp2env import ( + load_schema, + openai_response_format, + validate_tool_call, +) _SYSTEM = """\ You translate natural-language user requests (any language) into nlp2env MCP tool calls for SMTP/email .env configuration. -Return ONLY valid JSON, no markdown: -{"tool":"nlp2env_set_email","arguments":{"host":"smtp.example.com","user":"a@b.c","port":"587","from_addr":"a@b.c","password_env":"SMTP_PASSWORD"}} +Return ONLY JSON matching the nlp2env ToolCall 1.0.0 contract, no markdown: +{"contractVersion":"1.0.0","tool":"nlp2env_set_email","arguments":{"host":"smtp.example.com","user":"a@b.c","port":"587","from_addr":"a@b.c","password_env":"SMTP_PASSWORD"}} Allowed tools: nlp2env_set_email, nlp2env_email_status, nlp2env_list. Never put passwords in JSON — always use password_env=SMTP_PASSWORD. @@ -32,7 +38,9 @@ def ollama_reachable(base: str | None = None) -> bool: def resolve_llm_backend() -> tuple[str, str]: key = os.getenv("OPENROUTER_API_KEY", "").strip() if key: - model = os.getenv("LLM_MODEL", os.getenv("PFIX_MODEL", "openrouter/qwen/qwen3-coder-next")) + model = os.getenv( + "LLM_MODEL", os.getenv("PFIX_MODEL", "openrouter/z-ai/glm-5.2") + ) if not model.startswith("openrouter/"): model = f"openrouter/{model.removeprefix('ollama/')}" return "openrouter", model @@ -44,7 +52,9 @@ def resolve_llm_backend() -> tuple[str, str]: return "none", "" -def _http_post_json(url: str, payload: dict[str, Any], headers: dict[str, str] | None = None) -> dict[str, Any]: +def _http_post_json( + url: str, payload: dict[str, Any], headers: dict[str, str] | None = None +) -> dict[str, Any]: data = json.dumps(payload).encode("utf-8") req = urllib.request.Request( url, @@ -57,18 +67,33 @@ def _http_post_json(url: str, payload: dict[str, Any], headers: dict[str, str] | def _extract_json_object(text: str) -> dict[str, Any]: - text = text.strip() - fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL) - if fence: - text = fence.group(1) - elif not text.startswith("{"): - start, end = text.find("{"), text.rfind("}") - if start != -1 and end != -1: - text = text[start : end + 1] - return json.loads(text) - - -def translate_nl_to_mcp(nl: str, backend: str, model: str) -> tuple[str, dict[str, str]]: + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError("LLM response must be a single JSON object") from exc + if not isinstance(parsed, dict): + raise ValueError("LLM response must be a JSON object") + validate_tool_call(parsed) + return parsed + + +def _openrouter_headers(api_key: str) -> dict[str, str]: + app_name = ( + os.getenv("OPENROUTER_APP_NAME", "").strip() or Path.cwd().name or "testql" + ) + headers = { + "Authorization": f"Bearer {api_key}", + "X-Title": app_name, + } + site_url = os.getenv("OPENROUTER_SITE_URL", "").strip() + if site_url: + headers["HTTP-Referer"] = site_url + return headers + + +def translate_nl_to_mcp( + nl: str, backend: str, model: str +) -> tuple[str, dict[str, str]]: messages = [ {"role": "system", "content": _SYSTEM}, {"role": "user", "content": nl}, @@ -77,25 +102,29 @@ def translate_nl_to_mcp(nl: str, backend: str, model: str) -> tuple[str, dict[st key = os.getenv("OPENROUTER_API_KEY", "").strip() body = _http_post_json( "https://openrouter.ai/api/v1/chat/completions", - {"model": model.removeprefix("openrouter/"), "messages": messages, "temperature": 0.1}, - headers={"Authorization": f"Bearer {key}"}, + { + "model": model.removeprefix("openrouter/"), + "messages": messages, + "temperature": 0.1, + "response_format": openai_response_format(), + }, + headers=_openrouter_headers(key), ) content = body["choices"][0]["message"]["content"] elif backend == "ollama": base = os.getenv("OLLAMA_API_BASE", "http://localhost:11434").rstrip("/") body = _http_post_json( f"{base}/api/chat", - {"model": model.removeprefix("ollama/"), "messages": messages, "stream": False}, + { + "model": model.removeprefix("ollama/"), + "messages": messages, + "stream": False, + "format": load_schema(), + }, ) content = body["message"]["content"] else: raise RuntimeError("Brak backendu LLM (OPENROUTER_API_KEY lub Ollama)") parsed = _extract_json_object(content) - tool = str(parsed.get("tool", "")).strip() - arguments = parsed.get("arguments") or {} - if not tool: - raise ValueError(f"LLM nie zwrócił tool: {content[:300]}") - if not isinstance(arguments, dict): - raise ValueError("LLM arguments musi być obiektem JSON") - return tool, {str(k): str(v) for k, v in arguments.items()} + return cast(str, parsed["tool"]), cast(dict[str, str], parsed["arguments"]) diff --git a/tests/fixtures/contracts/nlp2env/v1/invalid-tool-call.json b/tests/fixtures/contracts/nlp2env/v1/invalid-tool-call.json new file mode 100644 index 0000000..253f737 --- /dev/null +++ b/tests/fixtures/contracts/nlp2env/v1/invalid-tool-call.json @@ -0,0 +1,8 @@ +{ + "contractVersion": "1.0.0", + "tool": "nlp2env_set_email", + "arguments": { + "host": "smtp.example.com", + "password": "literal-secret" + } +} diff --git a/tests/fixtures/contracts/nlp2env/v1/valid-tool-call.json b/tests/fixtures/contracts/nlp2env/v1/valid-tool-call.json new file mode 100644 index 0000000..5a3b083 --- /dev/null +++ b/tests/fixtures/contracts/nlp2env/v1/valid-tool-call.json @@ -0,0 +1,11 @@ +{ + "contractVersion": "1.0.0", + "tool": "nlp2env_set_email", + "arguments": { + "host": "smtp.example.com", + "user": "user@example.com", + "port": "587", + "from_addr": "user@example.com", + "password_env": "SMTP_PASSWORD" + } +} diff --git a/tests/test_environment_regressions.py b/tests/test_environment_regressions.py index 287d0c9..77e3672 100644 --- a/tests/test_environment_regressions.py +++ b/tests/test_environment_regressions.py @@ -16,7 +16,7 @@ def test_navigate_section_starts_browser_before_navigation(): commands = [line.command for line in convert_testtoon_to_oql(source).lines] - assert commands == ["GUI_START", "WAIT", "NAVIGATE", "WAIT"] + assert commands == ["GUI_START", "WAIT", "GUI_NAVIGATE", "WAIT"] def test_shell_result_keeps_bounded_stdout_and_stderr(): diff --git a/tests/test_nlp2env_llm_contract.py b/tests/test_nlp2env_llm_contract.py new file mode 100644 index 0000000..7e1aa7d --- /dev/null +++ b/tests/test_nlp2env_llm_contract.py @@ -0,0 +1,110 @@ +"""Offline contract tests for the nlp2env LLM-to-MCP boundary.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from testql.contracts.nlp2env import CONTRACT_VERSION, load_schema, validate_tool_call +from testql.nlp2env import llm + +FIXTURES = Path(__file__).parent / "fixtures" / "contracts" / "nlp2env" / "v1" +CONTRACTS = Path(llm.__file__).parents[1] / "contracts" / "nlp2env" / "v1" + + +def _fixture(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_valid_fixture_passes_runtime_validation() -> None: + payload = _fixture("valid-tool-call.json") + validate_tool_call(payload) + assert llm._extract_json_object(json.dumps(payload)) == payload + + +@pytest.mark.parametrize( + "payload", + [ + _fixture("invalid-tool-call.json"), + {"contractVersion": "2.0.0", "tool": "nlp2env_list", "arguments": {}}, + {"contractVersion": "1.0.0", "tool": "shell", "arguments": {}}, + {"contractVersion": "1.0.0", "tool": "nlp2env_list", "arguments": {"x": "y"}}, + ], +) +def test_invalid_or_unsafe_calls_fail_closed(payload: dict) -> None: + with pytest.raises(ValueError, match="violates nlp2env ToolCall v1"): + validate_tool_call(payload) + + +def test_parser_rejects_markdown_or_surrounding_prose() -> None: + with pytest.raises(ValueError, match="single JSON object"): + llm._extract_json_object('```json\n{"tool":"nlp2env_list"}\n```') + + +def test_openrouter_request_uses_schema_and_project_app_name( + monkeypatch, tmp_path: Path +) -> None: + project = tmp_path / "customer-project" + project.mkdir() + monkeypatch.chdir(project) + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key") + monkeypatch.delenv("OPENROUTER_APP_NAME", raising=False) + captured: dict = {} + + def fake_post(url, payload, headers=None): + captured.update(url=url, payload=payload, headers=headers) + content = json.dumps( + {"contractVersion": "1.0.0", "tool": "nlp2env_list", "arguments": {}} + ) + return {"choices": [{"message": {"content": content}}]} + + monkeypatch.setattr(llm, "_http_post_json", fake_post) + assert llm.translate_nl_to_mcp("list", "openrouter", "openrouter/z-ai/glm-5.2") == ( + "nlp2env_list", + {}, + ) + assert captured["headers"]["X-Title"] == "customer-project" + assert ( + captured["payload"]["response_format"]["json_schema"]["schema"] == load_schema() + ) + + +def test_ollama_request_uses_the_same_schema(monkeypatch) -> None: + captured: dict = {} + + def fake_post(url, payload, headers=None): + captured.update(url=url, payload=payload, headers=headers) + content = json.dumps( + { + "contractVersion": "1.0.0", + "tool": "nlp2env_email_status", + "arguments": {}, + } + ) + return {"message": {"content": content}} + + monkeypatch.setattr(llm, "_http_post_json", fake_post) + assert llm.translate_nl_to_mcp("status", "ollama", "ollama/glm") == ( + "nlp2env_email_status", + {}, + ) + assert captured["payload"]["format"] == load_schema() + + +def test_manifest_and_artifacts_bind_the_runtime_boundary() -> None: + manifest = json.loads((CONTRACTS / "manifest.json").read_text(encoding="utf-8")) + assert manifest["version"] == CONTRACT_VERSION + assert manifest["boundary"] == "testql.nlp2env.llm.translate_nl_to_mcp" + for name in manifest["artifacts"].values(): + assert (CONTRACTS / name).is_file() + grammar = (CONTRACTS / "tool-call.gbnf").read_text(encoding="utf-8") + proto = (CONTRACTS / "tool-call.proto").read_text(encoding="utf-8") + assert f'\\"{CONTRACT_VERSION}\\"' in grammar + assert '\\"SMTP_PASSWORD\\"' in grammar + schema_tools = set(load_schema()["properties"]["tool"]["enum"]) + proto_tools = set(re.findall(r"^ (nlp2env_\w+) = \d+;", proto, re.MULTILINE)) + assert proto_tools == schema_tools + assert 'json_name = "contractVersion"' in proto