feat(nlp2env): enforce versioned LLM tool-call contract

.env.example

@@ -0,0 +1,5 @@
+# OpenRouter identifies requests in its App column through X-Title.
+OPENROUTER_API_KEY=
+OPENROUTER_APP_NAME=testql
+OPENROUTER_SITE_URL=
+LLM_MODEL=openrouter/z-ai/glm-5.2

README.md

@@ -115,6 +115,19 @@ print(result.to_dict())  # testql.verification-result.v1
 
 Packaged JSON Schemas are available through `verification_contract_schema()`.
 
+### Versioned LLM contracts
+
+The live `nlp2env` path accepts only the versioned `ToolCall 1.0.0` response.
+Its GBNF, Protobuf, JSON Schema and boundary manifest are packaged under
+`testql/contracts/nlp2env/v1`. OpenRouter receives the schema through
+`response_format`, Ollama through `format`, and TestQL validates the complete
+JSON response again before invoking MCP. Markdown fences, surrounding prose,
+unknown tools, literal password fields and contract-version drift fail closed.
+
+Set `OPENROUTER_APP_NAME` to identify TestQL in OpenRouter logs. If it is not
+set, TestQL uses the current project folder name. See `.env.example` for the
+GLM 5.2 defaults.
+
 
 ## Artifact Discovery, Topology, and Web Inspection
 

pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "click>=8.0",
     "rich>=13.0",
     "pyyaml>=6.0",
+    "jsonschema>=4.21",
     "goal>=2.1.0",
     "costs>=0.1.20",
     "pfix>=0.1.60",
@@ -79,7 +80,13 @@ dev = ["pytest", "pytest-asyncio", "pytest-cov",
 include = ["testql*"]
 
 [tool.setuptools.package-data]
-testql = ["data/*.json", "interpreter/*.cjs"]
+testql = [
+    "contracts/nlp2env/v1/*.gbnf",
+    "contracts/nlp2env/v1/*.json",
+    "contracts/nlp2env/v1/*.proto",
+    "data/*.json",
+    "interpreter/*.cjs",
+]
 
 [tool.uv.sources]
 vdisplay = { path = "../../wronai/vdisplay", editable = true }

testql/contracts/__init__.py

@@ -0,0 +1 @@
+"""Versioned contracts for TestQL's LLM-facing boundaries."""

testql/contracts/nlp2env/__init__.py

@@ -0,0 +1,51 @@
+"""Runtime binding for the nlp2env LLM-to-MCP ToolCall contract."""
+
+from __future__ import annotations
+
+import json
+from importlib.resources import files
+from typing import Any
+
+from jsonschema import Draft202012Validator
+
+CONTRACT_VERSION = "1.0.0"
+
+
+def _contract_file(name: str):
+    return files(__package__).joinpath("v1", name)
+
+
+def load_schema() -> dict[str, Any]:
+    return json.loads(
+        _contract_file("tool-call.schema.json").read_text(encoding="utf-8")
+    )
+
+
+def validate_tool_call(payload: object) -> None:
+    validator = Draft202012Validator(load_schema())
+    errors = sorted(validator.iter_errors(payload), key=lambda error: list(error.path))
+    if errors:
+        first = errors[0]
+        location = ".".join(str(part) for part in first.absolute_path) or "$"
+        raise ValueError(
+            f"LLM response violates nlp2env ToolCall v1 at {location}: {first.message}"
+        )
+
+
+def openai_response_format() -> dict[str, Any]:
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "nlp2env_tool_call_v1",
+            "strict": True,
+            "schema": load_schema(),
+        },
+    }
+
+
+__all__ = [
+    "CONTRACT_VERSION",
+    "load_schema",
+    "openai_response_format",
+    "validate_tool_call",
+]

testql/contracts/nlp2env/v1/manifest.json

@@ -0,0 +1,19 @@
+{
+  "contract": "testql.nlp2env.ToolCall",
+  "version": "1.0.0",
+  "boundary": "testql.nlp2env.llm.translate_nl_to_mcp",
+  "mediaType": "application/json",
+  "artifacts": {
+    "grammar": "tool-call.gbnf",
+    "protobuf": "tool-call.proto",
+    "schema": "tool-call.schema.json"
+  },
+  "providers": {
+    "openrouter": "response_format.json_schema",
+    "ollama": "format"
+  },
+  "runtime": {
+    "parser": "testql.nlp2env.llm._extract_json_object",
+    "validator": "testql.contracts.nlp2env.validate_tool_call"
+  }
+}

testql/contracts/nlp2env/v1/tool-call.gbnf

@@ -0,0 +1,17 @@
+root ::= ws "{" ws version ws "," ws call ws "}" ws
+version ::= "\"contractVersion\"" ws ":" ws "\"1.0.0\""
+call ::= set-email | status | list
+
+set-email ::= "\"tool\"" ws ":" ws "\"nlp2env_set_email\"" ws "," ws "\"arguments\"" ws ":" ws "{" ws email-pair (ws "," ws email-pair)* ws "}"
+status ::= "\"tool\"" ws ":" ws "\"nlp2env_email_status\"" ws "," ws empty-arguments
+list ::= "\"tool\"" ws ":" ws "\"nlp2env_list\"" ws "," ws empty-arguments
+empty-arguments ::= "\"arguments\"" ws ":" ws "{" ws "}"
+email-pair ::= regular-pair | password-pair
+regular-pair ::= regular-key ws ":" ws string
+regular-key ::= "\"host\"" | "\"user\"" | "\"port\"" | "\"from_addr\""
+password-pair ::= "\"password_env\"" ws ":" ws "\"SMTP_PASSWORD\""
+
+string ::= "\"" char* "\""
+char ::= [^"\\\x00-\x1f] | "\\" (["\\/bfnrt] | "u" hex hex hex hex)
+hex ::= [0-9a-fA-F]
+ws ::= [ \t\n\r]*

testql/contracts/nlp2env/v1/tool-call.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package testql.contracts.nlp2env.v1;
+
+message ToolCall {
+  string contract_version = 1 [json_name = "contractVersion"];
+  Tool tool = 2;
+  Arguments arguments = 3;
+}
+
+enum Tool {
+  tool_unspecified = 0;
+  nlp2env_set_email = 1;
+  nlp2env_email_status = 2;
+  nlp2env_list = 3;
+}
+
+message Arguments {
+  string host = 1;
+  string user = 2;
+  string port = 3;
+  string from_addr = 4;
+  string password_env = 5;
+}

testql/contracts/nlp2env/v1/tool-call.schema.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://testql.dev/contracts/nlp2env/v1/tool-call.schema.json",
+  "title": "nlp2env ToolCall v1",
+  "type": "object",
+  "required": ["contractVersion", "tool", "arguments"],
+  "properties": {
+    "contractVersion": { "const": "1.0.0" },
+    "tool": {
+      "enum": ["nlp2env_set_email", "nlp2env_email_status", "nlp2env_list"]
+    },
+    "arguments": { "type": "object" }
+  },
+  "oneOf": [
+    {
+      "properties": {
+        "tool": { "const": "nlp2env_set_email" },
+        "arguments": {
+          "type": "object",
+          "minProperties": 1,
+          "properties": {
+            "host": { "type": "string", "minLength": 1, "maxLength": 253 },
+            "user": { "type": "string", "minLength": 1, "maxLength": 320 },
+            "port": { "type": "string", "pattern": "^[0-9]{1,5}$" },
+            "from_addr": { "type": "string", "minLength": 1, "maxLength": 320 },
+            "password_env": { "const": "SMTP_PASSWORD" }
+          },
+          "additionalProperties": false
+        }
+      }
+    },
+    {
+      "properties": {
+        "tool": { "enum": ["nlp2env_email_status", "nlp2env_list"] },
+        "arguments": { "type": "object", "maxProperties": 0 }
+      }
+    }
+  ],
+  "additionalProperties": false
+}

testql/nlp2env/llm.py

@@ -4,15 +4,21 @@
 
 import json
 import os
-import re
 import urllib.error
 import urllib.request
-from typing import Any
+from pathlib import Path
+from typing import Any, cast
+
+from testql.contracts.nlp2env import (
+    load_schema,
+    openai_response_format,
+    validate_tool_call,
+)
 
 _SYSTEM = """\
 You translate natural-language user requests (any language) into nlp2env MCP tool calls for SMTP/email .env configuration.
-Return ONLY valid JSON, no markdown:
-{"tool":"nlp2env_set_email","arguments":{"host":"smtp.example.com","user":"a@b.c","port":"587","from_addr":"a@b.c","password_env":"SMTP_PASSWORD"}}
+Return ONLY JSON matching the nlp2env ToolCall 1.0.0 contract, no markdown:
+{"contractVersion":"1.0.0","tool":"nlp2env_set_email","arguments":{"host":"smtp.example.com","user":"a@b.c","port":"587","from_addr":"a@b.c","password_env":"SMTP_PASSWORD"}}
 
 Allowed tools: nlp2env_set_email, nlp2env_email_status, nlp2env_list.
 Never put passwords in JSON — always use password_env=SMTP_PASSWORD.
@@ -32,7 +38,9 @@ def ollama_reachable(base: str | None = None) -> bool:
 def resolve_llm_backend() -> tuple[str, str]:
     key = os.getenv("OPENROUTER_API_KEY", "").strip()
     if key:
-        model = os.getenv("LLM_MODEL", os.getenv("PFIX_MODEL", "openrouter/qwen/qwen3-coder-next"))
+        model = os.getenv(
+            "LLM_MODEL", os.getenv("PFIX_MODEL", "openrouter/z-ai/glm-5.2")
+        )
         if not model.startswith("openrouter/"):
             model = f"openrouter/{model.removeprefix('ollama/')}"
         return "openrouter", model
@@ -44,7 +52,9 @@ def resolve_llm_backend() -> tuple[str, str]:
     return "none", ""
 
 
-def _http_post_json(url: str, payload: dict[str, Any], headers: dict[str, str] | None = None) -> dict[str, Any]:
+def _http_post_json(
+    url: str, payload: dict[str, Any], headers: dict[str, str] | None = None
+) -> dict[str, Any]:
     data = json.dumps(payload).encode("utf-8")
     req = urllib.request.Request(
         url,
@@ -57,18 +67,33 @@ def _http_post_json(url: str, payload: dict[str, Any], headers: dict[str, str] |
 
 
 def _extract_json_object(text: str) -> dict[str, Any]:
-    text = text.strip()
-    fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
-    if fence:
-        text = fence.group(1)
-    elif not text.startswith("{"):
-        start, end = text.find("{"), text.rfind("}")
-        if start != -1 and end != -1:
-            text = text[start : end + 1]
-    return json.loads(text)
-
-
-def translate_nl_to_mcp(nl: str, backend: str, model: str) -> tuple[str, dict[str, str]]:
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ValueError("LLM response must be a single JSON object") from exc
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM response must be a JSON object")
+    validate_tool_call(parsed)
+    return parsed
+
+
+def _openrouter_headers(api_key: str) -> dict[str, str]:
+    app_name = (
+        os.getenv("OPENROUTER_APP_NAME", "").strip() or Path.cwd().name or "testql"
+    )
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "X-Title": app_name,
+    }
+    site_url = os.getenv("OPENROUTER_SITE_URL", "").strip()
+    if site_url:
+        headers["HTTP-Referer"] = site_url
+    return headers
+
+
+def translate_nl_to_mcp(
+    nl: str, backend: str, model: str
+) -> tuple[str, dict[str, str]]:
     messages = [
         {"role": "system", "content": _SYSTEM},
         {"role": "user", "content": nl},
@@ -77,25 +102,29 @@ def translate_nl_to_mcp(nl: str, backend: str, model: str) -> tuple[str, dict[st
         key = os.getenv("OPENROUTER_API_KEY", "").strip()
         body = _http_post_json(
             "https://openrouter.ai/api/v1/chat/completions",
-            {"model": model.removeprefix("openrouter/"), "messages": messages, "temperature": 0.1},
-            headers={"Authorization": f"Bearer {key}"},
+            {
+                "model": model.removeprefix("openrouter/"),
+                "messages": messages,
+                "temperature": 0.1,
+                "response_format": openai_response_format(),
+            },
+            headers=_openrouter_headers(key),
         )
         content = body["choices"][0]["message"]["content"]
     elif backend == "ollama":
         base = os.getenv("OLLAMA_API_BASE", "http://localhost:11434").rstrip("/")
         body = _http_post_json(
             f"{base}/api/chat",
-            {"model": model.removeprefix("ollama/"), "messages": messages, "stream": False},
+            {
+                "model": model.removeprefix("ollama/"),
+                "messages": messages,
+                "stream": False,
+                "format": load_schema(),
+            },
         )
         content = body["message"]["content"]
     else:
         raise RuntimeError("Brak backendu LLM (OPENROUTER_API_KEY lub Ollama)")
 
     parsed = _extract_json_object(content)
-    tool = str(parsed.get("tool", "")).strip()
-    arguments = parsed.get("arguments") or {}
-    if not tool:
-        raise ValueError(f"LLM nie zwrócił tool: {content[:300]}")
-    if not isinstance(arguments, dict):
-        raise ValueError("LLM arguments musi być obiektem JSON")
-    return tool, {str(k): str(v) for k, v in arguments.items()}
+    return cast(str, parsed["tool"]), cast(dict[str, str], parsed["arguments"])

tests/fixtures/contracts/nlp2env/v1/invalid-tool-call.json

@@ -0,0 +1,8 @@
+{
+  "contractVersion": "1.0.0",
+  "tool": "nlp2env_set_email",
+  "arguments": {
+    "host": "smtp.example.com",
+    "password": "literal-secret"
+  }
+}

tests/fixtures/contracts/nlp2env/v1/valid-tool-call.json

@@ -0,0 +1,11 @@
+{
+  "contractVersion": "1.0.0",
+  "tool": "nlp2env_set_email",
+  "arguments": {
+    "host": "smtp.example.com",
+    "user": "user@example.com",
+    "port": "587",
+    "from_addr": "user@example.com",
+    "password_env": "SMTP_PASSWORD"
+  }
+}

tests/test_environment_regressions.py

@@ -16,7 +16,7 @@ def test_navigate_section_starts_browser_before_navigation():
 
     commands = [line.command for line in convert_testtoon_to_oql(source).lines]
 
-    assert commands == ["GUI_START", "WAIT", "NAVIGATE", "WAIT"]
+    assert commands == ["GUI_START", "WAIT", "GUI_NAVIGATE", "WAIT"]
 
 
 def test_shell_result_keeps_bounded_stdout_and_stderr():