diff --git a/.env.example b/.env.example index a87e973..ebb69aa 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,4 @@ OPENROUTER_API_KEY= OPENROUTER_APP_NAME=testql OPENROUTER_SITE_URL= LLM_MODEL=openrouter/z-ai/glm-5.2 +TESTQL_LIVE_LLM_MODEL=openrouter/z-ai/glm-5.2 diff --git a/README.md b/README.md index f6cba4f..0fe5b39 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ Set `OPENROUTER_APP_NAME` to identify TestQL in OpenRouter logs. If it is not set, TestQL uses the current project folder name. See `.env.example` for the GLM 5.2 defaults. +The optional live nlp2dsl conversation provider similarly uses +`ConversationFields 1.0.0`. It returns only requested missing fields, while +preserving the existing plain field mapping passed into `llmContext`. + ## Artifact Discovery, Topology, and Web Inspection diff --git a/pyproject.toml b/pyproject.toml index d822e1e..2ededbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,9 @@ testql = [ "contracts/nlp2env/v1/*.gbnf", "contracts/nlp2env/v1/*.json", "contracts/nlp2env/v1/*.proto", + "contracts/nlp2dsl_conversation/v1/*.gbnf", + "contracts/nlp2dsl_conversation/v1/*.json", + "contracts/nlp2dsl_conversation/v1/*.proto", "data/*.json", "interpreter/*.cjs", ] diff --git a/testql/adapters/nlp2dsl/live_llm.py b/testql/adapters/nlp2dsl/live_llm.py index 275d70b..55d1234 100644 --- a/testql/adapters/nlp2dsl/live_llm.py +++ b/testql/adapters/nlp2dsl/live_llm.py @@ -9,26 +9,38 @@ import httpx +from testql.contracts.nlp2dsl_conversation import response_format, validate_payload + @dataclass class LiveLLMProvider: """Call an OpenAI-compatible chat API to fill missing dialog fields.""" api_key: str - model: str = "openrouter/qwen/qwen3-coder-next" + model: str = "openrouter/z-ai/glm-5.2" base_url: str = "https://openrouter.ai/api/v1" timeout_s: float = 60.0 extra_headers: dict[str, str] = field(default_factory=dict) @classmethod def from_env(cls) -> "LiveLLMProvider": - api_key = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LLM_API_KEY") or "" + api_key = ( + os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LLM_API_KEY") or "" + ) if not api_key: - raise RuntimeError("TESTQL_LIVE_LLM=1 requires OPENROUTER_API_KEY or LLM_API_KEY") + raise RuntimeError( + "TESTQL_LIVE_LLM=1 requires OPENROUTER_API_KEY or LLM_API_KEY" + ) return cls( api_key=api_key, - model=os.environ.get("TESTQL_LIVE_LLM_MODEL", os.environ.get("LLM_MODEL", "openrouter/qwen/qwen3-coder-next")), - base_url=os.environ.get("TESTQL_LIVE_LLM_BASE_URL", os.environ.get("LLM_BASE_URL", "https://openrouter.ai/api/v1")).rstrip("/"), + model=os.environ.get( + "TESTQL_LIVE_LLM_MODEL", + os.environ.get("LLM_MODEL", "openrouter/z-ai/glm-5.2"), + ), + base_url=os.environ.get( + "TESTQL_LIVE_LLM_BASE_URL", + os.environ.get("LLM_BASE_URL", "https://openrouter.ai/api/v1"), + ).rstrip("/"), ) def reply_for( @@ -38,17 +50,28 @@ def reply_for( missing: list[str] | None = None, context: dict[str, Any] | None = None, ) -> dict[str, Any]: - prompt = self._build_prompt(conversation_id, missing=missing or [], context=context or {}) + prompt = self._build_prompt( + conversation_id, missing=missing or [], context=context or {} + ) content = self._chat(prompt) - return self._parse_json_object(content) + payload = self._parse_json_object(content) + fields = payload["fields"] + unexpected = set(fields) - set(missing or []) + if missing and unexpected: + names = ", ".join(sorted(unexpected)) + raise ValueError(f"live LLM returned fields outside missing set: {names}") + return fields - def _build_prompt(self, conversation_id: str, *, missing: list[str], context: dict[str, Any]) -> str: + def _build_prompt( + self, conversation_id: str, *, missing: list[str], context: dict[str, Any] + ) -> str: return ( "You are completing missing fields for an automated integration test.\n" f"conversationId: {conversation_id}\n" f"missing fields: {missing}\n" f"context: {json.dumps(context, ensure_ascii=False)}\n" - "Respond with a single JSON object only — keys should address the missing fields " + "Respond with a ConversationFields 1.0.0 JSON object only. " + "Put values under fields; keys must address the missing fields " "(e.g. attachmentPath, recipient). No markdown." ) @@ -57,12 +80,19 @@ def _chat(self, prompt: str) -> str: headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", + "X-Title": os.environ.get("OPENROUTER_APP_NAME", "").strip() + or os.path.basename(os.getcwd()) + or "testql", **self.extra_headers, } + site_url = os.environ.get("OPENROUTER_SITE_URL", "").strip() + if site_url and "HTTP-Referer" not in headers: + headers["HTTP-Referer"] = site_url payload = { - "model": self.model, + "model": self.model.removeprefix("openrouter/"), "messages": [{"role": "user", "content": prompt}], "temperature": 0, + "response_format": response_format(), } with httpx.Client(timeout=self.timeout_s) as client: response = client.post(url, headers=headers, json=payload) @@ -79,12 +109,11 @@ def _chat(self, prompt: str) -> str: @staticmethod def _parse_json_object(text: str) -> dict[str, Any]: - stripped = text.strip() - if stripped.startswith("```"): - stripped = stripped.strip("`") - if stripped.lower().startswith("json"): - stripped = stripped[4:].strip() - parsed = json.loads(stripped) + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError("live LLM response must be a single JSON object") from exc if not isinstance(parsed, dict): raise ValueError("live LLM response must be a JSON object") + validate_payload(parsed) return parsed diff --git a/testql/contracts/nlp2dsl_conversation/__init__.py b/testql/contracts/nlp2dsl_conversation/__init__.py new file mode 100644 index 0000000..845c8d6 --- /dev/null +++ b/testql/contracts/nlp2dsl_conversation/__init__.py @@ -0,0 +1,46 @@ +"""Runtime binding for the nlp2dsl ConversationFields response contract.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from jsonschema import Draft202012Validator + +CONTRACT_VERSION = "1.0.0" + + +def _contract_file(name: str): + return files(__package__).joinpath("v1", name) + + +def load_schema() -> dict[str, Any]: + return json.loads( + _contract_file("conversation-fields.schema.json").read_text(encoding="utf-8") + ) + + +def validate_payload(payload: object) -> None: + validator = Draft202012Validator(load_schema()) + errors = sorted(validator.iter_errors(payload), key=lambda error: list(error.path)) + if errors: + first = errors[0] + location = ".".join(str(part) for part in first.absolute_path) or "$" + raise ValueError( + f"live LLM response violates ConversationFields v1 at {location}: {first.message}" + ) + + +def response_format() -> dict[str, Any]: + return { + "type": "json_schema", + "json_schema": { + "name": "testql_conversation_fields_v1", + "strict": True, + "schema": load_schema(), + }, + } + + +__all__ = ["CONTRACT_VERSION", "load_schema", "response_format", "validate_payload"] diff --git a/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.gbnf b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.gbnf new file mode 100644 index 0000000..66ff72a --- /dev/null +++ b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.gbnf @@ -0,0 +1,9 @@ +root ::= ws "{" ws version ws "," ws fields ws "}" ws +version ::= "\"contractVersion\"" ws ":" ws "\"1.0.0\"" +fields ::= "\"fields\"" ws ":" ws "{" ws pair (ws "," ws pair)* ws "}" +pair ::= string ws ":" ws string + +string ::= "\"" char* "\"" +char ::= [^"\\\x00-\x1f] | "\\" (["\\/bfnrt] | "u" hex hex hex hex) +hex ::= [0-9a-fA-F] +ws ::= [ \t\n\r]* diff --git a/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.proto b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.proto new file mode 100644 index 0000000..1e3c8d8 --- /dev/null +++ b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package testql.contracts.nlp2dsl_conversation.v1; + +message ConversationFields { + string contract_version = 1 [json_name = "contractVersion"]; + map fields = 2; +} diff --git a/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.schema.json b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.schema.json new file mode 100644 index 0000000..f9c5982 --- /dev/null +++ b/testql/contracts/nlp2dsl_conversation/v1/conversation-fields.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://testql.dev/contracts/nlp2dsl-conversation/v1/conversation-fields.schema.json", + "title": "TestQL ConversationFields v1", + "type": "object", + "required": ["contractVersion", "fields"], + "properties": { + "contractVersion": { "const": "1.0.0" }, + "fields": { + "type": "object", + "minProperties": 1, + "maxProperties": 32, + "propertyNames": { + "pattern": "^[A-Za-z][A-Za-z0-9_.-]{0,127}$" + }, + "additionalProperties": { + "type": "string", + "maxLength": 8000 + } + } + }, + "additionalProperties": false +} diff --git a/testql/contracts/nlp2dsl_conversation/v1/manifest.json b/testql/contracts/nlp2dsl_conversation/v1/manifest.json new file mode 100644 index 0000000..2fe6ec1 --- /dev/null +++ b/testql/contracts/nlp2dsl_conversation/v1/manifest.json @@ -0,0 +1,16 @@ +{ + "contract": "testql.nlp2dsl.ConversationFields", + "version": "1.0.0", + "boundary": "testql.adapters.nlp2dsl.live_llm.LiveLLMProvider.reply_for", + "mediaType": "application/json", + "artifacts": { + "grammar": "conversation-fields.gbnf", + "protobuf": "conversation-fields.proto", + "schema": "conversation-fields.schema.json" + }, + "provider": "response_format.json_schema", + "runtime": { + "parser": "testql.adapters.nlp2dsl.live_llm.LiveLLMProvider._parse_json_object", + "validator": "testql.contracts.nlp2dsl_conversation.validate_payload" + } +} diff --git a/tests/fixtures/contracts/nlp2dsl-conversation/v1/invalid-conversation-fields.json b/tests/fixtures/contracts/nlp2dsl-conversation/v1/invalid-conversation-fields.json new file mode 100644 index 0000000..d09d12a --- /dev/null +++ b/tests/fixtures/contracts/nlp2dsl-conversation/v1/invalid-conversation-fields.json @@ -0,0 +1,6 @@ +{ + "contractVersion": "1.0.0", + "fields": { + "recipient": { "address": "test@example.com" } + } +} diff --git a/tests/fixtures/contracts/nlp2dsl-conversation/v1/valid-conversation-fields.json b/tests/fixtures/contracts/nlp2dsl-conversation/v1/valid-conversation-fields.json new file mode 100644 index 0000000..bff269f --- /dev/null +++ b/tests/fixtures/contracts/nlp2dsl-conversation/v1/valid-conversation-fields.json @@ -0,0 +1,7 @@ +{ + "contractVersion": "1.0.0", + "fields": { + "attachmentPath": "/tmp/invoice.pdf", + "recipient": "test@example.com" + } +} diff --git a/tests/test_conversation_live_llm.py b/tests/test_conversation_live_llm.py index 6d45eef..a680dcc 100644 --- a/tests/test_conversation_live_llm.py +++ b/tests/test_conversation_live_llm.py @@ -7,7 +7,11 @@ import httpx import pytest -from testql.adapters.nlp2dsl import LiveLLMProvider, live_llm_enabled, resolve_llm_provider +from testql.adapters.nlp2dsl import ( + LiveLLMProvider, + live_llm_enabled, + resolve_llm_provider, +) from testql.adapters.nlp2dsl.mock_llm import MockLLMProvider from testql.conversation import ConversationRunner @@ -34,10 +38,10 @@ def test_live_without_key_raises(self, monkeypatch): class TestLiveLLMParsing: - def test_parse_json_object_strips_fence(self): + def test_parse_json_object_rejects_fence(self): raw = '```json\n{"attachmentPath": "/tmp/x.pdf"}\n```' - parsed = LiveLLMProvider._parse_json_object(raw) - assert parsed["attachmentPath"] == "/tmp/x.pdf" + with pytest.raises(ValueError, match="single JSON object"): + LiveLLMProvider._parse_json_object(raw) @pytest.mark.live_llm @@ -67,16 +71,22 @@ def test_conversation_runner_with_live_llm_smoke(): from testql.ir import Capture, Nlp2DslStep, TestPlan - plan = TestPlan(steps=[ - Nlp2DslStep(endpoint="chatstart", payload={"userId": "live-test"}, captures=[ - Capture(var_name="conversationId", from_path="conversationId"), - ]), - Nlp2DslStep( - endpoint="chatmessage", - payload={"conversationId": "${conversationId}", "text": "ping"}, - mock_llm={}, - ), - ]) + plan = TestPlan( + steps=[ + Nlp2DslStep( + endpoint="chatstart", + payload={"userId": "live-test"}, + captures=[ + Capture(var_name="conversationId", from_path="conversationId"), + ], + ), + Nlp2DslStep( + endpoint="chatmessage", + payload={"conversationId": "${conversationId}", "text": "ping"}, + mock_llm={}, + ), + ] + ) runner = ConversationRunner(api_url=nlp2dsl_url, live_llm=True) result = runner.run(plan) assert any(t.kind == "nlp2dsl" for t in result.turns) diff --git a/tests/test_conversation_llm_contract.py b/tests/test_conversation_llm_contract.py new file mode 100644 index 0000000..55ca4c7 --- /dev/null +++ b/tests/test_conversation_llm_contract.py @@ -0,0 +1,115 @@ +"""Offline contract tests for the live nlp2dsl conversation provider.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from testql.adapters.nlp2dsl.live_llm import LiveLLMProvider +from testql.contracts.nlp2dsl_conversation import ( + CONTRACT_VERSION, + load_schema, + validate_payload, +) + +FIXTURES = ( + Path(__file__).parent / "fixtures" / "contracts" / "nlp2dsl-conversation" / "v1" +) +CONTRACTS = ( + Path(__file__).parents[1] / "testql" / "contracts" / "nlp2dsl_conversation" / "v1" +) + + +def _fixture(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_valid_fixture_passes_runtime_contract() -> None: + payload = _fixture("valid-conversation-fields.json") + validate_payload(payload) + assert LiveLLMProvider._parse_json_object(json.dumps(payload)) == payload + + +@pytest.mark.parametrize( + "payload", + [ + _fixture("invalid-conversation-fields.json"), + {"contractVersion": "2.0.0", "fields": {"recipient": "a@b.c"}}, + {"contractVersion": "1.0.0", "fields": {}}, + {"contractVersion": "1.0.0", "fields": {"bad key": "value"}}, + ], +) +def test_invalid_payloads_fail_closed(payload: dict) -> None: + with pytest.raises(ValueError, match="violates ConversationFields v1"): + validate_payload(payload) + + +def test_parser_rejects_markdown_fence() -> None: + with pytest.raises(ValueError, match="single JSON object"): + LiveLLMProvider._parse_json_object('```json\n{"contractVersion":"1.0.0"}\n```') + + +def test_reply_rejects_fields_not_requested(monkeypatch) -> None: + provider = LiveLLMProvider(api_key="test") + monkeypatch.setattr( + provider, + "_chat", + lambda prompt: json.dumps( + {"contractVersion": "1.0.0", "fields": {"admin": "true"}} + ), + ) + with pytest.raises(ValueError, match="outside missing set: admin"): + provider.reply_for("conv", missing=["recipient"]) + + +def test_openrouter_request_uses_schema_app_name_and_normalized_model( + monkeypatch, tmp_path: Path +) -> None: + project = tmp_path / "dialog-app" + project.mkdir() + monkeypatch.chdir(project) + monkeypatch.delenv("OPENROUTER_APP_NAME", raising=False) + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["headers"] = request.headers + captured["payload"] = json.loads(request.content) + content = json.dumps( + {"contractVersion": "1.0.0", "fields": {"recipient": "a@b.c"}} + ) + return httpx.Response( + 200, json={"choices": [{"message": {"content": content}}]} + ) + + provider = LiveLLMProvider( + api_key="test", + model="openrouter/z-ai/glm-5.2", + extra_headers={}, + ) + real_client = httpx.Client + monkeypatch.setattr( + httpx, + "Client", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler)), + ) + assert provider.reply_for("conv", missing=["recipient"]) == {"recipient": "a@b.c"} + assert captured["headers"]["x-title"] == "dialog-app" + assert captured["payload"]["model"] == "z-ai/glm-5.2" + assert ( + captured["payload"]["response_format"]["json_schema"]["schema"] == load_schema() + ) + + +def test_manifest_binds_artifacts_to_live_provider() -> None: + manifest = json.loads((CONTRACTS / "manifest.json").read_text(encoding="utf-8")) + assert manifest["version"] == CONTRACT_VERSION + assert manifest["boundary"].endswith("LiveLLMProvider.reply_for") + for artifact in manifest["artifacts"].values(): + assert (CONTRACTS / artifact).is_file() + proto = (CONTRACTS / "conversation-fields.proto").read_text(encoding="utf-8") + grammar = (CONTRACTS / "conversation-fields.gbnf").read_text(encoding="utf-8") + assert "map fields = 2;" in proto + assert f'\\"{CONTRACT_VERSION}\\"' in grammar