diff --git a/docs/api-reference.md b/docs/api-reference.md index 75bd343..dab9087 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -71,19 +71,23 @@ submodule. Public grammar surface: +- `CanonicalDirective` - `DirectiveKind` - `ValidatedDirective` +- `decompose_directive(text)` - `validate_directive(text)` - `is_canonical_directive(text)` - `render_directive(kind, /, **operands)` -Use this surface for exact canonical validation or canonical directive string -construction only. +Use this surface for exact canonical validation, canonical directive syntax +decomposition, or canonical directive string construction only. Boundary notes: -- no public parser is exposed +- decomposition exposes canonical syntax only +- operands are grammar-level text, not normalized semantic values - validation returns `None` for any non-canonical input +- decomposition returns `None` for any non-canonical input - rendering is syntax-only and performs no state interpretation - `engine.step(...)` remains the authority for clarification, state transitions, and mutation behavior @@ -95,6 +99,10 @@ Boundary notes: clarification-only runtime category; it follows the deterministic semantic rules defined in the specification +`CanonicalDirective.operands` preserves the grammar-recognized operand text. +Core does not lowercase operands, collapse internal operand whitespace, or +convert operand text into engine/domain identifiers at the grammar layer. + ### `engine.state` Read the current authoritative in-memory state snapshot. diff --git a/pyproject.toml b/pyproject.toml index 5f663fc..6e5cb9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "context-compiler" -version = "0.9.0dev2" +version = "0.9.0dev3" description = "Deterministic conversational state engine for LLM applications." readme = "README.md" requires-python = ">=3.11" diff --git a/src/context_compiler/engine.py b/src/context_compiler/engine.py index 3c41448..da5b88a 100644 --- a/src/context_compiler/engine.py +++ b/src/context_compiler/engine.py @@ -17,7 +17,7 @@ STATE_PREMISE, STATE_VERSION, ) -from .grammar import DirectiveKind, _parse_canonical_directive +from .grammar import DirectiveKind, decompose_directive PolicyValue = Literal["use", "prohibit"] @@ -259,9 +259,7 @@ def _apply_replacement_explicit(self, new_item: str, old_item: str) -> None: def _parse_directive(user_input: str) -> Action | None: - # Engine semantics intentionally depend on grammar's internal parsed form. - # This keeps syntax authority in grammar without making the parser a public API. - parsed = _parse_canonical_directive(user_input) + parsed = decompose_directive(user_input) if parsed is None: return None diff --git a/src/context_compiler/grammar.py b/src/context_compiler/grammar.py index f24a6e3..833477b 100644 --- a/src/context_compiler/grammar.py +++ b/src/context_compiler/grammar.py @@ -26,7 +26,7 @@ class ValidatedDirective: @dataclass(frozen=True, slots=True) -class _GrammarParsedDirective: +class CanonicalDirective: text: str kind: DirectiveKind operands: MappingProxyType[str, str] @@ -243,7 +243,7 @@ def _contains_multiple_canonical_directives(text: str) -> bool: return False -def _parse_replace_use(trimmed_text: str) -> _GrammarParsedDirective | None: +def _parse_replace_use(trimmed_text: str) -> CanonicalDirective | None: match = _REPLACE_RE.fullmatch(trimmed_text) if match is None: return None @@ -258,14 +258,14 @@ def _parse_replace_use(trimmed_text: str) -> _GrammarParsedDirective | None: normalized_payload = _normalized_for_matching(trimmed_text) if normalized_payload.count(_INSTEAD_OF_DELIMITER) != 1: return None - return _GrammarParsedDirective( + return CanonicalDirective( text=trimmed_text, kind=DirectiveKind.REPLACE_USE, operands=MappingProxyType({"new_item": new_item, "old_item": old_item}), ) -def _parse_directive(text: str) -> _GrammarParsedDirective | None: +def decompose_directive(text: str) -> CanonicalDirective | None: trimmed_text = _trim_ascii_whitespace(text) if trimmed_text == "": return None @@ -275,17 +275,17 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: normalized = _normalized_for_matching(trimmed_text) if normalized == "clear premise": - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.CLEAR_PREMISE, operands=MappingProxyType({}) ) if normalized == "reset policies": - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.RESET_POLICIES, operands=MappingProxyType({}), ) if normalized == "clear state": - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.CLEAR_STATE, operands=MappingProxyType({}) ) @@ -296,7 +296,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: value = match.group("value") if not _operand_has_content(value) or _operand_starts_with_token(value, "to"): return None - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.SET_PREMISE, operands=MappingProxyType({"value": value}), @@ -309,7 +309,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: value = match.group("value") if not _operand_has_content(value): return None - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.CHANGE_PREMISE, operands=MappingProxyType({"value": value}), @@ -332,7 +332,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: or _INSTEAD_OF_DELIMITER in normalized_item ): return None - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.USE_ITEM, operands=MappingProxyType({"item": item}), @@ -345,7 +345,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: item = match.group("item") if not _operand_has_content(item): return None - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.PROHIBIT_ITEM, operands=MappingProxyType({"item": item}), @@ -358,7 +358,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: item = match.group("item") if not _operand_has_content(item): return None - return _GrammarParsedDirective( + return CanonicalDirective( text=text, kind=DirectiveKind.REMOVE_POLICY, operands=MappingProxyType({"item": item}), @@ -368,16 +368,12 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None: def validate_directive(text: str) -> ValidatedDirective | None: - parsed = _parse_directive(text) + parsed = decompose_directive(text) if parsed is None: return None return ValidatedDirective(text=parsed.text, kind=parsed.kind) -def _parse_canonical_directive(text: str) -> _GrammarParsedDirective | None: - return _parse_directive(text) - - def match_canonical_directive_start(text: str, start: int) -> int | None: return _match_canonical_directive_start(text, start) @@ -426,8 +422,10 @@ def render_directive(kind: DirectiveKind, /, **operands: str) -> str: __all__ = [ + "CanonicalDirective", "DirectiveKind", "ValidatedDirective", + "decompose_directive", "is_canonical_directive", "render_directive", "validate_directive", diff --git a/tests/_api_contract_harness.py b/tests/_api_contract_harness.py index 8a26df6..4679a98 100644 --- a/tests/_api_contract_harness.py +++ b/tests/_api_contract_harness.py @@ -68,6 +68,10 @@ def assert_shape( assert value == grammar.validate_directive(shape["text"]) return + if "kind" in shape and shape["kind"] == "canonical_directive": + assert value == grammar.decompose_directive(shape["text"]) + return + expected_types = shape["type"] if isinstance(expected_types, str): expected_types = [expected_types] @@ -372,6 +376,15 @@ def _validate_shape_spec(shape: object, label: str) -> None: _assert_type(shape["text"], str, f"{label}.text") _assert_type(shape["directive_kind"], str, f"{label}.directive_kind") return + if kind == "canonical_directive": + _assert_closed_keys(shape, {"kind", "text", "directive_kind", "operands"}, label) + _require_fields(shape, {"kind", "text", "directive_kind", "operands"}, label) + _assert_type(shape["text"], str, f"{label}.text") + _assert_type(shape["directive_kind"], str, f"{label}.directive_kind") + _assert_string_keyed_dict(shape["operands"], f"{label}.operands") + for operand_name, operand_value in shape["operands"].items(): + _assert_type(operand_value, str, f"{label}.operands[{operand_name!r}]") + return raise AssertionError(f"{label}.kind has unsupported shape kind {kind!r}") if not has_type: diff --git a/tests/fixtures/conformance/api/public-grammar-v1.json b/tests/fixtures/conformance/api/public-grammar-v1.json index 8dabc8c..8ce1897 100644 --- a/tests/fixtures/conformance/api/public-grammar-v1.json +++ b/tests/fixtures/conformance/api/public-grammar-v1.json @@ -4,19 +4,59 @@ "module": "context_compiler.grammar", "exports": { "names": [ + "CanonicalDirective", "DirectiveKind", "ValidatedDirective", + "decompose_directive", "is_canonical_directive", "render_directive", "validate_directive" ], "members": { + "CanonicalDirective": { + "kind": "class" + }, "DirectiveKind": { "kind": "class" }, "ValidatedDirective": { "kind": "class" }, + "decompose_directive": { + "kind": "callable", + "signature": { + "params": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "has_default": false + } + ] + }, + "shape_probes": [ + { + "kwargs": { + "text": "use docker" + }, + "return_shape": { + "kind": "canonical_directive", + "text": "use docker", + "directive_kind": "use_item", + "operands": { + "item": "docker" + } + } + }, + { + "kwargs": { + "text": "please use docker" + }, + "return_shape": { + "type": "null" + } + } + ] + }, "is_canonical_directive": { "kind": "callable", "signature": { diff --git a/tests/fixtures/conformance/grammar/grammar_decompose_clear_state.json b/tests/fixtures/conformance/grammar/grammar_decompose_clear_state.json new file mode 100644 index 0000000..decc6fa --- /dev/null +++ b/tests/fixtures/conformance/grammar/grammar_decompose_clear_state.json @@ -0,0 +1,15 @@ +{ + "id": "grammar_decompose_clear_state", + "kind": "grammar", + "action": { + "fn": "decompose_directive", + "text": "clear state" + }, + "expected": { + "directive": { + "text": "clear state", + "kind": "clear_state", + "operands": {} + } + } +} diff --git a/tests/fixtures/conformance/grammar/grammar_decompose_invalid_noncanonical.json b/tests/fixtures/conformance/grammar/grammar_decompose_invalid_noncanonical.json new file mode 100644 index 0000000..5fd09be --- /dev/null +++ b/tests/fixtures/conformance/grammar/grammar_decompose_invalid_noncanonical.json @@ -0,0 +1,11 @@ +{ + "id": "grammar_decompose_invalid_noncanonical", + "kind": "grammar", + "action": { + "fn": "decompose_directive", + "text": "please use docker" + }, + "expected": { + "directive": null + } +} diff --git a/tests/fixtures/conformance/grammar/grammar_decompose_preserves_operand_text.json b/tests/fixtures/conformance/grammar/grammar_decompose_preserves_operand_text.json new file mode 100644 index 0000000..fc8f208 --- /dev/null +++ b/tests/fixtures/conformance/grammar/grammar_decompose_preserves_operand_text.json @@ -0,0 +1,17 @@ +{ + "id": "grammar_decompose_preserves_operand_text", + "kind": "grammar", + "action": { + "fn": "decompose_directive", + "text": "Use Docker Engine" + }, + "expected": { + "directive": { + "text": "Use Docker Engine", + "kind": "use_item", + "operands": { + "item": "Docker Engine" + } + } + } +} diff --git a/tests/fixtures/conformance/grammar/grammar_decompose_replace_use.json b/tests/fixtures/conformance/grammar/grammar_decompose_replace_use.json new file mode 100644 index 0000000..eb982a7 --- /dev/null +++ b/tests/fixtures/conformance/grammar/grammar_decompose_replace_use.json @@ -0,0 +1,18 @@ +{ + "id": "grammar_decompose_replace_use", + "kind": "grammar", + "action": { + "fn": "decompose_directive", + "text": "use podman instead of docker" + }, + "expected": { + "directive": { + "text": "use podman instead of docker", + "kind": "replace_use", + "operands": { + "new_item": "podman", + "old_item": "docker" + } + } + } +} diff --git a/tests/test_engine.py b/tests/test_engine.py index 3c134c1..e9541bd 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,6 +12,7 @@ ) from context_compiler.grammar import ( contains_multiple_canonical_directives, + decompose_directive, match_canonical_directive_start, ) @@ -45,6 +46,17 @@ def test_parse_directive_delegates_canonical_kinds_to_existing_actions() -> None assert _parse_directive("clear state") == Action(kind="clear_state") +def test_engine_parse_directive_matches_public_decomposition_boundary() -> None: + parsed = decompose_directive("use podman instead of docker") + + assert parsed is not None + assert _parse_directive(parsed.text) == Action( + kind="replace_use", + new_item=parsed.operands["new_item"], + old_item=parsed.operands["old_item"], + ) + + def test_parse_directive_returns_none_for_invalid_syntax_and_passthrough_inputs() -> None: assert _parse_directive("set premise") is None assert _parse_directive("change premise to") is None diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index 2978806..c40459e 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -6,7 +6,12 @@ from context_compiler import create_engine, get_decision_state from context_compiler.controller import get_step_state, preview, state_diff, step -from context_compiler.grammar import DirectiveKind, render_directive, validate_directive +from context_compiler.grammar import ( + DirectiveKind, + decompose_directive, + render_directive, + validate_directive, +) _STEP_FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" / "conformance" / "step" _STATE_JSON_FIXTURES_DIR = ( @@ -242,9 +247,26 @@ def _validate_grammar_fixture(fixture: dict[str, object], fixture_id: object) -> expected = fixture["expected"] assert isinstance(expected, dict), fixture_id fn = action["fn"] - assert fn in {"validate_directive", "render_directive"}, fixture_id + assert fn in {"decompose_directive", "validate_directive", "render_directive"}, fixture_id - if fn == "validate_directive": + if fn == "decompose_directive": + _assert_allowed_keys(action, {"fn", "text"}, fixture_id, "action") + assert isinstance(action["text"], str), fixture_id + _assert_allowed_keys(expected, {"directive"}, fixture_id, "expected") + directive = expected["directive"] + if directive is not None: + assert isinstance(directive, dict), fixture_id + _assert_allowed_keys( + directive, {"text", "kind", "operands"}, fixture_id, "expected.directive" + ) + assert isinstance(directive["text"], str), fixture_id + assert isinstance(directive["kind"], str), fixture_id + assert isinstance(directive["operands"], dict), fixture_id + assert all(isinstance(key, str) for key in directive["operands"]), fixture_id + assert all(isinstance(value, str) for value in directive["operands"].values()), ( + fixture_id + ) + elif fn == "validate_directive": _assert_allowed_keys(action, {"fn", "text"}, fixture_id, "action") assert isinstance(action["text"], str), fixture_id _assert_allowed_keys(expected, {"validated"}, fixture_id, "expected") @@ -411,7 +433,17 @@ def test_grammar_fixtures() -> None: expected = fixture["expected"] fn = action["fn"] - if fn == "validate_directive": + if fn == "decompose_directive": + directive = decompose_directive(action["text"]) + expected_directive = expected["directive"] + if expected_directive is None: + assert directive is None, fixture_id + else: + assert directive is not None, fixture_id + assert directive.text == expected_directive["text"], fixture_id + assert directive.kind.value == expected_directive["kind"], fixture_id + assert dict(directive.operands) == expected_directive["operands"], fixture_id + elif fn == "validate_directive": validated = validate_directive(action["text"]) expected_validated = expected["validated"] if expected_validated is None: diff --git a/tests/test_grammar.py b/tests/test_grammar.py index 75e012c..16886a0 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -5,9 +5,11 @@ import context_compiler.grammar as grammar_module from context_compiler.grammar import ( + CanonicalDirective, DirectiveKind, ValidatedDirective, contains_multiple_canonical_directives, + decompose_directive, is_canonical_directive, match_canonical_directive_start, render_directive, @@ -51,6 +53,17 @@ def test_validated_directive_is_frozen_and_slotted() -> None: validated.text = "change premise to concise replies" # type: ignore[misc] +def test_canonical_directive_is_frozen_and_slotted() -> None: + directive = CanonicalDirective( + text="use docker", + kind=DirectiveKind.USE_ITEM, + operands=MappingProxyType({"item": "docker"}), + ) + assert directive.__slots__ == ("text", "kind", "operands") + with pytest.raises(FrozenInstanceError): + directive.kind = DirectiveKind.PROHIBIT_ITEM # type: ignore[misc] + + @pytest.mark.parametrize( ("text", "expected_kind"), [ @@ -73,6 +86,35 @@ def test_validate_directive_accepts_each_canonical_family( assert is_canonical_directive(text) is True +@pytest.mark.parametrize( + ("text", "expected_kind", "expected_operands"), + [ + ("set premise concise replies", DirectiveKind.SET_PREMISE, {"value": "concise replies"}), + ("change premise to formal tone", DirectiveKind.CHANGE_PREMISE, {"value": "formal tone"}), + ("use docker", DirectiveKind.USE_ITEM, {"item": "docker"}), + ("prohibit peanuts", DirectiveKind.PROHIBIT_ITEM, {"item": "peanuts"}), + ("remove policy docker", DirectiveKind.REMOVE_POLICY, {"item": "docker"}), + ( + "use podman instead of docker", + DirectiveKind.REPLACE_USE, + {"new_item": "podman", "old_item": "docker"}, + ), + ("clear premise", DirectiveKind.CLEAR_PREMISE, {}), + ("reset policies", DirectiveKind.RESET_POLICIES, {}), + ("clear state", DirectiveKind.CLEAR_STATE, {}), + ], +) +def test_decompose_directive_accepts_each_canonical_family( + text: str, expected_kind: DirectiveKind, expected_operands: dict[str, str] +) -> None: + decomposed = decompose_directive(text) + assert decomposed == CanonicalDirective( + text=text, + kind=expected_kind, + operands=MappingProxyType(expected_operands), + ) + + @pytest.mark.parametrize( "text", [ @@ -93,6 +135,7 @@ def test_validate_directive_accepts_each_canonical_family( ) def test_validate_directive_rejects_non_canonical_inputs(text: str) -> None: assert validate_directive(text) is None + assert decompose_directive(text) is None assert is_canonical_directive(text) is False @@ -111,6 +154,24 @@ def test_validate_directive_accepts_lexically_normalized_canonical_input( assert validated == ValidatedDirective(text=text, kind=expected_kind) +@pytest.mark.parametrize( + ("text", "expected_operands"), + [ + ("Use docker", {"item": "docker"}), + ("use\tdocker", {"item": "docker"}), + (" use docker ", {"item": "docker"}), + ("Use Docker", {"item": "Docker"}), + ("use docker engine", {"item": "docker engine"}), + ], +) +def test_decompose_directive_preserves_current_operand_casing_and_whitespace( + text: str, expected_operands: dict[str, str] +) -> None: + decomposed = decompose_directive(text) + assert decomposed is not None + assert dict(decomposed.operands) == expected_operands + + @pytest.mark.parametrize( ("kind", "operands", "expected"), [ @@ -262,8 +323,8 @@ def test_contains_multiple_canonical_directives_reports_public_compound_detectio assert contains_multiple_canonical_directives(text) is expected -def test_private_parse_canonical_directive_preserves_internal_engine_seam() -> None: - parsed = grammar_module._parse_canonical_directive("use docker") +def test_decompose_directive_returns_canonical_operands_for_use_item() -> None: + parsed = decompose_directive("use docker") assert parsed is not None assert parsed.text == "use docker" @@ -271,6 +332,14 @@ def test_private_parse_canonical_directive_preserves_internal_engine_seam() -> N assert parsed.operands == {"item": "docker"} +def test_validate_directive_is_projection_of_decomposition() -> None: + decomposed = decompose_directive("use docker") + validated = validate_directive("use docker") + + assert decomposed is not None + assert validated == ValidatedDirective(text=decomposed.text, kind=decomposed.kind) + + def test_internal_match_directive_token_rejects_truncated_and_non_whitespace_separator() -> None: assert ( grammar_module._match_directive_token( diff --git a/tests/test_public_grammar_root_exports.py b/tests/test_public_grammar_root_exports.py index c75d976..73fd8fc 100644 --- a/tests/test_public_grammar_root_exports.py +++ b/tests/test_public_grammar_root_exports.py @@ -14,7 +14,9 @@ def test_root_does_not_export_public_grammar_surface() -> None: def test_grammar_submodule_preserves_public_grammar_surface() -> None: + assert grammar_module.CanonicalDirective is not None assert grammar_module.DirectiveKind is not None + assert grammar_module.decompose_directive is not None assert grammar_module.validate_directive is not None assert grammar_module.render_directive is not None assert grammar_module.is_canonical_directive is not None @@ -31,7 +33,6 @@ def test_root_does_not_export_private_grammar_implementation() -> None: "_REPLACE_RE", "_match_directive_token", "_match_canonical_directive_start", - "_parse_directive", ): assert name not in context_compiler.__all__ assert not hasattr(context_compiler, name) diff --git a/uv.lock b/uv.lock index 17d03c7..eb86f33 100644 --- a/uv.lock +++ b/uv.lock @@ -296,7 +296,7 @@ wheels = [ [[package]] name = "context-compiler" -version = "0.9.0.dev2" +version = "0.9.0.dev3" source = { editable = "." } [package.optional-dependencies]