feat: add public directive decomposition

docs/api-reference.md

@@ -71,19 +71,23 @@ submodule.
 
 Public grammar surface:
 
+- `CanonicalDirective`
 - `DirectiveKind`
 - `ValidatedDirective`
+- `decompose_directive(text)`
 - `validate_directive(text)`
 - `is_canonical_directive(text)`
 - `render_directive(kind, /, **operands)`
 
-Use this surface for exact canonical validation or canonical directive string
-construction only.
+Use this surface for exact canonical validation, canonical directive syntax
+decomposition, or canonical directive string construction only.
 
 Boundary notes:
 
-- no public parser is exposed
+- decomposition exposes canonical syntax only
+- operands are grammar-level text, not normalized semantic values
 - validation returns `None` for any non-canonical input
+- decomposition returns `None` for any non-canonical input
 - rendering is syntax-only and performs no state interpretation
 - `engine.step(...)` remains the authority for clarification, state
   transitions, and mutation behavior
@@ -95,6 +99,10 @@ Boundary notes:
   clarification-only runtime category; it follows the deterministic semantic
   rules defined in the specification
 
+`CanonicalDirective.operands` preserves the grammar-recognized operand text.
+Core does not lowercase operands, collapse internal operand whitespace, or
+convert operand text into engine/domain identifiers at the grammar layer.
+
 ### `engine.state`
 
 Read the current authoritative in-memory state snapshot.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "context-compiler"
-version = "0.9.0dev2"
+version = "0.9.0dev3"
 description = "Deterministic conversational state engine for LLM applications."
 readme = "README.md"
 requires-python = ">=3.11"

src/context_compiler/engine.py

@@ -17,7 +17,7 @@
     STATE_PREMISE,
     STATE_VERSION,
 )
-from .grammar import DirectiveKind, _parse_canonical_directive
+from .grammar import DirectiveKind, decompose_directive
 
 PolicyValue = Literal["use", "prohibit"]
 
@@ -259,9 +259,7 @@ def _apply_replacement_explicit(self, new_item: str, old_item: str) -> None:
 
 
 def _parse_directive(user_input: str) -> Action | None:
-    # Engine semantics intentionally depend on grammar's internal parsed form.
-    # This keeps syntax authority in grammar without making the parser a public API.
-    parsed = _parse_canonical_directive(user_input)
+    parsed = decompose_directive(user_input)
     if parsed is None:
         return None
 

src/context_compiler/grammar.py

@@ -26,7 +26,7 @@ class ValidatedDirective:
 
 
 @dataclass(frozen=True, slots=True)
-class _GrammarParsedDirective:
+class CanonicalDirective:
     text: str
     kind: DirectiveKind
     operands: MappingProxyType[str, str]
@@ -243,7 +243,7 @@ def _contains_multiple_canonical_directives(text: str) -> bool:
     return False
 
 
-def _parse_replace_use(trimmed_text: str) -> _GrammarParsedDirective | None:
+def _parse_replace_use(trimmed_text: str) -> CanonicalDirective | None:
     match = _REPLACE_RE.fullmatch(trimmed_text)
     if match is None:
         return None
@@ -258,14 +258,14 @@ def _parse_replace_use(trimmed_text: str) -> _GrammarParsedDirective | None:
     normalized_payload = _normalized_for_matching(trimmed_text)
     if normalized_payload.count(_INSTEAD_OF_DELIMITER) != 1:
         return None
-    return _GrammarParsedDirective(
+    return CanonicalDirective(
         text=trimmed_text,
         kind=DirectiveKind.REPLACE_USE,
         operands=MappingProxyType({"new_item": new_item, "old_item": old_item}),
     )
 
 
-def _parse_directive(text: str) -> _GrammarParsedDirective | None:
+def decompose_directive(text: str) -> CanonicalDirective | None:
     trimmed_text = _trim_ascii_whitespace(text)
     if trimmed_text == "":
         return None
@@ -275,17 +275,17 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
     normalized = _normalized_for_matching(trimmed_text)
 
     if normalized == "clear premise":
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text, kind=DirectiveKind.CLEAR_PREMISE, operands=MappingProxyType({})
         )
     if normalized == "reset policies":
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.RESET_POLICIES,
             operands=MappingProxyType({}),
         )
     if normalized == "clear state":
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text, kind=DirectiveKind.CLEAR_STATE, operands=MappingProxyType({})
         )
 
@@ -296,7 +296,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
         value = match.group("value")
         if not _operand_has_content(value) or _operand_starts_with_token(value, "to"):
             return None
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.SET_PREMISE,
             operands=MappingProxyType({"value": value}),
@@ -309,7 +309,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
         value = match.group("value")
         if not _operand_has_content(value):
             return None
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.CHANGE_PREMISE,
             operands=MappingProxyType({"value": value}),
@@ -332,7 +332,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
             or _INSTEAD_OF_DELIMITER in normalized_item
         ):
             return None
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.USE_ITEM,
             operands=MappingProxyType({"item": item}),
@@ -345,7 +345,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
         item = match.group("item")
         if not _operand_has_content(item):
             return None
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.PROHIBIT_ITEM,
             operands=MappingProxyType({"item": item}),
@@ -358,7 +358,7 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
         item = match.group("item")
         if not _operand_has_content(item):
             return None
-        return _GrammarParsedDirective(
+        return CanonicalDirective(
             text=text,
             kind=DirectiveKind.REMOVE_POLICY,
             operands=MappingProxyType({"item": item}),
@@ -368,16 +368,12 @@ def _parse_directive(text: str) -> _GrammarParsedDirective | None:
 
 
 def validate_directive(text: str) -> ValidatedDirective | None:
-    parsed = _parse_directive(text)
+    parsed = decompose_directive(text)
     if parsed is None:
         return None
     return ValidatedDirective(text=parsed.text, kind=parsed.kind)
 
 
-def _parse_canonical_directive(text: str) -> _GrammarParsedDirective | None:
-    return _parse_directive(text)
-
-
 def match_canonical_directive_start(text: str, start: int) -> int | None:
     return _match_canonical_directive_start(text, start)
 
@@ -426,8 +422,10 @@ def render_directive(kind: DirectiveKind, /, **operands: str) -> str:
 
 
 __all__ = [
+    "CanonicalDirective",
     "DirectiveKind",
     "ValidatedDirective",
+    "decompose_directive",
     "is_canonical_directive",
     "render_directive",
     "validate_directive",

tests/_api_contract_harness.py

@@ -68,6 +68,10 @@ def assert_shape(
         assert value == grammar.validate_directive(shape["text"])
         return
 
+    if "kind" in shape and shape["kind"] == "canonical_directive":
+        assert value == grammar.decompose_directive(shape["text"])
+        return
+
     expected_types = shape["type"]
     if isinstance(expected_types, str):
         expected_types = [expected_types]
@@ -372,6 +376,15 @@ def _validate_shape_spec(shape: object, label: str) -> None:
             _assert_type(shape["text"], str, f"{label}.text")
             _assert_type(shape["directive_kind"], str, f"{label}.directive_kind")
             return
+        if kind == "canonical_directive":
+            _assert_closed_keys(shape, {"kind", "text", "directive_kind", "operands"}, label)
+            _require_fields(shape, {"kind", "text", "directive_kind", "operands"}, label)
+            _assert_type(shape["text"], str, f"{label}.text")
+            _assert_type(shape["directive_kind"], str, f"{label}.directive_kind")
+            _assert_string_keyed_dict(shape["operands"], f"{label}.operands")
+            for operand_name, operand_value in shape["operands"].items():
+                _assert_type(operand_value, str, f"{label}.operands[{operand_name!r}]")
+            return
         raise AssertionError(f"{label}.kind has unsupported shape kind {kind!r}")
 
     if not has_type:

tests/fixtures/conformance/api/public-grammar-v1.json

@@ -4,19 +4,59 @@
   "module": "context_compiler.grammar",
   "exports": {
     "names": [
+      "CanonicalDirective",
       "DirectiveKind",
       "ValidatedDirective",
+      "decompose_directive",
       "is_canonical_directive",
       "render_directive",
       "validate_directive"
     ],
     "members": {
+      "CanonicalDirective": {
+        "kind": "class"
+      },
       "DirectiveKind": {
         "kind": "class"
       },
       "ValidatedDirective": {
         "kind": "class"
       },
+      "decompose_directive": {
+        "kind": "callable",
+        "signature": {
+          "params": [
+            {
+              "name": "text",
+              "kind": "POSITIONAL_OR_KEYWORD",
+              "has_default": false
+            }
+          ]
+        },
+        "shape_probes": [
+          {
+            "kwargs": {
+              "text": "use docker"
+            },
+            "return_shape": {
+              "kind": "canonical_directive",
+              "text": "use docker",
+              "directive_kind": "use_item",
+              "operands": {
+                "item": "docker"
+              }
+            }
+          },
+          {
+            "kwargs": {
+              "text": "please use docker"
+            },
+            "return_shape": {
+              "type": "null"
+            }
+          }
+        ]
+      },
       "is_canonical_directive": {
         "kind": "callable",
         "signature": {

tests/fixtures/conformance/grammar/grammar_decompose_clear_state.json

@@ -0,0 +1,15 @@
+{
+  "id": "grammar_decompose_clear_state",
+  "kind": "grammar",
+  "action": {
+    "fn": "decompose_directive",
+    "text": "clear state"
+  },
+  "expected": {
+    "directive": {
+      "text": "clear state",
+      "kind": "clear_state",
+      "operands": {}
+    }
+  }
+}

tests/fixtures/conformance/grammar/grammar_decompose_invalid_noncanonical.json

@@ -0,0 +1,11 @@
+{
+  "id": "grammar_decompose_invalid_noncanonical",
+  "kind": "grammar",
+  "action": {
+    "fn": "decompose_directive",
+    "text": "please use docker"
+  },
+  "expected": {
+    "directive": null
+  }
+}

tests/fixtures/conformance/grammar/grammar_decompose_preserves_operand_text.json

@@ -0,0 +1,17 @@
+{
+  "id": "grammar_decompose_preserves_operand_text",
+  "kind": "grammar",
+  "action": {
+    "fn": "decompose_directive",
+    "text": "Use    Docker Engine"
+  },
+  "expected": {
+    "directive": {
+      "text": "Use    Docker Engine",
+      "kind": "use_item",
+      "operands": {
+        "item": "Docker Engine"
+      }
+    }
+  }
+}

tests/fixtures/conformance/grammar/grammar_decompose_replace_use.json

@@ -0,0 +1,18 @@
+{
+  "id": "grammar_decompose_replace_use",
+  "kind": "grammar",
+  "action": {
+    "fn": "decompose_directive",
+    "text": "use podman instead of docker"
+  },
+  "expected": {
+    "directive": {
+      "text": "use podman instead of docker",
+      "kind": "replace_use",
+      "operands": {
+        "new_item": "podman",
+        "old_item": "docker"
+      }
+    }
+  }
+}

tests/test_engine.py

@@ -12,6 +12,7 @@
 )
 from context_compiler.grammar import (
     contains_multiple_canonical_directives,
+    decompose_directive,
     match_canonical_directive_start,
 )
 
@@ -45,6 +46,17 @@ def test_parse_directive_delegates_canonical_kinds_to_existing_actions() -> None
     assert _parse_directive("clear state") == Action(kind="clear_state")
 
 
+def test_engine_parse_directive_matches_public_decomposition_boundary() -> None:
+    parsed = decompose_directive("use podman instead of docker")
+
+    assert parsed is not None
+    assert _parse_directive(parsed.text) == Action(
+        kind="replace_use",
+        new_item=parsed.operands["new_item"],
+        old_item=parsed.operands["old_item"],
+    )
+
+
 def test_parse_directive_returns_none_for_invalid_syntax_and_passthrough_inputs() -> None:
     assert _parse_directive("set premise") is None
     assert _parse_directive("change premise to") is None