From a0f3dee3cd1b1ef35981c77d710fb6eea5f453ac Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Fri, 14 Aug 2026 18:34:11 -0700
Subject: [PATCH 01/13] Wire six 6f observability fixtures

Activates 145, 146, 147, 149, 150 and 151, cutting the fixtures that
run nowhere from fourteen to eight.

Adds proposal 0107's raises sub-directive to the retrieval mocks. The
category must still come from the declared status, so rather than
restate the status-to-category table in the harness, the real classifier
runs on a probe response and the harness raises a dynamically-named
subclass of whatever it returns: the category is inherited while the
class name and message supply the literal values 150 and 151 assert.

Adds the five token-budget invariants fixture 146 declares, covering a
null prompt_tokens leaving the input bound unevaluable while a sound
total bound still evaluates. The exceeded span signal must be absent
rather than false, since false would claim the bound was evaluated.

Fixtures 144 and 148 stay deferred with reasons naming what they need:
both want a driver rather than a dispatch entry.
---
 tests/conformance/test_observability.py | 153 +++++++++++++++++++-----
 1 file changed, 125 insertions(+), 28 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index 2cdd5a3..38e87c7 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -33,7 +33,7 @@
 
 import copy
 import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
@@ -194,6 +194,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "065-llm-completion-event-active-prompt-null",
         "067-llm-completion-event-call-id-always-present-and-distinct",
         "068-llm-completion-event-response-model-distinct-from-request",
+        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
         "071-llm-failure-event-call-id-distinct-from-completion-event",
         "072-llm-failure-event-mutual-exclusion-with-completion-event",
         # proposal 0082: the LlmFailedEvent response-side surface on a
@@ -227,6 +228,8 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # Captured via a private MeterProvider + InMemoryMetricReader (the
         # §6.9 metric-capture primitive). 089 (embeddings) is deferred.
         "088-llm-metrics-token-and-duration",
+        "145-llm-metrics-no-input-token-observation-on-null-counter",
+        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "090-metrics-error-type-on-duration",
         "091-metrics-disabled-no-measurements",
         # proposal 0082: the OTel error span (124) renders the response-side
@@ -245,6 +248,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # NOT on a no-usage provider_unavailable failure. The Langfuse WARNING
         # (130) runs in the dedicated test_observability_langfuse harness.
         "126-token-budget-input-exceeded",
+        "146-token-budget-input-bound-not-evaluated-on-null-counter",
         "127-token-budget-total-exceeded",
         "128-token-budget-under-budget-no-warning",
         "129-token-budget-absent-unchanged",
@@ -288,6 +292,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # harvested error message ABSENT with error_type retained, via the
         # metadata_absent directive that landed in #267.
         "137-langfuse-embedding-failure-observation",
+        "150-langfuse-embedding-failure-literal-error-fields",
         "139-otel-embedding-no-usage-input-tokens-omitted",
         "140-langfuse-embedding-no-usage-usagedetails-omitted",
         # proposal 0067 §11 embedding metrics: token.usage (input only) +
@@ -315,6 +320,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "107-otel-rerank-span-attributes",
         "108-langfuse-rerank-observation",
         "138-langfuse-rerank-failure-observation",
+        "151-langfuse-rerank-failure-literal-error-fields",
         "141-otel-rerank-no-usage-attributes-omitted",
         "142-langfuse-rerank-no-usage-usagedetails-omitted",
         "109-rerank-metrics-token-and-duration",
@@ -417,39 +423,25 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # conformance fixture wiring rides the v0.17.0 fixture-wiring PR.
     # Proposal 0101 (spec v0.96.0) malformed usage counter -> not reported,
     # threaded through the span / metric / budget observability surfaces.
-    "144-otel-llm-span-omits-input-usage-on-null-counter": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "145-llm-metrics-no-input-token-observation-on-null-counter": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "146-token-budget-input-bound-not-evaluated-on-null-counter": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "147-otel-null-counter-reaches-no-span-histogram-or-budget": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "148-langfuse-generation-usage-omits-input-on-null-counter": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": (
-        "Proposal 0101 null-counter observability; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
     # Proposal 0107 (spec v0.102.0) mock_embedding / mock_rerank raises
     # sub-directive -> literal error-field assertion.
     # The 0118 half of these two resolved at this pin (both cases moved to
     # disable_provider_payload=false, since asserting the message literally is
     # their purpose). What remains is the 0107 mock-raises harness wiring.
-    "150-langfuse-embedding-failure-literal-error-fields": (
-        "Proposal 0107 mock_embedding raises sub-directive not yet wired; rides the "
-        "remaining v0.17.0 fixture-wiring PR (observability 144-156)"
-    ),
-    "151-langfuse-rerank-failure-literal-error-fields": (
-        "Proposal 0107 mock_rerank raises sub-directive not yet wired; rides the "
-        "remaining v0.17.0 fixture-wiring PR (observability 144-156)"
-    ),
     # Spec v0.103.1 conformance coverage (0084 orphan-fallback arms + the
     # embedding failure-metrics counterpart).
+    # 0101 null-counter arms whose SIBLINGS (145 / 146 / 147 / 149) now run.
+    # These two need a driver rather than wiring, which is why they did not ride
+    # with the rest of the cluster.
+    "144-otel-llm-span-omits-input-usage-on-null-counter": (
+        "asserts `expected.span_tree` on an LLM span, and every span_tree driver in this file is "
+        "fixture-specific (038 / 132); needs a generic LLM span-tree driver, not a dispatch entry"
+    ),
+    "148-langfuse-generation-usage-omits-input-on-null-counter": (
+        "asserts `expected.langfuse_trace` on an LLM generation, and this runner has no LLM "
+        "Langfuse driver: the langfuse_trace fixtures live in the sibling runner, which has no "
+        "mock_llm usage plumbing"
+    ),
     "152-otel-parallel-branch-orphan-llm-fallback": (
         "Spec v0.103.1 orphan-fallback coverage; harness wiring rides the v0.17.0 fixture-wiring PR"
     ),
@@ -748,10 +740,12 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "064-llm-completion-event-active-prompt-populated",
         "065-llm-completion-event-active-prompt-null",
         "068-llm-completion-event-response-model-distinct-from-request",
+        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
     }:
         await _run_typed_event_cases(spec)
     elif fixture_id in {
         "072-llm-failure-event-mutual-exclusion-with-completion-event",
+        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
         "120-llm-failure-event-structured-output-truncation",
         "121-llm-failure-event-structured-output-schema-mismatch",
         "122-llm-failure-event-response-side-null-on-non-body-failure",
@@ -786,6 +780,8 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         await _run_llm_payload_fixture(spec)
     elif fixture_id in {
         "088-llm-metrics-token-and-duration",
+        "145-llm-metrics-no-input-token-observation-on-null-counter",
+        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "090-metrics-error-type-on-duration",
         "091-metrics-disabled-no-measurements",
         "125-metrics-token-usage-on-structured-output-failure",
@@ -795,6 +791,7 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         await _run_structured_output_error_span_fixture(spec)
     elif fixture_id in {
         "126-token-budget-input-exceeded",
+        "146-token-budget-input-bound-not-evaluated-on-null-counter",
         "127-token-budget-total-exceeded",
         "128-token-budget-under-budget-no-warning",
         "129-token-budget-absent-unchanged",
@@ -827,6 +824,7 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "140-langfuse-embedding-no-usage-usagedetails-omitted",
         "089-embedding-metrics-token-and-duration",
         "143-embedding-metrics-no-usage-no-token-observation",
+        "150-langfuse-embedding-failure-literal-error-fields",
     }:
         await _run_embedding_fixture(spec)
     elif fixture_id in {
@@ -844,6 +842,7 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "141-otel-rerank-no-usage-attributes-omitted",
         "142-langfuse-rerank-no-usage-usagedetails-omitted",
         "109-rerank-metrics-token-and-duration",
+        "151-langfuse-rerank-failure-literal-error-fields",
     }:
         await _run_rerank_fixture(spec)
     elif fixture_id in {
@@ -4622,13 +4621,14 @@ async def _run_token_budget_case(case: Mapping[str, Any]) -> None:
     expected_metrics = cast("list[dict[str, Any]] | None", expected.get("metrics"))
     if expected_metrics is not None:
         _assert_metric_points(points, expected_metrics)
-    _assert_token_budget_invariants(case, points, collectors)
+    _assert_token_budget_invariants(case, points, collectors, spans)
 
 
 def _assert_token_budget_invariants(
     case: Mapping[str, Any],
     points: Sequence[tuple[str, float, int, dict[str, Any]]],
     collectors: Mapping[str, Any],
+    spans: Sequence[Any] = (),
 ) -> None:
     """Evaluate the token-budget named invariants (128/129/131) as absence
     predicates over the captured measurement set + the typed events.
@@ -4645,6 +4645,12 @@ def _assert_token_budget_invariants(
         "no_token_budget_instrument_observations_without_usage",
         "token_budget_populated_but_not_evaluated_without_usage",
         "exception_propagates_alongside_typed_event",
+        # Proposal 0101 null-counter arms (fixture 146).
+        "input_bound_not_evaluated_when_prompt_tokens_null",
+        "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable",
+        "no_input_token_usage_observation_when_prompt_tokens_null",
+        "total_bound_evaluated_when_total_tokens_sound",
+        "no_input_kind_budget_observation_when_prompt_tokens_null",
     }
     unknown = set(invariants) - _known_invariants
     assert not unknown, f"unhandled token-budget invariant(s): {sorted(unknown)}"
@@ -4694,6 +4700,50 @@ def _assert_token_budget_invariants(
         assert not exceeded_points and not utilization_points, (
             "a no-usage failure MUST NOT evaluate the budget (no token-budget metric observation)"
         )
+    # ---- proposal 0101: a null prompt_tokens makes the INPUT bound unevaluable
+    # while leaving a sound total bound fully evaluated. Absence predicates,
+    # because the `metrics:` list shape asserts presence only.
+    _BUDGET_KIND = "openarmature.gen_ai.token_budget.kind"
+    _TOKEN_TYPE = "openarmature.gen_ai.token.type"
+    input_budget_points = [
+        p for p in exceeded_points + utilization_points if p[3].get(_BUDGET_KIND) == "input"
+    ]
+
+    if invariants.get("input_bound_not_evaluated_when_prompt_tokens_null") or invariants.get(
+        "no_input_kind_budget_observation_when_prompt_tokens_null"
+    ):
+        assert not input_budget_points, (
+            "a null prompt_tokens leaves the INPUT bound unevaluable, so it MUST record no "
+            f"token-budget observation of kind=input; got {input_budget_points}"
+        )
+    if invariants.get("total_bound_evaluated_when_total_tokens_sound"):
+        # The independence half. Without it the case above is satisfied by an
+        # implementation that skipped budget evaluation entirely.
+        total_points = [p for p in utilization_points if p[3].get(_BUDGET_KIND) == "total"]
+        assert total_points, (
+            "a sound total_tokens MUST still evaluate the total bound; got no kind=total "
+            f"utilization observation among {utilization_points}"
+        )
+    if invariants.get("no_input_token_usage_observation_when_prompt_tokens_null"):
+        input_usage = [
+            p
+            for p in points
+            if p[0] == "openarmature.gen_ai.client.token.usage" and p[3].get(_TOKEN_TYPE) == "input"
+        ]
+        assert not input_usage, (
+            f"a null prompt_tokens MUST record no input token.usage observation; got {input_usage}"
+        )
+    if invariants.get("exceeded_span_signal_absent_not_false_when_only_bound_unevaluable"):
+        # ABSENT, not present-and-false. A `False` would assert "evaluated, not
+        # breached", which is a different and wrong claim when nothing was
+        # evaluable at all.
+        attr = "openarmature.llm.token_budget.exceeded"
+        carrying = [s for s in spans if attr in dict(s.attributes or {})]
+        assert not carrying, (
+            f"with no evaluable bound the {attr} span attribute MUST be absent rather than false; "
+            f"present on {[(s.name, dict(s.attributes or {}).get(attr)) for s in carrying]}"
+        )
+
     if invariants.get("exception_propagates_alongside_typed_event"):
         # The runner's pytest.raises already asserts the exception propagated;
         # assert the typed failure event fired alongside it (both, not either).
@@ -5195,6 +5245,49 @@ async def _run_tool_case(case: Mapping[str, Any]) -> None:
 # appears, mirroring the tool path (_run_tool_case).
 
 
+def _raise_mock_provider_error(
+    spec_resp: Mapping[str, Any],
+    classify: Callable[[Any], Any],
+) -> None:
+    """Honour a ``raises`` sub-directive on a mock_embedding / mock_rerank entry."""
+    # Proposal 0107 (conformance-adapter §5.15), the retrieval analogue of the
+    # tool path's `mock_tool: {raises: {...}}`. Fixtures 137 / 138 can only assert
+    # error_type / error_message by FORMAT, because both are impl-derived: the
+    # class name comes from whichever exception the provider's own classifier
+    # picked, and the message from the vendor body. `raises` supplies literal
+    # values so 150 / 151 can assert them exactly.
+    #
+    # The category must still come from `status`. So rather than restate the
+    # status-to-category table here, which would silently drift from src, run the
+    # REAL classifier on a probe response of that status and subclass whatever it
+    # returns. `error_category` on the event reads `exc.category`, which the
+    # subclass inherits, while `error_type` reads `type(exc).__name__` and
+    # `error_message` reads `str(exc)`, which the rename and the message supply.
+    #
+    # Raised from inside the mock transport, which works because the exception is
+    # an LlmProviderError rather than an httpx.HTTPError: the provider's
+    # `except httpx.HTTPError` around the post does not catch it, so it reaches
+    # the `except LlmProviderError` that builds the failure event.
+    import json
+
+    import httpx
+
+    raises = spec_resp.get("raises")
+    if not isinstance(raises, Mapping):
+        return
+    raises_map = cast("Mapping[str, Any]", raises)
+    status = int(spec_resp.get("status", 500))
+    probe = httpx.Response(
+        status,
+        content=json.dumps(spec_resp.get("body") or {}).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+    )
+    classified = cast("Exception", classify(probe))
+    error_type = cast("str", raises_map.get("error_type") or type(classified).__name__)
+    renamed: type[Exception] = type(error_type, (type(classified),), {})
+    raise renamed(cast("str", raises_map.get("message", "")))
+
+
 def _embedding_model_from_first_response(case: Mapping[str, Any]) -> str | None:
     """Return the ``model`` on the first ``mock_embedding`` response body, so
     the provider binds to the model the fixture's expected event reports."""
@@ -5260,6 +5353,7 @@ def _build_embedding_graph(
 
     from openarmature.graph import END, GraphBuilder
     from openarmature.retrieval import OpenAIEmbeddingProvider
+    from openarmature.retrieval.providers.openai import _classify_embedding_http_error
 
     from .adapter import build_state_cls
 
@@ -5274,6 +5368,7 @@ def _handler(_request: httpx.Request) -> httpx.Response:
         if not mock_responses:
             raise AssertionError("mock_embedding queue exhausted")
         spec_resp = mock_responses.pop(0)
+        _raise_mock_provider_error(spec_resp, _classify_embedding_http_error)
         body = cast("dict[str, Any]", spec_resp.get("body") or {})
         return httpx.Response(
             int(spec_resp.get("status", 200)),
@@ -5540,6 +5635,7 @@ def _build_rerank_graph(
 
     from openarmature.graph import END, GraphBuilder
     from openarmature.retrieval import CohereRerankProvider
+    from openarmature.retrieval.providers.cohere import _classify_cohere_http_error
 
     from .adapter import build_state_cls
 
@@ -5554,6 +5650,7 @@ def _handler(_request: httpx.Request) -> httpx.Response:
         if not mock_responses:
             raise AssertionError("mock_rerank queue exhausted")
         spec_resp = mock_responses.pop(0)
+        _raise_mock_provider_error(spec_resp, _classify_cohere_http_error)
         body = cast("dict[str, Any]", spec_resp.get("body") or {})
         return httpx.Response(
             int(spec_resp.get("status", 200)),

From 0512aedfadb014b84e5b410cdcd4b5f87c1ccdbe Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 12:58:44 -0700
Subject: [PATCH 02/13] Make the metrics fixtures' invariants actually assert

The metrics driver read expected.metrics but never expected.invariants,
and unlike the token-budget driver it had no guard for an invariant it
did not handle. Every invariant on every metrics fixture was silently
dropped.

That mattered most for 145 and 147, whose invariants are the whole
assertion: both state in their own comments that the metrics list shape
can express presence only, so the negative claims live nowhere else.
Both passed while an input token.usage observation was recorded for a
null prompt_tokens, which is exactly what they forbid.

Adding the guard surfaced nine more dead invariants on four fixtures
that predate this cluster. All sixteen are implemented, and the guard
now fails loudly on the next unhandled name.
---
 tests/conformance/test_observability.py | 127 ++++++++++++++++++++++++
 1 file changed, 127 insertions(+)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index 38e87c7..f40109e 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -4517,6 +4517,133 @@ async def _run_metrics_case(case: Mapping[str, Any]) -> None:
     points = _collect_metric_points(reader)
     expected_metrics = cast("list[dict[str, Any]]", case["expected"].get("metrics") or [])
     _assert_metric_points(points, expected_metrics)
+    _assert_metrics_invariants(case, points, exporter.get_finished_spans())
+
+
+_METRICS_INVARIANTS = {
+    # Proposal 0101 null-counter arms (fixtures 145 / 147).
+    "no_input_token_usage_observation_when_prompt_tokens_null",
+    "output_token_usage_observation_recorded_when_completion_sound",
+    "null_prompt_tokens_reaches_no_span_histogram_or_budget_surface",
+    "span_omits_both_input_usage_attributes",
+    "no_input_token_usage_observation",
+    "no_token_budget_instrument_observation_for_unevaluable_input_bound",
+    "exceeded_span_signal_absent_not_false",
+    # Pre-existing metrics fixtures (088 / 090 / 091 / 125). These were being
+    # silently dropped too: the driver read no invariants at all, so adding the
+    # guard above surfaced them.
+    "one_duration_observation_per_call",
+    "two_token_usage_observations_input_and_output",
+    "errored_call_records_duration_with_error_type",
+    "no_token_usage_observation_on_failure",
+    "no_measurements_recorded_when_metrics_off",
+    "duration_records_error_type_on_failure",
+    "token_usage_recorded_on_structured_output_failure",
+    # Documentary: states that the duration VALUE is deliberately not asserted
+    # (it is wall-clock). Recognized so it cannot read as unhandled, but there is
+    # nothing to check.
+    "duration_value_not_asserted",
+}
+
+
+def _assert_metrics_invariants(
+    case: Mapping[str, Any],
+    points: Sequence[tuple[str, float, int, dict[str, Any]]],
+    spans: Sequence[Any],
+) -> None:
+    """Evaluate the named invariants on a metrics fixture."""
+    # These are ABSENCE predicates, which the `metrics:` list shape cannot
+    # express -- it asserts presence only. 145 and 147 say so in their own
+    # comments, so for those two the invariants ARE the assertion: without this
+    # the fixtures pass while checking nothing but the positive rows. Mirrors the
+    # token-budget driver's guard, whose absence here is what let that happen.
+    invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
+    unknown = set(invariants) - _METRICS_INVARIANTS
+    assert not unknown, f"unhandled metrics invariant(s): {sorted(unknown)}"
+    if not invariants:
+        return
+
+    usage_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token.usage"]
+    input_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "input"]
+    output_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "output"]
+    budget_points = [
+        p
+        for p in points
+        if p[0]
+        in (
+            "openarmature.gen_ai.client.token_budget.exceeded",
+            "openarmature.gen_ai.client.token_budget.utilization",
+        )
+    ]
+    input_attrs = ("openarmature.gen_ai.usage.input_tokens", "gen_ai.usage.input_tokens")
+
+    def _spans_carrying(*names: str) -> list[tuple[str, str]]:
+        return [(s.name, n) for s in spans for n in names if n in dict(s.attributes or {})]
+
+    if invariants.get("no_input_token_usage_observation_when_prompt_tokens_null") or invariants.get(
+        "no_input_token_usage_observation"
+    ):
+        assert not input_usage, (
+            f"a null prompt_tokens MUST record no input token.usage observation; got {input_usage}"
+        )
+    if invariants.get("output_token_usage_observation_recorded_when_completion_sound"):
+        # The non-vacuity partner: without it the absence above is satisfied by
+        # an implementation that recorded no usage observations at all.
+        assert output_usage, (
+            f"a sound completion_tokens MUST still record an output token.usage "
+            f"observation; got only {usage_points}"
+        )
+    if invariants.get("span_omits_both_input_usage_attributes"):
+        carrying = _spans_carrying(*input_attrs)
+        assert not carrying, (
+            f"a null prompt_tokens MUST leave both input-usage span attributes absent; present as {carrying}"
+        )
+    if invariants.get("no_token_budget_instrument_observation_for_unevaluable_input_bound"):
+        input_budget = [
+            p for p in budget_points if p[3].get("openarmature.gen_ai.token_budget.kind") == "input"
+        ]
+        assert not input_budget, (
+            f"an unevaluable input bound MUST record no token-budget observation; got {input_budget}"
+        )
+    if invariants.get("exceeded_span_signal_absent_not_false"):
+        attr = "openarmature.llm.token_budget.exceeded"
+        carrying = _spans_carrying(attr)
+        assert not carrying, (
+            f"with no evaluable bound the {attr} span attribute MUST be absent rather than "
+            f"false; present on {carrying}"
+        )
+    duration_points = [p for p in points if p[0] == "openarmature.gen_ai.client.operation.duration"]
+
+    if invariants.get("one_duration_observation_per_call"):
+        assert len(duration_points) == 1, (
+            f"expected exactly one operation.duration observation per call; got {duration_points}"
+        )
+    if invariants.get("two_token_usage_observations_input_and_output"):
+        assert len(input_usage) == 1 and len(output_usage) == 1, (
+            f"expected one input and one output token.usage observation; got {usage_points}"
+        )
+    if invariants.get("errored_call_records_duration_with_error_type") or invariants.get(
+        "duration_records_error_type_on_failure"
+    ):
+        assert duration_points, "a failed call MUST still record operation.duration"
+        assert all(p[3].get("error.type") for p in duration_points), (
+            f"a failed call's duration observation MUST carry error.type; got {duration_points}"
+        )
+    if invariants.get("no_token_usage_observation_on_failure"):
+        assert not usage_points, (
+            f"a failure with no usage MUST record no token.usage observation; got {usage_points}"
+        )
+    if invariants.get("token_usage_recorded_on_structured_output_failure"):
+        # A structured-output failure DOES carry usage, so the counterpart above
+        # must not be over-applied to every failure.
+        assert usage_points, "a structured-output failure carries usage, so token.usage MUST still record"
+    if invariants.get("no_measurements_recorded_when_metrics_off"):
+        assert not points, f"metrics disabled MUST record no measurements at all; got {points}"
+    if invariants.get("null_prompt_tokens_reaches_no_span_histogram_or_budget_surface"):
+        # The umbrella claim: none of the three surfaces sees the null counter.
+        assert not input_usage, f"histogram surface: {input_usage}"
+        assert not _spans_carrying(*input_attrs), "span surface carries an input-usage attribute"
+        assert not budget_points, f"budget surface: {budget_points}"
 
 
 # ---------------------------------------------------------------------------

From b23c7c4c90019b88bc510ab3280fcd3b44bedf71 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 13:54:47 -0700
Subject: [PATCH 03/13] Guard against a case whose only assertion is its
 invariants

Invariant blocks in this corpus are documentary: they restate in prose
what a concrete directive already pins, and runners deliberately do not
implement most of them. That convention is only safe while something
concrete is present to do the pinning.

A case whose expected block carries nothing but invariants has no such
backing, so it asserts nothing while reading like coverage. The metrics
driver reached that state by another route and two fixtures passed while
checking nothing, which is what this guard exists to prevent recurring.

It matches no fixture today, so it ships with a test that injects a
synthetic case and confirms it fires on both runner paths, plus a
non-vacuity check that the corpus it polices is actually reachable.
---
 tests/conformance/harness/capabilities.py     | 25 +++++++++++
 .../test_capability_gate_wiring.py            | 42 +++++++++++++++++++
 tests/conformance/test_observability.py       |  8 ++++
 .../test_observability_langfuse.py            |  7 ++++
 4 files changed, 82 insertions(+)

diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py
index f18c4d4..3b31516 100644
--- a/tests/conformance/harness/capabilities.py
+++ b/tests/conformance/harness/capabilities.py
@@ -149,3 +149,28 @@ def assert_some_case_ran(
     "capability_skip_reason",
     "report_recognized_skip",
 ]
+
+
+def assert_case_asserts_something(fixture_stem: str, case_name: str, expected: Mapping[str, object]) -> None:
+    """Fail a case whose ``expected`` block carries nothing but ``invariants``."""
+    # Invariant blocks in this corpus are DOCUMENTARY: they restate in prose what
+    # a concrete directive (span_tree / langfuse_trace / metrics / event_count)
+    # already pins, and runners deliberately do not implement most of them. That
+    # convention is only safe while something concrete is present to do the
+    # pinning. A case whose expected block is invariants-ONLY has no such backing,
+    # so it asserts exactly nothing while reading like coverage.
+    #
+    # Currently matches no fixture in the corpus, which is why it is a guard
+    # rather than a fix. It exists because the metrics driver reached that state
+    # by a different route -- reading `metrics` but never `invariants`, on two
+    # fixtures whose invariants were their only negative assertion -- and both
+    # passed while checking nothing.
+    concrete = set(expected) - {"invariants"}
+    if concrete or not expected.get("invariants"):
+        return
+    raise AssertionError(
+        f"{fixture_stem}::{case_name}: `expected` declares only `invariants` and no concrete "
+        f"directive, so nothing is asserted unless the runner implements every one of them. "
+        f"Add a concrete assertion, or implement the invariants and register them in the "
+        f"driver's known-invariant set."
+    )
diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py
index 588bc88..6b153ad 100644
--- a/tests/conformance/test_capability_gate_wiring.py
+++ b/tests/conformance/test_capability_gate_wiring.py
@@ -320,3 +320,45 @@ def test_every_gated_fixture_belongs_to_a_runner_that_implements_the_gate() -> N
         f"runner implements the gate. Wire it into their runner "
         f"(tests/conformance/harness/capabilities.py) before those fixtures are activated."
     )
+
+
+# -- invariants-only guard --------------------------------------------------
+
+
+def test_invariants_only_expected_block_is_rejected() -> None:
+    from .harness.capabilities import assert_case_asserts_something
+
+    with pytest.raises(AssertionError, match="only `invariants`"):
+        assert_case_asserts_something(
+            "999-synthetic",
+            "case_with_no_concrete_assertion",
+            {"invariants": {"some_absence_claim": True}},
+        )
+
+
+@pytest.mark.parametrize(
+    "expected",
+    [
+        {"span_tree": [], "invariants": {"x": True}},
+        {"metrics": [], "invariants": {"x": True}},
+        {"langfuse_trace": {}, "invariants": {"x": True}},
+        {"invariants": {}},
+        {},
+    ],
+)
+def test_a_case_with_any_concrete_directive_is_accepted(expected: dict[str, object]) -> None:
+    from .harness.capabilities import assert_case_asserts_something
+
+    assert_case_asserts_something("999-synthetic", "case", expected)
+
+
+def test_the_guard_is_pointed_at_the_real_corpus() -> None:
+    # Non-vacuity for the guard itself. The rejecting test above proves the
+    # PREDICATE works; this proves the corpus it is meant to police is actually
+    # reachable and non-empty, because a guard verified only against synthetic
+    # input is how a sweep ends up globbing an empty directory and reporting a
+    # clean result.
+    from .test_observability import _fixture_paths
+
+    paths = list(_fixture_paths())
+    assert len(paths) > 100, f"expected the observability corpus, found {len(paths)} fixtures"
diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index f40109e..cb7de60 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -680,6 +680,14 @@ async def test_observability_fixture(fixture_path: Path) -> None:
 
     spec = _load(fixture_path)
     _reject_unsupported_capability_gate(fixture_id, spec)
+    from .harness.capabilities import assert_case_asserts_something  # noqa: PLC0415
+
+    for _case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+        assert_case_asserts_something(
+            fixture_id,
+            cast("str", _case.get("name") or "<unnamed>"),
+            cast("Mapping[str, Any]", _case.get("expected") or {}),
+        )
     if fixture_id == "001-otel-basic-trace":
         await _run_fixture_001(spec)
     elif fixture_id == "002-otel-subgraph-hierarchy":
diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py
index 8c3b248..bd4bddf 100644
--- a/tests/conformance/test_observability_langfuse.py
+++ b/tests/conformance/test_observability_langfuse.py
@@ -52,6 +52,7 @@
 
 from .adapter import build_graph, build_state_cls
 from .harness.capabilities import (
+    assert_case_asserts_something,
     assert_some_case_ran,
     capability_skip_reason,
     report_recognized_skip,
@@ -558,6 +559,12 @@ async def fetch(
 async def test_langfuse_fixture(fixture_path: Path) -> None:
     spec = _load(fixture_path)
     fixture_stem = fixture_path.stem
+    for _case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+        assert_case_asserts_something(
+            fixture_stem,
+            cast("str", _case.get("name") or "<unnamed>"),
+            cast("Mapping[str, Any]", _case.get("expected") or {}),
+        )
     if fixture_stem in _ISOLATION_FIXTURES:
         cases_157 = cast("list[dict[str, Any]]", spec["cases"])
         excluded_157: dict[str, str] = {}

From 0cedfaeeffa6fb37d50fa1278d734aa3d834ea99 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 14:41:23 -0700
Subject: [PATCH 04/13] Wire fixture 154 and diagnose the remaining deferrals

154 asserts the failure-path counterpart of the embedding metrics
fixtures: the duration observation survives a failed embed and carries
the error category, while no token usage is recorded because no response
arrived. Its two invariants are implemented and mutation-verified.

Binding the provider needed a fallback. The harness reads the model off
the first mock response body, and a 503 has no body, so a failure-path
fixture bound to the harness default and every dimension match failed on
a model name the fixture never mentions. It now falls back to the model
the expected metric dimensions report.

119, 152 and 153 stay deferred, but with reasons diagnosed rather than
guessed: each reaches its sibling's driver and fails on a concrete gap.
119 drives a branch that raises under retry, which that driver has no
path for; 152 and 153 need a subgraphs block the orphan-fallback driver
does not build.
---
 tests/conformance/test_observability.py | 72 ++++++++++++++++++-------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index cb7de60..9ece49e 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -299,6 +299,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # operation.duration, operation="embeddings", recorded from the terminal
         # embedding event. 089 (usage) + 143 (no-usage: duration only).
         "089-embedding-metrics-token-and-duration",
+        "154-embedding-metrics-on-failure",
         "143-embedding-metrics-no-usage-no-token-observation",
         # v0.16.0 — proposal 0060 rerank observability (0060b). A calls_rerank
         # node awaits CohereRerankProvider.rerank() inside the node body; the
@@ -386,10 +387,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # attempt_index-under-node-retry coverage round-out -- mixes a graph-style
     # shape the cross-capability parser doesn't model (cf. 110) and has no
     # dedicated runner here yet; defer until the harness wires it.
-    "119-otel-callable-branch-attempt-index-under-node-retry": (
-        "0075 shipped v0.15.0; runner for this callable-branch-attempt-index-under-retry "
-        "case not yet wired -- harness gap, not unimplemented"
-    ),
     # Proposal 0082 (structured-output failure diagnostics, spec v0.77.0).
     # The event surface (120-122), the OTel error-span rendering (124) and the
     # §11 token-usage metric (125) run here; the Langfuse failed-Generation
@@ -433,6 +430,19 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # 0101 null-counter arms whose SIBLINGS (145 / 146 / 147 / 149) now run.
     # These two need a driver rather than wiring, which is why they did not ride
     # with the rest of the cluster.
+    # Diagnosed against their sibling drivers rather than guessed: each reaches a
+    # driver and fails on a concrete gap, not on "no driver".
+    "119-otel-callable-branch-attempt-index-under-node-retry": (
+        "reuses fixture 110's callable-branch driver, but drives a branch that RAISES under node "
+        "retry; that driver has no failure/retry path, so the branch exception escapes"
+    ),
+    "152-otel-parallel-branch-orphan-llm-fallback": (
+        "reuses fixture 133's orphan-fallback driver, which does not build the `subgraphs` block "
+        "152 adds (KeyError: 'subgraphs')"
+    ),
+    "153-otel-mixed-nesting-orphan-llm-fallback": (
+        "same driver gap as 152, one nesting level deeper (KeyError: 'leaf_sg')"
+    ),
     "144-otel-llm-span-omits-input-usage-on-null-counter": (
         "asserts `expected.span_tree` on an LLM span, and every span_tree driver in this file is "
         "fixture-specific (038 / 132); needs a generic LLM span-tree driver, not a dispatch entry"
@@ -442,15 +452,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "Langfuse driver: the langfuse_trace fixtures live in the sibling runner, which has no "
         "mock_llm usage plumbing"
     ),
-    "152-otel-parallel-branch-orphan-llm-fallback": (
-        "Spec v0.103.1 orphan-fallback coverage; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "153-otel-mixed-nesting-orphan-llm-fallback": (
-        "Spec v0.103.1 orphan-fallback coverage; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "154-embedding-metrics-on-failure": (
-        "Spec v0.103.1 embedding failure-metrics coverage; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
     # Proposal 0109 (spec v0.104.0) token-budget failure-path parity.
     "155-langfuse-token-budget-exceeded-flag-on-failure": (
         "Proposal 0109 token_budget_exceeded flag; harness wiring rides the v0.17.0 fixture-wiring PR"
@@ -714,11 +715,15 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         await _run_fixture_028(spec)
     elif fixture_id == "038-otel-parallel-branches-dispatch-span":
         await _run_fixture_038(spec)
-    elif fixture_id == "110-otel-callable-branch-span":
+    elif fixture_id in {
+        "110-otel-callable-branch-span",
+    }:
         await _run_fixture_110(spec)
     elif fixture_id == "132-otel-nested-fan-out-span-keying-and-llm-exact-match":
         await _run_fixture_132(spec)
-    elif fixture_id == "133-otel-nested-fan-out-orphan-llm-fallback":
+    elif fixture_id in {
+        "133-otel-nested-fan-out-orphan-llm-fallback",
+    }:
         await _run_fixture_133(spec)
     elif fixture_id in {
         "040-llm-cache-attribute-emission",
@@ -831,6 +836,7 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "139-otel-embedding-no-usage-input-tokens-omitted",
         "140-langfuse-embedding-no-usage-usagedetails-omitted",
         "089-embedding-metrics-token-and-duration",
+        "154-embedding-metrics-on-failure",
         "143-embedding-metrics-no-usage-no-token-observation",
         "150-langfuse-embedding-failure-literal-error-fields",
     }:
@@ -5427,11 +5433,21 @@ def _embedding_model_from_first_response(case: Mapping[str, Any]) -> str | None:
     """Return the ``model`` on the first ``mock_embedding`` response body, so
     the provider binds to the model the fixture's expected event reports."""
     responses = cast("list[dict[str, Any]] | None", case.get("mock_embedding")) or []
-    if not responses:
-        return None
-    body = cast("dict[str, Any] | None", responses[0].get("body")) or {}
-    model = body.get("model")
-    return model if isinstance(model, str) else None
+    body = cast("dict[str, Any] | None", responses[0].get("body")) if responses else None
+    model = (body or {}).get("model")
+    if isinstance(model, str):
+        return model
+    # A failure-path fixture (154) has no response body to carry a model, so bind
+    # to the one its expected metric dimensions report instead. Without this the
+    # provider binds to the harness default and every dimension match fails on a
+    # model name the fixture never mentions.
+    expected = cast("dict[str, Any]", case.get("expected") or {})
+    for point in cast("list[dict[str, Any]]", expected.get("metrics") or []):
+        dims = cast("dict[str, Any]", point.get("dimensions") or {})
+        declared = dims.get("gen_ai.request.model")
+        if isinstance(declared, str):
+            return declared
+    return None
 
 
 def _build_embedding_runtime_config(config_spec: Mapping[str, Any] | None) -> Any:
@@ -5668,6 +5684,9 @@ def _assert_embedding_metrics_invariants(
         "duration_recorded_when_usage_absent",
         "embedding_records_input_token_only",
         "no_output_token_observation_for_embedding",
+        # 154: the failure-path counterpart of 089 / 143.
+        "errored_embedding_records_duration_with_error_type",
+        "no_token_usage_observation_on_failure",
     }
     unknown = set(invariants) - _known
     assert not unknown, f"unhandled embedding-metrics invariant(s): {sorted(unknown)}"
@@ -5690,6 +5709,19 @@ def _assert_embedding_metrics_invariants(
         assert not output_token_points, (
             f"embedding MUST record no output token observation; got {output_token_points}"
         )
+    if invariants.get("errored_embedding_records_duration_with_error_type"):
+        # The §11 baseline survives the failure, carrying the §7 category. Both
+        # halves matter: a duration that records without error.type would lose
+        # the only dimension distinguishing a failed call from a slow one.
+        assert duration_points, "a failed embedding MUST still record operation.duration"
+        assert all(p[3].get("error.type") for p in duration_points), (
+            f"a failed embedding's duration MUST carry error.type; got {duration_points}"
+        )
+    if invariants.get("no_token_usage_observation_on_failure"):
+        assert not token_points, (
+            f"a failed embedding received no response, so it MUST record no token.usage "
+            f"observation; got {token_points}"
+        )
 
 
 # A calls_rerank node constructs a CohereRerankProvider on an httpx.MockTransport

From 2a36be851fd9b295716e92fb2596f60f7d4f4b73 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 14:50:20 -0700
Subject: [PATCH 05/13] Route the token-budget flag fixtures to the Langfuse
 runner

155 and 156 are near-clones of 130, which already runs in the sibling
Langfuse runner, so they belong there rather than needing a driver in
the generic one. Both now execute and the generic runner records them as
relocated rather than unaccounted.

Verified in both directions, since a one-way mutation proves nothing for
a pair asserting opposite outcomes: hardcoding the exceeded flag false
fails 155, hardcoding it true fails 156.
---
 tests/conformance/test_observability.py          | 8 ++------
 tests/conformance/test_observability_langfuse.py | 2 ++
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index 9ece49e..f17c64b 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -453,12 +453,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "mock_llm usage plumbing"
     ),
     # Proposal 0109 (spec v0.104.0) token-budget failure-path parity.
-    "155-langfuse-token-budget-exceeded-flag-on-failure": (
-        "Proposal 0109 token_budget_exceeded flag; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
-    "156-langfuse-token-budget-under-budget-flag-false": (
-        "Proposal 0109 token_budget_exceeded flag; harness wiring rides the v0.17.0 fixture-wiring PR"
-    ),
 }
 
 
@@ -484,6 +478,8 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "036-caller-invocation-id-non-uuid",
         "037-langfuse-trace-input-output",
         "059-implementation-attribution-langfuse",
+        "155-langfuse-token-budget-exceeded-flag-on-failure",
+        "156-langfuse-token-budget-under-budget-flag-false",
         # 134 -- proposal 0084 Langfuse parent resolution (nested exact-match +
         # orphan fallback). Driven by a dedicated hand-built runner in the
         # sibling harness; the generic topology path cannot model the
diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py
index bd4bddf..1e17863 100644
--- a/tests/conformance/test_observability_langfuse.py
+++ b/tests/conformance/test_observability_langfuse.py
@@ -159,6 +159,8 @@
         # to metadata.token_budget.*. Drives the input-exceeded path via
         # renders_prompt + the prompt's token_budget block.
         "130-langfuse-token-budget-warning-level",
+        "155-langfuse-token-budget-exceeded-flag-on-failure",
+        "156-langfuse-token-budget-under-budget-flag-false",
         # 134 (proposal 0084): the Langfuse Generation parent resolves by the same
         # chain-aware §5.5 rule as the OTel span parent -- both the nested
         # exact-match (case 1, mirrors OTel 132) and the orphan fallback (case 2,

From 30b1a93da959d8a63d4a192494f9b811dca26db9 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 16:22:13 -0700
Subject: [PATCH 06/13] Stop three fixtures being wired into drivers that
 ignore them

The adversarial review found this branch reproducing the defect it was
opened to fix. 147 and 149 declared span_tree blocks their drivers never
read: corrupting every span name in either left both green. With 144
deferred, that left the whole 0101 span surface with no live coverage
while the branch reported nine fixtures wired.

144's deferral reason was also false. It claimed no generic LLM
span-tree driver existed; _run_llm_payload_fixture is exactly that, and
144 passes through it unmodified. A reader trusting that reason would
have built a driver that already exists.

144 and 149 now run through that driver, and 147 through the
token-budget one, which reads observers and span_tree alongside metrics.
The two invariant registries validate against their union so a fixture
may declare names from both families while each guard implements only
its own subset.

The absence predicates gain a positive anchor. Every span claim is of
the form "X is absent", which an empty span set satisfies trivially:
renaming the LLM span made 147 green.

The structural fix is a guard asserting each fixture's driver reads
every expected key it declares, with the fixture-to-driver map parsed
from the dispatch chain rather than hand-maintained. It caught a real
error in this very commit, where a move landed in the wrong list.
---
 tests/conformance/test_observability.py | 148 +++++++++++++++++++-----
 1 file changed, 120 insertions(+), 28 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index f17c64b..bc01543 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -32,6 +32,7 @@
 from __future__ import annotations
 
 import copy
+import functools
 import re
 from collections.abc import Callable, Mapping, Sequence
 from pathlib import Path
@@ -194,6 +195,7 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "065-llm-completion-event-active-prompt-null",
         "067-llm-completion-event-call-id-always-present-and-distinct",
         "068-llm-completion-event-response-model-distinct-from-request",
+        "144-otel-llm-span-omits-input-usage-on-null-counter",
         "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
         "071-llm-failure-event-call-id-distinct-from-completion-event",
         "072-llm-failure-event-mutual-exclusion-with-completion-event",
@@ -229,7 +231,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # §6.9 metric-capture primitive). 089 (embeddings) is deferred.
         "088-llm-metrics-token-and-duration",
         "145-llm-metrics-no-input-token-observation-on-null-counter",
-        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "090-metrics-error-type-on-duration",
         "091-metrics-disabled-no-measurements",
         # proposal 0082: the OTel error span (124) renders the response-side
@@ -249,6 +250,9 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         # (130) runs in the dedicated test_observability_langfuse harness.
         "126-token-budget-input-exceeded",
         "146-token-budget-input-bound-not-evaluated-on-null-counter",
+        # Declares observers + span_tree as well as metrics; the metrics driver
+        # reads neither, so its span claims were dropped entirely.
+        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "127-token-budget-total-exceeded",
         "128-token-budget-under-budget-no-warning",
         "129-token-budget-absent-unchanged",
@@ -427,9 +431,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # their purpose). What remains is the 0107 mock-raises harness wiring.
     # Spec v0.103.1 conformance coverage (0084 orphan-fallback arms + the
     # embedding failure-metrics counterpart).
-    # 0101 null-counter arms whose SIBLINGS (145 / 146 / 147 / 149) now run.
-    # These two need a driver rather than wiring, which is why they did not ride
-    # with the rest of the cluster.
     # Diagnosed against their sibling drivers rather than guessed: each reaches a
     # driver and fails on a concrete gap, not on "no driver".
     "119-otel-callable-branch-attempt-index-under-node-retry": (
@@ -443,10 +444,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     "153-otel-mixed-nesting-orphan-llm-fallback": (
         "same driver gap as 152, one nesting level deeper (KeyError: 'leaf_sg')"
     ),
-    "144-otel-llm-span-omits-input-usage-on-null-counter": (
-        "asserts `expected.span_tree` on an LLM span, and every span_tree driver in this file is "
-        "fixture-specific (038 / 132); needs a generic LLM span-tree driver, not a dispatch entry"
-    ),
     "148-langfuse-generation-usage-omits-input-on-null-counter": (
         "asserts `expected.langfuse_trace` on an LLM generation, and this runner has no LLM "
         "Langfuse driver: the langfuse_trace fixtures live in the sibling runner, which has no "
@@ -657,6 +654,73 @@ def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any]
         )
 
 
+# Which `expected` keys each OTel driver actually READS. Derived from the driver
+# bodies, not from fixture usage: this exists because 147 and 149 were wired into
+# drivers that never look at `span_tree`, so their span claims were dropped while
+# both counted as covered. A fixture routed to a driver that ignores one of its
+# directives now fails at wiring time instead of passing hollow.
+_DRIVER_EXPECTED_KEYS: dict[str, frozenset[str]] = {
+    "_run_typed_event_cases": frozenset({"observers", "invariants", "node_completed_event_carries_error"}),
+    "_run_metrics_fixture": frozenset({"metrics", "invariants"}),
+    "_run_embedding_fixture": frozenset(
+        {"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}
+    ),
+    "_run_rerank_fixture": frozenset({"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}),
+    "_run_token_budget_fixture": frozenset({"span_tree", "metrics", "invariants", "observers"}),
+    "_run_llm_payload_fixture": frozenset({"span_tree", "invariants", "observers"}),
+}
+
+
+@functools.cache
+def _driver_for_fixture() -> dict[str, str]:
+    """Map each fixture id to the driver the dispatch chain sends it to."""
+    # Parsed from THIS file's own dispatch chain rather than hand-maintained, so
+    # the map cannot drift from what actually runs. A hand-written copy is the
+    # allowlist-derived-from-usage mistake one level up.
+    import ast  # noqa: PLC0415
+    import inspect  # noqa: PLC0415
+
+    tree = ast.parse(inspect.getsource(test_observability_fixture))
+    out: dict[str, str] = {}
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.If):
+            continue
+        called = [
+            c.func.id
+            for c in ast.walk(ast.Module(body=node.body, type_ignores=[]))
+            if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id.startswith("_run_")
+        ]
+        if not called:
+            continue
+        ids: list[str] = []
+        for cmp_node in ast.walk(node.test):
+            if isinstance(cmp_node, ast.Constant) and isinstance(cmp_node.value, str):
+                ids.append(cmp_node.value)
+        for fid in ids:
+            out.setdefault(fid, called[0])
+    return out
+
+
+def _assert_driver_reads_every_expected_key(fixture_id: str, spec: Mapping[str, Any]) -> None:
+    """Fail when a fixture declares an `expected` key its driver does not read."""
+    # Resolved lazily: the dispatch function it parses is defined below this point.
+    driver = _driver_for_fixture().get(fixture_id)
+    if driver is None:
+        return
+    handled = _DRIVER_EXPECTED_KEYS.get(driver)
+    if handled is None:
+        return
+    for case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+        declared = set(cast("dict[str, Any]", case.get("expected") or {}))
+        unread = sorted(declared - handled)
+        assert not unread, (
+            f"{fixture_id}::{case.get('name')}: declares expected key(s) {unread} that "
+            f"{driver} does not read, so those assertions would be silently dropped while the "
+            f"fixture still counts as covered. Route it to a driver that reads them, or "
+            f"implement them and register the key in _DRIVER_EXPECTED_KEYS."
+        )
+
+
 @pytest.mark.parametrize("fixture_path", _fixture_paths(), ids=_fixture_id)
 async def test_observability_fixture(fixture_path: Path) -> None:
     fixture_id = fixture_path.stem
@@ -679,6 +743,8 @@ async def test_observability_fixture(fixture_path: Path) -> None:
     _reject_unsupported_capability_gate(fixture_id, spec)
     from .harness.capabilities import assert_case_asserts_something  # noqa: PLC0415
 
+    _assert_driver_reads_every_expected_key(fixture_id, spec)
+
     for _case in cast("list[dict[str, Any]]", spec.get("cases") or []):
         assert_case_asserts_something(
             fixture_id,
@@ -749,12 +815,10 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "064-llm-completion-event-active-prompt-populated",
         "065-llm-completion-event-active-prompt-null",
         "068-llm-completion-event-response-model-distinct-from-request",
-        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
     }:
         await _run_typed_event_cases(spec)
     elif fixture_id in {
         "072-llm-failure-event-mutual-exclusion-with-completion-event",
-        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
         "120-llm-failure-event-structured-output-truncation",
         "121-llm-failure-event-structured-output-schema-mismatch",
         "122-llm-failure-event-response-side-null-on-non-body-failure",
@@ -785,12 +849,16 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         "085-llm-tool-call-request-attributes",
         "086-llm-tool-call-request-absent",
         "087-llm-tool-call-request-survives-payload-gating",
+        # 0101 null-counter span claims. This driver walks expected.span_tree
+        # including attributes_absent -- the half 149 silently dropped under the
+        # typed-event driver, and the driver 144 was wrongly deferred for lacking.
+        "144-otel-llm-span-omits-input-usage-on-null-counter",
+        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
     }:
         await _run_llm_payload_fixture(spec)
     elif fixture_id in {
         "088-llm-metrics-token-and-duration",
         "145-llm-metrics-no-input-token-observation-on-null-counter",
-        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "090-metrics-error-type-on-duration",
         "091-metrics-disabled-no-measurements",
         "125-metrics-token-usage-on-structured-output-failure",
@@ -801,6 +869,9 @@ async def test_observability_fixture(fixture_path: Path) -> None:
     elif fixture_id in {
         "126-token-budget-input-exceeded",
         "146-token-budget-input-bound-not-evaluated-on-null-counter",
+        # Declares observers + span_tree as well as metrics; the metrics driver
+        # reads neither, so its span claims were dropped entirely.
+        "147-otel-null-counter-reaches-no-span-histogram-or-budget",
         "127-token-budget-total-exceeded",
         "128-token-budget-under-budget-no-warning",
         "129-token-budget-absent-unchanged",
@@ -4530,6 +4601,25 @@ async def _run_metrics_case(case: Mapping[str, Any]) -> None:
     _assert_metrics_invariants(case, points, exporter.get_finished_spans())
 
 
+# Names only the token-budget driver implements. Kept beside the metrics set so
+# each guard can validate against the UNION while implementing only its own
+# subset: fixture 147 legitimately declares names from both families.
+_TOKEN_BUDGET_ONLY_INVARIANTS = {
+    "no_token_budget_exceeded_observation_when_under_budget",
+    "utilization_records_under_budget",
+    "no_token_budget_instrument_observations_when_no_budget",
+    "token_budget_null_on_completion_event",
+    "no_token_budget_instrument_observations_without_usage",
+    "token_budget_populated_but_not_evaluated_without_usage",
+    "exception_propagates_alongside_typed_event",
+    # Proposal 0101 null-counter arms (fixture 146).
+    "input_bound_not_evaluated_when_prompt_tokens_null",
+    "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable",
+    "no_input_token_usage_observation_when_prompt_tokens_null",
+    "total_bound_evaluated_when_total_tokens_sound",
+    "no_input_kind_budget_observation_when_prompt_tokens_null",
+}
+
 _METRICS_INVARIANTS = {
     # Proposal 0101 null-counter arms (fixtures 145 / 147).
     "no_input_token_usage_observation_when_prompt_tokens_null",
@@ -4568,11 +4658,22 @@ def _assert_metrics_invariants(
     # the fixtures pass while checking nothing but the positive rows. Mirrors the
     # token-budget driver's guard, whose absence here is what let that happen.
     invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
-    unknown = set(invariants) - _METRICS_INVARIANTS
+    unknown = set(invariants) - _METRICS_INVARIANTS - _TOKEN_BUDGET_ONLY_INVARIANTS
     assert not unknown, f"unhandled metrics invariant(s): {sorted(unknown)}"
     if not invariants:
         return
 
+    # Positive anchor BEFORE any absence predicate. Every claim below is of the
+    # form "X is absent", which an empty span set satisfies trivially -- deleting
+    # the LLM span outright made 147 green. Assert the span exists first, so the
+    # absences are read off a run that actually happened.
+    if spans and any(k.startswith(("span_", "null_prompt_tokens_reaches")) for k in invariants):
+        llm_spans = [s for s in spans if s.name == "openarmature.llm.complete"]
+        assert llm_spans, (
+            f"no openarmature.llm.complete span was recorded, so the span-absence invariants "
+            f"below would pass vacuously; got {[s.name for s in spans]}"
+        )
+
     usage_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token.usage"]
     input_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "input"]
     output_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "output"]
@@ -4759,6 +4860,7 @@ async def _run_token_budget_case(case: Mapping[str, Any]) -> None:
     if expected_metrics is not None:
         _assert_metric_points(points, expected_metrics)
     _assert_token_budget_invariants(case, points, collectors, spans)
+    _assert_metrics_invariants(case, points, spans)
 
 
 def _assert_token_budget_invariants(
@@ -4774,22 +4876,12 @@ def _assert_token_budget_invariants(
     invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
     # Fail loudly on a declared invariant this runner does not map, so a future
     # fixture's new invariant name cannot pass vacuously (harness-fidelity).
-    _known_invariants = {
-        "no_token_budget_exceeded_observation_when_under_budget",
-        "utilization_records_under_budget",
-        "no_token_budget_instrument_observations_when_no_budget",
-        "token_budget_null_on_completion_event",
-        "no_token_budget_instrument_observations_without_usage",
-        "token_budget_populated_but_not_evaluated_without_usage",
-        "exception_propagates_alongside_typed_event",
-        # Proposal 0101 null-counter arms (fixture 146).
-        "input_bound_not_evaluated_when_prompt_tokens_null",
-        "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable",
-        "no_input_token_usage_observation_when_prompt_tokens_null",
-        "total_bound_evaluated_when_total_tokens_sound",
-        "no_input_kind_budget_observation_when_prompt_tokens_null",
-    }
-    unknown = set(invariants) - _known_invariants
+    #
+    # Validated against the UNION with the metrics family, because a fixture may
+    # legitimately declare names from both (147 does). Each guard still implements
+    # only its own subset; the union governs what is RECOGNISED, not what is
+    # checked, and `_run_token_budget_case` calls both.
+    unknown = set(invariants) - _METRICS_INVARIANTS - _TOKEN_BUDGET_ONLY_INVARIANTS
     assert not unknown, f"unhandled token-budget invariant(s): {sorted(unknown)}"
     exceeded_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.exceeded"]
     utilization_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.utilization"]

From 7573bb6ad640b148bc3a40d991c615dd3e250b66 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 17:11:02 -0700
Subject: [PATCH 07/13] Fix the review's remaining findings

Two deferral reasons were wrong in the same way as 144's. 148's blamed
missing mock_llm plumbing in the Langfuse runner, which drives three such
fixtures already; the real gap is that its usage comparator subset-matches
expected keys, so an omitted input key cannot be expressed at all. 119's
blamed the callable-branch driver, but the adapter never translates a
node's YAML middleware block, so the retry is never installed and the
suggested fix would not have helped.

The invariants-only guard skipped every single-case fixture, because
those carry expected at the top level and it iterated `cases or []`. Its
non-vacuity test was itself vacuous: it counted fixtures on disk and
passed identically with both call sites deleted. It now grafts a
synthetic case onto a real fixture and pushes it through each runner's
entry point, and fails when either call site is removed.

The duration and token-usage count invariants asserted on aggregated
histogram data points rather than observation counts, so the
double-record regression they are named for could not fail them.

156's no_warning_level_under_budget is implemented because it is
unbacked: 155 pins its level concretely so its invariants are
documentary, but 156's expected block declares no level. A blanket guard
here would be wrong, since it would demand implementation of the
documentary majority.
---
 tests/conformance/harness/capabilities.py     |  6 ++-
 .../test_capability_gate_wiring.py            | 48 +++++++++++++++----
 tests/conformance/test_observability.py       | 33 +++++++++----
 .../test_observability_langfuse.py            | 15 +++++-
 4 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/tests/conformance/harness/capabilities.py b/tests/conformance/harness/capabilities.py
index 3b31516..f98f608 100644
--- a/tests/conformance/harness/capabilities.py
+++ b/tests/conformance/harness/capabilities.py
@@ -160,8 +160,10 @@ def assert_case_asserts_something(fixture_stem: str, case_name: str, expected: M
     # pinning. A case whose expected block is invariants-ONLY has no such backing,
     # so it asserts exactly nothing while reading like coverage.
     #
-    # Currently matches no fixture in the corpus, which is why it is a guard
-    # rather than a fix. It exists because the metrics driver reached that state
+    # Matches no ACTIVATED fixture. That is weaker than "no fixture": an
+    # invariants-only case does exist in the corpus and escapes only because it
+    # is skipped upstream, which makes this guard's placement relative to the
+    # skip chain load-bearing rather than incidental. It exists because the metrics driver reached that state
     # by a different route -- reading `metrics` but never `invariants`, on two
     # fixtures whose invariants were their only negative assertion -- and both
     # passed while checking nothing.
diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py
index 6b153ad..9b6bd02 100644
--- a/tests/conformance/test_capability_gate_wiring.py
+++ b/tests/conformance/test_capability_gate_wiring.py
@@ -352,13 +352,41 @@ def test_a_case_with_any_concrete_directive_is_accepted(expected: dict[str, obje
     assert_case_asserts_something("999-synthetic", "case", expected)
 
 
-def test_the_guard_is_pointed_at_the_real_corpus() -> None:
-    # Non-vacuity for the guard itself. The rejecting test above proves the
-    # PREDICATE works; this proves the corpus it is meant to police is actually
-    # reachable and non-empty, because a guard verified only against synthetic
-    # input is how a sweep ends up globbing an empty directory and reporting a
-    # clean result.
-    from .test_observability import _fixture_paths
-
-    paths = list(_fixture_paths())
-    assert len(paths) > 100, f"expected the observability corpus, found {len(paths)} fixtures"
+@pytest.mark.parametrize(
+    ("module_name", "runner_attr", "fixture_stem"),
+    [
+        # One per dispatch path, and one SINGLE-CASE fixture (022) because those
+        # put `expected` at the top level with no `cases` list -- the shape the
+        # first version of this wiring skipped entirely.
+        ("tests.conformance.test_observability", "test_observability_fixture", "001-otel-basic-trace"),
+        (
+            "tests.conformance.test_observability_langfuse",
+            "test_langfuse_fixture",
+            "022-langfuse-basic-trace",
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_the_guard_is_wired_into_each_runner(
+    monkeypatch: pytest.MonkeyPatch, module_name: str, runner_attr: str, fixture_stem: str
+) -> None:
+    # Non-vacuity for the WIRING, not the predicate. The rejecting test above
+    # already proves the predicate; this one failed to prove anything, because it
+    # only counted fixtures on disk and passed identically with both call sites
+    # deleted. Graft an invariants-only expected block onto a real fixture and
+    # push it through each runner's actual entry point.
+    import importlib
+
+    module = importlib.import_module(module_name)
+    path = next(p for p in module._fixture_paths() if p.stem == fixture_stem)  # noqa: SLF001
+    original_load = module._load  # noqa: SLF001
+
+    def _grafted(p: Any) -> Any:
+        spec = original_load(p)
+        target = spec["cases"][0] if spec.get("cases") else spec
+        target["expected"] = {"invariants": {"synthetic_unbacked_claim": True}}
+        return spec
+
+    monkeypatch.setattr(module, "_load", _grafted)
+    with pytest.raises(AssertionError, match="only `invariants`"):
+        await getattr(module, runner_attr)(path)
diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index bc01543..6b16f2d 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -434,8 +434,11 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # Diagnosed against their sibling drivers rather than guessed: each reaches a
     # driver and fails on a concrete gap, not on "no driver".
     "119-otel-callable-branch-attempt-index-under-node-retry": (
-        "reuses fixture 110's callable-branch driver, but drives a branch that RAISES under node "
-        "retry; that driver has no failure/retry path, so the branch exception escapes"
+        "the conformance adapter never translates a fixture node's YAML `middleware:` block, so "
+        "the retry is never installed and the transient failure is terminal: only attempt 0 runs "
+        "and the branch exception escapes. The fix is YAML middleware parsing in adapter.py plus "
+        "threading per_node_mw into add_parallel_branches_node, NOT a failure path in the 110 "
+        "driver -- that driver is reached and would still see an unretried raise"
     ),
     "152-otel-parallel-branch-orphan-llm-fallback": (
         "reuses fixture 133's orphan-fallback driver, which does not build the `subgraphs` block "
@@ -445,9 +448,11 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "same driver gap as 152, one nesting level deeper (KeyError: 'leaf_sg')"
     ),
     "148-langfuse-generation-usage-omits-input-on-null-counter": (
-        "asserts `expected.langfuse_trace` on an LLM generation, and this runner has no LLM "
-        "Langfuse driver: the langfuse_trace fixtures live in the sibling runner, which has no "
-        "mock_llm usage plumbing"
+        "the sibling Langfuse runner DOES drive mock_llm generations asserting usage (023, 155, "
+        "156), so plumbing is not the gap. The gap is its `usage` comparator: it iterates the "
+        "EXPECTED keys, a subset match, so an OMITTED `input` key -- the whole claim -- cannot "
+        "be expressed. Wiring it needs an exact-map or usage_absent comparator first, or the "
+        "fixture passes against an impl emitting usage.input = null"
     ),
     # Proposal 0109 (spec v0.104.0) token-budget failure-path parity.
 }
@@ -745,7 +750,9 @@ async def test_observability_fixture(fixture_path: Path) -> None:
 
     _assert_driver_reads_every_expected_key(fixture_id, spec)
 
-    for _case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+    # `or [spec]` because a single-case fixture puts `expected` at the top
+    # level with no `cases` list; `or []` skipped seven of them entirely.
+    for _case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]):
         assert_case_asserts_something(
             fixture_id,
             cast("str", _case.get("name") or "<unnamed>"),
@@ -4726,13 +4733,19 @@ def _spans_carrying(*names: str) -> list[tuple[str, str]]:
     duration_points = [p for p in points if p[0] == "openarmature.gen_ai.client.operation.duration"]
 
     if invariants.get("one_duration_observation_per_call"):
-        assert len(duration_points) == 1, (
+        # p[2] is the histogram's observation COUNT. Asserting only that one data
+        # point exists cannot fail the double-record regression this is named for:
+        # two observations aggregate into a single point with count 2.
+        assert len(duration_points) == 1 and duration_points[0][2] == 1, (
             f"expected exactly one operation.duration observation per call; got {duration_points}"
         )
     if invariants.get("two_token_usage_observations_input_and_output"):
-        assert len(input_usage) == 1 and len(output_usage) == 1, (
-            f"expected one input and one output token.usage observation; got {usage_points}"
-        )
+        assert (
+            len(input_usage) == 1
+            and len(output_usage) == 1
+            and input_usage[0][2] == 1
+            and output_usage[0][2] == 1
+        ), f"expected one input and one output token.usage observation; got {usage_points}"
     if invariants.get("errored_call_records_duration_with_error_type") or invariants.get(
         "duration_records_error_type_on_failure"
     ):
diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py
index 1e17863..5a3f83f 100644
--- a/tests/conformance/test_observability_langfuse.py
+++ b/tests/conformance/test_observability_langfuse.py
@@ -561,7 +561,9 @@ async def fetch(
 async def test_langfuse_fixture(fixture_path: Path) -> None:
     spec = _load(fixture_path)
     fixture_stem = fixture_path.stem
-    for _case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+    # `or [spec]` because a single-case fixture puts `expected` at the top
+    # level with no `cases` list; `or []` skipped seven of them entirely.
+    for _case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]):
         assert_case_asserts_something(
             fixture_stem,
             cast("str", _case.get("name") or "<unnamed>"),
@@ -2400,6 +2402,17 @@ def _assert_trace(
     *,
     expected_invariants: dict[str, Any],
 ) -> None:
+    if expected_invariants.get("no_warning_level_under_budget"):
+        # Implemented, unlike its neighbours, because it is UNBACKED: 155 pins its
+        # level concretely (`level: ERROR`) so its invariants are documentary, but
+        # 156's expected block declares no level at all, leaving this its only
+        # expression. A blanket guard here would be wrong -- it would demand
+        # implementation of the documentary majority.
+        offending = [o for o in trace.observations if str(getattr(o, "level", "") or "").upper() == "WARNING"]
+        assert not offending, (
+            f"an under-budget call MUST NOT render a WARNING-level observation; got "
+            f"{[(o.name, o.level) for o in offending]}"
+        )
     expected_id = expected.get("id")
     if expected_id is not None and not _is_placeholder(expected_id):
         # Fixtures 035/036: a LITERAL trace.id is the DERIVED Langfuse id; the

From 60ab6fb7753c0cd3dc29a631356472c34166e2f6 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sat, 15 Aug 2026 17:35:41 -0700
Subject: [PATCH 08/13] Record that a passing conformance fixture proves
 nothing

A green conformance run is the null result: a fixture wired into a driver
that ignores half its expected block passes exactly like a fully
asserted one, so passing discriminates nothing.

States the inverted acceptance criterion, that a fixture is wired when
the behaviour it covers has been broken and seen to go red, and the
traps around it: verify the mutation landed, treat a fixture that needs
no harness work as the highest risk rather than the easiest win,
diagnose by running instead of reading, and prefer a structural guard
over remembering any of it.

Written down because the rule already existed as guidance and was still
missed repeatedly across one session.
---
 AGENTS.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/AGENTS.md b/AGENTS.md
index c3ace78..d8415aa 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -75,6 +75,32 @@ by convention.
   `SubgraphNode.run`, projection variants, frozen-state mutation, etc.
 - `tests/test_smoke.py` — version sync.
 
+### Activating a conformance fixture is not done when it passes
+
+A green conformance run is the **null result**, not evidence. A fixture
+wired into a driver that ignores half its `expected` block passes exactly
+like one that is fully asserted, so "it passes" distinguishes nothing.
+
+The acceptance criterion is therefore inverted: a fixture is wired when
+you have **broken the behaviour it covers and watched it go red**.
+
+- Mutate the src behaviour, not the test. Confirm the mutation actually
+  landed before trusting the result: if the anchor text occurs twice and
+  you changed one, a partial mutation reads exactly like a surviving one.
+- A fixture that passes the moment you add a dispatch entry, with no
+  harness work, is the highest-risk case, not the easiest win.
+- Diagnose by running, not by reading. A deferral reason reached by
+  reasoning about which driver "looks" suitable is routinely wrong; a
+  ten-second `await _run_x(_load(path))` settles it.
+- Prefer a structural guard over remembering any of this. The
+  `_DRIVER_EXPECTED_KEYS` check in `tests/conformance/test_observability.py`
+  makes "wired into a driver that drops a directive" impossible rather
+  than catchable, and caught a live error in the commit that added it.
+
+This is written down because the rule existed as guidance and was still
+missed repeatedly across one session: four fixtures were reported wired
+while asserting nothing, and three deferral reasons were wrong.
+
 ## Tooling
 
 - `uv` for everything. Don't use `pip` directly.

From e1797c5f2201b1d255bbf6ae31729ed96a901002 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Sun, 16 Aug 2026 08:39:40 -0700
Subject: [PATCH 09/13] Correct the driver map and stop claiming 149 is wired

The delta review found the previous fix swapped which half of 149 was
dropped rather than fixing it. It moved from a driver reading only
observers to one reading only span_tree, and the new guard waved it
through because the hand-written map claimed the payload driver reads
observers. It does not; it reads span_tree alone.

The map now states what that driver actually reads. 149 goes back to
deferred, because no driver serves it: one reads observers, one reads
span_tree, and the third reads both but builds the graph from a mock
shape that does not reproduce the fixture's response model or id, so its
span match fails on attributes rather than behaviour. Its event half is
the fixture's stated discriminator, so wiring it under a span-only
driver drops the assertion it exists for.

The guard also iterated `cases or []`, the idiom this branch fixed forty
lines below it, making it a no-op for single-case fixtures. Its
fail-open bypass is now documented as fail-open rather than left to read
as blanket protection: most drivers are unregistered, so the guard is
dark for them.
---
 tests/conformance/test_observability.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index 6b16f2d..a46da68 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -196,7 +196,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
         "067-llm-completion-event-call-id-always-present-and-distinct",
         "068-llm-completion-event-response-model-distinct-from-request",
         "144-otel-llm-span-omits-input-usage-on-null-counter",
-        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
         "071-llm-failure-event-call-id-distinct-from-completion-event",
         "072-llm-failure-event-mutual-exclusion-with-completion-event",
         # proposal 0082: the LlmFailedEvent response-side surface on a
@@ -433,6 +432,15 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None:
     # embedding failure-metrics counterpart).
     # Diagnosed against their sibling drivers rather than guessed: each reaches a
     # driver and fails on a concrete gap, not on "no driver".
+    "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": (
+        "declares BOTH expected.span_tree and expected.observers.contains_event, and no driver "
+        "serves both: _run_typed_event_cases reads observers only, _run_llm_payload_case reads "
+        "span_tree only, and _run_token_budget_case reads both but builds the graph from a mock "
+        "shape that does not reproduce 149's response model / response id, so its span-tree match "
+        "fails on attributes rather than on behaviour. Needs a driver threading 149's mock_llm "
+        "through a typed collector AND a span exporter; the event half is the fixture's stated "
+        "discriminator, so wiring it under a span-only driver drops the assertion it exists for"
+    ),
     "119-otel-callable-branch-attempt-index-under-node-retry": (
         "the conformance adapter never translates a fixture node's YAML `middleware:` block, so "
         "the retry is never installed and the transient failure is terminal: only attempt 0 runs "
@@ -672,7 +680,10 @@ def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any]
     ),
     "_run_rerank_fixture": frozenset({"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}),
     "_run_token_budget_fixture": frozenset({"span_tree", "metrics", "invariants", "observers"}),
-    "_run_llm_payload_fixture": frozenset({"span_tree", "invariants", "observers"}),
+    # `invariants` is documentary here; `observers` is deliberately ABSENT because
+    # _run_llm_payload_case reads span_tree ONLY. Claiming observers is what let
+    # 149 be re-routed here with its contains_event half silently dropped.
+    "_run_llm_payload_fixture": frozenset({"span_tree", "invariants"}),
 }
 
 
@@ -714,8 +725,15 @@ def _assert_driver_reads_every_expected_key(fixture_id: str, spec: Mapping[str,
         return
     handled = _DRIVER_EXPECTED_KEYS.get(driver)
     if handled is None:
+        # FAIL-OPEN, and deliberately narrow. Most drivers are unregistered, so
+        # this exempts the majority; registering one is what turns the guard on
+        # for its fixtures. Recorded here rather than left implicit because a
+        # guard that is dark for two thirds of the corpus must not read as
+        # blanket protection -- see the note in AGENTS.md.
         return
-    for case in cast("list[dict[str, Any]]", spec.get("cases") or []):
+    # `or [spec]`, not `or []`: a single-case fixture carries `expected` at the top
+    # level with no `cases` list, and `or []` made this guard a no-op for all seven.
+    for case in cast("list[dict[str, Any]]", spec.get("cases") or [spec]):
         declared = set(cast("dict[str, Any]", case.get("expected") or {}))
         unread = sorted(declared - handled)
         assert not unread, (
@@ -860,7 +878,6 @@ async def test_observability_fixture(fixture_path: Path) -> None:
         # including attributes_absent -- the half 149 silently dropped under the
         # typed-event driver, and the driver 144 was wrongly deferred for lacking.
         "144-otel-llm-span-omits-input-usage-on-null-counter",
-        "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span",
     }:
         await _run_llm_payload_fixture(spec)
     elif fixture_id in {

From cc0820fbe4ee621a1995f02c9ada7acae3d41d53 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Mon, 17 Aug 2026 08:10:09 -0700
Subject: [PATCH 10/13] Fix two vacuity regressions in the metrics guards

The recognized-invariant check validated against the union of the
metrics and token-budget families from inside each guard, but the
metrics runner calls only one of them. A token-budget name declared
on a metrics fixture was therefore recognized while nothing checked
it. Each runner now supplies the union over the guards it actually
calls, so the metrics path rejects what it cannot evaluate.

The positive span anchor was gated on the span set being non-empty,
which short-circuited it in exactly the zero-span case it exists to
catch, and keyed on a name prefix that missed both
exceeded_span_signal_* claims. It now keys on an enumerated set and
fires regardless of how many spans were recorded.
---
 tests/conformance/test_observability.py | 84 +++++++++++++++++--------
 1 file changed, 57 insertions(+), 27 deletions(-)

diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py
index a46da68..7a7add4 100644
--- a/tests/conformance/test_observability.py
+++ b/tests/conformance/test_observability.py
@@ -35,6 +35,7 @@
 import functools
 import re
 from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Set as AbstractSet
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
@@ -4622,13 +4623,16 @@ async def _run_metrics_case(case: Mapping[str, Any]) -> None:
     points = _collect_metric_points(reader)
     expected_metrics = cast("list[dict[str, Any]]", case["expected"].get("metrics") or [])
     _assert_metric_points(points, expected_metrics)
+    # `_assert_metrics_invariants` is the ONLY guard on this path, so the
+    # token-budget family is deliberately not recognized here: a fixture routed
+    # to this driver that declares one must fail, not be silently dropped.
+    _assert_invariants_recognized(case, _METRICS_INVARIANTS, "metrics")
     _assert_metrics_invariants(case, points, exporter.get_finished_spans())
 
 
-# Names only the token-budget driver implements. Kept beside the metrics set so
-# each guard can validate against the UNION while implementing only its own
-# subset: fixture 147 legitimately declares names from both families.
-_TOKEN_BUDGET_ONLY_INVARIANTS = {
+# Names `_assert_token_budget_invariants` implements. Overlaps the metrics set by
+# one name that BOTH guards check, so this is not a disjoint "only" set.
+_TOKEN_BUDGET_INVARIANTS = {
     "no_token_budget_exceeded_observation_when_under_budget",
     "utilization_records_under_budget",
     "no_token_budget_instrument_observations_when_no_budget",
@@ -4669,6 +4673,49 @@ async def _run_metrics_case(case: Mapping[str, Any]) -> None:
     "duration_value_not_asserted",
 }
 
+# Invariant names whose claim is "some attribute is absent from a SPAN". Each is
+# satisfied trivially by a run that produced no spans at all, so declaring one
+# obliges the positive anchor below. Enumerated rather than matched by prefix: a
+# `startswith` heuristic silently skipped the anchor for the two
+# `exceeded_span_signal_*` names, which are exactly this shape.
+_SPAN_DEPENDENT_INVARIANTS = {
+    "span_omits_both_input_usage_attributes",
+    "exceeded_span_signal_absent_not_false",
+    "exceeded_span_signal_absent_not_false_when_only_bound_unevaluable",
+    "null_prompt_tokens_reaches_no_span_histogram_or_budget_surface",
+}
+
+
+def _assert_invariants_recognized(
+    case: Mapping[str, Any],
+    recognized: AbstractSet[str],
+    label: str,
+) -> None:
+    """Fail loudly on a declared invariant name no guard on this driver's path
+    implements, so a fixture's new invariant cannot pass vacuously."""
+    # `recognized` is supplied by the RUNNER, not by an individual guard: it is
+    # the union over the guards that runner actually calls. Validating inside a
+    # guard against a wider union is what let a token-budget name declared on a
+    # metrics fixture read as recognized while nothing checked it.
+    invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
+    unknown = set(invariants) - recognized
+    assert not unknown, f"unhandled {label} invariant(s): {sorted(unknown)}"
+
+
+def _assert_span_anchor(invariants: Mapping[str, Any], spans: Sequence[Any]) -> None:
+    """Assert the LLM span exists before any span-absence predicate reads it."""
+    # Every span-dependent claim is of the form "X is absent", which an empty
+    # span set satisfies trivially -- deleting the LLM span outright made 147
+    # green. Deliberately NOT gated on `spans` being non-empty: no spans at all
+    # is the very case this exists to catch.
+    if not (set(invariants) & _SPAN_DEPENDENT_INVARIANTS):
+        return
+    llm_spans = [s for s in spans if s.name == "openarmature.llm.complete"]
+    assert llm_spans, (
+        f"no openarmature.llm.complete span was recorded, so the span-absence invariants "
+        f"would pass vacuously; got {[s.name for s in spans]}"
+    )
+
 
 def _assert_metrics_invariants(
     case: Mapping[str, Any],
@@ -4682,21 +4729,9 @@ def _assert_metrics_invariants(
     # the fixtures pass while checking nothing but the positive rows. Mirrors the
     # token-budget driver's guard, whose absence here is what let that happen.
     invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
-    unknown = set(invariants) - _METRICS_INVARIANTS - _TOKEN_BUDGET_ONLY_INVARIANTS
-    assert not unknown, f"unhandled metrics invariant(s): {sorted(unknown)}"
     if not invariants:
         return
-
-    # Positive anchor BEFORE any absence predicate. Every claim below is of the
-    # form "X is absent", which an empty span set satisfies trivially -- deleting
-    # the LLM span outright made 147 green. Assert the span exists first, so the
-    # absences are read off a run that actually happened.
-    if spans and any(k.startswith(("span_", "null_prompt_tokens_reaches")) for k in invariants):
-        llm_spans = [s for s in spans if s.name == "openarmature.llm.complete"]
-        assert llm_spans, (
-            f"no openarmature.llm.complete span was recorded, so the span-absence invariants "
-            f"below would pass vacuously; got {[s.name for s in spans]}"
-        )
+    _assert_span_anchor(invariants, spans)
 
     usage_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token.usage"]
     input_usage = [p for p in usage_points if p[3].get("openarmature.gen_ai.token.type") == "input"]
@@ -4889,6 +4924,9 @@ async def _run_token_budget_case(case: Mapping[str, Any]) -> None:
     expected_metrics = cast("list[dict[str, Any]] | None", expected.get("metrics"))
     if expected_metrics is not None:
         _assert_metric_points(points, expected_metrics)
+    # Both guards run on this path, so the recognized set is their union: 147
+    # routes here and declares metrics-family names exclusively.
+    _assert_invariants_recognized(case, _METRICS_INVARIANTS | _TOKEN_BUDGET_INVARIANTS, "token-budget")
     _assert_token_budget_invariants(case, points, collectors, spans)
     _assert_metrics_invariants(case, points, spans)
 
@@ -4897,22 +4935,14 @@ def _assert_token_budget_invariants(
     case: Mapping[str, Any],
     points: Sequence[tuple[str, float, int, dict[str, Any]]],
     collectors: Mapping[str, Any],
-    spans: Sequence[Any] = (),
+    spans: Sequence[Any],
 ) -> None:
     """Evaluate the token-budget named invariants (128/129/131) as absence
     predicates over the captured measurement set + the typed events.
     The ``metrics:`` list shape asserts presence only, so these cover the
     "no observation recorded" claims the list cannot express."""
     invariants = cast("dict[str, Any]", case["expected"].get("invariants") or {})
-    # Fail loudly on a declared invariant this runner does not map, so a future
-    # fixture's new invariant name cannot pass vacuously (harness-fidelity).
-    #
-    # Validated against the UNION with the metrics family, because a fixture may
-    # legitimately declare names from both (147 does). Each guard still implements
-    # only its own subset; the union governs what is RECOGNISED, not what is
-    # checked, and `_run_token_budget_case` calls both.
-    unknown = set(invariants) - _METRICS_INVARIANTS - _TOKEN_BUDGET_ONLY_INVARIANTS
-    assert not unknown, f"unhandled token-budget invariant(s): {sorted(unknown)}"
+    _assert_span_anchor(invariants, spans)
     exceeded_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.exceeded"]
     utilization_points = [p for p in points if p[0] == "openarmature.gen_ai.client.token_budget.utilization"]
 

From de9b5753ea16c7f360ff099ef8ee412a071c3ef2 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Mon, 17 Aug 2026 08:10:30 -0700
Subject: [PATCH 11/13] Correct the AGENTS.md driver-guard claim

The note said the _DRIVER_EXPECTED_KEYS check makes a fixture wired
into a driver that drops a directive impossible rather than merely
catchable. That overstates it: the check is fail-open and returns
silently for any driver absent from the map, which is 30 of the 36
drivers the dispatch chain reaches. An agent reading the old wording
would treat a green run on an unregistered driver as evidence.
---
 AGENTS.md | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index d8415aa..446660a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -92,10 +92,14 @@ you have **broken the behaviour it covers and watched it go red**.
 - Diagnose by running, not by reading. A deferral reason reached by
   reasoning about which driver "looks" suitable is routinely wrong; a
   ten-second `await _run_x(_load(path))` settles it.
-- Prefer a structural guard over remembering any of this. The
-  `_DRIVER_EXPECTED_KEYS` check in `tests/conformance/test_observability.py`
-  makes "wired into a driver that drops a directive" impossible rather
-  than catchable, and caught a live error in the commit that added it.
+- Prefer a structural guard over remembering any of this, but check its
+  reach before trusting it. The `_DRIVER_EXPECTED_KEYS` check in
+  `tests/conformance/test_observability.py` catches "wired into a driver
+  that drops a directive" — it caught a live error in the commit that
+  added it — but only for the drivers registered in that map, which is a
+  minority of them. It is **fail-open**: a fixture routed to an
+  unregistered driver returns silently, so a green run there still means
+  nothing. Registering the driver is what turns the guard on.
 
 This is written down because the rule existed as guidance and was still
 missed repeatedly across one session: four fixtures were reported wired

From c110397b263c343263c26cc938f0eb138f09c225 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Mon, 17 Aug 2026 08:13:34 -0700
Subject: [PATCH 12/13] Guard harness allowlists against silent drift

Three of the harness's own allowlists had no check that they still
describe what its code does, and a conformance run cannot reveal the
difference: a dead assertion and a live one produce the same green.

_PER_TRACE_INVARIANTS omitted no_warning_level_under_budget, which
_assert_trace does check, so the multi-trace path would discard that
claim. Both sets are now derived from the function bodies and
compared, so the next omission fails instead of going quiet.

An unregistered driver is invisible to the mis-routing guard, so a
new one goes dark on arrival. Every dispatched driver must now be
registered or named in an unguarded list with its reason. Four shared
drivers covering 17 fixtures are recorded as a real remaining gap
rather than left implicit.

The span anchor itself was undetectable by the corpus: both fixtures
declaring a span-absence claim also declare span_tree, whose root
check fails first, so deleting the anchor left every fixture green.
Driving both guards directly with an empty span set closes that.
---
 tests/conformance/test_harness_fidelity.py    | 203 ++++++++++++++++++
 .../test_observability_langfuse.py            |  10 +-
 2 files changed, 212 insertions(+), 1 deletion(-)
 create mode 100644 tests/conformance/test_harness_fidelity.py

diff --git a/tests/conformance/test_harness_fidelity.py b/tests/conformance/test_harness_fidelity.py
new file mode 100644
index 0000000..3393920
--- /dev/null
+++ b/tests/conformance/test_harness_fidelity.py
@@ -0,0 +1,203 @@
+"""Structural guards on the conformance harness itself."""
+
+# Nothing here asserts engine behaviour. These tests assert that the harness's
+# own allowlists still describe what its code does, because both have already
+# drifted in ways no fixture run could reveal: a green conformance run is
+# identical whether an assertion is live or dead. See the AGENTS.md note
+# "Activating a conformance fixture is not done when it passes".
+
+from __future__ import annotations
+
+import ast
+import inspect
+from typing import Any
+
+import pytest
+
+from . import test_observability as otel_runner
+from . import test_observability_langfuse as langfuse_runner
+
+# Drivers whose `expected` keys are deliberately NOT declared in
+# `_DRIVER_EXPECTED_KEYS`, with the reason they are safe to leave unguarded.
+# The guard is fail-open, so an unregistered driver is invisible to it; listing
+# them here is what turns "invisible" into "decided". A new driver appears in
+# neither map and fails the test below, forcing that decision at the point it is
+# added rather than leaving it dark indefinitely.
+#
+# A single-fixture driver cannot suffer the defect the guard exists to catch --
+# a fixture routed to a SHARED driver that ignores one of its directives -- since
+# it was written against exactly one fixture's `expected` block.
+_UNGUARDED_SINGLE_FIXTURE_DRIVERS = frozenset(
+    {
+        "_run_fixture_001",
+        "_run_fixture_002",
+        "_run_fixture_003",
+        "_run_fixture_004",
+        "_run_fixture_005",
+        "_run_fixture_006",
+        "_run_fixture_007",
+        "_run_fixture_008",
+        "_run_fixture_009",
+        "_run_fixture_010",
+        "_run_fixture_011",
+        "_run_fixture_028",
+        "_run_fixture_038",
+        "_run_fixture_050",
+        "_run_fixture_051",
+        "_run_fixture_052",
+        "_run_fixture_053",
+        "_run_fixture_054",
+        "_run_fixture_055",
+        "_run_fixture_056",
+        "_run_fixture_058",
+        "_run_fixture_084",
+        "_run_fixture_110",
+        "_run_fixture_132",
+        "_run_fixture_133",
+        "_run_structured_output_error_span_fixture",
+    }
+)
+
+# Shared drivers that ARE exposed to the mis-routing defect and are not yet
+# registered. This is a real gap, recorded rather than hidden: registering one
+# means deriving its read set from its body (and its callees), which is the step
+# that produced a wrong `observers` claim when done by inspection alone.
+_UNGUARDED_SHARED_DRIVERS = frozenset(
+    {
+        "_run_get_invocation_metadata_fixture",
+        "_run_llm_cache_fixture",
+        "_run_tool_fixture",
+        "_run_typed_event_chain_cases",
+    }
+)
+
+
+def test_every_dispatched_driver_is_registered_or_explicitly_unguarded() -> None:
+    dispatched = set(otel_runner._driver_for_fixture().values())  # noqa: SLF001
+    accounted = (
+        set(otel_runner._DRIVER_EXPECTED_KEYS)  # noqa: SLF001
+        | _UNGUARDED_SINGLE_FIXTURE_DRIVERS
+        | _UNGUARDED_SHARED_DRIVERS
+    )
+    unaccounted = sorted(dispatched - accounted)
+    assert not unaccounted, (
+        f"drivers {unaccounted} are dispatched to but appear in neither `_DRIVER_EXPECTED_KEYS` nor "
+        f"the unguarded lists in this file. Register the driver's `expected` keys, or add it to an "
+        f"unguarded list with the reason -- an unregistered driver silently exempts its fixtures "
+        f"from the mis-routing guard."
+    )
+    # Non-vacuity on the INPUT: an empty dispatch map satisfies the subset check
+    # above trivially, which is how a guard over nothing reads as a clean pass.
+    assert dispatched, "the dispatch map is empty, so the check above compared nothing"
+    # And the lists must not accumulate names for drivers that no longer exist.
+    stale = sorted((_UNGUARDED_SINGLE_FIXTURE_DRIVERS | _UNGUARDED_SHARED_DRIVERS) - dispatched)
+    assert not stale, f"unguarded lists name drivers nothing dispatches to: {stale}"
+
+
+def _invariant_names_read_by(func: object, param: str) -> set[str]:
+    """String literals `func` looks up on the mapping named `param`."""
+    tree = ast.parse(inspect.getsource(func))  # pyright: ignore[reportArgumentType]
+    names: set[str] = set()
+    for node in ast.walk(tree):
+        if (
+            isinstance(node, ast.Call)
+            and isinstance(node.func, ast.Attribute)
+            and node.func.attr == "get"
+            and isinstance(node.func.value, ast.Name)
+            and node.func.value.id == param
+            and node.args
+            and isinstance(node.args[0], ast.Constant)
+            and isinstance(node.args[0].value, str)
+        ):
+            names.add(node.args[0].value)
+        elif (
+            isinstance(node, ast.Subscript)
+            and isinstance(node.value, ast.Name)
+            and node.value.id == param
+            and isinstance(node.slice, ast.Constant)
+            and isinstance(node.slice.value, str)
+        ):
+            names.add(node.slice.value)
+    return names
+
+
+def _span_dependent_invariants_in(func: object) -> set[str]:
+    """Invariant names whose `if invariants.get(...)` body reads the span set."""
+    tree = ast.parse(inspect.getsource(func))  # pyright: ignore[reportArgumentType]
+    found: set[str] = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.If):
+            continue
+        names = {
+            call.args[0].value
+            for call in ast.walk(node.test)
+            if isinstance(call, ast.Call)
+            and isinstance(call.func, ast.Attribute)
+            and call.func.attr == "get"
+            and isinstance(call.func.value, ast.Name)
+            and call.func.value.id == "invariants"
+            and call.args
+            and isinstance(call.args[0], ast.Constant)
+            and isinstance(call.args[0].value, str)
+        }
+        if not names:
+            continue
+        body = ast.Module(body=node.body, type_ignores=[])
+        if any(isinstance(n, ast.Name) and n.id in {"spans", "_spans_carrying"} for n in ast.walk(body)):
+            found |= names
+    return found
+
+
+def test_span_dependent_invariants_lists_every_span_reading_claim() -> None:
+    # `_assert_span_anchor` only fires for names in `_SPAN_DEPENDENT_INVARIANTS`.
+    # A span-reading claim missing from that set gets no positive anchor, so an
+    # empty span set satisfies it trivially -- which is what the `startswith`
+    # heuristic this set replaced actually did to the two
+    # `exceeded_span_signal_*` names.
+    derived = _span_dependent_invariants_in(
+        otel_runner._assert_metrics_invariants  # noqa: SLF001
+    ) | _span_dependent_invariants_in(otel_runner._assert_token_budget_invariants)  # noqa: SLF001
+    assert derived, "parsed no span-reading invariant blocks, so this compared nothing"
+    missing = sorted(derived - otel_runner._SPAN_DEPENDENT_INVARIANTS)  # noqa: SLF001
+    assert not missing, (
+        f"{missing} read the span set but are absent from `_SPAN_DEPENDENT_INVARIANTS`, so no "
+        f"positive anchor runs for them and a zero-span run satisfies them vacuously."
+    )
+    stale = sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS - derived)  # noqa: SLF001
+    assert not stale, f"`_SPAN_DEPENDENT_INVARIANTS` names claims that no longer read spans: {stale}"
+
+
+def _drive_metrics_guard(case: dict[str, Any]) -> None:
+    otel_runner._assert_metrics_invariants(case, [], [])  # noqa: SLF001
+
+
+def _drive_token_budget_guard(case: dict[str, Any]) -> None:
+    otel_runner._assert_token_budget_invariants(case, [], {}, [])  # noqa: SLF001
+
+
+@pytest.mark.parametrize("drive", [_drive_metrics_guard, _drive_token_budget_guard])
+@pytest.mark.parametrize("invariant", sorted(otel_runner._SPAN_DEPENDENT_INVARIANTS))  # noqa: SLF001
+def test_a_zero_span_run_cannot_satisfy_a_span_absence_claim(invariant: str, drive: Any) -> None:
+    # The test above keeps `_SPAN_DEPENDENT_INVARIANTS` honest about which names
+    # read spans; this one keeps the ANCHOR ITSELF live. Deleting
+    # `_assert_span_anchor`, or its call from either guard, leaves every shipping
+    # fixture green -- both fixtures that declare a span-absence claim also
+    # declare `span_tree`, whose own root-span check fails first, so nothing in
+    # the corpus would notice. Driving each guard directly is what makes the
+    # anchor's removal detectable at all.
+    with pytest.raises(AssertionError, match="pass vacuously"):
+        drive({"expected": {"invariants": {invariant: True}}})
+
+
+def test_per_trace_invariants_covers_everything_assert_trace_reads() -> None:
+    # `_assert_multi_traces` filters the fixture's invariants down to
+    # `_PER_TRACE_INVARIANTS` before delegating. A name `_assert_trace` checks
+    # but that is absent from the set is therefore dropped on that path, and the
+    # fixture declaring it still passes.
+    read = _invariant_names_read_by(langfuse_runner._assert_trace, "expected_invariants")  # noqa: SLF001
+    assert read, "parsed no invariant lookups out of _assert_trace, so this compared nothing"
+    missing = sorted(read - set(langfuse_runner._PER_TRACE_INVARIANTS))  # noqa: SLF001
+    assert not missing, (
+        f"`_assert_trace` checks {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the "
+        f"multi-trace runner discards those claims silently."
+    )
diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py
index 5a3f83f..270c928 100644
--- a/tests/conformance/test_observability_langfuse.py
+++ b/tests/conformance/test_observability_langfuse.py
@@ -2281,7 +2281,15 @@ def _runtime_config_from_spec(config_spec: dict[str, Any] | None) -> RuntimeConf
 # assertions; the rest (``distinct_trace_ids``,
 # ``correlation_id_consistent_across_traces``, etc.) stay in
 # ``_assert_multi_traces`` as cross-Trace checks.
-_PER_TRACE_INVARIANTS = frozenset({"trace_id_equals_invocation_id", "correlation_id_consistency"})
+#
+# Must list EVERY name ``_assert_trace`` reads: a name it checks but that is
+# missing here is silently discarded on the multi-trace path, so a fixture
+# declaring it there passes without the claim being evaluated.
+# ``test_harness_fidelity.py`` derives the read set from the function body and
+# fails on any omission -- ``no_warning_level_under_budget`` was one.
+_PER_TRACE_INVARIANTS = frozenset(
+    {"trace_id_equals_invocation_id", "correlation_id_consistency", "no_warning_level_under_budget"}
+)
 
 
 def _assert_multi_traces(

From 22c82ae9187f3de0ce6d3d8b04efd458779c4840 Mon Sep 17 00:00:00 2001
From: chris-colinsky <chris@lunarcommand.xyz>
Date: Mon, 17 Aug 2026 08:14:05 -0700
Subject: [PATCH 13/13] Cover both fixture shapes in the wiring test

The test named two fixtures and claimed one exercised the multi-case
shape, but both put expected at the top level with no cases list, so
the branch reading spec["cases"][0] never ran. It now names a
multi-case fixture per runner and asserts each row still has the
shape it was chosen for.

Both runners skip a fixture they no longer drive, and Skipped derives
from BaseException, so it passed straight through pytest.raises and
turned the whole test into a silent skip. A skip now fails, naming
the fixture, since a test that never ran is the failure mode this
file exists to prevent.
---
 .../test_capability_gate_wiring.py            | 43 ++++++++++++++++---
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/tests/conformance/test_capability_gate_wiring.py b/tests/conformance/test_capability_gate_wiring.py
index 9b6bd02..c42528a 100644
--- a/tests/conformance/test_capability_gate_wiring.py
+++ b/tests/conformance/test_capability_gate_wiring.py
@@ -355,15 +355,28 @@ def test_a_case_with_any_concrete_directive_is_accepted(expected: dict[str, obje
 @pytest.mark.parametrize(
     ("module_name", "runner_attr", "fixture_stem"),
     [
-        # One per dispatch path, and one SINGLE-CASE fixture (022) because those
-        # put `expected` at the top level with no `cases` list -- the shape the
-        # first version of this wiring skipped entirely.
+        # Each dispatch path x each fixture SHAPE. A single-case fixture puts
+        # `expected` at the top level with no `cases` list -- the shape the first
+        # version of this wiring skipped entirely -- while a multi-case fixture
+        # exercises the `spec["cases"][0]` branch of the graft below. Naming only
+        # single-case fixtures left that branch unexecuted, so the two shapes are
+        # now covered explicitly rather than assumed from a fixture id.
         ("tests.conformance.test_observability", "test_observability_fixture", "001-otel-basic-trace"),
+        (
+            "tests.conformance.test_observability",
+            "test_observability_fixture",
+            "005-otel-llm-provider-span-nested",
+        ),
         (
             "tests.conformance.test_observability_langfuse",
             "test_langfuse_fixture",
             "022-langfuse-basic-trace",
         ),
+        (
+            "tests.conformance.test_observability_langfuse",
+            "test_langfuse_fixture",
+            "023-langfuse-generation-rendering",
+        ),
     ],
 )
 @pytest.mark.asyncio
@@ -380,6 +393,8 @@ async def test_the_guard_is_wired_into_each_runner(
     module = importlib.import_module(module_name)
     path = next(p for p in module._fixture_paths() if p.stem == fixture_stem)  # noqa: SLF001
     original_load = module._load  # noqa: SLF001
+    spec_shape = original_load(path)
+    multi_case = bool(spec_shape.get("cases"))
 
     def _grafted(p: Any) -> Any:
         spec = original_load(p)
@@ -388,5 +403,23 @@ def _grafted(p: Any) -> Any:
         return spec
 
     monkeypatch.setattr(module, "_load", _grafted)
-    with pytest.raises(AssertionError, match="only `invariants`"):
-        await getattr(module, runner_attr)(path)
+    # Both runners `pytest.skip` for a fixture they do not currently drive, and
+    # `Skipped` derives from BaseException, so it sails through `pytest.raises`
+    # and turns this whole test into a silent skip. That is the exact failure
+    # this file exists to prevent, one level up: the day a named fixture leaves
+    # its runner's supported set, a skip must read as a wiring regression.
+    try:
+        with pytest.raises(AssertionError, match="only `invariants`"):
+            await getattr(module, runner_attr)(path)
+    except pytest.skip.Exception as exc:  # pragma: no cover - regression guard
+        pytest.fail(
+            f"{runner_attr} skipped {fixture_stem} ({exc}), so this test asserted nothing about "
+            f"the guard. Repoint it at a fixture that runner still drives."
+        )
+    # Non-vacuity on the parametrization itself: the two shapes must actually
+    # differ, or both rows exercise the same branch of the graft above.
+    multi_case_rows = {"005-otel-llm-provider-span-nested", "023-langfuse-generation-rendering"}
+    assert multi_case == (fixture_stem in multi_case_rows), (
+        f"{fixture_stem} no longer has the fixture shape this row was chosen for "
+        f"(multi_case={multi_case}); the single-case and multi-case branches are no longer both covered."
+    )