From 82b527f176373331364928d0eda953c31364f7e7 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:34:26 -0700 Subject: [PATCH 1/8] Activate fixture 148 and implement its usage claim 148 asserts that a counter the provider did not report is omitted from the Generation's fixed usage record. Nothing was checking that. The langfuse_trace comparator matches usage by iterating the EXPECTED keys, so declining to declare `input` looks at nothing, and both invariants carrying the claim were implemented in no runner. An implementation emitting input=0 passed the fixture unchanged. Both invariants are now implemented against the Generation set, with a positive anchor so no Generation at all cannot satisfy the omission half trivially. None is the faithful proxy for wire key-absence here: the SDK adapter builds usage_details by skipping exactly the None fields, while the in-memory double models usage as a dataclass where every field exists. --- .../test_observability_langfuse.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index 270c928..ee8d3c6 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -161,6 +161,13 @@ "130-langfuse-token-budget-warning-level", "155-langfuse-token-budget-exceeded-flag-on-failure", "156-langfuse-token-budget-under-budget-flag-false", + # 148 (proposal 0101): the Generation's FIXED `usage` record omits a + # counter the provider did not report, unlike the Embedding's open + # `usageDetails` map (140) where absence is naturally expressible. The + # `usage` comparator here iterates the EXPECTED keys, so declining to + # declare `input` asserts nothing; the case's two invariants carry the + # omission claim, and both are implemented in `_assert_trace`. + "148-langfuse-generation-usage-omits-input-on-null-counter", # 134 (proposal 0084): the Langfuse Generation parent resolves by the same # chain-aware §5.5 rule as the OTel span parent -- both the nested # exact-match (case 1, mirrors OTel 132) and the orphan fallback (case 2, @@ -2288,7 +2295,13 @@ def _runtime_config_from_spec(config_spec: dict[str, Any] | None) -> RuntimeConf # ``test_harness_fidelity.py`` derives the read set from the function body and # fails on any omission -- ``no_warning_level_under_budget`` was one. _PER_TRACE_INVARIANTS = frozenset( - {"trace_id_equals_invocation_id", "correlation_id_consistency", "no_warning_level_under_budget"} + { + "trace_id_equals_invocation_id", + "correlation_id_consistency", + "no_warning_level_under_budget", + "generation_usage_input_omitted_when_prompt_tokens_null", + "generation_usage_output_and_total_present_when_sound", + } ) @@ -2404,6 +2417,51 @@ def _assert_multi_traces( ) +def _assert_generation_usage_omission(trace: LangfuseTrace, expected_invariants: dict[str, Any]) -> None: + """Fixture 148: a counter the provider did not report is OMITTED from the + Generation's fixed `usage` record, while the sound counters stay present.""" + # These two are the whole of 148's claim. The `langfuse_trace` block cannot + # carry it: `usage` is compared by iterating the EXPECTED keys, so a key the + # fixture declines to declare is never looked at, and an implementation + # emitting `input: null` passes the block unchanged. + # + # `None` is how omission is spelled on this side of the boundary. The + # in-memory double models `usage` as a dataclass, where every field exists + # and an unreported counter reads as None; the SDK adapter builds the wire + # `usage_details` dict by skipping exactly the None fields + # (langfuse/adapter.py), so None here is the faithful proxy for key-absence + # there. Asserting `"input" not in ...` would be checking the double's + # dataclass shape rather than the mapping under test. + omitted = expected_invariants.get("generation_usage_input_omitted_when_prompt_tokens_null") + present = expected_invariants.get("generation_usage_output_and_total_present_when_sound") + if not omitted and not present: + return + generations = [o for o in trace.observations if o.type == "generation"] + # Positive anchor: every claim below reads a Generation, and no Generation + # at all would satisfy the omission half trivially. + assert generations, ( + f"no Generation observation was recorded, so the usage claims below would pass " + f"vacuously; got {[(o.name, o.type) for o in trace.observations]}" + ) + if omitted: + carrying = [ + (g.name, g.usage) for g in generations if g.usage is not None and g.usage.input is not None + ] + assert not carrying, ( + f"an unreported prompt_tokens MUST be omitted from generation.usage rather than " + f"rendered as null or zero; present on {carrying}" + ) + if present: + # The non-vacuity partner: without it the omission above is satisfied by + # an implementation that emitted no usage record at all. + for gen in generations: + assert gen.usage is not None, f"generation {gen.name!r} carries no usage record at all" + assert gen.usage.output is not None and gen.usage.total is not None, ( + f"generation {gen.name!r} MUST still carry the sound counters; got " + f"output={gen.usage.output!r} total={gen.usage.total!r}" + ) + + def _assert_trace( trace: LangfuseTrace, expected: dict[str, Any], @@ -2421,6 +2479,7 @@ def _assert_trace( f"an under-budget call MUST NOT render a WARNING-level observation; got " f"{[(o.name, o.level) for o in offending]}" ) + _assert_generation_usage_omission(trace, expected_invariants) expected_id = expected.get("id") if expected_id is not None and not _is_placeholder(expected_id): # Fixtures 035/036: a LITERAL trace.id is the DERIVED Langfuse id; the From a4d4a1dab42948cfb893579cad8cf9e757b39db8 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:34:40 -0700 Subject: [PATCH 2/8] Follow assert_trace's helpers in the per-trace guard The guard derived the read set from _assert_trace's own body, so delegating an invariant to a helper made it blind to that name. Moving 148's two invariants into _assert_generation_usage_omission did exactly that the moment it was written. It now walks the _assert_* helpers _assert_trace calls, one level, and fails if that walk finds nothing. Adds the reverse check too: a name in _PER_TRACE_INVARIANTS that no per-trace guard reads means the multi-trace path forwards a claim nothing evaluates. --- tests/conformance/test_harness_fidelity.py | 36 ++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/conformance/test_harness_fidelity.py b/tests/conformance/test_harness_fidelity.py index 3393920..e503837 100644 --- a/tests/conformance/test_harness_fidelity.py +++ b/tests/conformance/test_harness_fidelity.py @@ -121,6 +121,24 @@ def _invariant_names_read_by(func: object, param: str) -> set[str]: return names +def _assert_helpers_called_by(func: object, module: object) -> list[object]: + """Module-level `_assert_*` functions `func` calls directly.""" + # One level, deliberately. Reading a guard's own body misses names it + # delegates to a helper, which is not hypothetical: moving 148's two + # invariants into `_assert_generation_usage_omission` made the check below + # blind to them the moment it was written. Deeper recursion is the + # transitive-derivation problem tracked separately; one level covers the + # delegate-to-a-sibling shape that actually occurs here. + tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] + found: list[object] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + helper = getattr(module, node.func.id, None) + if node.func.id.startswith("_assert_") and callable(helper) and helper not in found: + found.append(helper) + return found + + def _span_dependent_invariants_in(func: object) -> set[str]: """Invariant names whose `if invariants.get(...)` body reads the span set.""" tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] @@ -194,10 +212,22 @@ def test_per_trace_invariants_covers_everything_assert_trace_reads() -> None: # `_PER_TRACE_INVARIANTS` before delegating. A name `_assert_trace` checks # but that is absent from the set is therefore dropped on that path, and the # fixture declaring it still passes. - read = _invariant_names_read_by(langfuse_runner._assert_trace, "expected_invariants") # noqa: SLF001 - assert read, "parsed no invariant lookups out of _assert_trace, so this compared nothing" + entry = langfuse_runner._assert_trace # noqa: SLF001 + helpers = _assert_helpers_called_by(entry, langfuse_runner) + # Non-vacuity on the callee walk: `_assert_trace` delegates today, so + # finding none means the walk broke and the check silently narrowed back to + # the entry point's own body. + assert helpers, "found no _assert_* helpers called by _assert_trace; the callee walk is broken" + read: set[str] = set() + for func in [entry, *helpers]: + read |= _invariant_names_read_by(func, "expected_invariants") + assert read, "parsed no invariant lookups out of the per-trace guards, so this compared nothing" missing = sorted(read - set(langfuse_runner._PER_TRACE_INVARIANTS)) # noqa: SLF001 assert not missing, ( - f"`_assert_trace` checks {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the " + f"the per-trace guards check {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the " f"multi-trace runner discards those claims silently." ) + # Stale names are the other direction: a set entry nothing reads means the + # multi-trace path forwards a claim no guard evaluates. + stale = sorted(set(langfuse_runner._PER_TRACE_INVARIANTS) - read) # noqa: SLF001 + assert not stale, f"`_PER_TRACE_INVARIANTS` names invariants no per-trace guard reads: {stale}" From b3083d30db81d7822b43f8bd54cefe6430f285d8 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:37:14 -0700 Subject: [PATCH 3/8] Compare sequence span attributes by value, not identity OTel stores a sequence attribute as a tuple while YAML parses the fixture's list syntax into a list, and the two never compare equal. The span-tree matcher used a plain ==, so any expected attribute holding a sequence was unsatisfiable: the span could carry exactly the right value and still be rejected. Nothing surfaced this because no activated fixture asserted one through this matcher until 149 asserts gen_ai.response.finish_reasons. The failure also reads as an implementation defect rather than a harness gap, since the message reports a span that does not match. Comparison stays strict in every other respect: order, length, a differing element, and an absent attribute are all still rejected. --- tests/conformance/test_observability.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 7a7add4..57c7541 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -2632,6 +2632,19 @@ async def _ask_body( ) +def _attr_equal(actual: Any, expected: Any) -> bool: + """Compare a span attribute against a fixture's expected value.""" + # OTel stores a sequence attribute as a TUPLE, while YAML parses the + # fixture's `[...]` into a LIST, and `['stop'] == ('stop',)` is False. A + # plain `==` therefore rejects every sequence-valued attribute, which read + # as "the implementation emitted the wrong value" rather than as a harness + # gap: 149 is the first fixture to assert `gen_ai.response.finish_reasons` + # through this matcher, so nothing surfaced it before. + if isinstance(expected, list) and isinstance(actual, tuple): + return list(cast("tuple[Any, ...]", actual)) == cast("list[Any]", expected) + return bool(actual == expected) + + def _assert_span_tree_matches( all_spans: Sequence[Any], actual_roots: Sequence[Any], expected_nodes: Sequence[Mapping[str, Any]] ) -> None: @@ -2650,7 +2663,7 @@ def _assert_span_tree_matches( def _matches(span: Any, eattrs: dict[str, Any] = expected_attrs) -> bool: attrs = dict(span.attributes or {}) - return all(attrs.get(k) == v for k, v in eattrs.items()) + return all(_attr_equal(attrs.get(k), v) for k, v in eattrs.items()) matching = [c for c in candidates if _matches(c)] assert len(matching) >= 1, ( From cd96ef15947c6e65157269bda60ae9a00f2afa64 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:37:47 -0700 Subject: [PATCH 4/8] Record 148 as Langfuse-harness driven Its deferral reason described the subset-match gap that has now been closed, and the fixture runs in the sibling Langfuse runner like every other Langfuse-mapping fixture. Move it onto the set that records exactly that, rather than leaving prose in the deferral map. --- tests/conformance/test_observability.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 57c7541..e4e4965 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -456,13 +456,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "153-otel-mixed-nesting-orphan-llm-fallback": ( "same driver gap as 152, one nesting level deeper (KeyError: 'leaf_sg')" ), - "148-langfuse-generation-usage-omits-input-on-null-counter": ( - "the sibling Langfuse runner DOES drive mock_llm generations asserting usage (023, 155, " - "156), so plumbing is not the gap. The gap is its `usage` comparator: it iterates the " - "EXPECTED keys, a subset match, so an OMITTED `input` key -- the whole claim -- cannot " - "be expressed. Wiring it needs an exact-map or usage_absent comparator first, or the " - "fixture passes against an impl emitting usage.input = null" - ), # Proposal 0109 (spec v0.104.0) token-budget failure-path parity. } @@ -491,6 +484,11 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "059-implementation-attribution-langfuse", "155-langfuse-token-budget-exceeded-flag-on-failure", "156-langfuse-token-budget-under-budget-flag-false", + # 148 (proposal 0101): the Generation's fixed `usage` record omits a + # counter the provider did not report. The `usage` comparator is a + # subset match, so the OMISSION -- the whole claim -- is carried by the + # case's two invariants rather than by the langfuse_trace block. + "148-langfuse-generation-usage-omits-input-on-null-counter", # 134 -- proposal 0084 Langfuse parent resolution (nested exact-match + # orphan fallback). Driven by a dedicated hand-built runner in the # sibling harness; the generic topology path cannot model the From 4a05d031499456e4030ac0e2d6810b3d1634bcd2 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:38:15 -0700 Subject: [PATCH 5/8] Wire fixture 149 through a two-surface driver 149 asserts that a malformed wire counter is nulled by the provider mapping BEFORE it reaches either the typed event or the span, so it declares observers.contains_event and span_tree together. No driver read both: the typed-event driver reads observers only, the LLM-payload driver reads span_tree only. Routing it to either dropped half the fixture, and an earlier attempt to re-route it swapped which half. The new driver attaches the typed collectors and an OTel observer to one invocation and reads both. Two details it depends on, both found by running rather than reading: the observer must be shut down before the exporter is read, since that is what ends the invocation root span, and the provider must be bound to an explicit request model, since the existing fallback binds it to the model the mock response reports and so collapses the request/response distinction the fixture draws. Its six invariants restate what the concrete directives already pin and are registered as documentary. One is recorded as a negative control rather than as verified: a null usage record is unrepresentable here because Response.usage is not optional, so no conforming mutation can violate that claim. --- tests/conformance/test_observability.py | 133 ++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 10 deletions(-) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index e4e4965..6aefdc8 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -257,6 +257,12 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "128-token-budget-under-budget-no-warning", "129-token-budget-absent-unchanged", "131-token-budget-on-structured-output-failure", + # 149 (proposal 0101): a MALFORMED wire counter ("abc" / -5 / true) is + # nulled by the real complete() mapping BEFORE the typed event and the + # span render. The event half is the fixture's stated discriminator -- + # an impl sourcing the event from `raw` surfaces the wire value there -- + # so it needs a driver reading both surfaces off one invocation. + "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span", # v0.69.0 — proposal 0063 (tool-execution observability). A # calls_tool node enters the with_tool_call scope; the typed # ToolCallEvent / ToolCallFailedEvent drive the OTel tool span + @@ -433,15 +439,6 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: # embedding failure-metrics counterpart). # Diagnosed against their sibling drivers rather than guessed: each reaches a # driver and fails on a concrete gap, not on "no driver". - "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": ( - "declares BOTH expected.span_tree and expected.observers.contains_event, and no driver " - "serves both: _run_typed_event_cases reads observers only, _run_llm_payload_case reads " - "span_tree only, and _run_token_budget_case reads both but builds the graph from a mock " - "shape that does not reproduce 149's response model / response id, so its span-tree match " - "fails on attributes rather than on behaviour. Needs a driver threading 149's mock_llm " - "through a typed collector AND a span exporter; the event half is the fixture's stated " - "discriminator, so wiring it under a span-only driver drops the assertion it exists for" - ), "119-otel-callable-branch-attempt-index-under-node-retry": ( "the conformance adapter never translates a fixture node's YAML `middleware:` block, so " "the retry is never installed and the transient failure is terminal: only attempt 0 runs " @@ -679,6 +676,8 @@ def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any] ), "_run_rerank_fixture": frozenset({"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}), "_run_token_budget_fixture": frozenset({"span_tree", "metrics", "invariants", "observers"}), + # 149's driver: the only one reading the typed-event and span halves together. + "_run_typed_event_with_span_cases": frozenset({"span_tree", "observers", "invariants"}), # `invariants` is documentary here; `observers` is deliberately ABSENT because # _run_llm_payload_case reads span_tree ONLY. Claiming observers is what let # 149 be re-routed here with its contains_event half silently dropped. @@ -901,6 +900,9 @@ async def test_observability_fixture(fixture_path: Path) -> None: "131-token-budget-on-structured-output-failure", }: await _run_token_budget_fixture(spec) + elif fixture_id == "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": + # The only fixture asserting the typed event AND the span from one run. + await _run_typed_event_with_span_cases(spec) elif fixture_id in { "092-tool-call-event-dispatch", "093-tool-call-failed-event-dispatch", @@ -6265,6 +6267,7 @@ def _build_simple_llm_graph( case: Mapping[str, Any], *, populate_caller_metadata: bool, + model: str | None = None, ) -> tuple[Any, type[Any], Any]: """Build a single-node graph that calls the LLM provider against a mock transport. Matches the simple entry → ask → END pattern used @@ -6293,8 +6296,16 @@ def _build_simple_llm_graph( # §5.5.7 -- 068 needs this to differ from the provider-returned # response_model), else the model the first mock response reports # (050-056 path), else a default. + # An explicit `model` from the caller wins over the mock fallback. Fixtures + # that assert `gen_ai.request.model` and `gen_ai.response.model` DIFFER (144, + # 149) declare no `calls_llm.model`, so the mock fallback below would bind + # the request model to the response model and collapse the distinction the + # fixture exists to draw. `_run_llm_payload_case`, which drives 144, resolves + # this by hardcoding "test-model"; this parameter is that same convention + # made available to the other drivers rather than duplicated again. bound_model = ( - cast("str | None", calls_llm_spec.get("model")) + model + or cast("str | None", calls_llm_spec.get("model")) or _mock_model_from_first_response(case) or "test-model" ) @@ -6678,6 +6689,108 @@ async def _run_typed_event_cases(spec: Mapping[str, Any], *, expect_failure: boo raise AssertionError(f"case {case_name!r}: {e}") from e +# Fixture 149's six invariant names, all DOCUMENTARY: each restates a claim the +# case's concrete directives already pin, so there is nothing here for a guard to +# evaluate. Recognized so they cannot read as unhandled. +# +# Documentary is a verdict, not an assumption, so the claims were mutation-tested +# against src rather than read off the fixture. Coercing a malformed counter to 0 +# instead of nulling it fails `contains_event`; emitting the input-usage span +# attribute anyway fails `attributes_absent`. Both directives are live. +# +# ONE name is a negative control here, and is recorded as such rather than left +# looking verified: `event_usage_is_present_record_of_nulls_not_null_record` +# cannot be violated by this implementation at all, because `Response.usage` is +# typed `Usage` rather than `Usage | None` (llm/response.py). A null usage record +# is unrepresentable, so pydantic rejects the mutation before any assertion runs. +# The fixture argues the nested `usage:` mapping discriminates a present record +# of nulls from a null record, which is true and useful for an implementation +# whose usage field is nullable; it just cannot fail for ours. +_MALFORMED_COUNTER_INVARIANTS = { + # Case 1: one malformed wire counter ("abc"). + "malformed_wire_prompt_tokens_nulled_before_event_and_span", + "event_usage_prompt_tokens_null_not_verbatim_wire_value", + "span_omits_both_input_usage_attributes_on_malformed_wire_counter", + # Case 2: all three malformed ("abc" / -5 / true). + "all_three_malformed_wire_counters_nulled_before_event_and_span", + "event_usage_is_present_record_of_nulls_not_null_record", + "span_omits_all_usage_attributes_when_all_counters_malformed", +} + + +async def _run_typed_event_with_span_cases(spec: Mapping[str, Any]) -> None: + """Fixture 149: cases asserting the SAME null-counter claim on the typed + event AND the OTel span, from one invocation.""" + # 149 declares `observers.contains_event` and `span_tree` together, and no + # other driver serves both: the typed-event driver reads observers only, the + # LLM-payload driver reads span_tree only. Routing it to either silently + # drops half the fixture, which is what the earlier deferral recorded and + # what an attempted re-route then repeated in the other direction. The whole + # point of the fixture is that the nulling happens BEFORE the fan-out to both + # surfaces, so both must be read off one run. + for case in cast("list[dict[str, Any]]", spec["cases"]): + case_name = cast("str", case["name"]) + try: + await _run_typed_event_with_span_case(case) + except AssertionError as e: + raise AssertionError(f"case {case_name!r}: {e}") from e + + +async def _run_typed_event_with_span_case(case: Mapping[str, Any]) -> None: + from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: PLC0415 + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: PLC0415 + InMemorySpanExporter, + ) + + collectors, populate_caller_metadata = _parse_typed_observers(case) + # 149 mirrors 144's span claims, and both assert that the REQUEST model + # differs from the model the provider returned, without declaring a + # `calls_llm.model`. Bind the same default 144's driver uses. + graph, state_cls, provider = _build_simple_llm_graph( + case, populate_caller_metadata=populate_caller_metadata, model="test-model" + ) + exporter = InMemorySpanExporter() + observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter)) + try: + # Deliberately NOT `_invoke_typed_fixture`: that helper removes the + # observer handles BEFORE `drain()`, so the invocation-completed event + # never reaches the observer and the root span is left open. It never + # mattered for the typed-event fixtures because none of them reads + # spans; here it dropped `openarmature.invocation` from the exporter + # entirely, leaving only the node and LLM spans. + try: + final, _exc = await _invoke_typed_fixture( + case, collectors, graph, state_cls, extra_observer=observer + ) + finally: + observer.shutdown() + assert final is not None, "expected a non-None final state on success path" + + expected = cast("dict[str, Any]", case.get("expected") or {}) + observer_expectations = cast("dict[str, Any]", expected.get("observers") or {}) + for name, expectations in observer_expectations.items(): + collector = collectors.get(name) + if collector is None: + raise AssertionError(f"fixture references unknown observer {name!r}") + _assert_observer_expectations(name, collector, cast("dict[str, Any]", expectations)) + + spans = exporter.get_finished_spans() + expected_tree = cast("list[dict[str, Any]] | None", expected.get("span_tree")) + if expected_tree is not None: + inv_root = next( + (s for s in spans if s.name == "openarmature.invocation" and cast("Any", s.parent) is None), + None, + ) + assert inv_root is not None, f"invocation root span missing; got {[s.name for s in spans]}" + _assert_span_tree_matches(spans, [inv_root], expected_tree) + _assert_error_span_extras(spans, expected_tree) + + _assert_invariants_recognized(case, _MALFORMED_COUNTER_INVARIANTS, "malformed-counter") + finally: + await provider.aclose() + observer.shutdown() + + async def _run_typed_event_chain_cases(spec: Mapping[str, Any], *, expect_failure: bool = False) -> None: """Iterate the multi-node-chain typed-event cases (067 success, 071 failure).""" From 0e7b9736d99ba59e010950698abbc31649af41e5 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 06:48:17 -0700 Subject: [PATCH 6/8] Correct the 149 driver's shutdown comment and dedupe it The comment claimed the driver deliberately avoids _invoke_typed_fixture because of that helper's detach-before-drain ordering. Both halves were wrong: the code calls the helper on the next line, and the ordering turned out to make no difference to any fixture. What the driver actually depends on is shutting the observer down before reading the exporter, since that ends the invocation root span, so the comment now says that and cites the mutation that shows it. The observer was also shut down twice, once in each finally. --- tests/conformance/test_observability.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 6aefdc8..7786d0f 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -6752,12 +6752,11 @@ async def _run_typed_event_with_span_case(case: Mapping[str, Any]) -> None: exporter = InMemorySpanExporter() observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter)) try: - # Deliberately NOT `_invoke_typed_fixture`: that helper removes the - # observer handles BEFORE `drain()`, so the invocation-completed event - # never reaches the observer and the root span is left open. It never - # mattered for the typed-event fixtures because none of them reads - # spans; here it dropped `openarmature.invocation` from the exporter - # entirely, leaving only the node and LLM spans. + # Shut the observer down BEFORE reading the exporter: that is what ends + # the `openarmature.invocation` root span, so asserting first sees only + # the node and LLM spans and the span_tree root lookup fails on harness + # ordering rather than on behaviour. Verified by moving it after the + # assertions, which fails with "invocation root span missing". try: final, _exc = await _invoke_typed_fixture( case, collectors, graph, state_cls, extra_observer=observer @@ -6787,8 +6786,10 @@ async def _run_typed_event_with_span_case(case: Mapping[str, Any]) -> None: _assert_invariants_recognized(case, _MALFORMED_COUNTER_INVARIANTS, "malformed-counter") finally: + # `observer.shutdown()` already ran in the inner finally above, on every + # path including the failure one; repeating it here would be a second + # shutdown of the same provider. await provider.aclose() - observer.shutdown() async def _run_typed_event_chain_cases(spec: Mapping[str, Any], *, expect_failure: bool = False) -> None: From 26f2bf94116f71429da2c9f80216aa497ecfc144 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 18:06:28 -0700 Subject: [PATCH 7/8] Fix the adversarial review's findings Correctness, in the code this PR added: - The comment recording one of 149's invariants as an unfalsifiable negative control was wrong. It reasoned about Response.usage, which is non-optional, but the invariant is about the EVENT, and LlmCompletionEvent.usage is nullable. Setting it to None is accepted and 149 goes red, so the claim is live like the other five. - The omitted arm of the 148 guard skipped any Generation whose usage was None, so declared on its own it was satisfied by an implementation emitting no usage record at all. It now anchors on record presence itself rather than relying on the sibling arm being co-declared, since declaration lives in the spec repo. - The new model parameter outranked the fixture's own calls_llm.model, so a driver's blanket default would silently replace a request model a case declared. It now sits below the declaration and above the mock fallback, which is what both adjacent comments already claimed. Structural, where a green run was proving less than it appeared: - Nothing cross-checked the two sets that activate a Langfuse fixture, so a half-done activation ran nowhere while the coverage guard counted it as covered and the skip message asserted it was tested elsewhere. - A guard keyed on an invariant name goes dead the moment the fixture stops declaring that name, which a spec-side rename does routinely. The names whose claim rests only on an invariant are now pinned to the fixture that must declare them. - The per-trace drift check counted a MENTION as evaluation, so gutting a guard to `if name: pass` left it green. It now requires a non-inert body, treats a bare early return as inert, resolves the local-binding idiom, and carries an explicit set for the one arm that is deliberately no-op. Three comments claimed 149's driver was the only one reading the typed-event and span halves together. That is false, and two of them sat within 25 lines of an entry contradicting them: the token-budget driver reads both. Running 149 through it shows what actually stops it, which is that it binds the request model from the mock response and validates invariants against families that reject 149's names. The deferral text this PR deleted had it right; the replacement did not. Also: the callee walk selects by signature rather than an _assert_ name prefix, the invariant-name extraction is one implementation rather than two copies that must agree, the 149 driver reports the exception it captured and drops a nested try for its sibling's flat shape, and both span-attribute comparators share one helper. --- tests/conformance/test_harness_fidelity.py | 232 +++++++++++++++--- tests/conformance/test_observability.py | 142 ++++++----- .../test_observability_langfuse.py | 31 ++- 3 files changed, 294 insertions(+), 111 deletions(-) diff --git a/tests/conformance/test_harness_fidelity.py b/tests/conformance/test_harness_fidelity.py index e503837..b599de8 100644 --- a/tests/conformance/test_harness_fidelity.py +++ b/tests/conformance/test_harness_fidelity.py @@ -10,7 +10,7 @@ import ast import inspect -from typing import Any +from typing import Any, cast import pytest @@ -96,49 +96,126 @@ def test_every_dispatched_driver_is_registered_or_explicitly_unguarded() -> None def _invariant_names_read_by(func: object, param: str) -> set[str]: """String literals `func` looks up on the mapping named `param`.""" - tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] - names: set[str] = set() - for node in ast.walk(tree): - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "get" - and isinstance(node.func.value, ast.Name) - and node.func.value.id == param - and node.args - and isinstance(node.args[0], ast.Constant) - and isinstance(node.args[0].value, str) - ): - names.add(node.args[0].value) - elif ( - isinstance(node, ast.Subscript) - and isinstance(node.value, ast.Name) - and node.value.id == param - and isinstance(node.slice, ast.Constant) - and isinstance(node.slice.value, str) - ): - names.add(node.slice.value) - return names + return _invariant_names_read_by_node( + ast.parse(inspect.getsource(func)), # pyright: ignore[reportArgumentType] + param, + ) -def _assert_helpers_called_by(func: object, module: object) -> list[object]: - """Module-level `_assert_*` functions `func` calls directly.""" +def _invariant_helpers_called_by(func: object, module: object) -> list[object]: + """Module-level callees of `func` that take an `expected_invariants` param.""" # One level, deliberately. Reading a guard's own body misses names it # delegates to a helper, which is not hypothetical: moving 148's two # invariants into `_assert_generation_usage_omission` made the check below # blind to them the moment it was written. Deeper recursion is the # transitive-derivation problem tracked separately; one level covers the # delegate-to-a-sibling shape that actually occurs here. + # + # Selected by SIGNATURE, not by an `_assert_` name prefix. A prefix filter + # silently excludes a future guard spelled `_check_...`, and excluding it + # from the walk is the fail-open direction this whole check exists to close. tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] found: list[object] = [] for node in ast.walk(tree): - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): - helper = getattr(module, node.func.id, None) - if node.func.id.startswith("_assert_") and callable(helper) and helper not in found: - found.append(helper) + if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)): + continue + helper = getattr(module, node.func.id, None) + if not callable(helper) or helper in found: + continue + try: + params = inspect.signature(helper).parameters + except (TypeError, ValueError): # pragma: no cover - builtins + continue + if "expected_invariants" in params: + found.append(helper) return found +def _invariant_names_evaluated_by(func: object, param: str) -> set[str]: + """Names from `_invariant_names_read_by` whose `if` body actually does work.""" + # A name is MENTIONED when `if invariants.get("x"):` appears; it is + # EVALUATED when that branch asserts something. `_assert_trace` contains a + # deliberate mention-only arm (`trace_id_equals_invocation_id`, whose body is + # a comment and `pass`), so counting mentions as evaluation lets a guard be + # gutted to `if x: pass` while every check here stays green. + tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] + # Guards spell the lookup two ways: inline in the test (`if inv.get("x"):`) + # and bound to a local first (`x = inv.get("x")` ... `if x:`), which is what + # a guard reading several names for one early-return does. Resolve the + # binding so the check does not quietly mandate one of the two idioms. + bound: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + names = _invariant_names_read_by_node(node.value, param) + if len(names) == 1: + bound[node.targets[0].id] = next(iter(names)) + + evaluated: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + body = [s for s in node.body if not _is_inert(s)] + if not body: + continue + evaluated |= _invariant_names_read_by_node(node.test, param) + evaluated |= {bound[n.id] for n in ast.walk(node.test) if isinstance(n, ast.Name) and n.id in bound} + return evaluated + + +def _is_inert(stmt: ast.stmt) -> bool: + """A statement that asserts nothing about the invariant.""" + # A BARE `return` counts as inert, which is not a nicety: a guard reading + # several names typically opens with `if not a and not b: return`, and that + # arm mentions every name while checking none of them. Counting it as + # evaluation let the gutted-to-`pass` mutant survive this whole check. + if isinstance(stmt, ast.Pass): + return True + if isinstance(stmt, ast.Return) and stmt.value is None: + return True + return ( + isinstance(stmt, ast.Expr) + and isinstance(stmt.value, ast.Constant) + and isinstance(stmt.value.value, str) + ) + + +def _invariant_names_read_by_node(node: ast.AST, param: str) -> set[str]: + """The `_invariant_names_read_by` extraction, over an arbitrary subtree.""" + names: set[str] = set() + for sub in ast.walk(node): + if ( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Attribute) + and sub.func.attr == "get" + and isinstance(sub.func.value, ast.Name) + and sub.func.value.id == param + and sub.args + and isinstance(sub.args[0], ast.Constant) + and isinstance(sub.args[0].value, str) + ): + names.add(sub.args[0].value) + elif ( + isinstance(sub, ast.Subscript) + and isinstance(sub.value, ast.Name) + and sub.value.id == param + and isinstance(sub.slice, ast.Constant) + and isinstance(sub.slice.value, str) + ): + names.add(sub.slice.value) + return names + + +# Per-trace invariant names whose arm is deliberately a no-op, with the reason. +# Listed so the stale check can tell "documented as inert" from "silently gutted". +_DOCUMENTARY_PER_TRACE_INVARIANTS = { + # §8.4.1 says trace.id == invocation_id, and there is no accessor for the + # invocation_id from outside the observer, so the claim degenerates to + # "trace.id matches the UUIDv4 pattern" -- already asserted via the + # `` placeholder on the id itself. + "trace_id_equals_invocation_id", +} + + def _span_dependent_invariants_in(func: object) -> set[str]: """Invariant names whose `if invariants.get(...)` body reads the span set.""" tree = ast.parse(inspect.getsource(func)) # pyright: ignore[reportArgumentType] @@ -207,27 +284,108 @@ def test_a_zero_span_run_cannot_satisfy_a_span_absence_claim(invariant: str, dri drive({"expected": {"invariants": {invariant: True}}}) -def test_per_trace_invariants_covers_everything_assert_trace_reads() -> None: +def test_per_trace_invariants_matches_what_the_per_trace_guards_evaluate() -> None: # `_assert_multi_traces` filters the fixture's invariants down to # `_PER_TRACE_INVARIANTS` before delegating. A name `_assert_trace` checks # but that is absent from the set is therefore dropped on that path, and the # fixture declaring it still passes. entry = langfuse_runner._assert_trace # noqa: SLF001 - helpers = _assert_helpers_called_by(entry, langfuse_runner) + helpers = _invariant_helpers_called_by(entry, langfuse_runner) # Non-vacuity on the callee walk: `_assert_trace` delegates today, so # finding none means the walk broke and the check silently narrowed back to # the entry point's own body. - assert helpers, "found no _assert_* helpers called by _assert_trace; the callee walk is broken" + assert helpers, "found no invariant helpers called by _assert_trace; the callee walk is broken" read: set[str] = set() + evaluated: set[str] = set() for func in [entry, *helpers]: read |= _invariant_names_read_by(func, "expected_invariants") + evaluated |= _invariant_names_evaluated_by(func, "expected_invariants") assert read, "parsed no invariant lookups out of the per-trace guards, so this compared nothing" + assert evaluated, "parsed no EVALUATED invariant arms; the body-triviality filter is broken" missing = sorted(read - set(langfuse_runner._PER_TRACE_INVARIANTS)) # noqa: SLF001 assert not missing, ( f"the per-trace guards check {missing}, but `_PER_TRACE_INVARIANTS` omits them, so the " f"multi-trace runner discards those claims silently." ) - # Stale names are the other direction: a set entry nothing reads means the - # multi-trace path forwards a claim no guard evaluates. - stale = sorted(set(langfuse_runner._PER_TRACE_INVARIANTS) - read) # noqa: SLF001 - assert not stale, f"`_PER_TRACE_INVARIANTS` names invariants no per-trace guard reads: {stale}" + # The other direction, and on EVALUATED rather than merely mentioned: a set + # entry whose arm is `if name: pass` reads as covered while asserting + # nothing, so gutting a guard to its `if` would otherwise stay green. + stale = sorted( + set(langfuse_runner._PER_TRACE_INVARIANTS) # noqa: SLF001 + - evaluated + - _DOCUMENTARY_PER_TRACE_INVARIANTS + ) + assert not stale, ( + f"`_PER_TRACE_INVARIANTS` names invariants no per-trace guard evaluates: {stale}. Either " + f"the guard was gutted, or the arm is deliberately inert and belongs in " + f"`_DOCUMENTARY_PER_TRACE_INVARIANTS` with its reason." + ) + + +def test_langfuse_harness_fixtures_are_all_driven_by_the_langfuse_runner() -> None: + # Activating a Langfuse-mapping fixture takes two independent edits in two + # files: adding it to `_LANGFUSE_HARNESS_FIXTURES` here, which makes the OTel + # runner skip it, and to `_LANGFUSE_FIXTURES` there, which makes the Langfuse + # runner parametrize it. Do only the first and the fixture runs NOWHERE: the + # OTel runner skips it saying it is tested by the sibling, the sibling never + # collects it, and the coverage guard counts it as accounted because the + # skip set is unioned into `accounted`. One green suite, zero assertions, + # and a skip message actively claiming coverage. + skipped_here = set(otel_runner._LANGFUSE_HARNESS_FIXTURES) # noqa: SLF001 + driven_there = set(langfuse_runner._LANGFUSE_FIXTURES) # noqa: SLF001 + assert skipped_here, "the skip set is empty, so the check below compared nothing" + undriven = sorted(skipped_here - driven_there) + assert not undriven, ( + f"{undriven} are skipped by the OTel runner as 'fixture-tested by the Langfuse " + f"harness', but the Langfuse runner does not collect them, so they run nowhere while " + f"the coverage guard counts them as accounted. Add them to `_LANGFUSE_FIXTURES`." + ) + + +# Invariant names the Langfuse runner IMPLEMENTS, mapped to the fixture whose +# claim rests on them. Most invariants in this corpus are documentary -- they +# restate what a concrete directive already pins -- so a blanket "every declared +# name must be read" guard would fail the ~50 working exactly as intended. +# These are the opposite: the fixture's `langfuse_trace` block CANNOT express +# the claim, so the invariant is the only thing asserting it. +_LOAD_BEARING_LANGFUSE_INVARIANTS = { + # 148: `usage` is compared by iterating the EXPECTED keys, so an omitted + # `input` key -- the whole claim -- is invisible to the directive. + "generation_usage_input_omitted_when_prompt_tokens_null": ( + "148-langfuse-generation-usage-omits-input-on-null-counter" + ), + "generation_usage_output_and_total_present_when_sound": ( + "148-langfuse-generation-usage-omits-input-on-null-counter" + ), + # 156 declares no `level` at all, so this is its only expression. 155 pins + # `level: ERROR` concretely, which is why only the 156 direction is here. + "no_warning_level_under_budget": "156-langfuse-token-budget-under-budget-flag-false", +} + + +def test_load_bearing_langfuse_invariants_are_still_declared() -> None: + # A guard keyed on an invariant name goes dark the moment the fixture stops + # declaring that name -- a spec-side rename is the routine way that happens, + # and a pin bump brings those. Nothing else catches it: the guard early- + # returns, the `langfuse_trace` block passes on its own, the + # assert-something check is satisfied by that block's presence, and the + # unknown-directive check never descends into `expected.invariants`. The + # fixture reports green while asserting nothing it was activated for. + import yaml # noqa: PLC0415 + + declared: dict[str, set[str]] = {} + for path in langfuse_runner._fixture_paths(): # noqa: SLF001 + spec = cast("dict[str, Any]", yaml.safe_load(path.read_text())) + cases = cast("list[dict[str, Any]]", spec.get("cases") or [spec]) + for case in cases: + expected = cast("dict[str, Any]", case.get("expected") or {}) + for name in cast("dict[str, Any]", expected.get("invariants") or {}): + declared.setdefault(name, set()).add(path.stem) + assert declared, "parsed no invariants out of the corpus, so this compared nothing" + + for name, fixture in sorted(_LOAD_BEARING_LANGFUSE_INVARIANTS.items()): + assert fixture in declared.get(name, set()), ( + f"{fixture} no longer declares {name!r}, but the Langfuse runner still implements " + f"a guard keyed on that name. The guard is now dead and the fixture's claim rests " + f"on nothing. Re-point the guard at the fixture's new spelling." + ) diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 7786d0f..22b8fdb 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -676,7 +676,7 @@ def _reject_unsupported_capability_gate(fixture_id: str, spec: Mapping[str, Any] ), "_run_rerank_fixture": frozenset({"span_tree", "metrics", "invariants", "observers", "langfuse_trace"}), "_run_token_budget_fixture": frozenset({"span_tree", "metrics", "invariants", "observers"}), - # 149's driver: the only one reading the typed-event and span halves together. + # 149's driver: reads the typed-event and span halves off one invocation. "_run_typed_event_with_span_cases": frozenset({"span_tree", "observers", "invariants"}), # `invariants` is documentary here; `observers` is deliberately ABSENT because # _run_llm_payload_case reads span_tree ONLY. Claiming observers is what let @@ -901,7 +901,7 @@ async def test_observability_fixture(fixture_path: Path) -> None: }: await _run_token_budget_fixture(spec) elif fixture_id == "149-malformed-wire-counter-nulled-through-mapping-to-event-and-span": - # The only fixture asserting the typed event AND the span from one run. + # Needs the request model bound independently of the mock; see the driver. await _run_typed_event_with_span_cases(spec) elif fixture_id in { "092-tool-call-event-dispatch", @@ -4524,10 +4524,12 @@ def _walk(expected_entries: list[dict[str, Any]]) -> None: # ``attributes:`` block — exact match per key. for k, v in cast("dict[str, Any]", entry.get("attributes") or {}).items(): actual: Any = attrs.get(k) - # OTel attribute arrays come back as tuples; normalize. - if isinstance(v, list) and isinstance(actual, tuple): - actual = list(cast("tuple[Any, ...]", actual)) - assert actual == v, f"span {name!r} attribute {k!r} mismatch: expected {v!r}, got {actual!r}" + # Shared with the span-tree walker: one fixture directive + # (`attributes:`) is consumed by two comparators in this file, + # so a fix to attribute matching must not have two landing sites. + assert _attr_equal(actual, v), ( + f"span {name!r} attribute {k!r} mismatch: expected {v!r}, got {actual!r}" + ) # ``attributes_absent:`` list of names that MUST NOT appear. absent = entry.get("attributes_absent") if absent: @@ -6296,16 +6298,18 @@ def _build_simple_llm_graph( # §5.5.7 -- 068 needs this to differ from the provider-returned # response_model), else the model the first mock response reports # (050-056 path), else a default. - # An explicit `model` from the caller wins over the mock fallback. Fixtures - # that assert `gen_ai.request.model` and `gen_ai.response.model` DIFFER (144, - # 149) declare no `calls_llm.model`, so the mock fallback below would bind - # the request model to the response model and collapse the distinction the - # fixture exists to draw. `_run_llm_payload_case`, which drives 144, resolves - # this by hardcoding "test-model"; this parameter is that same convention - # made available to the other drivers rather than duplicated again. + # A caller-supplied `model` sits BELOW the fixture's own declaration and above + # the mock fallback. Fixtures asserting that `gen_ai.request.model` and + # `gen_ai.response.model` DIFFER (144, 149) declare no `calls_llm.model`, so + # the mock fallback would bind the request model to the response model and + # collapse the distinction they exist to draw; `_run_llm_payload_case`, which + # drives 144, resolves that by hardcoding "test-model", and this parameter is + # the same convention made reusable. It must NOT outrank `calls_llm.model`: + # a driver's blanket default would then silently replace a request model a + # case declared, and any case not pinning the model attribute stays green. bound_model = ( - model - or cast("str | None", calls_llm_spec.get("model")) + cast("str | None", calls_llm_spec.get("model")) + or model or _mock_model_from_first_response(case) or "test-model" ) @@ -6698,14 +6702,15 @@ async def _run_typed_event_cases(spec: Mapping[str, Any], *, expect_failure: boo # instead of nulling it fails `contains_event`; emitting the input-usage span # attribute anyway fails `attributes_absent`. Both directives are live. # -# ONE name is a negative control here, and is recorded as such rather than left -# looking verified: `event_usage_is_present_record_of_nulls_not_null_record` -# cannot be violated by this implementation at all, because `Response.usage` is -# typed `Usage` rather than `Usage | None` (llm/response.py). A null usage record -# is unrepresentable, so pydantic rejects the mutation before any assertion runs. -# The fixture argues the nested `usage:` mapping discriminates a present record -# of nulls from a null record, which is true and useful for an implementation -# whose usage field is nullable; it just cannot fail for ours. +# `event_usage_is_present_record_of_nulls_not_null_record` is live like the rest. +# It was briefly recorded here as an unfalsifiable negative control, on the +# grounds that `Response.usage` is typed `Usage` rather than `Usage | None`. That +# reasoned about the wrong layer: the invariant and the fixture's contains_event +# block are about the EVENT, and `LlmCompletionEvent.usage` IS nullable +# (graph/events.py). Setting `usage=None` where the event is built +# (llm/providers/openai.py) is accepted and fails 149's contains_event, so the +# nested `usage:` mapping does discriminate a present record of nulls from a +# null record, exactly as the fixture argues. _MALFORMED_COUNTER_INVARIANTS = { # Case 1: one malformed wire counter ("abc"). "malformed_wire_prompt_tokens_nulled_before_event_and_span", @@ -6721,13 +6726,20 @@ async def _run_typed_event_cases(spec: Mapping[str, Any], *, expect_failure: boo async def _run_typed_event_with_span_cases(spec: Mapping[str, Any]) -> None: """Fixture 149: cases asserting the SAME null-counter claim on the typed event AND the OTel span, from one invocation.""" - # 149 declares `observers.contains_event` and `span_tree` together, and no - # other driver serves both: the typed-event driver reads observers only, the - # LLM-payload driver reads span_tree only. Routing it to either silently - # drops half the fixture, which is what the earlier deferral recorded and - # what an attempted re-route then repeated in the other direction. The whole - # point of the fixture is that the nulling happens BEFORE the fan-out to both - # surfaces, so both must be read off one run. + # 149 declares `observers.contains_event` and `span_tree` together, and the + # nulling it pins happens BEFORE the fan-out to both, so both must come off + # ONE invocation. The typed-event driver reads observers only and the + # LLM-payload driver reads span_tree only, so routing 149 to either silently + # drops the other half, which an attempted re-route did in each direction. + # + # `_run_token_budget_fixture` DOES read both, so this is not "no driver + # serves both". What stops it is mechanical, and worth recording because the + # next 0101-family fixture gets triaged against it: it binds the request + # model from the mock response, so 149's `gen_ai.request.model: test-model` + # cannot match a `test-model-2026-07-15` response, and it validates + # invariants against the metrics / token-budget families, which reject 149's + # six names. Confirmed by running 149 through it -- it reaches the span-tree + # match and fails on the model attribute. for case in cast("list[dict[str, Any]]", spec["cases"]): case_name = cast("str", case["name"]) try: @@ -6751,45 +6763,45 @@ async def _run_typed_event_with_span_case(case: Mapping[str, Any]) -> None: ) exporter = InMemorySpanExporter() observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter)) + # Teardown BEFORE the assertions, matching `_run_token_budget_case`: the + # shutdown ends the `openarmature.invocation` root span, so reading the + # exporter first sees only the node and LLM spans and the span_tree root + # lookup fails on harness ordering rather than on behaviour. Verified by + # moving it after, which fails with "invocation root span missing". try: - # Shut the observer down BEFORE reading the exporter: that is what ends - # the `openarmature.invocation` root span, so asserting first sees only - # the node and LLM spans and the span_tree root lookup fails on harness - # ordering rather than on behaviour. Verified by moving it after the - # assertions, which fails with "invocation root span missing". - try: - final, _exc = await _invoke_typed_fixture( - case, collectors, graph, state_cls, extra_observer=observer - ) - finally: - observer.shutdown() - assert final is not None, "expected a non-None final state on success path" + final, exc = await _invoke_typed_fixture(case, collectors, graph, state_cls, extra_observer=observer) + finally: + await provider.aclose() + observer.shutdown() - expected = cast("dict[str, Any]", case.get("expected") or {}) - observer_expectations = cast("dict[str, Any]", expected.get("observers") or {}) - for name, expectations in observer_expectations.items(): - collector = collectors.get(name) - if collector is None: - raise AssertionError(f"fixture references unknown observer {name!r}") - _assert_observer_expectations(name, collector, cast("dict[str, Any]", expectations)) + # `_invoke_typed_fixture` CAPTURES a NodeException rather than raising, so + # surface it: an implementation that raises on a malformed counter instead + # of nulling it is one of the two behaviours 149 discriminates, and + # reporting that as a bare state assertion drops the cause. + assert final is not None, ( + f"expected a non-None final state on the success path; the invocation raised {exc!r}" + ) - spans = exporter.get_finished_spans() - expected_tree = cast("list[dict[str, Any]] | None", expected.get("span_tree")) - if expected_tree is not None: - inv_root = next( - (s for s in spans if s.name == "openarmature.invocation" and cast("Any", s.parent) is None), - None, - ) - assert inv_root is not None, f"invocation root span missing; got {[s.name for s in spans]}" - _assert_span_tree_matches(spans, [inv_root], expected_tree) - _assert_error_span_extras(spans, expected_tree) + expected = cast("dict[str, Any]", case.get("expected") or {}) + observer_expectations = cast("dict[str, Any]", expected.get("observers") or {}) + for name, expectations in observer_expectations.items(): + collector = collectors.get(name) + if collector is None: + raise AssertionError(f"fixture references unknown observer {name!r}") + _assert_observer_expectations(name, collector, cast("dict[str, Any]", expectations)) - _assert_invariants_recognized(case, _MALFORMED_COUNTER_INVARIANTS, "malformed-counter") - finally: - # `observer.shutdown()` already ran in the inner finally above, on every - # path including the failure one; repeating it here would be a second - # shutdown of the same provider. - await provider.aclose() + spans = exporter.get_finished_spans() + expected_tree = cast("list[dict[str, Any]] | None", expected.get("span_tree")) + if expected_tree is not None: + inv_root = next( + (s for s in spans if s.name == "openarmature.invocation" and cast("Any", s.parent) is None), + None, + ) + assert inv_root is not None, f"invocation root span missing; got {[s.name for s in spans]}" + _assert_span_tree_matches(spans, [inv_root], expected_tree) + _assert_error_span_extras(spans, expected_tree) + + _assert_invariants_recognized(case, _MALFORMED_COUNTER_INVARIANTS, "malformed-counter") async def _run_typed_event_chain_cases(spec: Mapping[str, Any], *, expect_failure: bool = False) -> None: diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index ee8d3c6..edb9cd2 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -166,7 +166,8 @@ # `usageDetails` map (140) where absence is naturally expressible. The # `usage` comparator here iterates the EXPECTED keys, so declining to # declare `input` asserts nothing; the case's two invariants carry the - # omission claim, and both are implemented in `_assert_trace`. + # omission claim, and both are implemented in + # `_assert_generation_usage_omission`, which `_assert_trace` calls. "148-langfuse-generation-usage-omits-input-on-null-counter", # 134 (proposal 0084): the Langfuse Generation parent resolves by the same # chain-aware §5.5 rule as the OTel span parent -- both the nested @@ -2289,11 +2290,14 @@ def _runtime_config_from_spec(config_spec: dict[str, Any] | None) -> RuntimeConf # ``correlation_id_consistent_across_traces``, etc.) stay in # ``_assert_multi_traces`` as cross-Trace checks. # -# Must list EVERY name ``_assert_trace`` reads: a name it checks but that is -# missing here is silently discarded on the multi-trace path, so a fixture -# declaring it there passes without the claim being evaluated. -# ``test_harness_fidelity.py`` derives the read set from the function body and -# fails on any omission -- ``no_warning_level_under_budget`` was one. +# Must list EVERY name ``_assert_trace`` OR A HELPER IT DELEGATES TO reads: a +# name checked but missing here is silently discarded on the multi-trace path, +# so a fixture declaring it there passes without the claim being evaluated. +# The converse also binds: every name here must be EVALUATED by one of those +# guards, or listed in ``test_harness_fidelity._DOCUMENTARY_PER_TRACE_INVARIANTS`` +# with its reason, so a guard gutted to ``if name: pass`` cannot read as live. +# ``test_harness_fidelity.py`` derives both sets by walking those bodies and +# fails either way -- ``no_warning_level_under_budget`` was a missing one. _PER_TRACE_INVARIANTS = frozenset( { "trace_id_equals_invocation_id", @@ -2444,9 +2448,18 @@ def _assert_generation_usage_omission(trace: LangfuseTrace, expected_invariants: f"vacuously; got {[(o.name, o.type) for o in trace.observations]}" ) if omitted: - carrying = [ - (g.name, g.usage) for g in generations if g.usage is not None and g.usage.input is not None - ] + # Self-anchoring: skipping a Generation whose `usage` is None would make + # this satisfied by an implementation emitting no usage record at all. + # The `present` arm below rules that out too, but only when the fixture + # happens to declare BOTH names, and declaration lives in the spec repo. + carrying: list[tuple[str | None, Any]] = [] + for gen in generations: + assert gen.usage is not None, ( + f"generation {gen.name!r} carries no usage record at all, so the omission " + f"claim would pass vacuously" + ) + if gen.usage.input is not None: + carrying.append((gen.name, gen.usage)) assert not carrying, ( f"an unreported prompt_tokens MUST be omitted from generation.usage rather than " f"rendered as null or zero; present on {carrying}" From bcad8a8862add7a01387e89e55f47c0548ca4f67 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Wed, 19 Aug 2026 21:58:08 -0700 Subject: [PATCH 8/8] Assert the Generation usage map at the SDK boundary Fixture 148 pins that a counter the provider did not report is omitted from the Generation's usage record. Its conformance assertion runs against the in-memory double, where omission and a rendered null are the same state, so it can only check the record. The adapter is what turns that record into the wire usage_details map, and nothing covered it: neutralising the guard at all four sites left the whole suite green. The new test drives the real adapter and asserts the map it hands the SDK. A zero row rides along, because the obvious tidy of that guard is a truthiness check, and a genuinely reported 0 must still render. The conformance-side proxy is unchanged and still documented as a proxy. Making the double record the translated map is the larger fix and is tracked on its own, since it changes the evidence every Langfuse fixture rests on. --- .../test_observability_langfuse_adapter.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/unit/test_observability_langfuse_adapter.py b/tests/unit/test_observability_langfuse_adapter.py index a2f2527..cddbe4b 100644 --- a/tests/unit/test_observability_langfuse_adapter.py +++ b/tests/unit/test_observability_langfuse_adapter.py @@ -22,6 +22,7 @@ from __future__ import annotations import os +from types import SimpleNamespace from typing import Annotated, Any import pytest @@ -36,6 +37,7 @@ LangfuseClient, LangfuseObserver, LangfuseSDKAdapter, + LangfuseUsage, ) @@ -403,3 +405,45 @@ async def test_adapter_against_real_langfuse_cloud() -> None: # The trace_id in the dashboard is the 32-char hex form (no dashes) # of OA's UUID4 invocation_id; strip dashes from any logged # correlation_id / invocation_id to find it. + + +@pytest.mark.parametrize( + ("usage", "expected"), + [ + # 0101: a counter the provider did not report is OMITTED from the wire + # record, not rendered as null or zero. + (LangfuseUsage(input=None, output=5, total=15), {"output": 5, "total": 15}), + (LangfuseUsage(input=7, output=5, total=15), {"input": 7, "output": 5, "total": 15}), + (LangfuseUsage(input=0, output=5, total=15), {"input": 0, "output": 5, "total": 15}), + ], +) +def test_generation_usage_details_omit_unreported_counters( + monkeypatch: pytest.MonkeyPatch, + usage: LangfuseUsage, + expected: dict[str, int], +) -> None: + # The conformance fixture for this claim (148) asserts `usage.input is None` + # on the in-memory double, where omission and a rendered null are the SAME + # state. This is the other side of that proxy: the adapter is what turns the + # record into the wire `usage_details` map, and until this existed the guard + # there could be neutralised at all four sites with the whole suite green. + # + # The zero row is the reason `if usage.input is not None` cannot become a + # truthiness check: a genuinely reported 0 MUST still render. + adapter = LangfuseSDKAdapter(_dummy_client()) + captured: dict[str, Any] = {} + + def _capture(**kwargs: Any) -> Any: + captured.update(kwargs) + + def _noop(**_kwargs: Any) -> None: + return None + + return SimpleNamespace(id="obs-1", update=_noop, end=_noop) + + monkeypatch.setattr(adapter, "_start_observation", _capture) + adapter.generation(trace_id="tr-1", name="openarmature.llm.complete", usage=usage) + + assert captured["usage_details"] == expected, ( + f"usage_details must carry exactly the reported counters; got {captured.get('usage_details')!r}" + )