From f290753386c0c58c90d95777e016cb277a14e206 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Thu, 13 Aug 2026 17:52:08 -0700 Subject: [PATCH 1/2] Publish the external dependencies this implementation requires Records the version range, the version deliberately verified, and the private upstream surface the Langfuse adapter reaches into. Spec renders a per-implementation block from these keys on its compatibility page, alongside the matrix that records what the normative text is written against; the two answer different questions and the split is why this lives here rather than there. The internals list is the source the guard parametrizes over, not a copy of it, so the published record and the enforced record are the same list. A path declared here that no longer exists upstream fails, and one the adapter imports but nobody declared fails too. verified_on is the date the pin was last deliberately moved, not the date CI last passed. A date sitting well back while the range still admits newer versions is saying something true, so nothing nudges it forward on a run that merely passes. A new check pins verified to the installed version so the published number cannot drift from the tested one. --- conformance.toml | 36 +++++++++ tests/unit/test_langfuse_sdk_internals.py | 93 ++++++++++++++++++----- 2 files changed, 108 insertions(+), 21 deletions(-) diff --git a/conformance.toml b/conformance.toml index df40a64..fcccc60 100644 --- a/conformance.toml +++ b/conformance.toml @@ -55,6 +55,42 @@ spec_pin = "v0.112.0" # than the portable suppress floor. langfuse_bound_provider_detection = true +# External dependencies this implementation requires, and the private upstream +# surface it reaches into (coord thread discuss-external-dependency-tracking). +# +# The split from openarmature.org/compatibility's matrix is deliberate: that +# matrix records what the SPEC's normative text is written against, which is one +# implementation-independent row. This records what THIS implementation requires, +# has verified, and depends on, which differs per implementation. Spec renders a +# per-implementation block from these keys. +# +# `verified_on` is the date the pin was last deliberately moved or re-verified, +# NOT the date CI last passed. A date that sits well back while `requires` still +# admits newer versions is telling the reader something true, and is not nudged +# forward by runs that merely pass. +# +# `internals` is the SOURCE the guard in tests/unit/test_langfuse_sdk_internals.py +# parametrizes over, so the published list and the enforced list cannot drift: +# adding a path here without it existing upstream fails, and losing one upstream +# fails too. +[external_dependencies.langfuse] +requires = ">=4.6,<5" +verified = "4.7.1" +verified_on = "2026-08-13" +note = "The declared range currently resolves as far as 4.14.x, well past `verified`. Every listed internal still exists there and the suite passes against it, but 4.7.1 is the version this implementation deliberately tests against. Losing one of these internals does not raise to the caller: the graph observer isolates observer errors, so an observation simply stops being emitted and a leak assertion reads clean, which is why they are guarded rather than trusted." +internals = [ + "langfuse._client.client.Langfuse._resources", + "langfuse._client.client.Langfuse._tracing_enabled", + "langfuse._client.client.Langfuse._otel_tracer", + "langfuse._client.client.Langfuse._create_remote_parent_span", + "langfuse._client.resource_manager.LangfuseResourceManager.tracer_provider", + "langfuse._client.resource_manager.LangfuseResourceManager._instances", + "langfuse._client.span.LangfuseGeneration", + "langfuse._client.span.LangfuseTool", + "langfuse._client.span.LangfuseEmbedding", + "langfuse._client.span.LangfuseRetriever", +] + # Status values: # implemented — shipped behavior matches the proposal's contract # partial — partial impl; consult `note` for what's missing diff --git a/tests/unit/test_langfuse_sdk_internals.py b/tests/unit/test_langfuse_sdk_internals.py index 95fd775..934325a 100644 --- a/tests/unit/test_langfuse_sdk_internals.py +++ b/tests/unit/test_langfuse_sdk_internals.py @@ -21,10 +21,16 @@ from __future__ import annotations +import importlib import inspect +import tomllib +from pathlib import Path +from typing import Any, cast import pytest +_CONFORMANCE_TOML = Path(__file__).resolve().parents[2] / "conformance.toml" + langfuse = pytest.importorskip("langfuse", reason="the langfuse extra is optional") @@ -64,31 +70,70 @@ def test_the_per_credential_cache_is_where_the_adapter_looks() -> None: assert isinstance(LangfuseResourceManager._instances, dict) -@pytest.mark.parametrize( - ("symbol", "spelling"), - [("_otel_tracer", "self._otel_tracer"), ("_create_remote_parent_span", "def _create_remote_parent_span")], -) -def test_the_back_dated_observation_internals_exist(symbol: str, spelling: str) -> None: - # Every provider observation carrying a start_time goes through - # _start_back_dated_observation, which needs both. Losing either does not - # raise to the caller: the graph observer isolates observer errors, so the - # observation simply stops being emitted and a leak assertion reads clean. +def _declared() -> dict[str, Any]: + # Read at collection time to parametrize, so a missing section surfaces as a + # named failure rather than a bare KeyError from inside pytest's collector. + with _CONFORMANCE_TOML.open("rb") as handle: + manifest = tomllib.load(handle) + entry = cast("dict[str, Any]", manifest.get("external_dependencies", {})).get("langfuse") + assert entry is not None, ( + "conformance.toml has no [external_dependencies.langfuse] section. It is the published " + "record of the private SDK surface this implementation depends on, and the source these " + "guards parametrize over; without it nothing checks that surface." + ) + missing = sorted({"requires", "verified", "verified_on", "internals"} - set(entry)) + assert not missing, f"[external_dependencies.langfuse] is missing required keys: {missing}" + return cast("dict[str, Any]", entry) + + +def _resolve(path: str) -> tuple[Any, str]: + """Split a dotted path into the deepest importable owner and the final name.""" + parts = path.split(".") + for cut in range(len(parts) - 1, 0, -1): + try: + owner: Any = importlib.import_module(".".join(parts[:cut])) + except ImportError: + continue + for attr in parts[cut:-1]: + owner = getattr(owner, attr) + return owner, parts[-1] + raise AssertionError(f"no importable module in {path!r}") + + +@pytest.mark.parametrize("path", _declared()["internals"]) +def test_each_declared_internal_still_exists(path: str) -> None: + # Parametrized over conformance.toml's `internals` list rather than a copy of + # it, so the PUBLISHED surface and the ENFORCED surface are the same list. + # Restating it here would let the public record drift from what is checked, + # which is the failure this whole guard exists to prevent. # - # Checked in the source rather than with hasattr, because `_otel_tracer` is an - # INSTANCE attribute and a class-level hasattr would report it missing on a - # perfectly good SDK. - assert spelling in inspect.getsource(langfuse.Langfuse), ( - f"langfuse.Langfuse.{symbol} is gone; adapter._start_back_dated_observation depends " - f"on it, and its absence surfaces only as a swallowed observer warning" + # Losing any of these does not raise to the caller: the graph observer + # isolates observer errors, so an observation simply stops being emitted and + # a leak assertion reads clean. + owner, name = _resolve(path) + if hasattr(owner, name): + return + # Instance attributes (`self._resources`, `self._otel_tracer`) are not on the + # class, so a bare hasattr would report a perfectly good SDK as broken. + source = inspect.getsource(owner) + assert f"self.{name}" in source, ( + f"{path} is gone from the installed langfuse SDK. openarmature's shipped adapter " + f"depends on it, and its absence surfaces only as a swallowed observer warning." ) -@pytest.mark.parametrize( - "name", ["LangfuseGeneration", "LangfuseTool", "LangfuseEmbedding", "LangfuseRetriever"] -) -def test_the_span_classes_the_adapter_constructs_exist(name: str) -> None: - span_module = pytest.importorskip("langfuse._client.span") - assert hasattr(span_module, name) +def test_the_declared_internals_cover_what_the_adapter_imports() -> None: + # The list is only as good as its completeness, and completeness is not + # something the per-path checks above can see. The adapter constructs four + # private span classes; an earlier version of this file guarded two. + adapter_source = inspect.getsource(importlib.import_module("openarmature.observability.langfuse.adapter")) + declared = set(_declared()["internals"]) + for name in ("LangfuseGeneration", "LangfuseTool", "LangfuseEmbedding", "LangfuseRetriever"): + if name in adapter_source: + assert any(entry.endswith(f".{name}") for entry in declared), ( + f"the adapter imports {name} but conformance.toml does not declare it, so a " + f"rename upstream would go unguarded and unpublished" + ) def test_the_installed_version_is_within_the_declared_range() -> None: @@ -99,6 +144,12 @@ def test_the_installed_version_is_within_the_declared_range() -> None: from importlib.metadata import version installed = version("langfuse") + declared = _declared() + assert installed == declared["verified"], ( + f"conformance.toml publishes langfuse {declared['verified']} as the verified version " + f"but {installed} is installed. Either move the pin deliberately and update `verified` " + f"and `verified_on`, or restore the lock; the published number must be the tested one." + ) # Regex rather than int() on the split parts: a PEP 440 two-component # pre-release such as 4.7rc1 attaches its suffix to the MINOR, so splitting # raises ValueError on a version that is inside our declared range. A bare From 89b465fe0214bf6576b8dd6ccc3d58a467c7a8b2 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Thu, 13 Aug 2026 18:30:41 -0700 Subject: [PATCH 2/2] Name one authority for the version, and parse the real imports The adapter header claimed validation against 4.7.0 while the same file cited 4.7.1 behaviour and the manifest published 4.7.1. It no longer names a version at all: the manifest entry is enforced against what is installed, a comment is not, so having two claims meant the unenforced one drifted. The completeness check now parses the adapter's actual private span imports instead of testing a hardcoded list of four names against a substring search. A hardcoded list is not a completeness check, which is the one job it had: a fifth class added later would have gone unnoticed, and a name in prose or the same name from another module would both have matched. It also asserts it found imports at all, so a pattern that stops matching fails rather than passing having read nothing. Resolving a declared path now fails by name when an intermediate segment disappears, rather than raising a bare AttributeError. The message is this guard's product, since losing one of these paths surfaces nowhere else. --- .../observability/langfuse/adapter.py | 13 +++-- tests/unit/test_langfuse_sdk_internals.py | 47 +++++++++++++++---- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/openarmature/observability/langfuse/adapter.py b/src/openarmature/observability/langfuse/adapter.py index ce33453..faf1481 100644 --- a/src/openarmature/observability/langfuse/adapter.py +++ b/src/openarmature/observability/langfuse/adapter.py @@ -1,8 +1,13 @@ # Bridges the langfuse Python SDK (v4.6+) onto the LangfuseClient -# Protocol. Validated against langfuse==4.7.0; the [langfuse] extras -# pin to `>=4.6,<5`. SDK churn before v4 (v2/v3 API removed in v4) is -# not supported — projects on v2/v3 should write their own adapter or -# upgrade. +# Protocol. The version this adapter is verified against, and the private +# SDK surface it depends on, are published in `conformance.toml` under +# `[external_dependencies.langfuse]` and enforced by +# `tests/unit/test_langfuse_sdk_internals.py`. Named there rather than +# here so there is one authority: a version in a comment goes stale +# silently, which is exactly what that manifest entry exists to prevent. +# The [langfuse] extras pin to `>=4.6,<5`. SDK churn before v4 (v2/v3 +# API removed in v4) is not supported — projects on v2/v3 should write +# their own adapter or upgrade. # # Shape mismatch the adapter handles: # - v4 has no explicit `client.trace(...)` — traces are auto-created diff --git a/tests/unit/test_langfuse_sdk_internals.py b/tests/unit/test_langfuse_sdk_internals.py index 934325a..d2784cd 100644 --- a/tests/unit/test_langfuse_sdk_internals.py +++ b/tests/unit/test_langfuse_sdk_internals.py @@ -23,6 +23,7 @@ import importlib import inspect +import re import tomllib from pathlib import Path from typing import Any, cast @@ -95,9 +96,20 @@ def _resolve(path: str) -> tuple[Any, str]: except ImportError: continue for attr in parts[cut:-1]: + # Asserted rather than left to getattr: a renamed intermediate (a + # class, say) would otherwise surface as a bare AttributeError, and + # the message IS this guard's product. Losing one of these paths shows + # up nowhere else, because the graph observer swallows observer errors. + assert hasattr(owner, attr), ( + f"{path}: {attr!r} is missing from {owner!r}, so the rest of the path cannot be " + f"resolved. openarmature's shipped adapter depends on this path." + ) owner = getattr(owner, attr) return owner, parts[-1] - raise AssertionError(f"no importable module in {path!r}") + raise AssertionError( + f"no importable module in {path!r}; the declared internal names a module that no " + f"longer exists in the installed SDK" + ) @pytest.mark.parametrize("path", _declared()["internals"]) @@ -123,17 +135,32 @@ def test_each_declared_internal_still_exists(path: str) -> None: def test_the_declared_internals_cover_what_the_adapter_imports() -> None: - # The list is only as good as its completeness, and completeness is not - # something the per-path checks above can see. The adapter constructs four - # private span classes; an earlier version of this file guarded two. + # The completeness half: the per-path checks above verify what IS declared and + # can say nothing about what the adapter depends on but nobody declared. + # + # Parsed from the actual import statements rather than matched against names + # written out here. A hardcoded list is not a completeness check: an earlier + # version enumerated four span classes, so a fifth added later would have gone + # unnoticed by the very test meant to catch that. A substring search over the + # module source would also match a name in prose, and a suffix match would + # accept the same name exported by a different module. adapter_source = inspect.getsource(importlib.import_module("openarmature.observability.langfuse.adapter")) + imported = set(re.findall(r"from\s+langfuse\._client\.span\s+import\s+(\w+)", adapter_source)) + assert imported, ( + "found no `from langfuse._client.span import ...` in the adapter. Either it stopped " + "importing private span classes, in which case the declared internals should shrink, " + "or this pattern no longer matches and the check is reading nothing." + ) declared = set(_declared()["internals"]) - for name in ("LangfuseGeneration", "LangfuseTool", "LangfuseEmbedding", "LangfuseRetriever"): - if name in adapter_source: - assert any(entry.endswith(f".{name}") for entry in declared), ( - f"the adapter imports {name} but conformance.toml does not declare it, so a " - f"rename upstream would go unguarded and unpublished" - ) + missing = sorted( + f"langfuse._client.span.{name}" + for name in imported + if f"langfuse._client.span.{name}" not in declared + ) + assert not missing, ( + f"the adapter imports {missing} but conformance.toml does not declare them, so a rename " + f"upstream would go both unguarded and unpublished" + ) def test_the_installed_version_is_within_the_declared_range() -> None: