From db95a89ba179568f33e0c1cd9d188f5bdd460021 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 17 Aug 2026 16:07:08 -0500 Subject: [PATCH 01/12] fix(converter): keep only last message from full-history agent output (HYBIM-988) --- src/splunk_ao/converter/attribute_mapping.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 64b782d3..90ec0561 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -447,6 +447,10 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: return if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] + # Full history includes all messages in the run, not just the final response. + # Keep only the last message — it is always the agent's final output. + if full_history and len(output_messages) > 1: + output_messages = [output_messages[-1]] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) From 2ad296269a97fef9c8e94d74a1b61869b8399ad1 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 17 Aug 2026 17:11:23 -0500 Subject: [PATCH 02/12] test(converter): add edge case tests for full-history agent output reduction (HYBIM-988) --- tests/test_attribute_mapping.py | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 31571bfa..1b629b81 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -418,6 +418,90 @@ def test_orchestration_output_omits_repeated_input_history() -> None: ] +def test_orchestration_full_history_with_tool_call_keeps_last_message() -> None: + # LangGraph accumulated state: user → tool-call AI (empty content) → tool response → final AI + # The first post-dedup message has empty content; the UI would show "—" without the fix. + user = {"role": "user", "content": "What is the dosage of Lisinopril?"} + ai_toolcall = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}]} + tool_resp = {"role": "tool", "content": "Lisinopril: 10mg daily", "tool_call_id": "tc1"} + ai_final = {"role": "assistant", "content": "Common dosage is 10mg once daily."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, ai_toolcall, tool_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["content"] == "Common dosage is 10mg once daily." + + +def test_orchestration_full_history_multi_turn_keeps_last_message() -> None: + # Multi-turn: output contains the full conversation history after multiple exchanges. + # Only the last message should be kept regardless of role. + user1 = {"role": "user", "content": "Hello"} + ai1 = {"role": "assistant", "content": "Hi, how can I help?"} + user2 = {"role": "user", "content": "What is Lisinopril?"} + ai2 = {"role": "assistant", "content": "Lisinopril is a blood pressure medication."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user1]}), + output=json.dumps({"messages": [user1, ai1, user2, ai2]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + + +def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> None: + # Two tool call rounds before the final answer — last message is still the only output. + user = {"role": "user", "content": "Compare Lisinopril and Amlodipine"} + tc1_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}]} + tc1_resp = {"role": "tool", "content": "Lisinopril: ACE inhibitor", "tool_call_id": "tc1"} + tc2_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}]} + tc2_resp = {"role": "tool", "content": "Amlodipine: calcium channel blocker", "tool_call_id": "tc2"} + ai_final = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker."} + + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, tc1_ai, tc1_resp, tc2_ai, tc2_resp, ai_final]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert "Amlodipine" in output_messages[0]["parts"][0]["content"] + + +def test_orchestration_non_full_history_output_not_reduced() -> None: + # Plain string output (full_history=False) — the last-message reduction must NOT fire. + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input="What is Lisinopril?", + output="Lisinopril is a blood pressure medication.", + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + + def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( name="tool-workflow", From b727b9180871f1772ec30477e7969e70ae95ce88 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 10:22:30 -0500 Subject: [PATCH 03/12] fix(converter): preserve agent and workflow outputs --- CHANGELOG.md | 7 + src/splunk_ao/converter/attribute_mapping.py | 18 ++- tests/test_attribute_mapping.py | 144 ++++++++++++++++++- 3 files changed, 158 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b750ee4..de91d4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Agent and workflow output conversion now removes only confirmed repeated input + history, preserves multiple terminal assistant messages, and reports the + standard `tool_call` finish reason when an output requests a tool and no + source finish reason is available. + ## [0.2.1] - 2026-08-07 ### Fixed diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 90ec0561..f3929270 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -268,7 +268,12 @@ def _orchestration_messages(value: Any, default_role: str) -> tuple[list[dict[st def _with_finish_reasons(messages: list[dict[str, Any]], finish_reason: str | None = None) -> list[dict[str, Any]]: for message in messages: source_finish_reason = message.get("finish_reason") - message["finish_reason"] = finish_reason or source_finish_reason or "unknown" + inferred_finish_reason = ( + "tool_call" + if any(part.get("type") == "tool_call" for part in message.get("parts", []) if isinstance(part, Mapping)) + else "unknown" + ) + message["finish_reason"] = finish_reason or source_finish_reason or inferred_finish_reason return messages @@ -445,12 +450,13 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages, full_history = _orchestration_messages(span.output, "assistant") if output_messages is None: return - if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: + if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] - # Full history includes all messages in the run, not just the final response. - # Keep only the last message — it is always the agent's final output. - if full_history and len(output_messages) > 1: - output_messages = [output_messages[-1]] + + terminal_start = len(output_messages) + while terminal_start > 0 and output_messages[terminal_start - 1].get("role") == "assistant": + terminal_start -= 1 + output_messages = output_messages[terminal_start:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 1b629b81..5e0b01de 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -1,6 +1,6 @@ import json from types import SimpleNamespace -from typing import cast +from typing import Any, cast from uuid import uuid4 import pytest @@ -418,11 +418,134 @@ def test_orchestration_output_omits_repeated_input_history() -> None: ] +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_preserves_all_terminal_assistant_messages( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + # Given: a full-history result with two terminal assistant outputs after the exact input history. + user = {"role": "user", "content": "Give me two alternatives"} + first = {"role": "assistant", "content": "First alternative"} + second = {"role": "assistant", "content": "Second alternative"} + span_kwargs: dict[str, Any] = { + "name": "planner", + "input": json.dumps({"messages": [user]}), + "output": json.dumps({"messages": [user, first, second]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.planner + + # When: the orchestration content is converted. + attrs = build_span_attributes(span_type(**span_kwargs)) + + # Then: the repeated input prefix is removed without reducing the terminal outputs to one message. + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "First alternative", finish_reason="unknown"), + _text_message("assistant", "Second alternative", finish_reason="unknown"), + ] + + +@pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) +def test_orchestration_full_history_removes_confirmed_input_prefix( + span_type: type[WorkflowSpan] | type[AgentSpan], +) -> None: + # Given: the input is the complete history immediately before the final assistant response. + user = {"role": "user", "content": "What is the dosage?"} + tool_call = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + tool_response = {"role": "tool", "content": "10 mg daily", "tool_call_id": "call-1"} + final = {"role": "assistant", "content": "The common dosage is 10 mg daily."} + input_history = [user, tool_call, tool_response] + span_kwargs: dict[str, Any] = { + "name": "healthcare", + "input": json.dumps({"messages": input_history}), + "output": json.dumps({"messages": [*input_history, final]}), + } + if span_type is AgentSpan: + span_kwargs["agent_type"] = AgentType.default + + # When: the orchestration content is converted. + attrs = build_span_attributes(span_type(**span_kwargs)) + + # Then: only the newly produced terminal response is exported as output. + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "The common dosage is 10 mg daily.", finish_reason="unknown") + ] + + +def test_orchestration_infers_tool_call_finish_reason_when_absent() -> None: + # Given: a workflow emits an assistant tool call without a source finish reason. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: the standard tool-call finish reason is inferred from the output part. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "tool_call" + assert output_message["parts"][0]["type"] == "tool_call" + + +def test_orchestration_tool_response_uses_unknown_finish_reason() -> None: + # Given: a workflow emits a tool response, which has no model-generation finish reason. + output = { + "update": {"messages": [{"role": "tool", "content": {"dosage": "10 mg daily"}, "tool_call_id": "call-1"}]} + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: its valid tool response structure is preserved without inventing a model finish reason. + output_message = json.loads(attrs["gen_ai.output.messages"])[0] + assert output_message["finish_reason"] == "unknown" + assert output_message["parts"] == [ + {"type": "tool_call_response", "id": "call-1", "response": {"dosage": "10 mg daily"}} + ] + + +def test_orchestration_preserves_explicit_finish_reason_for_tool_call() -> None: + # Given: the source supplies its own finish reason for a message containing a tool call. + output = { + "update": { + "messages": [ + { + "role": "assistant", + "content": "", + "finish_reason": "provider_tool_calls", + "tool_calls": [{"id": "call-1", "function": {"name": "search", "arguments": '{"query":"dosage"}'}}], + } + ] + } + } + + # When: the workflow output is converted. + attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) + + # Then: inference does not overwrite source telemetry. + assert json.loads(attrs["gen_ai.output.messages"])[0]["finish_reason"] == "provider_tool_calls" + + def test_orchestration_full_history_with_tool_call_keeps_last_message() -> None: # LangGraph accumulated state: user → tool-call AI (empty content) → tool response → final AI # The first post-dedup message has empty content; the UI would show "—" without the fix. user = {"role": "user", "content": "What is the dosage of Lisinopril?"} - ai_toolcall = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}]} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "rag_search", "arguments": '{"query":"Lisinopril dosage"}'}}], + } tool_resp = {"role": "tool", "content": "Lisinopril: 10mg daily", "tool_call_id": "tc1"} ai_final = {"role": "assistant", "content": "Common dosage is 10mg once daily."} @@ -466,11 +589,22 @@ def test_orchestration_full_history_multi_turn_keeps_last_message() -> None: def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> None: # Two tool call rounds before the final answer — last message is still the only output. user = {"role": "user", "content": "Compare Lisinopril and Amlodipine"} - tc1_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}]} + tc1_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}], + } tc1_resp = {"role": "tool", "content": "Lisinopril: ACE inhibitor", "tool_call_id": "tc1"} - tc2_ai = {"role": "assistant", "content": "", "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}]} + tc2_ai = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc2", "function": {"name": "search", "arguments": '{"query":"Amlodipine"}'}}], + } tc2_resp = {"role": "tool", "content": "Amlodipine: calcium channel blocker", "tool_call_id": "tc2"} - ai_final = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker."} + ai_final = { + "role": "assistant", + "content": "Lisinopril is an ACE inhibitor; Amlodipine is a calcium channel blocker.", + } span = AgentSpan( name="Agent", From 766d70795004e16d55f05e1b6df65a5a9524a612 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 10:46:00 -0500 Subject: [PATCH 04/12] test(converter): add parallel tool node output coverage --- tests/test_attribute_mapping.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 5e0b01de..a0864e90 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -620,20 +620,24 @@ def test_orchestration_full_history_multiple_tool_rounds_keeps_last_message() -> assert "Amlodipine" in output_messages[0]["parts"][0]["content"] -def test_orchestration_non_full_history_output_not_reduced() -> None: - # Plain string output (full_history=False) — the last-message reduction must NOT fire. - span = AgentSpan( - name="Agent", - agent_type=AgentType.default, - input="What is Lisinopril?", - output="Lisinopril is a blood pressure medication.", +def test_orchestration_message_container_without_input_prefix_match_not_reduced() -> None: + # A WorkflowSpan (e.g. ToolNode) returning multiple messages whose output does NOT + # prefix-match the input state — dedup gate never fires, so all messages must survive. + # This is the parallel-tool-call shape: two ToolMessages from a single ToolNode invocation. + tool_msg_1 = {"role": "tool", "content": "Lisinopril: 10 mg daily", "tool_call_id": "tc1"} + tool_msg_2 = {"role": "tool", "content": "Amlodipine: 5 mg daily", "tool_call_id": "tc2"} + span = WorkflowSpan( + name="tools", + input=json.dumps({"messages": [{"role": "user", "content": "Compare dosages"}]}), + output=json.dumps({"messages": [tool_msg_1, tool_msg_2]}), ) attrs = build_span_attributes(span) output_messages = json.loads(attrs["gen_ai.output.messages"]) - assert len(output_messages) == 1 - assert output_messages[0]["parts"][0]["content"] == "Lisinopril is a blood pressure medication." + assert len(output_messages) == 2 + assert output_messages[0]["parts"][0]["response"] == "Lisinopril: 10 mg daily" + assert output_messages[1]["parts"][0]["response"] == "Amlodipine: 5 mg daily" def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: From ce3d54c62dc052e1c534d7a1a43d8a134a8b30a9 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 15:37:47 -0500 Subject: [PATCH 05/12] fix(converter): simplify full-history trim to last message and add edge case tests --- CHANGELOG.md | 10 ++-- src/splunk_ao/converter/attribute_mapping.py | 7 +-- tests/test_attribute_mapping.py | 57 +++++++++++++++++--- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de91d4bc..7f33e391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Agent and workflow output conversion now removes only confirmed repeated input - history, preserves multiple terminal assistant messages, and reports the - standard `tool_call` finish reason when an output requests a tool and no - source finish reason is available. +- Agent and workflow output conversion now removes confirmed repeated input + history and keeps only the last message as the terminal output; intermediate + tool-call and tool-response messages are no longer included in + `gen_ai.output.messages` for full-history spans. The `tool_call` finish reason + is inferred when an output requests a tool and no source finish reason is + available. ## [0.2.1] - 2026-08-07 diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index f3929270..03be3c3e 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -451,12 +451,7 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: if output_messages is None: return if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: - output_messages = output_messages[len(input_messages) :] - - terminal_start = len(output_messages) - while terminal_start > 0 and output_messages[terminal_start - 1].get("role") == "assistant": - terminal_start -= 1 - output_messages = output_messages[terminal_start:] + output_messages = output_messages[len(input_messages) :][-1:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index a0864e90..f9151e00 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -419,10 +419,9 @@ def test_orchestration_output_omits_repeated_input_history() -> None: @pytest.mark.parametrize("span_type", [WorkflowSpan, AgentSpan]) -def test_orchestration_full_history_preserves_all_terminal_assistant_messages( +def test_orchestration_full_history_keeps_last_terminal_message( span_type: type[WorkflowSpan] | type[AgentSpan], ) -> None: - # Given: a full-history result with two terminal assistant outputs after the exact input history. user = {"role": "user", "content": "Give me two alternatives"} first = {"role": "assistant", "content": "First alternative"} second = {"role": "assistant", "content": "Second alternative"} @@ -434,13 +433,10 @@ def test_orchestration_full_history_preserves_all_terminal_assistant_messages( if span_type is AgentSpan: span_kwargs["agent_type"] = AgentType.planner - # When: the orchestration content is converted. attrs = build_span_attributes(span_type(**span_kwargs)) - # Then: the repeated input prefix is removed without reducing the terminal outputs to one message. assert json.loads(attrs["gen_ai.output.messages"]) == [ - _text_message("assistant", "First alternative", finish_reason="unknown"), - _text_message("assistant", "Second alternative", finish_reason="unknown"), + _text_message("assistant", "Second alternative", finish_reason="unknown") ] @@ -640,6 +636,55 @@ def test_orchestration_message_container_without_input_prefix_match_not_reduced( assert output_messages[1]["parts"][0]["response"] == "Amlodipine: 5 mg daily" +def test_orchestration_full_history_ends_on_tool_message_keeps_last() -> None: + # return_direct=True tool: run ends on a tool response, no final assistant message. + # The dedup gate fires (prefix matches) but the last message is a tool, not assistant. + # Must return the tool message rather than an empty list. + user = {"role": "user", "content": "Get patient P001"} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "get_patient", "arguments": '{"id":"P001"}'}}], + } + tool_resp = {"role": "tool", "content": "George Rivera, Lisinopril 10mg", "tool_call_id": "tc1"} + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user, ai_toolcall]}), + output=json.dumps({"messages": [user, ai_toolcall, tool_resp]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["parts"][0]["response"] == "George Rivera, Lisinopril 10mg" + + +def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> None: + # interrupt_before=["tools"]: run ends on a tool-call AIMessage with empty content. + # The last message is assistant role but content="" — must not return empty list. + user = {"role": "user", "content": "Search for Lisinopril"} + ai_toolcall = { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tc1", "function": {"name": "search", "arguments": '{"query":"Lisinopril"}'}}], + } + span = AgentSpan( + name="Agent", + agent_type=AgentType.default, + input=json.dumps({"messages": [user]}), + output=json.dumps({"messages": [user, ai_toolcall]}), + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["finish_reason"] == "tool_call" + + def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( name="tool-workflow", From 86c3be1a5912561a59118c4340a01e1b9fa48ae5 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 22:45:36 -0500 Subject: [PATCH 06/12] refactor(converter): drop unrelated tool_call finish-reason inference and add WorkflowSpan trim test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the tool_call finish-reason inference added to _with_finish_reasons — it is unrelated to HYBIM-988 and widens blast radius to LLM spans. Updates CHANGELOG and adds WorkflowSpan coverage for the full-history trim path. Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 4 +--- src/splunk_ao/converter/attribute_mapping.py | 7 +----- tests/test_attribute_mapping.py | 25 ++++++++++++++++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f33e391..0f1bf14a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Agent and workflow output conversion now removes confirmed repeated input history and keeps only the last message as the terminal output; intermediate tool-call and tool-response messages are no longer included in - `gen_ai.output.messages` for full-history spans. The `tool_call` finish reason - is inferred when an output requests a tool and no source finish reason is - available. + `gen_ai.output.messages` for full-history spans. ## [0.2.1] - 2026-08-07 diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 03be3c3e..d14b8f6e 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -268,12 +268,7 @@ def _orchestration_messages(value: Any, default_role: str) -> tuple[list[dict[st def _with_finish_reasons(messages: list[dict[str, Any]], finish_reason: str | None = None) -> list[dict[str, Any]]: for message in messages: source_finish_reason = message.get("finish_reason") - inferred_finish_reason = ( - "tool_call" - if any(part.get("type") == "tool_call" for part in message.get("parts", []) if isinstance(part, Mapping)) - else "unknown" - ) - message["finish_reason"] = finish_reason or source_finish_reason or inferred_finish_reason + message["finish_reason"] = finish_reason or source_finish_reason or "unknown" return messages diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index f9151e00..4c19098a 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -471,7 +471,7 @@ def test_orchestration_full_history_removes_confirmed_input_prefix( ] -def test_orchestration_infers_tool_call_finish_reason_when_absent() -> None: +def test_orchestration_tool_call_assistant_message_finish_reason_unknown() -> None: # Given: a workflow emits an assistant tool call without a source finish reason. output = { "update": { @@ -488,9 +488,9 @@ def test_orchestration_infers_tool_call_finish_reason_when_absent() -> None: # When: the workflow output is converted. attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) - # Then: the standard tool-call finish reason is inferred from the output part. + # Then: finish_reason defaults to "unknown" — no inference from parts. output_message = json.loads(attrs["gen_ai.output.messages"])[0] - assert output_message["finish_reason"] == "tool_call" + assert output_message["finish_reason"] == "unknown" assert output_message["parts"][0]["type"] == "tool_call" @@ -682,7 +682,24 @@ def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> output_messages = json.loads(attrs["gen_ai.output.messages"]) assert len(output_messages) == 1 assert output_messages[0]["role"] == "assistant" - assert output_messages[0]["finish_reason"] == "tool_call" + assert output_messages[0]["finish_reason"] == "unknown" + + +def test_orchestration_full_history_workflow_span_trim() -> None: + # WorkflowSpan (non-root LangGraph node) that carries full state: the trim + # must fire the same way it does for AgentSpan when the prefix matches. + user = {"role": "user", "content": "What is Lisinopril?"} + assistant = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor."} + span = WorkflowSpan( + name="summarise", input=json.dumps({"messages": [user]}), output=json.dumps({"messages": [user, assistant]}) + ) + + attrs = build_span_attributes(span) + + output_messages = json.loads(attrs["gen_ai.output.messages"]) + assert len(output_messages) == 1 + assert output_messages[0]["role"] == "assistant" + assert output_messages[0]["parts"][0]["content"] == "Lisinopril is an ACE inhibitor." def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: From 6439657f967392ef86803b739f60e57ea0386c6d Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 18 Aug 2026 22:49:38 -0500 Subject: [PATCH 07/12] docs(converter): document prefix-match gate and threaded-history limitation Co-Authored-By: Claude Opus 4.7 --- src/splunk_ao/converter/attribute_mapping.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index d14b8f6e..25035f01 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -445,6 +445,11 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages, full_history = _orchestration_messages(span.output, "assistant") if output_messages is None: return + # Reduction is intentionally tied to a confirmed prefix match: we only trim when the + # output starts with an exact copy of the input, which is the LangGraph stateless + # (no checkpointer) full-history shape. With a checkpointer the input is the new turn + # only while the output carries the whole persisted thread, so the prefix never matches + # and no reduction fires — known limitation, tracked separately. if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :][-1:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) From 58a1f79b957c9081c730784cc8f82629b12ab659 Mon Sep 17 00:00:00 2001 From: etserend Date: Wed, 19 Aug 2026 19:36:16 -0500 Subject: [PATCH 08/12] fix(converter): restore tool_call finish-reason inference and gate trim on history_stripped - _with_finish_reasons: infer "tool_call" finish reason when a message has tool_call parts but no explicit source finish reason; "unknown" remains the fallback for all other cases - _set_orchestration_content: split dedup and [-1:] trim using a history_stripped flag so parallel tool-call outputs (ToolNode) are never collapsed to one message when the output does not echo the input prefix - Update two test assertions from "unknown" to "tool_call" for tool-call assistant message cases - CHANGELOG updated with both fixes --- CHANGELOG.md | 9 ++++++++- src/splunk_ao/converter/attribute_mapping.py | 17 +++++++++++++++-- tests/test_attribute_mapping.py | 6 +++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f1bf14a..10651f77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Agent and workflow output conversion now removes confirmed repeated input history and keeps only the last message as the terminal output; intermediate tool-call and tool-response messages are no longer included in - `gen_ai.output.messages` for full-history spans. + `gen_ai.output.messages` for full-history spans. The trim is now gated on the + prefix-match dedup check so parallel tool-call outputs (e.g. a ToolNode with + multiple simultaneous calls) are never collapsed to one message when the output + does not echo the input. +- `gen_ai.output.messages` assistant entries whose parts contain a `tool_call` + but carry no explicit `finish_reason` now receive `"tool_call"` instead of + `"unknown"`. Explicit source finish reasons and tool-response messages are + unaffected. ## [0.2.1] - 2026-08-07 diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 25035f01..357fdba1 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -268,7 +268,13 @@ def _orchestration_messages(value: Any, default_role: str) -> tuple[list[dict[st def _with_finish_reasons(messages: list[dict[str, Any]], finish_reason: str | None = None) -> list[dict[str, Any]]: for message in messages: source_finish_reason = message.get("finish_reason") - message["finish_reason"] = finish_reason or source_finish_reason or "unknown" + # Infer "tool_call" when the message has tool_call parts but no explicit finish reason. + inferred_finish_reason = ( + "tool_call" + if any(part.get("type") == "tool_call" for part in message.get("parts", []) if isinstance(part, Mapping)) + else "unknown" + ) + message["finish_reason"] = finish_reason or source_finish_reason or inferred_finish_reason return messages @@ -450,8 +456,15 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: # (no checkpointer) full-history shape. With a checkpointer the input is the new turn # only while the output carries the whole persisted thread, so the prefix never matches # and no reduction fires — known limitation, tracked separately. + # The [-1:] trim is gated on history_stripped so that parallel tool-call outputs (e.g. + # a ToolNode with multiple simultaneous calls) are never collapsed to one message when + # the output is not a full-history echo of the input. + history_stripped = False if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: - output_messages = output_messages[len(input_messages) :][-1:] + output_messages = output_messages[len(input_messages) :] + history_stripped = True + if history_stripped: + output_messages = output_messages[-1:] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 4c19098a..ca5946ec 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -488,9 +488,9 @@ def test_orchestration_tool_call_assistant_message_finish_reason_unknown() -> No # When: the workflow output is converted. attrs = build_span_attributes(WorkflowSpan(name="tools", output=json.dumps(output))) - # Then: finish_reason defaults to "unknown" — no inference from parts. + # Then: finish_reason is inferred as "tool_call" from the parts. output_message = json.loads(attrs["gen_ai.output.messages"])[0] - assert output_message["finish_reason"] == "unknown" + assert output_message["finish_reason"] == "tool_call" assert output_message["parts"][0]["type"] == "tool_call" @@ -682,7 +682,7 @@ def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> output_messages = json.loads(attrs["gen_ai.output.messages"]) assert len(output_messages) == 1 assert output_messages[0]["role"] == "assistant" - assert output_messages[0]["finish_reason"] == "unknown" + assert output_messages[0]["finish_reason"] == "tool_call" def test_orchestration_full_history_workflow_span_trim() -> None: From 73421d7dce4d0ac9d6af3c88bfa8eb178ade9100 Mon Sep 17 00:00:00 2001 From: etserend Date: Thu, 20 Aug 2026 09:36:01 -0500 Subject: [PATCH 09/12] =?UTF-8?q?fix(converter):=20apply=20reviewer=20sugg?= =?UTF-8?q?estions=20=E2=80=94=20replace=20block=20comment=20and=20drop=20?= =?UTF-8?q?redundant=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/splunk_ao/converter/attribute_mapping.py | 15 +++++++-------- tests/test_attribute_mapping.py | 16 ---------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 357fdba1..76e50384 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -451,14 +451,13 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages, full_history = _orchestration_messages(span.output, "assistant") if output_messages is None: return - # Reduction is intentionally tied to a confirmed prefix match: we only trim when the - # output starts with an exact copy of the input, which is the LangGraph stateless - # (no checkpointer) full-history shape. With a checkpointer the input is the new turn - # only while the output carries the whole persisted thread, so the prefix never matches - # and no reduction fires — known limitation, tracked separately. - # The [-1:] trim is gated on history_stripped so that parallel tool-call outputs (e.g. - # a ToolNode with multiple simultaneous calls) are never collapsed to one message when - # the output is not a full-history echo of the input. + # The trim is gated on a confirmed prefix match so that it only fires when the output + # is genuinely accumulated input history. Without this gate, any span whose output is a + # top-level {"messages": [...]} container would be reduced to a single message — a + # LangGraph ToolNode emitting one ToolMessage per parallel tool call would lose all but + # the last. Consequence: with a checkpointer the input is only the new turn while the + # output carries the whole persisted thread, so the prefix never matches and no + # reduction fires. That case remains unhandled. history_stripped = False if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index ca5946ec..31a01e7c 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -685,22 +685,6 @@ def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> assert output_messages[0]["finish_reason"] == "tool_call" -def test_orchestration_full_history_workflow_span_trim() -> None: - # WorkflowSpan (non-root LangGraph node) that carries full state: the trim - # must fire the same way it does for AgentSpan when the prefix matches. - user = {"role": "user", "content": "What is Lisinopril?"} - assistant = {"role": "assistant", "content": "Lisinopril is an ACE inhibitor."} - span = WorkflowSpan( - name="summarise", input=json.dumps({"messages": [user]}), output=json.dumps({"messages": [user, assistant]}) - ) - - attrs = build_span_attributes(span) - - output_messages = json.loads(attrs["gen_ai.output.messages"]) - assert len(output_messages) == 1 - assert output_messages[0]["role"] == "assistant" - assert output_messages[0]["parts"][0]["content"] == "Lisinopril is an ACE inhibitor." - def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( From 7fdd4739702bb538b05d2aaa981584f15647a711 Mon Sep 17 00:00:00 2001 From: Erdenesaikhan Tserendavga <105012329+etserend@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:31:04 -0500 Subject: [PATCH 10/12] Update src/splunk_ao/converter/attribute_mapping.py Co-authored-by: Fernando Correia --- src/splunk_ao/converter/attribute_mapping.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 76e50384..d8a7b05c 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -463,7 +463,15 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: output_messages = output_messages[len(input_messages) :] history_stripped = True if history_stripped: - output_messages = output_messages[-1:] + # Prefer the last non-user message: the terminal message may legitimately be a tool + # response (return_direct) or a tool-call AIMessage, but a trailing user turn is an + # input, not this span's output. + terminal = next( + (index for index in reversed(range(len(output_messages))) if output_messages[index].get("role") != "user"), + None, + ) + if terminal is not None: + output_messages = output_messages[terminal : terminal + 1] attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) From 7c19f7514a08e79c741b97ca7e1ac4c97487919e Mon Sep 17 00:00:00 2001 From: Erdenesaikhan Tserendavga <105012329+etserend@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:31:15 -0500 Subject: [PATCH 11/12] Update tests/test_attribute_mapping.py Co-authored-by: Fernando Correia --- tests/test_attribute_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 31a01e7c..689473c1 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -471,7 +471,7 @@ def test_orchestration_full_history_removes_confirmed_input_prefix( ] -def test_orchestration_tool_call_assistant_message_finish_reason_unknown() -> None: +def test_orchestration_tool_call_assistant_message_infers_tool_call_finish_reason() -> None: # Given: a workflow emits an assistant tool call without a source finish reason. output = { "update": { From f1d4ef620a3d8803f2c4a4c196c7a6719aa3d76f Mon Sep 17 00:00:00 2001 From: etserend Date: Thu, 20 Aug 2026 18:07:26 -0500 Subject: [PATCH 12/12] =?UTF-8?q?fix(converter):=20apply=20reviewer=20sugg?= =?UTF-8?q?estions=20=E2=80=94=20fold=20history=5Fstripped=20and=20prefer?= =?UTF-8?q?=20last=20non-user=20message?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove history_stripped flag; fold non-user-message trim into the single if block (r3824609327) - Replace [-1:] with last non-user-message search to avoid surfacing trailing user turns (r3824609186) - Rename test to reflect tool_call inference rather than unknown finish reason (r3824609453) - Remove extra blank line at line 688 (r3824609548) Co-Authored-By: Claude Opus 4.7 --- src/splunk_ao/converter/attribute_mapping.py | 3 --- tests/test_attribute_mapping.py | 1 - 2 files changed, 4 deletions(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index d8a7b05c..39b5dde9 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -458,11 +458,8 @@ def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: # the last. Consequence: with a checkpointer the input is only the new turn while the # output carries the whole persisted thread, so the prefix never matches and no # reduction fires. That case remains unhandled. - history_stripped = False if full_history and input_messages and output_messages[: len(input_messages)] == input_messages: output_messages = output_messages[len(input_messages) :] - history_stripped = True - if history_stripped: # Prefer the last non-user message: the terminal message may legitimately be a tool # response (return_direct) or a tool-call AIMessage, but a trailing user turn is an # input, not this span's output. diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 689473c1..7cdd6662 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -685,7 +685,6 @@ def test_orchestration_full_history_ends_on_tool_call_ai_message_keeps_last() -> assert output_messages[0]["finish_reason"] == "tool_call" - def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: span = WorkflowSpan( name="tool-workflow",