diff --git a/crates/jp_cli/src/cmd/conversation/edit.rs b/crates/jp_cli/src/cmd/conversation/edit.rs index 994d5e9fd..cfdff5aa2 100644 --- a/crates/jp_cli/src/cmd/conversation/edit.rs +++ b/crates/jp_cli/src/cmd/conversation/edit.rs @@ -356,7 +356,8 @@ async fn generate_titles( tool_choice: jp_config::assistant::tool_choice::ToolChoice::default(), }; - let retry_config = RetryConfig::default(); + let retry_config = + RetryConfig::default().with_max_response_bytes(config.assistant.request.max_response_bytes); let llm_events = collect_with_retry(provider.as_ref(), &model_details, query, &retry_config).await?; diff --git a/crates/jp_cli/src/cmd/conversation/summarize.rs b/crates/jp_cli/src/cmd/conversation/summarize.rs index e5b0385fe..4a519c7d5 100644 --- a/crates/jp_cli/src/cmd/conversation/summarize.rs +++ b/crates/jp_cli/src/cmd/conversation/summarize.rs @@ -96,6 +96,7 @@ pub async fn generate_summary( stream, instructions, &user_message, + app_cfg.assistant.request.max_response_bytes, ) .await } @@ -120,8 +121,9 @@ async fn summarize_stream( mut stream: ConversationStream, instructions: &str, user_message: &str, + max_response_bytes: u32, ) -> Result { - let retry_config = RetryConfig::default(); + let retry_config = RetryConfig::default().with_max_response_bytes(max_response_bytes); loop { let thread = ThreadBuilder::default() diff --git a/crates/jp_cli/src/cmd/conversation/summarize_tests.rs b/crates/jp_cli/src/cmd/conversation/summarize_tests.rs index 5b7d21194..6a8271266 100644 --- a/crates/jp_cli/src/cmd/conversation/summarize_tests.rs +++ b/crates/jp_cli/src/cmd/conversation/summarize_tests.rs @@ -10,7 +10,7 @@ use jp_llm::{ }; use super::{ - StreamOutcome, collect_range_events, failure_reason, summarize_events, summarize_stream, + Error, StreamOutcome, collect_range_events, failure_reason, summarize_events, summarize_stream, }; /// A stream that produced `text` and then stopped for `reason`. @@ -53,6 +53,14 @@ fn test_model_id() -> ModelIdConfig { async fn summarize_with( batches: Vec>, stream: ConversationStream, +) -> super::Result { + summarize_with_ceiling(batches, stream, 1_048_576).await +} + +async fn summarize_with_ceiling( + batches: Vec>, + stream: ConversationStream, + max_response_bytes: u32, ) -> super::Result { let provider = MockProvider::with_batches(batches); let model_id = test_model_id(); @@ -65,10 +73,39 @@ async fn summarize_with( stream, "instructions", "summarize", + max_response_bytes, ) .await } +/// A summary request honors the configured ceiling rather than a hardcoded +/// default, and does not re-request the response after breaching it. +#[tokio::test] +async fn summarize_applies_the_configured_output_ceiling() { + // One scripted batch is deliberate: `MockProvider` panics on a second + // request, so a ceiling misclassified as retryable fails loudly here. + // 30 bytes of content against a 25-byte ceiling. + let batches = vec![stream_with_text( + "012345678901234567890123456789", + FinishReason::Completed, + )]; + + let error = summarize_with_ceiling(batches, range_stream(&["sig"]), 25) + .await + .expect_err("the summary must stop at the configured ceiling"); + + // The default ceiling is 1 MiB; 30 bytes only breaches the configured 25, + // so reaching this arm proves the setting was threaded through. + assert!( + matches!( + error, + Error::Llm(jp_llm::Error::Stream(ref e)) + if e.kind == jp_llm::StreamErrorKind::OutputLimit + ), + "got: {error:?}" + ); +} + fn build_stream_with_turns(count: usize) -> ConversationStream { let mut stream = ConversationStream::new_test(); for i in 0..count { diff --git a/crates/jp_cli/src/cmd/query/interrupt/signals_tests.rs b/crates/jp_cli/src/cmd/query/interrupt/signals_tests.rs index 054bfca86..1a752c69c 100644 --- a/crates/jp_cli/src/cmd/query/interrupt/signals_tests.rs +++ b/crates/jp_cli/src/cmd/query/interrupt/signals_tests.rs @@ -46,6 +46,7 @@ fn make_retry_state(max_retries: u32) -> StreamRetryState { base_backoff_ms: 1, max_backoff_secs: 1, stream_idle_timeout_secs: 120, + max_response_bytes: 1_048_576, cache: CachePolicy::default(), }; StreamRetryState::new(config, false) diff --git a/crates/jp_cli/src/cmd/query/stream/retry_tests.rs b/crates/jp_cli/src/cmd/query/stream/retry_tests.rs index bb35daed3..a0fbbfa96 100644 --- a/crates/jp_cli/src/cmd/query/stream/retry_tests.rs +++ b/crates/jp_cli/src/cmd/query/stream/retry_tests.rs @@ -21,6 +21,7 @@ fn make_retry_state(max_retries: u32) -> StreamRetryState { base_backoff_ms: 1, // 1ms for fast tests max_backoff_secs: 1, stream_idle_timeout_secs: 120, + max_response_bytes: 1_048_576, cache: CachePolicy::default(), }; StreamRetryState::new(config, false) @@ -85,6 +86,7 @@ fn backoff_uses_retry_after_when_present() { base_backoff_ms: 1, max_backoff_secs: 120, stream_idle_timeout_secs: 120, + max_response_bytes: 1_048_576, cache: CachePolicy::default(), }; let state = StreamRetryState::new(config, false); @@ -341,6 +343,7 @@ async fn interrupt_during_backoff_cuts_wait_short() { base_backoff_ms: 1, max_backoff_secs: 120, stream_idle_timeout_secs: 120, + max_response_bytes: 1_048_576, cache: CachePolicy::default(), }; let mut retry_state = StreamRetryState::new(config, false); diff --git a/crates/jp_cli/src/cmd/query/tool/inquiry.rs b/crates/jp_cli/src/cmd/query/tool/inquiry.rs index 560e1f4f3..724813a1d 100644 --- a/crates/jp_cli/src/cmd/query/tool/inquiry.rs +++ b/crates/jp_cli/src/cmd/query/tool/inquiry.rs @@ -145,6 +145,10 @@ pub struct InquiryConfig { pub model: ModelDetails, pub system_prompt: Option, pub sections: Vec, + + /// Output ceiling for the inquiry request, from + /// `assistant.request.max_response_bytes`. + pub max_response_bytes: u32, } /// Resolves inquiries by making structured output calls to an LLM provider. @@ -188,7 +192,10 @@ impl LlmInquiryBackend { } /// Look up the effective config for this tool/question pair. - fn config_for(&self, tool_name: &str, question_id: &str) -> &InquiryConfig { + /// + /// Returns the per-question override when one exists, otherwise the default + /// built from the global inquiry config merged with the parent assistant. + pub(crate) fn config_for(&self, tool_name: &str, question_id: &str) -> &InquiryConfig { self.overrides .get(&(tool_name.to_owned(), question_id.to_owned())) .unwrap_or(&self.default_config) @@ -282,7 +289,8 @@ impl InquiryBackend for LlmInquiryBackend { tool_choice: ToolChoice::None, }; - let retry_config = RetryConfig::default(); + let retry_config = + RetryConfig::default().with_max_response_bytes(config.max_response_bytes); let llm_events = tokio::select! { biased; () = cancellation_token.cancelled() => { diff --git a/crates/jp_cli/src/cmd/query/tool/inquiry_tests.rs b/crates/jp_cli/src/cmd/query/tool/inquiry_tests.rs index 43c61018e..ff8c3533a 100644 --- a/crates/jp_cli/src/cmd/query/tool/inquiry_tests.rs +++ b/crates/jp_cli/src/cmd/query/tool/inquiry_tests.rs @@ -39,6 +39,7 @@ fn test_inquiry_config(provider: MockProvider) -> InquiryConfig { model: test_model(), system_prompt: None, sections: vec![], + max_response_bytes: 1_048_576, } } @@ -323,6 +324,7 @@ async fn llm_backend_uses_per_question_override() { model: test_model(), system_prompt: Some("Override prompt.".into()), sections: vec![], + max_response_bytes: 1_048_576, }; let overrides = IndexMap::from([(("test_tool".into(), "confirm".into()), override_config)]); @@ -557,6 +559,7 @@ async fn dedicated_model_backend_returns_answer() { }), system_prompt: Some("Answer concisely.".to_string()), sections: vec![], + max_response_bytes: 1_048_576, }; let backend = LlmInquiryBackend::new(config, IndexMap::new(), vec![], vec![]); diff --git a/crates/jp_cli/src/cmd/query/turn_loop.rs b/crates/jp_cli/src/cmd/query/turn_loop.rs index 6565e1e92..a751b0307 100644 --- a/crates/jp_cli/src/cmd/query/turn_loop.rs +++ b/crates/jp_cli/src/cmd/query/turn_loop.rs @@ -31,10 +31,11 @@ use jp_llm::{ error::StreamError, event::{Event, EventPart, FinishReason, ToolCallPart}, model::ModelDetails, + output_limit_bytes, provider::get_provider, query::ChatQuery, tool::{InvocationContext, ToolDefinition, executor::Executor}, - with_idle_timeout, + with_idle_timeout, with_output_limit, }; use jp_printer::{ErrChannel, Printer}; use jp_workspace::{ConversationLock, ConversationMut}; @@ -206,6 +207,7 @@ pub(super) async fn run_turn_loop( 0 => None, secs => Some(Duration::from_secs(u64::from(secs))), }; + let output_limit = output_limit_bytes(cfg.assistant.request.max_response_bytes); let mut turn_coordinator = TurnCoordinator::new( printer.clone(), cfg.style.clone(), @@ -342,6 +344,14 @@ pub(super) async fn run_turn_loop( Some(idle) => with_idle_timeout(raw_stream, idle), None => raw_stream, }; + // Wrapped outside the provider stream, so the bytes of every + // chained continuation accumulate against a single ceiling + // rather than resetting per link. Bytes the provider discards + // while merging those links are billed but never seen here. + let raw_stream = match output_limit { + Some(max) => with_output_limit(raw_stream, max), + None => raw_stream, + }; let llm_stream = StreamSource::Llm( raw_stream .fuse() @@ -873,6 +883,26 @@ async fn build_inquiry_backend( .clone() .or_else(|| cfg.assistant.system_prompt.clone()); + // Same fallback for the output ceiling: an inquiry can be held to a tighter + // (or looser) ceiling than the parent assistant. + // + // `AssistantOverrideConfig::request` is a resolved `RequestConfig`, so a + // block where the user set only a sibling field (say `cache`) arrives here + // with every other field at Rust's `Default` rather than its schematic + // default. A `0` therefore cannot be distinguished from "unset", and reading + // it as the ceiling's disable sentinel would silently drop the runaway guard + // for every inquiry. Treat it as "inherit" instead, matching the block's + // documented unset-means-inherit rule. A per-question override carries real + // `Option`s, so `0` still disables the ceiling there. + let default_max_response_bytes = inquiry_override + .request + .as_ref() + .map_or(0, |request| request.max_response_bytes); + let default_max_response_bytes = match default_max_response_bytes { + 0 => cfg.assistant.request.max_response_bytes, + bytes => bytes, + }; + // Track providers we've already constructed to avoid duplicates. let mut providers: IndexMap> = IndexMap::new(); @@ -920,6 +950,7 @@ async fn build_inquiry_backend( model: inquiry_model, system_prompt: default_system_prompt, sections: sections.clone(), + max_response_bytes: default_max_response_bytes, } } else { providers.insert(model.id.provider, Arc::clone(&provider)); @@ -929,6 +960,7 @@ async fn build_inquiry_backend( model: model.clone(), system_prompt: default_system_prompt, sections: sections.clone(), + max_response_bytes: default_max_response_bytes, } }; @@ -1018,11 +1050,20 @@ async fn build_inquiry_overrides( .map(|s| s.to_string()) .or_else(|| default_config.system_prompt.clone()); + // Output ceiling follows the same order. `default_config` already + // carries the global-inquiry-or-parent value, so an unset + // per-question ceiling inherits it. + let max_response_bytes = per_q + .request + .max_response_bytes + .unwrap_or(default_config.max_response_bytes); + overrides.insert((tool_name.to_owned(), question_id.clone()), InquiryConfig { provider: inq_provider, model: inq_model, system_prompt, sections: default_config.sections.clone(), + max_response_bytes, }); } } diff --git a/crates/jp_cli/src/cmd/query/turn_loop_tests.rs b/crates/jp_cli/src/cmd/query/turn_loop_tests.rs index 1e857b2d1..3fe664f9c 100644 --- a/crates/jp_cli/src/cmd/query/turn_loop_tests.rs +++ b/crates/jp_cli/src/cmd/query/turn_loop_tests.rs @@ -16,7 +16,11 @@ use futures::{StreamExt as _, stream}; use indexmap::IndexMap; use inquire::InquireError; use jp_config::{ - AppConfig, + AppConfig, PartialAppConfig, + assistant::{ + PartialAssistantConfig, + request::{CachePolicy, PartialRequestConfig, RequestConfig}, + }, conversation::tool::{ CommandConfigOrString, QuestionConfig, QuestionTarget, RunMode, ToolConfig, ToolSource, style::{DisplayStyleConfig, ErrorStyleConfig, InlineResults, LinkStyle, ParametersStyle}, @@ -193,6 +197,49 @@ impl Provider for AlwaysPrematureProvider { } } +/// A provider that streams content without end, counting calls so a test can +/// assert the response was not re-requested. +/// +/// After its parts the stream stays pending forever and it never sends a +/// terminal `Finished`, so the output ceiling is the only thing that can end +/// it. +/// A test built on this fixture hangs if the ceiling never fires, rather than +/// falling through to the retry path and passing for the wrong reason. +#[derive(Debug, Default)] +struct RunawayProvider { + calls: AtomicUsize, +} + +#[async_trait] +impl Provider for RunawayProvider { + async fn model_details(&self, name: &id::Name) -> Result { + Ok(ModelDetails::empty(id::ModelIdConfig { + provider: ProviderId::Test, + name: name.clone(), + })) + } + + async fn models(&self) -> Result, LlmError> { + Ok(vec![]) + } + + async fn chat_completion_stream( + &self, + _model: &ModelDetails, + _query: ChatQuery, + ) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + + // 7 bytes, then 40 bytes per part. + let mut events = vec![Event::message(0, "runaway")]; + events.extend(std::iter::repeat_with(|| Event::message(0, "0123456789".repeat(4))).take(9)); + + Ok(Box::pin( + stream::iter(events.into_iter().map(Ok)).chain(stream::pending()), + )) + } +} + /// A provider whose stream yields the given events and then stays pending /// forever, simulating an in-flight response that only an interrupt can stop. #[derive(Debug)] @@ -692,6 +739,94 @@ async fn premature_stream_end_exhausts_retry_budget() { ); } +/// A response that runs past `assistant.request.max_response_bytes` ends the +/// turn with an error, is not re-requested, and keeps the content streamed +/// before the ceiling was reached. +#[tokio::test] +async fn output_ceiling_ends_turn_without_re_requesting() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let storage = root.join(".jp"); + + let mut config = AppConfig::new_test(); + config.assistant.request.max_response_bytes = 64; + // A retry budget is left in place so the call-count assertion below has + // something to catch: were the ceiling classified as retryable, the loop + // would re-request the response instead of ending the turn. + config.assistant.request.max_retries = 5; + config.assistant.request.base_backoff_ms = 0; + // The fixture never goes idle-silent before the ceiling fires, but an + // enabled idle timeout would give the loop a second way out; disable it so + // only the ceiling can end this turn. + config.assistant.request.stream_idle_timeout_secs = 0; + + let fs = Arc::new(FsStorageBackend::new(&storage).expect("failed to create backend")); + let mut workspace = Workspace::new(root).with_backend(fs.clone()); + + let lock = workspace + .create_and_lock_conversation(Conversation::default(), config.clone().into(), None) + .unwrap(); + let conv_id = lock.id(); + + let provider = Arc::new(RunawayProvider::default()); + let dyn_provider: Arc = provider.clone(); + let model = dyn_provider + .model_details(&"test-model".parse().unwrap()) + .await + .unwrap(); + + let (printer, _out, _err) = Printer::memory(OutputFormat::TextPretty); + let printer = Arc::new(printer); + let mcp_client = jp_mcp::Client::default(); + let router = detached_router(); + + let result = timeout( + Duration::from_secs(5), + run_turn_loop( + dyn_provider, + &model, + &config, + &router, + &mcp_client, + root, + false, + &[], + &lock, + ToolChoice::Auto, + &[], + printer.clone(), + Arc::new(MockPromptBackend::new()), + ToolCoordinator::new(config.conversation.tools.clone(), empty_executor_source()), + ChatRequest::from("hi"), + InvocationContext::default(), + PendingStreamTrim::default(), + ), + ) + .await + .expect("the output ceiling should end the turn, not hang it"); + + let Err(Error::Llm(jp_llm::Error::Stream(error))) = result else { + panic!("expected a stream error from the output ceiling, got: {result:?}"); + }; + assert_eq!(error.kind, jp_llm::StreamErrorKind::OutputLimit); + + assert_eq!( + provider.calls.load(Ordering::SeqCst), + 1, + "a response that breached the ceiling must not be re-requested" + ); + + // Option 1 of the ceiling design: the turn ends, but content the user + // already saw stays in the conversation. + let content = fs + .read_test_events_raw(&conv_id) + .expect("events should be persisted"); + assert!( + content.contains("runaway"), + "content streamed before the ceiling must be persisted.\nFile contents:\n{content}" + ); +} + /// Regression: any `ToolCallRequest` already in the stream when a new /// `Streaming` cycle starts MUST be sanitized into a stream that's safe to send /// to the provider. @@ -4393,6 +4528,135 @@ fn inquiry_mock_model() -> ModelDetails { }) } +/// The global inquiry override for the output ceiling wins over the parent +/// assistant's value. +/// +/// `conversation.inquiry.assistant.request.max_response_bytes` is a public key, +/// so reading the parent value here would silently ignore it. +#[tokio::test] +async fn inquiry_ceiling_honors_the_global_inquiry_override() { + let mut config = AppConfig::new_test(); + config.assistant.request.max_response_bytes = 999_999; + config.conversation.inquiry.assistant.request = Some(RequestConfig { + max_response_bytes: 4096, + ..config.assistant.request + }); + + let provider: Arc = Arc::new(MockProvider::new(vec![])); + let model = inquiry_mock_model(); + + let backend = build_inquiry_backend(&config, vec![], model, provider, vec![]) + .await + .expect("the inquiry backend builds"); + + assert_eq!( + backend + .config_for("any_tool", "any_question") + .max_response_bytes, + 4096, + "the global inquiry override must win over the parent assistant" + ); +} + +/// A partially-set inquiry request block must not disable the ceiling. +/// +/// Built through the real loading path rather than by hand: because +/// `AssistantOverrideConfig::request` is a resolved struct, setting only a +/// sibling field leaves `max_response_bytes` at `0`, which is the ceiling's +/// disable sentinel. +/// Reading it verbatim would silently drop the runaway guard for every inquiry. +#[tokio::test] +async fn inquiry_ceiling_survives_a_sibling_only_request_override() { + let mut partial = PartialAppConfig::new_test(); + partial.assistant.request.max_response_bytes = Some(500_000); + + partial.conversation.inquiry.assistant.request = Some(PartialRequestConfig { + cache: Some(CachePolicy::Off), + ..PartialRequestConfig::default() + }); + + let config = AppConfig::from_partial_with_defaults(partial).expect("valid config"); + + // The resolution the guard has to cope with: the block is present, and its + // ceiling field is a zero the user never asked for. + assert_eq!( + config + .conversation + .inquiry + .assistant + .request + .expect("the block is set") + .max_response_bytes, + 0 + ); + + let provider: Arc = Arc::new(MockProvider::new(vec![])); + let model = inquiry_mock_model(); + + let backend = build_inquiry_backend(&config, vec![], model, provider, vec![]) + .await + .expect("the inquiry backend builds"); + + assert_eq!( + backend + .config_for("any_tool", "any_question") + .max_response_bytes, + 500_000, + "an unset inquiry ceiling must inherit the parent, not disable the guard" + ); +} + +/// A per-question ceiling wins over the global inquiry override, which in turn +/// wins over the parent assistant (RFD 034's resolution order). +#[tokio::test] +async fn inquiry_ceiling_honors_the_per_question_override() { + let mut config = AppConfig::new_test(); + config.assistant.request.max_response_bytes = 999_999; + config.conversation.inquiry.assistant.request = Some(RequestConfig { + max_response_bytes: 4096, + ..config.assistant.request + }); + + let mut per_question = PartialAssistantConfig::default(); + per_question.request.max_response_bytes = Some(512); + + let mut tool = inquiry_tool_config(&["confirm"]); + tool.questions + .insert("confirm".to_string(), QuestionConfig { + target: QuestionTarget::Assistant(Box::new(per_question)), + answer: None, + }); + config + .conversation + .tools + .insert("inquiry_tool".to_string(), tool); + + let provider: Arc = Arc::new(MockProvider::new(vec![])); + let model = inquiry_mock_model(); + + let backend = build_inquiry_backend(&config, vec![], model, provider, vec![]) + .await + .expect("the inquiry backend builds"); + + assert_eq!( + backend + .config_for("inquiry_tool", "confirm") + .max_response_bytes, + 512, + "the per-question override must win over the global inquiry value" + ); + + // A question without its own ceiling still inherits the global inquiry + // value, not the parent assistant's. + assert_eq!( + backend + .config_for("inquiry_tool", "other") + .max_response_bytes, + 4096, + "an unset per-question ceiling falls back to the global inquiry value" + ); +} + /// Tool has one boolean question with `QuestionTarget::Assistant`. /// Flow: LLM tool call → `NeedsInput` → inquiry → answer → tool completes. #[tokio::test] diff --git a/crates/jp_config/src/assistant/request.rs b/crates/jp_config/src/assistant/request.rs index dcf08ec68..86c998228 100644 --- a/crates/jp_config/src/assistant/request.rs +++ b/crates/jp_config/src/assistant/request.rs @@ -75,6 +75,37 @@ pub struct RequestConfig { #[setting(default = 60)] pub stream_idle_timeout_secs: u32, + /// Abort a response after it generates more than this many bytes. + /// + /// Defaults to `1048576` (1 MiB, roughly 260,000 tokens). + /// Set to `0` to disable the ceiling. + /// + /// This is a runaway guard rather than a length preference. + /// It exists so a model that gets stuck generating without end cannot run + /// up an unbounded bill while nobody is watching the terminal. + /// The default allows long responses assembled from several continuation + /// requests while bounding the cost of a runaway response. + /// + /// During a query, content streamed before the ceiling is reached stays in + /// the conversation and only the turn ends, with an error. + /// Background requests that collect a whole response before using it (title + /// generation, summarization, tool inquiries) keep no partial result: they + /// fail outright. + /// + /// Inquiry requests can be held to their own ceiling via + /// `conversation.inquiry.assistant.request.max_response_bytes`, or per + /// question via a question's assistant target. + /// In the `conversation.inquiry` block, `0` means "inherit this ceiling" + /// rather than "disable it"; disable it per question instead. + /// + /// The ceiling counts bytes rather than tokens because tokens cannot be + /// counted locally; four bytes per token is a rough guide. + /// It counts the bytes JP receives, which can be fewer than the bytes + /// billed: a provider that assembles a response from several continuation + /// requests may discard some of what it generated before returning it. + #[setting(default = 1_048_576)] + pub max_response_bytes: u32, + /// Prompt caching policy. /// /// Controls whether the provider applies prompt caching optimizations (e.g. @@ -122,6 +153,7 @@ impl AssignKeyValue for PartialRequestConfig { "stream_idle_timeout_secs" => { self.stream_idle_timeout_secs = kv.try_some_u32()?; } + "max_response_bytes" => self.max_response_bytes = kv.try_some_u32()?, "cache" => self.cache = kv.try_some_bool_or_from_str()?, _ => return missing_key(&kv), } @@ -140,6 +172,10 @@ impl PartialConfigDelta for PartialRequestConfig { self.stream_idle_timeout_secs.as_ref(), next.stream_idle_timeout_secs, ), + max_response_bytes: delta_opt( + self.max_response_bytes.as_ref(), + next.max_response_bytes, + ), cache: delta_opt(self.cache.as_ref(), next.cache), } } @@ -154,6 +190,7 @@ impl FillDefaults for PartialRequestConfig { stream_idle_timeout_secs: self .stream_idle_timeout_secs .or(defaults.stream_idle_timeout_secs), + max_response_bytes: self.max_response_bytes.or(defaults.max_response_bytes), cache: self.cache.or(defaults.cache), } } @@ -171,6 +208,7 @@ impl ToPartial for RequestConfig { &self.stream_idle_timeout_secs, defaults.stream_idle_timeout_secs, ), + max_response_bytes: partial_opt(&self.max_response_bytes, defaults.max_response_bytes), cache: partial_opt(&self.cache, defaults.cache), } } diff --git a/crates/jp_config/src/assistant/request_tests.rs b/crates/jp_config/src/assistant/request_tests.rs index a27431028..ea3ae0228 100644 --- a/crates/jp_config/src/assistant/request_tests.rs +++ b/crates/jp_config/src/assistant/request_tests.rs @@ -12,6 +12,7 @@ fn request_config(stream_idle_timeout_secs: u32) -> RequestConfig { base_backoff_ms: 1000, max_backoff_secs: 60, stream_idle_timeout_secs, + max_response_bytes: 1_048_576, cache: CachePolicy::default(), } } @@ -51,6 +52,16 @@ fn test_request_config_assign() { let kv = KvAssignment::try_from_cli("stream_idle_timeout_secs", "20").unwrap(); p.assign(kv).unwrap(); assert_eq!(p.stream_idle_timeout_secs, Some(20)); + + let kv = KvAssignment::try_from_cli("max_response_bytes", "4096").unwrap(); + p.assign(kv).unwrap(); + assert_eq!(p.max_response_bytes, Some(4096)); + + // `0` is the documented way to disable the ceiling, so it must survive + // assignment rather than being treated as "unset". + let kv = KvAssignment::try_from_cli("max_response_bytes", "0").unwrap(); + p.assign(kv).unwrap(); + assert_eq!(p.max_response_bytes, Some(0)); } #[test] diff --git a/crates/jp_config/src/lib_tests.rs b/crates/jp_config/src/lib_tests.rs index 8a4abccbd..241fba266 100644 --- a/crates/jp_config/src/lib_tests.rs +++ b/crates/jp_config/src/lib_tests.rs @@ -25,6 +25,51 @@ fn test_app_config_fields() { insta::assert_debug_snapshot!(AppConfig::fields()); } +/// Setting one field in the inquiry request block must not silently zero the +/// rest of it. +/// +/// `AssistantOverrideConfig::request` is a resolved `Option`, so +/// the conversion has no per-field presence to preserve: every field the user +/// did not set resolves to Rust's `Default` rather than the schematic default. +/// This test pins that behavior so a reader of the resolution code knows the +/// zeros are an artifact, not a user's choice. +#[test] +fn inquiry_request_override_zeroes_unset_fields() { + use crate::assistant::request::CachePolicy; + + let mut partial = PartialAppConfig::new_test(); + partial.assistant.request.max_response_bytes = Some(500_000); + + // Only a sibling field is set in the inquiry block. + partial.conversation.inquiry.assistant.request = + Some(crate::assistant::request::PartialRequestConfig { + cache: Some(CachePolicy::Off), + ..Default::default() + }); + + let config = AppConfig::from_partial_with_defaults(partial).expect("valid config"); + + let request = config + .conversation + .inquiry + .assistant + .request + .expect("the inquiry request block is set"); + + assert_eq!(request.cache, CachePolicy::Off, "the set field survives"); + assert_eq!( + request.max_response_bytes, 0, + "an unset field resolves to Rust's Default, not the schematic default" + ); + assert_eq!( + request.max_retries, 0, + "the same applies to every other field in the block" + ); + + // The parent keeps its own value; nothing merged into it. + assert_eq!(config.assistant.request.max_response_bytes, 500_000); +} + #[test] fn test_ensure_no_missing_assignments() { // Some fields cannot be assigned via CLI. diff --git a/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap b/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap index b879c750f..4be0258d8 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap @@ -105,6 +105,7 @@ expression: "AppConfig::fields()" "assistant.request.base_backoff_ms", "assistant.request.cache", "assistant.request.max_backoff_secs", + "assistant.request.max_response_bytes", "assistant.request.max_retries", "assistant.request.stream_idle_timeout_secs", "assistant.model.id", diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap index a94f2c7cd..3be493edb 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap @@ -38,6 +38,7 @@ PartialAppConfig { base_backoff_ms: None, max_backoff_secs: None, stream_idle_timeout_secs: None, + max_response_bytes: None, cache: None, }, }, diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap index 61b1220e0..af7cdbcb0 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap @@ -90,6 +90,9 @@ Ok( stream_idle_timeout_secs: Some( 60, ), + max_response_bytes: Some( + 1048576, + ), cache: None, }, }, diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap index e8d361b31..dd379bd7a 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap @@ -38,6 +38,7 @@ PartialAppConfig { base_backoff_ms: None, max_backoff_secs: None, stream_idle_timeout_secs: None, + max_response_bytes: None, cache: None, }, }, diff --git a/crates/jp_llm/src/error.rs b/crates/jp_llm/src/error.rs index 2279407f0..63dbe6b92 100644 --- a/crates/jp_llm/src/error.rs +++ b/crates/jp_llm/src/error.rs @@ -72,6 +72,12 @@ impl StreamError { Self::new(StreamErrorKind::Transient, message) } + /// Create an output-limit error. + #[must_use] + pub fn output_limit(message: impl Into) -> Self { + Self::new(StreamErrorKind::OutputLimit, message) + } + /// Returns the human-readable error message (without the source chain). #[must_use] pub fn message(&self) -> &str { @@ -288,6 +294,10 @@ pub enum StreamErrorKind { /// This is not retryable — the user needs to top up or change plans. InsufficientQuota, + /// The response exceeded the configured output byte ceiling. + /// This is not retryable because a retry regenerates the same runaway. + OutputLimit, + /// Other errors that are not categorized. /// These may or may not be retryable depending on the specific error. Other, @@ -303,6 +313,7 @@ impl StreamErrorKind { Self::RateLimit => "Rate limited", Self::Transient => "Server error", Self::InsufficientQuota => "Insufficient API quota", + Self::OutputLimit => "Output limit exceeded", Self::Other => "Stream Error", } } diff --git a/crates/jp_llm/src/lib.rs b/crates/jp_llm/src/lib.rs index 92d64a424..7cf1a311f 100644 --- a/crates/jp_llm/src/lib.rs +++ b/crates/jp_llm/src/lib.rs @@ -15,5 +15,8 @@ pub(crate) mod test; pub use error::{Error, StreamError, StreamErrorKind, ToolError}; pub use provider::Provider; pub use retry::exponential_backoff; -pub use stream::{EventStream, chain::EventChain, with_idle_timeout, with_tool_call_keepalive}; +pub use stream::{ + EventStream, chain::EventChain, output_limit_bytes, with_idle_timeout, with_output_limit, + with_tool_call_keepalive, +}; pub use tool::{CommandResult, ExecutionOutcome, ToolTrace, run_tool_command}; diff --git a/crates/jp_llm/src/provider/anthropic.rs b/crates/jp_llm/src/provider/anthropic.rs index 18d77eafe..791ad3b15 100644 --- a/crates/jp_llm/src/provider/anthropic.rs +++ b/crates/jp_llm/src/provider/anthropic.rs @@ -95,6 +95,17 @@ const CONTINUE_MESSAGE: &str = "Continue your response exactly from where you le /// Default minimum overlap for merge point detection during chaining. const CHAIN_MIN_OVERLAP: usize = 5; +/// How many continuation requests a single response may chain after hitting the +/// token ceiling. +/// +/// Each continuation re-sends the whole conversation, so the input cost is paid +/// again per link. +/// A model stuck emitting tokens without end hits the ceiling every time and +/// would otherwise chain forever. +/// Five links allow several long continuations while keeping the worst case +/// bounded. +const MAX_CHAIN_DEPTH: u8 = 5; + /// How many escalating soft-force retries to attempt for models that can't /// disable thinking, before accepting a response without the forced tool call. const SOFT_FORCE_MAX_RETRIES: u8 = 3; @@ -182,6 +193,11 @@ impl Provider for Anthropic { // tokens value, or when chaining is disabled in the provider config. let chain_on_max_tokens = !is_structured && max_tokens_config.is_none() && self.chain_on_max_tokens; + let chains_remaining = if chain_on_max_tokens { + MAX_CHAIN_DEPTH + } else { + 0 + }; debug!(stream = true, "Anthropic chat completion stream request."); trace!( @@ -193,7 +209,7 @@ impl Provider for Anthropic { call( client, request, - chain_on_max_tokens, + chains_remaining, is_structured, forced_tool, ), @@ -250,10 +266,12 @@ impl ForcedToolFallback { /// Create a streaming request and return a stream of [`Event`]s. /// -/// If `chain_on_max_tokens` is `true`, a new request is created when the last -/// one ends with a [`FinishReason::MaxTokens`] event, allowing the assistant to -/// continue from where it left off and the caller to receive the full response -/// as a single stream of events. +/// While `chains_remaining` is above zero, a new request is created when the +/// last one ends with a [`FinishReason::MaxTokens`] event, allowing the +/// assistant to continue from where it left off and the caller to receive the +/// full response as a single stream of events. +/// Each continuation decrements the budget, so a model that keeps hitting the +/// token ceiling cannot chain requests without bound. /// /// If `forced_tool_fallback` is `Some`, the model was asked to call a tool but /// the API restriction on reasoning + forced tool use required us to downgrade @@ -265,10 +283,11 @@ impl ForcedToolFallback { /// If the API rejects a thinking block in the request, emits [`Event::Patch`] /// instructions to fix the conversation stream and finishes with /// [`FinishReason::Retry`] so the caller can rebuild and retry. +#[expect(clippy::too_many_lines)] fn call( client: Client, request: types::CreateMessagesRequest, - chain_on_max_tokens: bool, + chains_remaining: u8, is_structured: bool, forced_tool_fallback: Option, ) -> EventStream { @@ -281,7 +300,7 @@ fn call( // The same buffer is used for the forced-tool fallback: if the model // finishes without calling a tool, we need the accumulated events to // build the assistant message for the retry request. - let needs_event_buffer = chain_on_max_tokens || forced_tool_fallback.is_some(); + let needs_event_buffer = chains_remaining > 0 || forced_tool_fallback.is_some(); let mut chain_builder = EventBuilder::new(); let mut chain_events = vec![]; @@ -331,13 +350,20 @@ fn call( // existing stream of events alive. // // TODO: generalize this for any provider. - event if should_chain(&event, tool_calls_requested, chain_on_max_tokens) => { - debug!("Max tokens reached, auto-requesting more tokens."); + event if should_chain(&event, tool_calls_requested, chains_remaining) => { + debug!( + chains_remaining, + "Max tokens reached, auto-requesting more tokens." + ); chain_events.extend(chain_builder.drain()); - for await event in - chain(client.clone(), request.clone(), chain_events, is_structured) - { + for await event in chain( + client.clone(), + request.clone(), + chain_events, + is_structured, + chains_remaining - 1, + ) { yield event?; } return; @@ -397,7 +423,7 @@ fn call( // chained request, we ignore the flush event, to allow more // event parts to be generated by the next response. if let Some(event) = next_event - && should_chain(event, tool_calls_requested, chain_on_max_tokens) + && should_chain(event, tool_calls_requested, chains_remaining) { continue; } @@ -418,9 +444,9 @@ fn call( } /// Check if we should chain more events from a new request. -fn should_chain(event: &Event, tool_calls_requested: bool, chain_on_max_tokens: bool) -> bool { +fn should_chain(event: &Event, tool_calls_requested: bool, chains_remaining: u8) -> bool { !tool_calls_requested - && chain_on_max_tokens + && chains_remaining > 0 && matches!(event, Event::Finished(FinishReason::MaxTokens)) } @@ -431,6 +457,7 @@ fn chain( mut request: types::CreateMessagesRequest, events: Vec, is_structured: bool, + chains_remaining: u8, ) -> EventStream { debug_assert!(!events.iter().any(ConversationEvent::is_tool_call_request)); @@ -472,7 +499,7 @@ fn chain( }); Box::pin(try_stream!({ - for await event in call(client, request, true, is_structured, None) { + for await event in call(client, request, chains_remaining, is_structured, None) { let mut event = event?; // When chaining new events, the reasoning content is irrelevant, as @@ -600,7 +627,7 @@ fn force_tool_retry( Box::pin(try_stream!({ // Pass `None` for forced_tool_fallback to prevent infinite retries. - for await event in call(client, request, false, is_structured, None) { + for await event in call(client, request, 0, is_structured, None) { let event = event?; // Skip reasoning (shouldn't appear with thinking disabled, but @@ -682,7 +709,7 @@ fn soft_force_retry( }); Box::pin(try_stream!({ - for await event in call(client, request, false, is_structured, next_fallback) { + for await event in call(client, request, 0, is_structured, next_fallback) { yield event?; } diff --git a/crates/jp_llm/src/provider/anthropic_tests.rs b/crates/jp_llm/src/provider/anthropic_tests.rs index cdd8eb404..9923c41ba 100644 --- a/crates/jp_llm/src/provider/anthropic_tests.rs +++ b/crates/jp_llm/src/provider/anthropic_tests.rs @@ -9,7 +9,10 @@ use jp_config::{ }, }; use jp_conversation::{event::ChatRequest, thread::Thread}; -use jp_test::{Result, function_name}; +use jp_test::{ + Result, function_name, + mock::{MockServer, POST}, +}; use serde_json::Map; use test_log::test; @@ -31,6 +34,119 @@ async fn test_redacted_thinking() -> Result { run_test(PROVIDER, function_name!(), requests).await } +/// An SSE response that always truncates on `max_tokens`, which is the +/// condition that triggers a continuation request. +fn max_tokens_sse_body() -> String { + [ + r"event: message_start", + r#"data: {"type":"message_start","message":{"model":"claude-test","id":"msg_test","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":1}}}"#, + "", + r"event: content_block_start", + r#"data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#, + "", + r"event: content_block_delta", + r#"data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"and on and on"}}"#, + "", + r"event: content_block_stop", + r#"data: {"type":"content_block_stop","index":0}"#, + "", + r"event: message_delta", + r#"data: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"input_tokens":1,"output_tokens":1}}"#, + "", + r"event: message_stop", + r#"data: {"type":"message_stop"}"#, + "", + ] + .join("\n") +} + +/// A model that truncates on every request must stop chaining once the +/// continuation budget is spent, rather than continuing forever. +/// +/// This drives the real `call` -\> `chain` -\> `call` recursion against a +/// server that always answers `max_tokens`, so it fails if the budget stops +/// being decremented or a continuation is handed the original budget. +#[test(tokio::test)] +async fn chaining_is_bounded_by_the_continuation_budget() { + let server = MockServer::start_async().await; + let endpoint = server + .mock_async(|when, then| { + when.method(POST).path("/v1/messages"); + then.status(200) + .header("content-type", "text/event-stream; charset=utf-8") + .body(max_tokens_sse_body()); + }) + .await; + + let mut builder = Client::builder(); + builder + .api_key("test-key") + .base_url(server.base_url()) + .version("2023-06-01"); + let client = builder.build().expect("a client for the mock server"); + + let request = types::CreateMessagesRequestBuilder::default() + .model("claude-test".to_owned()) + .messages(vec![types::Message { + role: types::MessageRole::User, + content: types::MessageContentList(vec![types::MessageContent::Text( + "go on then".into(), + )]), + }]) + .max_tokens(16) + .stream(true) + .build() + .expect("a valid request"); + + let events: Vec<_> = call(client, request, MAX_CHAIN_DEPTH, false, None) + .collect() + .await; + + assert!( + events.iter().all(std::result::Result::is_ok), + "the chain should end cleanly, got: {events:?}" + ); + + // One initial request plus MAX_CHAIN_DEPTH continuations. An off-by-one in + // the budget shows up here as a different count; a budget that never + // decrements never stops. + assert_eq!( + endpoint.calls_async().await, + usize::from(MAX_CHAIN_DEPTH) + 1, + "a model that always truncates must exhaust the continuation budget and stop" + ); +} + +#[test] +fn chaining_stops_when_the_budget_is_exhausted() { + let max_tokens = Event::Finished(FinishReason::MaxTokens); + + assert!( + should_chain(&max_tokens, false, MAX_CHAIN_DEPTH), + "the initial request may use its continuation budget" + ); + assert!( + should_chain(&max_tokens, false, 1), + "the final remaining continuation may be used" + ); + assert!( + !should_chain(&max_tokens, false, 0), + "an exhausted budget must terminate the response" + ); + assert!( + !should_chain(&max_tokens, true, MAX_CHAIN_DEPTH), + "tool calls cannot be continued without their results" + ); + assert!( + !should_chain( + &Event::Finished(FinishReason::Completed), + false, + MAX_CHAIN_DEPTH, + ), + "a completed response must not be continued" + ); +} + #[test(tokio::test)] async fn test_request_chaining() -> Result { let mut request = TestRequest::chat(PROVIDER) diff --git a/crates/jp_llm/src/retry.rs b/crates/jp_llm/src/retry.rs index 4f60df47b..099ea78a4 100644 --- a/crates/jp_llm/src/retry.rs +++ b/crates/jp_llm/src/retry.rs @@ -6,7 +6,12 @@ use futures::TryStreamExt as _; use tracing::{debug, warn}; use crate::{ - Provider, StreamError, error::Result, event::Event, model::ModelDetails, query::ChatQuery, + Provider, StreamError, + error::Result, + event::Event, + model::ModelDetails, + query::ChatQuery, + stream::{output_limit_bytes, with_output_limit}, }; /// Configuration for resilient stream retries. @@ -20,6 +25,11 @@ pub struct RetryConfig { /// Maximum backoff delay in seconds. pub max_backoff_secs: u64, + + /// Abort a response after it generates more than this many bytes. + /// + /// `0` disables the ceiling. + pub max_response_bytes: u32, } impl Default for RetryConfig { @@ -28,10 +38,26 @@ impl Default for RetryConfig { max_retries: 3, base_backoff_ms: 1000, max_backoff_secs: 30, + max_response_bytes: 1_048_576, } } } +impl RetryConfig { + /// Apply the user's configured output ceiling, leaving the retry and + /// backoff settings at their defaults. + /// + /// The retry settings are deliberately not taken from the same config + /// block: they are tuned for a query the user is watching, and a collect + /// call that inherited a large `max_retries` would keep an unattended + /// request alive far longer than the caller expects. + #[must_use] + pub fn with_max_response_bytes(mut self, max_response_bytes: u32) -> Self { + self.max_response_bytes = max_response_bytes; + self + } +} + /// Execute `chat_completion_stream` with automatic retries on transient errors. /// /// Collects the full event stream into a `Vec`. @@ -53,6 +79,14 @@ pub async fn collect_with_retry( .chat_completion_stream(model, query.clone()) .await?; + // Bound a runaway response. These collect-style requests run with no + // terminal attached (title generation, summarization, tool inquiries), + // so nobody is watching to interrupt one that never stops. + let stream = match output_limit_bytes(config.max_response_bytes) { + Some(max) => with_output_limit(stream, max), + None => stream, + }; + let error = match stream.try_collect::>().await { // Contract: a well-formed stream ends with `Event::Finished`. A // stream that ends without one was cut short (e.g. a dropped diff --git a/crates/jp_llm/src/retry_tests.rs b/crates/jp_llm/src/retry_tests.rs index eb4139057..a9c608ba1 100644 --- a/crates/jp_llm/src/retry_tests.rs +++ b/crates/jp_llm/src/retry_tests.rs @@ -1,5 +1,36 @@ +use jp_config::{ + assistant::tool_choice::ToolChoice, + model::id::{ModelIdConfig, ProviderId}, +}; +use jp_conversation::{ConversationStream, thread::Thread}; + use super::*; -use crate::error::StreamError; +use crate::{ + error::{Error, StreamError, StreamErrorKind}, + event::Event, + model::ModelDetails, + provider::mock::MockProvider, +}; + +fn empty_query() -> ChatQuery { + ChatQuery { + thread: Thread { + system_prompt: None, + sections: vec![], + attachments: vec![], + events: ConversationStream::new_test(), + }, + tools: vec![], + tool_choice: ToolChoice::Auto, + } +} + +fn model() -> ModelDetails { + ModelDetails::empty(ModelIdConfig { + provider: ProviderId::Test, + name: "mock-model".parse().expect("valid model name"), + }) +} /// Default base backoff for tests. const TEST_BASE_BACKOFF_MS: u64 = 1000; @@ -53,6 +84,38 @@ fn stream_error_is_retryable() { assert!(!StreamError::other("test").is_retryable()); } +#[tokio::test] +async fn collect_with_retry_applies_the_output_ceiling_without_retrying() { + // One scripted request is deliberate. If OutputLimit is misclassified as + // retryable, the mock panics when collect_with_retry asks for a second one. + let provider = MockProvider::with_batches(vec![vec![ + Event::message(0, "0123456789"), + Event::message(0, "0123456789"), + Event::message(0, "0123456789"), + ]]); + let config = RetryConfig { + max_retries: 5, + base_backoff_ms: 1, + max_backoff_secs: 1, + max_response_bytes: 25, + }; + + let error = collect_with_retry(&provider, &model(), empty_query(), &config) + .await + .expect_err("response must stop at the configured ceiling"); + + assert!( + matches!( + error, + Error::Stream(StreamError { + kind: StreamErrorKind::OutputLimit, + .. + }) + ), + "got: {error:?}" + ); +} + #[test] fn stream_error_with_retry_after() { let err = StreamError::rate_limit(Some(Duration::from_secs(30))); diff --git a/crates/jp_llm/src/stream.rs b/crates/jp_llm/src/stream.rs index a0387bbd1..92f71a52a 100644 --- a/crates/jp_llm/src/stream.rs +++ b/crates/jp_llm/src/stream.rs @@ -6,6 +6,7 @@ use std::{ use async_stream::stream; use futures::{Stream, StreamExt as _}; +use serde_json::{Map, Value}; use tokio::time::timeout; use crate::{ @@ -98,6 +99,108 @@ fn with_idle_timeout_at( .boxed() } +/// Wrap an event stream so a runaway response fails instead of generating +/// without bound. +/// +/// Counts the bytes of generated content: assistant messages, reasoning, +/// structured output, tool-call arguments, the identity of each tool call, and +/// any retained metadata carried alongside them. +/// Metadata matters because several providers put generated payload there +/// rather than in the content field: Anthropic sends redacted thinking as an +/// empty reasoning part with the data in metadata, and OpenAI attaches +/// encrypted reasoning content to a flush. +/// Once the running total passes `max_bytes`, the returned stream yields a +/// non-retryable [`StreamErrorKind::OutputLimit`] error and ends. +/// The event that crosses the threshold is forwarded before the error, so +/// content the provider has already generated is never withheld; the total may +/// therefore overshoot `max_bytes` by up to the size of one event. +/// +/// Patches and keep-alives carry nothing generated and do not count. +/// +/// The count covers bytes delivered downstream, which is not the same as bytes +/// the provider billed for. +/// A provider that assembles a response from several requests may discard some +/// of what it generated before yielding it here: Anthropic's continuation +/// chaining drops the reasoning of each continuation and trims the overlap +/// between links. +/// Those bytes are paid for but never counted, so this ceiling bounds retained +/// output rather than spend. +/// The number of continuation requests is bounded separately by the provider. +/// +/// [`StreamErrorKind::OutputLimit`]: crate::StreamErrorKind::OutputLimit +#[must_use] +pub fn with_output_limit(stream: EventStream, max_bytes: u64) -> EventStream { + stream! { + let mut stream = stream; + let mut total: u64 = 0; + while let Some(item) = stream.next().await { + total = total.saturating_add(content_byte_size(&item)); + yield item; + + if total > max_bytes { + yield Err(StreamError::output_limit(format!( + "response exceeded the {max_bytes} byte output limit after {total} bytes" + ))); + break; + } + } + } + .boxed() +} + +/// Translate a configured output ceiling into the argument for +/// [`with_output_limit`], where `0` means the ceiling is disabled. +/// +/// Returns `None` when no ceiling applies, in which case the caller leaves the +/// stream unwrapped. +#[must_use] +pub fn output_limit_bytes(configured: u32) -> Option { + match configured { + 0 => None, + bytes => Some(u64::from(bytes)), + } +} + +/// Byte size of the generated content carried by a stream item. +/// +/// A tool call's `id` and `name` are generated response bytes too, so a stream +/// of nothing but tool-call openings still advances the total. +/// Metadata counts on both parts and flushes, since a provider may deliver +/// reasoning payload there with an empty content field. +/// +/// Errors, patches and keep-alives count as zero. +fn content_byte_size(item: &Result) -> u64 { + let size = match item { + Ok(Event::Part { part, metadata, .. }) => { + let content = match part { + EventPart::Message(text) + | EventPart::Reasoning(text) + | EventPart::Structured(text) => text.len(), + EventPart::ToolCall(ToolCallPart::ArgumentChunk(json)) => json.len(), + EventPart::ToolCall(ToolCallPart::Start { id, name }) => id.len() + name.len(), + }; + + content + metadata_byte_size(metadata) + } + Ok(Event::Flush { metadata, .. }) => metadata_byte_size(metadata), + _ => 0, + }; + + size as u64 +} + +/// Byte size of the retained payload in an event's metadata. +/// +/// Only string values count: those carry the provider payloads that would +/// otherwise escape the ceiling (thinking signatures, redacted thinking, +/// encrypted reasoning content, thought signatures). +fn metadata_byte_size(metadata: &Map) -> usize { + metadata + .values() + .map(|value| value.as_str().map_or(0, str::len)) + .sum() +} + /// Inject a synthetic [`Event::KeepAlive`] every `interval` while a tool call /// is mid-stream. /// diff --git a/crates/jp_llm/src/stream_tests.rs b/crates/jp_llm/src/stream_tests.rs index 1bb7b69cb..2f4068523 100644 --- a/crates/jp_llm/src/stream_tests.rs +++ b/crates/jp_llm/src/stream_tests.rs @@ -7,9 +7,16 @@ use std::{ }; use futures::{StreamExt as _, future, stream}; +use serde_json::Map; -use super::{with_idle_timeout, with_idle_timeout_at, with_tool_call_keepalive}; -use crate::{StreamError, StreamErrorKind, event::Event}; +use super::{ + output_limit_bytes, with_idle_timeout, with_idle_timeout_at, with_output_limit, + with_tool_call_keepalive, +}; +use crate::{ + StreamError, StreamErrorKind, + event::{Event, EventPart, FinishReason}, +}; #[tokio::test(start_paused = true)] async fn idle_timeout_fires_when_wall_clock_exceeds_idle() { @@ -54,6 +61,199 @@ async fn active_stream_passes_through_without_timeout() { assert!(wrapped.next().await.is_none(), "inner stream exhausted"); } +#[tokio::test] +async fn output_limit_fires_once_the_ceiling_is_crossed() { + // Four 10-byte parts against a 25-byte ceiling. The third crosses it, so + // exactly three parts are forwarded before the guard errors: the crossing + // part is still delivered, because content already generated is never + // withheld. The fourth part is never polled. + let inner = stream::iter(vec![ + Ok(Event::reasoning(0, "0123456789")), + Ok(Event::message(1, "0123456789")), + Ok(Event::structured(2, "0123456789")), + Ok(Event::tool_call_args(3, "0123456789")), + ]) + .boxed(); + let mut wrapped = with_output_limit(inner, 25); + + for index in 0..3 { + assert!( + wrapped.next().await.expect("a forwarded part").is_ok(), + "part {index} is forwarded" + ); + } + + let err = wrapped + .next() + .await + .expect("an item before the stream ends") + .expect_err("an output limit error"); + assert_eq!(err.kind, StreamErrorKind::OutputLimit); + assert!( + !err.is_retryable(), + "regenerating a runaway response must not be retried" + ); + + assert!( + wrapped.next().await.is_none(), + "stream ends after the output limit fires" + ); +} + +#[tokio::test] +async fn output_under_the_ceiling_passes_through_untouched() { + let inner = stream::iter(vec![ + Ok(Event::message(0, "0123456789")), + Ok(Event::flush(0)), + Ok(Event::Finished(FinishReason::Completed)), + ]) + .boxed(); + let mut wrapped = with_output_limit(inner, 25); + + assert!(matches!(wrapped.next().await, Some(Ok(Event::Part { .. })))); + assert!(matches!( + wrapped.next().await, + Some(Ok(Event::Flush { .. })) + )); + assert!(matches!( + wrapped.next().await, + Some(Ok(Event::Finished(FinishReason::Completed))) + )); + assert!(wrapped.next().await.is_none(), "inner stream exhausted"); +} + +#[tokio::test] +async fn non_content_events_do_not_count_toward_the_ceiling() { + // Keep-alives, patches, and metadata-free flushes carry nothing generated, + // so a stream of nothing but those must survive a 1-byte ceiling. + let inner = stream::iter(vec![ + Ok(Event::flush(0)), + Ok(Event::KeepAlive), + Ok(Event::Patch(vec![])), + Ok(Event::flush(1)), + ]) + .boxed(); + + let items: Vec<_> = with_output_limit(inner, 1).collect().await; + + assert_eq!(items.len(), 4, "no extra error item is appended"); + assert!(items.iter().all(Result::is_ok), "got: {items:?}"); +} + +#[tokio::test] +async fn retained_part_metadata_counts_toward_the_ceiling() { + // Anthropic delivers redacted thinking as an empty reasoning part with the + // payload in metadata. Counting only the content field would let an + // arbitrarily large retained payload through at zero counted bytes. + let payload = "0123456789".repeat(4); + let inner = stream::iter(vec![ + Ok(Event::Part { + index: 0, + part: EventPart::Reasoning(String::new()), + metadata: Map::from_iter([( + "anthropic_redacted_thinking".to_owned(), + payload.clone().into(), + )]), + }), + Ok(Event::Part { + index: 0, + part: EventPart::Reasoning(String::new()), + metadata: Map::from_iter([("anthropic_redacted_thinking".to_owned(), payload.into())]), + }), + ]) + .boxed(); + + // 40 metadata bytes per part against a 50-byte ceiling: the second crosses. + let mut wrapped = with_output_limit(inner, 50); + + assert!( + wrapped.next().await.expect("the first part").is_ok(), + "the first part is under the ceiling" + ); + assert!(wrapped.next().await.expect("the second part").is_ok()); + + let err = wrapped + .next() + .await + .expect("an item before the stream ends") + .expect_err("an output limit error"); + assert_eq!(err.kind, StreamErrorKind::OutputLimit); +} + +#[tokio::test] +async fn retained_flush_metadata_counts_toward_the_ceiling() { + // OpenAI attaches encrypted reasoning content to the flush rather than to a + // part, so a flush is not automatically free. + let inner = stream::iter(vec![Ok(Event::flush_with_metadata_field( + 0, + "openai_encrypted_content", + "0123456789".repeat(4), + ))]) + .boxed(); + + let mut wrapped = with_output_limit(inner, 25); + + assert!( + wrapped.next().await.expect("the flush").is_ok(), + "the flush is forwarded before the guard errors" + ); + + let err = wrapped + .next() + .await + .expect("an item before the stream ends") + .expect_err("an output limit error"); + assert_eq!(err.kind, StreamErrorKind::OutputLimit); +} + +#[tokio::test] +async fn repeated_tool_call_openings_reach_the_ceiling() { + // A tool call's `id` and `name` are generated response bytes. A model that + // emits nothing but empty tool calls must still hit the ceiling, rather + // than streaming forever at zero counted bytes. + let inner = stream::iter( + std::iter::repeat_with(|| Ok(Event::tool_call_start(0, "call_01", "some_tool"))) + .take(20) + .collect::>(), + ) + .boxed(); + + // 16 bytes per opening (7 + 9) against a 40-byte ceiling: the third crosses + // it at 48 bytes. + let mut wrapped = with_output_limit(inner, 40); + + for index in 0..3 { + assert!( + wrapped.next().await.expect("a forwarded part").is_ok(), + "tool call opening {index} is forwarded" + ); + } + + let err = wrapped + .next() + .await + .expect("an item before the stream ends") + .expect_err("an output limit error"); + assert_eq!(err.kind, StreamErrorKind::OutputLimit); + + assert!( + wrapped.next().await.is_none(), + "stream ends after the output limit fires" + ); +} + +#[test] +fn output_ceiling_of_zero_is_disabled() { + // `0` means "no ceiling" at the config layer. Both call sites route through + // this translation so they cannot drift on what `0` means. + assert!(output_limit_bytes(0).is_none(), "zero disables the ceiling"); + assert_eq!( + output_limit_bytes(512), + Some(512), + "a non-zero ceiling is used as-is" + ); +} + #[tokio::test(start_paused = true)] async fn tool_call_keepalive_emitted_during_open_tool_call() { // A tool-call Start opens the call; the model then goes silent for longer diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap index 2e0108cc0..e641e1987 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap index 35adb93ae..d98887047 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap index 22b64608a..78788a717 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap index ab258e9a9..394faeb3c 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap index 3c7f11b42..9df2ebc3a 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap index 63ce95a79..60b2d2068 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap index 69d3de998..ebb97e09d 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap index 1edd9996c..b51c3e129 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap @@ -43,6 +43,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap index ad904ed07..0331ae9e4 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap index bdf970269..b6723cd27 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap index 8bcfa4c93..bb95a031e 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap index fd5629327..ccf3d15a8 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap index 25dcf5af0..238af4307 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap index c2ba57c97..4e044663d 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap index ca530799f..6901ed7d1 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap index 0531b42d3..e1610296a 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap index 17da7d956..e0a63cf3e 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap index 2f7a5f0fc..ba1f8ad1f 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap index 4fcf90a3b..d9ae51e49 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap index 472490d1e..a7ff7d519 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap index 3440e0399..38628169c 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap index 45787d343..e1d3241b9 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap index 43b88424e..65b8dd17b 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap index a33c4cdd9..b1e3e9e73 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap index b84957d25..5cf10281c 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap index fbe91e33b..968f220a8 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap index ce63c9fc8..9539b1ce9 100644 --- a/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap index f12ea3445..82c66beca 100644 --- a/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap index 138472de5..41e873985 100644 --- a/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap index 0b0a60f81..94be6d746 100644 --- a/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap index 5b3634221..b7b1694dc 100644 --- a/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap index b0dd2fff5..aeabda145 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap index ecd00f8d2..90ac4a0f9 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap index c64174900..e612fcd95 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap index daa217b8a..05e5db625 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap index afb7fda66..3edd2e01e 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap index 4dcce82c2..689d6dc71 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap index 834d2c39c..3cdcd5c76 100644 --- a/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap index 43f937460..d864bd91b 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap index d8c66a925..69c6035a4 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap index 2477c69be..15afbecf9 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap index 1a0b8c5d5..09eb44bcf 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap index 1ebae2a00..75fa76af2 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap index f8bc51d2c..965ca6990 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap index f9e64995a..ce46cc7a1 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap index 51f67f3bf..710faa290 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap index 5d1f1b61c..45081ac29 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap index b33b3a851..5701fe424 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap index 535b66f67..f1b686d2f 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap index 9883fd181..d81ab5abb 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap index 667a6be09..3e1f77c24 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap index 28d7e2c85..0cdb949e6 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap index 3db76ff28..52568a0dd 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap index b70e99e32..3211f380c 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap index 49ac21f65..f1153beed 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap index 97bbe2418..342e3747c 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap index 6c3d1e47f..8694f4761 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap index 739c1a2c0..b76f9410e 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap index 076b56dd5..c7476bc94 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap index 176cb2ebc..86122b774 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": false } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap index 5317cff76..5f12d4723 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap @@ -43,6 +43,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap index 27b0e85f5..1f95a9710 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap index f4160e26c..05ce769f6 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap index 65c7cf1c2..1df377104 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap index 134ea4ed7..01b6813f7 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap index 4faad3c57..242b3b7f8 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap index 257b79020..f91177d7c 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap index e456cf64e..5d386ae55 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap index 4672960ba..f4c1a457e 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap index 6dd8b1eb3..9bf46e034 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap index ab8d45ecd..dc636f997 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap index 0acd739df..b25a665dd 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap index e12846e54..a1a63fb4b 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap index c99fccb97..26d454dd2 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap index 021c931f9..af2c77bd0 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap index 9a31ff2cd..2bd5aeeac 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap index d1a0598b7..c5959d8f9 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap index 083cb1b71..e1b82ac1d 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap index 0bcd002b0..dfcdf7de8 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap index 8f4157f5c..2e5900b35 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap index 903b68a7e..20abfef2d 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap index 745486cd2..84f84627d 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap index 0acf0b0bc..3ec63d7fd 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap index 414892243..f1272e92d 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap index 05de0c9f5..34b824bf7 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap @@ -38,6 +38,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap index c250560bb..0b5cfc7de 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap @@ -41,6 +41,7 @@ expression: v "base_backoff_ms": 1000, "max_backoff_secs": 60, "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, "cache": true } }, diff --git a/crates/jp_task/src/task/title_generator.rs b/crates/jp_task/src/task/title_generator.rs index 708261f0e..f364f5a21 100644 --- a/crates/jp_task/src/task/title_generator.rs +++ b/crates/jp_task/src/task/title_generator.rs @@ -35,6 +35,9 @@ pub struct TitleGeneratorTask { pub providers: LlmProviderConfig, pub events: ConversationStream, pub title: Option, + /// Output ceiling for the title request, from + /// `assistant.request.max_response_bytes`. + pub max_response_bytes: u32, /// Whether the invoking process is attached to a terminal. /// When `false`, the OSC-2 title-update side effect on task sync is /// suppressed — the bytes would otherwise leak into a captured pipe. @@ -87,6 +90,7 @@ impl TitleGeneratorTask { providers: config.providers.llm.clone(), events, title: None, + max_response_bytes: config.assistant.request.max_response_bytes, is_tty, }) } @@ -117,7 +121,7 @@ impl TitleGeneratorTask { tool_choice: jp_config::assistant::tool_choice::ToolChoice::default(), }; - let retry_config = RetryConfig::default(); + let retry_config = RetryConfig::default().with_max_response_bytes(self.max_response_bytes); let llm_events = collect_with_retry(provider.as_ref(), &model, query, &retry_config).await?;