From 12e6a7f90de1e5e8567190116fd16b22c338d8d6 Mon Sep 17 00:00:00 2001 From: will wade Date: Tue, 18 Aug 2026 09:04:49 +0000 Subject: [PATCH] fix(cloud): azure/edge silent zero-audio for bare envelopes and MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Azure/Edge failure modes complete the turn normally (turn.end / HTTP 200) but synthesise zero audio, with no error anywhere — consumers saw a silent success: - A bare envelope (no version/xmlns/xml:lang) — exactly what speech-dispatcher's index-marking wrapper sends. normalize_ssml_envelope() now completes the envelope before the request goes out (xml:lang derived from the voice name, like build_azure_ssml). - An SSML element, which Azure/Edge don't support — speech-dispatcher injects around every pause. strip_unsupported_marks() drops the (empty, unspoken) elements. Defense in depth: a turn that finishes cleanly with zero audio now returns Err("synthesis completed with no audio") on both the WS path (azure/edge) and every REST response path, so any future silent zero-audio failure surfaces as an error. Verified against the live Edge endpoint: plain text, bare , and documents all synthesise (examples/edge-bare-envelope.rs). --- examples/edge-bare-envelope.rs | 34 ++++ src/cloud_engine.rs | 311 ++++++++++++++++++++++++++++++++- 2 files changed, 339 insertions(+), 6 deletions(-) create mode 100644 examples/edge-bare-envelope.rs diff --git a/examples/edge-bare-envelope.rs b/examples/edge-bare-envelope.rs new file mode 100644 index 0000000..a9990ef --- /dev/null +++ b/examples/edge-bare-envelope.rs @@ -0,0 +1,34 @@ +//! Live regression check (not part of CI): Edge must synthesise audio for +//! the bare `` envelope speech-dispatcher sends. Run with: +//! cargo run --no-default-features --features cloud --example edge-bare-envelope +//! Exits non-zero when either variant produces no audio. + +use rust_tts_wrapper::factory::create_engine; + +fn main() { + let engine = create_engine("edge", "{}").expect("edge engine"); + + for text in [ + "Plain text reference.", + "Repeat test number 3", + "With a mark", + "With a mark", + ] { + let mut bytes = 0usize; + let mut words = 0usize; + engine + .speak( + text, + Some("en-GB-SoniaNeural"), + 1.0, + 1.0, + 1.0, + Some(&mut |chunk: &[u8]| bytes += chunk.len()), + Some(&mut |_w, _s, _e, _o, _l| words += 1), + ) + .unwrap_or_else(|e| panic!("{text}: speak failed: {e}")); + println!("{text:?}: {bytes} PCM bytes, {words} word boundaries"); + assert!(bytes > 0, "{text}: no audio!"); + } + println!("PASS"); +} diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 6baf89c..1b09e1e 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1094,6 +1094,142 @@ fn inject_voice_if_missing(ssml: &str, voice: &str) -> String { ssml.to_string() } +/// The SSML 1.0 synthesis namespace (``). +const SSML_XMLNS: &str = "http://www.w3.org/2001/10/synthesis"; + +/// Derive a BCP-47 language tag from an Azure/Edge voice name +/// (`en-GB-SoniaNeural` → `en-GB`), defaulting to `en-US` when the name +/// doesn't look like a locale. +fn voice_lang(voice: &str) -> String { + let head: String = voice.chars().take(5).collect(); + let chars: Vec = head.chars().collect(); + let locale_like = chars.len() == 5 + && chars[2] == '-' + && [0, 1, 3, 4].iter().all(|&i| chars[i].is_ascii_alphabetic()); + if locale_like { + head + } else { + "en-US".to_string() + } +} + +/// Complete an SSML document's `` envelope with the attributes +/// Azure/Edge require: `version`, `xmlns` and `xml:lang`. +/// +/// A bare `` — exactly what speech-dispatcher's index-marking +/// wrapper produces, and common from other SSML emitters — is *accepted* +/// by the service: the turn completes with `turn.end` and no error, but +/// **zero audio is synthesised**. Filling in the missing attributes +/// before the request goes out makes such documents speak. Present +/// attributes and the rest of the document are passed through verbatim; +/// plain text (no ` String { + let trimmed = ssml.trim_start(); + if !trimmed.to_ascii_lowercase().starts_with("') else { + return ssml.to_string(); // unterminated tag — leave alone + }; + let inner = trimmed[..=tag_end] + .strip_prefix('<') + .and_then(|t| t.strip_suffix('>')) + .unwrap_or(""); + // Attribute names present in the tag (name only, up to `=`). + let mut has_version = false; + let mut has_xmlns = false; + let mut has_lang = false; + for attr in inner.split_whitespace().skip(1) { + match attr + .split('=') + .next() + .unwrap_or("") + .to_ascii_lowercase() + .as_str() + { + "version" => has_version = true, + "xmlns" | "xmlns:xmlns" => has_xmlns = true, + "xml:lang" => has_lang = true, + _ => {} + } + } + if has_version && has_xmlns && has_lang { + return ssml.to_string(); // nothing to do + } + let mut words = inner.split_whitespace(); + // XML tag names are case-sensitive: keep the original spelling so the + // opening tag still matches a `` close. + let first = words.next().unwrap_or("speak"); + let self_closing = first.ends_with('/'); + let name = first.trim_end_matches('/'); + let mut open = format!("<{name}"); + // Keep custom attributes, dropping a trailing self-closing slash. + let attrs = words.collect::>().join(" "); + let self_closing = self_closing || attrs.ends_with('/'); + let attrs = attrs.trim_end_matches('/'); + if !attrs.is_empty() { + open.push(' '); + open.push_str(attrs); + } + if !has_version { + open.push_str(" version=\"1.0\""); + } + if !has_xmlns { + open.push_str(" xmlns=\""); + open.push_str(SSML_XMLNS); + open.push('"'); + } + if !has_lang { + open.push_str(" xml:lang=\""); + open.push_str(&voice_lang(voice)); + open.push('"'); + } + if self_closing { + open.push('/'); + } + open.push('>'); + let lead = &ssml[..ssml.len() - trimmed.len()]; + format!("{lead}{open}{}", &trimmed[tag_end + 1..]) +} + +/// Remove `` elements from an SSML document for Azure/Edge. +/// +/// Azure/Edge do not support the SSML `` element — an utterance +/// containing one synthesises **zero audio**, again with no error from +/// the service. `` is an empty element (it only names a position), +/// so dropping it changes no spoken content; consumers that need the +/// positions should use word-boundary events. speech-dispatcher's +/// wrapper injects `` around every pause, so +/// pass-through SSML from SSIP clients hits this constantly. +#[cfg(feature = "cloud")] +fn strip_unsupported_marks(ssml: &str) -> String { + if !ssml.contains("` stays untouched. + let name_done = |s: &str, prefix: &str| { + s.strip_prefix(prefix) + .is_some_and(|tail| tail.starts_with([' ', '\t', '\r', '\n', '/', '>'])) + }; + if name_done(after, "') { + out.push_str(&rest[..pos]); + rest = &after[end + 1..]; + continue; + } + } + out.push_str(&rest[..=pos]); + rest = &rest[pos + 1..]; + } + out.push_str(rest); + out +} + /// Build SSML for Azure TTS. fn build_azure_ssml(text: &str, voice: &str, rate: f32, pitch: f32, volume: f32) -> String { let lang = voice.chars().take(5).collect::(); @@ -1781,6 +1917,12 @@ impl TtsEngine for CloudEngine { // tracks whether the session ended on `turn.end` (socket reusable → // check back in) so a broken connection is never pooled. let mut clean_finish = false; + // Total synthesis audio received on the wire. A turn that ends + // cleanly (turn.end, no error) with zero audio means the request + // was malformed in a way the service doesn't report — the classic + // case being a bare envelope — and must surface as an + // error, not a silent success. + let mut ws_audio_bytes = 0usize; let mut socket = match ws_checkout(&ws_url_str) { Some(pooled) => pooled, None => { @@ -1860,9 +2002,14 @@ impl TtsEngine for CloudEngine { // send it directly without build_azure_ssml wrapping (which would // XML-escape the tags). If the SSML lacks a tag but the // caller set one via tts_set_voice, inject it so the voice takes - // effect. + // effect. The envelope is completed first (a bare is + // accepted but synthesises zero audio) and unsupported + // elements are dropped (same silent zero-audio failure). let ssml = if is_ssml { - inject_voice_if_missing(&text, &voice_to_use) + inject_voice_if_missing( + &strip_unsupported_marks(&normalize_ssml_envelope(&text, &voice_to_use)), + &voice_to_use, + ) } else { build_azure_ssml(&text, &voice_to_use, rate, pitch, volume) }; @@ -2018,6 +2165,7 @@ impl TtsEngine for CloudEngine { let header_length = ((b[0] as usize) << 8) | (b[1] as usize); if b.len() > 2 + header_length { let audio = &b[2 + header_length..]; + ws_audio_bytes += audio.len(); if self.config.response_is_pcm { // Azure raw-PCM frames — deliver straight through. if let Some(cb) = on_audio.as_mut() { @@ -2090,6 +2238,12 @@ impl TtsEngine for CloudEngine { if clean_finish { ws_checkin(ws_url_str, socket); } + if ws_audio_bytes == 0 { + return Err(TtsError(format!( + "{} synthesis completed with no audio (malformed SSML envelope?)", + self.config.provider_id + ))); + } return Ok(()); } @@ -2114,9 +2268,14 @@ impl TtsEngine for CloudEngine { let resp = if self.config.body_is_ssml { // Azure: send SSML XML body. When is_ssml=true, the text is // already SSML — send it directly (don't escape/wrap with - // build_azure_ssml). Inject voice if the SSML lacks a tag. + // build_azure_ssml). Inject voice if the SSML lacks a + // tag, after completing the envelope and dropping unsupported + // elements (both make Azure synthesise zero audio). let ssml = if is_ssml { - inject_voice_if_missing(&text, &voice_to_use) + inject_voice_if_missing( + &strip_unsupported_marks(&normalize_ssml_envelope(&text, &voice_to_use)), + &voice_to_use, + ) } else { build_azure_ssml(&text, &voice_to_use, rate, pitch, volume) }; @@ -2175,6 +2334,12 @@ impl TtsEngine for CloudEngine { return Err(TtsError(format!("API error {status}: {body_text}"))); } + // Total audio delivered for this utterance. A 2xx response with no + // audio at all is a failure (malformed request the service didn't + // reject, empty synthesis, …) — reported as an error rather than a + // silent success. + let mut audio_total = 0usize; + if self.config.provider_id == "elevenlabs" && on_boundary.is_some() { let resp_text = resp .text() @@ -2188,6 +2353,7 @@ impl TtsEngine for CloudEngine { .decode(b64) .map_err(|e| TtsError(format!("Base64 decode: {e}")))?; let pcm = decode_mp3_to_pcm16_mono(&mp3_bytes); + audio_total += pcm.len(); if let Some(cb) = on_audio.as_mut() { for chunk in pcm.chunks(STREAMING_CHUNK_SIZE) { cb(chunk); @@ -2228,6 +2394,7 @@ impl TtsEngine for CloudEngine { .decode(b64) .map_err(|e| TtsError(format!("Base64 decode: {e}")))?; let pcm = decode_mp3_to_pcm16_mono(&mp3_bytes); + audio_total += pcm.len(); if let Some(cb) = on_audio.as_mut() { for chunk in pcm.chunks(STREAMING_CHUNK_SIZE) { cb(chunk); @@ -2289,7 +2456,10 @@ impl TtsEngine for CloudEngine { // delivered audio, instead of all-at-once afterwards. let plan = on_boundary.is_some().then(|| EstimatePlan::build(&text)); let mut on_event = |ev: StreamEvt<'_>| match ev { - StreamEvt::Audio(bytes) => cb(bytes), + StreamEvt::Audio(bytes) => { + audio_total += bytes.len(); + cb(bytes); + } StreamEvt::Boundary(word, start, end, offset, len) => { if let Some(bcb) = on_boundary.as_mut() { bcb(word, start, end, offset, len); @@ -2307,9 +2477,16 @@ impl TtsEngine for CloudEngine { ) .map_err(TtsError)?; } else { - let _audio_bytes = resp + let audio_bytes = resp .bytes() .map_err(|e| TtsError(format!("Read error: {e}")))?; + audio_total += audio_bytes.len(); + } + if audio_total == 0 { + return Err(TtsError(format!( + "{} synthesis returned no audio", + self.config.provider_id + ))); } Ok(()) } @@ -3284,6 +3461,128 @@ mod tests { assert!(ssml.contains("volume=\"+40%\"")); } + // ===== normalize_ssml_envelope ===== + + #[test] + fn test_normalize_envelope_fills_missing_attributes() { + // Exactly what speech-dispatcher's index-marking wrapper sends: + // a bare . Azure/Edge accept it but synthesise no audio. + let result = normalize_ssml_envelope( + "Repeat test", + "en-GB-SoniaNeural", + ); + let expected_prefix = + format!(""); + assert!( + result.starts_with(&expected_prefix), + "envelope attributes must be added in order: {result}" + ); + assert!(result.ends_with("Repeat test")); + } + + #[test] + fn test_normalize_envelope_keeps_present_attributes() { + let ssml = "Hallo"; + assert_eq!(normalize_ssml_envelope(ssml, "en-US-AriaNeural"), ssml); + } + + #[test] + fn test_normalize_envelope_fills_only_gaps() { + let result = normalize_ssml_envelope( + "Bonjour", + "en-US-AriaNeural", + ); + assert!(result.contains("xml:lang='fr-FR'"), "existing lang kept"); + assert!(result.contains("version=\"1.0\""), "version added"); + assert!(result.contains("xmlns="), "xmlns added"); + // The added xml:lang must not duplicate the existing one. + assert_eq!(result.matches("xml:lang").count(), 1); + } + + #[test] + fn test_normalize_envelope_leaves_plain_text_and_fragments_alone() { + assert_eq!( + normalize_ssml_envelope("Angle < bracket", "en-US-AriaNeural"), + "Angle < bracket" + ); + // Envelope-less SSML fragment (no hi", "en-US-AriaNeural"), + "hi" + ); + // Unterminated tag — left alone rather than mangled. + assert_eq!( + normalize_ssml_envelope("Hi", "en-US-AriaNeural"); + assert!(result.starts_with("")); + + let result = normalize_ssml_envelope("", "en-US-AriaNeural"); + let expected = + format!(""); + assert_eq!(result, expected); + } + + #[test] + fn test_normalize_envelope_composes_with_voice_injection() { + // The WS/REST send path normalizes first, then injects . + let result = inject_voice_if_missing( + &normalize_ssml_envelope("hello", "en-GB-SoniaNeural"), + "en-GB-SoniaNeural", + ); + assert!(result.contains("version=\"1.0\"")); + assert!(result.contains("xml:lang=\"en-GB\"")); + assert!(result.contains("")); + } + + #[test] + fn test_voice_lang_from_voice_name() { + assert_eq!(voice_lang("en-GB-SoniaNeural"), "en-GB"); + assert_eq!(voice_lang("en-US-AvaMultilingualNeural"), "en-US"); + assert_eq!(voice_lang("alloy"), "en-US"); // not a locale + assert_eq!(voice_lang(""), "en-US"); + } + + // ===== strip_unsupported_marks ===== + + #[test] + fn test_strip_marks_removes_self_closing_and_paired() { + // The exact shape speech-dispatcher wraps around pauses. + assert_eq!( + strip_unsupported_marks("A B"), + "A B" + ); + assert_eq!( + strip_unsupported_marks("A B"), + "A B" + ); + } + + #[test] + fn test_strip_marks_leaves_similar_names_and_text_alone() { + assert_eq!( + strip_unsupported_marks(""), + "" + ); + assert_eq!(strip_unsupported_marks("a < b"), "a < b"); + assert_eq!(strip_unsupported_marks(" world"; + let stripped = strip_unsupported_marks(ssml); + assert!(!stripped.contains("mark")); + assert_eq!(stripped, "Hello world"); + } + // ===== inject_voice_if_missing ===== #[test]