From dc53d323fc76d5caa6faa7dc36ae1b392751e585 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:26:10 -0400 Subject: [PATCH 01/74] feat(voice): keep conversations active in background --- src-tauri/src/commands/mod.rs | 2 + src-tauri/src/commands/native_voice.rs | 199 +++++++++++++----- src-tauri/src/commands/voice_menu_bar.rs | 108 ++++++++++ src-tauri/src/commands/window_session.rs | 2 + src-tauri/src/lib.rs | 2 + src/app/AppShell.navigation.test.tsx | 56 ++--- src/app/AppShell.tsx | 58 +++-- .../api/voiceConversation.test.ts | 24 ++- .../api/voiceConversation.ts | 40 ++++ .../hooks/usePocketVoiceSetup.test.ts | 1 + .../useVoiceConversationController.test.ts | 77 +------ .../hooks/useVoiceConversationController.ts | 60 ------ .../lib/nativeAssistantSpeech.test.ts | 1 + .../stores/voiceConversationStore.test.ts | 1 + .../stores/voiceConversationStore.ts | 30 ++- 15 files changed, 397 insertions(+), 264 deletions(-) create mode 100644 src-tauri/src/commands/voice_menu_bar.rs diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 485664fe5..4af7e3bec 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -52,6 +52,8 @@ pub mod telemetry; pub mod terminal; pub mod updates; pub mod voice_capture; +#[cfg(target_os = "macos")] +pub mod voice_menu_bar; pub mod whoami; pub mod window_session; pub mod workspace_context; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index ffaa4a376..dffaf7f6d 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -48,6 +48,7 @@ pub struct NativeVoiceStatus { lifecycle: Lifecycle, session_id: Option, owner_window_label: Option, + microphone_muted: bool, revision: u64, native_microphone_mute_control: bool, native_microphone_muted: bool, @@ -103,6 +104,11 @@ enum NativeVoiceEvent { muted: bool, revision: u64, }, + MicrophoneMute { + session_id: String, + muted: bool, + revision: u64, + }, CleanShutdown { session_id: String, revision: u64, @@ -137,6 +143,7 @@ pub struct NativeVoiceState { capture_suppressions: Arc, input_muted: Arc, input_mute_epoch: Arc, + microphone_muted: Arc, } #[must_use = "capture suppression ends when the guard is dropped"] @@ -170,6 +177,54 @@ impl NativeVoiceState { fn capture_is_suppressed(&self) -> bool { self.capture_suppressions.load(Ordering::SeqCst) > 0 } + + pub fn microphone_is_muted(&self) -> bool { + self.microphone_muted.load(Ordering::SeqCst) + } + + #[cfg(target_os = "macos")] + pub fn active_session_target(&self) -> Option<(String, String)> { + let runtime = self.runtime.lock().ok()?; + Some(( + runtime.session_id.clone()?, + runtime.owner.as_ref()?.window_label.clone(), + )) + } + + pub fn set_microphone_muted(&self, app: &AppHandle, muted: bool) -> Result<(), String> { + let (session_id, owner_window_label, revision) = { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + let session_id = runtime + .session_id + .clone() + .ok_or_else(|| "No native voice conversation is active.".to_string())?; + let owner_window_label = runtime + .owner + .as_ref() + .map(|owner| owner.window_label.clone()) + .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; + (session_id, owner_window_label, runtime.revision) + }; + self.microphone_muted.store(muted, Ordering::SeqCst); + #[cfg(target_os = "macos")] + if let Err(error) = super::voice_menu_bar::set_muted(app, muted) { + log::warn!("Failed to update the voice menu bar mute state: {error}"); + } + if let Some(window) = app.get_webview_window(&owner_window_label) { + let _ = window.emit( + EVENT_NAME, + NativeVoiceEvent::MicrophoneMute { + session_id, + muted, + revision, + }, + ); + } + Ok(()) + } } enum SttMessage { @@ -333,6 +388,7 @@ fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { .owner .as_ref() .map(|owner| owner.window_label.clone()), + microphone_muted: state.microphone_is_muted(), revision: runtime.revision, native_microphone_mute_control: runtime.native_microphone_mute_control, native_microphone_muted: runtime.session_id.is_some() @@ -499,12 +555,18 @@ pub async fn start_native_voice_conversation( }, ); }); + state.microphone_muted.store(false, Ordering::SeqCst); ( runtime.revision, runtime.lifecycle_id.clone().unwrap_or_default(), runtime.native_microphone_mute_control, ) }; + #[cfg(target_os = "macos")] + if let Err(error) = super::voice_menu_bar::install(&app, false) { + state.stop_active(&app, &capture).await?; + return Err(format!("Could not install the voice menu bar: {error}")); + } let _ = webview_window.emit( EVENT_NAME, NativeVoiceEvent::Startup { @@ -521,6 +583,7 @@ pub async fn start_native_voice_conversation( let runtime = Arc::clone(&state.runtime); let pending = Arc::clone(&state.pending); let input_muted = Arc::clone(&state.input_muted); + let event_state = state.inner().clone(); tauri::async_runtime::spawn(async move { while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { @@ -604,6 +667,9 @@ pub async fn start_native_voice_conversation( if let Some(pipeline) = pipeline { shutdown_pipeline(pipeline).await; } + event_state.microphone_muted.store(false, Ordering::SeqCst); + #[cfg(target_os = "macos")] + super::voice_menu_bar::remove(&event_app); event_app .state::() .release_owner(&window_label, &owner_id); @@ -624,6 +690,16 @@ pub async fn start_native_voice_conversation( Ok(status(&app, &state)) } +#[tauri::command] +pub fn set_native_voice_microphone_muted( + app: AppHandle, + state: State<'_, NativeVoiceState>, + muted: bool, +) -> Result { + state.set_microphone_muted(&app, muted)?; + Ok(status(&app, &state)) +} + #[tauri::command] pub async fn stop_native_voice_conversation( app: AppHandle, @@ -634,58 +710,7 @@ pub async fn stop_native_voice_conversation( renderer_epoch: u64, ) -> Result { capture.activate_renderer(webview_window.label(), &renderer_id, renderer_epoch)?; - let (session_id, revision, pipeline, owner) = { - let mut runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - let owner = runtime.owner.clone(); - let session_id = runtime.session_id.clone(); - let owner_id = session_id.as_deref().map(native_owner_id); - let revision = runtime.revision; - ( - session_id, - revision, - runtime.pipeline.take(), - owner.zip(owner_id), - ) - }; - // Keep the lifecycle current while the worker flushes its final buffered - // utterance into the durable pending queue. - if let Some(pipeline) = pipeline { - shutdown_pipeline(pipeline).await; - } - let revision = { - let mut runtime = state - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.revision == revision && runtime.session_id == session_id { - native_input_mute::stop(&state.input_muted); - runtime.native_microphone_mute_control = false; - runtime.session_id = None; - runtime.lifecycle_id = None; - runtime.owner = None; - runtime.revision = runtime.revision.wrapping_add(1); - } - runtime.revision - }; - if let Some((owner, owner_id)) = owner.as_ref() { - capture.release_owner(&owner.window_label, owner_id); - } - if let Some(session_id) = session_id { - let target = owner - .as_ref() - .and_then(|(owner, _)| app.get_webview_window(&owner.window_label)) - .unwrap_or(webview_window); - let _ = target.emit( - EVENT_NAME, - NativeVoiceEvent::CleanShutdown { - session_id, - revision, - }, - ); - } + state.stop_active(&app, &capture).await?; Ok(status(&app, &state)) } @@ -694,6 +719,65 @@ fn native_owner_id(session_id: &str) -> String { } impl NativeVoiceState { + pub async fn stop_active( + &self, + app: &AppHandle, + capture: &VoiceCaptureState, + ) -> Result<(), String> { + let (session_id, revision, pipeline, owner) = { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + let owner = runtime.owner.clone(); + let session_id = runtime.session_id.clone(); + let owner_id = session_id.as_deref().map(native_owner_id); + let revision = runtime.revision; + ( + session_id, + revision, + runtime.pipeline.take(), + owner.zip(owner_id), + ) + }; + // Keep the lifecycle current while the worker flushes its final buffered + // utterance into the durable pending queue. + if let Some(pipeline) = pipeline { + shutdown_pipeline(pipeline).await; + } + let next_revision = { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.revision == revision && runtime.session_id == session_id { + runtime.session_id = None; + runtime.lifecycle_id = None; + runtime.owner = None; + runtime.revision = runtime.revision.wrapping_add(1); + } + runtime.revision + }; + self.microphone_muted.store(false, Ordering::SeqCst); + #[cfg(target_os = "macos")] + super::voice_menu_bar::remove(app); + if let Some((owner, owner_id)) = owner.as_ref() { + capture.release_owner(&owner.window_label, owner_id); + } + if let (Some(session_id), Some((owner, _))) = (session_id, owner) { + if let Some(target) = app.get_webview_window(&owner.window_label) { + let _ = target.emit( + EVENT_NAME, + NativeVoiceEvent::CleanShutdown { + session_id, + revision: next_revision, + }, + ); + } + } + Ok(()) + } + pub async fn stop_for_model_removal( &self, app: &AppHandle, @@ -729,6 +813,9 @@ impl NativeVoiceState { } runtime.revision }; + self.microphone_muted.store(false, Ordering::SeqCst); + #[cfg(target_os = "macos")] + super::voice_menu_bar::remove(app); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); if let Some(window) = app.get_webview_window(&owner.window_label) { @@ -807,6 +894,7 @@ impl NativeVoiceState { ) }; drop(pipeline); + self.microphone_muted.store(false, Ordering::SeqCst); if let Ok(mut runtime) = self.runtime.lock() { if runtime.revision == revision && runtime.session_id == session_id { runtime.session_id = None; @@ -879,7 +967,7 @@ fn push_audio_for_window( { return Err("Only the owning window may send native voice audio.".to_string()); } - if state.capture_is_suppressed() { + if state.capture_is_suppressed() || state.microphone_is_muted() { return Ok(()); } if let Some(pipeline) = runtime.pipeline.as_ref() { @@ -1361,6 +1449,11 @@ mod tests { assert!(push_audio_for_window(&state, "other-window", vec![0; 4]).is_err()); assert!(receiver.try_recv().is_err()); + state.microphone_muted.store(true, Ordering::SeqCst); + push_audio_for_window(&state, "owner-window", vec![0; 4]) + .expect("muted owner audio is ignored"); + assert!(receiver.try_recv().is_err()); + state.microphone_muted.store(false, Ordering::SeqCst); push_audio_for_window(&state, "owner-window", vec![0; 4]).expect("owner can send audio"); assert_eq!( receiver.try_recv().expect("owner audio queued").bytes, diff --git a/src-tauri/src/commands/voice_menu_bar.rs b/src-tauri/src/commands/voice_menu_bar.rs new file mode 100644 index 000000000..59dad4182 --- /dev/null +++ b/src-tauri/src/commands/voice_menu_bar.rs @@ -0,0 +1,108 @@ +//! macOS menu bar controls for the process-wide native voice conversation. + +use serde::Serialize; +use tauri::{ + menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem}, + tray::TrayIconBuilder, + AppHandle, Emitter, Manager, WebviewWindow, +}; + +use super::{native_voice::NativeVoiceState, voice_capture::VoiceCaptureState}; + +const TRAY_ID: &str = "voice-conversation"; +const MUTE_ID: &str = "voice-conversation-mute"; +const OPEN_ID: &str = "voice-conversation-open"; +const STOP_ID: &str = "voice-conversation-stop"; +pub const OPEN_SESSION_EVENT: &str = "voice-conversation:open-session"; + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpenSessionPayload { + session_id: String, +} + +fn menu(app: &AppHandle, muted: bool) -> tauri::Result> { + let status = MenuItem::new(app, "Voice conversation active", false, None::<&str>)?; + let mute = CheckMenuItem::with_id(app, MUTE_ID, "Mute Microphone", true, muted, None::<&str>)?; + let open = MenuItem::with_id(app, OPEN_ID, "Open Voice Session", true, None::<&str>)?; + let stop = MenuItem::with_id(app, STOP_ID, "Stop Voice Conversation", true, None::<&str>)?; + let separator = PredefinedMenuItem::separator(app)?; + Menu::with_items(app, &[&status, &separator, &mute, &open, &stop]) +} + +pub fn install(app: &AppHandle, muted: bool) -> Result<(), String> { + remove(app); + let menu = menu(app, muted).map_err(|error| error.to_string())?; + TrayIconBuilder::with_id(TRAY_ID) + .menu(&menu) + .title(if muted { "🔇" } else { "🎙" }) + .tooltip("Berd voice conversation") + .build(app) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +pub fn set_muted(app: &AppHandle, muted: bool) -> Result<(), String> { + let Some(tray) = app.tray_by_id(TRAY_ID) else { + return Ok(()); + }; + tray.set_title(Some(if muted { "🔇" } else { "🎙" })) + .map_err(|error| error.to_string())?; + tray.set_menu(Some(menu(app, muted).map_err(|error| error.to_string())?)) + .map_err(|error| error.to_string()) +} + +pub fn remove(app: &AppHandle) { + let _ = app.remove_tray_by_id(TRAY_ID); +} + +fn focus_window(window: &WebviewWindow) { + let _ = window.show(); + let _ = window.unminimize(); + let _ = window.set_focus(); +} + +fn open_voice_session(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let Some((session_id, owner_window_label)) = state.active_session_target() else { + return Ok(()); + }; + let window = app + .get_webview_window(&owner_window_label) + .ok_or_else(|| "The voice session window is no longer available.".to_string())?; + focus_window(&window); + if owner_window_label == "main" { + window + .emit(OPEN_SESSION_EVENT, OpenSessionPayload { session_id }) + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +pub fn handle_menu_event(app: &AppHandle, event: tauri::menu::MenuEvent) { + match event.id().as_ref() { + MUTE_ID => { + let state = app.state::(); + let muted = !state.microphone_is_muted(); + if let Err(error) = state.set_microphone_muted(app, muted) { + log::warn!("Failed to update voice microphone mute: {error}"); + } + } + OPEN_ID => { + if let Err(error) = open_voice_session(app) { + log::warn!("Failed to open the voice session: {error}"); + } + } + STOP_ID => { + let app = app.clone(); + tauri::async_runtime::spawn(async move { + let state = app.state::().inner().clone(); + let capture = app.state::(); + if let Err(error) = state.stop_active(&app, capture.inner()).await { + log::warn!("Failed to stop the voice conversation from the menu bar: {error}"); + } + }); + } + _ => {} + } +} diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index 6af73a640..6e8f851f2 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -684,6 +684,8 @@ pub fn open_session_window( .state::() .stop_for_window_destroyed(&label_for_close); if stopped_native_voice { + #[cfg(target_os = "macos")] + crate::commands::voice_menu_bar::remove(&app_for_close); app_for_close .state::() .stop_for_window_destroyed(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8672de987..f67053603 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -384,6 +384,7 @@ pub fn run() { set_dev_dock_icon(); refresh_traffic_light_position_on_window_changes(app); attach_main_window_lifecycle(app); + app.on_menu_event(commands::voice_menu_bar::handle_menu_event); let app_menu = SubmenuBuilder::new(app, "Berd") .about_with_text( @@ -640,6 +641,7 @@ pub fn run() { commands::siri_voice::finish_siri_voice_stream, commands::siri_voice::stop_siri_voice, commands::native_voice::get_native_voice_conversation_status, + commands::native_voice::set_native_voice_microphone_muted, commands::native_voice::drain_native_voice_conversation_transcripts, commands::native_voice::acknowledge_native_voice_conversation_transcript, commands::native_voice::reject_native_voice_conversation_transcript, diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index 095c4cb48..b15233cbc 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -29,6 +29,7 @@ import { OPEN_SETTINGS_EVENT } from "@/features/settings/lib/settingsEvents"; import { SHORTCUT_PREFERENCES_STORAGE_KEY } from "@/features/shortcuts/lib/shortcutRegistry"; import { useShortcutsDialogStore } from "@/features/shortcuts/stores/shortcutsDialogStore"; import { useProjectStore } from "@/features/projects/stores/projectStore"; +import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; import { dispatchOnboarding } from "@/features/onboarding/model"; import { resetHomeWidgetStoreForTests, @@ -60,7 +61,6 @@ import { import { AppShell, shouldStopVoiceConversationOnExperimentChange, - shouldStopVoiceConversationOnSessionChange, } from "./AppShell"; import type { NavigationPanesViewProps } from "@/app/views/NavigationPanesView"; import type { AppShellContent as AppShellContentType } from "./ui/AppShellContent"; @@ -872,46 +872,6 @@ describe("AppShell global navigation", () => { ).toBe(false); }); - it("stops voice only when navigation leaves its bound chat", () => { - const base = { - previousSessionId: "session-1", - boundSessionId: "session-1", - lifecycle: "running", - }; - - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - }), - ).toBe(true); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: null, - }), - ).toBe(true); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-1", - }), - ).toBe(false); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - boundSessionId: "session-elsewhere", - }), - ).toBe(false); - expect( - shouldStopVoiceConversationOnSessionChange({ - ...base, - nextSessionId: "session-2", - lifecycle: "stopped", - }), - ).toBe(false); - }); afterEach(cleanup); beforeEach(() => { @@ -5546,6 +5506,7 @@ describe("AppShell global navigation", () => { it("cycles sessions with Ctrl+Tab and Ctrl+Shift+Tab", async () => { const user = userEvent.setup(); + const stopVoiceConversation = vi.fn(); const sessionBase = { executionTarget: { harnessId: "goose" }, workingDir: "~/goose artifacts", @@ -5569,6 +5530,18 @@ describe("AppShell global navigation", () => { ] as ChatSession[], activeSessionId: null, }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); renderAppShell(); @@ -5601,5 +5574,6 @@ describe("AppShell global navigation", () => { expect(screen.getByTestId("rendered-session-id")).toHaveTextContent( "session-1", ); + expect(stopVoiceConversation).not.toHaveBeenCalled(); }); }); diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index c08504812..5b9e90b60 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -224,6 +224,7 @@ import { useExperiment } from "@/features/experiments/experimentPreferences"; import { OnboardingFlow } from "@/features/onboarding/ui/OnboardingFlow"; import { useOnboardingState } from "@/features/onboarding/model"; import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; +import { listenToVoiceConversationOpenSession } from "@/features/voice-conversation/api/voiceConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { PocketVoiceSetupDialog } from "@/features/voice-conversation/ui/PocketVoiceSetupDialog"; @@ -362,25 +363,6 @@ function getSessionArchiveInterruptionReason( type GlobalComposerPlacement = "docked" | "centered" | "handoff"; -export function shouldStopVoiceConversationOnSessionChange({ - previousSessionId, - nextSessionId, - boundSessionId, - lifecycle, -}: { - previousSessionId: string | null; - nextSessionId: string | null; - boundSessionId: string | null; - lifecycle: string; -}): boolean { - return ( - previousSessionId !== null && - previousSessionId !== nextSessionId && - boundSessionId === previousSessionId && - lifecycle !== "stopped" && - lifecycle !== "unavailable" - ); -} const current = (id: string, label: string): TopBarBreadcrumb => ({ id, label, @@ -764,18 +746,7 @@ export function AppShell({ ) { voice.clearRequestedStart(previousSessionId); } - if ( - !shouldStopVoiceConversationOnSessionChange({ - previousSessionId, - nextSessionId: activeSessionId, - boundSessionId: voice.status.sessionId, - lifecycle: voice.status.lifecycle, - }) - ) { - return; - } - void stopVoiceConversation().catch(() => undefined); - }, [activeSessionId, stopVoiceConversation]); + }, [activeSessionId]); const sidebarIsResizing = isResizing; const sidebarDockedPanelOuterWidth = sidebarPanelOuterWidth; const sidebarDockedOuterWidth = sidebarCollapsed ? 0 : sidebarPanelOuterWidth; @@ -3895,6 +3866,31 @@ export function AppShell({ [activeView, guardAppNavigation, isMultiWindowEnabled, selectSessionDirect], ); + useEffect(() => { + let cancelled = false; + let unlisten: (() => void) | null = null; + void listenToVoiceConversationOpenSession((sessionId) => { + const voice = useVoiceConversationStore.getState().status; + if (voice.lifecycle === "running" && voice.sessionId === sessionId) { + handleSelectSession(sessionId); + } + }) + .then((cleanup) => { + if (cancelled) cleanup(); + else unlisten = cleanup; + }) + .catch((error) => { + console.error( + "Failed to listen for voice session open requests:", + error, + ); + }); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [handleSelectSession]); + const handleSelectSearchResult = useCallback( (sessionId: string, messageId?: string, query?: string) => { guardAppNavigation(() => { diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index 15c4fac47..ffe9e42cc 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -67,6 +67,7 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, } as const; mocks.invoke @@ -132,6 +133,7 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, } as const; mocks.invoke.mockResolvedValue(status); @@ -150,6 +152,7 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, } as const; @@ -187,6 +190,7 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "session-window", + microphoneMuted: false, revision: 3, } as const; @@ -202,19 +206,30 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, } as const; await reconcileVoiceConversationMicrophone(status); await setVoiceConversationMicrophoneMuted(true, status); - await reconcileVoiceConversationMicrophone(status); + await reconcileVoiceConversationMicrophone({ + ...status, + microphoneMuted: true, + }); await setVoiceConversationMicrophoneMuted(false, status); expect(mocks.startMicrophone).toHaveBeenCalledOnce(); expect(mocks.stopMicrophone).not.toHaveBeenCalled(); - expect(mocks.setMicrophoneMuted).toHaveBeenCalledWith(true); - expect(mocks.setMicrophoneMuted).toHaveBeenLastCalledWith(false); - expect(mocks.invoke).not.toHaveBeenCalled(); + expect(mocks.setMicrophoneMuted.mock.calls).toEqual([ + [false], + [true], + [true], + [false], + ]); + expect(mocks.invoke.mock.calls).toEqual([ + ["set_native_voice_microphone_muted", { muted: true }], + ["set_native_voice_microphone_muted", { muted: false }], + ]); }); it("routes UI mute through macOS while keeping browser capture in sync", async () => { @@ -384,6 +399,7 @@ describe("voice conversation API", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, } as const; diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index f8ca73e9e..b542bd0b4 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -58,6 +58,7 @@ async function ensureActiveMicrophone(): Promise { export async function reconcileVoiceConversationMicrophone( status: VoiceConversationStatus, ): Promise { + microphoneMuted = status.microphoneMuted; if ( status.lifecycle === "running" && status.ownerWindowLabel === getCurrentWindow().label @@ -91,6 +92,7 @@ export async function setVoiceConversationMicrophoneMuted( await reconcileVoiceConversationMicrophone(status); if (intent !== microphoneMuteIntent) return; activeMicrophone?.setMuted(microphoneMuted); + await invoke("set_native_voice_microphone_muted", { muted }); if (status.nativeMicrophoneMuteControl) { await invoke("set_native_voice_input_muted", { sessionId: status.sessionId, @@ -163,6 +165,8 @@ export interface VoiceConversationStatus { sessionId: string | null; /** Trusted Tauri window allowed to attach capture and send raw PCM. */ ownerWindowLabel: string | null; + /** Whether microphone samples are currently withheld from recognition. */ + microphoneMuted: boolean; /** Monotonic native lifecycle revision used to reject stale responses/events. */ revision: number; /** macOS owns an input session capable of receiving headset mute controls. */ @@ -205,6 +209,12 @@ export type VoiceConversationEvent = muted: boolean; revision: number; } + | { + type: "microphoneMute"; + sessionId: string; + muted: boolean; + revision: number; + } | { type: "cleanShutdown"; sessionId: string; @@ -219,6 +229,8 @@ export type VoiceConversationEvent = }; export const VOICE_CONVERSATION_EVENT = "voice-conversation:event"; +export const VOICE_CONVERSATION_OPEN_SESSION_EVENT = + "voice-conversation:open-session"; export function getVoiceConversationStatus(): Promise { return invoke( @@ -311,6 +323,34 @@ export function listenToVoiceConversation( onEvent: (event: VoiceConversationEvent) => void, ): Promise { return listen(VOICE_CONVERSATION_EVENT, (event) => { + if ( + event.payload.type === "inputMute" || + event.payload.type === "microphoneMute" + ) { + applyVoiceConversationMicrophoneMuteEvent(event.payload.muted); + } + if ( + event.payload.type === "cleanShutdown" || + (event.payload.type === "error" && event.payload.terminal) + ) { + microphoneMuted = false; + stopActiveMicrophone(); + } onEvent(event.payload); }); } + +export function listenToVoiceConversationOpenSession( + onOpen: (sessionId: string) => void, +): Promise { + const internals = window.__TAURI_INTERNALS__ as + | { transformCallback?: unknown } + | undefined; + if (typeof internals?.transformCallback !== "function") { + return Promise.resolve(() => undefined); + } + return listen<{ sessionId: string }>( + VOICE_CONVERSATION_OPEN_SESSION_EVENT, + (event) => onOpen(event.payload.sessionId), + ); +} diff --git a/src/features/voice-conversation/hooks/usePocketVoiceSetup.test.ts b/src/features/voice-conversation/hooks/usePocketVoiceSetup.test.ts index 8feb6f5bb..0ccebb315 100644 --- a/src/features/voice-conversation/hooks/usePocketVoiceSetup.test.ts +++ b/src/features/voice-conversation/hooks/usePocketVoiceSetup.test.ts @@ -248,6 +248,7 @@ describe("mergePocketVoiceStatus", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 1, }, stop, diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 68e06b5c6..af0c59b9f 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -18,14 +18,12 @@ vi.mock("../lib/nativeAssistantSpeech", () => ({ import { canBindVoiceSendRoute, canClaimVoiceSendRoute, - createVoiceRouteMountRegistry, createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, resetVoiceUiWhenRunSettles, resolveVoiceRouteMount, resolveVoiceToggleAction, shouldStartRequestedVoiceConversation, - shouldStopVoiceWhenRouteUnmounts, startPendingTranscriptRecovery, useVoiceConversationController, waitForVoiceDeliveryOpportunity, @@ -68,6 +66,7 @@ describe("voice transcript delivery coordination", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 3, }, uiState: "agent-working", @@ -143,6 +142,7 @@ describe("voice transcript delivery coordination", () => { lifecycle: "stopped", sessionId: null, ownerWindowLabel: null, + microphoneMuted: false, revision: 4, }, }); @@ -292,6 +292,7 @@ describe("voice transcript delivery coordination", () => { lifecycle: "starting" as const, sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 1, }); useVoiceConversationStore.setState({ @@ -301,6 +302,7 @@ describe("voice transcript delivery coordination", () => { lifecycle: "unavailable", sessionId: null, ownerWindowLabel: null, + microphoneMuted: false, revision: 0, }, hydrated: true, @@ -354,77 +356,6 @@ describe("voice transcript delivery coordination", () => { expect(canClaimVoiceSendRoute(null, null, "session-2")).toBe(true); }); - it("stops only after the final view for the bound chat unmounts", () => { - const scheduled: Array<() => void> = []; - const registry = createVoiceRouteMountRegistry((callback) => - scheduled.push(callback), - ); - const onLastUnmount = vi.fn(); - const unregisterFirst = registry.register("session-1", onLastUnmount); - const unregisterSecond = registry.register("session-1", onLastUnmount); - - unregisterFirst(); - scheduled.splice(0).forEach((callback) => { - callback(); - }); - expect(onLastUnmount).not.toHaveBeenCalled(); - - unregisterSecond(); - const remounted = registry.register("session-1", onLastUnmount); - scheduled.splice(0).forEach((callback) => { - callback(); - }); - expect(onLastUnmount).not.toHaveBeenCalled(); - - remounted(); - scheduled.splice(0).forEach((callback) => { - callback(); - }); - expect(onLastUnmount).toHaveBeenCalledOnce(); - }); - - it("stops a starting or running voice lifecycle when its chat disappears", () => { - expect( - shouldStopVoiceWhenRouteUnmounts( - { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - }, - "session-1", - ), - ).toBe(true); - expect( - shouldStopVoiceWhenRouteUnmounts( - { - available: true, - unavailableReason: null, - lifecycle: "starting", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - }, - "session-1", - ), - ).toBe(true); - expect( - shouldStopVoiceWhenRouteUnmounts( - { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - revision: 3, - }, - "session-2", - ), - ).toBe(false); - }); - it("drains retained transcripts without stealing a stopped session route", () => { expect( resolveVoiceRouteMount({ diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index cec0a633d..61361f97b 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -16,7 +16,6 @@ import { stopNativeAssistantSpeech, takeVoicePlaybackNotices, } from "../lib/nativeAssistantSpeech"; -import type { VoiceConversationStatus } from "../api/voiceConversation"; interface VoiceSendRoute { sessionId: string; @@ -30,45 +29,6 @@ let activeSendRoute: VoiceSendRoute | null = null; let deliveryInitialized = false; let operationInFlight = false; -export function createVoiceRouteMountRegistry( - schedule: (callback: () => void) => void = queueMicrotask, -) { - const mountsBySession = new Map>(); - return { - register(sessionId: string, onLastUnmount: () => void): () => void { - const token = Symbol(sessionId); - const mounts = mountsBySession.get(sessionId) ?? new Set(); - mounts.add(token); - mountsBySession.set(sessionId, mounts); - - return () => { - const currentMounts = mountsBySession.get(sessionId); - currentMounts?.delete(token); - if (currentMounts?.size === 0) mountsBySession.delete(sessionId); - - // React development mode may immediately remount the same view. Defer - // the ownership check so that remount can reclaim the session first. - schedule(() => { - if (!mountsBySession.has(sessionId)) onLastUnmount(); - }); - }; - }, - }; -} - -const voiceRouteMountRegistry = createVoiceRouteMountRegistry(); - -export function shouldStopVoiceWhenRouteUnmounts( - status: VoiceConversationStatus, - unmountedSessionId: string, -): boolean { - return ( - status.sessionId === unmountedSessionId && - status.lifecycle !== "stopped" && - status.lifecycle !== "unavailable" - ); -} - export function createVoiceTranscriptDeliveryQueue() { const queues = new Map>(); return (sessionId: string, task: () => Promise): Promise => { @@ -512,26 +472,6 @@ export function useVoiceConversationController({ ); const previousPocketReady = useRef(pocketReady); - useEffect( - () => - voiceRouteMountRegistry.register(sessionId, () => { - const voice = useVoiceConversationStore.getState(); - if (!shouldStopVoiceWhenRouteUnmounts(voice.status, sessionId)) return; - - void voice - .stop() - .catch((stopError) => { - addErrorNotification(sessionId, errorText(stopError)); - }) - .finally(() => { - if (activeSendRoute?.sessionId === sessionId) { - activeSendRoute = null; - } - }); - }), - [sessionId], - ); - useEffect(() => { if (!enabled || !isGooseSession) return; void init().catch((initError) => { diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 87d9416f5..a2e9dd620 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -110,6 +110,7 @@ describe("native assistant speech stream", () => { lifecycle: "running", sessionId: "session-1", ownerWindowLabel: "main", + microphoneMuted: false, revision: 1, }, uiState: "listening", diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index ce6691b04..cfc9ae69b 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -48,6 +48,7 @@ function status( lifecycle, sessionId, ownerWindowLabel: lifecycle === "running" ? "main" : null, + microphoneMuted: false, revision, }; } diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index b4b48459b..870057d87 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -35,6 +35,7 @@ export const VOICE_CONVERSATION_OFF_STATUS: VoiceConversationStatus = { lifecycle: "stopped", sessionId: null, ownerWindowLabel: null, + microphoneMuted: false, revision: 0, }; @@ -255,7 +256,8 @@ export const useVoiceConversationStore = create( : uiStateForStatus(status), microphoneMuted: applyHydratedMute ? status.lifecycle === "running" - ? (status.nativeMicrophoneMuted ?? false) + ? status.microphoneMuted || + (status.nativeMicrophoneMuted ?? false) : false : state.microphoneMuted, hydrated: true, @@ -267,10 +269,12 @@ export const useVoiceConversationStore = create( ...state.status, available: status.available, unavailableReason: status.unavailableReason, + microphoneMuted: status.microphoneMuted, }, microphoneMuted: applyHydratedMute ? status.lifecycle === "running" - ? (status.nativeMicrophoneMuted ?? false) + ? status.microphoneMuted || + (status.nativeMicrophoneMuted ?? false) : false : state.microphoneMuted, hydrated: true, @@ -321,6 +325,7 @@ export const useVoiceConversationStore = create( lifecycle: "running" as const, sessionId: event.sessionId, ownerWindowLabel: event.ownerWindowLabel, + microphoneMuted: false, revision: event.revision, nativeMicrophoneMuteControl: event.nativeMicrophoneMuteControl, @@ -328,6 +333,24 @@ export const useVoiceConversationStore = create( uiState: "listening", error: null, }; + case "microphoneMute": { + const nextState = { + ...state, + microphoneMuted: event.muted, + userSpeaking: event.muted ? false : state.userSpeaking, + status: { + ...state.status, + lifecycle: "running" as const, + sessionId: event.sessionId, + microphoneMuted: event.muted, + revision: event.revision, + }, + }; + return { + ...nextState, + uiState: activityUiState(nextState), + }; + } case "user": return { ...state, @@ -397,6 +420,7 @@ export const useVoiceConversationStore = create( lifecycle: "stopped", sessionId: null, ownerWindowLabel: null, + microphoneMuted: false, revision: event.revision, nativeMicrophoneMuteControl: false, }, @@ -416,6 +440,7 @@ export const useVoiceConversationStore = create( lifecycle: "stopped", sessionId: null, ownerWindowLabel: null, + microphoneMuted: false, revision: event.revision, nativeMicrophoneMuteControl: false, } @@ -543,6 +568,7 @@ export const useVoiceConversationStore = create( ? { status, uiState: uiStateForStatus(status), + microphoneMuted: status.microphoneMuted, error: null, } : state, From 4a2438212d19361b38bbe94ef07a11641034fabb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:36:07 -0400 Subject: [PATCH 02/74] fix(voice): close background lifecycle gaps --- src/app/AppShell.navigation.test.tsx | 25 ++++++++++++++++++- src/app/AppShell.tsx | 16 ++++++++++++ .../api/voiceConversation.test.ts | 4 +++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index b15233cbc..2e683879f 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -29,7 +29,10 @@ import { OPEN_SETTINGS_EVENT } from "@/features/settings/lib/settingsEvents"; import { SHORTCUT_PREFERENCES_STORAGE_KEY } from "@/features/shortcuts/lib/shortcutRegistry"; import { useShortcutsDialogStore } from "@/features/shortcuts/stores/shortcutsDialogStore"; import { useProjectStore } from "@/features/projects/stores/projectStore"; -import { useVoiceConversationStore } from "@/features/voice-conversation/stores/voiceConversationStore"; +import { + useVoiceConversationStore, + VOICE_CONVERSATION_OFF_STATUS, +} from "@/features/voice-conversation/stores/voiceConversationStore"; import { dispatchOnboarding } from "@/features/onboarding/model"; import { resetHomeWidgetStoreForTests, @@ -88,6 +91,7 @@ const gitMocks = vi.hoisted(() => ({ removeWorktree: vi.fn(), })); const mockIsExternalAgentReady = vi.hoisted(() => vi.fn()); +const originalStopVoiceConversation = useVoiceConversationStore.getState().stop; const mockAgentStatus = vi.hoisted(() => ({ readyAgentIds: new Set(["goose"]), })); @@ -894,6 +898,11 @@ describe("AppShell global navigation", () => { mockSessionWindowSupport.supported = false; mockFocusSessionWindow.mockReset(); useSessionWindowStore.getState().setSnapshot([]); + useVoiceConversationStore.setState({ + status: VOICE_CONVERSATION_OFF_STATUS, + microphoneMuted: false, + stop: originalStopVoiceConversation, + }); mockListExtensions.mockReset(); mockListExtensions.mockResolvedValue([]); mockAcpCreateSession.mockReset(); @@ -2536,6 +2545,7 @@ describe("AppShell global navigation", () => { it("archives the active session with Cmd+E", async () => { const user = userEvent.setup(); + const stopVoiceConversation = vi.fn().mockResolvedValue(undefined); const session: ChatSession = { id: "session-1", title: "Active chat", @@ -2549,6 +2559,18 @@ describe("AppShell global navigation", () => { sessions: [session], activeSessionId: null, }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); renderAppShell(); @@ -2563,6 +2585,7 @@ describe("AppShell global navigation", () => { expect(screen.getByTestId("active-view")).toHaveTextContent("home"); }); expect(mockAcpArchiveSession).toHaveBeenCalledWith("session-1"); + expect(stopVoiceConversation).toHaveBeenCalledOnce(); expect(useChatSessionStore.getState().activeSessionId).toBeNull(); expect( useChatSessionStore.getState().getSession("session-1")?.archivedAt, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 5b9e90b60..710ecfa65 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -3695,6 +3695,22 @@ export function AppShell({ }; } + const voice = useVoiceConversationStore.getState(); + if ( + voice.status.sessionId === sessionId && + voice.status.lifecycle !== "stopped" && + voice.status.lifecycle !== "unavailable" + ) { + try { + await voice.stop(); + } catch (error) { + console.error( + "Failed to stop voice for the archived session:", + error, + ); + } + } + let cleanupFailureReason: | "target_session_running" | "workspace_cleanup_failed" diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index ffe9e42cc..dd3658b29 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -230,6 +230,9 @@ describe("voice conversation API", () => { ["set_native_voice_microphone_muted", { muted: true }], ["set_native_voice_microphone_muted", { muted: false }], ]); + expect(mocks.setMicrophoneMuted.mock.invocationCallOrder[1]).toBeLessThan( + mocks.invoke.mock.invocationCallOrder[0], + ); }); it("routes UI mute through macOS while keeping browser capture in sync", async () => { @@ -408,6 +411,7 @@ describe("voice conversation API", () => { await expect( setVoiceConversationMicrophoneMuted(true, status), ).rejects.toThrow("capture failed"); + expect(mocks.invoke).not.toHaveBeenCalled(); mocks.startMicrophone.mockResolvedValueOnce({ setMuted: mocks.setMicrophoneMuted, stop: mocks.stopMicrophone, From 3ea1f912ca124b29ba88e12c6c6ce4303e16b587 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:43:44 -0400 Subject: [PATCH 03/74] fix(voice): protect active sessions from archival --- src/app/AppShell.navigation.test.tsx | 32 ++++++++++++++++++++++++++++ src/app/AppShell.tsx | 14 ++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index 2e683879f..e91ade286 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -2513,6 +2513,38 @@ describe("AppShell global navigation", () => { expect(gitMocks.removeWorktree).not.toHaveBeenCalled(); }); + it("does not auto-archive a background voice session", async () => { + const stopVoiceConversation = vi.fn().mockResolvedValue(undefined); + useChatSessionStore.setState({ + sessions: [makeManagedWorktreeSession("background-voice")], + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + stop: stopVoiceConversation, + }); + renderAppShell(); + + const outcome = await getAppNavigationController().archiveSession( + "session-1", + "reject", + ); + + expect(outcome).toEqual({ + ok: false, + reason: "target_session_running", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + expect(stopVoiceConversation).not.toHaveBeenCalled(); + }); + it("rechecks running state before noninteractive archival", async () => { const inspection = deferred(); mockPathExists.mockResolvedValue(true); diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 710ecfa65..8f28d6163 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -3589,6 +3589,19 @@ export function AppShell({ if (!session || session.id !== sessionId) { return { ok: false as const, reason: "session_not_found" as const }; } + await useVoiceConversationStore.getState().init(); + const voiceBeforeArchive = useVoiceConversationStore.getState().status; + if ( + cleanupPolicy === "reject" && + voiceBeforeArchive.sessionId === sessionId && + voiceBeforeArchive.lifecycle !== "stopped" && + voiceBeforeArchive.lifecycle !== "unavailable" + ) { + return { + ok: false as const, + reason: "target_session_running" as const, + }; + } let plans: InspectedSessionWorkspaceCleanupPlan[] = []; if (hasSessionWorkspaceCleanupTargets(session)) { @@ -3695,6 +3708,7 @@ export function AppShell({ }; } + await useVoiceConversationStore.getState().init(); const voice = useVoiceConversationStore.getState(); if ( voice.status.sessionId === sessionId && From 55a328bb04094d4cb1d4e8500a8dd94907d36689 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:50:02 -0400 Subject: [PATCH 04/74] fix(voice): recheck archive ownership --- src/app/AppShell.navigation.test.tsx | 46 ++++++++++++++++++++++++++-- src/app/AppShell.tsx | 15 ++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/app/AppShell.navigation.test.tsx b/src/app/AppShell.navigation.test.tsx index e91ade286..30eaaed17 100644 --- a/src/app/AppShell.navigation.test.tsx +++ b/src/app/AppShell.navigation.test.tsx @@ -2513,7 +2513,10 @@ describe("AppShell global navigation", () => { expect(gitMocks.removeWorktree).not.toHaveBeenCalled(); }); - it("does not auto-archive a background voice session", async () => { + it.each([ + "reject", + "discard", + ] as const)("does not use the %s archive policy on a background voice session", async (cleanupPolicy) => { const stopVoiceConversation = vi.fn().mockResolvedValue(undefined); useChatSessionStore.setState({ sessions: [makeManagedWorktreeSession("background-voice")], @@ -2534,7 +2537,7 @@ describe("AppShell global navigation", () => { const outcome = await getAppNavigationController().archiveSession( "session-1", - "reject", + cleanupPolicy, ); expect(outcome).toEqual({ @@ -2545,6 +2548,45 @@ describe("AppShell global navigation", () => { expect(stopVoiceConversation).not.toHaveBeenCalled(); }); + it("rechecks background voice immediately before auto-archive", async () => { + const inspection = deferred(); + mockPathExists.mockResolvedValue(true); + gitMocks.getGitState.mockReturnValue(inspection.promise); + useChatSessionStore.setState({ + sessions: [makeManagedWorktreeSession("voice-starts-during-inspection")], + }); + renderAppShell(); + + const outcome = getAppNavigationController().archiveSession( + "session-1", + "reject", + ); + await waitFor(() => { + expect(gitMocks.getGitState).toHaveBeenCalled(); + }); + + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + }); + inspection.resolve( + managedWorktreeGitState("voice-starts-during-inspection"), + ); + + await expect(outcome).resolves.toEqual({ + ok: false, + reason: "target_session_running", + }); + expect(mockAcpArchiveSession).not.toHaveBeenCalled(); + }); + it("rechecks running state before noninteractive archival", async () => { const inspection = deferred(); mockPathExists.mockResolvedValue(true); diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 8f28d6163..d8e481a35 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -3592,7 +3592,7 @@ export function AppShell({ await useVoiceConversationStore.getState().init(); const voiceBeforeArchive = useVoiceConversationStore.getState().status; if ( - cleanupPolicy === "reject" && + cleanupPolicy !== "confirm" && voiceBeforeArchive.sessionId === sessionId && voiceBeforeArchive.lifecycle !== "stopped" && voiceBeforeArchive.lifecycle !== "unavailable" @@ -3676,6 +3676,19 @@ export function AppShell({ reason: "blocked_unsaved_changes" as const, }; } + await useVoiceConversationStore.getState().init(); + const voiceBeforeMutation = useVoiceConversationStore.getState().status; + if ( + cleanupPolicy !== "confirm" && + voiceBeforeMutation.sessionId === sessionId && + voiceBeforeMutation.lifecycle !== "stopped" && + voiceBeforeMutation.lifecycle !== "unavailable" + ) { + return { + ok: false as const, + reason: "target_session_running" as const, + }; + } try { await useChatSessionStore From f359432d2900fde48977d3a60feec76104d92d99 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 20 Aug 2026 15:59:06 -0400 Subject: [PATCH 05/74] fix(voice): enable Tauri tray support --- src-tauri/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 4c57c6e2d..f697817ef 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -79,7 +79,7 @@ yaml_serde = "0.10.4" sha2 = "0.10" sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio-rustls", "sqlite", "migrate", "macros"] } sysinfo = { version = "0.33", default-features = false, features = ["system"] } -tauri = { version = "2", features = ["protocol-asset"] } +tauri = { version = "2", features = ["protocol-asset", "tray-icon"] } toml = "1.1.4" tauri-plugin-berdctl = { path = "plugins/berdctl" } tauri-plugin-app-test-driver = { path = "plugins/app-test-driver" } From 9357d655e4294dd92c6d311d7e26f7268e9575b7 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 10:58:59 -0400 Subject: [PATCH 06/74] fix(voice): remove menu bar on main thread --- src-tauri/src/commands/voice_menu_bar.rs | 66 ++++++++++++++++++------ 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/src-tauri/src/commands/voice_menu_bar.rs b/src-tauri/src/commands/voice_menu_bar.rs index 59dad4182..3b70ae8c6 100644 --- a/src-tauri/src/commands/voice_menu_bar.rs +++ b/src-tauri/src/commands/voice_menu_bar.rs @@ -1,6 +1,7 @@ //! macOS menu bar controls for the process-wide native voice conversation. use serde::Serialize; +use std::sync::mpsc; use tauri::{ menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem}, tray::TrayIconBuilder, @@ -21,6 +22,28 @@ struct OpenSessionPayload { session_id: String, } +// AppKit traps if an NSStatusItem is created, mutated, or dropped off its main +// queue. Tauri's tray wrapper drops the native item when it leaves the manager. +fn on_main_thread(app: &AppHandle, operation: F) -> Result +where + T: Send + 'static, + F: FnOnce(&AppHandle) -> Result + Send + 'static, +{ + if objc2::MainThreadMarker::new().is_some() { + return operation(app); + } + + let (sender, receiver) = mpsc::sync_channel(1); + let main_thread_app = app.clone(); + app.run_on_main_thread(move || { + let _ = sender.send(operation(&main_thread_app)); + }) + .map_err(|error| error.to_string())?; + receiver + .recv() + .map_err(|_| "The voice menu bar main-thread operation was interrupted.".to_string())? +} + fn menu(app: &AppHandle, muted: bool) -> tauri::Result> { let status = MenuItem::new(app, "Voice conversation active", false, None::<&str>)?; let mute = CheckMenuItem::with_id(app, MUTE_ID, "Mute Microphone", true, muted, None::<&str>)?; @@ -31,29 +54,38 @@ fn menu(app: &AppHandle, muted: bool) -> tauri::Result> { } pub fn install(app: &AppHandle, muted: bool) -> Result<(), String> { - remove(app); - let menu = menu(app, muted).map_err(|error| error.to_string())?; - TrayIconBuilder::with_id(TRAY_ID) - .menu(&menu) - .title(if muted { "🔇" } else { "🎙" }) - .tooltip("Berd voice conversation") - .build(app) - .map(|_| ()) - .map_err(|error| error.to_string()) + on_main_thread(app, move |app| { + let _ = app.remove_tray_by_id(TRAY_ID); + let menu = menu(app, muted).map_err(|error| error.to_string())?; + TrayIconBuilder::with_id(TRAY_ID) + .menu(&menu) + .title(if muted { "🔇" } else { "🎙" }) + .tooltip("Berd voice conversation") + .build(app) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) } pub fn set_muted(app: &AppHandle, muted: bool) -> Result<(), String> { - let Some(tray) = app.tray_by_id(TRAY_ID) else { - return Ok(()); - }; - tray.set_title(Some(if muted { "🔇" } else { "🎙" })) - .map_err(|error| error.to_string())?; - tray.set_menu(Some(menu(app, muted).map_err(|error| error.to_string())?)) - .map_err(|error| error.to_string()) + on_main_thread(app, move |app| { + let Some(tray) = app.tray_by_id(TRAY_ID) else { + return Ok(()); + }; + tray.set_title(Some(if muted { "🔇" } else { "🎙" })) + .map_err(|error| error.to_string())?; + tray.set_menu(Some(menu(app, muted).map_err(|error| error.to_string())?)) + .map_err(|error| error.to_string()) + }) } pub fn remove(app: &AppHandle) { - let _ = app.remove_tray_by_id(TRAY_ID); + if let Err(error) = on_main_thread(app, |app| { + let _ = app.remove_tray_by_id(TRAY_ID); + Ok(()) + }) { + log::warn!("Failed to remove the voice menu bar: {error}"); + } } fn focus_window(window: &WebviewWindow) { From 00eab54244eda6449191e1ee104069df7d611a24 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 11:27:34 -0400 Subject: [PATCH 07/74] feat(voice): add cross-platform Gloopie buddy --- src-tauri/capabilities/voice-buddy.json | 11 + src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/native_voice.rs | 56 +++-- src-tauri/src/commands/voice_buddy.rs | 133 +++++++++++ src-tauri/src/commands/voice_menu_bar.rs | 46 +--- src-tauri/src/commands/window_session.rs | 1 + src-tauri/src/lib.rs | 3 + src/app/lib/rendererDiagnostics.ts | 2 +- .../api/voiceConversation.test.ts | 17 ++ .../api/voiceConversation.ts | 14 ++ .../useVoiceConversationController.test.ts | 10 + .../hooks/useVoiceConversationController.ts | 19 ++ .../ui/VoiceBuddyApp.test.tsx | 95 ++++++++ .../voice-conversation/ui/VoiceBuddyApp.tsx | 217 ++++++++++++++++++ src/main.tsx | 36 ++- src/shared/i18n/locales/en/chat.json | 11 + src/shared/i18n/locales/es/chat.json | 11 + 17 files changed, 619 insertions(+), 64 deletions(-) create mode 100644 src-tauri/capabilities/voice-buddy.json create mode 100644 src-tauri/src/commands/voice_buddy.rs create mode 100644 src/features/voice-conversation/ui/VoiceBuddyApp.test.tsx create mode 100644 src/features/voice-conversation/ui/VoiceBuddyApp.tsx diff --git a/src-tauri/capabilities/voice-buddy.json b/src-tauri/capabilities/voice-buddy.json new file mode 100644 index 000000000..b00e713eb --- /dev/null +++ b/src-tauri/capabilities/voice-buddy.json @@ -0,0 +1,11 @@ +{ + "$schema": "../gen/schemas/desktop-schema.json", + "identifier": "voice-buddy", + "description": "Capability for the always-on-top voice conversation buddy", + "windows": ["voice-buddy"], + "permissions": [ + "core:default", + "core:window:allow-start-dragging", + "core:window:allow-close" + ] +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4af7e3bec..ef5fedac1 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -51,6 +51,7 @@ pub mod system; pub mod telemetry; pub mod terminal; pub mod updates; +pub mod voice_buddy; pub mod voice_capture; #[cfg(target_os = "macos")] pub mod voice_menu_bar; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index dffaf7f6d..b44339fdd 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -20,7 +20,7 @@ use super::{ native_input_mute, pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState, }; -const EVENT_NAME: &str = "voice-conversation:event"; +pub(crate) const EVENT_NAME: &str = "voice-conversation:event"; const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; const AUDIO_QUEUE_DEPTH: usize = 50; const MAX_PENDING_TRANSCRIPTS: usize = 64; @@ -182,7 +182,6 @@ impl NativeVoiceState { self.microphone_muted.load(Ordering::SeqCst) } - #[cfg(target_os = "macos")] pub fn active_session_target(&self) -> Option<(String, String)> { let runtime = self.runtime.lock().ok()?; Some(( @@ -213,16 +212,15 @@ impl NativeVoiceState { if let Err(error) = super::voice_menu_bar::set_muted(app, muted) { log::warn!("Failed to update the voice menu bar mute state: {error}"); } + let event = NativeVoiceEvent::MicrophoneMute { + session_id, + muted, + revision, + }; if let Some(window) = app.get_webview_window(&owner_window_label) { - let _ = window.emit( - EVENT_NAME, - NativeVoiceEvent::MicrophoneMute { - session_id, - muted, - revision, - }, - ); + let _ = window.emit(EVENT_NAME, event.clone()); } + super::voice_buddy::emit(app, event); Ok(()) } } @@ -562,10 +560,9 @@ pub async fn start_native_voice_conversation( runtime.native_microphone_mute_control, ) }; - #[cfg(target_os = "macos")] - if let Err(error) = super::voice_menu_bar::install(&app, false) { + if let Err(error) = super::voice_buddy::install(&app) { state.stop_active(&app, &capture).await?; - return Err(format!("Could not install the voice menu bar: {error}")); + return Err(format!("Could not show the Gloopie voice buddy: {error}")); } let _ = webview_window.emit( EVENT_NAME, @@ -577,6 +574,15 @@ pub async fn start_native_voice_conversation( native_microphone_mute_control: runtime_mute_control, }, ); + super::voice_buddy::emit( + &app, + NativeVoiceEvent::Startup { + session_id: session_id.clone(), + owner_window_label: window_label.clone(), + line: "Native Parakeet voice conversation is on".to_string(), + revision, + }, + ); let event_app = app.clone(); let event_window = webview_window.clone(); @@ -595,18 +601,17 @@ pub async fn start_native_voice_conversation( } match event { SttMessage::Speaking(speaking) => { - let _ = event_window.emit( - EVENT_NAME, - NativeVoiceEvent::Activity { - session_id: session_id.clone(), - activity: if speaking { - "user-speaking" - } else { - "user-idle" - }, - revision, + let event = NativeVoiceEvent::Activity { + session_id: session_id.clone(), + activity: if speaking { + "user-speaking" + } else { + "user-idle" }, - ); + revision, + }; + let _ = event_window.emit(EVENT_NAME, event.clone()); + super::voice_buddy::emit(&event_app, event); } SttMessage::Final { text, delivered } => { let transcript = PendingTranscript { @@ -670,6 +675,7 @@ pub async fn start_native_voice_conversation( event_state.microphone_muted.store(false, Ordering::SeqCst); #[cfg(target_os = "macos")] super::voice_menu_bar::remove(&event_app); + super::voice_buddy::remove(&event_app); event_app .state::() .release_owner(&window_label, &owner_id); @@ -761,6 +767,7 @@ impl NativeVoiceState { self.microphone_muted.store(false, Ordering::SeqCst); #[cfg(target_os = "macos")] super::voice_menu_bar::remove(app); + super::voice_buddy::remove(app); if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); } @@ -816,6 +823,7 @@ impl NativeVoiceState { self.microphone_muted.store(false, Ordering::SeqCst); #[cfg(target_os = "macos")] super::voice_menu_bar::remove(app); + super::voice_buddy::remove(app); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); if let Some(window) = app.get_webview_window(&owner.window_label) { diff --git a/src-tauri/src/commands/voice_buddy.rs b/src-tauri/src/commands/voice_buddy.rs new file mode 100644 index 000000000..b0a075af4 --- /dev/null +++ b/src-tauri/src/commands/voice_buddy.rs @@ -0,0 +1,133 @@ +//! Cross-platform always-on-top controls for the process-wide voice conversation. + +use serde::Serialize; +use tauri::{ + AppHandle, Emitter, Manager, PhysicalPosition, WebviewUrl, WebviewWindow, WebviewWindowBuilder, +}; + +use super::{native_voice::NativeVoiceState, voice_capture::VoiceCaptureState}; + +pub const WINDOW_LABEL: &str = "voice-buddy"; +pub const OPEN_SESSION_EVENT: &str = "voice-conversation:open-session"; +const WINDOW_WIDTH: f64 = 248.0; +const WINDOW_HEIGHT: f64 = 196.0; +const SCREEN_INSET: i32 = 24; + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpenSessionPayload { + session_id: String, +} + +fn focus_window(window: &WebviewWindow) { + let _ = window.show(); + let _ = window.unminimize(); + let _ = window.set_focus(); +} + +pub fn open_active_session(app: &AppHandle) -> Result<(), String> { + let state = app.state::(); + let Some((session_id, owner_window_label)) = state.active_session_target() else { + return Ok(()); + }; + let window = app + .get_webview_window(&owner_window_label) + .ok_or_else(|| "The voice session window is no longer available.".to_string())?; + focus_window(&window); + if owner_window_label == "main" { + window + .emit(OPEN_SESSION_EVENT, OpenSessionPayload { session_id }) + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +fn position_near_bottom_right(window: &WebviewWindow) { + let Ok(Some(monitor)) = window.primary_monitor() else { + return; + }; + let monitor_position = monitor.position(); + let monitor_size = monitor.size(); + let Ok(window_size) = window.outer_size() else { + return; + }; + let x = monitor_position.x + + i32::try_from(monitor_size.width.saturating_sub(window_size.width)).unwrap_or_default() + - SCREEN_INSET; + let y = monitor_position.y + + i32::try_from(monitor_size.height.saturating_sub(window_size.height)).unwrap_or_default() + - SCREEN_INSET; + let _ = window.set_position(PhysicalPosition::new(x, y)); +} + +pub fn install(app: &AppHandle) -> Result<(), String> { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + let _ = window.show(); + return Ok(()); + } + + let entrypoint = if cfg!(target_os = "macos") { + "index.html?voiceBuddy=1&menuBar=1" + } else { + "index.html?voiceBuddy=1" + }; + let window = WebviewWindowBuilder::new(app, WINDOW_LABEL, WebviewUrl::App(entrypoint.into())) + .title("Berd voice conversation") + .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) + .resizable(false) + .maximizable(false) + .minimizable(false) + .decorations(false) + .always_on_top(true) + .skip_taskbar(true) + .focused(false) + .visible(false) + .build() + .map_err(|error| error.to_string())?; + position_near_bottom_right(&window); + window.show().map_err(|error| error.to_string()) +} + +pub fn remove(app: &AppHandle) { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + let _ = window.close(); + } +} + +pub fn emit(app: &AppHandle, payload: T) { + if let Some(window) = app.get_webview_window(WINDOW_LABEL) { + let _ = window.emit(super::native_voice::EVENT_NAME, payload); + } +} + +#[tauri::command] +pub fn open_voice_conversation_session(app: AppHandle) -> Result<(), String> { + open_active_session(&app) +} + +#[tauri::command] +pub async fn stop_voice_conversation_from_buddy( + app: AppHandle, + state: tauri::State<'_, NativeVoiceState>, + capture: tauri::State<'_, VoiceCaptureState>, +) -> Result<(), String> { + state.stop_active(&app, capture.inner()).await +} + +#[tauri::command] +pub fn send_voice_conversation_to_menu_bar( + app: AppHandle, + state: tauri::State<'_, NativeVoiceState>, +) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + super::voice_menu_bar::install(&app, state.microphone_is_muted())?; + remove(&app); + Ok(()) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (app, state); + Err("The menu bar voice surface is available only on macOS.".to_string()) + } +} diff --git a/src-tauri/src/commands/voice_menu_bar.rs b/src-tauri/src/commands/voice_menu_bar.rs index 3b70ae8c6..8f2a1615c 100644 --- a/src-tauri/src/commands/voice_menu_bar.rs +++ b/src-tauri/src/commands/voice_menu_bar.rs @@ -1,11 +1,10 @@ //! macOS menu bar controls for the process-wide native voice conversation. -use serde::Serialize; use std::sync::mpsc; use tauri::{ menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem}, tray::TrayIconBuilder, - AppHandle, Emitter, Manager, WebviewWindow, + AppHandle, Manager, }; use super::{native_voice::NativeVoiceState, voice_capture::VoiceCaptureState}; @@ -14,13 +13,7 @@ const TRAY_ID: &str = "voice-conversation"; const MUTE_ID: &str = "voice-conversation-mute"; const OPEN_ID: &str = "voice-conversation-open"; const STOP_ID: &str = "voice-conversation-stop"; -pub const OPEN_SESSION_EVENT: &str = "voice-conversation:open-session"; - -#[derive(Clone, Serialize)] -#[serde(rename_all = "camelCase")] -struct OpenSessionPayload { - session_id: String, -} +const SHOW_BUDDY_ID: &str = "voice-conversation-show-buddy"; // AppKit traps if an NSStatusItem is created, mutated, or dropped off its main // queue. Tauri's tray wrapper drops the native item when it leaves the manager. @@ -48,9 +41,13 @@ fn menu(app: &AppHandle, muted: bool) -> tauri::Result> { let status = MenuItem::new(app, "Voice conversation active", false, None::<&str>)?; let mute = CheckMenuItem::with_id(app, MUTE_ID, "Mute Microphone", true, muted, None::<&str>)?; let open = MenuItem::with_id(app, OPEN_ID, "Open Voice Session", true, None::<&str>)?; + let show_buddy = MenuItem::with_id(app, SHOW_BUDDY_ID, "Show Gloopie", true, None::<&str>)?; let stop = MenuItem::with_id(app, STOP_ID, "Stop Voice Conversation", true, None::<&str>)?; let separator = PredefinedMenuItem::separator(app)?; - Menu::with_items(app, &[&status, &separator, &mute, &open, &stop]) + Menu::with_items( + app, + &[&status, &separator, &mute, &open, &show_buddy, &stop], + ) } pub fn install(app: &AppHandle, muted: bool) -> Result<(), String> { @@ -88,29 +85,6 @@ pub fn remove(app: &AppHandle) { } } -fn focus_window(window: &WebviewWindow) { - let _ = window.show(); - let _ = window.unminimize(); - let _ = window.set_focus(); -} - -fn open_voice_session(app: &AppHandle) -> Result<(), String> { - let state = app.state::(); - let Some((session_id, owner_window_label)) = state.active_session_target() else { - return Ok(()); - }; - let window = app - .get_webview_window(&owner_window_label) - .ok_or_else(|| "The voice session window is no longer available.".to_string())?; - focus_window(&window); - if owner_window_label == "main" { - window - .emit(OPEN_SESSION_EVENT, OpenSessionPayload { session_id }) - .map_err(|error| error.to_string())?; - } - Ok(()) -} - pub fn handle_menu_event(app: &AppHandle, event: tauri::menu::MenuEvent) { match event.id().as_ref() { MUTE_ID => { @@ -121,10 +95,14 @@ pub fn handle_menu_event(app: &AppHandle, event: tauri::menu::MenuEvent) { } } OPEN_ID => { - if let Err(error) = open_voice_session(app) { + if let Err(error) = super::voice_buddy::open_active_session(app) { log::warn!("Failed to open the voice session: {error}"); } } + SHOW_BUDDY_ID => match super::voice_buddy::install(app) { + Ok(()) => remove(app), + Err(error) => log::warn!("Failed to restore the Gloopie voice buddy: {error}"), + }, STOP_ID => { let app = app.clone(); tauri::async_runtime::spawn(async move { diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index 6e8f851f2..d57094d15 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -686,6 +686,7 @@ pub fn open_session_window( if stopped_native_voice { #[cfg(target_os = "macos")] crate::commands::voice_menu_bar::remove(&app_for_close); + crate::commands::voice_buddy::remove(&app_for_close); app_for_close .state::() .stop_for_window_destroyed(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index f67053603..e5608de74 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -649,6 +649,9 @@ pub fn run() { commands::native_voice::stop_native_voice_conversation, commands::native_voice::push_native_voice_audio, commands::native_voice::set_native_voice_input_muted, + commands::voice_buddy::open_voice_conversation_session, + commands::voice_buddy::stop_voice_conversation_from_buddy, + commands::voice_buddy::send_voice_conversation_to_menu_bar, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, diff --git a/src/app/lib/rendererDiagnostics.ts b/src/app/lib/rendererDiagnostics.ts index 22343711a..5dbc59250 100644 --- a/src/app/lib/rendererDiagnostics.ts +++ b/src/app/lib/rendererDiagnostics.ts @@ -5,7 +5,7 @@ const SECRET_VALUE_PATTERN = /\b(authorization|refresh_token|access_token|secret_key|api_key|apikey|password|secret|token)\b\s*[:=]\s*(['"]?)[^,\s;&'"]+/gi; interface RendererDiagnosticsContext { - windowKind: "main" | "session"; + windowKind: "main" | "session" | "voice-buddy"; } let installed = false; diff --git a/src/features/voice-conversation/api/voiceConversation.test.ts b/src/features/voice-conversation/api/voiceConversation.test.ts index dd3658b29..0db980358 100644 --- a/src/features/voice-conversation/api/voiceConversation.test.ts +++ b/src/features/voice-conversation/api/voiceConversation.test.ts @@ -30,10 +30,13 @@ import { getVoiceConversationStatus, hydrateVoiceConversationMicrophone, listenToVoiceConversation, + openVoiceConversationSession, reconcileVoiceConversationMicrophone, setVoiceConversationMicrophoneMuted, + sendVoiceConversationToMenuBar, startVoiceConversation, stopActiveMicrophoneForTest, + stopVoiceConversationFromBuddy, stopVoiceConversation, } from "./voiceConversation"; @@ -126,6 +129,20 @@ describe("voice conversation API", () => { ); }); + it("exposes the buddy control commands", async () => { + mocks.invoke.mockResolvedValue(undefined); + + await openVoiceConversationSession(); + await stopVoiceConversationFromBuddy(); + await sendVoiceConversationToMenuBar(); + + expect(mocks.invoke.mock.calls).toEqual([ + ["open_voice_conversation_session"], + ["stop_voice_conversation_from_buddy"], + ["send_voice_conversation_to_menu_bar"], + ]); + }); + it("can stop only the browser microphone for deterministic development tests", async () => { const status = { available: true, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index b542bd0b4..b0c996f39 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -238,6 +238,20 @@ export function getVoiceConversationStatus(): Promise { ); } +export function openVoiceConversationSession(): Promise { + return invoke("open_voice_conversation_session"); +} + +export function stopVoiceConversationFromBuddy(): Promise { + microphoneMuted = false; + stopActiveMicrophone(); + return invoke("stop_voice_conversation_from_buddy"); +} + +export function sendVoiceConversationToMenuBar(): Promise { + return invoke("send_voice_conversation_to_menu_bar"); +} + export interface PendingVoiceTranscript { sessionId: string; lifecycleId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index af0c59b9f..62578eb66 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -21,6 +21,7 @@ import { createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, resetVoiceUiWhenRunSettles, + resolveActiveVoiceButtonAction, resolveVoiceRouteMount, resolveVoiceToggleAction, shouldStartRequestedVoiceConversation, @@ -356,6 +357,15 @@ describe("voice transcript delivery coordination", () => { expect(canClaimVoiceSendRoute(null, null, "session-2")).toBe(true); }); + it("opens the owner instead of stopping voice from another session", () => { + expect(resolveActiveVoiceButtonAction("session-1", "session-2")).toBe( + "open-owner", + ); + expect(resolveActiveVoiceButtonAction("session-1", "session-1")).toBe( + "stop", + ); + }); + it("drains retained transcripts without stealing a stopped session route", () => { expect( resolveVoiceRouteMount({ diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 61361f97b..a2e407cf8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -16,6 +16,7 @@ import { stopNativeAssistantSpeech, takeVoicePlaybackNotices, } from "../lib/nativeAssistantSpeech"; +import { openVoiceConversationSession } from "../api/voiceConversation"; interface VoiceSendRoute { sessionId: string; @@ -61,6 +62,13 @@ export function canBindVoiceSendRoute(options: { ); } +export function resolveActiveVoiceButtonAction( + activeSessionId: string | null, + candidateSessionId: string, +): "stop" | "open-owner" { + return activeSessionId === candidateSessionId ? "stop" : "open-owner"; +} + export function shouldStartRequestedVoiceConversation({ requestedStartSessionId, sessionId, @@ -607,6 +615,17 @@ export function useVoiceConversationController({ }); if (action === "stop") { const boundSessionId = currentStatus.sessionId; + if ( + resolveActiveVoiceButtonAction(boundSessionId, sessionId) === + "open-owner" + ) { + try { + await openVoiceConversationSession(); + } catch (openError) { + addErrorNotification(boundSessionId, errorText(openError)); + } + return; + } try { await stop(); } catch (stopError) { diff --git a/src/features/voice-conversation/ui/VoiceBuddyApp.test.tsx b/src/features/voice-conversation/ui/VoiceBuddyApp.test.tsx new file mode 100644 index 000000000..eec827d81 --- /dev/null +++ b/src/features/voice-conversation/ui/VoiceBuddyApp.test.tsx @@ -0,0 +1,95 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + getStatus: vi.fn(), + listen: vi.fn(), + openSession: vi.fn(), + sendToMenuBar: vi.fn(), + setMuted: vi.fn(), + stop: vi.fn(), +})); + +vi.mock("react-i18next", () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})); +vi.mock("@/shared/hooks/useAvatarSrc", () => ({ + useAvatarMediaState: () => ({ media: undefined }), +})); +vi.mock("@/features/voice-conversation/api/voiceConversation", () => ({ + getVoiceConversationStatus: mocks.getStatus, + listenToVoiceConversation: mocks.listen, + openVoiceConversationSession: mocks.openSession, + sendVoiceConversationToMenuBar: mocks.sendToMenuBar, + setVoiceConversationMicrophoneMuted: mocks.setMuted, + stopVoiceConversationFromBuddy: mocks.stop, +})); + +import { VoiceBuddyApp } from "./VoiceBuddyApp"; + +const runningStatus = { + available: true, + unavailableReason: null, + lifecycle: "running" as const, + sessionId: "session-a", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 3, +}; + +describe("VoiceBuddyApp", () => { + beforeEach(() => { + window.history.replaceState({}, "", "/?voiceBuddy=1&menuBar=1"); + mocks.getStatus.mockReset().mockResolvedValue(runningStatus); + mocks.listen.mockReset().mockResolvedValue(vi.fn()); + mocks.openSession.mockReset().mockResolvedValue(undefined); + mocks.sendToMenuBar.mockReset().mockResolvedValue(undefined); + mocks.setMuted.mockReset().mockResolvedValue(undefined); + mocks.stop.mockReset().mockResolvedValue(undefined); + }); + + it("opens the owner and exposes mute, hang-up, and macOS menu controls", async () => { + const user = userEvent.setup(); + render(); + + await waitFor(() => expect(mocks.getStatus).toHaveBeenCalledOnce()); + await user.click( + screen.getByRole("button", { + name: "composer.voiceConversation.buddy.openSession", + }), + ); + await user.click( + screen.getByRole("button", { + name: "composer.voiceConversation.muteMicrophone", + }), + ); + await user.click( + screen.getByRole("button", { + name: "composer.voiceConversation.buddy.hangUp", + }), + ); + await user.click( + screen.getByRole("button", { + name: "composer.voiceConversation.buddy.sendToMenuBar", + }), + ); + + expect(mocks.openSession).toHaveBeenCalledOnce(); + expect(mocks.setMuted).toHaveBeenCalledWith(true, runningStatus); + expect(mocks.stop).toHaveBeenCalledOnce(); + expect(mocks.sendToMenuBar).toHaveBeenCalledOnce(); + }); + + it("omits the menu-bar control on Windows and Linux", async () => { + window.history.replaceState({}, "", "/?voiceBuddy=1"); + render(); + + await waitFor(() => expect(mocks.getStatus).toHaveBeenCalledOnce()); + expect( + screen.queryByRole("button", { + name: "composer.voiceConversation.buddy.sendToMenuBar", + }), + ).not.toBeInTheDocument(); + }); +}); diff --git a/src/features/voice-conversation/ui/VoiceBuddyApp.tsx b/src/features/voice-conversation/ui/VoiceBuddyApp.tsx new file mode 100644 index 000000000..093c1d197 --- /dev/null +++ b/src/features/voice-conversation/ui/VoiceBuddyApp.tsx @@ -0,0 +1,217 @@ +import { Menu, Mic, MicOff, PhoneOff } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { + getVoiceConversationStatus, + listenToVoiceConversation, + openVoiceConversationSession, + sendVoiceConversationToMenuBar, + setVoiceConversationMicrophoneMuted, + stopVoiceConversationFromBuddy, + type VoiceConversationEvent, + type VoiceConversationStatus, +} from "@/features/voice-conversation/api/voiceConversation"; +import { useAvatarMediaState } from "@/shared/hooks/useAvatarSrc"; +import { AvatarMedia } from "@/shared/ui/avatar-media"; +import { Button } from "@/shared/ui/button"; + +type Activity = "listening" | "user-speaking" | "agent-speaking"; + +function activityFromEvent( + event: VoiceConversationEvent, + current: Activity, +): Activity { + if (event.type !== "activity") return current; + if (event.activity === "user-speaking") return "user-speaking"; + if (event.activity === "assistant-speaking") return "agent-speaking"; + return "listening"; +} + +export function VoiceBuddyApp() { + const { t } = useTranslation("chat"); + const [status, setStatus] = useState(null); + const [activity, setActivity] = useState("listening"); + const [busyAction, setBusyAction] = useState<"mute" | "stop" | "menu" | null>( + null, + ); + const [error, setError] = useState(null); + const avatar = useAvatarMediaState("app-avatar:gloopies-22"); + const menuBarAvailable = useMemo( + () => new URLSearchParams(window.location.search).has("menuBar"), + [], + ); + + useEffect(() => { + let cancelled = false; + let unlisten: (() => void) | undefined; + void getVoiceConversationStatus() + .then((nextStatus) => { + if (!cancelled) setStatus(nextStatus); + }) + .catch((cause) => { + if (!cancelled) setError(String(cause)); + }); + void listenToVoiceConversation((event) => { + setActivity((current) => activityFromEvent(event, current)); + setStatus((current) => { + if (!current || event.revision < current.revision) return current; + switch (event.type) { + case "startup": + return { + ...current, + lifecycle: "running", + sessionId: event.sessionId, + ownerWindowLabel: event.ownerWindowLabel, + microphoneMuted: false, + revision: event.revision, + }; + case "microphoneMute": + return { + ...current, + microphoneMuted: event.muted, + revision: event.revision, + }; + case "cleanShutdown": + return { + ...current, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + microphoneMuted: false, + revision: event.revision, + }; + case "error": + if (event.terminal) setError(event.message); + return { ...current, revision: event.revision }; + default: + return { ...current, revision: event.revision }; + } + }); + }).then((nextUnlisten) => { + if (cancelled) nextUnlisten(); + else unlisten = nextUnlisten; + }); + return () => { + cancelled = true; + unlisten?.(); + }; + }, []); + + const microphoneMuted = status?.microphoneMuted ?? false; + const activityLabel = microphoneMuted + ? t("composer.voiceConversation.buddy.muted") + : t(`composer.voiceConversation.buddy.${activity}`); + + const run = async ( + action: "mute" | "stop" | "menu", + operation: () => Promise, + ) => { + setBusyAction(action); + setError(null); + try { + await operation(); + } catch (cause) { + setError(String(cause)); + } finally { + setBusyAction(null); + } + }; + + const toggleMute = () => { + if (!status) return; + void run("mute", async () => { + await setVoiceConversationMicrophoneMuted(!microphoneMuted, status); + setStatus((current) => + current ? { ...current, microphoneMuted: !microphoneMuted } : current, + ); + }); + }; + + return ( +
+
+ {t("composer.voiceConversation.buddy.title")} +
+ +

+ {error ?? activityLabel} +

+
+ + + {menuBarAvailable ? ( + + ) : null} +
+
+ ); +} diff --git a/src/main.tsx b/src/main.tsx index 7adb705af..79787eb08 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -117,9 +117,9 @@ function OptionalBerdctlBridge() { return Bridge ? : null; } -const sessionKey = new URLSearchParams(window.location.search).get( - "sessionKey", -); +const entrypointParams = new URLSearchParams(window.location.search); +const sessionKey = entrypointParams.get("sessionKey"); +const voiceBuddy = entrypointParams.has("voiceBuddy"); let sessionId: string | null = null; let bootError: string | null = null; if (sessionKey) { @@ -132,9 +132,35 @@ if (sessionKey) { } } -installRendererDiagnostics({ windowKind: sessionId ? "session" : "main" }); +installRendererDiagnostics({ + windowKind: voiceBuddy ? "voice-buddy" : sessionId ? "session" : "main", +}); -if (bootError) { +if (voiceBuddy) { + import("@/features/voice-conversation/ui/VoiceBuddyApp") + .then(({ VoiceBuddyApp }) => { + reactRoot.render( + + + + + + + + + + + + + , + ); + }) + .catch((error) => { + console.error("Failed to load voice buddy bundle:", error); + reportRendererError("voice_buddy_bundle_load_failed", error); + renderBootError("The voice buddy could not be loaded."); + }); +} else if (bootError) { renderBootError(bootError); } else if (sessionId) { const decodedSessionId = sessionId; diff --git a/src/shared/i18n/locales/en/chat.json b/src/shared/i18n/locales/en/chat.json index c867305da..6196140bd 100644 --- a/src/shared/i18n/locales/en/chat.json +++ b/src/shared/i18n/locales/en/chat.json @@ -548,6 +548,17 @@ "start": "Start voice conversation", "muteMicrophone": "Mute microphone", "unmuteMicrophone": "Unmute microphone", + "buddy": { + "title": "Voice conversation", + "gloopieAlt": "Gloopie voice buddy", + "openSession": "Open voice session", + "hangUp": "Hang up", + "sendToMenuBar": "Send to menu bar", + "listening": "Listening…", + "user-speaking": "Listening to you…", + "agent-speaking": "Speaking…", + "muted": "Microphone muted" + }, "states": { "off": "Start voice conversation", "starting": "Starting voice conversation…", diff --git a/src/shared/i18n/locales/es/chat.json b/src/shared/i18n/locales/es/chat.json index beb0be99f..ae30c5b1a 100644 --- a/src/shared/i18n/locales/es/chat.json +++ b/src/shared/i18n/locales/es/chat.json @@ -545,6 +545,17 @@ "start": "Iniciar conversación de voz", "muteMicrophone": "Silenciar micrófono", "unmuteMicrophone": "Activar micrófono", + "buddy": { + "title": "Conversación por voz", + "gloopieAlt": "Asistente Gloopie de voz", + "openSession": "Abrir sesión de voz", + "hangUp": "Colgar", + "sendToMenuBar": "Enviar a la barra de menús", + "listening": "Escuchando…", + "user-speaking": "Escuchándote…", + "agent-speaking": "Hablando…", + "muted": "Micrófono silenciado" + }, "states": { "off": "Iniciar conversación de voz", "starting": "Iniciando conversación de voz…", From 1da5a5ac861e432a169aff6f7385339644517f6e Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 21 Aug 2026 11:46:35 -0400 Subject: [PATCH 08/74] feat(voice): float Gloopie controls --- src-tauri/Cargo.toml | 2 +- src-tauri/capabilities/voice-buddy.json | 6 +- src-tauri/src/commands/mod.rs | 2 - src-tauri/src/commands/native_voice.rs | 19 ++- src-tauri/src/commands/notifications.rs | 14 ++- src-tauri/src/commands/voice_buddy.rs | 86 ++++++------- src-tauri/src/commands/voice_menu_bar.rs | 118 ------------------ src-tauri/src/commands/window_session.rs | 2 - src-tauri/src/lib.rs | 17 +-- src/features/chat/ui/ChatInputToolbar.tsx | 73 +++-------- .../chat/ui/__tests__/ChatInput.test.tsx | 70 +++-------- .../api/voiceConversation.test.ts | 3 - .../api/voiceConversation.ts | 4 - .../ui/VoiceBuddyApp.test.tsx | 48 +++---- .../voice-conversation/ui/VoiceBuddyApp.tsx | 92 +++++--------- src/main.tsx | 1 + .../useCompletionNotifications.test.ts | 53 +++++++- .../hooks/useCompletionNotifications.ts | 88 ++++++++----- src/shared/i18n/locales/en/chat.json | 4 +- src/shared/i18n/locales/es/chat.json | 4 +- src/shared/styles/globals.css | 6 + 21 files changed, 279 insertions(+), 433 deletions(-) delete mode 100644 src-tauri/src/commands/voice_menu_bar.rs diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index f697817ef..4c57c6e2d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -79,7 +79,7 @@ yaml_serde = "0.10.4" sha2 = "0.10" sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio-rustls", "sqlite", "migrate", "macros"] } sysinfo = { version = "0.33", default-features = false, features = ["system"] } -tauri = { version = "2", features = ["protocol-asset", "tray-icon"] } +tauri = { version = "2", features = ["protocol-asset"] } toml = "1.1.4" tauri-plugin-berdctl = { path = "plugins/berdctl" } tauri-plugin-app-test-driver = { path = "plugins/app-test-driver" } diff --git a/src-tauri/capabilities/voice-buddy.json b/src-tauri/capabilities/voice-buddy.json index b00e713eb..3ca7cbfa5 100644 --- a/src-tauri/capabilities/voice-buddy.json +++ b/src-tauri/capabilities/voice-buddy.json @@ -3,9 +3,5 @@ "identifier": "voice-buddy", "description": "Capability for the always-on-top voice conversation buddy", "windows": ["voice-buddy"], - "permissions": [ - "core:default", - "core:window:allow-start-dragging", - "core:window:allow-close" - ] + "permissions": ["core:default", "core:window:allow-start-dragging"] } diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index ef5fedac1..7b1c1c700 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -53,8 +53,6 @@ pub mod terminal; pub mod updates; pub mod voice_buddy; pub mod voice_capture; -#[cfg(target_os = "macos")] -pub mod voice_menu_bar; pub mod whoami; pub mod window_session; pub mod workspace_context; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index b44339fdd..3a680d89d 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -190,6 +190,14 @@ impl NativeVoiceState { )) } + pub fn is_active_for_session(&self, session_id: &str) -> bool { + self.runtime + .lock() + .ok() + .and_then(|runtime| runtime.session_id.clone()) + .is_some_and(|active_session_id| active_session_id == session_id) + } + pub fn set_microphone_muted(&self, app: &AppHandle, muted: bool) -> Result<(), String> { let (session_id, owner_window_label, revision) = { let runtime = self @@ -208,10 +216,6 @@ impl NativeVoiceState { (session_id, owner_window_label, runtime.revision) }; self.microphone_muted.store(muted, Ordering::SeqCst); - #[cfg(target_os = "macos")] - if let Err(error) = super::voice_menu_bar::set_muted(app, muted) { - log::warn!("Failed to update the voice menu bar mute state: {error}"); - } let event = NativeVoiceEvent::MicrophoneMute { session_id, muted, @@ -611,7 +615,6 @@ pub async fn start_native_voice_conversation( revision, }; let _ = event_window.emit(EVENT_NAME, event.clone()); - super::voice_buddy::emit(&event_app, event); } SttMessage::Final { text, delivered } => { let transcript = PendingTranscript { @@ -673,8 +676,6 @@ pub async fn start_native_voice_conversation( shutdown_pipeline(pipeline).await; } event_state.microphone_muted.store(false, Ordering::SeqCst); - #[cfg(target_os = "macos")] - super::voice_menu_bar::remove(&event_app); super::voice_buddy::remove(&event_app); event_app .state::() @@ -765,8 +766,6 @@ impl NativeVoiceState { runtime.revision }; self.microphone_muted.store(false, Ordering::SeqCst); - #[cfg(target_os = "macos")] - super::voice_menu_bar::remove(app); super::voice_buddy::remove(app); if let Some((owner, owner_id)) = owner.as_ref() { capture.release_owner(&owner.window_label, owner_id); @@ -821,8 +820,6 @@ impl NativeVoiceState { runtime.revision }; self.microphone_muted.store(false, Ordering::SeqCst); - #[cfg(target_os = "macos")] - super::voice_menu_bar::remove(app); super::voice_buddy::remove(app); if let (Some(owner), Some(session_id)) = (owner, session_id) { capture.release_owner(&owner.window_label, &native_owner_id(&session_id)); diff --git a/src-tauri/src/commands/notifications.rs b/src-tauri/src/commands/notifications.rs index e657294fb..4defc296d 100644 --- a/src-tauri/src/commands/notifications.rs +++ b/src-tauri/src/commands/notifications.rs @@ -2,9 +2,9 @@ use std::path::{Component, Path, PathBuf}; #[cfg(target_os = "macos")] use std::process::Command; -use tauri::AppHandle; #[cfg(target_os = "macos")] use tauri::Manager; +use tauri::{AppHandle, State}; struct CompletionNotificationRequest { session_id: String, @@ -27,10 +27,14 @@ struct CompletionNotificationState { #[tauri::command] pub fn show_completion_notification( app: AppHandle, + voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, session_id: String, body: String, sound: Option, ) -> Result<(), String> { + if voice_state.is_active_for_session(&session_id) { + return Ok(()); + } show_platform_completion_notification( app, CompletionNotificationRequest { @@ -41,6 +45,14 @@ pub fn show_completion_notification( ) } +#[tauri::command] +pub fn should_suppress_completion_notification( + voice_state: State<'_, crate::commands::native_voice::NativeVoiceState>, + session_id: String, +) -> bool { + voice_state.is_active_for_session(&session_id) +} + #[cfg(target_os = "macos")] pub fn init_completion_notifications(app: &tauri::AppHandle) -> Result<(), String> { macos_completion::init_completion_notifications(app) diff --git a/src-tauri/src/commands/voice_buddy.rs b/src-tauri/src/commands/voice_buddy.rs index b0a075af4..c3ba2f33e 100644 --- a/src-tauri/src/commands/voice_buddy.rs +++ b/src-tauri/src/commands/voice_buddy.rs @@ -3,6 +3,7 @@ use serde::Serialize; use tauri::{ AppHandle, Emitter, Manager, PhysicalPosition, WebviewUrl, WebviewWindow, WebviewWindowBuilder, + WindowEvent, }; use super::{native_voice::NativeVoiceState, voice_capture::VoiceCaptureState}; @@ -42,20 +43,25 @@ pub fn open_active_session(app: &AppHandle) -> Result<(), String> { Ok(()) } -fn position_near_bottom_right(window: &WebviewWindow) { - let Ok(Some(monitor)) = window.primary_monitor() else { +fn position_near_bottom_right(app: &AppHandle, window: &WebviewWindow) { + let owner_monitor = app + .state::() + .active_session_target() + .and_then(|(_, label)| app.get_webview_window(&label)) + .and_then(|owner| owner.current_monitor().ok().flatten()); + let Some(monitor) = owner_monitor.or_else(|| window.primary_monitor().ok().flatten()) else { return; }; - let monitor_position = monitor.position(); - let monitor_size = monitor.size(); + let work_area = monitor.work_area(); let Ok(window_size) = window.outer_size() else { return; }; - let x = monitor_position.x - + i32::try_from(monitor_size.width.saturating_sub(window_size.width)).unwrap_or_default() + let x = work_area.position.x + + i32::try_from(work_area.size.width.saturating_sub(window_size.width)).unwrap_or_default() - SCREEN_INSET; - let y = monitor_position.y - + i32::try_from(monitor_size.height.saturating_sub(window_size.height)).unwrap_or_default() + let y = work_area.position.y + + i32::try_from(work_area.size.height.saturating_sub(window_size.height)) + .unwrap_or_default() - SCREEN_INSET; let _ = window.set_position(PhysicalPosition::new(x, y)); } @@ -66,31 +72,37 @@ pub fn install(app: &AppHandle) -> Result<(), String> { return Ok(()); } - let entrypoint = if cfg!(target_os = "macos") { - "index.html?voiceBuddy=1&menuBar=1" - } else { - "index.html?voiceBuddy=1" - }; - let window = WebviewWindowBuilder::new(app, WINDOW_LABEL, WebviewUrl::App(entrypoint.into())) - .title("Berd voice conversation") - .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) - .resizable(false) - .maximizable(false) - .minimizable(false) - .decorations(false) - .always_on_top(true) - .skip_taskbar(true) - .focused(false) - .visible(false) - .build() - .map_err(|error| error.to_string())?; - position_near_bottom_right(&window); + let window = WebviewWindowBuilder::new( + app, + WINDOW_LABEL, + WebviewUrl::App("index.html?voiceBuddy=1".into()), + ) + .title("Berd voice conversation") + .inner_size(WINDOW_WIDTH, WINDOW_HEIGHT) + .resizable(false) + .maximizable(false) + .minimizable(false) + .decorations(false) + .transparent(true) + .shadow(false) + .always_on_top(true) + .skip_taskbar(true) + .focused(false) + .visible(false) + .build() + .map_err(|error| error.to_string())?; + window.on_window_event(|event| { + if let WindowEvent::CloseRequested { api, .. } = event { + api.prevent_close(); + } + }); + position_near_bottom_right(app, &window); window.show().map_err(|error| error.to_string()) } pub fn remove(app: &AppHandle) { if let Some(window) = app.get_webview_window(WINDOW_LABEL) { - let _ = window.close(); + let _ = window.destroy(); } } @@ -113,21 +125,3 @@ pub async fn stop_voice_conversation_from_buddy( ) -> Result<(), String> { state.stop_active(&app, capture.inner()).await } - -#[tauri::command] -pub fn send_voice_conversation_to_menu_bar( - app: AppHandle, - state: tauri::State<'_, NativeVoiceState>, -) -> Result<(), String> { - #[cfg(target_os = "macos")] - { - super::voice_menu_bar::install(&app, state.microphone_is_muted())?; - remove(&app); - Ok(()) - } - #[cfg(not(target_os = "macos"))] - { - let _ = (app, state); - Err("The menu bar voice surface is available only on macOS.".to_string()) - } -} diff --git a/src-tauri/src/commands/voice_menu_bar.rs b/src-tauri/src/commands/voice_menu_bar.rs deleted file mode 100644 index 8f2a1615c..000000000 --- a/src-tauri/src/commands/voice_menu_bar.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! macOS menu bar controls for the process-wide native voice conversation. - -use std::sync::mpsc; -use tauri::{ - menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem}, - tray::TrayIconBuilder, - AppHandle, Manager, -}; - -use super::{native_voice::NativeVoiceState, voice_capture::VoiceCaptureState}; - -const TRAY_ID: &str = "voice-conversation"; -const MUTE_ID: &str = "voice-conversation-mute"; -const OPEN_ID: &str = "voice-conversation-open"; -const STOP_ID: &str = "voice-conversation-stop"; -const SHOW_BUDDY_ID: &str = "voice-conversation-show-buddy"; - -// AppKit traps if an NSStatusItem is created, mutated, or dropped off its main -// queue. Tauri's tray wrapper drops the native item when it leaves the manager. -fn on_main_thread(app: &AppHandle, operation: F) -> Result -where - T: Send + 'static, - F: FnOnce(&AppHandle) -> Result + Send + 'static, -{ - if objc2::MainThreadMarker::new().is_some() { - return operation(app); - } - - let (sender, receiver) = mpsc::sync_channel(1); - let main_thread_app = app.clone(); - app.run_on_main_thread(move || { - let _ = sender.send(operation(&main_thread_app)); - }) - .map_err(|error| error.to_string())?; - receiver - .recv() - .map_err(|_| "The voice menu bar main-thread operation was interrupted.".to_string())? -} - -fn menu(app: &AppHandle, muted: bool) -> tauri::Result> { - let status = MenuItem::new(app, "Voice conversation active", false, None::<&str>)?; - let mute = CheckMenuItem::with_id(app, MUTE_ID, "Mute Microphone", true, muted, None::<&str>)?; - let open = MenuItem::with_id(app, OPEN_ID, "Open Voice Session", true, None::<&str>)?; - let show_buddy = MenuItem::with_id(app, SHOW_BUDDY_ID, "Show Gloopie", true, None::<&str>)?; - let stop = MenuItem::with_id(app, STOP_ID, "Stop Voice Conversation", true, None::<&str>)?; - let separator = PredefinedMenuItem::separator(app)?; - Menu::with_items( - app, - &[&status, &separator, &mute, &open, &show_buddy, &stop], - ) -} - -pub fn install(app: &AppHandle, muted: bool) -> Result<(), String> { - on_main_thread(app, move |app| { - let _ = app.remove_tray_by_id(TRAY_ID); - let menu = menu(app, muted).map_err(|error| error.to_string())?; - TrayIconBuilder::with_id(TRAY_ID) - .menu(&menu) - .title(if muted { "🔇" } else { "🎙" }) - .tooltip("Berd voice conversation") - .build(app) - .map(|_| ()) - .map_err(|error| error.to_string()) - }) -} - -pub fn set_muted(app: &AppHandle, muted: bool) -> Result<(), String> { - on_main_thread(app, move |app| { - let Some(tray) = app.tray_by_id(TRAY_ID) else { - return Ok(()); - }; - tray.set_title(Some(if muted { "🔇" } else { "🎙" })) - .map_err(|error| error.to_string())?; - tray.set_menu(Some(menu(app, muted).map_err(|error| error.to_string())?)) - .map_err(|error| error.to_string()) - }) -} - -pub fn remove(app: &AppHandle) { - if let Err(error) = on_main_thread(app, |app| { - let _ = app.remove_tray_by_id(TRAY_ID); - Ok(()) - }) { - log::warn!("Failed to remove the voice menu bar: {error}"); - } -} - -pub fn handle_menu_event(app: &AppHandle, event: tauri::menu::MenuEvent) { - match event.id().as_ref() { - MUTE_ID => { - let state = app.state::(); - let muted = !state.microphone_is_muted(); - if let Err(error) = state.set_microphone_muted(app, muted) { - log::warn!("Failed to update voice microphone mute: {error}"); - } - } - OPEN_ID => { - if let Err(error) = super::voice_buddy::open_active_session(app) { - log::warn!("Failed to open the voice session: {error}"); - } - } - SHOW_BUDDY_ID => match super::voice_buddy::install(app) { - Ok(()) => remove(app), - Err(error) => log::warn!("Failed to restore the Gloopie voice buddy: {error}"), - }, - STOP_ID => { - let app = app.clone(); - tauri::async_runtime::spawn(async move { - let state = app.state::().inner().clone(); - let capture = app.state::(); - if let Err(error) = state.stop_active(&app, capture.inner()).await { - log::warn!("Failed to stop the voice conversation from the menu bar: {error}"); - } - }); - } - _ => {} - } -} diff --git a/src-tauri/src/commands/window_session.rs b/src-tauri/src/commands/window_session.rs index d57094d15..9327ccabe 100644 --- a/src-tauri/src/commands/window_session.rs +++ b/src-tauri/src/commands/window_session.rs @@ -684,8 +684,6 @@ pub fn open_session_window( .state::() .stop_for_window_destroyed(&label_for_close); if stopped_native_voice { - #[cfg(target_os = "macos")] - crate::commands::voice_menu_bar::remove(&app_for_close); crate::commands::voice_buddy::remove(&app_for_close); app_for_close .state::() diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index e5608de74..783292a86 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -23,9 +23,9 @@ use services::{bundled_agents, bundled_skills, distro_bundle::DistroBundleState} use std::path::PathBuf; #[cfg(target_os = "macos")] use tauri::menu::{AboutMetadataBuilder, MenuBuilder, SubmenuBuilder}; -use tauri::{Manager, RunEvent}; #[cfg(target_os = "macos")] -use tauri::{WebviewWindow, WindowEvent}; +use tauri::WebviewWindow; +use tauri::{Manager, RunEvent, WindowEvent}; use tauri_plugin_window_state::StateFlags; #[cfg(target_os = "macos")] @@ -376,6 +376,8 @@ pub fn run() { // Surface WKWebView renderer memory and detect silent OOM reaps. services::renderer_monitor::start(app.handle().clone()); + attach_main_window_lifecycle(app); + // Build a custom macOS application menu so that the app submenu, // "About" item, and "Quit" item use the product name "Berd" // instead of the Cargo binary name. @@ -383,8 +385,6 @@ pub fn run() { { set_dev_dock_icon(); refresh_traffic_light_position_on_window_changes(app); - attach_main_window_lifecycle(app); - app.on_menu_event(commands::voice_menu_bar::handle_menu_event); let app_menu = SubmenuBuilder::new(app, "Berd") .about_with_text( @@ -651,7 +651,7 @@ pub fn run() { commands::native_voice::set_native_voice_input_muted, commands::voice_buddy::open_voice_conversation_session, commands::voice_buddy::stop_voice_conversation_from_buddy, - commands::voice_buddy::send_voice_conversation_to_menu_bar, + commands::notifications::should_suppress_completion_notification, commands::voice_capture::register_voice_renderer_instance, commands::window_session::get_session_window_support, commands::window_session::open_session_window, @@ -716,7 +716,6 @@ fn refresh_traffic_light_position_on_window_changes(app: &tauri::App) { } } -#[cfg(target_os = "macos")] fn attach_main_window_lifecycle(app: &tauri::App) { let Some(main) = app.get_webview_window("main") else { return; @@ -729,8 +728,12 @@ fn attach_main_window_lifecycle(app: &tauri::App) { .webview_windows() .keys() .any(|label| label != "main"); + let should_preserve = app_handle + .get_webview_window(commands::voice_buddy::WINDOW_LABEL) + .is_some() + || (cfg!(target_os = "macos") && has_secondary_window); - if has_secondary_window { + if should_preserve { api.prevent_close(); if let Some(main) = app_handle.get_webview_window("main") { let _ = main.hide(); diff --git a/src/features/chat/ui/ChatInputToolbar.tsx b/src/features/chat/ui/ChatInputToolbar.tsx index 8d03d40a0..05214effd 100644 --- a/src/features/chat/ui/ChatInputToolbar.tsx +++ b/src/features/chat/ui/ChatInputToolbar.tsx @@ -1,13 +1,14 @@ import { useMemo, useState } from "react"; import { Mic, + MicOff, Headphones, + PhoneOff, ArrowUp, File, FolderOpen, Settings2, Plus, - Volume2, } from "lucide-react"; import { useTranslation } from "react-i18next"; import { useLocaleFormatting } from "@/shared/i18n"; @@ -63,35 +64,6 @@ interface ChatInputToolbarComposerActions { voiceConversation?: ChatInputVoiceConversation; } -function UserVoiceActivityIndicator() { - return ( -