From 99d0a02d496da1df0c1c083be16d0eba132861b3 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Sun, 19 Jul 2026 21:05:27 -0500 Subject: [PATCH 01/13] feat(voice): add Read Aloud button for on-demand TTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a single Read Aloud button next to the existing Voice Mode/Mic buttons in the chat input footer. Reads everything the assistant said since the user's last turn (not just the latest message), since an agent can emit several consecutive assistant messages — tool calls, intermediate steps, a final reply — before control returns to the user. Reuses the existing voice infrastructure end to end: - getAssistantMessagesAfterLastUser for the turn boundary - useVoiceMode's queueSentence/stopSpeaking for playback - flushForTts for markdown-to-speech normalization and chunking New useReadAloud hook owns its own useVoiceMode() instance and is disabled whenever live Voice Mode is active on the same surface, since the two instances share global voice state but have independent audio playback and can't stop each other's audio. Wired into all three chat surfaces: AiChatView, GlobalAssistantView, and SidebarChatTab. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../components/ai/chat/input/ChatInput.tsx | 12 ++++ .../page-views/ai-page/AiChatView.tsx | 14 +++++ .../dashboard/GlobalAssistantView.tsx | 14 +++++ .../ai-assistant/SidebarChatTab.tsx | 14 +++++ .../ui/floating-input/InputFooter.tsx | 45 ++++++++++++- apps/web/src/hooks/useReadAloud.ts | 33 ++++++++++ .../getTextSinceLastUserTurn.test.ts | 63 +++++++++++++++++++ .../ai/streams/getTextSinceLastUserTurn.ts | 22 +++++++ 8 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/hooks/useReadAloud.ts create mode 100644 apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts create mode 100644 apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts diff --git a/apps/web/src/components/ai/chat/input/ChatInput.tsx b/apps/web/src/components/ai/chat/input/ChatInput.tsx index fc2df373a7..f419cb179b 100644 --- a/apps/web/src/components/ai/chat/input/ChatInput.tsx +++ b/apps/web/src/components/ai/chat/input/ChatInput.tsx @@ -65,6 +65,12 @@ export interface ChatInputProps { onVoiceModeClick?: () => void; /** Whether voice mode is currently active */ isVoiceModeActive?: boolean; + /** Callback when the read-aloud button is clicked */ + onReadAloudClick?: () => void; + /** Whether read-aloud is currently playing */ + isReadingAloud?: boolean; + /** Whether there is anything eligible to read aloud right now */ + canReadAloud?: boolean; /** Image attachments for vision support */ attachments?: ImageAttachment[]; /** Handler to add image files */ @@ -127,6 +133,9 @@ export const ChatInput = forwardRef( onProviderModelChange, onVoiceModeClick, isVoiceModeActive = false, + onReadAloudClick, + isReadingAloud = false, + canReadAloud = false, attachments, onAddFiles, onRemoveFile, @@ -305,6 +314,9 @@ export const ChatInput = forwardRef( onClearMicError={clearSpeechError} onVoiceModeClick={onVoiceModeClick} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={onReadAloudClick} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} selectedProvider={currentProvider} selectedModel={currentModel} onProviderModelChange={handleProviderModelChange} diff --git a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx index ec70b63551..f97d81cfd2 100644 --- a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx @@ -48,6 +48,8 @@ import { shouldRefreshAfterUndo } from '@/lib/ai/streams/shouldRefreshAfterUndo' import { shouldPrependConversation } from '@/lib/ai/streams/shouldPrependConversation'; import { shouldReloadOnComountComplete } from '@/lib/ai/streams/shouldReloadOnComountComplete'; import { getBrowserSessionId } from '@/lib/ai/core/browser-session-id'; +import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; +import { useReadAloud } from '@/hooks/useReadAloud'; // Shared hooks and components import { @@ -581,6 +583,15 @@ const AiChatView: React.FC = ({ page }) => { const renderedMessages = useRenderedMessages(page.id, currentConversationId); const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn. Owns its own useVoiceMode() instance, so it's disabled + // whenever VoiceCallPanel's live-call instance is active on this surface. + const { isReadingAloud, toggleReadAloud } = useReadAloud(); + const canReadAloud = useMemo( + () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeActive] + ); + // "Load older" (epic leaf 6.6, scroll-to-top): AiChatView's route IS the agent-conversation // route (page.id is the agentId), so the shared agent-mode loader applies directly. const { isLoadingOlder } = useConversationOlderPageState(currentConversationId); @@ -1518,6 +1529,9 @@ const AiChatView: React.FC = ({ page }) => { }} onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={() => toggleReadAloud(plainMessages)} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx index f060306994..9885e3a9fb 100644 --- a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx @@ -108,6 +108,8 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText'; import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline'; import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; +import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; +import { useReadAloud } from '@/hooks/useReadAloud'; import { createId } from '@paralleldrive/cuid2'; const VOICE_OWNER: VoiceModeOwner = 'global-assistant'; @@ -436,6 +438,15 @@ const GlobalAssistantView: React.FC = () => { // above) never renders post-cutover — it stays the transport/controller only. const renderedMessages = useRenderedMessages(streamChannelId ?? '', currentConversationId); const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); + + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn. Owns its own useVoiceMode() instance, so it's disabled + // whenever VoiceCallPanel's live-call instance is active on this surface. + const { isReadingAloud, toggleReadAloud } = useReadAloud(); + const canReadAloud = useMemo( + () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeActive] + ); // Loading/error UI reads the cache entry's state (replaces the context's // isMessagesLoading and the dashboard store's isConversationMessagesLoading). const messagesLoadState = useConversationLoadState(currentConversationId); @@ -1095,6 +1106,9 @@ const GlobalAssistantView: React.FC = () => { popupPlacement={props.inputPosition === 'centered' ? 'bottom' : 'top'} onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={() => toggleReadAloud(plainMessages)} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx index f1da34ce20..81a8f66f39 100644 --- a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx +++ b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx @@ -44,6 +44,8 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText'; import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline'; import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; +import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; +import { useReadAloud } from '@/hooks/useReadAloud'; import { createId } from '@paralleldrive/cuid2'; import { useStopStream } from '@/hooks/useStopStream'; import { useOwnStreamMirror } from '@/hooks/useOwnStreamMirror'; @@ -542,6 +544,15 @@ const SidebarChatTab: React.FC = () => { const disableVoiceMode = useVoiceModeStore((s) => s.disable); const isVoiceModeActive = isVoiceModeEnabled && voiceOwner === VOICE_OWNER; + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn. Owns its own useVoiceMode() instance, so it's disabled + // whenever VoiceCallPanel's live-call instance is active on this surface. + const { isReadingAloud, toggleReadAloud } = useReadAloud(); + const canReadAloud = useMemo( + () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeActive] + ); + // Display preferences const { preferences: displayPreferences } = useDisplayPreferences(); @@ -1115,6 +1126,9 @@ const SidebarChatTab: React.FC = () => { variant="sidebar" onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={() => toggleReadAloud(plainMessages)} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/ui/floating-input/InputFooter.tsx b/apps/web/src/components/ui/floating-input/InputFooter.tsx index 3fb5b856c2..237581e793 100644 --- a/apps/web/src/components/ui/floating-input/InputFooter.tsx +++ b/apps/web/src/components/ui/floating-input/InputFooter.tsx @@ -7,7 +7,7 @@ import { TooltipTrigger, TooltipContent, } from '@/components/ui/tooltip'; -import { Mic, AudioLines, MicOff } from 'lucide-react'; +import { Mic, AudioLines, MicOff, Volume2 } from 'lucide-react'; import { cn } from '@/lib/utils'; import { ProviderModelSelector } from '@/components/ai/chat/input/ProviderModelSelector'; import { ToolsPopover } from './ToolsPopover'; @@ -61,6 +61,12 @@ export interface InputFooterProps { onVoiceModeClick?: () => void; /** Whether voice mode is currently active */ isVoiceModeActive?: boolean; + /** Callback when the read-aloud button is clicked */ + onReadAloudClick?: () => void; + /** Whether read-aloud is currently playing */ + isReadingAloud?: boolean; + /** Whether there is anything eligible to read aloud right now */ + canReadAloud?: boolean; /** Error message from microphone/speech recognition */ micError?: string | null; /** Callback to clear the mic error */ @@ -110,6 +116,9 @@ export function InputFooter({ isMicSupported = true, onVoiceModeClick, isVoiceModeActive = false, + onReadAloudClick, + isReadingAloud = false, + canReadAloud = false, micError, onClearMicError, selectedProvider, @@ -165,6 +174,40 @@ export function InputFooter({ /> )} + {/* Read Aloud button (on-demand TTS for the assistant's last turn) */} + + + + + + {isVoiceProGated + ? 'Read aloud requires a Pro plan' + : isReadingAloud + ? 'Stop reading aloud' + : 'Read aloud'} + + + {/* Voice Mode button (hands-free STT/TTS) */} diff --git a/apps/web/src/hooks/useReadAloud.ts b/apps/web/src/hooks/useReadAloud.ts new file mode 100644 index 0000000000..bec689baea --- /dev/null +++ b/apps/web/src/hooks/useReadAloud.ts @@ -0,0 +1,33 @@ +'use client'; + +import { useCallback } from 'react'; +import type { UIMessage } from 'ai'; +import { useVoiceMode } from './useVoiceMode'; +import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; +import { flushForTts } from '@/lib/voice/chunkForTts'; + +/** + * On-demand TTS for "read the assistant's last turn aloud" — distinct from + * full hands-free Voice Mode. Owns its own `useVoiceMode()` instance, so + * callers must not use this while Voice Mode is active on the same surface: + * both instances share the same global voice-state store but have + * independent audio playback, so one can't stop audio started by the other. + */ +export function useReadAloud() { + const { isSpeaking, queueSentence, stopSpeaking } = useVoiceMode(); + + const toggleReadAloud = useCallback( + (messages: readonly UIMessage[]) => { + if (isSpeaking) { + stopSpeaking(); + return; + } + const text = getTextSinceLastUserTurn(messages); + if (!text.trim()) return; + flushForTts(text).forEach((chunk) => queueSentence(chunk)); + }, + [isSpeaking, stopSpeaking, queueSentence] + ); + + return { isReadingAloud: isSpeaking, toggleReadAloud }; +} diff --git a/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts b/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts new file mode 100644 index 0000000000..13f9c4b9d0 --- /dev/null +++ b/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect } from 'vitest'; +import type { UIMessage } from 'ai'; +import { getTextSinceLastUserTurn } from '../getTextSinceLastUserTurn'; + +const msg = (id: string, role: UIMessage['role'], parts: UIMessage['parts']): UIMessage => ({ + id, + role, + parts, +}); + +const text = (t: string) => ({ type: 'text' as const, text: t }); + +describe('getTextSinceLastUserTurn', () => { + it('given no user message at all, returns empty string', () => { + const messages = [msg('a1', 'assistant', [text('hello')])]; + expect(getTextSinceLastUserTurn(messages)).toBe(''); + }); + + it('given a single assistant reply, returns its text', () => { + const messages = [msg('u1', 'user', [text('hi')]), msg('a1', 'assistant', [text('hello there')])]; + expect(getTextSinceLastUserTurn(messages)).toBe('hello there'); + }); + + it('given multiple consecutive assistant messages, joins their text in order', () => { + const messages = [ + msg('u1', 'user', [text('go')]), + msg('a1', 'assistant', [text('step one')]), + msg('a2', 'assistant', [text('step two')]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('step one\n\nstep two'); + }); + + it('skips tool-call parts and only reads text parts within a message', () => { + const messages = [ + msg('u1', 'user', [text('run it')]), + msg('a1', 'assistant', [ + { type: 'tool-run', toolCallId: 't1', state: 'output-available' } as unknown as UIMessage['parts'][number], + text('done running'), + ]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('done running'); + }); + + it('given a trailing tool-only message with no text, drops it from the joined output', () => { + const messages = [ + msg('u1', 'user', [text('go')]), + msg('a1', 'assistant', [text('here is the answer')]), + msg('a2', 'assistant', [ + { type: 'tool-run', toolCallId: 't2', state: 'output-available' } as unknown as UIMessage['parts'][number], + ]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('here is the answer'); + }); + + it('given the last user message has no reply yet, returns empty string', () => { + const messages = [msg('a1', 'assistant', [text('old reply')]), msg('u1', 'user', [text('new question')])]; + expect(getTextSinceLastUserTurn(messages)).toBe(''); + }); + + it('given an empty array, returns empty string', () => { + expect(getTextSinceLastUserTurn([])).toBe(''); + }); +}); diff --git a/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts b/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts new file mode 100644 index 0000000000..60c272056e --- /dev/null +++ b/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts @@ -0,0 +1,22 @@ +import type { UIMessage } from 'ai'; +import { getAssistantMessagesAfterLastUser } from './getAssistantMessagesAfterLastUser'; + +const textOf = (message: UIMessage): string => + (message.parts ?? []) + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join(''); + +/** + * Plain-text speech source for "read aloud": every assistant message since + * the user's last turn, joined in order. An agent can emit several + * consecutive assistant messages (tool calls, intermediate steps, a final + * reply) before control returns to the user, so this reads all of them + * rather than just the latest one. + */ +export function getTextSinceLastUserTurn(messages: readonly UIMessage[]): string { + return getAssistantMessagesAfterLastUser(messages) + .map(textOf) + .filter(Boolean) + .join('\n\n'); +} From 16db1f79354a78fc4d6382e1a3c35ae6bf8c2870 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Sun, 19 Jul 2026 21:18:59 -0500 Subject: [PATCH 02/13] fix(voice): centralize read-aloud audio into a module-level singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses two Codex review findings on useReadAloud.ts (PR #2173): 1. Multiple mounted chat surfaces (e.g. the right-sidebar chat tab alongside the main AiChatView/GlobalAssistantView content) each ran their own useVoiceMode() instance. All instances shared global voiceState/isSpeaking via the store, but each held its own local AudioContext/audioSource — so Stop clicked on one surface cleared shared state but couldn't reach audio actually playing from another surface's instance. 2. useVoiceMode's unmount cleanup stopped that instance's own audio but never reset the shared store's speaking state, so navigating away mid-playback could leave every other surface stuck showing "Stop reading aloud". Fix: apps/web/src/lib/voice/readAloudPlayer.ts is a module-singleton player (one AudioContext, one audioSource, one queue) completely decoupled from useVoiceMode/useVoiceModeStore's voiceState. Every useReadAloud() call site now just subscribes to it via useSyncExternalStore, so starting or stopping from any surface acts on the one real audio source everywhere. Also switches the "disable while Voice Mode is active" gate in all three chat views from the surface-scoped isVoiceModeActive to the global isVoiceModeEnabled flag — a live call on a DIFFERENT surface still plays through its own separate AudioContext and would overlap with read-aloud audio, which the surface-scoped check missed. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../page-views/ai-page/AiChatView.tsx | 10 +- .../dashboard/GlobalAssistantView.tsx | 10 +- .../ai-assistant/SidebarChatTab.tsx | 10 +- apps/web/src/hooks/useReadAloud.ts | 30 +++-- .../voice/__tests__/readAloudPlayer.test.ts | 114 ++++++++++++++++ apps/web/src/lib/voice/readAloudPlayer.ts | 126 ++++++++++++++++++ 6 files changed, 276 insertions(+), 24 deletions(-) create mode 100644 apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts create mode 100644 apps/web/src/lib/voice/readAloudPlayer.ts diff --git a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx index f97d81cfd2..ab4f7e7920 100644 --- a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx @@ -584,12 +584,14 @@ const AiChatView: React.FC = ({ page }) => { const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); // Read Aloud: on-demand TTS for everything the assistant said since the - // user's last turn. Owns its own useVoiceMode() instance, so it's disabled - // whenever VoiceCallPanel's live-call instance is active on this surface. + // user's last turn, via a shared playback singleton (see readAloudPlayer). + // Disabled whenever Voice Mode is enabled on ANY surface (not just this + // one) since a live call elsewhere plays through its own separate + // AudioContext and would overlap with this audio. const { isReadingAloud, toggleReadAloud } = useReadAloud(); const canReadAloud = useMemo( - () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, - [plainMessages, isVoiceModeActive] + () => !isVoiceModeEnabled && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeEnabled] ); // "Load older" (epic leaf 6.6, scroll-to-top): AiChatView's route IS the agent-conversation diff --git a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx index 9885e3a9fb..ce2df5cda9 100644 --- a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx @@ -440,12 +440,14 @@ const GlobalAssistantView: React.FC = () => { const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); // Read Aloud: on-demand TTS for everything the assistant said since the - // user's last turn. Owns its own useVoiceMode() instance, so it's disabled - // whenever VoiceCallPanel's live-call instance is active on this surface. + // user's last turn, via a shared playback singleton (see readAloudPlayer). + // Disabled whenever Voice Mode is enabled on ANY surface (not just this + // one) since a live call elsewhere plays through its own separate + // AudioContext and would overlap with this audio. const { isReadingAloud, toggleReadAloud } = useReadAloud(); const canReadAloud = useMemo( - () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, - [plainMessages, isVoiceModeActive] + () => !isVoiceModeEnabled && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeEnabled] ); // Loading/error UI reads the cache entry's state (replaces the context's // isMessagesLoading and the dashboard store's isConversationMessagesLoading). diff --git a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx index 81a8f66f39..ac216fc1b9 100644 --- a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx +++ b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx @@ -545,12 +545,14 @@ const SidebarChatTab: React.FC = () => { const isVoiceModeActive = isVoiceModeEnabled && voiceOwner === VOICE_OWNER; // Read Aloud: on-demand TTS for everything the assistant said since the - // user's last turn. Owns its own useVoiceMode() instance, so it's disabled - // whenever VoiceCallPanel's live-call instance is active on this surface. + // user's last turn, via a shared playback singleton (see readAloudPlayer). + // Disabled whenever Voice Mode is enabled on ANY surface (not just this + // one) since a live call elsewhere plays through its own separate + // AudioContext and would overlap with this audio. const { isReadingAloud, toggleReadAloud } = useReadAloud(); const canReadAloud = useMemo( - () => !isVoiceModeActive && getTextSinceLastUserTurn(plainMessages).trim().length > 0, - [plainMessages, isVoiceModeActive] + () => !isVoiceModeEnabled && getTextSinceLastUserTurn(plainMessages).trim().length > 0, + [plainMessages, isVoiceModeEnabled] ); // Display preferences diff --git a/apps/web/src/hooks/useReadAloud.ts b/apps/web/src/hooks/useReadAloud.ts index bec689baea..f993c5642b 100644 --- a/apps/web/src/hooks/useReadAloud.ts +++ b/apps/web/src/hooks/useReadAloud.ts @@ -1,33 +1,39 @@ 'use client'; -import { useCallback } from 'react'; +import { useCallback, useSyncExternalStore } from 'react'; import type { UIMessage } from 'ai'; -import { useVoiceMode } from './useVoiceMode'; import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; import { flushForTts } from '@/lib/voice/chunkForTts'; +import { + startReadAloud, + stopReadAloud, + isReadAloudPlaying, + subscribeReadAloud, +} from '@/lib/voice/readAloudPlayer'; + +const getServerSnapshot = () => false; /** * On-demand TTS for "read the assistant's last turn aloud" — distinct from - * full hands-free Voice Mode. Owns its own `useVoiceMode()` instance, so - * callers must not use this while Voice Mode is active on the same surface: - * both instances share the same global voice-state store but have - * independent audio playback, so one can't stop audio started by the other. + * full hands-free Voice Mode. Every call site shares the same module-level + * playback singleton (`readAloudPlayer`), so starting or stopping from any + * mounted chat surface acts on the one real audio source. */ export function useReadAloud() { - const { isSpeaking, queueSentence, stopSpeaking } = useVoiceMode(); + const isReadingAloud = useSyncExternalStore(subscribeReadAloud, isReadAloudPlaying, getServerSnapshot); const toggleReadAloud = useCallback( (messages: readonly UIMessage[]) => { - if (isSpeaking) { - stopSpeaking(); + if (isReadAloudPlaying()) { + stopReadAloud(); return; } const text = getTextSinceLastUserTurn(messages); if (!text.trim()) return; - flushForTts(text).forEach((chunk) => queueSentence(chunk)); + startReadAloud(flushForTts(text)); }, - [isSpeaking, stopSpeaking, queueSentence] + [] ); - return { isReadingAloud: isSpeaking, toggleReadAloud }; + return { isReadingAloud, toggleReadAloud }; } diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts new file mode 100644 index 0000000000..fceb87daa1 --- /dev/null +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { + startReadAloud, + stopReadAloud, + isReadAloudPlaying, + subscribeReadAloud, +} from '../readAloudPlayer'; + +class FakeAudioBufferSourceNode { + buffer: unknown = null; + onended: (() => void) | null = null; + connect(): void {} + start(): void {} + stop = vi.fn(); +} + +class FakeAudioContext { + state = 'running'; + destination = {}; + createBufferSource(): FakeAudioBufferSourceNode { + return new FakeAudioBufferSourceNode(); + } + decodeAudioData(): Promise { + return Promise.resolve({}); + } + resume(): Promise { + return Promise.resolve(); + } +} + +describe('readAloudPlayer', () => { + beforeEach(() => { + vi.stubGlobal('AudioContext', FakeAudioContext); + vi.mocked(fetchWithAuth).mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + }); + + afterEach(() => { + stopReadAloud(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('is not playing before anything starts', () => { + expect(isReadAloudPlaying()).toBe(false); + }); + + it('flips to playing synchronously when starting, before synthesis resolves', () => { + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + }); + + it('given no chunks, does not start playing', () => { + startReadAloud([]); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('stop clears the playing state immediately', () => { + startReadAloud(['hello there']); + stopReadAloud(); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('notifies subscribers on start and stop, regardless of which call site triggers it', () => { + // Simulates two independent useReadAloud() call sites (e.g. sidebar + + // main content) both observing the one shared player. + const surfaceA = vi.fn(); + const surfaceB = vi.fn(); + const unsubscribeA = subscribeReadAloud(surfaceA); + const unsubscribeB = subscribeReadAloud(surfaceB); + + startReadAloud(['hello there']); + expect(surfaceA).toHaveBeenCalledTimes(1); + expect(surfaceB).toHaveBeenCalledTimes(1); + + // A stop triggered from "surface B" must be observable by "surface A" — + // this is the cross-surface ownership guarantee the singleton provides. + stopReadAloud(); + expect(surfaceA).toHaveBeenCalledTimes(2); + expect(surfaceB).toHaveBeenCalledTimes(2); + expect(isReadAloudPlaying()).toBe(false); + + unsubscribeA(); + unsubscribeB(); + }); + + it('unsubscribing stops further notifications', () => { + const listener = vi.fn(); + const unsubscribe = subscribeReadAloud(listener); + unsubscribe(); + + startReadAloud(['hello there']); + expect(listener).not.toHaveBeenCalled(); + }); + + it('starting a new read-aloud while one is in flight replaces the previous queue', () => { + startReadAloud(['first attempt', 'more of the first']); + startReadAloud(['second attempt']); + expect(isReadAloudPlaying()).toBe(true); + }); + + it('stopping when nothing is playing is a no-op that does not throw', () => { + expect(() => stopReadAloud()).not.toThrow(); + expect(isReadAloudPlaying()).toBe(false); + }); +}); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts new file mode 100644 index 0000000000..e7d68a70e3 --- /dev/null +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -0,0 +1,126 @@ +'use client'; + +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; + +/** + * Module-singleton "read aloud" audio player, deliberately independent of + * React component lifecycles and of `useVoiceMode`'s per-instance + * AudioContext/audioSource refs. + * + * Read Aloud can be triggered from multiple mounted chat surfaces at once + * (e.g. the right-sidebar chat tab alongside the main AiChatView/ + * GlobalAssistantView content). A per-component `useVoiceMode()` instance + * only tracks its OWN AudioContext locally while sharing global voice state + * via the store — so a second instance's "stop" call clears shared state but + * can't reach the first instance's actual playing AudioBufferSourceNode, and + * an unmounting instance never resets that shared state either. Keeping the + * one real audio source at module scope means there is only ever one thing + * to stop, reachable from anywhere `useReadAloud()` is called. + */ + +type Listener = () => void; + +let audioContext: AudioContext | null = null; +let audioSource: AudioBufferSourceNode | null = null; +let queue: string[] = []; +let playing = false; +const listeners = new Set(); + +function notify(): void { + listeners.forEach((listener) => { listener(); }); +} + +function getAudioContext(): AudioContext { + if (!audioContext) { + audioContext = new AudioContext(); + } + return audioContext; +} + +async function synthesize(text: string): Promise { + const { ttsVoice, ttsSpeed } = useVoiceModeStore.getState(); + // Created before the network await so the browser still credits this + // AudioContext to the user gesture that triggered playback. + const ctx = getAudioContext(); + try { + const response = await fetchWithAuth('/api/voice/synthesize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, voice: ttsVoice, speed: ttsSpeed }), + }); + if (!response.ok) return null; + const audioData = await response.arrayBuffer(); + if (ctx.state === 'suspended') { + await ctx.resume(); + } + return await ctx.decodeAudioData(audioData); + } catch { + return null; + } +} + +async function playNext(): Promise { + if (!playing) return; + const text = queue.shift(); + if (text === undefined) { + playing = false; + notify(); + return; + } + + const buffer = await synthesize(text); + // Stopped while this chunk was being synthesized — discard the result. + if (!playing) return; + if (!buffer) { + // Skip a chunk that failed to synthesize rather than abandoning the rest. + void playNext(); + return; + } + + const ctx = getAudioContext(); + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + audioSource = source; + source.onended = () => { + if (audioSource === source) { + audioSource = null; + void playNext(); + } + }; + source.start(); +} + +export function startReadAloud(chunks: string[]): void { + stopReadAloud(); + if (chunks.length === 0) return; + queue = [...chunks]; + playing = true; + notify(); + void playNext(); +} + +export function stopReadAloud(): void { + const wasPlaying = playing; + if (audioSource) { + try { + audioSource.stop(); + } catch { + // Already stopped. + } + audioSource = null; + } + queue = []; + playing = false; + if (wasPlaying) notify(); +} + +export function isReadAloudPlaying(): boolean { + return playing; +} + +export function subscribeReadAloud(listener: Listener): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} From e0643f70021ee59652a6c0a81fb26a13669aefc6 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Sun, 19 Jul 2026 21:43:02 -0500 Subject: [PATCH 03/13] test(voice): cover readAloudPlayer's async playback path Existing tests only checked synchronous start/stop state transitions. Adds coverage for the actual promise chain: a chunk synthesizing and playing, multiple chunks playing back to back via onended, a failed synthesis being skipped without aborting the rest of the queue, and a chunk that finishes synthesizing after stop() was already called being correctly discarded rather than resurrecting playback. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../voice/__tests__/readAloudPlayer.test.ts | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts index fceb87daa1..3734ed040f 100644 --- a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -16,15 +16,19 @@ class FakeAudioBufferSourceNode { buffer: unknown = null; onended: (() => void) | null = null; connect(): void {} - start(): void {} + start = vi.fn(); stop = vi.fn(); } +const createdSources: FakeAudioBufferSourceNode[] = []; + class FakeAudioContext { state = 'running'; destination = {}; createBufferSource(): FakeAudioBufferSourceNode { - return new FakeAudioBufferSourceNode(); + const node = new FakeAudioBufferSourceNode(); + createdSources.push(node); + return node; } decodeAudioData(): Promise { return Promise.resolve({}); @@ -36,6 +40,7 @@ class FakeAudioContext { describe('readAloudPlayer', () => { beforeEach(() => { + createdSources.length = 0; vi.stubGlobal('AudioContext', FakeAudioContext); vi.mocked(fetchWithAuth).mockResolvedValue({ ok: true, @@ -111,4 +116,54 @@ describe('readAloudPlayer', () => { expect(() => stopReadAloud()).not.toThrow(); expect(isReadAloudPlaying()).toBe(false); }); + + it('synthesizes and starts playback of a single chunk, then finishes naturally when it ends', async () => { + startReadAloud(['only chunk']); + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + expect(createdSources[0].start).toHaveBeenCalledTimes(1); + expect(isReadAloudPlaying()).toBe(true); + + // Simulate the browser firing onended once the clip finishes. + createdSources[0].onended?.(); + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + }); + + it('plays multiple queued chunks back to back in order', async () => { + startReadAloud(['first', 'second']); + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + + createdSources[0].onended?.(); + await vi.waitFor(() => expect(createdSources).toHaveLength(2)); + expect(isReadAloudPlaying()).toBe(true); + + createdSources[1].onended?.(); + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + }); + + it('skips a chunk that fails to synthesize and continues with the rest', async () => { + vi.mocked(fetchWithAuth) + .mockResolvedValueOnce({ ok: false } as Response) + .mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + + startReadAloud(['broken chunk', 'good chunk']); + // The first chunk's failed synthesis is skipped without ever creating a + // source; only the second, successful chunk should end up playing. + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + expect(isReadAloudPlaying()).toBe(true); + + createdSources[0].onended?.(); + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + }); + + it('discards a chunk that finishes synthesizing after stop was already called', async () => { + startReadAloud(['only chunk']); + stopReadAloud(); + // Let the in-flight synthesis resolve; it must not resurrect playback. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(createdSources).toHaveLength(0); + expect(isReadAloudPlaying()).toBe(false); + }); }); From b6f3a1ac2f04612ebe2056c0a6dfe80f059fc798 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Sun, 19 Jul 2026 22:07:44 -0500 Subject: [PATCH 04/13] fix(voice): close 3 more races/gaps found on the centralized player MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three fresh Codex findings on commit 16db1f793 (PR #2173): 1. (P1) readAloudPlayer.ts: a stop-then-restart while the stopped run's synthesis was still in flight could let the stale run's audio start playing after the new run had already begun, since both checked the same shared `playing` boolean. Added a monotonic `generation` token, captured per playNext() call and re-checked after every await, so a stale run is discarded even when a newer run has since flipped `playing` back to true. New regression test reproduces the exact interleaving (stale fetch resolves after the new run has started) and asserts only the new run ever creates an audio source. 2. (P1) None of the three handleVoiceModeToggle implementations (AiChatView, GlobalAssistantView, SidebarChatTab) stopped read-aloud playback before enabling Voice Mode, so VoiceCallPanel's mic capture could start while read-aloud audio was still playing — synthesized speech could get picked up as if it were user input. Now calls stopReadAloud() before enabling. 3. (P2) InputFooter's Read Aloud button was fully disabled whenever the footer's `disabled` prop was true (e.g. while a new message is streaming), which meant the only control that could stop ongoing playback became unreachable if the user sent a follow-up message mid-read. The button now stays enabled/clickable whenever isReadingAloud is true, regardless of the rest of the footer's disabled state. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../page-views/ai-page/AiChatView.tsx | 4 ++ .../dashboard/GlobalAssistantView.tsx | 4 ++ .../ai-assistant/SidebarChatTab.tsx | 4 ++ .../ui/floating-input/InputFooter.tsx | 4 +- .../voice/__tests__/readAloudPlayer.test.ts | 38 +++++++++++++++++++ apps/web/src/lib/voice/readAloudPlayer.ts | 26 +++++++++---- 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx index ab4f7e7920..e78d28af30 100644 --- a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx @@ -50,6 +50,7 @@ import { shouldReloadOnComountComplete } from '@/lib/ai/streams/shouldReloadOnCo import { getBrowserSessionId } from '@/lib/ai/core/browser-session-id'; import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; import { useReadAloud } from '@/hooks/useReadAloud'; +import { stopReadAloud } from '@/lib/voice/readAloudPlayer'; // Shared hooks and components import { @@ -1256,6 +1257,9 @@ const AiChatView: React.FC = ({ page }) => { if (isVoiceModeActive) { disableVoiceMode(); } else { + // Voice Mode's mic capture would otherwise pick up read-aloud audio + // still playing through the shared singleton. + stopReadAloud(); enableVoiceMode(VOICE_OWNER); } }, [isVoiceModeActive, enableVoiceMode, disableVoiceMode]); diff --git a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx index ce2df5cda9..03c2c6a4bc 100644 --- a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx @@ -110,6 +110,7 @@ import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActiv import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; import { useReadAloud } from '@/hooks/useReadAloud'; +import { stopReadAloud } from '@/lib/voice/readAloudPlayer'; import { createId } from '@paralleldrive/cuid2'; const VOICE_OWNER: VoiceModeOwner = 'global-assistant'; @@ -917,6 +918,9 @@ const GlobalAssistantView: React.FC = () => { if (isVoiceModeActive) { disableVoiceMode(); } else { + // Voice Mode's mic capture would otherwise pick up read-aloud audio + // still playing through the shared singleton. + stopReadAloud(); enableVoiceMode(VOICE_OWNER); } }, [isVoiceModeActive, enableVoiceMode, disableVoiceMode]); diff --git a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx index ac216fc1b9..0220bb8dc4 100644 --- a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx +++ b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx @@ -46,6 +46,7 @@ import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActiv import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; import { getTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; import { useReadAloud } from '@/hooks/useReadAloud'; +import { stopReadAloud } from '@/lib/voice/readAloudPlayer'; import { createId } from '@paralleldrive/cuid2'; import { useStopStream } from '@/hooks/useStopStream'; import { useOwnStreamMirror } from '@/hooks/useOwnStreamMirror'; @@ -882,6 +883,9 @@ const SidebarChatTab: React.FC = () => { if (isVoiceModeActive) { disableVoiceMode(); } else { + // Voice Mode's mic capture would otherwise pick up read-aloud audio + // still playing through the shared singleton. + stopReadAloud(); enableVoiceMode(VOICE_OWNER); } }, [isVoiceModeActive, enableVoiceMode, disableVoiceMode]); diff --git a/apps/web/src/components/ui/floating-input/InputFooter.tsx b/apps/web/src/components/ui/floating-input/InputFooter.tsx index 237581e793..f610bb8e31 100644 --- a/apps/web/src/components/ui/floating-input/InputFooter.tsx +++ b/apps/web/src/components/ui/floating-input/InputFooter.tsx @@ -180,8 +180,8 @@ export function InputFooter({ @@ -204,7 +206,9 @@ export function InputFooter({ ? 'Read aloud requires a Pro plan' : isReadingAloud ? 'Stop reading aloud' - : 'Read aloud'} + : isListening + ? 'Read aloud unavailable while dictating' + : 'Read aloud'} From ac734cbd84962a6873fbbd0747ae82c15d470a54 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 20 Jul 2026 00:24:27 -0500 Subject: [PATCH 09/13] fix(voice): coordinate dictation cross-surface, load settings, cancel billing on stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from Codex's review of 6584ad0dc (PR #2173): 1. (P2) Load persisted voice settings before Read Aloud starts. synthesize() reads ttsVoice/ttsSpeed straight from useVoiceModeStore, but that store's loadSettings() (hydrates from localStorage) previously only ran when useVoiceMode() mounted inside the Voice Mode panel — never for Read Aloud, which no longer depends on that hook at all. A user's persisted voice choice would silently be ignored the first time they used Read Aloud without ever opening Voice Mode. Now called on every startReadAloud() (idempotent and cheap, so no one-time guard needed). 2. (P2) Coordinate mic dictation across mounted chat surfaces. The basic Mic control (useSpeechRecognition) is local per-ChatInput-instance state, unlike Voice Mode's single global store — so the earlier same-surface-only fix missed the case where a DIFFERENT surface's dictation is active while this surface starts Read Aloud, letting TTS audio get transcribed into that other draft. Added a small shared useDictationActivityStore (a count, not a boolean, so two simultaneously-listening surfaces don't have one's stop clear the other's still-active state) that every useSpeechRecognition() instance feeds. readAloudPlayer subscribes to it the same way it already does for Voice Mode, and useReadAloud()'s canReadAloud/toggleReadAloud now check it too — which also let the now-redundant local isListening prop check in InputFooter's Read Aloud button be removed. 3. (P2) Propagate the client's Stop cancellation to the OpenAI request itself. The previous AbortController fix only cancelled the browser's own fetch; /api/voice/synthesize started its upstream OpenAI request without forwarding request.signal, so a stopped chunk's provider call (and its billing) ran to completion regardless. Route now passes `signal: request.signal` to the upstream fetch — the existing hold-release-on-non-billed-path logic already handles the resulting abort correctly with no other changes needed. Also required a `@vitest-environment node` pragma on the synthesize route test file: it tests server code, but ran under the project's jsdom default, which has its own competing AbortController/AbortSignal globals — jsdom's Request constructor rejected a real AbortController's signal as "not an instance of AbortSignal" once a test actually exercised one. node is the more accurate environment for testing a route handler, not a workaround. Verified via typecheck plus the full related suite (92 tests: readAloudPlayer, both voice API routes, getTextSinceLastUserTurn, AiChatView ×2, ChatInput), all passing. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../voice/synthesize/__tests__/route.test.ts | 45 ++++++++++++++++ .../web/src/app/api/voice/synthesize/route.ts | 6 ++- .../ui/floating-input/InputFooter.tsx | 2 +- apps/web/src/hooks/useReadAloud.ts | 21 ++++---- apps/web/src/hooks/useSpeechRecognition.ts | 43 +++++++++++++++ .../voice/__tests__/readAloudPlayer.test.ts | 24 +++++++++ apps/web/src/lib/voice/readAloudPlayer.ts | 52 +++++++++++++------ 7 files changed, 167 insertions(+), 26 deletions(-) diff --git a/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts b/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts index 23aad58a68..e0bfcde181 100644 --- a/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts +++ b/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts @@ -1,3 +1,12 @@ +// @vitest-environment node +// +// This tests a Node.js server route handler, not browser code — running it +// under jsdom (the project default) creates two competing AbortController/ +// AbortSignal globals (jsdom's polyfill vs. Node's native one), and jsdom's +// Request constructor rejects a signal from the "wrong" realm with +// "Expected signal to be an instance of AbortSignal". The route itself runs +// in a real Node/Edge runtime with a single Fetch API implementation, so +// `node` here is the more accurate environment, not a workaround. import { describe, it, expect, vi, beforeEach } from 'vitest'; // ── Mocks ──────────────────────────────────────────────────────────────────── @@ -146,4 +155,40 @@ describe('POST /api/voice/synthesize — metering', () => { expect(mockTrackUsage).not.toHaveBeenCalled(); expect(mockReleaseHold).toHaveBeenCalledWith('hold_1'); }); + + it('propagates the caller abort signal to the upstream OpenAI request, so a cancelled client request releases the hold without billing', async () => { + let capturedSignal: AbortSignal | undefined; + vi.stubGlobal('fetch', vi.fn().mockImplementation((_url: string, opts: RequestInit) => { + capturedSignal = opts.signal ?? undefined; + return new Promise((_resolve, reject) => { + const abort = () => reject(new DOMException('The operation was aborted', 'AbortError')); + // The route awaits auth/gating before ever reaching fetch(), so by + // then the signal may already be aborted — a listener alone would + // miss an abort event that already fired in the past. + if (opts.signal?.aborted) { + abort(); + return; + } + opts.signal?.addEventListener('abort', abort); + }); + })); + + const controller = new AbortController(); + const request = new Request('http://localhost/api/voice/synthesize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text: 'hello' }), + signal: controller.signal, + }); + + const resPromise = POST(request); + controller.abort(); + const res = await resPromise; + + expect(capturedSignal).toBeDefined(); + expect(capturedSignal?.aborted).toBe(true); + expect(res.status).toBe(500); + expect(mockTrackUsage).not.toHaveBeenCalled(); + expect(mockReleaseHold).toHaveBeenCalledWith('hold_1'); + }); }); diff --git a/apps/web/src/app/api/voice/synthesize/route.ts b/apps/web/src/app/api/voice/synthesize/route.ts index 6d852234fe..e508f1e9be 100644 --- a/apps/web/src/app/api/voice/synthesize/route.ts +++ b/apps/web/src/app/api/voice/synthesize/route.ts @@ -158,7 +158,10 @@ export async function POST(request: Request) { } holdId = gate.holdId; - // Call OpenAI TTS API + // Call OpenAI TTS API. Forwards the caller's abort signal so a client + // that cancels mid-request (e.g. Read Aloud's Stop button) also cancels + // the upstream request — otherwise it runs to completion and gets + // billed regardless of the client having already discarded it. const response = await fetch('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: { @@ -172,6 +175,7 @@ export async function POST(request: Request) { speed: clampedSpeed, response_format: 'mp3', }), + signal: request.signal, }); if (!response.ok) { diff --git a/apps/web/src/components/ui/floating-input/InputFooter.tsx b/apps/web/src/components/ui/floating-input/InputFooter.tsx index 7c5f05da5b..2570d8483b 100644 --- a/apps/web/src/components/ui/floating-input/InputFooter.tsx +++ b/apps/web/src/components/ui/floating-input/InputFooter.tsx @@ -181,7 +181,7 @@ export function InputFooter({ variant="ghost" size="sm" onClick={onReadAloudClick} - disabled={!isReadingAloud && (disabled || isVoiceProGated || !canReadAloud || isListening)} + disabled={!isReadingAloud && (disabled || isVoiceProGated || !canReadAloud)} className={cn( 'h-8 w-8 p-0 transition-all duration-200 hover:bg-transparent dark:hover:bg-transparent', isReadingAloud diff --git a/apps/web/src/hooks/useReadAloud.ts b/apps/web/src/hooks/useReadAloud.ts index a8884ae617..3c8d045f8d 100644 --- a/apps/web/src/hooks/useReadAloud.ts +++ b/apps/web/src/hooks/useReadAloud.ts @@ -3,6 +3,7 @@ import { useCallback } from 'react'; import type { UIMessage } from 'ai'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; import { getTextSinceLastUserTurn, hasTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; import { flushForTts } from '@/lib/voice/chunkForTts'; import { @@ -17,15 +18,17 @@ import { * playback singleton (`readAloudPlayer`), so starting or stopping from any * mounted chat surface acts on the one real audio source. * - * Unavailable while Voice Mode is enabled on ANY surface (not just the - * current one): a live call elsewhere plays through its own separate - * AudioContext and would overlap with this audio. `readAloudPlayer` itself - * also stops any in-progress read-aloud the moment Voice Mode turns on, so - * this is a pre-check for starting a new read, not the only guard. + * Unavailable while Voice Mode is enabled, or mic dictation is active, on + * ANY surface (not just the current one) — both are separate microphone + * captures elsewhere that would overlap with this audio. `readAloudPlayer` + * itself also stops any in-progress read-aloud the moment either turns on, + * so this is a pre-check for starting a new read, not the only guard. */ export function useReadAloud() { const isReadingAloud = useReadAloudPlayerStore((s) => s.isPlaying); const isVoiceModeEnabled = useVoiceModeStore((s) => s.isEnabled); + const isDictationActive = useDictationActivityStore((s) => s.activeCount > 0); + const blocked = isVoiceModeEnabled || isDictationActive; const toggleReadAloud = useCallback( (messages: readonly UIMessage[]) => { @@ -33,17 +36,17 @@ export function useReadAloud() { stopReadAloud(); return; } - if (isVoiceModeEnabled) return; + if (blocked) return; const text = getTextSinceLastUserTurn(messages); if (!text.trim()) return; startReadAloud(flushForTts(text)); }, - [isVoiceModeEnabled] + [blocked] ); const canReadAloud = useCallback( - (messages: readonly UIMessage[]) => !isVoiceModeEnabled && hasTextSinceLastUserTurn(messages), - [isVoiceModeEnabled] + (messages: readonly UIMessage[]) => !blocked && hasTextSinceLastUserTurn(messages), + [blocked] ); return { isReadingAloud, toggleReadAloud, canReadAloud }; diff --git a/apps/web/src/hooks/useSpeechRecognition.ts b/apps/web/src/hooks/useSpeechRecognition.ts index 85bd6e9aa3..d1a729e5f8 100644 --- a/apps/web/src/hooks/useSpeechRecognition.ts +++ b/apps/web/src/hooks/useSpeechRecognition.ts @@ -1,6 +1,28 @@ 'use client'; import { useState, useEffect, useRef, useCallback } from 'react'; +import { create } from 'zustand'; + +/** + * Cross-surface dictation activity. Each `useSpeechRecognition()` instance + * is otherwise local state — a mounted sidebar chat and main chat each own + * an independent SpeechRecognition object — so consumers that need to know + * "is dictation active ANYWHERE" (e.g. to avoid Read Aloud's TTS getting + * transcribed back into a draft) can't read that off any single instance. + * A count (not a boolean) so two simultaneously-listening surfaces don't + * have one's stop clear the other's still-active state. + */ +interface DictationActivityState { + activeCount: number; +} + +export const useDictationActivityStore = create(() => ({ + activeCount: 0, +})); + +export function isDictationActive(): boolean { + return useDictationActivityStore.getState().activeCount > 0; +} export interface UseSpeechRecognitionOptions { /** Callback when speech is transcribed */ @@ -45,6 +67,10 @@ export function useSpeechRecognition({ const recognitionRef = useRef(null); const errorTimerRef = useRef | null>(null); const onTranscriptRef = useRef(onTranscript); + // Guards against double-decrementing the shared activeCount — onerror is + // typically followed by onend for the same session, and this instance's + // unmount cleanup could otherwise also fire after one of those already did. + const isCountedActiveRef = useRef(false); // Keep callback ref updated onTranscriptRef.current = onTranscript; @@ -67,12 +93,25 @@ export function useSpeechRecognition({ recognition.interimResults = true; recognition.lang = lang; + const markActive = () => { + if (isCountedActiveRef.current) return; + isCountedActiveRef.current = true; + useDictationActivityStore.setState((s) => ({ activeCount: s.activeCount + 1 })); + }; + const markInactive = () => { + if (!isCountedActiveRef.current) return; + isCountedActiveRef.current = false; + useDictationActivityStore.setState((s) => ({ activeCount: Math.max(0, s.activeCount - 1) })); + }; + recognition.onstart = () => { setIsListening(true); + markActive(); }; recognition.onend = () => { setIsListening(false); + markInactive(); }; recognition.onresult = (event) => { @@ -93,6 +132,7 @@ export function useSpeechRecognition({ recognition.onerror = (event) => { console.error('Speech recognition error:', event.error); setIsListening(false); + markInactive(); const errorMessages: Record = { 'not-allowed': 'Microphone access denied. Check your browser permissions.', @@ -115,6 +155,9 @@ export function useSpeechRecognition({ return () => { recognition.stop(); + // Belt-and-suspenders in case onend never fires before unmount — + // guarded by isCountedActiveRef, so this is a no-op when it already did. + markInactive(); if (errorTimerRef.current) clearTimeout(errorTimerRef.current); }; }, [lang, continuous]); diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts index 4c8884fcd4..d70ba846b1 100644 --- a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -6,6 +6,7 @@ vi.mock('@/lib/auth/auth-fetch', () => ({ import { fetchWithAuth } from '@/lib/auth/auth-fetch'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; import { startReadAloud, stopReadAloud, @@ -52,6 +53,7 @@ describe('readAloudPlayer', () => { afterEach(() => { stopReadAloud(); useVoiceModeStore.getState().disable(); + useDictationActivityStore.setState({ activeCount: 0 }); vi.unstubAllGlobals(); vi.restoreAllMocks(); }); @@ -234,4 +236,26 @@ describe('readAloudPlayer', () => { stopReadAloud(); expect(capturedSignal?.aborted).toBe(true); }); + + it('stops playback the moment mic dictation becomes active anywhere, regardless of which ChatInput instance started it', () => { + // Dictation is per-ChatInput-instance local state (useSpeechRecognition), + // unlike Voice Mode's single global store — this simulates a DIFFERENT + // mounted surface's mic starting, via the shared activeCount it feeds. + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + useDictationActivityStore.setState({ activeCount: 1 }); + + expect(isReadAloudPlaying()).toBe(false); + }); + + it('reloads persisted voice settings on every start, not just once', () => { + const loadSettingsSpy = vi.spyOn(useVoiceModeStore.getState(), 'loadSettings'); + + startReadAloud(['first']); + stopReadAloud(); + startReadAloud(['second']); + + expect(loadSettingsSpy).toHaveBeenCalledTimes(2); + }); }); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts index 7fe2e24d1d..b7083291dd 100644 --- a/apps/web/src/lib/voice/readAloudPlayer.ts +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -3,6 +3,7 @@ import { create } from 'zustand'; import { fetchWithAuth } from '@/lib/auth/auth-fetch'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; /** * Module-singleton "read aloud" audio player, deliberately independent of @@ -25,6 +26,13 @@ import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; * `useVoiceModeStore`). The actual audio machinery (AudioContext, the * current source, the chunk queue, the race-guard generation counter) stays * in plain module variables outside React entirely. + * + * Mutually exclusive with two other microphone-capturing features, each + * with the same cross-surface problem: a live Voice Mode call + * (`useVoiceModeStore`) and basic mic dictation (`useDictationActivityStore` + * in `useSpeechRecognition.ts`, one per `ChatInput` instance). Either one's + * mic could otherwise pick up this module's own TTS audio and transcribe it + * back into a draft or into Voice Mode's own turn. See `ensureReadAloudReady`. */ interface ReadAloudPlayerState { @@ -132,7 +140,14 @@ async function playNext(runId: number): Promise { } export function startReadAloud(chunks: string[]): void { - ensureVoiceModeStopsReadAloud(); + ensureMutualExclusionSubscriptions(); + // synthesize() reads ttsVoice/ttsSpeed straight from useVoiceModeStore + // without ever mounting useVoiceMode() — which is what used to trigger + // this load on mount — so a user's persisted voice choice would otherwise + // be silently ignored the first time they use Read Aloud without ever + // having opened the full Voice Mode panel. Idempotent and cheap, so it's + // just re-run on every start rather than gated behind a one-time flag. + useVoiceModeStore.getState().loadSettings(); stopReadAloud(); if (chunks.length === 0) return; queue = [...chunks]; @@ -165,24 +180,31 @@ export function isReadAloudPlaying(): boolean { return useReadAloudPlayerStore.getState().isPlaying; } -// Enforced here (not per call-site) so the invariant holds no matter which UI -// entry point enables Voice Mode, present or future: a mic-capturing live -// call must never run concurrently with this module's own TTS audio, since -// each has its own separate AudioContext and would otherwise be picked up by -// Voice Mode's microphone as if it were user speech. -// // Registered lazily (on first startReadAloud(), not at module import time) -// so merely importing this module — e.g. a component under test that renders -// but never exercises Read Aloud — never touches useVoiceModeStore.subscribe. -// By the time any audio could actually be playing, this has always already -// run, since startReadAloud() is the only path that starts playback. -let voiceModeSubscriptionRegistered = false; -function ensureVoiceModeStopsReadAloud(): void { - if (voiceModeSubscriptionRegistered) return; - voiceModeSubscriptionRegistered = true; +// so merely importing this module — e.g. a component under test that +// renders but never exercises Read Aloud — never touches these other +// stores' `.subscribe`. By the time any audio could actually be playing, +// this has always already run, since startReadAloud() is the only path +// that starts playback. Guarded by a one-time flag (unlike loadSettings() +// above) because subscribing more than once would leak duplicate listeners. +// +// A live Voice Mode call or active mic dictation each capture the +// microphone — this module's own TTS audio must never play concurrently +// with either, or it risks being picked up as if it were user speech. +// Enforced here (not per call-site) so the invariant holds no matter which +// UI entry point triggers either, present or future. +let subscriptionsRegistered = false; +function ensureMutualExclusionSubscriptions(): void { + if (subscriptionsRegistered) return; + subscriptionsRegistered = true; useVoiceModeStore.subscribe((state, prevState) => { if (state.isEnabled && !prevState.isEnabled) { stopReadAloud(); } }); + useDictationActivityStore.subscribe((state, prevState) => { + if (state.activeCount > 0 && prevState.activeCount === 0) { + stopReadAloud(); + } + }); } From 836db6c0b476f65a8f87ff129dc65dd2c30dfba9 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 20 Jul 2026 00:48:55 -0500 Subject: [PATCH 10/13] fix(ci): drop unused isDictationActive export flagged by knip's dead-code gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useReadAloud.ts ended up using the reactive useDictationActivityStore((s) => s.activeCount > 0) selector directly instead of the standalone isDictationActive() helper, leaving it an unused export — caught immediately by the blocking knip:check gate on CI. Removed; useDictationActivityStore itself is still exported and used. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- apps/web/src/hooks/useSpeechRecognition.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/web/src/hooks/useSpeechRecognition.ts b/apps/web/src/hooks/useSpeechRecognition.ts index d1a729e5f8..7460ebcd19 100644 --- a/apps/web/src/hooks/useSpeechRecognition.ts +++ b/apps/web/src/hooks/useSpeechRecognition.ts @@ -20,10 +20,6 @@ export const useDictationActivityStore = create(() => ({ activeCount: 0, })); -export function isDictationActive(): boolean { - return useDictationActivityStore.getState().activeCount > 0; -} - export interface UseSpeechRecognitionOptions { /** Callback when speech is transcribed */ onTranscript: (text: string) => void; From f73c45ddc45aa7d0c72d1b9d820ac4e2c38a1805 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 20 Jul 2026 01:28:32 -0500 Subject: [PATCH 11/13] fix(voice): recheck live mic-capture state at start, surface synthesis failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more findings from Codex's review of 836db6c0b (PR #2173): 1. (P2) Recheck active microphone modes before starting playback. useReadAloud()'s toggleReadAloud can hold a stale `blocked` React closure, and readAloudPlayer's mutual-exclusion subscriptions only fire on a FUTURE inactive-to-active transition — neither catches "Voice Mode or dictation was ALREADY active by the time this specific startReadAloud() call runs" (e.g. rapid clicks across two mounted surfaces). startReadAloud() now independently re-checks live useVoiceModeStore/useDictationActivityStore state right before actually starting, closing the gap regardless of caller staleness. 2. (P2) Surface synthesis failures instead of silently skipping them. A systemic /api/voice/synthesize failure (out of credits, rate-limited, misconfigured) was converted to null and treated as one skippable chunk — every remaining chunk in a multi-chunk reply would then fail identically and silently, with the control just going quiet. Non-ok responses now surface the server's error message via toast.error() and stop the whole run (via the existing stopReadAloud()/generation-token mechanism) rather than retry-skipping a doomed request per chunk. Network/decode failures in the catch block remain skip-and-continue, since those genuinely can be per-chunk-transient. Verified via typecheck plus readAloudPlayer's suite (20 tests, +3 new: live Voice-Mode-already-on, live-dictation-already-active, and the systemic failure toast+stop behavior) and the full related suite (95 tests total), all passing. knip clean (5 baseline issues, no new ones). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../voice/__tests__/readAloudPlayer.test.ts | 42 +++++++++++++++++++ apps/web/src/lib/voice/readAloudPlayer.ts | 25 ++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts index d70ba846b1..f35c178281 100644 --- a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -4,6 +4,11 @@ vi.mock('@/lib/auth/auth-fetch', () => ({ fetchWithAuth: vi.fn(), })); +const { toastErrorMock } = vi.hoisted(() => ({ toastErrorMock: vi.fn() })); +vi.mock('sonner', () => ({ + toast: { error: toastErrorMock, success: vi.fn() }, +})); + import { fetchWithAuth } from '@/lib/auth/auth-fetch'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; @@ -43,6 +48,7 @@ class FakeAudioContext { describe('readAloudPlayer', () => { beforeEach(() => { createdSources.length = 0; + toastErrorMock.mockClear(); vi.stubGlobal('AudioContext', FakeAudioContext); vi.mocked(fetchWithAuth).mockResolvedValue({ ok: true, @@ -258,4 +264,40 @@ describe('readAloudPlayer', () => { expect(loadSettingsSpy).toHaveBeenCalledTimes(2); }); + + it('refuses to start when Voice Mode is already enabled at call time, even with no prior transition to observe', () => { + // Not a "stop while playing" case — Voice Mode was ALREADY on before + // startReadAloud() ever ran, so the subscription (which only fires on a + // future inactive-to-active transition) never gets a chance to fire. + useVoiceModeStore.getState().enable('ai-page'); + + startReadAloud(['hello there']); + + expect(isReadAloudPlaying()).toBe(false); + expect(createdSources).toHaveLength(0); + }); + + it('refuses to start when dictation is already active at call time, even with no prior transition to observe', () => { + useDictationActivityStore.setState({ activeCount: 1 }); + + startReadAloud(['hello there']); + + expect(isReadAloudPlaying()).toBe(false); + expect(createdSources).toHaveLength(0); + }); + + it('surfaces a systemic synthesis failure and stops the run instead of retry-skipping every remaining chunk', async () => { + vi.mocked(fetchWithAuth).mockResolvedValue({ + ok: false, + json: () => Promise.resolve({ message: 'Voice synthesis is not configured on this deployment.' }), + } as Response); + + startReadAloud(['first chunk', 'second chunk']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(createdSources).toHaveLength(0); + expect(toastErrorMock).toHaveBeenCalledWith('Voice synthesis is not configured on this deployment.'); + // Stopped after the first failure — never even attempted the second chunk. + expect(fetchWithAuth).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts index b7083291dd..74f1232b67 100644 --- a/apps/web/src/lib/voice/readAloudPlayer.ts +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -1,6 +1,7 @@ 'use client'; import { create } from 'zustand'; +import { toast } from 'sonner'; import { fetchWithAuth } from '@/lib/auth/auth-fetch'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; @@ -85,7 +86,21 @@ async function synthesize(text: string): Promise { body: JSON.stringify({ text, voice: ttsVoice, speed: ttsSpeed }), signal: controller.signal, }); - if (!response.ok) return null; + if (!response.ok) { + // A systemic failure (out of credits, rate-limited, misconfigured) — + // every remaining chunk would fail identically, so surface it and stop + // the whole run instead of silently skip-retrying it chunk by chunk. + // playNext()'s generation check (bumped by the stopReadAloud() call + // below) is what actually prevents the skip-retry, not this branch. + const errorData = await response.json().catch(() => ({}) as Record); + const message = + (typeof errorData.message === 'string' && errorData.message) || + (typeof errorData.error === 'string' && errorData.error) || + 'Could not read this reply aloud.'; + toast.error(message); + stopReadAloud(); + return null; + } const audioData = await response.arrayBuffer(); if (ctx.state === 'suspended') { await ctx.resume(); @@ -150,6 +165,14 @@ export function startReadAloud(chunks: string[]): void { useVoiceModeStore.getState().loadSettings(); stopReadAloud(); if (chunks.length === 0) return; + // Re-validate live state here, not just via the subscriptions above: a + // caller's "may I start?" check (useReadAloud's `blocked`) can be a stale + // React closure, and the subscriptions only fire on a FUTURE + // inactive-to-active transition — neither catches "already active by the + // time this specific call runs" (e.g. rapid clicks across two surfaces). + if (useVoiceModeStore.getState().isEnabled || useDictationActivityStore.getState().activeCount > 0) { + return; + } queue = [...chunks]; setPlaying(true); void playNext(generation); From c5c343e94377f98c1ebb43e855607948b1de4d00 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 20 Jul 2026 02:02:18 -0500 Subject: [PATCH 12/13] fix(voice): surface catch-block synthesis failures, fix stale-run stop bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's review of f73c45ddc (PR #2173) found the catch block still silently dropped genuine failures — the earlier fix only handled non-ok HTTP responses, not exceptions from fetchWithAuth/arrayBuffer/resume/ decodeAudioData. Any of those throwing (network failure, a corrupted or undecodable response, AudioContext.resume() failing) got converted to null and treated as a skippable chunk, silently truncating or fully dropping a reply with zero explanation. Distinguished an intentional Stop-triggered AbortError (expected, silent) from any other exception (now gets the same surface-and-stop treatment as a non-ok response). Threaded the run's generation-token ID into synthesize() so both failure paths only toast/stop for a run that's still current — this caught a real bug while writing the test for it: the previous version called stopReadAloud() unconditionally on any non-abort failure, so a STALE, already-superseded run (from a stop-then-restart) independently failing for its own unrelated reason would incorrectly tear down a newer, legitimately-playing run. Both the toast and the stop are now gated behind the same "is this run still current" check, making a stale run's failure a full no-op rather than a spurious toast or interruption. Removed a now-outdated test whose entire premise (skip a failed HTTP response and continue to the next chunk) was the exact behavior the "surface systemic failures" fix from the previous commit intentionally replaced. Added tests for: catching a genuine exception (toast + stop), an intentional abort producing no toast, and — the one that caught the bug — a stale run's own independent failure producing no toast AND not disturbing a newer run already in progress. Verified via typecheck plus readAloudPlayer's suite (22 tests) and the full related suite (97 tests total), all passing. knip clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../voice/__tests__/readAloudPlayer.test.ts | 77 ++++++++++++++----- apps/web/src/lib/voice/readAloudPlayer.ts | 33 ++++++-- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts index f35c178281..4dcb077bb1 100644 --- a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -150,24 +150,6 @@ describe('readAloudPlayer', () => { await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); }); - it('skips a chunk that fails to synthesize and continues with the rest', async () => { - vi.mocked(fetchWithAuth) - .mockResolvedValueOnce({ ok: false } as Response) - .mockResolvedValue({ - ok: true, - arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), - } as Response); - - startReadAloud(['broken chunk', 'good chunk']); - // The first chunk's failed synthesis is skipped without ever creating a - // source; only the second, successful chunk should end up playing. - await vi.waitFor(() => expect(createdSources).toHaveLength(1)); - expect(isReadAloudPlaying()).toBe(true); - - createdSources[0].onended?.(); - await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); - }); - it('discards a chunk that finishes synthesizing after stop was already called', async () => { startReadAloud(['only chunk']); stopReadAloud(); @@ -300,4 +282,63 @@ describe('readAloudPlayer', () => { // Stopped after the first failure — never even attempted the second chunk. expect(fetchWithAuth).toHaveBeenCalledTimes(1); }); + + it('does not surface an error toast when synthesis is cancelled by an intentional stop', async () => { + // Mirrors a real fetch()'s behavior on abort: rejects with an AbortError, + // either immediately (if already aborted) or once the signal fires. + vi.mocked(fetchWithAuth).mockImplementation((_url, options) => { + const signal = (options as RequestInit | undefined)?.signal; + return new Promise((_resolve, reject) => { + const onAbort = () => reject(new DOMException('The operation was aborted', 'AbortError')); + if (signal?.aborted) { + onAbort(); + return; + } + signal?.addEventListener('abort', onAbort); + }); + }); + + startReadAloud(['only chunk']); + stopReadAloud(); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(toastErrorMock).not.toHaveBeenCalled(); + }); + + it('surfaces an error toast and stops the run when synthesis throws for a reason other than an intentional stop', async () => { + vi.mocked(fetchWithAuth).mockRejectedValue(new Error('network blip')); + + startReadAloud(['first chunk', 'second chunk']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(toastErrorMock).toHaveBeenCalledWith('Could not read this reply aloud. Please try again.'); + // Stopped after the first failure — never even attempted the second chunk. + expect(fetchWithAuth).toHaveBeenCalledTimes(1); + }); + + it('does not surface a toast for a stale, already-superseded run that fails on its own after a restart', async () => { + let rejectFirst: ((err: Error) => void) | undefined; + const firstResponse = new Promise((_resolve, reject) => { + rejectFirst = reject; + }); + vi.mocked(fetchWithAuth) + .mockReturnValueOnce(firstResponse) + .mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + + startReadAloud(['stale chunk']); // run A: fetch pending + stopReadAloud(); // stopped before run A's fetch ever resolved + startReadAloud(['fresh chunk']); // run B: a fresh, successful fetch + + // Run A's fetch fails for its own, unrelated reason — AFTER run B has + // already begun. The user already moved on; this shouldn't surface a + // confusing error about a read they're no longer waiting on. + rejectFirst?.(new Error('unrelated network blip')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(toastErrorMock).not.toHaveBeenCalled(); + expect(isReadAloudPlaying()).toBe(true); + }); }); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts index 74f1232b67..51f383426a 100644 --- a/apps/web/src/lib/voice/readAloudPlayer.ts +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -72,13 +72,23 @@ function getAudioContext(): AudioContext { return audioContext; } -async function synthesize(text: string): Promise { +async function synthesize(text: string, runId: number): Promise { const { ttsVoice, ttsSpeed } = useVoiceModeStore.getState(); // Created before the network await so the browser still credits this // AudioContext to the user gesture that triggered playback. const ctx = getAudioContext(); const controller = new AbortController(); activeAbortController = controller; + // A stale, already-superseded run (from a stop-then-restart) failing for + // its own unrelated reason must be a full no-op: no toast, and — crucially + // — no stopReadAloud() either, since that would tear down a newer run that + // has nothing to do with this one's failure. Only the run that is still + // current gets to surface an error and stop the queue. + const handleFailure = (message: string) => { + if (runId !== generation) return; + toast.error(message); + stopReadAloud(); + }; try { const response = await fetchWithAuth('/api/voice/synthesize', { method: 'POST', @@ -90,15 +100,15 @@ async function synthesize(text: string): Promise { // A systemic failure (out of credits, rate-limited, misconfigured) — // every remaining chunk would fail identically, so surface it and stop // the whole run instead of silently skip-retrying it chunk by chunk. - // playNext()'s generation check (bumped by the stopReadAloud() call - // below) is what actually prevents the skip-retry, not this branch. + // playNext()'s generation check (bumped by handleFailure's + // stopReadAloud() call) is what actually prevents the skip-retry, not + // this branch. const errorData = await response.json().catch(() => ({}) as Record); const message = (typeof errorData.message === 'string' && errorData.message) || (typeof errorData.error === 'string' && errorData.error) || 'Could not read this reply aloud.'; - toast.error(message); - stopReadAloud(); + handleFailure(message); return null; } const audioData = await response.arrayBuffer(); @@ -106,7 +116,16 @@ async function synthesize(text: string): Promise { await ctx.resume(); } return await ctx.decodeAudioData(audioData); - } catch { + } catch (err) { + // An AbortError here means WE intentionally cancelled this request (via + // stopReadAloud()'s controller.abort()) — expected, nothing to surface. + // Any other exception (network failure, a corrupted/undecodable + // response, AudioContext.resume() failing) is unexpected and would + // otherwise silently truncate or drop the reply with zero explanation, + // so it gets the same surface-and-stop treatment as a non-ok response. + if (!(err instanceof DOMException && err.name === 'AbortError')) { + handleFailure('Could not read this reply aloud. Please try again.'); + } return null; } finally { // Only clear if still ours — a stop-then-restart race can leave a newer @@ -127,7 +146,7 @@ async function playNext(runId: number): Promise { return; } - const buffer = await synthesize(text); + const buffer = await synthesize(text, runId); // A stop, or a stop-then-restart, happened while this chunk was // synthesizing — discard the result rather than letting a stale run play. if (runId !== generation) return; From 7b5edc2e3d662441f907ea8d3309fe620200329a Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 20 Jul 2026 02:43:51 -0500 Subject: [PATCH 13/13] fix(voice): stop on last-surface unmount, handle AudioContext creation failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more findings from Codex's review of c5c343e94 (PR #2173): 1. (P2) Stop playback when the final read-aloud surface unmounts. The player is deliberately independent of any single component's lifecycle (that's the point of the module singleton), but if EVERY mounted chat surface unmounts — e.g. the user navigates to a route with no chat UI at all — there's no longer a Stop control reachable anywhere, while synthesis keeps running (and billing) and audio keeps playing. Added a ref count of currently-mounted useReadAloud() consumers; stopReadAloud() only fires when the count drops to zero, so closing just the sidebar while the main chat stays open correctly leaves a main-chat-initiated read running. 2. (P2) Handle AudioContext creation failures. `new AudioContext()` ran before synthesize()'s try block, so a creation failure (quota exhausted, Web Audio unsupported) rejected the fire-and-forget playNext() call unhandled, leaving isPlaying stuck true forever with no audio and no explanation. Moved the call inside the try block so it gets the same surface-and-stop handling as any other failure. New tests: apps/web/src/hooks/__tests__/useReadAloud.test.ts covers the ref-counted unmount behavior (single consumer, and one-of-several vs. last-of-several). The AudioContext-failure test lives in its own file (readAloudPlayer.audioContextFailure.test.ts) rather than the main readAloudPlayer.test.ts — the module's AudioContext is a lazy singleton that, once created by an earlier test, is never recreated, so a throwing stub registered in a later test in the same file would never actually be invoked; a fresh file gives Vitest a fresh module registry instead. Verified via typecheck plus the full related suite (100 tests total, all passing). knip clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NHGBY8FgjREJXDGaPRAyig --- .../src/hooks/__tests__/useReadAloud.test.ts | 66 +++++++++++++++++++ apps/web/src/hooks/useReadAloud.ts | 24 ++++++- ...eadAloudPlayer.audioContextFailure.test.ts | 41 ++++++++++++ apps/web/src/lib/voice/readAloudPlayer.ts | 11 +++- 4 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 apps/web/src/hooks/__tests__/useReadAloud.test.ts create mode 100644 apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts diff --git a/apps/web/src/hooks/__tests__/useReadAloud.test.ts b/apps/web/src/hooks/__tests__/useReadAloud.test.ts new file mode 100644 index 0000000000..4423e271ee --- /dev/null +++ b/apps/web/src/hooks/__tests__/useReadAloud.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { renderHook } from '@testing-library/react'; + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +import { useReadAloud } from '../useReadAloud'; +import { startReadAloud, stopReadAloud, isReadAloudPlaying } from '@/lib/voice/readAloudPlayer'; + +class FakeAudioBufferSourceNode { + buffer: unknown = null; + onended: (() => void) | null = null; + connect(): void {} + start(): void {} + stop(): void {} +} + +class FakeAudioContext { + state = 'running'; + destination = {}; + createBufferSource(): FakeAudioBufferSourceNode { + return new FakeAudioBufferSourceNode(); + } + decodeAudioData(): Promise { + return Promise.resolve({}); + } + resume(): Promise { + return Promise.resolve(); + } +} + +describe('useReadAloud — unmount lifecycle', () => { + beforeEach(() => { + vi.stubGlobal('AudioContext', FakeAudioContext); + }); + + afterEach(() => { + stopReadAloud(); + vi.unstubAllGlobals(); + }); + + it('stops playback when the only mounted consumer unmounts', () => { + const { unmount } = renderHook(() => useReadAloud()); + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + unmount(); + + expect(isReadAloudPlaying()).toBe(false); + }); + + it('does not stop playback when one of several mounted consumers unmounts, only when the last one does', () => { + // Simulates the sidebar chat and main chat both being mounted at once. + const first = renderHook(() => useReadAloud()); + const second = renderHook(() => useReadAloud()); + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + first.unmount(); + expect(isReadAloudPlaying()).toBe(true); + + second.unmount(); + expect(isReadAloudPlaying()).toBe(false); + }); +}); diff --git a/apps/web/src/hooks/useReadAloud.ts b/apps/web/src/hooks/useReadAloud.ts index 3c8d045f8d..bb0099734e 100644 --- a/apps/web/src/hooks/useReadAloud.ts +++ b/apps/web/src/hooks/useReadAloud.ts @@ -1,6 +1,6 @@ 'use client'; -import { useCallback } from 'react'; +import { useCallback, useEffect } from 'react'; import type { UIMessage } from 'ai'; import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; @@ -24,7 +24,29 @@ import { * itself also stops any in-progress read-aloud the moment either turns on, * so this is a pre-check for starting a new read, not the only guard. */ + +// The player is deliberately independent of any one component's lifecycle +// (that's the whole point of the module singleton — see readAloudPlayer.ts), +// but if EVERY mounted chat surface unmounts (e.g. the user navigates to a +// route with no chat UI at all) there is no longer a Stop control reachable +// anywhere, while synthesis keeps running and billing and audio keeps +// playing. Ref-counts how many useReadAloud() consumers are currently +// mounted; stops playback only when the count drops to zero, not on every +// individual unmount (closing just the sidebar while the main chat is still +// open must not interrupt a main-chat-initiated read). +let mountedConsumers = 0; + export function useReadAloud() { + useEffect(() => { + mountedConsumers += 1; + return () => { + mountedConsumers -= 1; + if (mountedConsumers === 0) { + stopReadAloud(); + } + }; + }, []); + const isReadingAloud = useReadAloudPlayerStore((s) => s.isPlaying); const isVoiceModeEnabled = useVoiceModeStore((s) => s.isEnabled); const isDictationActive = useDictationActivityStore((s) => s.activeCount > 0); diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts new file mode 100644 index 0000000000..8e1859ed45 --- /dev/null +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Kept in its own file, separate from readAloudPlayer.test.ts: the module's +// AudioContext is a lazy singleton, created once and never recreated — a +// prior test in the same file that successfully starts playback leaves it +// cached, so a throwing AudioContext stub registered in a LATER test would +// never actually get invoked. A fresh file gives Vitest a fresh module +// registry, guaranteeing this is the first (and only) call to +// getAudioContext() in this environment. + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +const { toastErrorMock } = vi.hoisted(() => ({ toastErrorMock: vi.fn() })); +vi.mock('sonner', () => ({ + toast: { error: toastErrorMock, success: vi.fn() }, +})); + +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { startReadAloud, isReadAloudPlaying } from '../readAloudPlayer'; + +describe('readAloudPlayer — AudioContext construction failure', () => { + it('surfaces an error and stops the run when creating the AudioContext itself fails', async () => { + vi.stubGlobal( + 'AudioContext', + class { + constructor() { + throw new Error('Web Audio unavailable'); + } + } + ); + + startReadAloud(['hello there']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(toastErrorMock).toHaveBeenCalledWith('Could not read this reply aloud. Please try again.'); + // Never even reached the network call — failed before fetchWithAuth. + expect(fetchWithAuth).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts index 51f383426a..401c6f8268 100644 --- a/apps/web/src/lib/voice/readAloudPlayer.ts +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -74,9 +74,6 @@ function getAudioContext(): AudioContext { async function synthesize(text: string, runId: number): Promise { const { ttsVoice, ttsSpeed } = useVoiceModeStore.getState(); - // Created before the network await so the browser still credits this - // AudioContext to the user gesture that triggered playback. - const ctx = getAudioContext(); const controller = new AbortController(); activeAbortController = controller; // A stale, already-superseded run (from a stop-then-restart) failing for @@ -90,6 +87,14 @@ async function synthesize(text: string, runId: number): Promise