diff --git a/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts b/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts index 23aad58a68..e0bfcde181 100644 --- a/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts +++ b/apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts @@ -1,3 +1,12 @@ +// @vitest-environment node +// +// This tests a Node.js server route handler, not browser code — running it +// under jsdom (the project default) creates two competing AbortController/ +// AbortSignal globals (jsdom's polyfill vs. Node's native one), and jsdom's +// Request constructor rejects a signal from the "wrong" realm with +// "Expected signal to be an instance of AbortSignal". The route itself runs +// in a real Node/Edge runtime with a single Fetch API implementation, so +// `node` here is the more accurate environment, not a workaround. import { describe, it, expect, vi, beforeEach } from 'vitest'; // ── Mocks ──────────────────────────────────────────────────────────────────── @@ -146,4 +155,40 @@ describe('POST /api/voice/synthesize — metering', () => { expect(mockTrackUsage).not.toHaveBeenCalled(); expect(mockReleaseHold).toHaveBeenCalledWith('hold_1'); }); + + it('propagates the caller abort signal to the upstream OpenAI request, so a cancelled client request releases the hold without billing', async () => { + let capturedSignal: AbortSignal | undefined; + vi.stubGlobal('fetch', vi.fn().mockImplementation((_url: string, opts: RequestInit) => { + capturedSignal = opts.signal ?? undefined; + return new Promise((_resolve, reject) => { + const abort = () => reject(new DOMException('The operation was aborted', 'AbortError')); + // The route awaits auth/gating before ever reaching fetch(), so by + // then the signal may already be aborted — a listener alone would + // miss an abort event that already fired in the past. + if (opts.signal?.aborted) { + abort(); + return; + } + opts.signal?.addEventListener('abort', abort); + }); + })); + + const controller = new AbortController(); + const request = new Request('http://localhost/api/voice/synthesize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text: 'hello' }), + signal: controller.signal, + }); + + const resPromise = POST(request); + controller.abort(); + const res = await resPromise; + + expect(capturedSignal).toBeDefined(); + expect(capturedSignal?.aborted).toBe(true); + expect(res.status).toBe(500); + expect(mockTrackUsage).not.toHaveBeenCalled(); + expect(mockReleaseHold).toHaveBeenCalledWith('hold_1'); + }); }); diff --git a/apps/web/src/app/api/voice/synthesize/route.ts b/apps/web/src/app/api/voice/synthesize/route.ts index 6d852234fe..e508f1e9be 100644 --- a/apps/web/src/app/api/voice/synthesize/route.ts +++ b/apps/web/src/app/api/voice/synthesize/route.ts @@ -158,7 +158,10 @@ export async function POST(request: Request) { } holdId = gate.holdId; - // Call OpenAI TTS API + // Call OpenAI TTS API. Forwards the caller's abort signal so a client + // that cancels mid-request (e.g. Read Aloud's Stop button) also cancels + // the upstream request — otherwise it runs to completion and gets + // billed regardless of the client having already discarded it. const response = await fetch('https://api.openai.com/v1/audio/speech', { method: 'POST', headers: { @@ -172,6 +175,7 @@ export async function POST(request: Request) { speed: clampedSpeed, response_format: 'mp3', }), + signal: request.signal, }); if (!response.ok) { diff --git a/apps/web/src/components/ai/chat/input/ChatInput.tsx b/apps/web/src/components/ai/chat/input/ChatInput.tsx index fc2df373a7..053714183a 100644 --- a/apps/web/src/components/ai/chat/input/ChatInput.tsx +++ b/apps/web/src/components/ai/chat/input/ChatInput.tsx @@ -12,6 +12,7 @@ import { useAssistantSettingsStore } from '@/stores/useAssistantSettingsStore'; import { isImageGenerationAllowed } from '@/lib/ai/core/image-gen-access'; import { useSpeechRecognition } from '@/hooks/useSpeechRecognition'; import { useMobileKeyboard } from '@/hooks/useMobileKeyboard'; +import { stopReadAloud } from '@/lib/voice/readAloudPlayer'; import type { ImageAttachment } from '@/lib/ai/shared/hooks/useImageAttachments'; export interface ChatInputProps { @@ -65,6 +66,12 @@ export interface ChatInputProps { onVoiceModeClick?: () => void; /** Whether voice mode is currently active */ isVoiceModeActive?: boolean; + /** Callback when the read-aloud button is clicked */ + onReadAloudClick?: () => void; + /** Whether read-aloud is currently playing */ + isReadingAloud?: boolean; + /** Whether there is anything eligible to read aloud right now */ + canReadAloud?: boolean; /** Image attachments for vision support */ attachments?: ImageAttachment[]; /** Handler to add image files */ @@ -127,6 +134,9 @@ export const ChatInput = forwardRef( onProviderModelChange, onVoiceModeClick, isVoiceModeActive = false, + onReadAloudClick, + isReadingAloud = false, + canReadAloud = false, attachments, onAddFiles, onRemoveFile, @@ -174,6 +184,15 @@ export const ChatInput = forwardRef( }, }); + // Starting mic dictation stops Read Aloud first — otherwise the mic can + // transcribe the TTS audio it hears right back into the draft. + const handleMicClick = useCallback(() => { + if (!isListening) { + stopReadAloud(); + } + toggleListening(); + }, [isListening, toggleListening]); + // Mobile keyboard management const keyboard = useMobileKeyboard(); const prevStreamingRef = useRef(isStreaming); @@ -298,13 +317,16 @@ export const ChatInput = forwardRef( isMcpServerEnabled={isMcpServerEnabled} onMcpServerToggle={onMcpServerToggle} showMcp={showMcp} - onMicClick={toggleListening} + onMicClick={handleMicClick} isListening={isListening} isMicSupported={isSupported} micError={speechError} onClearMicError={clearSpeechError} onVoiceModeClick={onVoiceModeClick} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={onReadAloudClick} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} selectedProvider={currentProvider} selectedModel={currentModel} onProviderModelChange={handleProviderModelChange} diff --git a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx index ec70b63551..5e680330ef 100644 --- a/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/ai-page/AiChatView.tsx @@ -48,6 +48,7 @@ import { shouldRefreshAfterUndo } from '@/lib/ai/streams/shouldRefreshAfterUndo' import { shouldPrependConversation } from '@/lib/ai/streams/shouldPrependConversation'; import { shouldReloadOnComountComplete } from '@/lib/ai/streams/shouldReloadOnComountComplete'; import { getBrowserSessionId } from '@/lib/ai/core/browser-session-id'; +import { useReadAloud } from '@/hooks/useReadAloud'; // Shared hooks and components import { @@ -581,6 +582,15 @@ const AiChatView: React.FC = ({ page }) => { const renderedMessages = useRenderedMessages(page.id, currentConversationId); const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn, via a shared playback singleton (see readAloudPlayer). + const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud(); + const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]); + const handleReadAloudClick = useCallback( + () => toggleReadAloud(plainMessages), + [toggleReadAloud, plainMessages] + ); + // "Load older" (epic leaf 6.6, scroll-to-top): AiChatView's route IS the agent-conversation // route (page.id is the agentId), so the shared agent-mode loader applies directly. const { isLoadingOlder } = useConversationOlderPageState(currentConversationId); @@ -1238,7 +1248,9 @@ const AiChatView: React.FC = ({ page }) => { prepareSend, ]); - // Voice mode toggle handler + // Voice mode toggle handler. Enabling Voice Mode also stops any + // in-progress read-aloud playback — enforced inside readAloudPlayer itself + // (subscribed to the voice-mode store), not here. const handleVoiceModeToggle = useCallback(() => { if (isVoiceModeActive) { disableVoiceMode(); @@ -1518,6 +1530,9 @@ const AiChatView: React.FC = ({ page }) => { }} onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={handleReadAloudClick} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx index f060306994..6882f81f45 100644 --- a/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx +++ b/apps/web/src/components/layout/middle-content/page-views/dashboard/GlobalAssistantView.tsx @@ -108,6 +108,7 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText'; import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline'; import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; +import { useReadAloud } from '@/hooks/useReadAloud'; import { createId } from '@paralleldrive/cuid2'; const VOICE_OWNER: VoiceModeOwner = 'global-assistant'; @@ -436,6 +437,15 @@ const GlobalAssistantView: React.FC = () => { // above) never renders post-cutover — it stays the transport/controller only. const renderedMessages = useRenderedMessages(streamChannelId ?? '', currentConversationId); const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]); + + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn, via a shared playback singleton (see readAloudPlayer). + const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud(); + const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]); + const handleReadAloudClick = useCallback( + () => toggleReadAloud(plainMessages), + [toggleReadAloud, plainMessages] + ); // Loading/error UI reads the cache entry's state (replaces the context's // isMessagesLoading and the dashboard store's isConversationMessagesLoading). const messagesLoadState = useConversationLoadState(currentConversationId); @@ -899,7 +909,9 @@ const GlobalAssistantView: React.FC = () => { setLastAIResponse((current) => (current?.id === next.id ? current : next)); }, [renderedMessages, isVoiceModeActive]); - // Voice mode toggle handler + // Voice mode toggle handler. Enabling Voice Mode also stops any + // in-progress read-aloud playback — enforced inside readAloudPlayer itself + // (subscribed to the voice-mode store), not here. const handleVoiceModeToggle = useCallback(() => { if (isVoiceModeActive) { disableVoiceMode(); @@ -1095,6 +1107,9 @@ const GlobalAssistantView: React.FC = () => { popupPlacement={props.inputPosition === 'centered' ? 'bottom' : 'top'} onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={handleReadAloudClick} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx index f1da34ce20..0055b34fb1 100644 --- a/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx +++ b/apps/web/src/components/layout/right-sidebar/ai-assistant/SidebarChatTab.tsx @@ -44,6 +44,7 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText'; import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline'; import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage'; +import { useReadAloud } from '@/hooks/useReadAloud'; import { createId } from '@paralleldrive/cuid2'; import { useStopStream } from '@/hooks/useStopStream'; import { useOwnStreamMirror } from '@/hooks/useOwnStreamMirror'; @@ -542,6 +543,15 @@ const SidebarChatTab: React.FC = () => { const disableVoiceMode = useVoiceModeStore((s) => s.disable); const isVoiceModeActive = isVoiceModeEnabled && voiceOwner === VOICE_OWNER; + // Read Aloud: on-demand TTS for everything the assistant said since the + // user's last turn, via a shared playback singleton (see readAloudPlayer). + const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud(); + const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]); + const handleReadAloudClick = useCallback( + () => toggleReadAloud(plainMessages), + [toggleReadAloud, plainMessages] + ); + // Display preferences const { preferences: displayPreferences } = useDisplayPreferences(); @@ -864,7 +874,9 @@ const SidebarChatTab: React.FC = () => { ), }); - // Voice mode toggle handler + // Voice mode toggle handler. Enabling Voice Mode also stops any + // in-progress read-aloud playback — enforced inside readAloudPlayer itself + // (subscribed to the voice-mode store), not here. const handleVoiceModeToggle = useCallback(() => { if (isVoiceModeActive) { disableVoiceMode(); @@ -1115,6 +1127,9 @@ const SidebarChatTab: React.FC = () => { variant="sidebar" onVoiceModeClick={handleVoiceModeToggle} isVoiceModeActive={isVoiceModeActive} + onReadAloudClick={handleReadAloudClick} + isReadingAloud={isReadingAloud} + canReadAloud={canReadAloud} attachments={attachments} onAddFiles={addFiles} onRemoveFile={removeFile} diff --git a/apps/web/src/components/ui/floating-input/InputFooter.tsx b/apps/web/src/components/ui/floating-input/InputFooter.tsx index 3fb5b856c2..2570d8483b 100644 --- a/apps/web/src/components/ui/floating-input/InputFooter.tsx +++ b/apps/web/src/components/ui/floating-input/InputFooter.tsx @@ -7,7 +7,7 @@ import { TooltipTrigger, TooltipContent, } from '@/components/ui/tooltip'; -import { Mic, AudioLines, MicOff } from 'lucide-react'; +import { Mic, AudioLines, MicOff, Volume2 } from 'lucide-react'; import { cn } from '@/lib/utils'; import { ProviderModelSelector } from '@/components/ai/chat/input/ProviderModelSelector'; import { ToolsPopover } from './ToolsPopover'; @@ -61,6 +61,12 @@ export interface InputFooterProps { onVoiceModeClick?: () => void; /** Whether voice mode is currently active */ isVoiceModeActive?: boolean; + /** Callback when the read-aloud button is clicked */ + onReadAloudClick?: () => void; + /** Whether read-aloud is currently playing */ + isReadingAloud?: boolean; + /** Whether there is anything eligible to read aloud right now */ + canReadAloud?: boolean; /** Error message from microphone/speech recognition */ micError?: string | null; /** Callback to clear the mic error */ @@ -110,6 +116,9 @@ export function InputFooter({ isMicSupported = true, onVoiceModeClick, isVoiceModeActive = false, + onReadAloudClick, + isReadingAloud = false, + canReadAloud = false, micError, onClearMicError, selectedProvider, @@ -165,6 +174,44 @@ export function InputFooter({ /> )} + {/* Read Aloud button (on-demand TTS for the assistant's last turn) */} + + + + + + {isVoiceProGated + ? 'Read aloud requires a Pro plan' + : isReadingAloud + ? 'Stop reading aloud' + : isListening + ? 'Read aloud unavailable while dictating' + : 'Read aloud'} + + + {/* Voice Mode button (hands-free STT/TTS) */} diff --git a/apps/web/src/hooks/__tests__/useReadAloud.test.ts b/apps/web/src/hooks/__tests__/useReadAloud.test.ts new file mode 100644 index 0000000000..4423e271ee --- /dev/null +++ b/apps/web/src/hooks/__tests__/useReadAloud.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { renderHook } from '@testing-library/react'; + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +import { useReadAloud } from '../useReadAloud'; +import { startReadAloud, stopReadAloud, isReadAloudPlaying } from '@/lib/voice/readAloudPlayer'; + +class FakeAudioBufferSourceNode { + buffer: unknown = null; + onended: (() => void) | null = null; + connect(): void {} + start(): void {} + stop(): void {} +} + +class FakeAudioContext { + state = 'running'; + destination = {}; + createBufferSource(): FakeAudioBufferSourceNode { + return new FakeAudioBufferSourceNode(); + } + decodeAudioData(): Promise { + return Promise.resolve({}); + } + resume(): Promise { + return Promise.resolve(); + } +} + +describe('useReadAloud — unmount lifecycle', () => { + beforeEach(() => { + vi.stubGlobal('AudioContext', FakeAudioContext); + }); + + afterEach(() => { + stopReadAloud(); + vi.unstubAllGlobals(); + }); + + it('stops playback when the only mounted consumer unmounts', () => { + const { unmount } = renderHook(() => useReadAloud()); + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + unmount(); + + expect(isReadAloudPlaying()).toBe(false); + }); + + it('does not stop playback when one of several mounted consumers unmounts, only when the last one does', () => { + // Simulates the sidebar chat and main chat both being mounted at once. + const first = renderHook(() => useReadAloud()); + const second = renderHook(() => useReadAloud()); + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + first.unmount(); + expect(isReadAloudPlaying()).toBe(true); + + second.unmount(); + expect(isReadAloudPlaying()).toBe(false); + }); +}); diff --git a/apps/web/src/hooks/useReadAloud.ts b/apps/web/src/hooks/useReadAloud.ts new file mode 100644 index 0000000000..bb0099734e --- /dev/null +++ b/apps/web/src/hooks/useReadAloud.ts @@ -0,0 +1,75 @@ +'use client'; + +import { useCallback, useEffect } from 'react'; +import type { UIMessage } from 'ai'; +import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; +import { getTextSinceLastUserTurn, hasTextSinceLastUserTurn } from '@/lib/ai/streams/getTextSinceLastUserTurn'; +import { flushForTts } from '@/lib/voice/chunkForTts'; +import { + startReadAloud, + stopReadAloud, + useReadAloudPlayerStore, +} from '@/lib/voice/readAloudPlayer'; + +/** + * On-demand TTS for "read the assistant's last turn aloud" — distinct from + * full hands-free Voice Mode. Every call site shares the same module-level + * playback singleton (`readAloudPlayer`), so starting or stopping from any + * mounted chat surface acts on the one real audio source. + * + * Unavailable while Voice Mode is enabled, or mic dictation is active, on + * ANY surface (not just the current one) — both are separate microphone + * captures elsewhere that would overlap with this audio. `readAloudPlayer` + * itself also stops any in-progress read-aloud the moment either turns on, + * so this is a pre-check for starting a new read, not the only guard. + */ + +// The player is deliberately independent of any one component's lifecycle +// (that's the whole point of the module singleton — see readAloudPlayer.ts), +// but if EVERY mounted chat surface unmounts (e.g. the user navigates to a +// route with no chat UI at all) there is no longer a Stop control reachable +// anywhere, while synthesis keeps running and billing and audio keeps +// playing. Ref-counts how many useReadAloud() consumers are currently +// mounted; stops playback only when the count drops to zero, not on every +// individual unmount (closing just the sidebar while the main chat is still +// open must not interrupt a main-chat-initiated read). +let mountedConsumers = 0; + +export function useReadAloud() { + useEffect(() => { + mountedConsumers += 1; + return () => { + mountedConsumers -= 1; + if (mountedConsumers === 0) { + stopReadAloud(); + } + }; + }, []); + + const isReadingAloud = useReadAloudPlayerStore((s) => s.isPlaying); + const isVoiceModeEnabled = useVoiceModeStore((s) => s.isEnabled); + const isDictationActive = useDictationActivityStore((s) => s.activeCount > 0); + const blocked = isVoiceModeEnabled || isDictationActive; + + const toggleReadAloud = useCallback( + (messages: readonly UIMessage[]) => { + if (useReadAloudPlayerStore.getState().isPlaying) { + stopReadAloud(); + return; + } + if (blocked) return; + const text = getTextSinceLastUserTurn(messages); + if (!text.trim()) return; + startReadAloud(flushForTts(text)); + }, + [blocked] + ); + + const canReadAloud = useCallback( + (messages: readonly UIMessage[]) => !blocked && hasTextSinceLastUserTurn(messages), + [blocked] + ); + + return { isReadingAloud, toggleReadAloud, canReadAloud }; +} diff --git a/apps/web/src/hooks/useSpeechRecognition.ts b/apps/web/src/hooks/useSpeechRecognition.ts index 85bd6e9aa3..7460ebcd19 100644 --- a/apps/web/src/hooks/useSpeechRecognition.ts +++ b/apps/web/src/hooks/useSpeechRecognition.ts @@ -1,6 +1,24 @@ 'use client'; import { useState, useEffect, useRef, useCallback } from 'react'; +import { create } from 'zustand'; + +/** + * Cross-surface dictation activity. Each `useSpeechRecognition()` instance + * is otherwise local state — a mounted sidebar chat and main chat each own + * an independent SpeechRecognition object — so consumers that need to know + * "is dictation active ANYWHERE" (e.g. to avoid Read Aloud's TTS getting + * transcribed back into a draft) can't read that off any single instance. + * A count (not a boolean) so two simultaneously-listening surfaces don't + * have one's stop clear the other's still-active state. + */ +interface DictationActivityState { + activeCount: number; +} + +export const useDictationActivityStore = create(() => ({ + activeCount: 0, +})); export interface UseSpeechRecognitionOptions { /** Callback when speech is transcribed */ @@ -45,6 +63,10 @@ export function useSpeechRecognition({ const recognitionRef = useRef(null); const errorTimerRef = useRef | null>(null); const onTranscriptRef = useRef(onTranscript); + // Guards against double-decrementing the shared activeCount — onerror is + // typically followed by onend for the same session, and this instance's + // unmount cleanup could otherwise also fire after one of those already did. + const isCountedActiveRef = useRef(false); // Keep callback ref updated onTranscriptRef.current = onTranscript; @@ -67,12 +89,25 @@ export function useSpeechRecognition({ recognition.interimResults = true; recognition.lang = lang; + const markActive = () => { + if (isCountedActiveRef.current) return; + isCountedActiveRef.current = true; + useDictationActivityStore.setState((s) => ({ activeCount: s.activeCount + 1 })); + }; + const markInactive = () => { + if (!isCountedActiveRef.current) return; + isCountedActiveRef.current = false; + useDictationActivityStore.setState((s) => ({ activeCount: Math.max(0, s.activeCount - 1) })); + }; + recognition.onstart = () => { setIsListening(true); + markActive(); }; recognition.onend = () => { setIsListening(false); + markInactive(); }; recognition.onresult = (event) => { @@ -93,6 +128,7 @@ export function useSpeechRecognition({ recognition.onerror = (event) => { console.error('Speech recognition error:', event.error); setIsListening(false); + markInactive(); const errorMessages: Record = { 'not-allowed': 'Microphone access denied. Check your browser permissions.', @@ -115,6 +151,9 @@ export function useSpeechRecognition({ return () => { recognition.stop(); + // Belt-and-suspenders in case onend never fires before unmount — + // guarded by isCountedActiveRef, so this is a no-op when it already did. + markInactive(); if (errorTimerRef.current) clearTimeout(errorTimerRef.current); }; }, [lang, continuous]); diff --git a/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts b/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts new file mode 100644 index 0000000000..fe23631572 --- /dev/null +++ b/apps/web/src/lib/ai/streams/__tests__/getTextSinceLastUserTurn.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import type { UIMessage } from 'ai'; +import { getTextSinceLastUserTurn, hasTextSinceLastUserTurn } from '../getTextSinceLastUserTurn'; + +const msg = (id: string, role: UIMessage['role'], parts: UIMessage['parts']): UIMessage => ({ + id, + role, + parts, +}); + +const text = (t: string) => ({ type: 'text' as const, text: t }); + +describe('getTextSinceLastUserTurn', () => { + it('given no user message at all, returns empty string', () => { + const messages = [msg('a1', 'assistant', [text('hello')])]; + expect(getTextSinceLastUserTurn(messages)).toBe(''); + }); + + it('given a single assistant reply, returns its text', () => { + const messages = [msg('u1', 'user', [text('hi')]), msg('a1', 'assistant', [text('hello there')])]; + expect(getTextSinceLastUserTurn(messages)).toBe('hello there'); + }); + + it('given multiple consecutive assistant messages, joins their text in order', () => { + const messages = [ + msg('u1', 'user', [text('go')]), + msg('a1', 'assistant', [text('step one')]), + msg('a2', 'assistant', [text('step two')]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('step one\n\nstep two'); + }); + + it('skips tool-call parts and only reads text parts within a message', () => { + const messages = [ + msg('u1', 'user', [text('run it')]), + msg('a1', 'assistant', [ + { type: 'tool-run', toolCallId: 't1', state: 'output-available' } as unknown as UIMessage['parts'][number], + text('done running'), + ]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('done running'); + }); + + it('given a trailing tool-only message with no text, drops it from the joined output', () => { + const messages = [ + msg('u1', 'user', [text('go')]), + msg('a1', 'assistant', [text('here is the answer')]), + msg('a2', 'assistant', [ + { type: 'tool-run', toolCallId: 't2', state: 'output-available' } as unknown as UIMessage['parts'][number], + ]), + ]; + expect(getTextSinceLastUserTurn(messages)).toBe('here is the answer'); + }); + + it('given the last user message has no reply yet, returns empty string', () => { + const messages = [msg('a1', 'assistant', [text('old reply')]), msg('u1', 'user', [text('new question')])]; + expect(getTextSinceLastUserTurn(messages)).toBe(''); + }); + + it('given an empty array, returns empty string', () => { + expect(getTextSinceLastUserTurn([])).toBe(''); + }); +}); + +describe('hasTextSinceLastUserTurn', () => { + it('given no user message at all, returns false', () => { + expect(hasTextSinceLastUserTurn([msg('a1', 'assistant', [text('hello')])])).toBe(false); + }); + + it('given a single assistant reply with text, returns true', () => { + const messages = [msg('u1', 'user', [text('hi')]), msg('a1', 'assistant', [text('hello there')])]; + expect(hasTextSinceLastUserTurn(messages)).toBe(true); + }); + + it('given only tool-call parts and no text, returns false', () => { + const messages = [ + msg('u1', 'user', [text('run it')]), + msg('a1', 'assistant', [ + { type: 'tool-run', toolCallId: 't1', state: 'output-available' } as unknown as UIMessage['parts'][number], + ]), + ]; + expect(hasTextSinceLastUserTurn(messages)).toBe(false); + }); + + it('given a text part that is only whitespace, returns false', () => { + const messages = [msg('u1', 'user', [text('hi')]), msg('a1', 'assistant', [text(' ')])]; + expect(hasTextSinceLastUserTurn(messages)).toBe(false); + }); + + it('given the last user message has no reply yet, returns false', () => { + const messages = [msg('a1', 'assistant', [text('old reply')]), msg('u1', 'user', [text('new question')])]; + expect(hasTextSinceLastUserTurn(messages)).toBe(false); + }); +}); diff --git a/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts b/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts new file mode 100644 index 0000000000..f98ba88d30 --- /dev/null +++ b/apps/web/src/lib/ai/streams/getTextSinceLastUserTurn.ts @@ -0,0 +1,35 @@ +import type { UIMessage } from 'ai'; +import { getAssistantMessagesAfterLastUser } from './getAssistantMessagesAfterLastUser'; + +const textOf = (message: UIMessage): string => + (message.parts ?? []) + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join(''); + +/** + * Plain-text speech source for "read aloud": every assistant message since + * the user's last turn, joined in order. An agent can emit several + * consecutive assistant messages (tool calls, intermediate steps, a final + * reply) before control returns to the user, so this reads all of them + * rather than just the latest one. + */ +export function getTextSinceLastUserTurn(messages: readonly UIMessage[]): string { + return getAssistantMessagesAfterLastUser(messages) + .map(textOf) + .filter(Boolean) + .join('\n\n'); +} + +/** + * Cheap yes/no check for "is there anything to read aloud" — short-circuits + * on the first non-empty text part instead of joining the full reply text. + * Callers that only need a boolean (e.g. to enable/disable a button on every + * render, including every token of a live stream) should use this instead of + * checking `getTextSinceLastUserTurn(...).trim().length > 0`. + */ +export function hasTextSinceLastUserTurn(messages: readonly UIMessage[]): boolean { + return getAssistantMessagesAfterLastUser(messages).some((message) => + (message.parts ?? []).some((p) => p.type === 'text' && p.text.trim().length > 0) + ); +} diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts new file mode 100644 index 0000000000..8e1859ed45 --- /dev/null +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.audioContextFailure.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Kept in its own file, separate from readAloudPlayer.test.ts: the module's +// AudioContext is a lazy singleton, created once and never recreated — a +// prior test in the same file that successfully starts playback leaves it +// cached, so a throwing AudioContext stub registered in a LATER test would +// never actually get invoked. A fresh file gives Vitest a fresh module +// registry, guaranteeing this is the first (and only) call to +// getAudioContext() in this environment. + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +const { toastErrorMock } = vi.hoisted(() => ({ toastErrorMock: vi.fn() })); +vi.mock('sonner', () => ({ + toast: { error: toastErrorMock, success: vi.fn() }, +})); + +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { startReadAloud, isReadAloudPlaying } from '../readAloudPlayer'; + +describe('readAloudPlayer — AudioContext construction failure', () => { + it('surfaces an error and stops the run when creating the AudioContext itself fails', async () => { + vi.stubGlobal( + 'AudioContext', + class { + constructor() { + throw new Error('Web Audio unavailable'); + } + } + ); + + startReadAloud(['hello there']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(toastErrorMock).toHaveBeenCalledWith('Could not read this reply aloud. Please try again.'); + // Never even reached the network call — failed before fetchWithAuth. + expect(fetchWithAuth).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts new file mode 100644 index 0000000000..4dcb077bb1 --- /dev/null +++ b/apps/web/src/lib/voice/__tests__/readAloudPlayer.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +vi.mock('@/lib/auth/auth-fetch', () => ({ + fetchWithAuth: vi.fn(), +})); + +const { toastErrorMock } = vi.hoisted(() => ({ toastErrorMock: vi.fn() })); +vi.mock('sonner', () => ({ + toast: { error: toastErrorMock, success: vi.fn() }, +})); + +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; +import { + startReadAloud, + stopReadAloud, + isReadAloudPlaying, + useReadAloudPlayerStore, +} from '../readAloudPlayer'; + +class FakeAudioBufferSourceNode { + buffer: unknown = null; + onended: (() => void) | null = null; + connect(): void {} + start = vi.fn(); + stop = vi.fn(); +} + +const createdSources: FakeAudioBufferSourceNode[] = []; + +class FakeAudioContext { + state = 'running'; + destination = {}; + createBufferSource(): FakeAudioBufferSourceNode { + const node = new FakeAudioBufferSourceNode(); + createdSources.push(node); + return node; + } + decodeAudioData(): Promise { + return Promise.resolve({}); + } + resume(): Promise { + return Promise.resolve(); + } +} + +describe('readAloudPlayer', () => { + beforeEach(() => { + createdSources.length = 0; + toastErrorMock.mockClear(); + vi.stubGlobal('AudioContext', FakeAudioContext); + vi.mocked(fetchWithAuth).mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + }); + + afterEach(() => { + stopReadAloud(); + useVoiceModeStore.getState().disable(); + useDictationActivityStore.setState({ activeCount: 0 }); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('is not playing before anything starts', () => { + expect(isReadAloudPlaying()).toBe(false); + }); + + it('flips to playing synchronously when starting, before synthesis resolves', () => { + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + }); + + it('given no chunks, does not start playing', () => { + startReadAloud([]); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('stop clears the playing state immediately', () => { + startReadAloud(['hello there']); + stopReadAloud(); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('notifies subscribers on start and stop, regardless of which call site triggers it', () => { + // Simulates two independent useReadAloud() call sites (e.g. sidebar + + // main content) both observing the one shared player. + const surfaceA = vi.fn(); + const surfaceB = vi.fn(); + const unsubscribeA = useReadAloudPlayerStore.subscribe(surfaceA); + const unsubscribeB = useReadAloudPlayerStore.subscribe(surfaceB); + + startReadAloud(['hello there']); + expect(surfaceA).toHaveBeenCalledTimes(1); + expect(surfaceB).toHaveBeenCalledTimes(1); + + // A stop triggered from "surface B" must be observable by "surface A" — + // this is the cross-surface ownership guarantee the singleton provides. + stopReadAloud(); + expect(surfaceA).toHaveBeenCalledTimes(2); + expect(surfaceB).toHaveBeenCalledTimes(2); + expect(isReadAloudPlaying()).toBe(false); + + unsubscribeA(); + unsubscribeB(); + }); + + it('unsubscribing stops further notifications', () => { + const listener = vi.fn(); + const unsubscribe = useReadAloudPlayerStore.subscribe(listener); + unsubscribe(); + + startReadAloud(['hello there']); + expect(listener).not.toHaveBeenCalled(); + }); + + it('starting a new read-aloud while one is in flight replaces the previous queue', () => { + startReadAloud(['first attempt', 'more of the first']); + startReadAloud(['second attempt']); + expect(isReadAloudPlaying()).toBe(true); + }); + + it('stopping when nothing is playing is a no-op that does not throw', () => { + expect(() => stopReadAloud()).not.toThrow(); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('synthesizes and starts playback of a single chunk, then finishes naturally when it ends', async () => { + startReadAloud(['only chunk']); + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + expect(createdSources[0].start).toHaveBeenCalledTimes(1); + expect(isReadAloudPlaying()).toBe(true); + + // Simulate the browser firing onended once the clip finishes. + createdSources[0].onended?.(); + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + }); + + it('plays multiple queued chunks back to back in order', async () => { + startReadAloud(['first', 'second']); + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + + createdSources[0].onended?.(); + await vi.waitFor(() => expect(createdSources).toHaveLength(2)); + expect(isReadAloudPlaying()).toBe(true); + + createdSources[1].onended?.(); + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + }); + + it('discards a chunk that finishes synthesizing after stop was already called', async () => { + startReadAloud(['only chunk']); + stopReadAloud(); + // Let the in-flight synthesis resolve; it must not resurrect playback. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(createdSources).toHaveLength(0); + expect(isReadAloudPlaying()).toBe(false); + }); + + it('given a stop-then-restart while the stopped run is still synthesizing, only the new run ever plays', async () => { + // Two independently-resolvable synthesis calls, so the first run's + // fetch can be made to resolve AFTER a second run has already started — + // reproducing the exact race the generation token guards against. + let resolveFirst: (() => void) | undefined; + let resolveSecond: (() => void) | undefined; + const firstResponse = new Promise((resolve) => { + resolveFirst = () => resolve({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + }); + const secondResponse = new Promise((resolve) => { + resolveSecond = () => resolve({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + }); + vi.mocked(fetchWithAuth) + .mockReturnValueOnce(firstResponse) + .mockReturnValueOnce(secondResponse); + + startReadAloud(['stale chunk']); // run A: fetch is now pending + stopReadAloud(); // stopped before run A's fetch ever resolved + startReadAloud(['fresh chunk']); // run B: a second, independent fetch is pending + + // Resolve the STALE run's fetch first, after the fresh run has already + // begun — without the generation guard this would create and start a + // second, overlapping AudioBufferSourceNode. + resolveFirst?.(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(createdSources).toHaveLength(0); + + resolveSecond?.(); + await vi.waitFor(() => expect(createdSources).toHaveLength(1)); + expect(isReadAloudPlaying()).toBe(true); + }); + + it('stops playback the moment Voice Mode is enabled, regardless of how it was enabled', () => { + // Enforced at the module level (subscribed to useVoiceModeStore) rather + // than by any specific UI call site — so calling the store's enable() + // action directly, bypassing every view's own handleVoiceModeToggle, + // must still stop read-aloud. + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + useVoiceModeStore.getState().enable('ai-page'); + + expect(isReadAloudPlaying()).toBe(false); + }); + + it('aborts the in-flight synthesis fetch when stopped, instead of only discarding its result', () => { + let capturedSignal: AbortSignal | undefined; + vi.mocked(fetchWithAuth).mockImplementation((_url, options) => { + capturedSignal = (options as RequestInit | undefined)?.signal ?? undefined; + return new Promise(() => {}); // never resolves — only the signal matters here + }); + + startReadAloud(['only chunk']); + expect(capturedSignal).toBeDefined(); + expect(capturedSignal?.aborted).toBe(false); + + stopReadAloud(); + expect(capturedSignal?.aborted).toBe(true); + }); + + it('stops playback the moment mic dictation becomes active anywhere, regardless of which ChatInput instance started it', () => { + // Dictation is per-ChatInput-instance local state (useSpeechRecognition), + // unlike Voice Mode's single global store — this simulates a DIFFERENT + // mounted surface's mic starting, via the shared activeCount it feeds. + startReadAloud(['hello there']); + expect(isReadAloudPlaying()).toBe(true); + + useDictationActivityStore.setState({ activeCount: 1 }); + + expect(isReadAloudPlaying()).toBe(false); + }); + + it('reloads persisted voice settings on every start, not just once', () => { + const loadSettingsSpy = vi.spyOn(useVoiceModeStore.getState(), 'loadSettings'); + + startReadAloud(['first']); + stopReadAloud(); + startReadAloud(['second']); + + expect(loadSettingsSpy).toHaveBeenCalledTimes(2); + }); + + it('refuses to start when Voice Mode is already enabled at call time, even with no prior transition to observe', () => { + // Not a "stop while playing" case — Voice Mode was ALREADY on before + // startReadAloud() ever ran, so the subscription (which only fires on a + // future inactive-to-active transition) never gets a chance to fire. + useVoiceModeStore.getState().enable('ai-page'); + + startReadAloud(['hello there']); + + expect(isReadAloudPlaying()).toBe(false); + expect(createdSources).toHaveLength(0); + }); + + it('refuses to start when dictation is already active at call time, even with no prior transition to observe', () => { + useDictationActivityStore.setState({ activeCount: 1 }); + + startReadAloud(['hello there']); + + expect(isReadAloudPlaying()).toBe(false); + expect(createdSources).toHaveLength(0); + }); + + it('surfaces a systemic synthesis failure and stops the run instead of retry-skipping every remaining chunk', async () => { + vi.mocked(fetchWithAuth).mockResolvedValue({ + ok: false, + json: () => Promise.resolve({ message: 'Voice synthesis is not configured on this deployment.' }), + } as Response); + + startReadAloud(['first chunk', 'second chunk']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(createdSources).toHaveLength(0); + expect(toastErrorMock).toHaveBeenCalledWith('Voice synthesis is not configured on this deployment.'); + // Stopped after the first failure — never even attempted the second chunk. + expect(fetchWithAuth).toHaveBeenCalledTimes(1); + }); + + it('does not surface an error toast when synthesis is cancelled by an intentional stop', async () => { + // Mirrors a real fetch()'s behavior on abort: rejects with an AbortError, + // either immediately (if already aborted) or once the signal fires. + vi.mocked(fetchWithAuth).mockImplementation((_url, options) => { + const signal = (options as RequestInit | undefined)?.signal; + return new Promise((_resolve, reject) => { + const onAbort = () => reject(new DOMException('The operation was aborted', 'AbortError')); + if (signal?.aborted) { + onAbort(); + return; + } + signal?.addEventListener('abort', onAbort); + }); + }); + + startReadAloud(['only chunk']); + stopReadAloud(); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(toastErrorMock).not.toHaveBeenCalled(); + }); + + it('surfaces an error toast and stops the run when synthesis throws for a reason other than an intentional stop', async () => { + vi.mocked(fetchWithAuth).mockRejectedValue(new Error('network blip')); + + startReadAloud(['first chunk', 'second chunk']); + + await vi.waitFor(() => expect(isReadAloudPlaying()).toBe(false)); + expect(toastErrorMock).toHaveBeenCalledWith('Could not read this reply aloud. Please try again.'); + // Stopped after the first failure — never even attempted the second chunk. + expect(fetchWithAuth).toHaveBeenCalledTimes(1); + }); + + it('does not surface a toast for a stale, already-superseded run that fails on its own after a restart', async () => { + let rejectFirst: ((err: Error) => void) | undefined; + const firstResponse = new Promise((_resolve, reject) => { + rejectFirst = reject; + }); + vi.mocked(fetchWithAuth) + .mockReturnValueOnce(firstResponse) + .mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + } as Response); + + startReadAloud(['stale chunk']); // run A: fetch pending + stopReadAloud(); // stopped before run A's fetch ever resolved + startReadAloud(['fresh chunk']); // run B: a fresh, successful fetch + + // Run A's fetch fails for its own, unrelated reason — AFTER run B has + // already begun. The user already moved on; this shouldn't surface a + // confusing error about a read they're no longer waiting on. + rejectFirst?.(new Error('unrelated network blip')); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(toastErrorMock).not.toHaveBeenCalled(); + expect(isReadAloudPlaying()).toBe(true); + }); +}); diff --git a/apps/web/src/lib/voice/readAloudPlayer.ts b/apps/web/src/lib/voice/readAloudPlayer.ts new file mode 100644 index 0000000000..401c6f8268 --- /dev/null +++ b/apps/web/src/lib/voice/readAloudPlayer.ts @@ -0,0 +1,257 @@ +'use client'; + +import { create } from 'zustand'; +import { toast } from 'sonner'; +import { fetchWithAuth } from '@/lib/auth/auth-fetch'; +import { useVoiceModeStore } from '@/stores/useVoiceModeStore'; +import { useDictationActivityStore } from '@/hooks/useSpeechRecognition'; + +/** + * Module-singleton "read aloud" audio player, deliberately independent of + * React component lifecycles and of `useVoiceMode`'s per-instance + * AudioContext/audioSource refs. + * + * Read Aloud can be triggered from multiple mounted chat surfaces at once + * (e.g. the right-sidebar chat tab alongside the main AiChatView/ + * GlobalAssistantView content). A per-component `useVoiceMode()` instance + * only tracks its OWN AudioContext locally while sharing global voice state + * via the store — so a second instance's "stop" call clears shared state but + * can't reach the first instance's actual playing AudioBufferSourceNode, and + * an unmounting instance never resets that shared state either. Keeping the + * one real audio source at module scope means there is only ever one thing + * to stop, reachable from anywhere `useReadAloud()` is called. + * + * Only the boolean "is something playing" needs to be observable by React, + * so that alone lives in a tiny zustand store (this codebase's established + * pattern for shared, cross-component state — see the sibling + * `useVoiceModeStore`). The actual audio machinery (AudioContext, the + * current source, the chunk queue, the race-guard generation counter) stays + * in plain module variables outside React entirely. + * + * Mutually exclusive with two other microphone-capturing features, each + * with the same cross-surface problem: a live Voice Mode call + * (`useVoiceModeStore`) and basic mic dictation (`useDictationActivityStore` + * in `useSpeechRecognition.ts`, one per `ChatInput` instance). Either one's + * mic could otherwise pick up this module's own TTS audio and transcribe it + * back into a draft or into Voice Mode's own turn. See `ensureReadAloudReady`. + */ + +interface ReadAloudPlayerState { + isPlaying: boolean; +} + +export const useReadAloudPlayerStore = create(() => ({ + isPlaying: false, +})); + +function setPlaying(isPlaying: boolean): void { + useReadAloudPlayerStore.setState({ isPlaying }); +} + +let audioContext: AudioContext | null = null; +let audioSource: AudioBufferSourceNode | null = null; +let queue: string[] = []; +// The in-flight synthesis request, if any — aborted by stopReadAloud() so a +// stopped chunk's billed TTS call is actually cancelled, not just ignored. +let activeAbortController: AbortController | null = null; + +// Bumped by every stopReadAloud() call (including the implicit one at the +// start of startReadAloud()). A `playNext`/`synthesize` chain captures the +// generation it was started under and re-checks it after every await: +// `isPlaying` alone can't tell "this run was stopped" apart from "this run +// was stopped AND THEN a newer run began" — in the latter case `isPlaying` +// is true again by the time the stale chain resumes, so it would otherwise +// create a second AudioBufferSourceNode that plays concurrently with the +// new run's. +let generation = 0; + +function getAudioContext(): AudioContext { + if (!audioContext) { + audioContext = new AudioContext(); + } + return audioContext; +} + +async function synthesize(text: string, runId: number): Promise { + const { ttsVoice, ttsSpeed } = useVoiceModeStore.getState(); + const controller = new AbortController(); + activeAbortController = controller; + // A stale, already-superseded run (from a stop-then-restart) failing for + // its own unrelated reason must be a full no-op: no toast, and — crucially + // — no stopReadAloud() either, since that would tear down a newer run that + // has nothing to do with this one's failure. Only the run that is still + // current gets to surface an error and stop the queue. + const handleFailure = (message: string) => { + if (runId !== generation) return; + toast.error(message); + stopReadAloud(); + }; + try { + // Created before the network await so the browser still credits this + // AudioContext to the user gesture that triggered playback. Inside the + // try block (not before it) so a creation failure — quota exhausted, + // Web Audio unsupported — gets the same surface-and-stop handling as any + // other synthesis failure, rather than rejecting playNext()'s + // fire-and-forget call unhandled and leaving isPlaying stuck true with + // no explanation and no audio. + const ctx = getAudioContext(); + const response = await fetchWithAuth('/api/voice/synthesize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, voice: ttsVoice, speed: ttsSpeed }), + signal: controller.signal, + }); + if (!response.ok) { + // A systemic failure (out of credits, rate-limited, misconfigured) — + // every remaining chunk would fail identically, so surface it and stop + // the whole run instead of silently skip-retrying it chunk by chunk. + // playNext()'s generation check (bumped by handleFailure's + // stopReadAloud() call) is what actually prevents the skip-retry, not + // this branch. + const errorData = await response.json().catch(() => ({}) as Record); + const message = + (typeof errorData.message === 'string' && errorData.message) || + (typeof errorData.error === 'string' && errorData.error) || + 'Could not read this reply aloud.'; + handleFailure(message); + return null; + } + const audioData = await response.arrayBuffer(); + if (ctx.state === 'suspended') { + await ctx.resume(); + } + return await ctx.decodeAudioData(audioData); + } catch (err) { + // An AbortError here means WE intentionally cancelled this request (via + // stopReadAloud()'s controller.abort()) — expected, nothing to surface. + // Any other exception (network failure, a corrupted/undecodable + // response, AudioContext.resume() failing) is unexpected and would + // otherwise silently truncate or drop the reply with zero explanation, + // so it gets the same surface-and-stop treatment as a non-ok response. + if (!(err instanceof DOMException && err.name === 'AbortError')) { + handleFailure('Could not read this reply aloud. Please try again.'); + } + return null; + } finally { + // Only clear if still ours — a stop-then-restart race can leave a newer + // call's controller as the active one while this (stale) call is still + // unwinding; clearing unconditionally would drop the newer reference and + // leave IT un-abortable by a later stop. + if (activeAbortController === controller) { + activeAbortController = null; + } + } +} + +async function playNext(runId: number): Promise { + if (runId !== generation) return; + const text = queue.shift(); + if (text === undefined) { + setPlaying(false); + return; + } + + const buffer = await synthesize(text, runId); + // A stop, or a stop-then-restart, happened while this chunk was + // synthesizing — discard the result rather than letting a stale run play. + if (runId !== generation) return; + if (!buffer) { + // Skip a chunk that failed to synthesize rather than abandoning the rest. + void playNext(runId); + return; + } + + const ctx = getAudioContext(); + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + audioSource = source; + source.onended = () => { + if (runId !== generation) return; + // Within one generation only one source is ever live at a time — a new + // one is only created after the previous one's onended fired (or after a + // stop, which bumps generation and is already excluded above) — so + // audioSource is guaranteed to still be this source here. + audioSource = null; + void playNext(runId); + }; + source.start(); +} + +export function startReadAloud(chunks: string[]): void { + ensureMutualExclusionSubscriptions(); + // synthesize() reads ttsVoice/ttsSpeed straight from useVoiceModeStore + // without ever mounting useVoiceMode() — which is what used to trigger + // this load on mount — so a user's persisted voice choice would otherwise + // be silently ignored the first time they use Read Aloud without ever + // having opened the full Voice Mode panel. Idempotent and cheap, so it's + // just re-run on every start rather than gated behind a one-time flag. + useVoiceModeStore.getState().loadSettings(); + stopReadAloud(); + if (chunks.length === 0) return; + // Re-validate live state here, not just via the subscriptions above: a + // caller's "may I start?" check (useReadAloud's `blocked`) can be a stale + // React closure, and the subscriptions only fire on a FUTURE + // inactive-to-active transition — neither catches "already active by the + // time this specific call runs" (e.g. rapid clicks across two surfaces). + if (useVoiceModeStore.getState().isEnabled || useDictationActivityStore.getState().activeCount > 0) { + return; + } + queue = [...chunks]; + setPlaying(true); + void playNext(generation); +} + +export function stopReadAloud(): void { + generation++; + activeAbortController?.abort(); + activeAbortController = null; + if (audioSource) { + try { + audioSource.stop(); + } catch { + // Already stopped. + } + audioSource = null; + } + queue = []; + // Guard against a redundant store update (and subscriber notification): + // startReadAloud() always calls this first to reset any prior run, even + // when nothing was playing. + if (useReadAloudPlayerStore.getState().isPlaying) { + setPlaying(false); + } +} + +export function isReadAloudPlaying(): boolean { + return useReadAloudPlayerStore.getState().isPlaying; +} + +// Registered lazily (on first startReadAloud(), not at module import time) +// so merely importing this module — e.g. a component under test that +// renders but never exercises Read Aloud — never touches these other +// stores' `.subscribe`. By the time any audio could actually be playing, +// this has always already run, since startReadAloud() is the only path +// that starts playback. Guarded by a one-time flag (unlike loadSettings() +// above) because subscribing more than once would leak duplicate listeners. +// +// A live Voice Mode call or active mic dictation each capture the +// microphone — this module's own TTS audio must never play concurrently +// with either, or it risks being picked up as if it were user speech. +// Enforced here (not per call-site) so the invariant holds no matter which +// UI entry point triggers either, present or future. +let subscriptionsRegistered = false; +function ensureMutualExclusionSubscriptions(): void { + if (subscriptionsRegistered) return; + subscriptionsRegistered = true; + useVoiceModeStore.subscribe((state, prevState) => { + if (state.isEnabled && !prevState.isEnabled) { + stopReadAloud(); + } + }); + useDictationActivityStore.subscribe((state, prevState) => { + if (state.activeCount > 0 && prevState.activeCount === 0) { + stopReadAloud(); + } + }); +}