Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions apps/web/src/app/api/voice/synthesize/__tests__/route.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
// @vitest-environment node
//
// This tests a Node.js server route handler, not browser code — running it
// under jsdom (the project default) creates two competing AbortController/
// AbortSignal globals (jsdom's polyfill vs. Node's native one), and jsdom's
// Request constructor rejects a signal from the "wrong" realm with
// "Expected signal to be an instance of AbortSignal". The route itself runs
// in a real Node/Edge runtime with a single Fetch API implementation, so
// `node` here is the more accurate environment, not a workaround.
import { describe, it, expect, vi, beforeEach } from 'vitest';

// ── Mocks ────────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -146,4 +155,40 @@ describe('POST /api/voice/synthesize — metering', () => {
expect(mockTrackUsage).not.toHaveBeenCalled();
expect(mockReleaseHold).toHaveBeenCalledWith('hold_1');
});

it('propagates the caller abort signal to the upstream OpenAI request, so a cancelled client request releases the hold without billing', async () => {
let capturedSignal: AbortSignal | undefined;
vi.stubGlobal('fetch', vi.fn().mockImplementation((_url: string, opts: RequestInit) => {
capturedSignal = opts.signal ?? undefined;
return new Promise((_resolve, reject) => {
const abort = () => reject(new DOMException('The operation was aborted', 'AbortError'));
// The route awaits auth/gating before ever reaching fetch(), so by
// then the signal may already be aborted — a listener alone would
// miss an abort event that already fired in the past.
if (opts.signal?.aborted) {
abort();
return;
}
opts.signal?.addEventListener('abort', abort);
});
}));

const controller = new AbortController();
const request = new Request('http://localhost/api/voice/synthesize', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ text: 'hello' }),
signal: controller.signal,
});

const resPromise = POST(request);
controller.abort();
const res = await resPromise;

expect(capturedSignal).toBeDefined();
expect(capturedSignal?.aborted).toBe(true);
expect(res.status).toBe(500);
expect(mockTrackUsage).not.toHaveBeenCalled();
expect(mockReleaseHold).toHaveBeenCalledWith('hold_1');
});
});
6 changes: 5 additions & 1 deletion apps/web/src/app/api/voice/synthesize/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,10 @@ export async function POST(request: Request) {
}
holdId = gate.holdId;

// Call OpenAI TTS API
// Call OpenAI TTS API. Forwards the caller's abort signal so a client
// that cancels mid-request (e.g. Read Aloud's Stop button) also cancels
// the upstream request — otherwise it runs to completion and gets
// billed regardless of the client having already discarded it.
const response = await fetch('https://api.openai.com/v1/audio/speech', {
method: 'POST',
headers: {
Expand All @@ -172,6 +175,7 @@ export async function POST(request: Request) {
speed: clampedSpeed,
response_format: 'mp3',
}),
signal: request.signal,
});

if (!response.ok) {
Expand Down
24 changes: 23 additions & 1 deletion apps/web/src/components/ai/chat/input/ChatInput.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { useAssistantSettingsStore } from '@/stores/useAssistantSettingsStore';
import { isImageGenerationAllowed } from '@/lib/ai/core/image-gen-access';
import { useSpeechRecognition } from '@/hooks/useSpeechRecognition';
import { useMobileKeyboard } from '@/hooks/useMobileKeyboard';
import { stopReadAloud } from '@/lib/voice/readAloudPlayer';
import type { ImageAttachment } from '@/lib/ai/shared/hooks/useImageAttachments';

export interface ChatInputProps {
Expand Down Expand Up @@ -65,6 +66,12 @@ export interface ChatInputProps {
onVoiceModeClick?: () => void;
/** Whether voice mode is currently active */
isVoiceModeActive?: boolean;
/** Callback when the read-aloud button is clicked */
onReadAloudClick?: () => void;
/** Whether read-aloud is currently playing */
isReadingAloud?: boolean;
/** Whether there is anything eligible to read aloud right now */
canReadAloud?: boolean;
/** Image attachments for vision support */
attachments?: ImageAttachment[];
/** Handler to add image files */
Expand Down Expand Up @@ -127,6 +134,9 @@ export const ChatInput = forwardRef<ChatInputRef, ChatInputProps>(
onProviderModelChange,
onVoiceModeClick,
isVoiceModeActive = false,
onReadAloudClick,
isReadingAloud = false,
canReadAloud = false,
attachments,
onAddFiles,
onRemoveFile,
Expand Down Expand Up @@ -174,6 +184,15 @@ export const ChatInput = forwardRef<ChatInputRef, ChatInputProps>(
},
});

// Starting mic dictation stops Read Aloud first — otherwise the mic can
// transcribe the TTS audio it hears right back into the draft.
const handleMicClick = useCallback(() => {
if (!isListening) {
stopReadAloud();
}
toggleListening();
}, [isListening, toggleListening]);

// Mobile keyboard management
const keyboard = useMobileKeyboard();
const prevStreamingRef = useRef(isStreaming);
Expand Down Expand Up @@ -298,13 +317,16 @@ export const ChatInput = forwardRef<ChatInputRef, ChatInputProps>(
isMcpServerEnabled={isMcpServerEnabled}
onMcpServerToggle={onMcpServerToggle}
showMcp={showMcp}
onMicClick={toggleListening}
onMicClick={handleMicClick}
isListening={isListening}
isMicSupported={isSupported}
micError={speechError}
onClearMicError={clearSpeechError}
onVoiceModeClick={onVoiceModeClick}
isVoiceModeActive={isVoiceModeActive}
onReadAloudClick={onReadAloudClick}
isReadingAloud={isReadingAloud}
canReadAloud={canReadAloud}
selectedProvider={currentProvider}
selectedModel={currentModel}
onProviderModelChange={handleProviderModelChange}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import { shouldRefreshAfterUndo } from '@/lib/ai/streams/shouldRefreshAfterUndo'
import { shouldPrependConversation } from '@/lib/ai/streams/shouldPrependConversation';
import { shouldReloadOnComountComplete } from '@/lib/ai/streams/shouldReloadOnComountComplete';
import { getBrowserSessionId } from '@/lib/ai/core/browser-session-id';
import { useReadAloud } from '@/hooks/useReadAloud';

// Shared hooks and components
import {
Expand Down Expand Up @@ -581,6 +582,15 @@ const AiChatView: React.FC<AiChatViewProps> = ({ page }) => {
const renderedMessages = useRenderedMessages(page.id, currentConversationId);
const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]);

// Read Aloud: on-demand TTS for everything the assistant said since the
// user's last turn, via a shared playback singleton (see readAloudPlayer).
const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud();
const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]);
const handleReadAloudClick = useCallback(
() => toggleReadAloud(plainMessages),
[toggleReadAloud, plainMessages]
);

// "Load older" (epic leaf 6.6, scroll-to-top): AiChatView's route IS the agent-conversation
// route (page.id is the agentId), so the shared agent-mode loader applies directly.
const { isLoadingOlder } = useConversationOlderPageState(currentConversationId);
Expand Down Expand Up @@ -1238,7 +1248,9 @@ const AiChatView: React.FC<AiChatViewProps> = ({ page }) => {
prepareSend,
]);

// Voice mode toggle handler
// Voice mode toggle handler. Enabling Voice Mode also stops any
// in-progress read-aloud playback — enforced inside readAloudPlayer itself
// (subscribed to the voice-mode store), not here.
const handleVoiceModeToggle = useCallback(() => {
if (isVoiceModeActive) {
disableVoiceMode();
Expand Down Expand Up @@ -1518,6 +1530,9 @@ const AiChatView: React.FC<AiChatViewProps> = ({ page }) => {
}}
onVoiceModeClick={handleVoiceModeToggle}
isVoiceModeActive={isVoiceModeActive}
onReadAloudClick={handleReadAloudClick}
isReadingAloud={isReadingAloud}
canReadAloud={canReadAloud}
attachments={attachments}
onAddFiles={addFiles}
onRemoveFile={removeFile}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi
import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText';
import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline';
import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage';
import { useReadAloud } from '@/hooks/useReadAloud';
import { createId } from '@paralleldrive/cuid2';

const VOICE_OWNER: VoiceModeOwner = 'global-assistant';
Expand Down Expand Up @@ -436,6 +437,15 @@ const GlobalAssistantView: React.FC = () => {
// above) never renders post-cutover — it stays the transport/controller only.
const renderedMessages = useRenderedMessages(streamChannelId ?? '', currentConversationId);
const plainMessages = useMemo(() => renderedMessages.map((r) => r.message), [renderedMessages]);

// Read Aloud: on-demand TTS for everything the assistant said since the
// user's last turn, via a shared playback singleton (see readAloudPlayer).
const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud();
const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]);
const handleReadAloudClick = useCallback(
() => toggleReadAloud(plainMessages),
[toggleReadAloud, plainMessages]
);
// Loading/error UI reads the cache entry's state (replaces the context's
// isMessagesLoading and the dashboard store's isConversationMessagesLoading).
const messagesLoadState = useConversationLoadState(currentConversationId);
Expand Down Expand Up @@ -899,7 +909,9 @@ const GlobalAssistantView: React.FC = () => {
setLastAIResponse((current) => (current?.id === next.id ? current : next));
}, [renderedMessages, isVoiceModeActive]);

// Voice mode toggle handler
// Voice mode toggle handler. Enabling Voice Mode also stops any
// in-progress read-aloud playback — enforced inside readAloudPlayer itself
// (subscribed to the voice-mode store), not here.
const handleVoiceModeToggle = useCallback(() => {
if (isVoiceModeActive) {
disableVoiceMode();
Expand Down Expand Up @@ -1095,6 +1107,9 @@ const GlobalAssistantView: React.FC = () => {
popupPlacement={props.inputPosition === 'centered' ? 'bottom' : 'top'}
onVoiceModeClick={handleVoiceModeToggle}
isVoiceModeActive={isVoiceModeActive}
onReadAloudClick={handleReadAloudClick}
isReadingAloud={isReadingAloud}
canReadAloud={canReadAloud}
attachments={attachments}
onAddFiles={addFiles}
onRemoveFile={removeFile}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import { rollbackOptimisticSendOnFailure } from '@/lib/ai/streams/rollbackOptimi
import { selectVoiceStreamText } from '@/lib/ai/streams/selectVoiceStreamText';
import { selectVoiceActivationBaseline } from '@/lib/ai/streams/selectVoiceActivationBaseline';
import { selectPostBaselineAssistantMessage } from '@/lib/ai/streams/selectPostBaselineAssistantMessage';
import { useReadAloud } from '@/hooks/useReadAloud';
import { createId } from '@paralleldrive/cuid2';
import { useStopStream } from '@/hooks/useStopStream';
import { useOwnStreamMirror } from '@/hooks/useOwnStreamMirror';
Expand Down Expand Up @@ -542,6 +543,15 @@ const SidebarChatTab: React.FC = () => {
const disableVoiceMode = useVoiceModeStore((s) => s.disable);
const isVoiceModeActive = isVoiceModeEnabled && voiceOwner === VOICE_OWNER;

// Read Aloud: on-demand TTS for everything the assistant said since the
// user's last turn, via a shared playback singleton (see readAloudPlayer).
const { isReadingAloud, toggleReadAloud, canReadAloud: canReadAloudFor } = useReadAloud();
const canReadAloud = useMemo(() => canReadAloudFor(plainMessages), [canReadAloudFor, plainMessages]);
const handleReadAloudClick = useCallback(
() => toggleReadAloud(plainMessages),
[toggleReadAloud, plainMessages]
);

// Display preferences
const { preferences: displayPreferences } = useDisplayPreferences();

Expand Down Expand Up @@ -864,7 +874,9 @@ const SidebarChatTab: React.FC = () => {
),
});

// Voice mode toggle handler
// Voice mode toggle handler. Enabling Voice Mode also stops any
// in-progress read-aloud playback — enforced inside readAloudPlayer itself
// (subscribed to the voice-mode store), not here.
const handleVoiceModeToggle = useCallback(() => {
if (isVoiceModeActive) {
disableVoiceMode();
Expand Down Expand Up @@ -1115,6 +1127,9 @@ const SidebarChatTab: React.FC = () => {
variant="sidebar"
onVoiceModeClick={handleVoiceModeToggle}
isVoiceModeActive={isVoiceModeActive}
onReadAloudClick={handleReadAloudClick}
isReadingAloud={isReadingAloud}
canReadAloud={canReadAloud}
attachments={attachments}
onAddFiles={addFiles}
onRemoveFile={removeFile}
Expand Down
49 changes: 48 additions & 1 deletion apps/web/src/components/ui/floating-input/InputFooter.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
TooltipTrigger,
TooltipContent,
} from '@/components/ui/tooltip';
import { Mic, AudioLines, MicOff } from 'lucide-react';
import { Mic, AudioLines, MicOff, Volume2 } from 'lucide-react';
import { cn } from '@/lib/utils';
import { ProviderModelSelector } from '@/components/ai/chat/input/ProviderModelSelector';
import { ToolsPopover } from './ToolsPopover';
Expand Down Expand Up @@ -61,6 +61,12 @@ export interface InputFooterProps {
onVoiceModeClick?: () => void;
/** Whether voice mode is currently active */
isVoiceModeActive?: boolean;
/** Callback when the read-aloud button is clicked */
onReadAloudClick?: () => void;
/** Whether read-aloud is currently playing */
isReadingAloud?: boolean;
/** Whether there is anything eligible to read aloud right now */
canReadAloud?: boolean;
/** Error message from microphone/speech recognition */
micError?: string | null;
/** Callback to clear the mic error */
Expand Down Expand Up @@ -110,6 +116,9 @@ export function InputFooter({
isMicSupported = true,
onVoiceModeClick,
isVoiceModeActive = false,
onReadAloudClick,
isReadingAloud = false,
canReadAloud = false,
micError,
onClearMicError,
selectedProvider,
Expand Down Expand Up @@ -165,6 +174,44 @@ export function InputFooter({
/>
)}

{/* Read Aloud button (on-demand TTS for the assistant's last turn) */}
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="ghost"
size="sm"
onClick={onReadAloudClick}
disabled={!isReadingAloud && (disabled || isVoiceProGated || !canReadAloud)}
className={cn(
'h-8 w-8 p-0 transition-all duration-200 hover:bg-transparent dark:hover:bg-transparent',
isReadingAloud
? 'animate-pulse text-primary'
: 'text-muted-foreground hover:text-foreground'
)}
>
<Volume2 className="h-4 w-4" />
<span className="sr-only">
{isVoiceProGated
? 'Read aloud requires Pro'
: isReadingAloud
? 'Stop reading aloud'
: isListening
? 'Read aloud unavailable while dictating'
: 'Read aloud'}
</span>
</Button>
</TooltipTrigger>
<TooltipContent side="top">
{isVoiceProGated
? 'Read aloud requires a Pro plan'
: isReadingAloud
? 'Stop reading aloud'
: isListening
? 'Read aloud unavailable while dictating'
: 'Read aloud'}
</TooltipContent>
</Tooltip>

{/* Voice Mode button (hands-free STT/TTS) */}
<Tooltip>
<TooltipTrigger asChild>
Expand Down
Loading