diff --git a/.gitignore b/.gitignore index 71eac191..20170499 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ tmp/ temp/ ui/dist/ *.css -test/ debug_*.png debug_*.html proxylist.txt diff --git a/README.md b/README.md index 473beea2..1307aa8e 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ services: - `GET /v1/models`: 列出模型。 - `POST /v1/chat/completions`: 聊天补全和图片生成，支持非流式、真流式和假流式。 +- `POST /v1/audio/speech`: 使用 Gemini TTS 生成二进制语音，支持 WAV（默认）和原始 PCM 输出。 - `POST /v1/embeddings`: 生成文本嵌入向量。 - `POST /v1/responses`: OpenAI Responses API 兼容接口，用于对话生成，不支持图像生成，支持非流式、真流式和假流式。 - `POST /v1/responses/input_tokens`: 计算 OpenAI Responses API 请求的输入 token 数量。 diff --git a/README_EN.md b/README_EN.md index 9aa299f4..4922b653 100644 --- a/README_EN.md +++ b/README_EN.md @@ -199,6 +199,7 @@ This endpoint is processed and then forwarded to the Gemini API format endpoint. - `GET /v1/models`: List models. - `POST /v1/chat/completions`: Chat completion and image generation, supports non-streaming, real streaming, and fake streaming. +- `POST /v1/audio/speech`: Generate binary speech audio with Gemini TTS. Supports WAV (default) and raw PCM output. - `POST /v1/embeddings`: Generate text embedding vectors. - `POST /v1/responses`: OpenAI Responses API compatible endpoint for conversation generation, does not support image generation, and supports non-streaming, real streaming, and fake streaming. - `POST /v1/responses/input_tokens`: Count input tokens for an OpenAI Responses API request. diff --git a/docs/en/api-examples.md b/docs/en/api-examples.md index ebc05bf6..1fe3e9e7 100644 --- a/docs/en/api-examples.md +++ b/docs/en/api-examples.md @@ -122,6 +122,25 @@ curl -X POST http://localhost:7860/v1/responses \ }' ``` +### 🎤 Speech Generation + +The OpenAI-compatible speech endpoint returns binary audio directly. Gemini-native PCM is wrapped in a WAV container when `response_format` is `wav` (the default): + +```bash +curl -X POST http://localhost:7860/v1/audio/speech \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-api-key-1" \ + -d '{ + "model": "gemini-3.1-flash-tts-preview", + "input": "Hello, this is a text to speech test.", + "voice": "Kore", + "response_format": "wav" + }' \ + --output speech.wav +``` + +Supported response formats are `wav` and `pcm`. The `pcm` option returns Gemini's raw PCM bytes with the sample format declared in the response `Content-Type`. MP3, AAC, FLAC, and Opus are not returned because this project does not include an audio encoder; requesting them returns an OpenAI-style `400` error. Unsupported speech parameters, including `speed`, `instructions`, `stream`, and `stream_format`, also return `400` instead of being silently ignored. + ## ♊ Gemini Native API Format ```bash diff --git a/docs/zh/api-examples.md b/docs/zh/api-examples.md index 68881394..cac27c4b 100644 --- a/docs/zh/api-examples.md +++ b/docs/zh/api-examples.md @@ -122,6 +122,25 @@ curl -X POST http://localhost:7860/v1/responses \ }' ``` +### 🎤 语音生成 + +OpenAI 兼容的语音端点会直接返回二进制音频。当 `response_format` 为 `wav`（默认值）时，服务会将 Gemini 原生 PCM 封装为 WAV： + +```bash +curl -X POST http://localhost:7860/v1/audio/speech \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-api-key-1" \ + -d '{ + "model": "gemini-3.1-flash-tts-preview", + "input": "你好，这是一个语音合成测试。", + "voice": "Kore", + "response_format": "wav" + }' \ + --output speech.wav +``` + +支持的响应格式为 `wav` 和 `pcm`。选择 `pcm` 时会返回 Gemini 的原始 PCM 字节，并在响应 `Content-Type` 中声明采样格式。本项目未包含音频编码器，因此不会返回 MP3、AAC、FLAC 或 Opus；请求这些格式时会返回 OpenAI 风格的 `400` 错误。不支持的语音参数（包括 `speed`、`instructions`、`stream` 和 `stream_format`）同样会返回 `400`，不会被静默忽略。 + ## ♊ Gemini 原生 API 格式 ```bash diff --git a/package.json b/package.json index a3161dd1..ea262a5b 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "build:ui": "vite build", "preview:ui": "vite preview", "start": "cross-env NODE_ENV=production node main.js", + "test": "node --test test/*.test.js", "quick-start": "cross-env NODE_ENV=production node main.js", "prestart": "npm run build:ui", "save-auth": "node scripts/auth/saveAuth.js", diff --git a/src/core/FormatConverter.js b/src/core/FormatConverter.js index f13959ae..cc994769 100644 --- a/src/core/FormatConverter.js +++ b/src/core/FormatConverter.js @@ -7,6 +7,7 @@ const axios = require("axios"); const mime = require("mime-types"); +const { convertGeminiAudioResponse } = require("../utils/AudioUtils"); /** * Format Converter Module @@ -1036,6 +1037,84 @@ class FormatConverter { return { cleanModelName, googleRequest, path }; } + /** + * Convert an OpenAI speech request into Gemini native TTS format. + * Only WAV and raw PCM responses are supported because Gemini returns PCM and this + * project does not include a lossy audio encoder. + * + * @param {object} openaiBody - OpenAI speech request body + * @returns {{ cleanModelName: string, googleRequest: object, responseFormat: "wav"|"pcm" }} + */ + translateOpenAISpeechToGoogle(openaiBody) { + if (!openaiBody || typeof openaiBody !== "object" || Array.isArray(openaiBody)) { + throw new Error("Request body must be a JSON object."); + } + + const requiredStringFields = ["model", "input", "voice"]; + for (const field of requiredStringFields) { + if (typeof openaiBody[field] !== "string" || openaiBody[field].trim().length === 0) { + throw new Error(`Missing required parameter: '${field}'.`); + } + } + + const supportedFields = new Set(["input", "model", "response_format", "voice"]); + const unsupportedFields = Object.keys(openaiBody).filter(field => !supportedFields.has(field)); + if (unsupportedFields.length > 0) { + const fieldList = unsupportedFields.map(field => `'${field}'`).join(", "); + throw new Error(`Unsupported parameter${unsupportedFields.length === 1 ? "" : "s"}: ${fieldList}.`); + } + + const responseFormat = openaiBody.response_format === undefined ? "wav" : openaiBody.response_format; + if (typeof responseFormat !== "string" || !["pcm", "wav"].includes(responseFormat.toLowerCase())) { + const requestedFormat = typeof responseFormat === "string" ? responseFormat : typeof responseFormat; + throw new Error( + `Unsupported response_format '${requestedFormat}'. Supported response formats are 'wav' and 'pcm'.` + ); + } + + const cleanModelName = openaiBody.model.trim().replace(/^models\//, ""); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(cleanModelName)) { + throw new Error("Invalid 'model': expected a Gemini model name without path or query parameters."); + } + + const googleRequest = { + contents: [ + { + parts: [{ text: openaiBody.input }], + role: "user", + }, + ], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { + voiceName: openaiBody.voice.trim(), + }, + }, + }, + }, + }; + + this.logger.info(`[Adapter] OpenAI speech request translated for model "${cleanModelName}".`); + return { + cleanModelName, + googleRequest, + responseFormat: responseFormat.toLowerCase(), + }; + } + + /** + * Decode Gemini inline audio and convert it to the requested OpenAI speech format. + * + * @param {object} googleResponse - Gemini generateContent response + * @param {"wav"|"pcm"} responseFormat - Validated output format + * @returns {{ audioBuffer: Buffer, contentType: string }} + */ + convertGoogleToOpenAISpeech(googleResponse, responseFormat) { + return convertGeminiAudioResponse(googleResponse, responseFormat); + } + /** * Common final processing for Gemini requests: * 1. Inject force features (Search, URL Context) diff --git a/src/core/ProxyServerSystem.js b/src/core/ProxyServerSystem.js index 06332d8d..0177d3bc 100644 --- a/src/core/ProxyServerSystem.js +++ b/src/core/ProxyServerSystem.js @@ -486,6 +486,10 @@ class ProxyServerSystem extends EventEmitter { this.requestHandler.processOpenAIRequest(req, res); }); + app.post("/v1/audio/speech", (req, res) => { + this.requestHandler.processOpenAISpeechRequest(req, res); + }); + app.post(["/v1/embeddings", "/v1/openai/embeddings"], (req, res) => { this.requestHandler.processOpenAIEmbeddingsRequest(req, res); }); diff --git a/src/core/RequestHandler.js b/src/core/RequestHandler.js index ba65309c..c0aacd97 100644 --- a/src/core/RequestHandler.js +++ b/src/core/RequestHandler.js @@ -1042,6 +1042,99 @@ class RequestHandler { } } + // Process OpenAI speech synthesis requests + async processOpenAISpeechRequest(req, res) { + const requestId = this._generateRequestId(); + this._startTrackedRequest(requestId, req, { + apiFormat: "openai", + isStreaming: false, + requestCategory: "generation", + streamMode: null, + }); + this._setResponseApiFormat(res, "openai"); + res.__proxyResponseStreamMode = null; + + try { + let cleanModelName, googleRequest, responseFormat; + try { + const translatedRequest = this.formatConverter.translateOpenAISpeechToGoogle(req.body); + cleanModelName = translatedRequest.cleanModelName; + googleRequest = translatedRequest.googleRequest; + responseFormat = translatedRequest.responseFormat; + } catch (error) { + this.logger.warn( + `[Adapter] OpenAI speech request validation failed: ${error.message}, request ID: ${requestId}` + ); + return this._sendErrorResponse(res, 400, error.message, "invalid_request_error"); + } + + if (!(await this._ensureBrowserBackedRequestReady(res, { waitErrorType: "service_unavailable" }))) { + return; + } + + const usageCount = this.authSwitcher.incrementUsageCount(); + if (usageCount > 0) { + const rotationCountText = + this.config.switchOnUses > 0 ? `${usageCount}/${this.config.switchOnUses}` : `${usageCount}`; + this.logger.info( + `[Request] OpenAI speech generation request - account rotation count: ${rotationCountText} (Current account: ${this.currentAuthIndex}), request ID: ${requestId}` + ); + if (this.authSwitcher.shouldSwitchByUsage()) { + this.needsSwitchingAfterRequest = true; + } + } + + const proxyRequest = { + body: JSON.stringify(googleRequest), + headers: { "Content-Type": "application/json" }, + is_generative: true, + method: "POST", + path: `/v1beta/models/${cleanModelName}:generateContent`, + query_params: {}, + request_id: requestId, + response_format: responseFormat, + response_transform: "geminiTtsToOpenAIAudio", + streaming_mode: "fake", + tracking_model: cleanModelName, + }; + this._initializeProxyRequestAttempt(proxyRequest); + this._updateTrackedRequest(requestId, { + isStreaming: false, + model: cleanModelName, + path: proxyRequest.path, + requestCategory: "generation", + streamMode: null, + }); + + try { + const messageQueue = this.connectionRegistry.createMessageQueue( + requestId, + this.currentAuthIndex, + proxyRequest.request_attempt_id + ); + this._setupClientDisconnectHandler(res, requestId); + await this._handleNonStreamResponse(proxyRequest, messageQueue, req, res); + } catch (error) { + this._handleQueueTimeout(error, requestId); + this._handleRequestError(error, res, requestId); + } finally { + this.connectionRegistry.removeMessageQueue(requestId, "request_complete"); + if (this.needsSwitchingAfterRequest) { + this.logger.info( + `[Auth] Rotation count reached switching threshold (${this.authSwitcher.usageCount}/${this.config.switchOnUses}), will automatically switch account in background...` + ); + this.authSwitcher.switchToNextAuth().catch(error => { + this.logger.error(`[Auth] Background account switching task failed: ${error.message}`); + }); + this.needsSwitchingAfterRequest = false; + } + if (!res.writableEnded) res.end(); + } + } finally { + this._finalizeTrackedRequest(requestId, res); + } + } + // Process File Upload requests async processUploadRequest(req, res) { const requestId = this._generateRequestId(); @@ -3160,11 +3253,17 @@ class RequestHandler { const fullBodyBuffer = Buffer.concat(chunks); let responseBodyBuffer = fullBodyBuffer; - try { - const fullResponse = JSON.parse(responseBodyBuffer.toString()); - this._logGeminiNativeResponseDebug(fullResponse, "non-stream"); - } catch (e) { - // Ignore JSON parsing errors for finish reason + if (proxyRequest.response_transform === "geminiTtsToOpenAIAudio") { + this.logger.debug( + `[Request] Received Gemini TTS response body (${responseBodyBuffer.length} bytes), request ID: ${proxyRequest.request_id}` + ); + } else { + try { + const fullResponse = JSON.parse(responseBodyBuffer.toString()); + this._logGeminiNativeResponseDebug(fullResponse, "non-stream"); + } catch (e) { + // Ignore JSON parsing errors for finish reason + } } if (proxyRequest.response_transform === "batchEmbedToEmbedContent") { @@ -3177,6 +3276,46 @@ class RequestHandler { } } + if (proxyRequest.response_transform === "geminiTtsToOpenAIAudio") { + try { + const upstreamStatus = Number(headerMessage.status || 200); + if (upstreamStatus < 200 || upstreamStatus >= 300) { + throw new Error(`Gemini returned unexpected status ${upstreamStatus}.`); + } + + let googleResponse; + try { + googleResponse = JSON.parse(responseBodyBuffer.toString()); + } catch { + throw new Error("Gemini response was not valid JSON."); + } + const { audioBuffer, contentType } = this.formatConverter.convertGoogleToOpenAISpeech( + googleResponse, + proxyRequest.response_format + ); + res.status(200).set({ + "Cache-Control": "no-store", + "Content-Length": String(audioBuffer.length), + "Content-Type": contentType, + }); + res.send(audioBuffer); + this.logger.info( + `✅ [Request] Response completed (OpenAI speech, ${proxyRequest.response_format}), request ID: ${proxyRequest.request_id}` + ); + } catch (error) { + this.logger.error( + `❌ [Adapter] Failed to decode Gemini speech response: ${error.message}, request ID: ${proxyRequest.request_id}` + ); + this._sendErrorResponse( + res, + 502, + `Failed to decode audio from Gemini response: ${error.message}`, + "api_error" + ); + } + return; + } + this._setResponseHeaders(res, headerMessage, req); // Ensure Content-Type is set (Express defaults Buffer to application/octet-stream) diff --git a/src/utils/AudioUtils.js b/src/utils/AudioUtils.js new file mode 100644 index 00000000..0582de23 --- /dev/null +++ b/src/utils/AudioUtils.js @@ -0,0 +1,205 @@ +/** + * File: src/utils/AudioUtils.js + * Description: Helpers for decoding Gemini audio responses and packaging raw PCM as WAV + */ + +const DEFAULT_PCM_BITS_PER_SAMPLE = 16; +const DEFAULT_PCM_CHANNELS = 1; +const DEFAULT_PCM_SAMPLE_RATE = 24000; + +function extractGeminiInlineAudio(googleResponse) { + const candidates = googleResponse?.candidates; + if (!Array.isArray(candidates)) { + throw new Error("Gemini response did not contain an audio candidate."); + } + + let foundInlineData = false; + for (const candidate of candidates) { + const parts = candidate?.content?.parts; + if (!Array.isArray(parts)) continue; + + for (const part of parts) { + const inlineData = part?.inlineData; + if (!inlineData || typeof inlineData !== "object") continue; + foundInlineData = true; + + if ( + typeof inlineData.mimeType === "string" && + inlineData.mimeType.toLowerCase().startsWith("audio/") && + typeof inlineData.data === "string" && + inlineData.data.length > 0 + ) { + return { + data: inlineData.data, + mimeType: inlineData.mimeType, + }; + } + } + } + + if (foundInlineData) { + throw new Error("Gemini response contained malformed inline audio data."); + } + throw new Error("Gemini response did not contain inline audio data."); +} + +function decodeBase64Audio(data) { + const compactData = data.replace(/\s/g, ""); + if ( + compactData.length === 0 || + compactData.length % 4 === 1 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(compactData) || + (compactData.includes("=") && compactData.length % 4 !== 0) || + (compactData.includes("=") && compactData.indexOf("=") < compactData.length - 2) + ) { + throw new Error("Gemini response contained invalid base64 audio data."); + } + + const audioBuffer = Buffer.from(compactData, "base64"); + const canonicalInput = compactData.replace(/=+$/, ""); + const canonicalDecoded = audioBuffer.toString("base64").replace(/=+$/, ""); + if (audioBuffer.length === 0 || canonicalInput !== canonicalDecoded) { + throw new Error("Gemini response contained invalid base64 audio data."); + } + + return audioBuffer; +} + +function parseMimeParameters(parameterParts) { + const parameters = new Map(); + for (const part of parameterParts) { + const separatorIndex = part.indexOf("="); + if (separatorIndex === -1) continue; + const key = part.slice(0, separatorIndex).trim().toLowerCase(); + const value = part + .slice(separatorIndex + 1) + .trim() + .replace(/^"|"$/g, ""); + if (key) parameters.set(key, value); + } + return parameters; +} + +function parsePcmMimeType(mimeType) { + if (typeof mimeType !== "string" || /[\r\n]/.test(mimeType)) { + throw new Error("Gemini response contained an invalid audio MIME type."); + } + + const [rawMediaType, ...parameterParts] = mimeType.split(";"); + const mediaType = rawMediaType.trim().toLowerCase(); + const subtypeMatch = mediaType.match(/^audio\/l(\d+)$/); + const supportedPcmMediaTypes = new Set(["audio/pcm", "audio/raw", "audio/x-pcm", "audio/x-raw"]); + if (!subtypeMatch && !supportedPcmMediaTypes.has(mediaType)) { + throw new Error(`Unsupported Gemini audio MIME type: ${rawMediaType.trim() || "unknown"}.`); + } + + const parameters = parseMimeParameters(parameterParts); + const codec = parameters.get("codec"); + if (codec && !["lpcm", "pcm"].includes(codec.toLowerCase())) { + throw new Error(`Unsupported Gemini audio codec: ${codec}.`); + } + + const bitsPerSampleValue = subtypeMatch?.[1] || parameters.get("bits") || DEFAULT_PCM_BITS_PER_SAMPLE; + const channelsValue = parameters.get("channels") || DEFAULT_PCM_CHANNELS; + const sampleRateValue = parameters.get("rate") || DEFAULT_PCM_SAMPLE_RATE; + const bitsPerSample = Number(bitsPerSampleValue); + const channels = Number(channelsValue); + const sampleRate = Number(sampleRateValue); + + if (!Number.isInteger(bitsPerSample) || bitsPerSample <= 0 || bitsPerSample > 32 || bitsPerSample % 8 !== 0) { + throw new Error("Gemini response contained an invalid PCM bit depth."); + } + if (!Number.isInteger(channels) || channels <= 0 || channels > 65535) { + throw new Error("Gemini response contained an invalid PCM channel count."); + } + if (!Number.isInteger(sampleRate) || sampleRate <= 0 || sampleRate > 0xffffffff) { + throw new Error("Gemini response contained an invalid PCM sample rate."); + } + + return { + bitsPerSample, + channels, + contentType: `audio/L${bitsPerSample};codec=pcm;rate=${sampleRate}${ + channels === DEFAULT_PCM_CHANNELS ? "" : `;channels=${channels}` + }`, + sampleRate, + }; +} + +function wrapPcmInWav(pcmBuffer, metadata) { + const { bitsPerSample, channels, sampleRate } = metadata; + const blockAlign = (channels * bitsPerSample) / 8; + const byteRate = sampleRate * blockAlign; + if (blockAlign > 0xffff || byteRate > 0xffffffff || pcmBuffer.length > 0xffffffff - 44) { + throw new Error("Gemini PCM audio metadata exceeds WAV format limits."); + } + if (pcmBuffer.length % blockAlign !== 0) { + throw new Error("Gemini PCM audio length is not aligned to its sample format."); + } + + const paddingLength = pcmBuffer.length % 2; + const wavBuffer = Buffer.alloc(44 + pcmBuffer.length + paddingLength); + + wavBuffer.write("RIFF", 0, "ascii"); + wavBuffer.writeUInt32LE(wavBuffer.length - 8, 4); + wavBuffer.write("WAVE", 8, "ascii"); + wavBuffer.write("fmt ", 12, "ascii"); + wavBuffer.writeUInt32LE(16, 16); + wavBuffer.writeUInt16LE(1, 20); + wavBuffer.writeUInt16LE(channels, 22); + wavBuffer.writeUInt32LE(sampleRate, 24); + wavBuffer.writeUInt32LE(byteRate, 28); + wavBuffer.writeUInt16LE(blockAlign, 32); + wavBuffer.writeUInt16LE(bitsPerSample, 34); + wavBuffer.write("data", 36, "ascii"); + wavBuffer.writeUInt32LE(pcmBuffer.length, 40); + pcmBuffer.copy(wavBuffer, 44); + + return wavBuffer; +} + +function isWavMimeType(mimeType) { + const mediaType = String(mimeType).split(";", 1)[0].trim().toLowerCase(); + return mediaType === "audio/wav" || mediaType === "audio/wave" || mediaType === "audio/x-wav"; +} + +function convertGeminiAudioResponse(googleResponse, responseFormat) { + const inlineAudio = extractGeminiInlineAudio(googleResponse); + const audioBuffer = decodeBase64Audio(inlineAudio.data); + + if (responseFormat === "wav" && isWavMimeType(inlineAudio.mimeType)) { + if ( + audioBuffer.length < 12 || + audioBuffer.toString("ascii", 0, 4) !== "RIFF" || + audioBuffer.toString("ascii", 8, 12) !== "WAVE" + ) { + throw new Error("Gemini response contained malformed WAV audio data."); + } + return { audioBuffer, contentType: "audio/wav" }; + } + + const pcmMetadata = parsePcmMimeType(inlineAudio.mimeType); + if (responseFormat === "pcm") { + return { + audioBuffer, + contentType: pcmMetadata.contentType, + }; + } + + if (responseFormat === "wav") { + return { + audioBuffer: wrapPcmInWav(audioBuffer, pcmMetadata), + contentType: "audio/wav", + }; + } + + throw new Error(`Unsupported audio response format: ${responseFormat}.`); +} + +module.exports = { + convertGeminiAudioResponse, + decodeBase64Audio, + extractGeminiInlineAudio, + parsePcmMimeType, + wrapPcmInWav, +}; diff --git a/test/audioSpeech.test.js b/test/audioSpeech.test.js new file mode 100644 index 00000000..c8baff2a --- /dev/null +++ b/test/audioSpeech.test.js @@ -0,0 +1,464 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const FormatConverter = require("../src/core/FormatConverter"); +const ProxyServerSystem = require("../src/core/ProxyServerSystem"); +const RequestHandler = require("../src/core/RequestHandler"); + +const logger = { + debug() {}, + error() {}, + info() {}, + warn() {}, +}; + +function createMockResponse() { + return { + body: null, + destroyed: false, + end(body) { + if (body !== undefined) this.body = body; + this.headersSent = true; + this.writableEnded = true; + return this; + }, + get(name) { + return this.headers[String(name).toLowerCase()]; + }, + getHeader(name) { + return this.headers[String(name).toLowerCase()]; + }, + headers: {}, + headersSent: false, + on() { + return this; + }, + send(body) { + this.body = body; + this.headersSent = true; + this.writableEnded = true; + return this; + }, + set(nameOrHeaders, value) { + if (typeof nameOrHeaders === "string") { + this.headers[nameOrHeaders.toLowerCase()] = value; + } else { + for (const [name, headerValue] of Object.entries(nameOrHeaders)) { + this.headers[name.toLowerCase()] = headerValue; + } + } + return this; + }, + setHeader(name, value) { + this.headers[String(name).toLowerCase()] = value; + }, + socket: { destroyed: false, writable: true }, + status(statusCode) { + this.statusCode = statusCode; + return this; + }, + statusCode: 200, + type(contentType) { + this.headers["content-type"] = contentType; + return this; + }, + writableEnded: false, + write() { + this.headersSent = true; + return true; + }, + }; +} + +function createRequest(body) { + return { + body, + headers: { "content-type": "application/json" }, + method: "POST", + path: "/v1/audio/speech", + query: {}, + }; +} + +function createQueue(messages) { + const pendingMessages = [...messages]; + return { + async dequeue() { + assert.notEqual(pendingMessages.length, 0, "test queue ran out of messages"); + return pendingMessages.shift(); + }, + }; +} + +function createHandler(executeResultFactory) { + const formatConverter = new FormatConverter(logger, { config: {} }); + const handler = Object.create(RequestHandler.prototype); + handler.authSwitcher = { + currentAuthIndex: 0, + failureCount: 0, + handleRequestFailureAndSwitch: async () => {}, + incrementUsageCount: () => 1, + shouldSwitchByUsage: () => false, + switchToNextAuth: async () => {}, + usageCount: 1, + }; + handler.browserManager = { notifyUserActivity() {} }; + handler.config = { + maxRetries: 1, + retryDelay: 0, + switchOnUses: 0, + }; + handler.connectionRegistry = { + createMessageQueue: () => ({}), + removeMessageQueue() {}, + }; + handler.formatConverter = formatConverter; + handler.logger = logger; + handler.needsSwitchingAfterRequest = false; + handler.serverSystem = { + usageStatsService: null, + webRoutes: { authRoutes: { getClientIP: () => "127.0.0.1" } }, + }; + handler.timeouts = { FAKE_STREAM: 1000, STREAM_CHUNK: 1000 }; + handler._ensureBrowserBackedRequestReady = async () => true; + handler._executeRequestWithRetries = executeResultFactory; + handler._handleQueueTimeout = () => {}; + handler._setupClientDisconnectHandler = () => {}; + return handler; +} + +function createGeminiAudioResponse(pcmBuffer, mimeType = "audio/L16;codec=pcm;rate=24000") { + return { + candidates: [ + { + content: { + parts: [ + { + inlineData: { + data: pcmBuffer.toString("base64"), + mimeType, + }, + }, + ], + }, + }, + ], + }; +} + +test("registers the OpenAI speech route before the catch-all without replacing chat completions", () => { + let chatCalls = 0; + let speechCalls = 0; + const system = Object.create(ProxyServerSystem.prototype); + system.config = { apiKeys: [], modelList: [] }; + system.logger = logger; + system.requestHandler = { + processClaudeCountTokens() {}, + processClaudeRequest() {}, + processOpenAIEmbeddingsRequest() {}, + processOpenAIRequest() { + chatCalls += 1; + }, + processOpenAIResponseInputTokens() {}, + processOpenAIResponseRequest() {}, + processOpenAISpeechRequest() { + speechCalls += 1; + }, + processRequest() {}, + processUploadRequest() {}, + }; + system.webRoutes = { + authRoutes: { getClientIP: () => "127.0.0.1" }, + setupSession() {}, + }; + + const app = system._createExpressApp(); + const routeLayers = app._router.stack.filter(layer => layer.route); + const speechIndex = routeLayers.findIndex(layer => layer.route.path === "/v1/audio/speech"); + const chatIndex = routeLayers.findIndex(layer => layer.route.path === "/v1/chat/completions"); + const catchAllIndex = routeLayers.findIndex(layer => String(layer.route.path).includes("(.*)")); + + assert.ok(speechIndex >= 0); + assert.ok(chatIndex >= 0); + assert.ok(catchAllIndex > speechIndex); + + routeLayers[speechIndex].route.stack[0].handle({}, {}); + routeLayers[chatIndex].route.stack[0].handle({}, {}); + assert.equal(speechCalls, 1); + assert.equal(chatCalls, 1); +}); + +test("validates required OpenAI speech fields before checking browser readiness", async t => { + for (const missingField of ["model", "input", "voice"]) { + await t.test(`missing ${missingField}`, async () => { + let readinessChecks = 0; + const handler = createHandler(async () => { + throw new Error("upstream should not be called"); + }); + handler._ensureBrowserBackedRequestReady = async () => { + readinessChecks += 1; + return true; + }; + const body = { + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }; + delete body[missingField]; + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest(createRequest(body), res); + + assert.equal(res.statusCode, 400); + assert.equal(readinessChecks, 0); + const errorPayload = JSON.parse(res.body); + assert.equal(errorPayload.error.type, "invalid_request_error"); + assert.match(errorPayload.error.message, new RegExp(missingField)); + }); + } +}); + +test("builds the Gemini native TTS payload and returns binary WAV with MIME-derived metadata", async () => { + const pcmBuffer = Buffer.from([0x00, 0x00, 0x10, 0x00, 0xf0, 0xff, 0x00, 0x00]); + const googleResponse = createGeminiAudioResponse(pcmBuffer, "audio/L16;codec=pcm;rate=16000;channels=1"); + let capturedProxyRequest; + const responseQueue = createQueue([ + { data: JSON.stringify(googleResponse), event_type: "chunk" }, + { type: "STREAM_END" }, + ]); + const handler = createHandler(async proxyRequest => { + capturedProxyRequest = proxyRequest; + return { + message: { headers: { "content-type": "application/json" }, status: 200 }, + queue: responseQueue, + success: true, + }; + }); + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest( + createRequest({ + input: "Hello, this is a text to speech test.", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }), + res + ); + + assert.equal(capturedProxyRequest.path, "/v1beta/models/gemini-3.1-flash-tts-preview:generateContent"); + assert.equal(capturedProxyRequest.is_generative, true); + assert.equal(capturedProxyRequest.streaming_mode, "fake"); + assert.deepEqual(JSON.parse(capturedProxyRequest.body), { + contents: [ + { + parts: [{ text: "Hello, this is a text to speech test." }], + role: "user", + }, + ], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { voiceName: "Kore" }, + }, + }, + }, + }); + + assert.equal(res.statusCode, 200); + assert.equal(res.headers["content-type"], "audio/wav"); + assert.equal(Number(res.headers["content-length"]), res.body.length); + assert.ok(Buffer.isBuffer(res.body)); + assert.equal(res.body.toString("ascii", 0, 4), "RIFF"); + assert.equal(res.body.toString("ascii", 8, 12), "WAVE"); + assert.equal(res.body.readUInt16LE(20), 1); + assert.equal(res.body.readUInt16LE(22), 1); + assert.equal(res.body.readUInt32LE(24), 16000); + assert.equal(res.body.readUInt16LE(34), 16); + assert.equal(res.body.readUInt32LE(40), pcmBuffer.length); + assert.deepEqual(res.body.subarray(44), pcmBuffer); +}); + +test("returns raw PCM bytes with an accurate content type", async () => { + const pcmBuffer = Buffer.from([0x01, 0x00, 0x02, 0x00]); + const responseQueue = createQueue([ + { data: JSON.stringify(createGeminiAudioResponse(pcmBuffer)), event_type: "chunk" }, + { type: "STREAM_END" }, + ]); + const handler = createHandler(async () => ({ + message: { headers: {}, status: 200 }, + queue: responseQueue, + success: true, + })); + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest( + createRequest({ + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "pcm", + voice: "Kore", + }), + res + ); + + assert.equal(res.headers["content-type"], "audio/L16;codec=pcm;rate=24000"); + assert.equal(Number(res.headers["content-length"]), pcmBuffer.length); + assert.deepEqual(res.body, pcmBuffer); +}); + +test("returns an OpenAI-style 400 for unsupported formats and fields", async t => { + const invalidBodies = [ + { + expected: "response_format", + request: { + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "mp3", + voice: "Kore", + }, + }, + { + expected: "speed", + request: { + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + speed: 1.25, + voice: "Kore", + }, + }, + { + expected: "instructions", + request: { + input: "Hello", + instructions: "Speak slowly", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }, + }, + { + expected: "stream", + request: { + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + stream: false, + voice: "Kore", + }, + }, + { + expected: "stream_format", + request: { + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + stream_format: "audio", + voice: "Kore", + }, + }, + ]; + + for (const { expected, request } of invalidBodies) { + await t.test(expected, async () => { + const handler = createHandler(async () => { + throw new Error("upstream should not be called"); + }); + const res = createMockResponse(); + await handler.processOpenAISpeechRequest(createRequest(request), res); + assert.equal(res.statusCode, 400); + const errorPayload = JSON.parse(res.body); + assert.equal(errorPayload.error.type, "invalid_request_error"); + assert.match(errorPayload.error.message, new RegExp(expected)); + }); + } +}); + +test("returns a 502 OpenAI error when Gemini omits inline audio", async () => { + const responseQueue = createQueue([ + { + data: JSON.stringify({ candidates: [{ content: { parts: [{ text: "No audio" }] } }] }), + event_type: "chunk", + }, + { type: "STREAM_END" }, + ]); + const handler = createHandler(async () => ({ + message: { headers: {}, status: 200 }, + queue: responseQueue, + success: true, + })); + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest( + createRequest({ + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }), + res + ); + + assert.equal(res.statusCode, 502); + const errorPayload = JSON.parse(res.body); + assert.equal(errorPayload.error.type, "api_error"); + assert.match(errorPayload.error.message, /inline audio data/); +}); + +test("returns a safe 502 OpenAI error for malformed Gemini JSON", async () => { + const responseQueue = createQueue([ + { data: "not-json-with-private-upstream-content", event_type: "chunk" }, + { type: "STREAM_END" }, + ]); + const handler = createHandler(async () => ({ + message: { headers: {}, status: 200 }, + queue: responseQueue, + success: true, + })); + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest( + createRequest({ + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }), + res + ); + + assert.equal(res.statusCode, 502); + const errorPayload = JSON.parse(res.body); + assert.match(errorPayload.error.message, /not valid JSON/); + assert.doesNotMatch(errorPayload.error.message, /private-upstream-content/); +}); + +test("preserves upstream failure status through the shared retry pipeline", async () => { + let accountFailureCalls = 0; + const handler = createHandler(async () => ({ + error: { message: "Upstream quota exhausted", status: 429 }, + success: false, + })); + handler.authSwitcher.handleRequestFailureAndSwitch = async () => { + accountFailureCalls += 1; + }; + const res = createMockResponse(); + + await handler.processOpenAISpeechRequest( + createRequest({ + input: "Hello", + model: "gemini-3.1-flash-tts-preview", + response_format: "wav", + voice: "Kore", + }), + res + ); + + assert.equal(res.statusCode, 429); + assert.equal(accountFailureCalls, 1); + const errorPayload = JSON.parse(res.body); + assert.equal(errorPayload.error.message, "Upstream quota exhausted"); +});