From 17bc052512c62bf209680ec6f5224731e572b152 Mon Sep 17 00:00:00 2001 From: Santiago Date: Fri, 14 Aug 2026 21:13:56 -0300 Subject: [PATCH 1/2] Add OpenRouter transcription support --- .env.example | 3 + README.md | 12 + class-transcriber-backend-prd.md | 2 + ...scriber-backend-tech-stack-requirements.md | 3 +- class-transcriber-frontend-prd.md | 1 + class-transcriber-shared-api-contract.md | 2 + .../OpenRouterTranscriptionEngineTests.cs | 278 ++++++++++++++++++ src/ClassTranscriber.Api/Contracts/Enums.cs | 3 +- src/ClassTranscriber.Api/Program.cs | 18 ++ .../OpenRouterTranscriptionEngine.cs | 197 +++++++++++++ .../OpenAiAudioTranscriptionHelper.cs | 94 +++++- .../OpenRouterSpeechToTextClient.cs | 61 ++++ src/ClassTranscriber.Api/appsettings.json | 8 + src/frontend/e2e/frontend.spec.ts | 31 ++ .../src/config/transcriptionOptions.ts | 1 + src/frontend/src/pages/SettingsPage.tsx | 24 +- .../src/test/utils/transcription.test.ts | 8 + src/frontend/src/utils/transcription.ts | 2 + 18 files changed, 733 insertions(+), 15 deletions(-) create mode 100644 src/ClassTranscriber.Api.Tests/OpenRouterTranscriptionEngineTests.cs create mode 100644 src/ClassTranscriber.Api/Transcription/OpenRouterTranscriptionEngine.cs create mode 100644 src/ClassTranscriber.Api/Transcription/SpeechToText/OpenRouterSpeechToTextClient.cs diff --git a/.env.example b/.env.example index ab94238..1a7f500 100644 --- a/.env.example +++ b/.env.example @@ -15,5 +15,8 @@ TRANSCRIPTION_CONCURRENCY=1 ENABLE_GPU=false DEFAULT_MODEL=small +# Optional hosted speech-to-text +Transcription__OpenRouter__ApiKey=YOUR_OPENROUTER_API_KEY + # Media tooling FFMPEG_PATH=/usr/bin/ffmpeg diff --git a/README.md b/README.md index 00ef67b..d17e2e8 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,18 @@ Uses the supported OpenVINO Whisper sidecar for Intel GPU acceleration. The back OpenVINO Whisper models are stored under `data/models/openvino-genai/` and can be downloaded on first use or through `POST /api/settings/models/manage`. +#### OpenRouter (hosted speech-to-text) + +OpenRouter is available as a first-class remote engine when a server-side API key is configured. TranscriptLab discovers the current speech-to-text model catalog at runtime, stores the selected model with each project, and keeps remote models out of the local Model Manager. + +Configure the key with an environment variable so it is never committed: + +```bash +export Transcription__OpenRouter__ApiKey=YOUR_OPENROUTER_API_KEY +``` + +Optional settings can override `Transcription__OpenRouter__BaseUrl`, `FallbackModels`, and `TimeoutSeconds`. `BaseUrl` must be an absolute HTTPS URL. Prepared audio is uploaded to OpenRouter's hosted `/api/v1/audio/transcriptions` endpoint. + Configuration for all WhisperNet engines in `appsettings.json`: ```json { diff --git a/class-transcriber-backend-prd.md b/class-transcriber-backend-prd.md index 2ade599..4bc76d4 100644 --- a/class-transcriber-backend-prd.md +++ b/class-transcriber-backend-prd.md @@ -341,6 +341,7 @@ Current backend extension points may additionally expose: - `OpenVinoWhisperSidecar` - `OnnxWhisper` - `OpenAiCompatible` +- `OpenRouter` Implementation note: - `SherpaOnnx` may run on a local .NET runtime path or isolated helper worker as long as it stays behind the transcription engine abstraction. @@ -349,6 +350,7 @@ Implementation note: - `OpenVinoWhisperSidecar` runs through a long-lived Python FastAPI sidecar with an OpenAI-compatible API. The sidecar manages its own model downloads. The C# engine uses `ISpeechToTextClient` (Microsoft.Extensions.AI experimental) internally. It is the recommended OpenVINO engine for deployments with local GPU hardware. - `OnnxWhisper` is a reserved placeholder for a future native .NET ONNX Whisper engine. It reports unavailable and must not be used in production. - `OpenAiCompatible` proxies transcription to any configured OpenAI-compatible API. It must not appear in the engine selector when `BaseUrl` is not configured. +- `OpenRouter` sends prepared audio to OpenRouter's hosted speech-to-text endpoint and discovers transcription-capable models dynamically. It must not appear in the engine selector when its server-side API key is not configured, must use the model stored in each project's settings, and must remain outside the local filesystem Model Manager. ## Suggested model values for MVP - `tiny` diff --git a/class-transcriber-backend-tech-stack-requirements.md b/class-transcriber-backend-tech-stack-requirements.md index 13ae7ee..2c7a4e1 100644 --- a/class-transcriber-backend-tech-stack-requirements.md +++ b/class-transcriber-backend-tech-stack-requirements.md @@ -58,8 +58,9 @@ - **SherpaOnnx** via the official local **.NET runtime/package** is approved behind the engine abstraction; running it through an isolated helper worker process is allowed when needed for cancellation or runtime isolation - **Whisper.net** managed library with **Whisper.net.Runtime** (CPU), **Whisper.net.Runtime.Cuda** (NVIDIA GPU), and **Whisper.net.Runtime.CoreML** (native macOS Apple Silicon) runtimes are approved behind the engine abstraction, but CPU, CUDA, and CoreML execution must run through isolated helper worker processes because Whisper.net runtime loading is process-global. CoreML is only approved for native macOS ARM64 runs; Docker/Linux on macOS should be treated as CPU-only unless it calls a native host sidecar. - A separate **Python FastAPI sidecar** backed by **openvino-genai**, **fastapi**, and **uvicorn** is approved for the `OpenVinoWhisperSidecar` engine; the sidecar runs as a long-lived localhost HTTP server managed by the API process (spawned lazily on first use, killed on shutdown), caches loaded Whisper models in memory between jobs, and avoids the native library version conflict between the .NET `Whisper.net.Runtime.OpenVino` binding and newer OpenVINO Python package installs; the sidecar exposes an OpenAI-compatible `/v1/audio/transcriptions` endpoint plus a model management API with SSE-streamed download progress; it manages its own model downloads independently of the C# download infrastructure; the C# engine communicates with it via `ISpeechToTextClient` from `Microsoft.Extensions.AI.Abstractions` -- **Microsoft.Extensions.AI.Abstractions** is approved as an internal calling abstraction for HTTP-based transcription engines (`OpenVinoWhisperSidecar`, `OpenAiCompatible`); it is used as an implementation detail within the engine and must not replace `IRegisteredTranscriptionEngine` as the public engine contract; the `MEAI001` experimental diagnostic should be suppressed project-wide via `` in the `.csproj` file when the package is added +- **Microsoft.Extensions.AI.Abstractions** is approved as an internal calling abstraction for HTTP-based transcription engines (`OpenVinoWhisperSidecar`, `OpenAiCompatible`, `OpenRouter`); it is used as an implementation detail within the engine and must not replace `IRegisteredTranscriptionEngine` as the public engine contract; the `MEAI001` experimental diagnostic should be suppressed project-wide via `` in the `.csproj` file when the package is added - A generic **`OpenAiCompatible`** proxy engine is approved that forwards transcription requests to any OpenAI-compatible `/v1/audio/transcriptions` endpoint; it shares HTTP multipart construction code with the `OpenVinoWhisperSidecar` client and is hidden from the engine selector when `BaseUrl` is not configured +- A first-class **`OpenRouter`** engine is approved for hosted speech-to-text through `/api/v1/audio/transcriptions`; it reuses the shared OpenAI-compatible multipart helper, authenticates only with a server-side API key, discovers models with `GET /api/v1/models?output_modalities=transcription`, uses the project-selected model, retries with the default JSON response when a provider rejects `verbose_json`, and stays outside local model download/install management - An **`OnnxWhisper`** engine placeholder value is approved in the `TranscriptionEngine` enum; do not add `Microsoft.ML.OnnxRuntime` or any ONNX inference package until the full implementation is planned - SSE (Server-Sent Events) streaming is the approved pattern for long-running sidecar model download progress; the C# caller must consume the SSE stream until `status=complete` or `status=error` - Keep engine-specific logic behind a dedicated transcription service and engine interface diff --git a/class-transcriber-frontend-prd.md b/class-transcriber-frontend-prd.md index ce23fae..cc70ee4 100644 --- a/class-transcriber-frontend-prd.md +++ b/class-transcriber-frontend-prd.md @@ -533,6 +533,7 @@ The app must expose a global settings page for future uploads. - upload modal should start from global defaults but allow override per batch - batch and retry flows should allow diarization to be enabled or disabled per request - engine selectors in settings, upload, retry, diagnostics, and model management should surface runtime-available engines from the backend, including Intel GPU options such as `OpenVinoWhisperSidecar` and native Apple Silicon options such as `WhisperNetCoreML` when those runtimes are installed +- when `OpenRouter` is available, settings, upload, and retry selectors should show its backend-discovered transcription models and clearly disclose that audio is sent to a remote provider; OpenRouter models must not appear in the local filesystem Model Manager - the settings page should also expose a model manager below the defaults form in a vertical stack layout - the model manager should show known engine/model combinations, local install state, install path, and the latest probe result - installed models should be probed on page load so runtime problems are visible without queueing an upload diff --git a/class-transcriber-shared-api-contract.md b/class-transcriber-shared-api-contract.md index 6daf5a6..acc69d9 100644 --- a/class-transcriber-shared-api-contract.md +++ b/class-transcriber-shared-api-contract.md @@ -119,6 +119,7 @@ WhisperNetCoreML OpenVinoWhisperSidecar OnnxWhisper OpenAiCompatible +OpenRouter ``` Implementation note: @@ -131,6 +132,7 @@ Implementation note: - `OpenVinoWhisperSidecar` uses a long-lived Python FastAPI sidecar backed by `openvino_genai`. The sidecar exposes an OpenAI-compatible `/v1/audio/transcriptions` endpoint and a model management API. It caches loaded Whisper pipelines in memory between jobs and manages its own model downloads internally. The C# engine communicates with it via `ISpeechToTextClient` (Microsoft.Extensions.AI experimental). It is the recommended OpenVINO engine for deployments with a local GPU. - `OnnxWhisper` is a reserved placeholder for a future native .NET ONNX Whisper engine using `Microsoft.ML.OnnxRuntime`. It is not yet implemented and reports unavailable in all current releases. - `OpenAiCompatible` is a generic proxy engine that forwards transcription to any external service that exposes an OpenAI-compatible `/v1/audio/transcriptions` endpoint (e.g., the local `OpenVinoWhisperSidecar`, Whisper.cpp server, Ollama). It is hidden from the engine selector when `BaseUrl` is not configured. +- `OpenRouter` is a first-class hosted speech-to-text engine using OpenRouter's `/api/v1/audio/transcriptions` endpoint. It is hidden from the engine selector until a server-side API key is configured. Its transcription-capable models are discovered through `/api/v1/models?output_modalities=transcription`, the selected model is stored in each project's existing `model` field, and its remote models are not included in the local Model Manager catalog. - `WhisperNet`, `WhisperNetCuda`, and `WhisperNetCoreML` use shared ggml model files and support auto-download. `WhisperNetCoreML` also requires the matching CoreML encoder `.mlmodelc` package beside the ggml model. - `OpenVinoWhisperSidecar` uses curated pre-exported model directories under `models/openvino-genai/` and supports managed download/redownload/probe from the settings model manager. Download is proxied to the sidecar's own model management API. diff --git a/src/ClassTranscriber.Api.Tests/OpenRouterTranscriptionEngineTests.cs b/src/ClassTranscriber.Api.Tests/OpenRouterTranscriptionEngineTests.cs new file mode 100644 index 0000000..6b0678a --- /dev/null +++ b/src/ClassTranscriber.Api.Tests/OpenRouterTranscriptionEngineTests.cs @@ -0,0 +1,278 @@ +using System.Net; +using System.Net.Http.Json; +using ClassTranscriber.Api.Domain; +using ClassTranscriber.Api.Transcription; +using ClassTranscriber.Api.Transcription.SpeechToText; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace ClassTranscriber.Api.Tests; + +public sealed class OpenRouterTranscriptionEngineTests +{ + [Fact] + public void GetAvailabilityError_ReturnsConfigurationError_WhenApiKeyIsMissing() + { + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var engine = CreateEngine(factory, apiKey: string.Empty); + + var error = engine.GetAvailabilityError(); + + error.Should().Contain("Transcription:OpenRouter:ApiKey"); + } + + [Fact] + public void GetAvailabilityError_RequiresHttpsBaseUrl() + { + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var engine = CreateEngine(factory, baseUrl: "http://openrouter.ai/api/v1"); + + var error = engine.GetAvailabilityError(); + + error.Should().Contain("absolute HTTPS URL"); + } + + [Fact] + public void SupportedModels_FiltersOpenRouterCatalogToTranscriptionModels() + { + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = JsonContent.Create(new + { + data = new[] + { + new { id = "openai/whisper-large-v3" }, + new { id = "openai/gpt-4o-mini-transcribe" }, + }, + }), + }); + var engine = CreateEngine(factory); + + var models = engine.SupportedModels; + + models.Should().Equal("openai/whisper-large-v3", "openai/gpt-4o-mini-transcribe"); + factory.Requests.Should().ContainSingle(); + factory.Requests[0].RequestUri.Should().Be("https://openrouter.ai/api/v1/models?output_modalities=transcription"); + } + + [Fact] + public void SupportedModels_UsesConfiguredFallback_WhenCatalogIsUnavailable() + { + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)); + var engine = CreateEngine(factory); + + var models = engine.SupportedModels; + + models.Should().Equal("openai/whisper-large-v3"); + } + + [Fact] + public async Task TranscribeAsync_UsesProjectModelAndFixedLanguage_AndMapsVerboseSegments() + { + var audioPath = Path.Combine(CreateTempDirectory(), "lecture.wav"); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3]); + var factory = new RecordingHttpClientFactory(request => + { + request.Method.Should().Be(HttpMethod.Post); + request.RequestUri.Should().Be("https://openrouter.ai/api/v1/audio/transcriptions"); + request.Headers.Authorization?.Scheme.Should().Be("Bearer"); + request.Headers.Authorization?.Parameter.Should().Be("test-openrouter-key"); + var multipart = request.Content.Should().BeOfType().Subject; + ReadPartAsync(multipart, "model").GetAwaiter().GetResult().Should().Be("openai/gpt-4o-mini-transcribe"); + ReadPartAsync(multipart, "language").GetAwaiter().GetResult().Should().Be("es"); + ReadPartAsync(multipart, "response_format").GetAwaiter().GetResult().Should().Be("verbose_json"); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = JsonContent.Create(new + { + task = "transcribe", + language = "es", + duration = 2.5, + text = "hola mundo", + segments = new[] + { + new { id = 0, start = 0.25, end = 2.5, text = "hola mundo" }, + }, + }), + }; + }); + var engine = CreateEngine(factory); + + var result = await engine.TranscribeAsync(audioPath, new ProjectSettings + { + Engine = "OpenRouter", + Model = "openai/gpt-4o-mini-transcribe", + LanguageMode = "Fixed", + LanguageCode = "es", + }); + + result.PlainText.Should().Be("hola mundo"); + result.DetectedLanguage.Should().Be("es"); + result.DurationMs.Should().Be(2500); + result.Segments.Should().ContainSingle(); + result.Segments[0].StartMs.Should().Be(250); + result.Segments[0].EndMs.Should().Be(2500); + } + + [Fact] + public async Task TranscribeAsync_RetriesWithJson_WhenProviderRejectsVerboseJson() + { + var audioPath = Path.Combine(CreateTempDirectory(), "lecture.wav"); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3]); + var responseFormats = new List(); + var factory = new RecordingHttpClientFactory(request => + { + var multipart = request.Content.Should().BeOfType().Subject; + var responseFormat = ReadPartAsync(multipart, "response_format").GetAwaiter().GetResult(); + responseFormats.Add(responseFormat); + return responseFormat == "verbose_json" + ? new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = JsonContent.Create(new { error = new { message = "response_format 'verbose_json' is unsupported" } }), + } + : new HttpResponseMessage(HttpStatusCode.OK) + { + Content = JsonContent.Create(new + { + text = "provider fallback", + usage = new { seconds = 4.2 }, + }), + }; + }); + var engine = CreateEngine(factory); + + var result = await engine.TranscribeAsync(audioPath, new ProjectSettings + { + Engine = "OpenRouter", + Model = "deepgram/nova-3", + LanguageMode = "Auto", + }); + + responseFormats.Should().Equal("verbose_json", "json"); + result.PlainText.Should().Be("provider fallback"); + result.Segments.Should().ContainSingle(); + result.Segments[0].EndMs.Should().Be(4200); + result.DurationMs.Should().Be(4200); + } + + [Fact] + public async Task TranscribeAsync_DoesNotRetryUnrelatedBadRequests() + { + var audioPath = Path.Combine(CreateTempDirectory(), "lecture.wav"); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3]); + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = JsonContent.Create(new { error = new { message = "Invalid language" } }), + }); + var engine = CreateEngine(factory); + + var act = () => engine.TranscribeAsync(audioPath, new ProjectSettings + { + Engine = "OpenRouter", + Model = "deepgram/nova-3", + LanguageMode = "Fixed", + LanguageCode = "invalid", + }); + + await act.Should().ThrowAsync(); + factory.Requests.Should().ContainSingle(); + } + + [Fact] + public async Task TranscribeAsync_DoesNotLeakApiKey_WhenOpenRouterReturnsAnError() + { + const string apiKey = "sensitive-openrouter-key"; + var audioPath = Path.Combine(CreateTempDirectory(), "lecture.wav"); + await File.WriteAllBytesAsync(audioPath, [1, 2, 3]); + var factory = new RecordingHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.Unauthorized) + { + Content = JsonContent.Create(new { error = new { message = $"Invalid API key: {apiKey}" } }), + }); + var engine = CreateEngine(factory, apiKey); + + var act = () => engine.TranscribeAsync(audioPath, new ProjectSettings + { + Engine = "OpenRouter", + Model = "openai/whisper-large-v3", + LanguageMode = "Auto", + }); + + var exception = await act.Should().ThrowAsync(); + exception.Which.Message.Should().NotContain(apiKey); + factory.Requests.Should().ContainSingle(); + factory.Requests[0].AuthorizationParameter.Should().Be(apiKey); + } + + private static OpenRouterTranscriptionEngine CreateEngine( + RecordingHttpClientFactory factory, + string apiKey = "test-openrouter-key", + string baseUrl = "https://openrouter.ai/api/v1") + { + factory.AuthorizationParameter = apiKey; + var options = Options.Create(new OpenRouterOptions + { + BaseUrl = baseUrl, + ApiKey = apiKey, + FallbackModels = ["openai/whisper-large-v3"], + }); + var client = new OpenRouterSpeechToTextClient(factory); + return new OpenRouterTranscriptionEngine( + options, + client, + factory, + NullLogger.Instance); + } + + private static async Task ReadPartAsync(MultipartFormDataContent content, string name) + { + var part = content.Single(item => item.Headers.ContentDisposition?.Name?.Trim('"') == name); + return await part.ReadAsStringAsync(); + } + + private static string CreateTempDirectory() + { + var path = Path.Combine(Path.GetTempPath(), $"transcriptlab-openrouter-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(path); + return path; + } + + private sealed class RecordingHttpClientFactory( + Func createResponse) : IHttpClientFactory + { + public List Requests { get; } = []; + public string AuthorizationParameter { get; set; } = "test-openrouter-key"; + + public HttpClient CreateClient(string name) + => new(new RecordingHttpMessageHandler(request => + { + Requests.Add(new RecordedRequest( + request.Method, + request.RequestUri?.ToString(), + request.Headers.Authorization?.Parameter)); + return createResponse(request); + })) + { + BaseAddress = new Uri("https://openrouter.ai/api/v1/"), + DefaultRequestHeaders = + { + Authorization = new("Bearer", AuthorizationParameter), + }, + }; + } + + private sealed record RecordedRequest( + HttpMethod Method, + string? RequestUri, + string? AuthorizationParameter); + + private sealed class RecordingHttpMessageHandler( + Func createResponse) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + => Task.FromResult(createResponse(request)); + } +} diff --git a/src/ClassTranscriber.Api/Contracts/Enums.cs b/src/ClassTranscriber.Api/Contracts/Enums.cs index 09fa207..d2c9651 100644 --- a/src/ClassTranscriber.Api/Contracts/Enums.cs +++ b/src/ClassTranscriber.Api/Contracts/Enums.cs @@ -39,5 +39,6 @@ public enum TranscriptionEngine WhisperNetCoreML, OpenVinoWhisperSidecar, OnnxWhisper, - OpenAiCompatible + OpenAiCompatible, + OpenRouter } diff --git a/src/ClassTranscriber.Api/Program.cs b/src/ClassTranscriber.Api/Program.cs index f91e17c..8d8614e 100644 --- a/src/ClassTranscriber.Api/Program.cs +++ b/src/ClassTranscriber.Api/Program.cs @@ -240,6 +240,24 @@ client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", opts.ApiKey); }); + builder.Services.Configure(builder.Configuration.GetSection("Transcription:OpenRouter")); + builder.Services.AddSingleton(); + builder.Services.AddKeyedSingleton("OpenRouter"); + builder.Services.AddHttpClient(OpenRouterTranscriptionEngine.HttpClientName, (sp, client) => + { + var opts = sp.GetRequiredService>().Value; + if (Uri.TryCreate(opts.BaseUrl.TrimEnd('/') + "/", UriKind.Absolute, out var baseUri) + && string.Equals(baseUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + client.BaseAddress = baseUri; + if (!string.IsNullOrWhiteSpace(opts.ApiKey)) + { + client.DefaultRequestHeaders.Authorization = + new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", opts.ApiKey); + } + } + client.Timeout = TimeSpan.FromSeconds(opts.TimeoutSeconds > 0 ? opts.TimeoutSeconds : 120); + }); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/ClassTranscriber.Api/Transcription/OpenRouterTranscriptionEngine.cs b/src/ClassTranscriber.Api/Transcription/OpenRouterTranscriptionEngine.cs new file mode 100644 index 0000000..21ef4d1 --- /dev/null +++ b/src/ClassTranscriber.Api/Transcription/OpenRouterTranscriptionEngine.cs @@ -0,0 +1,197 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using ClassTranscriber.Api.Contracts; +using ClassTranscriber.Api.Domain; +using ClassTranscriber.Api.Transcription.SpeechToText; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace ClassTranscriber.Api.Transcription; + +public sealed class OpenRouterOptions +{ + public string BaseUrl { get; set; } = "https://openrouter.ai/api/v1"; + public string ApiKey { get; set; } = string.Empty; + public string[] FallbackModels { get; set; } = ["openai/whisper-large-v3"]; + public int TimeoutSeconds { get; set; } = 120; +} + +public sealed class OpenRouterTranscriptionEngine : IRegisteredTranscriptionEngine +{ + public const string HttpClientName = "OpenRouter"; + + private readonly OpenRouterOptions _options; + private readonly ISpeechToTextClient _speechToTextClient; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public OpenRouterTranscriptionEngine( + IOptions options, + [FromKeyedServices("OpenRouter")] ISpeechToTextClient speechToTextClient, + IHttpClientFactory httpClientFactory, + ILogger logger) + { + _options = options.Value; + _speechToTextClient = speechToTextClient; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + public string EngineId => "OpenRouter"; + + public IReadOnlyCollection SupportedModels + { + get + { + if (GetAvailabilityError() is not null) + return []; + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var client = _httpClientFactory.CreateClient(HttpClientName); + using var response = client.GetAsync("models?output_modalities=transcription", cts.Token) + .GetAwaiter().GetResult(); + + if (response.IsSuccessStatusCode) + { + var json = response.Content.ReadAsStringAsync(cts.Token).GetAwaiter().GetResult(); + var catalog = JsonSerializer.Deserialize(json); + var models = catalog?.Data? + .Select(model => model.Id) + .Where(model => !string.IsNullOrWhiteSpace(model)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (models is { Length: > 0 }) + return models; + } + } + catch (HttpRequestException) + { + } + catch (TaskCanceledException) + { + } + catch (JsonException) + { + } + + return GetFallbackModels(); + } + } + + public string? GetAvailabilityError() + { + if (string.IsNullOrWhiteSpace(_options.BaseUrl)) + return "OpenRouter engine requires Transcription:OpenRouter:BaseUrl to be set."; + if (!Uri.TryCreate(_options.BaseUrl, UriKind.Absolute, out var baseUri) + || !string.Equals(baseUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + return "OpenRouter engine requires Transcription:OpenRouter:BaseUrl to be an absolute HTTPS URL."; + } + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + return "OpenRouter engine requires Transcription:OpenRouter:ApiKey to be set."; + if (GetFallbackModels().Count == 0) + return "OpenRouter engine requires at least one Transcription:OpenRouter:FallbackModels entry."; + return null; + } + + public string? GetProbeError() + { + var availabilityError = GetAvailabilityError(); + if (availabilityError is not null) + return availabilityError; + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var client = _httpClientFactory.CreateClient(HttpClientName); + using var response = client.GetAsync("models?output_modalities=transcription", cts.Token) + .GetAwaiter().GetResult(); + return response.IsSuccessStatusCode + ? null + : $"OpenRouter models endpoint returned HTTP {(int)response.StatusCode}."; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + return $"OpenRouter models endpoint is not reachable: {ex.Message}"; + } + } + + public async Task TranscribeAsync( + string audioPath, + ProjectSettings settings, + CancellationToken ct = default) + { + var availabilityError = GetAvailabilityError(); + if (availabilityError is not null) + throw new InvalidOperationException(availabilityError); + + _logger.LogInformation( + "Starting {Engine} transcription for {AudioPath} with model {Model}", + EngineId, + audioPath, + settings.Model); + + await using var audioStream = File.OpenRead(audioPath); + var speechOptions = new SpeechToTextOptions { ModelId = settings.Model }; + if (string.Equals(settings.LanguageMode, "Fixed", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(settings.LanguageCode)) + { + speechOptions.SpeechLanguage = settings.LanguageCode; + } + + var response = await _speechToTextClient.GetTextAsync(audioStream, speechOptions, ct); + var result = MapResponse(response); + + _logger.LogInformation( + "{Engine} transcription completed: {SegmentCount} segments", + EngineId, + result.Segments.Length); + + return result; + } + + private IReadOnlyCollection GetFallbackModels() + => _options.FallbackModels + .Where(model => !string.IsNullOrWhiteSpace(model)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + private static TranscriptionResult MapResponse(SpeechToTextResponse response) + { + if (response.RawRepresentation is not OpenAiVerboseTranscriptionResponse raw) + { + var fallbackSegments = string.IsNullOrWhiteSpace(response.Text) + ? [] + : new[] { new TranscriptSegmentDto { StartMs = 0, EndMs = 0, Text = response.Text } }; + return new TranscriptionResult(response.Text, fallbackSegments, null, null); + } + + var durationSeconds = raw.Duration > 0 ? raw.Duration : raw.Usage?.Seconds; + var durationMs = durationSeconds is > 0 ? (long?)(durationSeconds.Value * 1000) : null; + var segments = raw.Segments is { Length: > 0 } + ? raw.Segments.Select(segment => new TranscriptSegmentDto + { + StartMs = (long)(segment.Start * 1000), + EndMs = (long)(segment.End * 1000), + Text = segment.Text, + }).ToArray() + : string.IsNullOrWhiteSpace(response.Text) + ? [] + : [new TranscriptSegmentDto { StartMs = 0, EndMs = durationMs ?? 0, Text = response.Text }]; + + return new TranscriptionResult(response.Text, segments, raw.Language, durationMs); + } +} + +file sealed class OpenRouterModelListResponse +{ + [JsonPropertyName("data")] public List? Data { get; set; } +} + +file sealed class OpenRouterModelEntry +{ + [JsonPropertyName("id")] public string Id { get; set; } = string.Empty; +} diff --git a/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenAiAudioTranscriptionHelper.cs b/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenAiAudioTranscriptionHelper.cs index dabff60..b15cd5d 100644 --- a/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenAiAudioTranscriptionHelper.cs +++ b/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenAiAudioTranscriptionHelper.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Serialization; @@ -16,6 +17,12 @@ internal sealed class OpenAiVerboseTranscriptionResponse [JsonPropertyName("duration")] public double Duration { get; set; } [JsonPropertyName("text")] public string? Text { get; set; } [JsonPropertyName("segments")] public OpenAiTranscriptionSegment[]? Segments { get; set; } + [JsonPropertyName("usage")] public OpenAiTranscriptionUsage? Usage { get; set; } +} + +internal sealed class OpenAiTranscriptionUsage +{ + [JsonPropertyName("seconds")] public double? Seconds { get; set; } } internal sealed class OpenAiTranscriptionSegment @@ -26,6 +33,16 @@ internal sealed class OpenAiTranscriptionSegment [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; } +internal sealed class OpenAiTranscriptionException( + HttpStatusCode statusCode, + bool responseFormatRejected, + string message) + : InvalidOperationException(message) +{ + public HttpStatusCode StatusCode { get; } = statusCode; + public bool ResponseFormatRejected { get; } = responseFormatRejected; +} + // --------------------------------------------------------------------------- // Shared helper for OpenAI-compatible /v1/audio/transcriptions requests // --------------------------------------------------------------------------- @@ -49,15 +66,19 @@ public static async Task TranscribeAsync( string? language, Stream audioStream, CancellationToken cancellationToken, - string? device = null) + string? device = null, + string responseFormat = "verbose_json", + bool leaveAudioStreamOpen = false, + bool includeProviderErrorDetail = true) { using var content = new MultipartFormDataContent(); - var streamContent = new StreamContent(audioStream); + var streamContent = new StreamContent( + leaveAudioStreamOpen ? new LeaveOpenStream(audioStream) : audioStream); streamContent.Headers.ContentType = new("audio/wav"); content.Add(streamContent, "file", "audio.wav"); content.Add(new StringContent(modelId), "model"); - content.Add(new StringContent("verbose_json"), "response_format"); + content.Add(new StringContent(responseFormat), "response_format"); if (!string.IsNullOrWhiteSpace(language)) content.Add(new StringContent(language), "language"); if (!string.IsNullOrWhiteSpace(device)) @@ -72,9 +93,13 @@ public static async Task TranscribeAsync( if (!response.IsSuccessStatusCode) { var detail = await response.Content.ReadAsStringAsync(cancellationToken); - var normalizedDetail = NormalizeErrorDetail(detail); - throw new InvalidOperationException( - $"OpenAI-compatible transcription API returned HTTP {(int)response.StatusCode}: {normalizedDetail}"); + var message = $"OpenAI-compatible transcription API returned HTTP {(int)response.StatusCode}."; + if (includeProviderErrorDetail) + message = $"OpenAI-compatible transcription API returned HTTP {(int)response.StatusCode}: {NormalizeErrorDetail(detail)}"; + throw new OpenAiTranscriptionException( + response.StatusCode, + IsResponseFormatRejection(detail), + message); } var parsed = await response.Content.ReadFromJsonAsync(cancellationToken); @@ -90,6 +115,31 @@ public static async Task TranscribeAsync( return speechResponse; } + private static bool IsResponseFormatRejection(string detail) + { + var trimmed = detail.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + return false; + + try + { + var payload = JsonSerializer.Deserialize(trimmed); + var providerMessage = payload?.Detail ?? payload?.Error?.Message; + if (!string.IsNullOrWhiteSpace(providerMessage)) + return ContainsResponseFormatName(providerMessage); + } + catch (JsonException) + { + return ContainsResponseFormatName(trimmed); + } + + return ContainsResponseFormatName(trimmed); + } + + private static bool ContainsResponseFormatName(string value) + => value.Contains("response_format", StringComparison.OrdinalIgnoreCase) + || value.Contains("verbose_json", StringComparison.OrdinalIgnoreCase); + private static string NormalizeErrorDetail(string detail) { var trimmed = detail.Trim(); @@ -99,19 +149,45 @@ private static string NormalizeErrorDetail(string detail) try { var payload = JsonSerializer.Deserialize(trimmed); - if (!string.IsNullOrWhiteSpace(payload?.Detail)) - return payload.Detail.Trim(); + var providerMessage = payload?.Detail ?? payload?.Error?.Message; + if (!string.IsNullOrWhiteSpace(providerMessage)) + return providerMessage.Trim(); } catch (JsonException) { - // Fall back to the raw body when the endpoint did not return JSON. + return trimmed; } return trimmed; } } +file sealed class LeaveOpenStream(Stream inner) : Stream +{ + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count); + + protected override void Dispose(bool disposing) + { + } +} + file sealed class OpenAiErrorResponse { [JsonPropertyName("detail")] public string? Detail { get; set; } + [JsonPropertyName("error")] public OpenAiNestedError? Error { get; set; } +} + +file sealed class OpenAiNestedError +{ + [JsonPropertyName("message")] public string? Message { get; set; } } diff --git a/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenRouterSpeechToTextClient.cs b/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenRouterSpeechToTextClient.cs new file mode 100644 index 0000000..d770f56 --- /dev/null +++ b/src/ClassTranscriber.Api/Transcription/SpeechToText/OpenRouterSpeechToTextClient.cs @@ -0,0 +1,61 @@ +using System.Net; +using Microsoft.Extensions.AI; + +namespace ClassTranscriber.Api.Transcription.SpeechToText; + +public sealed class OpenRouterSpeechToTextClient : ISpeechToTextClient +{ + private readonly HttpClient _httpClient; + + public OpenRouterSpeechToTextClient(IHttpClientFactory httpClientFactory) + { + _httpClient = httpClientFactory.CreateClient(OpenRouterTranscriptionEngine.HttpClientName); + } + + public async Task GetTextAsync( + Stream audioSpeechStream, + SpeechToTextOptions? options = null, + CancellationToken cancellationToken = default) + { + var baseUrl = _httpClient.BaseAddress?.ToString()?.TrimEnd('/') ?? string.Empty; + var initialPosition = audioSpeechStream.CanSeek ? audioSpeechStream.Position : (long?)null; + + try + { + return await TranscribeAsync("verbose_json"); + } + catch (OpenAiTranscriptionException ex) when ( + ex.StatusCode == HttpStatusCode.BadRequest + && initialPosition is not null + && ex.ResponseFormatRejected) + { + audioSpeechStream.Position = initialPosition.Value; + return await TranscribeAsync("json"); + } + + Task TranscribeAsync(string responseFormat) + { + return OpenAiAudioTranscriptionHelper.TranscribeAsync( + _httpClient, + $"{baseUrl}/audio/transcriptions", + apiKey: null, + modelId: options?.ModelId ?? string.Empty, + language: options?.SpeechLanguage, + audioStream: audioSpeechStream, + cancellationToken: cancellationToken, + responseFormat: responseFormat, + leaveAudioStreamOpen: true, + includeProviderErrorDetail: false); + } + } + + public IAsyncEnumerable GetStreamingTextAsync( + Stream audioSpeechStream, + SpeechToTextOptions? options = null, + CancellationToken cancellationToken = default) + => throw new NotSupportedException("OpenRouterSpeechToTextClient does not support streaming transcription."); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() { } +} diff --git a/src/ClassTranscriber.Api/appsettings.json b/src/ClassTranscriber.Api/appsettings.json index e934b97..310a903 100644 --- a/src/ClassTranscriber.Api/appsettings.json +++ b/src/ClassTranscriber.Api/appsettings.json @@ -75,6 +75,14 @@ "ApiKey": "", "ModelName": "", "TimeoutSeconds": 120 + }, + "OpenRouter": { + "BaseUrl": "https://openrouter.ai/api/v1", + "ApiKey": "", + "FallbackModels": [ + "openai/whisper-large-v3" + ], + "TimeoutSeconds": 120 } } } diff --git a/src/frontend/e2e/frontend.spec.ts b/src/frontend/e2e/frontend.spec.ts index 702aa9f..704f26e 100644 --- a/src/frontend/e2e/frontend.spec.ts +++ b/src/frontend/e2e/frontend.spec.ts @@ -224,6 +224,7 @@ async function installMockApi(page: Page, options?: { seedCompletedProject?: boo { engine: 'WhisperNet', models: ['tiny', 'base', 'small'] }, { engine: 'WhisperNetCuda', models: ['small'] }, { engine: 'OpenVinoGenAi', models: ['base-int8', 'small-fp16', 'tiny-int8'] }, + { engine: 'OpenRouter', models: ['openai/whisper-large-v3'] }, ], }); } @@ -600,6 +601,36 @@ test('supports folder creation, upload review, queue monitoring, project polling await expect(page.getByRole('menuitem', { name: 'PDF (.pdf)' })).toBeVisible(); }); +test('selects OpenRouter defaults without treating remote models as local installs', async ({ page }) => { + await installMockApi(page); + + await page.goto('/settings'); + await page.getByRole('combobox').nth(0).click(); + await page.getByRole('option', { name: 'OpenRouter' }).click(); + + await expect(page.getByRole('combobox').nth(1)).toHaveText(/openai\/whisper-large-v3/); + await expect(page.getByText(/hosted speech-to-text API/i)).toBeVisible(); + await expect(page.getByText(/requires an OpenRouter API key configured on the server/i)).toBeVisible(); + await expect(page.getByText(/audio is sent to the selected remote transcription provider/i)).toBeVisible(); + await expect(page.getByRole('table').getByText('OpenRouter', { exact: true })).toHaveCount(0); + + await page.setViewportSize({ width: 375, height: 812 }); + await expect(page.getByText(/swipe horizontally to view model status and actions/i)).toBeVisible(); + const modelManagerScroll = page.getByTestId('model-manager-scroll'); + await expect.poll(() => modelManagerScroll.evaluate((element) => element.scrollWidth > element.clientWidth)).toBe(true); + await modelManagerScroll.evaluate((element) => { + element.scrollLeft = element.scrollWidth; + }); + const probeIsHorizontallyReachable = await page.getByRole('button', { name: 'Probe' }).first().evaluate((button) => { + const scrollContainer = button.closest('[data-testid="model-manager-scroll"]'); + if (!scrollContainer) return false; + const buttonRect = button.getBoundingClientRect(); + const containerRect = scrollContainer.getBoundingClientRect(); + return buttonRect.left >= containerRect.left && buttonRect.right <= containerRect.right; + }); + expect(probeIsHorizontallyReachable).toBe(true); +}); + for (const viewport of MOBILE_VIEWPORTS) { test(`renders mobile layouts without horizontal overflow on ${viewport.label}`, async ({ page }) => { await page.setViewportSize({ width: viewport.width, height: viewport.height }); diff --git a/src/frontend/src/config/transcriptionOptions.ts b/src/frontend/src/config/transcriptionOptions.ts index 9185979..355e380 100644 --- a/src/frontend/src/config/transcriptionOptions.ts +++ b/src/frontend/src/config/transcriptionOptions.ts @@ -6,6 +6,7 @@ const ENGINE_MODEL_OPTIONS = { WhisperNetCoreML: ['tiny', 'base', 'small', 'medium', 'large', 'large-v3-turbo'], OpenVinoWhisperSidecar: ['tiny-int8', 'tiny-fp16', 'base-int8', 'base-fp16', 'small-int8', 'small-fp16', 'medium-int8', 'medium-fp16', 'large-v3-int8', 'large-v3-fp16'], OpenAiCompatible: [] as string[], + OpenRouter: [] as string[], } as const; export const TRANSCRIPTION_ENGINES = Object.keys(ENGINE_MODEL_OPTIONS) as Array; diff --git a/src/frontend/src/pages/SettingsPage.tsx b/src/frontend/src/pages/SettingsPage.tsx index 372ce35..d72d586 100644 --- a/src/frontend/src/pages/SettingsPage.tsx +++ b/src/frontend/src/pages/SettingsPage.tsx @@ -261,7 +261,9 @@ export default function SettingsPage() { - {(form.defaultEngine === 'OpenVinoWhisperSidecar' || form.defaultEngine === 'OpenAiCompatible') && ( + {(form.defaultEngine === 'OpenVinoWhisperSidecar' + || form.defaultEngine === 'OpenAiCompatible' + || form.defaultEngine === 'OpenRouter') && ( {form.defaultEngine === 'OpenVinoWhisperSidecar' ? 'Uses a local OpenVINO GPU sidecar. Requires the OpenVINO Python environment to be configured.' - : 'Requires backend configuration in appsettings.json. Contact your administrator to configure the target URL and model.'} + : form.defaultEngine === 'OpenRouter' + ? "Uses OpenRouter's hosted speech-to-text API. Requires an OpenRouter API key configured on the server. Audio is sent to the selected remote transcription provider." + : 'Requires backend configuration in appsettings.json. Contact your administrator to configure the target URL and model.'} )} @@ -412,7 +416,18 @@ export default function SettingsPage() { {modelsLoading && !modelCatalog ? ( ) : ( - + <> + + Swipe horizontally to view model status and actions. + + @@ -539,7 +554,8 @@ export default function SettingsPage() { ))}
- + + )} diff --git a/src/frontend/src/test/utils/transcription.test.ts b/src/frontend/src/test/utils/transcription.test.ts index fa5b315..d504961 100644 --- a/src/frontend/src/test/utils/transcription.test.ts +++ b/src/frontend/src/test/utils/transcription.test.ts @@ -6,6 +6,10 @@ describe('formatEngineLabel', () => { it('formats WhisperNetCoreML', () => { expect(formatEngineLabel('WhisperNetCoreML')).toBe('WhisperNet.CoreML'); }); + + it('formats OpenRouter', () => { + expect(formatEngineLabel('OpenRouter')).toBe('OpenRouter'); + }); }); describe('getModelsForEngine', () => { @@ -19,4 +23,8 @@ describe('getModelsForEngine', () => { 'large-v3-turbo', ]); }); + + it('keeps OpenRouter models data-driven', () => { + expect(getModelsForEngine('OpenRouter')).toEqual([]); + }); }); diff --git a/src/frontend/src/utils/transcription.ts b/src/frontend/src/utils/transcription.ts index e04b250..1cacc0f 100644 --- a/src/frontend/src/utils/transcription.ts +++ b/src/frontend/src/utils/transcription.ts @@ -14,6 +14,8 @@ export function formatEngineLabel(engine: string): string { return 'ONNX Whisper (coming soon)'; case 'OpenAiCompatible': return 'OpenAI-Compatible API'; + case 'OpenRouter': + return 'OpenRouter'; default: return engine; } From 7dd62f93ba136f272a604f5894f472c5b8eb0319 Mon Sep 17 00:00:00 2001 From: Santiago Date: Fri, 14 Aug 2026 21:14:01 -0300 Subject: [PATCH 2/2] Run browser tests against production preview --- src/frontend/package.json | 2 +- src/frontend/playwright.config.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/package.json b/src/frontend/package.json index 9bdcb4f..50f2536 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -13,7 +13,7 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", - "test:e2e": "playwright test" + "test:e2e": "npm run build && playwright test" }, "dependencies": { "@emotion/react": "^11.14.0", diff --git a/src/frontend/playwright.config.ts b/src/frontend/playwright.config.ts index cc214bf..6432bcf 100644 --- a/src/frontend/playwright.config.ts +++ b/src/frontend/playwright.config.ts @@ -8,8 +8,8 @@ export default defineConfig({ headless: true, }, webServer: { - command: 'npm run dev -- --host 127.0.0.1 --port 4173 --strictPort', + command: 'npm run preview -- --host 127.0.0.1 --port 4173 --strictPort', url: 'http://127.0.0.1:4173', - reuseExistingServer: true, + reuseExistingServer: false, }, });