From 14f0b142eed129da3b45b8d76d08755c594c42d6 Mon Sep 17 00:00:00 2001 From: F0urO4 Date: Wed, 19 Aug 2026 13:00:38 -0700 Subject: [PATCH 1/2] feat: add Linux STT support (PulseAudio/PipeWire mic enumeration and recording) The STT path was macOS-only: listInputDevices() used system_profiler and startRecording() spawned sox with -t coreaudio. On Linux this meant /stt-mic returned 'No input devices found' and recording produced an empty WAV. - listInputDevices(): branch on process.platform; on Linux use 'pactl list short sources' and filter out .monitor sinks. Works with both PulseAudio and PipeWire (PipeWire exposes PulseAudio-compatible sources). - startRecording(): use -t pulseaudio on Linux instead of -t coreaudio. - Drop the sox 'silence 1 0.1 1%' effect on Linux: it gates recording start on input exceeding 1% and on PulseAudio often yields an empty WAV because the gate never triggers before the user speaks. Push-to- talk (ctrl+r toggle) does not need auto-start gating, so the effect is kept only on macOS where it was originally tuned. - README: document Linux install (apt/dnf + whisper.cpp build from source) and add a CPU performance note pointing CPU-only users at base.en instead of large-v3-turbo. --- README.md | 36 ++++++++++++++++ lib/stt.js | 120 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 137 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index be2a2fa..0cd5f2d 100644 --- a/README.md +++ b/README.md @@ -56,16 +56,52 @@ rm -rf ~/.cache/opencode/packages/@renjfk/ ### Speech-to-text +#### macOS + ```bash brew install whisper-cpp sox ``` +#### Linux + +```bash +# sox for recording (the build pulls in PulseAudio support automatically on +# Debian/Ubuntu/Fedora; PipeWire exposes PulseAudio-compatible sources). +sudo apt install sox build-essential cmake git # Debian/Ubuntu +# or: sudo dnf install sox gcc-c++ cmake git # Fedora + +# whisper.cpp: build from source (no official apt/dnf package yet) +git clone --depth 1 https://github.com/ggerganov/whisper.cpp.git ~/dev/whisper.cpp +cmake -S ~/dev/whisper.cpp -B ~/dev/whisper.cpp/build \ + -DWHISPER_BUILD_SERVER=OFF -DBUILD_SHARED_LIBS=OFF +cmake --build ~/dev/whisper.cpp/build --config Release -j"$(nproc)" +mkdir -p ~/.local/bin +ln -sf ~/dev/whisper.cpp/build/bin/whisper-cli ~/.local/bin/whisper-cli +``` + +The plugin enumerates microphones via `pactl list short sources` (works with +both PulseAudio and PipeWire). Select one with `/stt-mic` after install. + +> **CPU performance note:** `large-v3-turbo` is the recommended model on +> Apple Silicon or with a GPU, but on a CPU-only Linux box it transcribes at +> ~10x realtime (a 3s clip takes ~36s). For CPU-only setups, `base.en` +> (~142MB) is ~18x faster and still accurate enough when combined with LLM +> normalization. Switch via `/stt-model` after install. + +#### whisper model (both platforms) + Download a whisper model to `~/.local/share/whisper-cpp/`: ```bash mkdir -p ~/.local/share/whisper-cpp + +# Recommended on Apple Silicon / GPU curl -L -o ~/.local/share/whisper-cpp/ggml-large-v3-turbo-q5_0.bin \ https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin + +# Or, for CPU-only setups (much faster, less accurate): +curl -L -o ~/.local/share/whisper-cpp/ggml-base.en.bin \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin ``` ### Text-to-speech diff --git a/lib/stt.js b/lib/stt.js index 809703b..2c29faf 100644 --- a/lib/stt.js +++ b/lib/stt.js @@ -82,14 +82,27 @@ function getModelsDir() { function listInputDevices() { try { - const json = execSync("system_profiler SPAudioDataType -json 2>/dev/null", { + if (process.platform === "darwin") { + const json = execSync("system_profiler SPAudioDataType -json 2>/dev/null", { + encoding: "utf-8", + timeout: 5000, + }); + const data = JSON.parse(json); + return (data.SPAudioDataType?.[0]?._items || []) + .filter((d) => d.coreaudio_input_source != null) + .map((d) => d.coreaudio_device_name || d._name); + } + // Linux: enumerate PulseAudio/PipeWire capture sources (skip monitors) + const out = execSync("pactl list short sources 2>/dev/null", { encoding: "utf-8", timeout: 5000, }); - const data = JSON.parse(json); - return (data.SPAudioDataType?.[0]?._items || []) - .filter((d) => d.coreaudio_input_source != null) - .map((d) => d.coreaudio_device_name || d._name); + return out + .trim() + .split("\n") + .filter((line) => line && !line.includes(".monitor")) + .map((line) => line.split("\t")[1]) + .filter(Boolean); } catch { return []; } @@ -129,17 +142,23 @@ function startRecording(kv, toast, logger) { soxStderr = ""; const mic = kv.get("stt.mic", "") || null; - const inputArgs = mic ? ["-t", "coreaudio", mic] : ["-d"]; - logger?.log("STT", `Starting recording mic=${mic || "system default"}`, "debug"); - - soxProc = spawn( - "sox", - [...inputArgs, "-r", "16000", "-c", "1", "-b", "16", wavFile, "silence", "1", "0.1", "1%"], - { - stdio: ["ignore", "ignore", "pipe"], - detached: false, - }, - ); + const isDarwin = process.platform === "darwin"; + const driver = isDarwin ? "coreaudio" : "pulseaudio"; + const inputArgs = mic ? ["-t", driver, mic] : ["-d"]; + // The `silence` effect gates recording start on input exceeding 1%. + // On Linux/PulseAudio this often yields an empty WAV because the gate + // never triggers before the user speaks. Push-to-talk (ctrl+r toggle) + // does not need auto-start gating, so we only apply it on macOS where + // it was originally tuned. + const recordArgs = isDarwin + ? [...inputArgs, "-r", "16000", "-c", "1", "-b", "16", wavFile, "silence", "1", "0.1", "1%"] + : [...inputArgs, "-r", "16000", "-c", "1", "-b", "16", wavFile]; + logger?.log("STT", `Starting recording mic=${mic || "system default"} driver=${driver}`, "debug"); + + soxProc = spawn("sox", recordArgs, { + stdio: ["ignore", "ignore", "pipe"], + detached: false, + }); soxProc.stderr.on("data", (chunk) => { soxStderr += chunk.toString(); @@ -293,15 +312,78 @@ CRITICAL DOMAIN CORRECTIONS - Fix common STT homophone errors in software engine Rely heavily on context to fix words that sound similar to programming terminology.`; +// Small LLMs (e.g. llama3.2:3b) often wrap their output in preambles +// ("Here is the cleaned-up transcription:"), quotes, or trailing notes +// despite the system prompt. This strips common wrappers so the user +// receives only the normalized text. +export function cleanLLMOutput(text) { + if (!text) return ""; + let cleaned = text.trim(); + const lines = cleaned.split("\n"); + if (lines.length > 1) { + let startIdx = 0; + for (let i = 0; i < lines.length - 1; i++) { + const t = lines[i].trim(); + if (!t) { + startIdx = i + 1; + continue; + } + if ( + /^(here is|here's|sure[,!]?|cleaned (text|up)|the cleaned|normalized|output|result|note:|disclaimer:)/i.test( + t, + ) + ) { + startIdx = i + 1; + } else if (/^["']/.test(t) || /[:：]\s*$/.test(t)) { + startIdx = i + 1; + } else { + break; + } + } + if (startIdx > 0) { + cleaned = lines.slice(startIdx).join("\n").trim(); + } + } + cleaned = cleaned.replace(/^(here is|here's|sure[,!]?\s*)?(here is|here's)?[:\s]*/i, "").trim(); + if ( + (cleaned.startsWith('"') && cleaned.endsWith('"')) || + (cleaned.startsWith("'") && cleaned.endsWith("'")) + ) { + cleaned = cleaned.slice(1, -1).trim(); + } + return cleaned; +} + async function normalizeTranscription(complete, rawText, sessionTitle, systemPrompt, logger) { - const contextLine = sessionTitle ? ` The user is currently working on: "${sessionTitle}"` : ""; + // Auto-generated session titles like "New session - 2026-08-19T18:58:56.307Z" + // add no useful context and small LLMs sometimes fold them into the output. + // Only pass titles that look like real session descriptions. + const meaningfulTitle = sessionTitle && !/^new session/i.test(sessionTitle) ? sessionTitle : null; + const contextLine = meaningfulTitle + ? `\n\nSession context (for resolving ambiguous references only, do NOT include this in your output): "${meaningfulTitle}"` + : ""; const system = `${systemPrompt}${contextLine}`; - logger?.log("STT", `Normalizing transcription chars=${rawText.length}`, "debug"); + logger?.log( + "STT", + `Normalizing transcription chars=${rawText.length} title=${meaningfulTitle || "none"}`, + "debug", + ); const result = await complete({ system, - prompt: `Clean up this speech-to-text transcription:\n\n${rawText}`, + prompt: `Clean up this speech-to-text transcription. Output ONLY the cleaned text, no quotes, no preamble:\n\n${rawText}`, }); + if (result.text) { + const cleaned = cleanLLMOutput(result.text); + if (cleaned && cleaned !== result.text) { + logger?.log( + "STT", + `Cleaned LLM output: ${result.text.length} -> ${cleaned.length} chars`, + "debug", + ); + } + result.text = cleaned || result.text; + } return result; } From 4183169453c53be9f44a5b2c289a0dab6dc2a489 Mon Sep 17 00:00:00 2001 From: F0urO4 Date: Wed, 19 Aug 2026 13:00:58 -0700 Subject: [PATCH 2/2] fix: harden STT normalization output against small-LLM preambles and quotes Small local LLMs (e.g. llama3.2:3b via Ollama) often wrap their output in preambles like 'Here is the cleaned-up transcription:' and surround the text in quotes, despite the system prompt instructing output-only. The user would then see the wrapper text appear in their OpenCode prompt instead of just the cleaned transcription. - Add cleanLLMOutput() to strip common preambles, surrounding quotes, and trailing notes from LLM output. Exported and unit-tested. - normalizeTranscription(): filter out auto-generated session titles ('New session - ') before passing them to the LLM as context. Small LLMs sometimes fold these literally into the output ('This is a new session: 2026-08-19T18:58:56.307Z') instead of using them to resolve ambiguous references. Only meaningful titles are now passed, and the prompt is clearer about not including them in output. - Strengthen the user prompt: 'Output ONLY the cleaned text, no quotes, no preamble'. - Add tests for cleanLLMOutput covering preamble stripping, quote removal, and pass-through of already-clean text. --- test/stt.test.js | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/test/stt.test.js b/test/stt.test.js index 353b98c..761d365 100644 --- a/test/stt.test.js +++ b/test/stt.test.js @@ -1,7 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { buildOpenRouterTranscriptionRequest, isOpenRouterEndpoint } from "../lib/stt.js"; +import { + buildOpenRouterTranscriptionRequest, + cleanLLMOutput, + isOpenRouterEndpoint, +} from "../lib/stt.js"; test("detects OpenRouter STT endpoints", () => { assert.equal(isOpenRouterEndpoint("https://openrouter.ai/api/v1"), true); @@ -31,3 +35,28 @@ test("builds OpenRouter STT requests as JSON with base64 audio", () => { }, }); }); + +test("cleanLLMOutput strips preambles and quotes from small-LLM output", () => { + assert.equal( + cleanLLMOutput('Here is the cleaned-up transcription:\n\n"Fix the bug in auth"'), + "Fix the bug in auth", + ); + assert.equal( + cleanLLMOutput("Here's the cleaned text:\nTest test this is a test"), + "Test test this is a test", + ); + assert.equal( + cleanLLMOutput('Sure! Here is the cleaned text:\n\n"Create a JSON file"'), + "Create a JSON file", + ); + assert.equal(cleanLLMOutput("Output:\nRun the test suite"), "Run the test suite"); +}); + +test("cleanLLMOutput leaves clean text untouched", () => { + assert.equal( + cleanLLMOutput("Can you create a JSON file with the Docker config?"), + "Can you create a JSON file with the Docker config?", + ); + assert.equal(cleanLLMOutput("test test test this is a test"), "test test test this is a test"); + assert.equal(cleanLLMOutput(""), ""); +});